diff --git a/.gitignore b/.gitignore index 3af66272d6f1..127012c1f717 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,7 @@ modules.order /tags /TAGS /linux +/modules-only.symvers /vmlinux /vmlinux.32 /vmlinux.symvers diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 04545725f187..25422ac02dad 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -1825,6 +1825,18 @@ initcall functions. Useful for debugging built-in modules and initcalls. + initramfs_async= [KNL] + Format: + Default: 1 + This parameter controls whether the initramfs + image is unpacked asynchronously, concurrently + with devices being probed and + initialized. This should normally just work, + but as a debugging aid, one can get the + historical behaviour of the initramfs + unpacking being completed before device_ and + late_ initcalls. + initrd= [BOOT] Specify the location of the initial ramdisk initrdmem= [KNL] Specify a physical address and size from which to @@ -1869,13 +1881,6 @@ bypassed by not enabling DMAR with this option. In this case, gfx device will use physical address for DMA. - forcedac [X86-64] - With this option iommu will not optimize to look - for io virtual address below 32-bit forcing dual - address cycle on pci bus for cards supporting greater - than 32-bit addressing. The default is to look - for translation below 32-bit and if not available - then look in the higher range. strict [Default Off] With this option on every unmap_single operation will result in a hardware IOTLB flush operation as opposed @@ -1964,6 +1969,14 @@ nobypass [PPC/POWERNV] Disable IOMMU bypass, using IOMMU for PCI devices. + iommu.forcedac= [ARM64, X86] Control IOVA allocation for PCI devices. + Format: { "0" | "1" } + 0 - Try to allocate a 32-bit DMA address first, before + falling back to the full range if needed. + 1 - Allocate directly from the full usable range, + forcing Dual Address Cycle for PCI cards supporting + greater than 32-bit addressing. + iommu.strict= [ARM64] Configure TLB invalidation behaviour Format: { "0" | "1" } 0 - Lazy mode. diff --git a/Documentation/admin-guide/sysctl/vm.rst b/Documentation/admin-guide/sysctl/vm.rst index 586cd4b86428..16826b31debb 100644 --- a/Documentation/admin-guide/sysctl/vm.rst +++ b/Documentation/admin-guide/sysctl/vm.rst @@ -69,6 +69,8 @@ Currently, these files are in /proc/sys/vm: - stat_refresh - numa_stat - swappiness +- unevictable_file_kbytes_low +- unevictable_file_kbytes_min - unprivileged_userfaultfd - user_reserve_kbytes - vfs_cache_pressure @@ -886,6 +888,31 @@ calls without any restrictions. The default value is 0. +unevictable_file_kbytes_low +=========================== + +Keep some file pages still mapped under memory pressure to avoid potential +disk thrashing that may occur due to evicting running executables code. +This implements soft eviction throttling, and some file pages can still +be discarded. + +Setting it to 0 effectively disables this feature. + +The default value is 256 MiB. + + +unevictable_file_kbytes_min +=========================== + +Keep all file pages still mapped under memory pressure to avoid potential +disk thrashing that may occur due to evicting running executables code. +This is the hard limit. + +Setting it to 0 effectively disables this feature. + +The default value is 128 MiB. + + user_reserve_kbytes =================== diff --git a/Documentation/devicetree/bindings/media/renesas,vin.yaml b/Documentation/devicetree/bindings/media/renesas,vin.yaml index fe7c4cbfe4ba..dd1a5ce5896c 100644 --- a/Documentation/devicetree/bindings/media/renesas,vin.yaml +++ b/Documentation/devicetree/bindings/media/renesas,vin.yaml @@ -193,23 +193,35 @@ required: - interrupts - clocks - power-domains - - resets - -if: - properties: - compatible: - contains: - enum: - - renesas,vin-r8a7778 - - renesas,vin-r8a7779 - - renesas,rcar-gen2-vin -then: - required: - - port -else: - required: - - renesas,id - - ports + +allOf: + - if: + not: + properties: + compatible: + contains: + enum: + - renesas,vin-r8a7778 + - renesas,vin-r8a7779 + then: + required: + - resets + + - if: + properties: + compatible: + contains: + enum: + - renesas,vin-r8a7778 + - renesas,vin-r8a7779 + - renesas,rcar-gen2-vin + then: + required: + - port + else: + required: + - renesas,id + - ports additionalProperties: false diff --git a/Documentation/devicetree/bindings/pci/rcar-pci-host.yaml b/Documentation/devicetree/bindings/pci/rcar-pci-host.yaml index 4a2bcc0158e2..8fdfbc763d70 100644 --- a/Documentation/devicetree/bindings/pci/rcar-pci-host.yaml +++ b/Documentation/devicetree/bindings/pci/rcar-pci-host.yaml @@ -17,6 +17,7 @@ allOf: properties: compatible: oneOf: + - const: renesas,pcie-r8a7779 # R-Car H1 - items: - enum: - renesas,pcie-r8a7742 # RZ/G1H @@ -74,7 +75,16 @@ required: - clocks - clock-names - power-domains - - resets + +if: + not: + properties: + compatible: + contains: + const: renesas,pcie-r8a7779 +then: + required: + - resets unevaluatedProperties: false diff --git a/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml index 626447fee092..7808ec8bc712 100644 --- a/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml @@ -25,11 +25,13 @@ properties: - qcom,msm8998-qmp-pcie-phy - qcom,msm8998-qmp-ufs-phy - qcom,msm8998-qmp-usb3-phy + - qcom,sc7180-qmp-usb3-phy - qcom,sc8180x-qmp-ufs-phy - qcom,sc8180x-qmp-usb3-phy - qcom,sdm845-qhp-pcie-phy - qcom,sdm845-qmp-pcie-phy - qcom,sdm845-qmp-ufs-phy + - qcom,sdm845-qmp-usb3-phy - qcom,sdm845-qmp-usb3-uni-phy - qcom,sm8150-qmp-ufs-phy - qcom,sm8150-qmp-usb3-phy diff --git a/Documentation/devicetree/bindings/phy/qcom,qmp-usb3-dp-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,qmp-usb3-dp-phy.yaml index 33974ad10afe..62c0179d1765 100644 --- a/Documentation/devicetree/bindings/phy/qcom,qmp-usb3-dp-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,qmp-usb3-dp-phy.yaml @@ -14,9 +14,7 @@ properties: compatible: enum: - qcom,sc7180-qmp-usb3-dp-phy - - qcom,sc7180-qmp-usb3-phy - qcom,sdm845-qmp-usb3-dp-phy - - qcom,sdm845-qmp-usb3-phy reg: items: - description: Address and length of PHY's USB serdes block. diff --git a/Documentation/devicetree/bindings/serial/8250.yaml b/Documentation/devicetree/bindings/serial/8250.yaml index f54cae9ff7b2..d3f87f2bfdc2 100644 --- a/Documentation/devicetree/bindings/serial/8250.yaml +++ b/Documentation/devicetree/bindings/serial/8250.yaml @@ -93,11 +93,6 @@ properties: - mediatek,mt7622-btif - mediatek,mt7623-btif - const: mediatek,mtk-btif - - items: - - enum: - - mediatek,mt7622-btif - - mediatek,mt7623-btif - - const: mediatek,mtk-btif - items: - const: mrvl,mmp-uart - const: intel,xscale-uart diff --git a/Documentation/devicetree/bindings/serial/st,stm32-uart.yaml b/Documentation/devicetree/bindings/serial/st,stm32-uart.yaml index 8631678283f9..865be05083c3 100644 --- a/Documentation/devicetree/bindings/serial/st,stm32-uart.yaml +++ b/Documentation/devicetree/bindings/serial/st,stm32-uart.yaml @@ -80,7 +80,8 @@ required: - interrupts - clocks -additionalProperties: false +additionalProperties: + type: object examples: - | diff --git a/Documentation/devicetree/bindings/thermal/rcar-gen3-thermal.yaml b/Documentation/devicetree/bindings/thermal/rcar-gen3-thermal.yaml index b33a76eeac4e..f963204e0b16 100644 --- a/Documentation/devicetree/bindings/thermal/rcar-gen3-thermal.yaml +++ b/Documentation/devicetree/bindings/thermal/rcar-gen3-thermal.yaml @@ -28,14 +28,7 @@ properties: - renesas,r8a77980-thermal # R-Car V3H - renesas,r8a779a0-thermal # R-Car V3U - reg: - minItems: 2 - maxItems: 4 - items: - - description: TSC1 registers - - description: TSC2 registers - - description: TSC3 registers - - description: TSC4 registers + reg: true interrupts: items: @@ -71,8 +64,25 @@ if: enum: - renesas,r8a779a0-thermal then: + properties: + reg: + minItems: 2 + maxItems: 3 + items: + - description: TSC1 registers + - description: TSC2 registers + - description: TSC3 registers required: - interrupts +else: + properties: + reg: + items: + - description: TSC0 registers + - description: TSC1 registers + - description: TSC2 registers + - description: TSC3 registers + - description: TSC4 registers additionalProperties: false @@ -111,3 +121,20 @@ examples: }; }; }; + - | + #include + #include + #include + + tsc_r8a779a0: thermal@e6190000 { + compatible = "renesas,r8a779a0-thermal"; + reg = <0xe6190000 0x200>, + <0xe6198000 0x200>, + <0xe61a0000 0x200>, + <0xe61a8000 0x200>, + <0xe61b0000 0x200>; + clocks = <&cpg CPG_MOD 919>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 919>; + #thermal-sensor-cells = <1>; + }; diff --git a/Documentation/dontdiff b/Documentation/dontdiff index e361fc95ca29..82e3eee7363b 100644 --- a/Documentation/dontdiff +++ b/Documentation/dontdiff @@ -178,6 +178,7 @@ mktables mktree mkutf8data modpost +modules-only.symvers modules.builtin modules.builtin.modinfo modules.nsdeps diff --git a/Documentation/driver-api/xilinx/eemi.rst b/Documentation/driver-api/xilinx/eemi.rst index 9dcbc6f18d75..c1bc47b9000d 100644 --- a/Documentation/driver-api/xilinx/eemi.rst +++ b/Documentation/driver-api/xilinx/eemi.rst @@ -16,35 +16,8 @@ components running across different processing clusters on a chip or device to communicate with a power management controller (PMC) on a device to issue or respond to power management requests. -EEMI ops is a structure containing all eemi APIs supported by Zynq MPSoC. -The zynqmp-firmware driver maintain all EEMI APIs in zynqmp_eemi_ops -structure. Any driver who want to communicate with PMC using EEMI APIs -can call zynqmp_pm_get_eemi_ops(). - -Example of EEMI ops:: - - /* zynqmp-firmware driver maintain all EEMI APIs */ - struct zynqmp_eemi_ops { - int (*get_api_version)(u32 *version); - int (*query_data)(struct zynqmp_pm_query_data qdata, u32 *out); - }; - - static const struct zynqmp_eemi_ops eemi_ops = { - .get_api_version = zynqmp_pm_get_api_version, - .query_data = zynqmp_pm_query_data, - }; - -Example of EEMI ops usage:: - - static const struct zynqmp_eemi_ops *eemi_ops; - u32 ret_payload[PAYLOAD_ARG_CNT]; - int ret; - - eemi_ops = zynqmp_pm_get_eemi_ops(); - if (IS_ERR(eemi_ops)) - return PTR_ERR(eemi_ops); - - ret = eemi_ops->query_data(qdata, ret_payload); +Any driver who wants to communicate with PMC using EEMI APIs use the +functions provided for each function. IOCTL ------ diff --git a/Documentation/filesystems/ntfs3.rst b/Documentation/filesystems/ntfs3.rst new file mode 100644 index 000000000000..fb29067360cc --- /dev/null +++ b/Documentation/filesystems/ntfs3.rst @@ -0,0 +1,107 @@ +.. SPDX-License-Identifier: GPL-2.0 + +===== +NTFS3 +===== + + +Summary and Features +==================== + +NTFS3 is fully functional NTFS Read-Write driver. The driver works with +NTFS versions up to 3.1, normal/compressed/sparse files +and journal replaying. File system type to use on mount is 'ntfs3'. + +- This driver implements NTFS read/write support for normal, sparse and + compressed files. +- Supports native journal replaying; +- Supports extended attributes + Predefined extended attributes: + - 'system.ntfs_security' gets/sets security + descriptor (SECURITY_DESCRIPTOR_RELATIVE) + - 'system.ntfs_attrib' gets/sets ntfs file/dir attributes. + Note: applied to empty files, this allows to switch type between + sparse(0x200), compressed(0x800) and normal; +- Supports NFS export of mounted NTFS volumes. + +Mount Options +============= + +The list below describes mount options supported by NTFS3 driver in addition to +generic ones. + +=============================================================================== + +nls=name This option informs the driver how to interpret path + strings and translate them to Unicode and back. If + this option is not set, the default codepage will be + used (CONFIG_NLS_DEFAULT). + Examples: + 'nls=utf8' + +uid= +gid= +umask= Controls the default permissions for files/directories created + after the NTFS volume is mounted. + +fmask= +dmask= Instead of specifying umask which applies both to + files and directories, fmask applies only to files and + dmask only to directories. + +nohidden Files with the Windows-specific HIDDEN (FILE_ATTRIBUTE_HIDDEN) + attribute will not be shown under Linux. + +sys_immutable Files with the Windows-specific SYSTEM + (FILE_ATTRIBUTE_SYSTEM) attribute will be marked as system + immutable files. + +discard Enable support of the TRIM command for improved performance + on delete operations, which is recommended for use with the + solid-state drives (SSD). + +force Forces the driver to mount partitions even if 'dirty' flag + (volume dirty) is set. Not recommended for use. + +sparse Create new files as "sparse". + +showmeta Use this parameter to show all meta-files (System Files) on + a mounted NTFS partition. + By default, all meta-files are hidden. + +prealloc Preallocate space for files excessively when file size is + increasing on writes. Decreases fragmentation in case of + parallel write operations to different files. + +no_acs_rules "No access rules" mount option sets access rights for + files/folders to 777 and owner/group to root. This mount + option absorbs all other permissions: + - permissions change for files/folders will be reported + as successful, but they will remain 777; + - owner/group change will be reported as successful, but + they will stay as root + +acl Support POSIX ACLs (Access Control Lists). Effective if + supported by Kernel. Not to be confused with NTFS ACLs. + The option specified as acl enables support for POSIX ACLs. + +noatime All files and directories will not update their last access + time attribute if a partition is mounted with this parameter. + This option can speed up file system operation. + +=============================================================================== + +ToDo list +========= + +- Full journaling support (currently journal replaying is supported) over JBD. + + +References +========== +https://www.paragon-software.com/home/ntfs-linux-professional/ + - Commercial version of the NTFS driver for Linux. + +almaz.alexandrovich@paragon-software.com + - Direct e-mail address for feedback and requests on the NTFS3 implementation. + diff --git a/Documentation/filesystems/proc.rst b/Documentation/filesystems/proc.rst index 48fbfc336ebf..80a294b914f9 100644 --- a/Documentation/filesystems/proc.rst +++ b/Documentation/filesystems/proc.rst @@ -47,6 +47,7 @@ fixes/update part 1.1 Stefani Seibold June 9 2009 3.10 /proc//timerslack_ns - Task timerslack value 3.11 /proc//patch_state - Livepatch patch operation state 3.12 /proc//arch_status - Task architecture specific information + 3.13 /proc//ksm - Remote KSM 4 Configuring procfs 4.1 Mount options @@ -2138,6 +2139,19 @@ AVX512_elapsed_ms the task is unlikely an AVX512 user, but depends on the workload and the scheduling scenario, it also could be a false negative mentioned above. +3.13 /proc//ksm - Remote KSM +------------------------------------ +This write-only file allows marking memory of another task for merging +and unmerging via KSM. + +The following actions are available: + + * mark task's memory as mergeable: + # echo merge > /proc//ksm + + * unmerging all the task's memory: + # echo unmerge > /proc//ksm + Chapter 4: Configuring procfs ============================= diff --git a/Documentation/userspace-api/media/v4l/subdev-formats.rst b/Documentation/userspace-api/media/v4l/subdev-formats.rst index 7f16cbe46e5c..e6a9faa81197 100644 --- a/Documentation/userspace-api/media/v4l/subdev-formats.rst +++ b/Documentation/userspace-api/media/v4l/subdev-formats.rst @@ -1567,8 +1567,8 @@ The following tables list existing packed RGB formats. - MEDIA_BUS_FMT_RGB101010_1X30 - 0x1018 - - - 0 - - 0 + - + - - r\ :sub:`9` - r\ :sub:`8` - r\ :sub:`7` diff --git a/MAINTAINERS b/MAINTAINERS index 9450e052f1b1..fefb8b6a6823 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -12775,6 +12775,13 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/aia21/ntfs.git F: Documentation/filesystems/ntfs.rst F: fs/ntfs/ +NTFS3 FILESYSTEM +M: Konstantin Komarov +S: Supported +W: http://www.paragon-software.com/ +F: Documentation/filesystems/ntfs3.rst +F: fs/ntfs3/ + NUBUS SUBSYSTEM M: Finn Thain L: linux-m68k@lists.linux-m68k.org @@ -19912,6 +19919,18 @@ F: Documentation/vm/zsmalloc.rst F: include/linux/zsmalloc.h F: mm/zsmalloc.c +ZSTD +M: Nick Terrell +S: Maintained +B: https://github.com/facebook/zstd/issues +T: git git://github.com/terrelln/linux.git +F: include/linux/zstd* +F: lib/zstd/ +F: lib/decompress_unzstd.c +F: crypto/zstd.c +N: zstd +K: zstd + ZSWAP COMPRESSED SWAP CACHING M: Seth Jennings M: Dan Streetman diff --git a/Makefile b/Makefile index 3a10a8e08b6d..8d49e1089d84 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ VERSION = 5 PATCHLEVEL = 12 SUBLEVEL = 0 -EXTRAVERSION = +EXTRAVERSION = -pf3 NAME = Frozen Wasteland # *DOCUMENTATION* @@ -775,16 +775,16 @@ KBUILD_CFLAGS += -Wno-gnu KBUILD_CFLAGS += -mno-global-merge else -# These warnings generated too much noise in a regular build. -# Use make W=1 to enable them (see scripts/Makefile.extrawarn) -KBUILD_CFLAGS += -Wno-unused-but-set-variable - # Warn about unmarked fall-throughs in switch statement. # Disabled for clang while comment to attribute conversion happens and # https://github.com/ClangBuiltLinux/linux/issues/636 is discussed. KBUILD_CFLAGS += $(call cc-option,-Wimplicit-fallthrough,) endif +# These warnings generated too much noise in a regular build. +# Use make W=1 to enable them (see scripts/Makefile.extrawarn) +KBUILD_CFLAGS += $(call cc-disable-warning, unused-but-set-variable) + KBUILD_CFLAGS += $(call cc-disable-warning, unused-const-variable) ifdef CONFIG_FRAME_POINTER KBUILD_CFLAGS += -fno-omit-frame-pointer -fno-optimize-sibling-calls @@ -1066,8 +1066,8 @@ endif # INSTALL_MOD_STRIP export mod_strip_cmd # CONFIG_MODULE_COMPRESS, if defined, will cause module to be compressed -# after they are installed in agreement with CONFIG_MODULE_COMPRESS_GZIP -# or CONFIG_MODULE_COMPRESS_XZ. +# after they are installed in agreement with CONFIG_MODULE_COMPRESS_GZIP, +# CONFIG_MODULE_COMPRESS_XZ, or CONFIG_MODULE_COMPRESS_ZSTD. mod_compress_cmd = true ifdef CONFIG_MODULE_COMPRESS @@ -1077,6 +1077,9 @@ ifdef CONFIG_MODULE_COMPRESS ifdef CONFIG_MODULE_COMPRESS_XZ mod_compress_cmd = $(XZ) --lzma2=dict=2MiB -f endif # CONFIG_MODULE_COMPRESS_XZ + ifdef CONFIG_MODULE_COMPRESS_ZSTD + mod_compress_cmd = $(ZSTD) -T0 --rm -f -q + endif # CONFIG_MODULE_COMPRESS_ZSTD endif # CONFIG_MODULE_COMPRESS export mod_compress_cmd @@ -1513,7 +1516,7 @@ endif # CONFIG_MODULES # make distclean Remove editor backup files, patch leftover files and the like # Directories & files removed with 'make clean' -CLEAN_FILES += include/ksym vmlinux.symvers \ +CLEAN_FILES += include/ksym vmlinux.symvers modules-only.symvers \ modules.builtin modules.builtin.modinfo modules.nsdeps \ compile_commands.json .thinlto-cache diff --git a/arch/arc/include/asm/page.h b/arch/arc/include/asm/page.h index ad9b7fe4dba3..4a9d33372fe2 100644 --- a/arch/arc/include/asm/page.h +++ b/arch/arc/include/asm/page.h @@ -7,6 +7,18 @@ #include +#ifdef CONFIG_ARC_HAS_PAE40 + +#define MAX_POSSIBLE_PHYSMEM_BITS 40 +#define PAGE_MASK_PHYS (0xff00000000ull | PAGE_MASK) + +#else /* CONFIG_ARC_HAS_PAE40 */ + +#define MAX_POSSIBLE_PHYSMEM_BITS 32 +#define PAGE_MASK_PHYS PAGE_MASK + +#endif /* CONFIG_ARC_HAS_PAE40 */ + #ifndef __ASSEMBLY__ #define clear_page(paddr) memset((paddr), 0, PAGE_SIZE) diff --git a/arch/arc/include/asm/pgtable.h b/arch/arc/include/asm/pgtable.h index 163641726a2b..5878846f00cf 100644 --- a/arch/arc/include/asm/pgtable.h +++ b/arch/arc/include/asm/pgtable.h @@ -107,8 +107,8 @@ #define ___DEF (_PAGE_PRESENT | _PAGE_CACHEABLE) /* Set of bits not changed in pte_modify */ -#define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_SPECIAL) - +#define _PAGE_CHG_MASK (PAGE_MASK_PHYS | _PAGE_ACCESSED | _PAGE_DIRTY | \ + _PAGE_SPECIAL) /* More Abbrevaited helpers */ #define PAGE_U_NONE __pgprot(___DEF) #define PAGE_U_R __pgprot(___DEF | _PAGE_READ) @@ -132,13 +132,7 @@ #define PTE_BITS_IN_PD0 (_PAGE_GLOBAL | _PAGE_PRESENT | _PAGE_HW_SZ) #define PTE_BITS_RWX (_PAGE_EXECUTE | _PAGE_WRITE | _PAGE_READ) -#ifdef CONFIG_ARC_HAS_PAE40 -#define PTE_BITS_NON_RWX_IN_PD1 (0xff00000000 | PAGE_MASK | _PAGE_CACHEABLE) -#define MAX_POSSIBLE_PHYSMEM_BITS 40 -#else -#define PTE_BITS_NON_RWX_IN_PD1 (PAGE_MASK | _PAGE_CACHEABLE) -#define MAX_POSSIBLE_PHYSMEM_BITS 32 -#endif +#define PTE_BITS_NON_RWX_IN_PD1 (PAGE_MASK_PHYS | _PAGE_CACHEABLE) /************************************************************************** * Mapping of vm_flags (Generic VM) to PTE flags (arch specific) diff --git a/arch/arc/include/uapi/asm/page.h b/arch/arc/include/uapi/asm/page.h index 2a97e2718a21..2a4ad619abfb 100644 --- a/arch/arc/include/uapi/asm/page.h +++ b/arch/arc/include/uapi/asm/page.h @@ -33,5 +33,4 @@ #define PAGE_MASK (~(PAGE_SIZE-1)) - #endif /* _UAPI__ASM_ARC_PAGE_H */ diff --git a/arch/arc/kernel/entry.S b/arch/arc/kernel/entry.S index 1743506081da..2cb8dfe866b6 100644 --- a/arch/arc/kernel/entry.S +++ b/arch/arc/kernel/entry.S @@ -177,7 +177,7 @@ tracesys: ; Do the Sys Call as we normally would. ; Validate the Sys Call number - cmp r8, NR_syscalls + cmp r8, NR_syscalls - 1 mov.hi r0, -ENOSYS bhi tracesys_exit @@ -255,7 +255,7 @@ ENTRY(EV_Trap) ;============ Normal syscall case ; syscall num shd not exceed the total system calls avail - cmp r8, NR_syscalls + cmp r8, NR_syscalls - 1 mov.hi r0, -ENOSYS bhi .Lret_from_system_call diff --git a/arch/arc/mm/init.c b/arch/arc/mm/init.c index ce07e697916c..1bcc6985b9a0 100644 --- a/arch/arc/mm/init.c +++ b/arch/arc/mm/init.c @@ -157,7 +157,16 @@ void __init setup_arch_memory(void) min_high_pfn = PFN_DOWN(high_mem_start); max_high_pfn = PFN_DOWN(high_mem_start + high_mem_sz); - max_zone_pfn[ZONE_HIGHMEM] = min_low_pfn; + /* + * max_high_pfn should be ok here for both HIGHMEM and HIGHMEM+PAE. + * For HIGHMEM without PAE max_high_pfn should be less than + * min_low_pfn to guarantee that these two regions don't overlap. + * For PAE case highmem is greater than lowmem, so it is natural + * to use max_high_pfn. + * + * In both cases, holes should be handled by pfn_valid(). + */ + max_zone_pfn[ZONE_HIGHMEM] = max_high_pfn; high_memory = (void *)(min_high_pfn << PAGE_SHIFT); diff --git a/arch/arc/mm/ioremap.c b/arch/arc/mm/ioremap.c index fac4adc90204..95c649fbc95a 100644 --- a/arch/arc/mm/ioremap.c +++ b/arch/arc/mm/ioremap.c @@ -53,9 +53,10 @@ EXPORT_SYMBOL(ioremap); void __iomem *ioremap_prot(phys_addr_t paddr, unsigned long size, unsigned long flags) { + unsigned int off; unsigned long vaddr; struct vm_struct *area; - phys_addr_t off, end; + phys_addr_t end; pgprot_t prot = __pgprot(flags); /* Don't allow wraparound, zero size */ @@ -72,7 +73,7 @@ void __iomem *ioremap_prot(phys_addr_t paddr, unsigned long size, /* Mappings have to be page-aligned */ off = paddr & ~PAGE_MASK; - paddr &= PAGE_MASK; + paddr &= PAGE_MASK_PHYS; size = PAGE_ALIGN(end + 1) - paddr; /* diff --git a/arch/arc/mm/tlb.c b/arch/arc/mm/tlb.c index 9bb3c24f3677..9c7c68247289 100644 --- a/arch/arc/mm/tlb.c +++ b/arch/arc/mm/tlb.c @@ -576,7 +576,7 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long vaddr_unaligned, pte_t *ptep) { unsigned long vaddr = vaddr_unaligned & PAGE_MASK; - phys_addr_t paddr = pte_val(*ptep) & PAGE_MASK; + phys_addr_t paddr = pte_val(*ptep) & PAGE_MASK_PHYS; struct page *page = pfn_to_page(pte_pfn(*ptep)); create_tlb(vma, vaddr, ptep); diff --git a/arch/arm/boot/compressed/Makefile b/arch/arm/boot/compressed/Makefile index fd94e27ba4fa..c1f804768621 100644 --- a/arch/arm/boot/compressed/Makefile +++ b/arch/arm/boot/compressed/Makefile @@ -118,8 +118,8 @@ asflags-y := -DZIMAGE # Supply kernel BSS size to the decompressor via a linker symbol. KBSS_SZ = $(shell echo $$(($$($(NM) $(obj)/../../../../vmlinux | \ - sed -n -e 's/^\([^ ]*\) [AB] __bss_start$$/-0x\1/p' \ - -e 's/^\([^ ]*\) [AB] __bss_stop$$/+0x\1/p') )) ) + sed -n -e 's/^\([^ ]*\) [ABD] __bss_start$$/-0x\1/p' \ + -e 's/^\([^ ]*\) [ABD] __bss_stop$$/+0x\1/p') )) ) LDFLAGS_vmlinux = --defsym _kernel_bss_size=$(KBSS_SZ) # Supply ZRELADDR to the decompressor via a linker symbol. ifneq ($(CONFIG_AUTO_ZRELADDR),y) diff --git a/arch/arm/boot/dts/aspeed-bmc-ibm-rainier.dts b/arch/arm/boot/dts/aspeed-bmc-ibm-rainier.dts index 6c9804d2f3b4..6df1ce545061 100644 --- a/arch/arm/boot/dts/aspeed-bmc-ibm-rainier.dts +++ b/arch/arm/boot/dts/aspeed-bmc-ibm-rainier.dts @@ -713,9 +713,9 @@ multi-master; status = "okay"; - si7021-a20@20 { + si7021-a20@40 { compatible = "silabs,si7020"; - reg = <0x20>; + reg = <0x40>; }; tmp275@48 { diff --git a/arch/arm/boot/dts/at91-sam9x60ek.dts b/arch/arm/boot/dts/at91-sam9x60ek.dts index 775ceb3acb6c..edca66c232c1 100644 --- a/arch/arm/boot/dts/at91-sam9x60ek.dts +++ b/arch/arm/boot/dts/at91-sam9x60ek.dts @@ -8,6 +8,7 @@ */ /dts-v1/; #include "sam9x60.dtsi" +#include / { model = "Microchip SAM9X60-EK"; @@ -84,7 +85,7 @@ sw1 { label = "SW1"; gpios = <&pioD 18 GPIO_ACTIVE_LOW>; - linux,code=<0x104>; + linux,code=; wakeup-source; }; }; diff --git a/arch/arm/boot/dts/at91-sama5d27_som1_ek.dts b/arch/arm/boot/dts/at91-sama5d27_som1_ek.dts index 84e1180f3e89..a9e6fee55a2a 100644 --- a/arch/arm/boot/dts/at91-sama5d27_som1_ek.dts +++ b/arch/arm/boot/dts/at91-sama5d27_som1_ek.dts @@ -11,6 +11,7 @@ #include "at91-sama5d27_som1.dtsi" #include #include +#include / { model = "Atmel SAMA5D27 SOM1 EK"; @@ -466,7 +467,7 @@ pb4 { label = "USER"; gpios = <&pioA PIN_PA29 GPIO_ACTIVE_LOW>; - linux,code = <0x104>; + linux,code = ; wakeup-source; }; }; diff --git a/arch/arm/boot/dts/at91-sama5d27_wlsom1_ek.dts b/arch/arm/boot/dts/at91-sama5d27_wlsom1_ek.dts index 180a08765cb8..ff83967fd008 100644 --- a/arch/arm/boot/dts/at91-sama5d27_wlsom1_ek.dts +++ b/arch/arm/boot/dts/at91-sama5d27_wlsom1_ek.dts @@ -8,6 +8,7 @@ */ /dts-v1/; #include "at91-sama5d27_wlsom1.dtsi" +#include / { model = "Microchip SAMA5D27 WLSOM1 EK"; @@ -35,7 +36,7 @@ sw4 { label = "USER BUTTON"; gpios = <&pioA PIN_PB2 GPIO_ACTIVE_LOW>; - linux,code = <0x104>; + linux,code = ; wakeup-source; }; }; diff --git a/arch/arm/boot/dts/at91-sama5d2_icp.dts b/arch/arm/boot/dts/at91-sama5d2_icp.dts index 46722a163184..bd64721fa23c 100644 --- a/arch/arm/boot/dts/at91-sama5d2_icp.dts +++ b/arch/arm/boot/dts/at91-sama5d2_icp.dts @@ -12,6 +12,7 @@ #include "sama5d2.dtsi" #include "sama5d2-pinfunc.h" #include +#include #include / { @@ -51,7 +52,7 @@ sw4 { label = "USER_PB1"; gpios = <&pioA PIN_PD0 GPIO_ACTIVE_LOW>; - linux,code = <0x104>; + linux,code = ; wakeup-source; }; }; diff --git a/arch/arm/boot/dts/at91-sama5d2_ptc_ek.dts b/arch/arm/boot/dts/at91-sama5d2_ptc_ek.dts index 8de57d164acd..dfd150eb0fd8 100644 --- a/arch/arm/boot/dts/at91-sama5d2_ptc_ek.dts +++ b/arch/arm/boot/dts/at91-sama5d2_ptc_ek.dts @@ -11,6 +11,7 @@ #include "sama5d2-pinfunc.h" #include #include +#include #include / { @@ -402,7 +403,7 @@ bp1 { label = "PB_USER"; gpios = <&pioA PIN_PA10 GPIO_ACTIVE_LOW>; - linux,code = <0x104>; + linux,code = ; wakeup-source; }; }; diff --git a/arch/arm/boot/dts/at91-sama5d2_xplained.dts b/arch/arm/boot/dts/at91-sama5d2_xplained.dts index 4e7cf21f124c..509c732a0d8b 100644 --- a/arch/arm/boot/dts/at91-sama5d2_xplained.dts +++ b/arch/arm/boot/dts/at91-sama5d2_xplained.dts @@ -10,6 +10,7 @@ #include "sama5d2-pinfunc.h" #include #include +#include #include / { @@ -712,7 +713,7 @@ bp1 { label = "PB_USER"; gpios = <&pioA PIN_PB9 GPIO_ACTIVE_LOW>; - linux,code = <0x104>; + linux,code = ; wakeup-source; }; }; diff --git a/arch/arm/boot/dts/at91-sama5d3_xplained.dts b/arch/arm/boot/dts/at91-sama5d3_xplained.dts index 5179258f9247..9c55a921263b 100644 --- a/arch/arm/boot/dts/at91-sama5d3_xplained.dts +++ b/arch/arm/boot/dts/at91-sama5d3_xplained.dts @@ -7,6 +7,7 @@ */ /dts-v1/; #include "sama5d36.dtsi" +#include / { model = "SAMA5D3 Xplained"; @@ -354,7 +355,7 @@ bp3 { label = "PB_USER"; gpios = <&pioE 29 GPIO_ACTIVE_LOW>; - linux,code = <0x104>; + linux,code = ; wakeup-source; }; }; diff --git a/arch/arm/boot/dts/at91sam9260ek.dts b/arch/arm/boot/dts/at91sam9260ek.dts index d3446e42b598..ce96345d28a3 100644 --- a/arch/arm/boot/dts/at91sam9260ek.dts +++ b/arch/arm/boot/dts/at91sam9260ek.dts @@ -7,6 +7,7 @@ */ /dts-v1/; #include "at91sam9260.dtsi" +#include / { model = "Atmel at91sam9260ek"; @@ -156,7 +157,7 @@ btn4 { label = "Button 4"; gpios = <&pioA 31 GPIO_ACTIVE_LOW>; - linux,code = <0x104>; + linux,code = ; wakeup-source; }; }; diff --git a/arch/arm/boot/dts/at91sam9g20ek_common.dtsi b/arch/arm/boot/dts/at91sam9g20ek_common.dtsi index 6e6e672c0b86..87bb39060e8b 100644 --- a/arch/arm/boot/dts/at91sam9g20ek_common.dtsi +++ b/arch/arm/boot/dts/at91sam9g20ek_common.dtsi @@ -5,6 +5,7 @@ * Copyright (C) 2012 Jean-Christophe PLAGNIOL-VILLARD */ #include "at91sam9g20.dtsi" +#include / { @@ -234,7 +235,7 @@ btn4 { label = "Button 4"; gpios = <&pioA 31 GPIO_ACTIVE_LOW>; - linux,code = <0x104>; + linux,code = ; wakeup-source; }; }; diff --git a/arch/arm/boot/dts/bcm4708-asus-rt-ac56u.dts b/arch/arm/boot/dts/bcm4708-asus-rt-ac56u.dts index 6a96655d8626..8ed403767540 100644 --- a/arch/arm/boot/dts/bcm4708-asus-rt-ac56u.dts +++ b/arch/arm/boot/dts/bcm4708-asus-rt-ac56u.dts @@ -21,8 +21,8 @@ memory@0 { device_type = "memory"; - reg = <0x00000000 0x08000000 - 0x88000000 0x08000000>; + reg = <0x00000000 0x08000000>, + <0x88000000 0x08000000>; }; leds { diff --git a/arch/arm/boot/dts/bcm4708-asus-rt-ac68u.dts b/arch/arm/boot/dts/bcm4708-asus-rt-ac68u.dts index 3b0029e61b4c..667b118ba4ee 100644 --- a/arch/arm/boot/dts/bcm4708-asus-rt-ac68u.dts +++ b/arch/arm/boot/dts/bcm4708-asus-rt-ac68u.dts @@ -21,8 +21,8 @@ memory@0 { device_type = "memory"; - reg = <0x00000000 0x08000000 - 0x88000000 0x08000000>; + reg = <0x00000000 0x08000000>, + <0x88000000 0x08000000>; }; leds { diff --git a/arch/arm/boot/dts/bcm4708-buffalo-wzr-1750dhp.dts b/arch/arm/boot/dts/bcm4708-buffalo-wzr-1750dhp.dts index 90f57bad6b24..ff31ce45831a 100644 --- a/arch/arm/boot/dts/bcm4708-buffalo-wzr-1750dhp.dts +++ b/arch/arm/boot/dts/bcm4708-buffalo-wzr-1750dhp.dts @@ -21,8 +21,8 @@ memory@0 { device_type = "memory"; - reg = <0x00000000 0x08000000 - 0x88000000 0x18000000>; + reg = <0x00000000 0x08000000>, + <0x88000000 0x18000000>; }; spi { diff --git a/arch/arm/boot/dts/bcm4708-netgear-r6250.dts b/arch/arm/boot/dts/bcm4708-netgear-r6250.dts index fed75e6ab58c..61c7b137607e 100644 --- a/arch/arm/boot/dts/bcm4708-netgear-r6250.dts +++ b/arch/arm/boot/dts/bcm4708-netgear-r6250.dts @@ -22,8 +22,8 @@ memory { device_type = "memory"; - reg = <0x00000000 0x08000000 - 0x88000000 0x08000000>; + reg = <0x00000000 0x08000000>, + <0x88000000 0x08000000>; }; leds { diff --git a/arch/arm/boot/dts/bcm4708-netgear-r6300-v2.dts b/arch/arm/boot/dts/bcm4708-netgear-r6300-v2.dts index 79542e18915c..4c60eda296d9 100644 --- a/arch/arm/boot/dts/bcm4708-netgear-r6300-v2.dts +++ b/arch/arm/boot/dts/bcm4708-netgear-r6300-v2.dts @@ -21,8 +21,8 @@ memory@0 { device_type = "memory"; - reg = <0x00000000 0x08000000 - 0x88000000 0x08000000>; + reg = <0x00000000 0x08000000>, + <0x88000000 0x08000000>; }; leds { diff --git a/arch/arm/boot/dts/bcm4708-smartrg-sr400ac.dts b/arch/arm/boot/dts/bcm4708-smartrg-sr400ac.dts index 51c64f0b2560..9ca6d1b2590d 100644 --- a/arch/arm/boot/dts/bcm4708-smartrg-sr400ac.dts +++ b/arch/arm/boot/dts/bcm4708-smartrg-sr400ac.dts @@ -21,8 +21,8 @@ memory@0 { device_type = "memory"; - reg = <0x00000000 0x08000000 - 0x88000000 0x08000000>; + reg = <0x00000000 0x08000000>, + <0x88000000 0x08000000>; }; leds { diff --git a/arch/arm/boot/dts/bcm47081-asus-rt-n18u.dts b/arch/arm/boot/dts/bcm47081-asus-rt-n18u.dts index c29950b43a95..0e273c598732 100644 --- a/arch/arm/boot/dts/bcm47081-asus-rt-n18u.dts +++ b/arch/arm/boot/dts/bcm47081-asus-rt-n18u.dts @@ -21,8 +21,8 @@ memory@0 { device_type = "memory"; - reg = <0x00000000 0x08000000 - 0x88000000 0x08000000>; + reg = <0x00000000 0x08000000>, + <0x88000000 0x08000000>; }; leds { diff --git a/arch/arm/boot/dts/bcm47081-buffalo-wzr-600dhp2.dts b/arch/arm/boot/dts/bcm47081-buffalo-wzr-600dhp2.dts index 2f2d2b0a6893..d857751ec507 100644 --- a/arch/arm/boot/dts/bcm47081-buffalo-wzr-600dhp2.dts +++ b/arch/arm/boot/dts/bcm47081-buffalo-wzr-600dhp2.dts @@ -21,8 +21,8 @@ memory@0 { device_type = "memory"; - reg = <0x00000000 0x08000000 - 0x88000000 0x08000000>; + reg = <0x00000000 0x08000000>, + <0x88000000 0x08000000>; }; spi { diff --git a/arch/arm/boot/dts/bcm47081-buffalo-wzr-900dhp.dts b/arch/arm/boot/dts/bcm47081-buffalo-wzr-900dhp.dts index 0e349e39f608..8b1a05a0f1a1 100644 --- a/arch/arm/boot/dts/bcm47081-buffalo-wzr-900dhp.dts +++ b/arch/arm/boot/dts/bcm47081-buffalo-wzr-900dhp.dts @@ -21,8 +21,8 @@ memory@0 { device_type = "memory"; - reg = <0x00000000 0x08000000 - 0x88000000 0x08000000>; + reg = <0x00000000 0x08000000>, + <0x88000000 0x08000000>; }; spi { diff --git a/arch/arm/boot/dts/bcm4709-asus-rt-ac87u.dts b/arch/arm/boot/dts/bcm4709-asus-rt-ac87u.dts index 8f1e565c3db4..6c6bb7b17d27 100644 --- a/arch/arm/boot/dts/bcm4709-asus-rt-ac87u.dts +++ b/arch/arm/boot/dts/bcm4709-asus-rt-ac87u.dts @@ -21,8 +21,8 @@ memory { device_type = "memory"; - reg = <0x00000000 0x08000000 - 0x88000000 0x08000000>; + reg = <0x00000000 0x08000000>, + <0x88000000 0x08000000>; }; leds { diff --git a/arch/arm/boot/dts/bcm4709-buffalo-wxr-1900dhp.dts b/arch/arm/boot/dts/bcm4709-buffalo-wxr-1900dhp.dts index ce888b1835d1..d29e7f80ea6a 100644 --- a/arch/arm/boot/dts/bcm4709-buffalo-wxr-1900dhp.dts +++ b/arch/arm/boot/dts/bcm4709-buffalo-wxr-1900dhp.dts @@ -21,8 +21,8 @@ memory { device_type = "memory"; - reg = <0x00000000 0x08000000 - 0x88000000 0x18000000>; + reg = <0x00000000 0x08000000>, + <0x88000000 0x18000000>; }; leds { diff --git a/arch/arm/boot/dts/bcm4709-linksys-ea9200.dts b/arch/arm/boot/dts/bcm4709-linksys-ea9200.dts index ed8619b54d69..38fbefdf2e4e 100644 --- a/arch/arm/boot/dts/bcm4709-linksys-ea9200.dts +++ b/arch/arm/boot/dts/bcm4709-linksys-ea9200.dts @@ -18,8 +18,8 @@ memory { device_type = "memory"; - reg = <0x00000000 0x08000000 - 0x88000000 0x08000000>; + reg = <0x00000000 0x08000000>, + <0x88000000 0x08000000>; }; gpio-keys { diff --git a/arch/arm/boot/dts/bcm4709-netgear-r7000.dts b/arch/arm/boot/dts/bcm4709-netgear-r7000.dts index 1f87993eae1d..7989a53597d4 100644 --- a/arch/arm/boot/dts/bcm4709-netgear-r7000.dts +++ b/arch/arm/boot/dts/bcm4709-netgear-r7000.dts @@ -21,8 +21,8 @@ memory { device_type = "memory"; - reg = <0x00000000 0x08000000 - 0x88000000 0x08000000>; + reg = <0x00000000 0x08000000>, + <0x88000000 0x08000000>; }; leds { diff --git a/arch/arm/boot/dts/bcm4709-netgear-r8000.dts b/arch/arm/boot/dts/bcm4709-netgear-r8000.dts index 6c6199a53d09..87b655be674c 100644 --- a/arch/arm/boot/dts/bcm4709-netgear-r8000.dts +++ b/arch/arm/boot/dts/bcm4709-netgear-r8000.dts @@ -32,8 +32,8 @@ memory { device_type = "memory"; - reg = <0x00000000 0x08000000 - 0x88000000 0x08000000>; + reg = <0x00000000 0x08000000>, + <0x88000000 0x08000000>; }; leds { diff --git a/arch/arm/boot/dts/bcm47094-dlink-dir-885l.dts b/arch/arm/boot/dts/bcm47094-dlink-dir-885l.dts index 911c65fbf251..e635a15041dd 100644 --- a/arch/arm/boot/dts/bcm47094-dlink-dir-885l.dts +++ b/arch/arm/boot/dts/bcm47094-dlink-dir-885l.dts @@ -21,8 +21,8 @@ memory@0 { device_type = "memory"; - reg = <0x00000000 0x08000000 - 0x88000000 0x08000000>; + reg = <0x00000000 0x08000000>, + <0x88000000 0x08000000>; }; nand: nand@18028000 { diff --git a/arch/arm/boot/dts/bcm47094-linksys-panamera.dts b/arch/arm/boot/dts/bcm47094-linksys-panamera.dts index 3725f2b0d60b..4b24b25389b5 100644 --- a/arch/arm/boot/dts/bcm47094-linksys-panamera.dts +++ b/arch/arm/boot/dts/bcm47094-linksys-panamera.dts @@ -18,8 +18,8 @@ memory@0 { device_type = "memory"; - reg = <0x00000000 0x08000000 - 0x88000000 0x08000000>; + reg = <0x00000000 0x08000000>, + <0x88000000 0x08000000>; }; gpio-keys { diff --git a/arch/arm/boot/dts/bcm47094-luxul-abr-4500.dts b/arch/arm/boot/dts/bcm47094-luxul-abr-4500.dts index 50f7cd08cfbb..a6dc99955e19 100644 --- a/arch/arm/boot/dts/bcm47094-luxul-abr-4500.dts +++ b/arch/arm/boot/dts/bcm47094-luxul-abr-4500.dts @@ -18,8 +18,8 @@ memory@0 { device_type = "memory"; - reg = <0x00000000 0x08000000 - 0x88000000 0x18000000>; + reg = <0x00000000 0x08000000>, + <0x88000000 0x18000000>; }; leds { diff --git a/arch/arm/boot/dts/bcm47094-luxul-xbr-4500.dts b/arch/arm/boot/dts/bcm47094-luxul-xbr-4500.dts index bcc420f85b56..ff98837bc0db 100644 --- a/arch/arm/boot/dts/bcm47094-luxul-xbr-4500.dts +++ b/arch/arm/boot/dts/bcm47094-luxul-xbr-4500.dts @@ -18,8 +18,8 @@ memory@0 { device_type = "memory"; - reg = <0x00000000 0x08000000 - 0x88000000 0x18000000>; + reg = <0x00000000 0x08000000>, + <0x88000000 0x18000000>; }; leds { diff --git a/arch/arm/boot/dts/bcm47094-luxul-xwc-2000.dts b/arch/arm/boot/dts/bcm47094-luxul-xwc-2000.dts index 4f8d777ae18d..452b8d0ab180 100644 --- a/arch/arm/boot/dts/bcm47094-luxul-xwc-2000.dts +++ b/arch/arm/boot/dts/bcm47094-luxul-xwc-2000.dts @@ -18,8 +18,8 @@ memory { device_type = "memory"; - reg = <0x00000000 0x08000000 - 0x88000000 0x18000000>; + reg = <0x00000000 0x08000000>, + <0x88000000 0x18000000>; }; leds { diff --git a/arch/arm/boot/dts/bcm47094-luxul-xwr-3100.dts b/arch/arm/boot/dts/bcm47094-luxul-xwr-3100.dts index e17e9a17fb00..b76bfe6efcd4 100644 --- a/arch/arm/boot/dts/bcm47094-luxul-xwr-3100.dts +++ b/arch/arm/boot/dts/bcm47094-luxul-xwr-3100.dts @@ -18,8 +18,8 @@ memory@0 { device_type = "memory"; - reg = <0x00000000 0x08000000 - 0x88000000 0x08000000>; + reg = <0x00000000 0x08000000>, + <0x88000000 0x08000000>; }; leds { diff --git a/arch/arm/boot/dts/bcm47094-luxul-xwr-3150-v1.dts b/arch/arm/boot/dts/bcm47094-luxul-xwr-3150-v1.dts index 60cc87ecc7ec..32d5a50578ec 100644 --- a/arch/arm/boot/dts/bcm47094-luxul-xwr-3150-v1.dts +++ b/arch/arm/boot/dts/bcm47094-luxul-xwr-3150-v1.dts @@ -18,8 +18,8 @@ memory@0 { device_type = "memory"; - reg = <0x00000000 0x08000000 - 0x88000000 0x18000000>; + reg = <0x00000000 0x08000000>, + <0x88000000 0x18000000>; }; leds { diff --git a/arch/arm/boot/dts/bcm47094-netgear-r8500.dts b/arch/arm/boot/dts/bcm47094-netgear-r8500.dts index f42a1703f4ab..42097a4c2659 100644 --- a/arch/arm/boot/dts/bcm47094-netgear-r8500.dts +++ b/arch/arm/boot/dts/bcm47094-netgear-r8500.dts @@ -18,8 +18,8 @@ memory@0 { device_type = "memory"; - reg = <0x00000000 0x08000000 - 0x88000000 0x18000000>; + reg = <0x00000000 0x08000000>, + <0x88000000 0x18000000>; }; leds { diff --git a/arch/arm/boot/dts/bcm47094-phicomm-k3.dts b/arch/arm/boot/dts/bcm47094-phicomm-k3.dts index ac3a4483dcb3..a2566ad4619c 100644 --- a/arch/arm/boot/dts/bcm47094-phicomm-k3.dts +++ b/arch/arm/boot/dts/bcm47094-phicomm-k3.dts @@ -15,8 +15,8 @@ memory@0 { device_type = "memory"; - reg = <0x00000000 0x08000000 - 0x88000000 0x18000000>; + reg = <0x00000000 0x08000000>, + <0x88000000 0x18000000>; }; gpio-keys { diff --git a/arch/arm/boot/dts/dra7-l4.dtsi b/arch/arm/boot/dts/dra7-l4.dtsi index 3bf90d9e3335..a294a02f2d23 100644 --- a/arch/arm/boot/dts/dra7-l4.dtsi +++ b/arch/arm/boot/dts/dra7-l4.dtsi @@ -1168,7 +1168,7 @@ }; }; - target-module@34000 { /* 0x48034000, ap 7 46.0 */ + timer3_target: target-module@34000 { /* 0x48034000, ap 7 46.0 */ compatible = "ti,sysc-omap4-timer", "ti,sysc"; reg = <0x34000 0x4>, <0x34010 0x4>; @@ -1195,7 +1195,7 @@ }; }; - target-module@36000 { /* 0x48036000, ap 9 4e.0 */ + timer4_target: target-module@36000 { /* 0x48036000, ap 9 4e.0 */ compatible = "ti,sysc-omap4-timer", "ti,sysc"; reg = <0x36000 0x4>, <0x36010 0x4>; diff --git a/arch/arm/boot/dts/dra7.dtsi b/arch/arm/boot/dts/dra7.dtsi index ce1194744f84..53d68786a61f 100644 --- a/arch/arm/boot/dts/dra7.dtsi +++ b/arch/arm/boot/dts/dra7.dtsi @@ -46,6 +46,7 @@ timer { compatible = "arm,armv7-timer"; + status = "disabled"; /* See ARM architected timer wrap erratum i940 */ interrupts = , , , @@ -1241,3 +1242,22 @@ assigned-clock-parents = <&sys_32k_ck>; }; }; + +/* Local timers, see ARM architected timer wrap erratum i940 */ +&timer3_target { + ti,no-reset-on-init; + ti,no-idle; + timer@0 { + assigned-clocks = <&l4per_clkctrl DRA7_L4PER_TIMER3_CLKCTRL 24>; + assigned-clock-parents = <&timer_sys_clk_div>; + }; +}; + +&timer4_target { + ti,no-reset-on-init; + ti,no-idle; + timer@0 { + assigned-clocks = <&l4per_clkctrl DRA7_L4PER_TIMER4_CLKCTRL 24>; + assigned-clock-parents = <&timer_sys_clk_div>; + }; +}; diff --git a/arch/arm/boot/dts/exynos4210-i9100.dts b/arch/arm/boot/dts/exynos4210-i9100.dts index 304a8ee2364c..d98c78207aaf 100644 --- a/arch/arm/boot/dts/exynos4210-i9100.dts +++ b/arch/arm/boot/dts/exynos4210-i9100.dts @@ -136,7 +136,7 @@ compatible = "maxim,max17042"; interrupt-parent = <&gpx2>; - interrupts = <3 IRQ_TYPE_EDGE_FALLING>; + interrupts = <3 IRQ_TYPE_LEVEL_LOW>; pinctrl-0 = <&max17042_fuel_irq>; pinctrl-names = "default"; diff --git a/arch/arm/boot/dts/exynos4412-midas.dtsi b/arch/arm/boot/dts/exynos4412-midas.dtsi index 111c32bae02c..fc77c1bfd844 100644 --- a/arch/arm/boot/dts/exynos4412-midas.dtsi +++ b/arch/arm/boot/dts/exynos4412-midas.dtsi @@ -173,7 +173,7 @@ pmic@66 { compatible = "maxim,max77693"; interrupt-parent = <&gpx1>; - interrupts = <5 IRQ_TYPE_EDGE_FALLING>; + interrupts = <5 IRQ_TYPE_LEVEL_LOW>; pinctrl-names = "default"; pinctrl-0 = <&max77693_irq>; reg = <0x66>; @@ -221,7 +221,7 @@ fuel-gauge@36 { compatible = "maxim,max17047"; interrupt-parent = <&gpx2>; - interrupts = <3 IRQ_TYPE_EDGE_FALLING>; + interrupts = <3 IRQ_TYPE_LEVEL_LOW>; pinctrl-names = "default"; pinctrl-0 = <&max77693_fuel_irq>; reg = <0x36>; @@ -665,7 +665,7 @@ max77686: pmic@9 { compatible = "maxim,max77686"; interrupt-parent = <&gpx0>; - interrupts = <7 IRQ_TYPE_NONE>; + interrupts = <7 IRQ_TYPE_LEVEL_LOW>; pinctrl-0 = <&max77686_irq>; pinctrl-names = "default"; reg = <0x09>; diff --git a/arch/arm/boot/dts/exynos4412-odroid-common.dtsi b/arch/arm/boot/dts/exynos4412-odroid-common.dtsi index 2b20d9095d9f..eebe6a3952ce 100644 --- a/arch/arm/boot/dts/exynos4412-odroid-common.dtsi +++ b/arch/arm/boot/dts/exynos4412-odroid-common.dtsi @@ -278,7 +278,7 @@ max77686: pmic@9 { compatible = "maxim,max77686"; interrupt-parent = <&gpx3>; - interrupts = <2 IRQ_TYPE_NONE>; + interrupts = <2 IRQ_TYPE_LEVEL_LOW>; pinctrl-names = "default"; pinctrl-0 = <&max77686_irq>; reg = <0x09>; diff --git a/arch/arm/boot/dts/exynos4412-p4note.dtsi b/arch/arm/boot/dts/exynos4412-p4note.dtsi index b2f9d5448a18..9e750890edb8 100644 --- a/arch/arm/boot/dts/exynos4412-p4note.dtsi +++ b/arch/arm/boot/dts/exynos4412-p4note.dtsi @@ -146,7 +146,7 @@ pinctrl-0 = <&fuel_alert_irq>; pinctrl-names = "default"; interrupt-parent = <&gpx2>; - interrupts = <3 IRQ_TYPE_EDGE_FALLING>; + interrupts = <3 IRQ_TYPE_LEVEL_LOW>; maxim,rsns-microohm = <10000>; maxim,over-heat-temp = <600>; maxim,over-volt = <4300>; @@ -322,7 +322,7 @@ max77686: pmic@9 { compatible = "maxim,max77686"; interrupt-parent = <&gpx0>; - interrupts = <7 IRQ_TYPE_NONE>; + interrupts = <7 IRQ_TYPE_LEVEL_LOW>; pinctrl-0 = <&max77686_irq>; pinctrl-names = "default"; reg = <0x09>; diff --git a/arch/arm/boot/dts/exynos5250-smdk5250.dts b/arch/arm/boot/dts/exynos5250-smdk5250.dts index 8b5a79a8720c..39bbe18145cf 100644 --- a/arch/arm/boot/dts/exynos5250-smdk5250.dts +++ b/arch/arm/boot/dts/exynos5250-smdk5250.dts @@ -134,7 +134,7 @@ compatible = "maxim,max77686"; reg = <0x09>; interrupt-parent = <&gpx3>; - interrupts = <2 IRQ_TYPE_NONE>; + interrupts = <2 IRQ_TYPE_LEVEL_LOW>; pinctrl-names = "default"; pinctrl-0 = <&max77686_irq>; #clock-cells = <1>; diff --git a/arch/arm/boot/dts/exynos5250-snow-common.dtsi b/arch/arm/boot/dts/exynos5250-snow-common.dtsi index 6635f6184051..2335c4687349 100644 --- a/arch/arm/boot/dts/exynos5250-snow-common.dtsi +++ b/arch/arm/boot/dts/exynos5250-snow-common.dtsi @@ -292,7 +292,7 @@ max77686: pmic@9 { compatible = "maxim,max77686"; interrupt-parent = <&gpx3>; - interrupts = <2 IRQ_TYPE_NONE>; + interrupts = <2 IRQ_TYPE_LEVEL_LOW>; pinctrl-names = "default"; pinctrl-0 = <&max77686_irq>; wakeup-source; diff --git a/arch/arm/boot/dts/qcom-msm8974-lge-nexus5-hammerhead.dts b/arch/arm/boot/dts/qcom-msm8974-lge-nexus5-hammerhead.dts index 0cda654371ae..56ee02ceba7d 100644 --- a/arch/arm/boot/dts/qcom-msm8974-lge-nexus5-hammerhead.dts +++ b/arch/arm/boot/dts/qcom-msm8974-lge-nexus5-hammerhead.dts @@ -575,7 +575,7 @@ maxim,rcomp = /bits/ 8 <0x4d>; interrupt-parent = <&msmgpio>; - interrupts = <9 IRQ_TYPE_EDGE_FALLING>; + interrupts = <9 IRQ_TYPE_LEVEL_LOW>; pinctrl-names = "default"; pinctrl-0 = <&fuelgauge_pin>; diff --git a/arch/arm/boot/dts/qcom-msm8974-samsung-klte.dts b/arch/arm/boot/dts/qcom-msm8974-samsung-klte.dts index a0f7f461f48c..2dadb836c5fe 100644 --- a/arch/arm/boot/dts/qcom-msm8974-samsung-klte.dts +++ b/arch/arm/boot/dts/qcom-msm8974-samsung-klte.dts @@ -717,7 +717,7 @@ maxim,rcomp = /bits/ 8 <0x56>; interrupt-parent = <&pma8084_gpios>; - interrupts = <21 IRQ_TYPE_EDGE_FALLING>; + interrupts = <21 IRQ_TYPE_LEVEL_LOW>; pinctrl-names = "default"; pinctrl-0 = <&fuelgauge_pin>; diff --git a/arch/arm/boot/dts/r8a7790-lager.dts b/arch/arm/boot/dts/r8a7790-lager.dts index 09a152b91557..1d6f0c5d02e9 100644 --- a/arch/arm/boot/dts/r8a7790-lager.dts +++ b/arch/arm/boot/dts/r8a7790-lager.dts @@ -53,6 +53,9 @@ i2c11 = &i2cexio1; i2c12 = &i2chdmi; i2c13 = &i2cpwr; + mmc0 = &mmcif1; + mmc1 = &sdhi0; + mmc2 = &sdhi2; }; chosen { diff --git a/arch/arm/boot/dts/r8a7791-koelsch.dts b/arch/arm/boot/dts/r8a7791-koelsch.dts index f603cba5441f..6af1727b8269 100644 --- a/arch/arm/boot/dts/r8a7791-koelsch.dts +++ b/arch/arm/boot/dts/r8a7791-koelsch.dts @@ -53,6 +53,9 @@ i2c12 = &i2cexio1; i2c13 = &i2chdmi; i2c14 = &i2cexio4; + mmc0 = &sdhi0; + mmc1 = &sdhi1; + mmc2 = &sdhi2; }; chosen { diff --git a/arch/arm/boot/dts/r8a7791-porter.dts b/arch/arm/boot/dts/r8a7791-porter.dts index c6d563fb7ec7..bf51e29c793a 100644 --- a/arch/arm/boot/dts/r8a7791-porter.dts +++ b/arch/arm/boot/dts/r8a7791-porter.dts @@ -28,6 +28,8 @@ serial0 = &scif0; i2c9 = &gpioi2c2; i2c10 = &i2chdmi; + mmc0 = &sdhi0; + mmc1 = &sdhi2; }; chosen { diff --git a/arch/arm/boot/dts/r8a7793-gose.dts b/arch/arm/boot/dts/r8a7793-gose.dts index abf487e8fe0f..2b59a0491350 100644 --- a/arch/arm/boot/dts/r8a7793-gose.dts +++ b/arch/arm/boot/dts/r8a7793-gose.dts @@ -49,6 +49,9 @@ i2c10 = &gpioi2c4; i2c11 = &i2chdmi; i2c12 = &i2cexio4; + mmc0 = &sdhi0; + mmc1 = &sdhi1; + mmc2 = &sdhi2; }; chosen { diff --git a/arch/arm/boot/dts/r8a7794-alt.dts b/arch/arm/boot/dts/r8a7794-alt.dts index 3f1cc5bbf329..32025986b3b9 100644 --- a/arch/arm/boot/dts/r8a7794-alt.dts +++ b/arch/arm/boot/dts/r8a7794-alt.dts @@ -19,6 +19,9 @@ i2c10 = &gpioi2c4; i2c11 = &i2chdmi; i2c12 = &i2cexio4; + mmc0 = &mmcif0; + mmc1 = &sdhi0; + mmc2 = &sdhi1; }; chosen { diff --git a/arch/arm/boot/dts/r8a7794-silk.dts b/arch/arm/boot/dts/r8a7794-silk.dts index 677596f6c9c9..af066ee5e275 100644 --- a/arch/arm/boot/dts/r8a7794-silk.dts +++ b/arch/arm/boot/dts/r8a7794-silk.dts @@ -31,6 +31,8 @@ serial0 = &scif2; i2c9 = &gpioi2c1; i2c10 = &i2chdmi; + mmc0 = &mmcif0; + mmc1 = &sdhi1; }; chosen { diff --git a/arch/arm/boot/dts/s5pv210-fascinate4g.dts b/arch/arm/boot/dts/s5pv210-fascinate4g.dts index ca064359dd30..b47d8300e536 100644 --- a/arch/arm/boot/dts/s5pv210-fascinate4g.dts +++ b/arch/arm/boot/dts/s5pv210-fascinate4g.dts @@ -115,7 +115,7 @@ compatible = "maxim,max77836-battery"; interrupt-parent = <&gph3>; - interrupts = <3 IRQ_TYPE_EDGE_FALLING>; + interrupts = <3 IRQ_TYPE_LEVEL_LOW>; pinctrl-names = "default"; pinctrl-0 = <&fg_irq>; diff --git a/arch/arm/boot/dts/ste-href-tvk1281618-r3.dtsi b/arch/arm/boot/dts/ste-href-tvk1281618-r3.dtsi index cb3677f0a1cb..b580397ede83 100644 --- a/arch/arm/boot/dts/ste-href-tvk1281618-r3.dtsi +++ b/arch/arm/boot/dts/ste-href-tvk1281618-r3.dtsi @@ -8,37 +8,43 @@ / { soc { i2c@80128000 { - /* Marked: - * 129 - * M35 - * L3GD20 - */ - l3gd20@6a { - /* Gyroscope */ - compatible = "st,l3gd20"; - status = "disabled"; + accelerometer@19 { + compatible = "st,lsm303dlhc-accel"; st,drdy-int-pin = <1>; - drive-open-drain; - reg = <0x6a>; // 0x6a or 0x6b + reg = <0x19>; vdd-supply = <&ab8500_ldo_aux1_reg>; vddio-supply = <&db8500_vsmps2_reg>; + interrupt-parent = <&gpio2>; + interrupts = <18 IRQ_TYPE_EDGE_RISING>, + <19 IRQ_TYPE_EDGE_RISING>; + pinctrl-names = "default"; + pinctrl-0 = <&accel_tvk_mode>; }; - /* - * Marked: - * 2122 - * C3H - * DQEEE - * LIS3DH? - */ - lis3dh@18 { - /* Accelerometer */ - compatible = "st,lis3dh-accel"; + magnetometer@1e { + compatible = "st,lsm303dlm-magn"; st,drdy-int-pin = <1>; - reg = <0x18>; + reg = <0x1e>; vdd-supply = <&ab8500_ldo_aux1_reg>; vddio-supply = <&db8500_vsmps2_reg>; + // This interrupt is not properly working with the driver + // interrupt-parent = <&gpio1>; + // interrupts = <0 IRQ_TYPE_EDGE_RISING>; pinctrl-names = "default"; - pinctrl-0 = <&accel_tvk_mode>; + pinctrl-0 = <&magn_tvk_mode>; + }; + gyroscope@68 { + /* Gyroscope */ + compatible = "st,l3g4200d-gyro"; + reg = <0x68>; + vdd-supply = <&ab8500_ldo_aux1_reg>; + vddio-supply = <&db8500_vsmps2_reg>; + }; + pressure@5c { + /* Barometer/pressure sensor */ + compatible = "st,lps001wp-press"; + reg = <0x5c>; + vdd-supply = <&ab8500_ldo_aux1_reg>; + vddio-supply = <&db8500_vsmps2_reg>; }; }; @@ -54,5 +60,26 @@ }; }; }; + + pinctrl { + accelerometer { + accel_tvk_mode: accel_tvk { + /* Accelerometer interrupt lines 1 & 2 */ + tvk_cfg { + pins = "GPIO82_C1", "GPIO83_D3"; + ste,config = <&gpio_in_pd>; + }; + }; + }; + magnetometer { + magn_tvk_mode: magn_tvk { + /* GPIO 32 used for DRDY, pull this down */ + tvk_cfg { + pins = "GPIO32_V2"; + ste,config = <&gpio_in_pd>; + }; + }; + }; + }; }; }; diff --git a/arch/arm/boot/dts/stm32mp15-pinctrl.dtsi b/arch/arm/boot/dts/stm32mp15-pinctrl.dtsi index 7b4249ed1983..060baa8b7e9d 100644 --- a/arch/arm/boot/dts/stm32mp15-pinctrl.dtsi +++ b/arch/arm/boot/dts/stm32mp15-pinctrl.dtsi @@ -1891,10 +1891,15 @@ usart2_idle_pins_c: usart2-idle-2 { pins1 { pinmux = , /* USART2_TX */ - , /* USART2_RTS */ ; /* USART2_CTS_NSS */ }; pins2 { + pinmux = ; /* USART2_RTS */ + bias-disable; + drive-push-pull; + slew-rate = <3>; + }; + pins3 { pinmux = ; /* USART2_RX */ bias-disable; }; @@ -1940,10 +1945,15 @@ usart3_idle_pins_b: usart3-idle-1 { pins1 { pinmux = , /* USART3_TX */ - , /* USART3_RTS */ ; /* USART3_CTS_NSS */ }; pins2 { + pinmux = ; /* USART3_RTS */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + pins3 { pinmux = ; /* USART3_RX */ bias-disable; }; @@ -1976,10 +1986,15 @@ usart3_idle_pins_c: usart3-idle-2 { pins1 { pinmux = , /* USART3_TX */ - , /* USART3_RTS */ ; /* USART3_CTS_NSS */ }; pins2 { + pinmux = ; /* USART3_RTS */ + bias-disable; + drive-push-pull; + slew-rate = <0>; + }; + pins3 { pinmux = ; /* USART3_RX */ bias-disable; }; diff --git a/arch/arm/boot/dts/tegra20-acer-a500-picasso.dts b/arch/arm/boot/dts/tegra20-acer-a500-picasso.dts index d3b99535d755..f9c0f6884cc1 100644 --- a/arch/arm/boot/dts/tegra20-acer-a500-picasso.dts +++ b/arch/arm/boot/dts/tegra20-acer-a500-picasso.dts @@ -448,7 +448,7 @@ reset-gpios = <&gpio TEGRA_GPIO(Q, 7) GPIO_ACTIVE_LOW>; - avdd-supply = <&vdd_3v3_sys>; + vdda-supply = <&vdd_3v3_sys>; vdd-supply = <&vdd_3v3_sys>; }; diff --git a/arch/arm/boot/dts/uniphier-pxs2.dtsi b/arch/arm/boot/dts/uniphier-pxs2.dtsi index b0b15c97306b..e81e5937a60a 100644 --- a/arch/arm/boot/dts/uniphier-pxs2.dtsi +++ b/arch/arm/boot/dts/uniphier-pxs2.dtsi @@ -583,7 +583,7 @@ clocks = <&sys_clk 6>; reset-names = "ether"; resets = <&sys_rst 6>; - phy-mode = "rgmii"; + phy-mode = "rgmii-id"; local-mac-address = [00 00 00 00 00 00]; socionext,syscon-phy-mode = <&soc_glue 0>; diff --git a/arch/arm/crypto/blake2s-core.S b/arch/arm/crypto/blake2s-core.S index bed897e9a181..86345751bbf3 100644 --- a/arch/arm/crypto/blake2s-core.S +++ b/arch/arm/crypto/blake2s-core.S @@ -8,6 +8,7 @@ */ #include +#include // Registers used to hold message words temporarily. There aren't // enough ARM registers to hold the whole message block, so we have to @@ -38,6 +39,23 @@ #endif .endm +.macro _le32_bswap a, tmp +#ifdef __ARMEB__ + rev_l \a, \tmp +#endif +.endm + +.macro _le32_bswap_8x a, b, c, d, e, f, g, h, tmp + _le32_bswap \a, \tmp + _le32_bswap \b, \tmp + _le32_bswap \c, \tmp + _le32_bswap \d, \tmp + _le32_bswap \e, \tmp + _le32_bswap \f, \tmp + _le32_bswap \g, \tmp + _le32_bswap \h, \tmp +.endm + // Execute a quarter-round of BLAKE2s by mixing two columns or two diagonals. // (a0, b0, c0, d0) and (a1, b1, c1, d1) give the registers containing the two // columns/diagonals. s0-s1 are the word offsets to the message words the first @@ -180,8 +198,10 @@ ENTRY(blake2s_compress_arch) tst r1, #3 bne .Lcopy_block_misaligned ldmia r1!, {r2-r9} + _le32_bswap_8x r2, r3, r4, r5, r6, r7, r8, r9, r14 stmia r12!, {r2-r9} ldmia r1!, {r2-r9} + _le32_bswap_8x r2, r3, r4, r5, r6, r7, r8, r9, r14 stmia r12, {r2-r9} .Lcopy_block_done: str r1, [sp, #68] // Update message pointer @@ -268,6 +288,7 @@ ENTRY(blake2s_compress_arch) 1: #ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS ldr r3, [r1], #4 + _le32_bswap r3, r4 #else ldrb r3, [r1, #0] ldrb r4, [r1, #1] diff --git a/arch/arm/crypto/curve25519-core.S b/arch/arm/crypto/curve25519-core.S index be18af52e7dc..b697fa5d059a 100644 --- a/arch/arm/crypto/curve25519-core.S +++ b/arch/arm/crypto/curve25519-core.S @@ -10,8 +10,8 @@ #include .text -.fpu neon .arch armv7-a +.fpu neon .align 4 ENTRY(curve25519_neon) diff --git a/arch/arm/crypto/poly1305-glue.c b/arch/arm/crypto/poly1305-glue.c index 3023c1acfa19..c31bd8f7c092 100644 --- a/arch/arm/crypto/poly1305-glue.c +++ b/arch/arm/crypto/poly1305-glue.c @@ -29,7 +29,7 @@ void __weak poly1305_blocks_neon(void *state, const u8 *src, u32 len, u32 hibit) static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); -void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 *key) +void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) { poly1305_init_arm(&dctx->h, key); dctx->s[0] = get_unaligned_le32(key + 16); diff --git a/arch/arm/kernel/asm-offsets.c b/arch/arm/kernel/asm-offsets.c index be8050b0c3df..70993af22d80 100644 --- a/arch/arm/kernel/asm-offsets.c +++ b/arch/arm/kernel/asm-offsets.c @@ -24,6 +24,7 @@ #include #include #include +#include #include "signal.h" /* @@ -148,6 +149,8 @@ int main(void) DEFINE(SLEEP_SAVE_SP_PHYS, offsetof(struct sleep_save_sp, save_ptr_stash_phys)); DEFINE(SLEEP_SAVE_SP_VIRT, offsetof(struct sleep_save_sp, save_ptr_stash)); #endif + DEFINE(ARM_SMCCC_QUIRK_ID_OFFS, offsetof(struct arm_smccc_quirk, id)); + DEFINE(ARM_SMCCC_QUIRK_STATE_OFFS, offsetof(struct arm_smccc_quirk, state)); BLANK(); DEFINE(DMA_BIDIRECTIONAL, DMA_BIDIRECTIONAL); DEFINE(DMA_TO_DEVICE, DMA_TO_DEVICE); diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c index 08660ae9dcbc..b1423fb130ea 100644 --- a/arch/arm/kernel/hw_breakpoint.c +++ b/arch/arm/kernel/hw_breakpoint.c @@ -886,7 +886,7 @@ static void breakpoint_handler(unsigned long unknown, struct pt_regs *regs) info->trigger = addr; pr_debug("breakpoint fired: address = 0x%x\n", addr); perf_bp_event(bp, regs); - if (!bp->overflow_handler) + if (is_default_overflow_handler(bp)) enable_single_step(bp, addr); goto unlock; } diff --git a/arch/arm/kernel/smccc-call.S b/arch/arm/kernel/smccc-call.S index 00664c78faca..931df62a7831 100644 --- a/arch/arm/kernel/smccc-call.S +++ b/arch/arm/kernel/smccc-call.S @@ -3,7 +3,9 @@ * Copyright (c) 2015, Linaro Limited */ #include +#include +#include #include #include #include @@ -27,7 +29,14 @@ UNWIND( .fnstart) UNWIND( .save {r4-r7}) ldm r12, {r4-r7} \instr - pop {r4-r7} + ldr r4, [sp, #36] + cmp r4, #0 + beq 1f // No quirk structure + ldr r5, [r4, #ARM_SMCCC_QUIRK_ID_OFFS] + cmp r5, #ARM_SMCCC_QUIRK_QCOM_A6 + bne 1f // No quirk present + str r6, [r4, #ARM_SMCCC_QUIRK_STATE_OFFS] +1: pop {r4-r7} ldr r12, [sp, #(4 * 4)] stm r12, {r0-r3} bx lr diff --git a/arch/arm/kernel/suspend.c b/arch/arm/kernel/suspend.c index 24bd20564be7..43f0a3ebf390 100644 --- a/arch/arm/kernel/suspend.c +++ b/arch/arm/kernel/suspend.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include #include @@ -25,6 +26,13 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) if (!idmap_pgd) return -EINVAL; + /* + * Function graph tracer state gets incosistent when the kernel + * calls functions that never return (aka suspend finishers) hence + * disable graph tracing during their execution. + */ + pause_graph_tracing(); + /* * Provide a temporary page table with an identity mapping for * the MMU-enable code, required for resuming. On successful @@ -32,6 +40,9 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) * back to the correct page tables. */ ret = __cpu_suspend(arg, fn, __mpidr); + + unpause_graph_tracing(); + if (ret == 0) { cpu_switch_mm(mm->pgd, mm); local_flush_bp_all(); @@ -45,7 +56,13 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) { u32 __mpidr = cpu_logical_map(smp_processor_id()); - return __cpu_suspend(arg, fn, __mpidr); + int ret; + + pause_graph_tracing(); + ret = __cpu_suspend(arg, fn, __mpidr); + unpause_graph_tracing(); + + return ret; } #define idmap_pgd NULL #endif diff --git a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908-asus-gt-ac5300.dts b/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908-asus-gt-ac5300.dts index 6e4ad66ff536..8d5d368dbe90 100644 --- a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908-asus-gt-ac5300.dts +++ b/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908-asus-gt-ac5300.dts @@ -65,6 +65,7 @@ port@7 { label = "sw"; reg = <7>; + phy-mode = "rgmii"; fixed-link { speed = <1000>; diff --git a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908.dtsi b/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908.dtsi index 9354077f74cd..9e799328c6db 100644 --- a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908.dtsi +++ b/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908.dtsi @@ -131,7 +131,7 @@ status = "disabled"; }; - ethernet-switch@80000 { + bus@80000 { compatible = "simple-bus"; #size-cells = <1>; #address-cells = <1>; diff --git a/arch/arm64/boot/dts/freescale/imx8mq-librem5-r3.dts b/arch/arm64/boot/dts/freescale/imx8mq-librem5-r3.dts index 0d38327043f8..cd3c3edd48fa 100644 --- a/arch/arm64/boot/dts/freescale/imx8mq-librem5-r3.dts +++ b/arch/arm64/boot/dts/freescale/imx8mq-librem5-r3.dts @@ -28,6 +28,10 @@ ti,termination-current = <144000>; /* uA */ }; +&buck3_reg { + regulator-always-on; +}; + &proximity { proximity-near-level = <25>; }; diff --git a/arch/arm64/boot/dts/marvell/armada-37xx.dtsi b/arch/arm64/boot/dts/marvell/armada-37xx.dtsi index 7a2df148c6a3..456dcd4a7793 100644 --- a/arch/arm64/boot/dts/marvell/armada-37xx.dtsi +++ b/arch/arm64/boot/dts/marvell/armada-37xx.dtsi @@ -156,7 +156,8 @@ }; nb_periph_clk: nb-periph-clk@13000 { - compatible = "marvell,armada-3700-periph-clock-nb"; + compatible = "marvell,armada-3700-periph-clock-nb", + "syscon"; reg = <0x13000 0x100>; clocks = <&tbg 0>, <&tbg 1>, <&tbg 2>, <&tbg 3>, <&xtalclk>; diff --git a/arch/arm64/boot/dts/mediatek/mt8173-evb.dts b/arch/arm64/boot/dts/mediatek/mt8173-evb.dts index 6dffada2e66b..28aa634c9780 100644 --- a/arch/arm64/boot/dts/mediatek/mt8173-evb.dts +++ b/arch/arm64/boot/dts/mediatek/mt8173-evb.dts @@ -294,7 +294,7 @@ &pwrap { /* Only MT8173 E1 needs USB power domain */ - power-domains = <&scpsys MT8173_POWER_DOMAIN_USB>; + power-domains = <&spm MT8173_POWER_DOMAIN_USB>; pmic: mt6397 { compatible = "mediatek,mt6397"; diff --git a/arch/arm64/boot/dts/mediatek/mt8173.dtsi b/arch/arm64/boot/dts/mediatek/mt8173.dtsi index 7fa870e4386a..ecb37a7e6870 100644 --- a/arch/arm64/boot/dts/mediatek/mt8173.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8173.dtsi @@ -1235,7 +1235,7 @@ <&mmsys CLK_MM_DSI1_DIGITAL>, <&mipi_tx1>; clock-names = "engine", "digital", "hs"; - phy = <&mipi_tx1>; + phys = <&mipi_tx1>; phy-names = "dphy"; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/mediatek/mt8183.dtsi b/arch/arm64/boot/dts/mediatek/mt8183.dtsi index 80519a145f13..16f4b1fc0fb9 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8183.dtsi @@ -983,6 +983,9 @@ compatible = "mediatek,mt8183-mmsys", "syscon"; reg = <0 0x14000000 0 0x1000>; #clock-cells = <1>; + mboxes = <&gce 0 CMDQ_THR_PRIO_HIGHEST>, + <&gce 1 CMDQ_THR_PRIO_HIGHEST>; + mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0 0x1000>; }; ovl0: ovl@14008000 { @@ -1058,6 +1061,7 @@ interrupts = ; power-domains = <&spm MT8183_POWER_DOMAIN_DISP>; clocks = <&mmsys CLK_MM_DISP_CCORR0>; + mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0xf000 0x1000>; }; aal0: aal@14010000 { @@ -1067,6 +1071,7 @@ interrupts = ; power-domains = <&spm MT8183_POWER_DOMAIN_DISP>; clocks = <&mmsys CLK_MM_DISP_AAL0>; + mediatek,gce-client-reg = <&gce SUBSYS_1401XXXX 0 0x1000>; }; gamma0: gamma@14011000 { @@ -1075,6 +1080,7 @@ interrupts = ; power-domains = <&spm MT8183_POWER_DOMAIN_DISP>; clocks = <&mmsys CLK_MM_DISP_GAMMA0>; + mediatek,gce-client-reg = <&gce SUBSYS_1401XXXX 0x1000 0x1000>; }; dither0: dither@14012000 { @@ -1083,6 +1089,7 @@ interrupts = ; power-domains = <&spm MT8183_POWER_DOMAIN_DISP>; clocks = <&mmsys CLK_MM_DISP_DITHER0>; + mediatek,gce-client-reg = <&gce SUBSYS_1401XXXX 0x2000 0x1000>; }; dsi0: dsi@14014000 { diff --git a/arch/arm64/boot/dts/mediatek/pumpkin-common.dtsi b/arch/arm64/boot/dts/mediatek/pumpkin-common.dtsi index 63fd70086bb8..9f27e7ed5e22 100644 --- a/arch/arm64/boot/dts/mediatek/pumpkin-common.dtsi +++ b/arch/arm64/boot/dts/mediatek/pumpkin-common.dtsi @@ -56,7 +56,7 @@ tca6416: gpio@20 { compatible = "ti,tca6416"; reg = <0x20>; - reset-gpios = <&pio 65 GPIO_ACTIVE_HIGH>; + reset-gpios = <&pio 65 GPIO_ACTIVE_LOW>; pinctrl-names = "default"; pinctrl-0 = <&tca6416_pins>; diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi b/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi index 07c8b2c926c0..b8f7cf5cbdab 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi @@ -22,9 +22,11 @@ thermal-sensors = <&pm6150_adc_tm 1>; trips { - temperature = <125000>; - hysteresis = <1000>; - type = "critical"; + charger-crit { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; }; }; }; @@ -768,17 +770,17 @@ hp_i2c: &i2c9 { }; &spi0 { - pinctrl-0 = <&qup_spi0_cs_gpio>; + pinctrl-0 = <&qup_spi0_cs_gpio_init_high>, <&qup_spi0_cs_gpio>; cs-gpios = <&tlmm 37 GPIO_ACTIVE_LOW>; }; &spi6 { - pinctrl-0 = <&qup_spi6_cs_gpio>; + pinctrl-0 = <&qup_spi6_cs_gpio_init_high>, <&qup_spi6_cs_gpio>; cs-gpios = <&tlmm 62 GPIO_ACTIVE_LOW>; }; ap_spi_fp: &spi10 { - pinctrl-0 = <&qup_spi10_cs_gpio>; + pinctrl-0 = <&qup_spi10_cs_gpio_init_high>, <&qup_spi10_cs_gpio>; cs-gpios = <&tlmm 89 GPIO_ACTIVE_LOW>; cros_ec_fp: ec@0 { @@ -1339,6 +1341,27 @@ ap_spi_fp: &spi10 { }; }; + qup_spi0_cs_gpio_init_high: qup-spi0-cs-gpio-init-high { + pinconf { + pins = "gpio37"; + output-high; + }; + }; + + qup_spi6_cs_gpio_init_high: qup-spi6-cs-gpio-init-high { + pinconf { + pins = "gpio62"; + output-high; + }; + }; + + qup_spi10_cs_gpio_init_high: qup-spi10-cs-gpio-init-high { + pinconf { + pins = "gpio89"; + output-high; + }; + }; + qup_uart3_sleep: qup-uart3-sleep { pinmux { pins = "gpio38", "gpio39", diff --git a/arch/arm64/boot/dts/qcom/sdm845-db845c.dts b/arch/arm64/boot/dts/qcom/sdm845-db845c.dts index c4ac6f5dc008..96d36b38f269 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-db845c.dts +++ b/arch/arm64/boot/dts/qcom/sdm845-db845c.dts @@ -1015,7 +1015,7 @@ left_spkr: wsa8810-left{ compatible = "sdw10217201000"; reg = <0 1>; - powerdown-gpios = <&wcdgpio 2 GPIO_ACTIVE_HIGH>; + powerdown-gpios = <&wcdgpio 1 GPIO_ACTIVE_HIGH>; #thermal-sensor-cells = <0>; sound-name-prefix = "SpkrLeft"; #sound-dai-cells = <0>; @@ -1023,7 +1023,7 @@ right_spkr: wsa8810-right{ compatible = "sdw10217201000"; - powerdown-gpios = <&wcdgpio 2 GPIO_ACTIVE_HIGH>; + powerdown-gpios = <&wcdgpio 1 GPIO_ACTIVE_HIGH>; reg = <0 2>; #thermal-sensor-cells = <0>; sound-name-prefix = "SpkrRight"; diff --git a/arch/arm64/boot/dts/qcom/sdm845.dtsi b/arch/arm64/boot/dts/qcom/sdm845.dtsi index 454f794af547..6a2ed02d383d 100644 --- a/arch/arm64/boot/dts/qcom/sdm845.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm845.dtsi @@ -2382,7 +2382,7 @@ #gpio-cells = <2>; interrupt-controller; #interrupt-cells = <2>; - gpio-ranges = <&tlmm 0 0 150>; + gpio-ranges = <&tlmm 0 0 151>; wakeup-parent = <&pdc_intc>; cci0_default: cci0-default { diff --git a/arch/arm64/boot/dts/qcom/sm8150.dtsi b/arch/arm64/boot/dts/qcom/sm8150.dtsi index e5bb17bc2f46..778613d3410b 100644 --- a/arch/arm64/boot/dts/qcom/sm8150.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8150.dtsi @@ -914,7 +914,7 @@ <0x0 0x03D00000 0x0 0x300000>; reg-names = "west", "east", "north", "south"; interrupts = ; - gpio-ranges = <&tlmm 0 0 175>; + gpio-ranges = <&tlmm 0 0 176>; gpio-controller; #gpio-cells = <2>; interrupt-controller; diff --git a/arch/arm64/boot/dts/qcom/sm8250.dtsi b/arch/arm64/boot/dts/qcom/sm8250.dtsi index 947e1accae3a..46a6c18cea91 100644 --- a/arch/arm64/boot/dts/qcom/sm8250.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8250.dtsi @@ -279,7 +279,7 @@ pmu { compatible = "arm,armv8-pmuv3"; - interrupts = ; + interrupts = ; }; psci { @@ -2327,10 +2327,9 @@ reg = <0 0x0ae00000 0 0x1000>; reg-names = "mdss"; - interconnects = <&gem_noc MASTER_AMPSS_M0 &config_noc SLAVE_DISPLAY_CFG>, - <&mmss_noc MASTER_MDP_PORT0 &mc_virt SLAVE_EBI_CH0>, + interconnects = <&mmss_noc MASTER_MDP_PORT0 &mc_virt SLAVE_EBI_CH0>, <&mmss_noc MASTER_MDP_PORT1 &mc_virt SLAVE_EBI_CH0>; - interconnect-names = "notused", "mdp0-mem", "mdp1-mem"; + interconnect-names = "mdp0-mem", "mdp1-mem"; power-domains = <&dispcc MDSS_GDSC>; @@ -2580,7 +2579,7 @@ dispcc: clock-controller@af00000 { compatible = "qcom,sm8250-dispcc"; - reg = <0 0x0af00000 0 0x20000>; + reg = <0 0x0af00000 0 0x10000>; mmcx-supply = <&mmcx_reg>; clocks = <&rpmhcc RPMH_CXO_CLK>, <&dsi0_phy 0>, @@ -2588,28 +2587,14 @@ <&dsi1_phy 0>, <&dsi1_phy 1>, <0>, - <0>, - <0>, - <0>, - <0>, - <0>, - <0>, - <0>, - <&sleep_clk>; + <0>; clock-names = "bi_tcxo", "dsi0_phy_pll_out_byteclk", "dsi0_phy_pll_out_dsiclk", "dsi1_phy_pll_out_byteclk", "dsi1_phy_pll_out_dsiclk", - "dp_link_clk_divsel_ten", - "dp_vco_divided_clk_src_mux", - "dptx1_phy_pll_link_clk", - "dptx1_phy_pll_vco_div_clk", - "dptx2_phy_pll_link_clk", - "dptx2_phy_pll_vco_div_clk", - "edp_phy_pll_link_clk", - "edp_phy_pll_vco_div_clk", - "sleep_clk"; + "dp_phy_pll_link_clk", + "dp_phy_pll_vco_div_clk"; #clock-cells = <1>; #reset-cells = <1>; #power-domain-cells = <1>; @@ -2689,7 +2674,7 @@ #gpio-cells = <2>; interrupt-controller; #interrupt-cells = <2>; - gpio-ranges = <&tlmm 0 0 180>; + gpio-ranges = <&tlmm 0 0 181>; wakeup-parent = <&pdc>; pri_mi2s_active: pri-mi2s-active { @@ -3754,7 +3739,7 @@ (GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_LOW)>, , - ; }; diff --git a/arch/arm64/boot/dts/qcom/sm8350.dtsi b/arch/arm64/boot/dts/qcom/sm8350.dtsi index 5ef460458f5c..e2fca420e518 100644 --- a/arch/arm64/boot/dts/qcom/sm8350.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8350.dtsi @@ -153,7 +153,7 @@ pmu { compatible = "arm,armv8-pmuv3"; - interrupts = ; + interrupts = ; }; psci { @@ -382,7 +382,7 @@ #gpio-cells = <2>; interrupt-controller; #interrupt-cells = <2>; - gpio-ranges = <&tlmm 0 0 203>; + gpio-ranges = <&tlmm 0 0 204>; qup_uart3_default_state: qup-uart3-default-state { rx { diff --git a/arch/arm64/boot/dts/renesas/hihope-common.dtsi b/arch/arm64/boot/dts/renesas/hihope-common.dtsi index 7a3da9b06f67..0c7e6f790590 100644 --- a/arch/arm64/boot/dts/renesas/hihope-common.dtsi +++ b/arch/arm64/boot/dts/renesas/hihope-common.dtsi @@ -12,6 +12,9 @@ aliases { serial0 = &scif2; serial1 = &hscif0; + mmc0 = &sdhi3; + mmc1 = &sdhi0; + mmc2 = &sdhi2; }; chosen { diff --git a/arch/arm64/boot/dts/renesas/r8a774a1-beacon-rzg2m-kit.dts b/arch/arm64/boot/dts/renesas/r8a774a1-beacon-rzg2m-kit.dts index 501cb05da228..3cf2e076940f 100644 --- a/arch/arm64/boot/dts/renesas/r8a774a1-beacon-rzg2m-kit.dts +++ b/arch/arm64/boot/dts/renesas/r8a774a1-beacon-rzg2m-kit.dts @@ -21,6 +21,9 @@ serial4 = &hscif2; serial5 = &scif5; ethernet0 = &avb; + mmc0 = &sdhi3; + mmc1 = &sdhi0; + mmc2 = &sdhi2; }; chosen { diff --git a/arch/arm64/boot/dts/renesas/r8a774b1-beacon-rzg2n-kit.dts b/arch/arm64/boot/dts/renesas/r8a774b1-beacon-rzg2n-kit.dts index 71763f4402a7..3c0d59def8ee 100644 --- a/arch/arm64/boot/dts/renesas/r8a774b1-beacon-rzg2n-kit.dts +++ b/arch/arm64/boot/dts/renesas/r8a774b1-beacon-rzg2n-kit.dts @@ -22,6 +22,9 @@ serial5 = &scif5; serial6 = &scif4; ethernet0 = &avb; + mmc0 = &sdhi3; + mmc1 = &sdhi0; + mmc2 = &sdhi2; }; chosen { diff --git a/arch/arm64/boot/dts/renesas/r8a774c0-cat874.dts b/arch/arm64/boot/dts/renesas/r8a774c0-cat874.dts index ea87cb5a459c..33257c6440b2 100644 --- a/arch/arm64/boot/dts/renesas/r8a774c0-cat874.dts +++ b/arch/arm64/boot/dts/renesas/r8a774c0-cat874.dts @@ -17,6 +17,8 @@ aliases { serial0 = &scif2; serial1 = &hscif2; + mmc0 = &sdhi0; + mmc1 = &sdhi3; }; chosen { diff --git a/arch/arm64/boot/dts/renesas/r8a774e1-beacon-rzg2h-kit.dts b/arch/arm64/boot/dts/renesas/r8a774e1-beacon-rzg2h-kit.dts index 273f062f2909..7b6649a3ded0 100644 --- a/arch/arm64/boot/dts/renesas/r8a774e1-beacon-rzg2h-kit.dts +++ b/arch/arm64/boot/dts/renesas/r8a774e1-beacon-rzg2h-kit.dts @@ -22,6 +22,9 @@ serial5 = &scif5; serial6 = &scif4; ethernet0 = &avb; + mmc0 = &sdhi3; + mmc1 = &sdhi0; + mmc2 = &sdhi2; }; chosen { diff --git a/arch/arm64/boot/dts/renesas/r8a77980.dtsi b/arch/arm64/boot/dts/renesas/r8a77980.dtsi index ec7ca72399ec..1ffa4a995a7a 100644 --- a/arch/arm64/boot/dts/renesas/r8a77980.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a77980.dtsi @@ -992,8 +992,8 @@ reg = <1>; - vin4csi41: endpoint@2 { - reg = <2>; + vin4csi41: endpoint@3 { + reg = <3>; remote-endpoint = <&csi41vin4>; }; }; @@ -1020,8 +1020,8 @@ reg = <1>; - vin5csi41: endpoint@2 { - reg = <2>; + vin5csi41: endpoint@3 { + reg = <3>; remote-endpoint = <&csi41vin5>; }; }; @@ -1048,8 +1048,8 @@ reg = <1>; - vin6csi41: endpoint@2 { - reg = <2>; + vin6csi41: endpoint@3 { + reg = <3>; remote-endpoint = <&csi41vin6>; }; }; @@ -1076,8 +1076,8 @@ reg = <1>; - vin7csi41: endpoint@2 { - reg = <2>; + vin7csi41: endpoint@3 { + reg = <3>; remote-endpoint = <&csi41vin7>; }; }; diff --git a/arch/arm64/boot/dts/renesas/r8a77990-ebisu.dts b/arch/arm64/boot/dts/renesas/r8a77990-ebisu.dts index f74f8b9993f1..6d6cdc4c324b 100644 --- a/arch/arm64/boot/dts/renesas/r8a77990-ebisu.dts +++ b/arch/arm64/boot/dts/renesas/r8a77990-ebisu.dts @@ -16,6 +16,9 @@ aliases { serial0 = &scif2; ethernet0 = &avb; + mmc0 = &sdhi3; + mmc1 = &sdhi0; + mmc2 = &sdhi1; }; chosen { diff --git a/arch/arm64/boot/dts/renesas/r8a779a0-falcon-cpu.dtsi b/arch/arm64/boot/dts/renesas/r8a779a0-falcon-cpu.dtsi index fa284a7260d6..e202e8aa6941 100644 --- a/arch/arm64/boot/dts/renesas/r8a779a0-falcon-cpu.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a779a0-falcon-cpu.dtsi @@ -12,6 +12,14 @@ model = "Renesas Falcon CPU board"; compatible = "renesas,falcon-cpu", "renesas,r8a779a0"; + aliases { + serial0 = &scif0; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + memory@48000000 { device_type = "memory"; /* first 128MB is reserved for secure area. */ diff --git a/arch/arm64/boot/dts/renesas/r8a779a0-falcon.dts b/arch/arm64/boot/dts/renesas/r8a779a0-falcon.dts index 5617b81dd7dc..273857ae38f3 100644 --- a/arch/arm64/boot/dts/renesas/r8a779a0-falcon.dts +++ b/arch/arm64/boot/dts/renesas/r8a779a0-falcon.dts @@ -14,11 +14,6 @@ aliases { ethernet0 = &avb0; - serial0 = &scif0; - }; - - chosen { - stdout-path = "serial0:115200n8"; }; }; diff --git a/arch/arm64/boot/dts/renesas/r8a779a0.dtsi b/arch/arm64/boot/dts/renesas/r8a779a0.dtsi index dfd6ae8b564f..86ac48e2c849 100644 --- a/arch/arm64/boot/dts/renesas/r8a779a0.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a779a0.dtsi @@ -60,10 +60,7 @@ pmu_a76 { compatible = "arm,cortex-a76-pmu"; - interrupts-extended = <&gic GIC_SPI 139 IRQ_TYPE_LEVEL_HIGH>, - <&gic GIC_SPI 140 IRQ_TYPE_LEVEL_HIGH>, - <&gic GIC_SPI 141 IRQ_TYPE_LEVEL_HIGH>, - <&gic GIC_SPI 142 IRQ_TYPE_LEVEL_HIGH>; + interrupts-extended = <&gic GIC_PPI 7 IRQ_TYPE_LEVEL_LOW>; }; /* External SCIF clock - to be overridden by boards that provide it */ diff --git a/arch/arm64/boot/dts/renesas/salvator-common.dtsi b/arch/arm64/boot/dts/renesas/salvator-common.dtsi index c22bb38994e8..15bb1eeb6601 100644 --- a/arch/arm64/boot/dts/renesas/salvator-common.dtsi +++ b/arch/arm64/boot/dts/renesas/salvator-common.dtsi @@ -36,6 +36,9 @@ serial0 = &scif2; serial1 = &hscif1; ethernet0 = &avb; + mmc0 = &sdhi2; + mmc1 = &sdhi0; + mmc2 = &sdhi3; }; chosen { diff --git a/arch/arm64/boot/dts/renesas/ulcb-kf.dtsi b/arch/arm64/boot/dts/renesas/ulcb-kf.dtsi index e9ed2597f1c2..61bd4df09df0 100644 --- a/arch/arm64/boot/dts/renesas/ulcb-kf.dtsi +++ b/arch/arm64/boot/dts/renesas/ulcb-kf.dtsi @@ -16,6 +16,7 @@ aliases { serial1 = &hscif0; serial2 = &scif1; + mmc2 = &sdhi3; }; clksndsel: clksndsel { diff --git a/arch/arm64/boot/dts/renesas/ulcb.dtsi b/arch/arm64/boot/dts/renesas/ulcb.dtsi index a04eae55dd6c..3d88e95c65a5 100644 --- a/arch/arm64/boot/dts/renesas/ulcb.dtsi +++ b/arch/arm64/boot/dts/renesas/ulcb.dtsi @@ -23,6 +23,8 @@ aliases { serial0 = &scif2; ethernet0 = &avb; + mmc0 = &sdhi2; + mmc1 = &sdhi0; }; chosen { diff --git a/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi b/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi index a87b8a678719..8f2c1c1e2c64 100644 --- a/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi +++ b/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi @@ -734,7 +734,7 @@ clocks = <&sys_clk 6>; reset-names = "ether"; resets = <&sys_rst 6>; - phy-mode = "rgmii"; + phy-mode = "rgmii-id"; local-mac-address = [00 00 00 00 00 00]; socionext,syscon-phy-mode = <&soc_glue 0>; diff --git a/arch/arm64/boot/dts/socionext/uniphier-pxs3.dtsi b/arch/arm64/boot/dts/socionext/uniphier-pxs3.dtsi index 0e52dadf54b3..be97da132258 100644 --- a/arch/arm64/boot/dts/socionext/uniphier-pxs3.dtsi +++ b/arch/arm64/boot/dts/socionext/uniphier-pxs3.dtsi @@ -564,7 +564,7 @@ clocks = <&sys_clk 6>; reset-names = "ether"; resets = <&sys_rst 6>; - phy-mode = "rgmii"; + phy-mode = "rgmii-id"; local-mac-address = [00 00 00 00 00 00]; socionext,syscon-phy-mode = <&soc_glue 0>; @@ -585,7 +585,7 @@ clocks = <&sys_clk 7>; reset-names = "ether"; resets = <&sys_rst 7>; - phy-mode = "rgmii"; + phy-mode = "rgmii-id"; local-mac-address = [00 00 00 00 00 00]; socionext,syscon-phy-mode = <&soc_glue 1>; diff --git a/arch/arm64/boot/dts/ti/k3-j721e-main.dtsi b/arch/arm64/boot/dts/ti/k3-j721e-main.dtsi index 8c84dafb7125..f1e7da3dfa27 100644 --- a/arch/arm64/boot/dts/ti/k3-j721e-main.dtsi +++ b/arch/arm64/boot/dts/ti/k3-j721e-main.dtsi @@ -1042,13 +1042,16 @@ assigned-clocks = <&k3_clks 91 1>; assigned-clock-parents = <&k3_clks 91 2>; bus-width = <8>; - mmc-hs400-1_8v; + mmc-hs200-1_8v; mmc-ddr-1_8v; ti,otap-del-sel-legacy = <0xf>; ti,otap-del-sel-mmc-hs = <0xf>; ti,otap-del-sel-ddr52 = <0x5>; ti,otap-del-sel-hs200 = <0x6>; ti,otap-del-sel-hs400 = <0x0>; + ti,itap-del-sel-legacy = <0x10>; + ti,itap-del-sel-mmc-hs = <0xa>; + ti,itap-del-sel-ddr52 = <0x3>; ti,trm-icp = <0x8>; ti,strobe-sel = <0x77>; dma-coherent; @@ -1069,9 +1072,15 @@ ti,otap-del-sel-sdr25 = <0xf>; ti,otap-del-sel-sdr50 = <0xc>; ti,otap-del-sel-ddr50 = <0xc>; + ti,itap-del-sel-legacy = <0x0>; + ti,itap-del-sel-sd-hs = <0x0>; + ti,itap-del-sel-sdr12 = <0x0>; + ti,itap-del-sel-sdr25 = <0x0>; + ti,itap-del-sel-ddr50 = <0x2>; ti,trm-icp = <0x8>; ti,clkbuf-sel = <0x7>; dma-coherent; + sdhci-caps-mask = <0x2 0x0>; }; main_sdhci2: mmc@4f98000 { @@ -1089,9 +1098,15 @@ ti,otap-del-sel-sdr25 = <0xf>; ti,otap-del-sel-sdr50 = <0xc>; ti,otap-del-sel-ddr50 = <0xc>; + ti,itap-del-sel-legacy = <0x0>; + ti,itap-del-sel-sd-hs = <0x0>; + ti,itap-del-sel-sdr12 = <0x0>; + ti,itap-del-sel-sdr25 = <0x0>; + ti,itap-del-sel-ddr50 = <0x2>; ti,trm-icp = <0x8>; ti,clkbuf-sel = <0x7>; dma-coherent; + sdhci-caps-mask = <0x2 0x0>; }; usbss0: cdns-usb@4104000 { diff --git a/arch/arm64/crypto/aes-modes.S b/arch/arm64/crypto/aes-modes.S index bbdb54702aa7..247011356d11 100644 --- a/arch/arm64/crypto/aes-modes.S +++ b/arch/arm64/crypto/aes-modes.S @@ -359,6 +359,7 @@ ST5( mov v4.16b, vctr.16b ) ins vctr.d[0], x8 /* apply carry to N counter blocks for N := x12 */ + cbz x12, 2f adr x16, 1f sub x16, x16, x12, lsl #3 br x16 diff --git a/arch/arm64/crypto/poly1305-glue.c b/arch/arm64/crypto/poly1305-glue.c index 683de671741a..9c3d86e397bf 100644 --- a/arch/arm64/crypto/poly1305-glue.c +++ b/arch/arm64/crypto/poly1305-glue.c @@ -25,7 +25,7 @@ asmlinkage void poly1305_emit(void *state, u8 *digest, const u32 *nonce); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); -void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 *key) +void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) { poly1305_init_arm64(&dctx->h, key); dctx->s[0] = get_unaligned_le32(key + 16); diff --git a/arch/arm64/include/asm/daifflags.h b/arch/arm64/include/asm/daifflags.h index 1c26d7baa67f..cfdde3a56805 100644 --- a/arch/arm64/include/asm/daifflags.h +++ b/arch/arm64/include/asm/daifflags.h @@ -131,6 +131,9 @@ static inline void local_daif_inherit(struct pt_regs *regs) if (interrupts_enabled(regs)) trace_hardirqs_on(); + if (system_uses_irq_prio_masking()) + gic_write_pmr(regs->pmr_save); + /* * We can't use local_daif_restore(regs->pstate) here as * system_has_prio_mask_debugging() won't restore the I bit if it can diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 3d10e6527f7d..858c2fcfc043 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -713,6 +713,7 @@ static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {} static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {} void kvm_arm_init_debug(void); +void kvm_arm_vcpu_init_debug(struct kvm_vcpu *vcpu); void kvm_arm_setup_debug(struct kvm_vcpu *vcpu); void kvm_arm_clear_debug(struct kvm_vcpu *vcpu); void kvm_arm_reset_debug_ptr(struct kvm_vcpu *vcpu); diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index 9d3588450473..117412bae915 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -226,14 +226,6 @@ static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr) { unsigned long far = read_sysreg(far_el1); - /* - * The CPU masked interrupts, and we are leaving them masked during - * do_debug_exception(). Update PMR as if we had called - * local_daif_mask(). - */ - if (system_uses_irq_prio_masking()) - gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); - arm64_enter_el1_dbg(regs); if (!cortex_a76_erratum_1463225_debug_handler(regs)) do_debug_exception(far, esr, regs); @@ -398,9 +390,6 @@ static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr) /* Only watchpoints write FAR_EL1, otherwise its UNKNOWN */ unsigned long far = read_sysreg(far_el1); - if (system_uses_irq_prio_masking()) - gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); - enter_from_user_mode(); do_debug_exception(far, esr, regs); local_daif_restore(DAIF_PROCCTX_NOIRQ); @@ -408,9 +397,6 @@ static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr) static void noinstr el0_svc(struct pt_regs *regs) { - if (system_uses_irq_prio_masking()) - gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); - enter_from_user_mode(); cortex_a76_erratum_1463225_svc_handler(); do_el0_svc(regs); @@ -486,9 +472,6 @@ static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr) static void noinstr el0_svc_compat(struct pt_regs *regs) { - if (system_uses_irq_prio_masking()) - gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); - enter_from_user_mode(); cortex_a76_erratum_1463225_svc_handler(); do_el0_svc_compat(regs); diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 6acfc5e6b5e0..e03fba3ae2a0 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -263,16 +263,16 @@ alternative_else_nop_endif stp lr, x21, [sp, #S_LR] /* - * For exceptions from EL0, terminate the callchain here. + * For exceptions from EL0, create a terminal frame record. * For exceptions from EL1, create a synthetic frame record so the * interrupted code shows up in the backtrace. */ .if \el == 0 - mov x29, xzr + stp xzr, xzr, [sp, #S_STACKFRAME] .else stp x29, x22, [sp, #S_STACKFRAME] - add x29, sp, #S_STACKFRAME .endif + add x29, sp, #S_STACKFRAME #ifdef CONFIG_ARM64_SW_TTBR0_PAN alternative_if_not ARM64_HAS_PAN @@ -292,6 +292,8 @@ alternative_else_nop_endif alternative_if ARM64_HAS_IRQ_PRIO_MASKING mrs_s x20, SYS_ICC_PMR_EL1 str x20, [sp, #S_PMR_SAVE] + mov x20, #GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET + msr_s SYS_ICC_PMR_EL1, x20 alternative_else_nop_endif /* Re-enable tag checking (TCO set on exception entry) */ @@ -493,8 +495,8 @@ tsk .req x28 // current thread_info /* * Interrupt handling. */ - .macro irq_handler - ldr_l x1, handle_arch_irq + .macro irq_handler, handler:req + ldr_l x1, \handler mov x0, sp irq_stack_entry blr x1 @@ -524,13 +526,41 @@ alternative_endif #endif .endm - .macro gic_prio_irq_setup, pmr:req, tmp:req -#ifdef CONFIG_ARM64_PSEUDO_NMI - alternative_if ARM64_HAS_IRQ_PRIO_MASKING - orr \tmp, \pmr, #GIC_PRIO_PSR_I_SET - msr_s SYS_ICC_PMR_EL1, \tmp - alternative_else_nop_endif + .macro el1_interrupt_handler, handler:req + enable_da_f + + mov x0, sp + bl enter_el1_irq_or_nmi + + irq_handler \handler + +#ifdef CONFIG_PREEMPTION + ldr x24, [tsk, #TSK_TI_PREEMPT] // get preempt count +alternative_if ARM64_HAS_IRQ_PRIO_MASKING + /* + * DA_F were cleared at start of handling. If anything is set in DAIF, + * we come back from an NMI, so skip preemption + */ + mrs x0, daif + orr x24, x24, x0 +alternative_else_nop_endif + cbnz x24, 1f // preempt count != 0 || NMI return path + bl arm64_preempt_schedule_irq // irq en/disable is done inside +1: #endif + + mov x0, sp + bl exit_el1_irq_or_nmi + .endm + + .macro el0_interrupt_handler, handler:req + user_exit_irqoff + enable_da_f + + tbz x22, #55, 1f + bl do_el0_irq_bp_hardening +1: + irq_handler \handler .endm .text @@ -662,32 +692,7 @@ SYM_CODE_END(el1_sync) .align 6 SYM_CODE_START_LOCAL_NOALIGN(el1_irq) kernel_entry 1 - gic_prio_irq_setup pmr=x20, tmp=x1 - enable_da_f - - mov x0, sp - bl enter_el1_irq_or_nmi - - irq_handler - -#ifdef CONFIG_PREEMPTION - ldr x24, [tsk, #TSK_TI_PREEMPT] // get preempt count -alternative_if ARM64_HAS_IRQ_PRIO_MASKING - /* - * DA_F were cleared at start of handling. If anything is set in DAIF, - * we come back from an NMI, so skip preemption - */ - mrs x0, daif - orr x24, x24, x0 -alternative_else_nop_endif - cbnz x24, 1f // preempt count != 0 || NMI return path - bl arm64_preempt_schedule_irq // irq en/disable is done inside -1: -#endif - - mov x0, sp - bl exit_el1_irq_or_nmi - + el1_interrupt_handler handle_arch_irq kernel_exit 1 SYM_CODE_END(el1_irq) @@ -727,22 +732,13 @@ SYM_CODE_END(el0_error_compat) SYM_CODE_START_LOCAL_NOALIGN(el0_irq) kernel_entry 0 el0_irq_naked: - gic_prio_irq_setup pmr=x20, tmp=x0 - user_exit_irqoff - enable_da_f - - tbz x22, #55, 1f - bl do_el0_irq_bp_hardening -1: - irq_handler - + el0_interrupt_handler handle_arch_irq b ret_to_user SYM_CODE_END(el0_irq) SYM_CODE_START_LOCAL(el1_error) kernel_entry 1 mrs x1, esr_el1 - gic_prio_kentry_setup tmp=x2 enable_dbg mov x0, sp bl do_serror @@ -753,7 +749,6 @@ SYM_CODE_START_LOCAL(el0_error) kernel_entry 0 el0_error_naked: mrs x25, esr_el1 - gic_prio_kentry_setup tmp=x2 user_exit_irqoff enable_dbg mov x0, sp diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c index d55bdfb7789c..7032a5f9e624 100644 --- a/arch/arm64/kernel/stacktrace.c +++ b/arch/arm64/kernel/stacktrace.c @@ -44,10 +44,6 @@ int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame) unsigned long fp = frame->fp; struct stack_info info; - /* Terminal record; nothing to unwind */ - if (!fp) - return -ENOENT; - if (fp & 0xf) return -EINVAL; @@ -108,6 +104,12 @@ int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame) frame->pc = ptrauth_strip_insn_pac(frame->pc); + /* + * This is a terminal record, so we have finished unwinding. + */ + if (!frame->fp && !frame->pc) + return -ENOENT; + return 0; } NOKPROBE_SYMBOL(unwind_frame); diff --git a/arch/arm64/kernel/vdso/vdso.lds.S b/arch/arm64/kernel/vdso/vdso.lds.S index 61dbb4c838ef..a5e61e09ea92 100644 --- a/arch/arm64/kernel/vdso/vdso.lds.S +++ b/arch/arm64/kernel/vdso/vdso.lds.S @@ -31,6 +31,13 @@ SECTIONS .gnu.version_d : { *(.gnu.version_d) } .gnu.version_r : { *(.gnu.version_r) } + /* + * Discard .note.gnu.property sections which are unused and have + * different alignment requirement from vDSO note sections. + */ + /DISCARD/ : { + *(.note.GNU-stack .note.gnu.property) + } .note : { *(.note.*) } :text :note . = ALIGN(16); @@ -48,7 +55,6 @@ SECTIONS PROVIDE(end = .); /DISCARD/ : { - *(.note.GNU-stack) *(.data .data.* .gnu.linkonce.d.* .sdata*) *(.bss .sbss .dynbss .dynsbss) *(.eh_frame .eh_frame_hdr) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 7f06ba76698d..84b5f79c9eab 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -580,6 +580,8 @@ static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu) vcpu->arch.has_run_once = true; + kvm_arm_vcpu_init_debug(vcpu); + if (likely(irqchip_in_kernel(kvm))) { /* * Map the VGIC hardware resources before running a vcpu the @@ -1808,8 +1810,10 @@ static int init_hyp_mode(void) if (is_protected_kvm_enabled()) { init_cpu_logical_map(); - if (!init_psci_relay()) + if (!init_psci_relay()) { + err = -ENODEV; goto out_err; + } } return 0; diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c index dbc890511631..2484b2cca74b 100644 --- a/arch/arm64/kvm/debug.c +++ b/arch/arm64/kvm/debug.c @@ -68,6 +68,64 @@ void kvm_arm_init_debug(void) __this_cpu_write(mdcr_el2, kvm_call_hyp_ret(__kvm_get_mdcr_el2)); } +/** + * kvm_arm_setup_mdcr_el2 - configure vcpu mdcr_el2 value + * + * @vcpu: the vcpu pointer + * + * This ensures we will trap access to: + * - Performance monitors (MDCR_EL2_TPM/MDCR_EL2_TPMCR) + * - Debug ROM Address (MDCR_EL2_TDRA) + * - OS related registers (MDCR_EL2_TDOSA) + * - Statistical profiler (MDCR_EL2_TPMS/MDCR_EL2_E2PB) + * - Self-hosted Trace Filter controls (MDCR_EL2_TTRF) + */ +static void kvm_arm_setup_mdcr_el2(struct kvm_vcpu *vcpu) +{ + /* + * This also clears MDCR_EL2_E2PB_MASK to disable guest access + * to the profiling buffer. + */ + vcpu->arch.mdcr_el2 = __this_cpu_read(mdcr_el2) & MDCR_EL2_HPMN_MASK; + vcpu->arch.mdcr_el2 |= (MDCR_EL2_TPM | + MDCR_EL2_TPMS | + MDCR_EL2_TTRF | + MDCR_EL2_TPMCR | + MDCR_EL2_TDRA | + MDCR_EL2_TDOSA); + + /* Is the VM being debugged by userspace? */ + if (vcpu->guest_debug) + /* Route all software debug exceptions to EL2 */ + vcpu->arch.mdcr_el2 |= MDCR_EL2_TDE; + + /* + * Trap debug register access when one of the following is true: + * - Userspace is using the hardware to debug the guest + * (KVM_GUESTDBG_USE_HW is set). + * - The guest is not using debug (KVM_ARM64_DEBUG_DIRTY is clear). + */ + if ((vcpu->guest_debug & KVM_GUESTDBG_USE_HW) || + !(vcpu->arch.flags & KVM_ARM64_DEBUG_DIRTY)) + vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA; + + trace_kvm_arm_set_dreg32("MDCR_EL2", vcpu->arch.mdcr_el2); +} + +/** + * kvm_arm_vcpu_init_debug - setup vcpu debug traps + * + * @vcpu: the vcpu pointer + * + * Set vcpu initial mdcr_el2 value. + */ +void kvm_arm_vcpu_init_debug(struct kvm_vcpu *vcpu) +{ + preempt_disable(); + kvm_arm_setup_mdcr_el2(vcpu); + preempt_enable(); +} + /** * kvm_arm_reset_debug_ptr - reset the debug ptr to point to the vcpu state */ @@ -83,13 +141,7 @@ void kvm_arm_reset_debug_ptr(struct kvm_vcpu *vcpu) * @vcpu: the vcpu pointer * * This is called before each entry into the hypervisor to setup any - * debug related registers. Currently this just ensures we will trap - * access to: - * - Performance monitors (MDCR_EL2_TPM/MDCR_EL2_TPMCR) - * - Debug ROM Address (MDCR_EL2_TDRA) - * - OS related registers (MDCR_EL2_TDOSA) - * - Statistical profiler (MDCR_EL2_TPMS/MDCR_EL2_E2PB) - * - Self-hosted Trace Filter controls (MDCR_EL2_TTRF) + * debug related registers. * * Additionally, KVM only traps guest accesses to the debug registers if * the guest is not actively using them (see the KVM_ARM64_DEBUG_DIRTY @@ -101,28 +153,14 @@ void kvm_arm_reset_debug_ptr(struct kvm_vcpu *vcpu) void kvm_arm_setup_debug(struct kvm_vcpu *vcpu) { - bool trap_debug = !(vcpu->arch.flags & KVM_ARM64_DEBUG_DIRTY); unsigned long mdscr, orig_mdcr_el2 = vcpu->arch.mdcr_el2; trace_kvm_arm_setup_debug(vcpu, vcpu->guest_debug); - /* - * This also clears MDCR_EL2_E2PB_MASK to disable guest access - * to the profiling buffer. - */ - vcpu->arch.mdcr_el2 = __this_cpu_read(mdcr_el2) & MDCR_EL2_HPMN_MASK; - vcpu->arch.mdcr_el2 |= (MDCR_EL2_TPM | - MDCR_EL2_TPMS | - MDCR_EL2_TTRF | - MDCR_EL2_TPMCR | - MDCR_EL2_TDRA | - MDCR_EL2_TDOSA); + kvm_arm_setup_mdcr_el2(vcpu); /* Is Guest debugging in effect? */ if (vcpu->guest_debug) { - /* Route all software debug exceptions to EL2 */ - vcpu->arch.mdcr_el2 |= MDCR_EL2_TDE; - /* Save guest debug state */ save_guest_debug_regs(vcpu); @@ -176,7 +214,6 @@ void kvm_arm_setup_debug(struct kvm_vcpu *vcpu) vcpu->arch.debug_ptr = &vcpu->arch.external_debug_state; vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY; - trap_debug = true; trace_kvm_arm_set_regset("BKPTS", get_num_brps(), &vcpu->arch.debug_ptr->dbg_bcr[0], @@ -191,10 +228,6 @@ void kvm_arm_setup_debug(struct kvm_vcpu *vcpu) BUG_ON(!vcpu->guest_debug && vcpu->arch.debug_ptr != &vcpu->arch.vcpu_debug_state); - /* Trap debug register access */ - if (trap_debug) - vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA; - /* If KDE or MDE are set, perform a full save/restore cycle. */ if (vcpu_read_sys_reg(vcpu, MDSCR_EL1) & (DBG_MDSCR_KDE | DBG_MDSCR_MDE)) vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY; @@ -203,7 +236,6 @@ void kvm_arm_setup_debug(struct kvm_vcpu *vcpu) if (has_vhe() && orig_mdcr_el2 != vcpu->arch.mdcr_el2) write_sysreg(vcpu->arch.mdcr_el2, mdcr_el2); - trace_kvm_arm_set_dreg32("MDCR_EL2", vcpu->arch.mdcr_el2); trace_kvm_arm_set_dreg32("MDSCR_EL1", vcpu_read_sys_reg(vcpu, MDSCR_EL1)); } diff --git a/arch/arm64/kvm/hyp/nvhe/gen-hyprel.c b/arch/arm64/kvm/hyp/nvhe/gen-hyprel.c index ead02c6a7628..6bc88a756cb7 100644 --- a/arch/arm64/kvm/hyp/nvhe/gen-hyprel.c +++ b/arch/arm64/kvm/hyp/nvhe/gen-hyprel.c @@ -50,6 +50,18 @@ #ifndef R_AARCH64_ABS64 #define R_AARCH64_ABS64 257 #endif +#ifndef R_AARCH64_PREL64 +#define R_AARCH64_PREL64 260 +#endif +#ifndef R_AARCH64_PREL32 +#define R_AARCH64_PREL32 261 +#endif +#ifndef R_AARCH64_PREL16 +#define R_AARCH64_PREL16 262 +#endif +#ifndef R_AARCH64_PLT32 +#define R_AARCH64_PLT32 314 +#endif #ifndef R_AARCH64_LD_PREL_LO19 #define R_AARCH64_LD_PREL_LO19 273 #endif @@ -371,6 +383,12 @@ static void emit_rela_section(Elf64_Shdr *sh_rela) case R_AARCH64_ABS64: emit_rela_abs64(rela, sh_orig_name); break; + /* Allow position-relative data relocations. */ + case R_AARCH64_PREL64: + case R_AARCH64_PREL32: + case R_AARCH64_PREL16: + case R_AARCH64_PLT32: + break; /* Allow relocations to generate PC-relative addressing. */ case R_AARCH64_LD_PREL_LO19: case R_AARCH64_ADR_PREL_LO21: diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c index bd354cd45d28..4b5acd84b8c8 100644 --- a/arch/arm64/kvm/reset.c +++ b/arch/arm64/kvm/reset.c @@ -242,6 +242,11 @@ int kvm_reset_vcpu(struct kvm_vcpu *vcpu) /* Reset core registers */ memset(vcpu_gp_regs(vcpu), 0, sizeof(*vcpu_gp_regs(vcpu))); + memset(&vcpu->arch.ctxt.fp_regs, 0, sizeof(vcpu->arch.ctxt.fp_regs)); + vcpu->arch.ctxt.spsr_abt = 0; + vcpu->arch.ctxt.spsr_und = 0; + vcpu->arch.ctxt.spsr_irq = 0; + vcpu->arch.ctxt.spsr_fiq = 0; vcpu_gp_regs(vcpu)->pstate = pstate; /* Reset system registers */ diff --git a/arch/arm64/kvm/vgic/vgic-kvm-device.c b/arch/arm64/kvm/vgic/vgic-kvm-device.c index 44419679f91a..7740995de982 100644 --- a/arch/arm64/kvm/vgic/vgic-kvm-device.c +++ b/arch/arm64/kvm/vgic/vgic-kvm-device.c @@ -87,8 +87,8 @@ int kvm_vgic_addr(struct kvm *kvm, unsigned long type, u64 *addr, bool write) r = vgic_v3_set_redist_base(kvm, 0, *addr, 0); goto out; } - rdreg = list_first_entry(&vgic->rd_regions, - struct vgic_redist_region, list); + rdreg = list_first_entry_or_null(&vgic->rd_regions, + struct vgic_redist_region, list); if (!rdreg) addr_ptr = &undef_value; else @@ -226,6 +226,9 @@ static int vgic_get_common_attr(struct kvm_device *dev, u64 addr; unsigned long type = (unsigned long)attr->attr; + if (copy_from_user(&addr, uaddr, sizeof(addr))) + return -EFAULT; + r = kvm_vgic_addr(dev->kvm, type, &addr, false); if (r) return (r == -ENODEV) ? -ENXIO : r; diff --git a/arch/arm64/mm/flush.c b/arch/arm64/mm/flush.c index ac485163a4a7..6d44c028d1c9 100644 --- a/arch/arm64/mm/flush.c +++ b/arch/arm64/mm/flush.c @@ -55,8 +55,10 @@ void __sync_icache_dcache(pte_t pte) { struct page *page = pte_page(pte); - if (!test_and_set_bit(PG_dcache_clean, &page->flags)) + if (!test_bit(PG_dcache_clean, &page->flags)) { sync_icache_aliases(page_address(page), page_size(page)); + set_bit(PG_dcache_clean, &page->flags); + } } EXPORT_SYMBOL_GPL(__sync_icache_dcache); diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index c967bfd30d2b..b183216a591c 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -444,6 +444,18 @@ SYM_FUNC_START(__cpu_setup) mov x10, #(SYS_GCR_EL1_RRND | SYS_GCR_EL1_EXCL_MASK) msr_s SYS_GCR_EL1, x10 + /* + * If GCR_EL1.RRND=1 is implemented the same way as RRND=0, then + * RGSR_EL1.SEED must be non-zero for IRG to produce + * pseudorandom numbers. As RGSR_EL1 is UNKNOWN out of reset, we + * must initialize it. + */ + mrs x10, CNTVCT_EL0 + ands x10, x10, #SYS_RGSR_EL1_SEED_MASK + csinc x10, x10, xzr, ne + lsl x10, x10, #SYS_RGSR_EL1_SEED_SHIFT + msr_s SYS_RGSR_EL1, x10 + /* clear any pending tag check faults in TFSR*_EL1 */ msr_s SYS_TFSR_EL1, xzr msr_s SYS_TFSRE0_EL1, xzr diff --git a/arch/ia64/include/asm/module.h b/arch/ia64/include/asm/module.h index 5a29652e6def..7271b9c5fc76 100644 --- a/arch/ia64/include/asm/module.h +++ b/arch/ia64/include/asm/module.h @@ -14,16 +14,20 @@ struct elf64_shdr; /* forward declration */ struct mod_arch_specific { + /* Used only at module load time. */ struct elf64_shdr *core_plt; /* core PLT section */ struct elf64_shdr *init_plt; /* init PLT section */ struct elf64_shdr *got; /* global offset table */ struct elf64_shdr *opd; /* official procedure descriptors */ struct elf64_shdr *unwind; /* unwind-table section */ unsigned long gp; /* global-pointer for module */ + unsigned int next_got_entry; /* index of next available got entry */ + /* Used at module run and cleanup time. */ void *core_unw_table; /* core unwind-table cookie returned by unwinder */ void *init_unw_table; /* init unwind-table cookie returned by unwinder */ - unsigned int next_got_entry; /* index of next available got entry */ + void *opd_addr; /* symbolize uses .opd to get to actual function */ + unsigned long opd_size; }; #define ARCH_SHF_SMALL SHF_IA_64_SHORT diff --git a/arch/ia64/kernel/acpi.c b/arch/ia64/kernel/acpi.c index a5636524af76..e2af6b172200 100644 --- a/arch/ia64/kernel/acpi.c +++ b/arch/ia64/kernel/acpi.c @@ -446,7 +446,8 @@ void __init acpi_numa_fixup(void) if (srat_num_cpus == 0) { node_set_online(0); node_cpuid[0].phys_id = hard_smp_processor_id(); - return; + slit_distance(0, 0) = LOCAL_DISTANCE; + goto out; } /* @@ -489,7 +490,7 @@ void __init acpi_numa_fixup(void) for (j = 0; j < MAX_NUMNODES; j++) slit_distance(i, j) = i == j ? LOCAL_DISTANCE : REMOTE_DISTANCE; - return; + goto out; } memset(numa_slit, -1, sizeof(numa_slit)); @@ -514,6 +515,8 @@ void __init acpi_numa_fixup(void) printk("\n"); } #endif +out: + node_possible_map = node_online_map; } #endif /* CONFIG_ACPI_NUMA */ diff --git a/arch/ia64/kernel/efi.c b/arch/ia64/kernel/efi.c index c5fe21de46a8..31149e41f9be 100644 --- a/arch/ia64/kernel/efi.c +++ b/arch/ia64/kernel/efi.c @@ -415,10 +415,10 @@ efi_get_pal_addr (void) mask = ~((1 << IA64_GRANULE_SHIFT) - 1); printk(KERN_INFO "CPU %d: mapping PAL code " - "[0x%lx-0x%lx) into [0x%lx-0x%lx)\n", - smp_processor_id(), md->phys_addr, - md->phys_addr + efi_md_size(md), - vaddr & mask, (vaddr & mask) + IA64_GRANULE_SIZE); + "[0x%llx-0x%llx) into [0x%llx-0x%llx)\n", + smp_processor_id(), md->phys_addr, + md->phys_addr + efi_md_size(md), + vaddr & mask, (vaddr & mask) + IA64_GRANULE_SIZE); #endif return __va(md->phys_addr); } @@ -560,6 +560,7 @@ efi_init (void) { efi_memory_desc_t *md; void *p; + unsigned int i; for (i = 0, p = efi_map_start; p < efi_map_end; ++i, p += efi_desc_size) @@ -586,7 +587,7 @@ efi_init (void) } printk("mem%02d: %s " - "range=[0x%016lx-0x%016lx) (%4lu%s)\n", + "range=[0x%016llx-0x%016llx) (%4lu%s)\n", i, efi_md_typeattr_format(buf, sizeof(buf), md), md->phys_addr, md->phys_addr + efi_md_size(md), size, unit); diff --git a/arch/ia64/kernel/module.c b/arch/ia64/kernel/module.c index 00a496cb346f..2cba53c1da82 100644 --- a/arch/ia64/kernel/module.c +++ b/arch/ia64/kernel/module.c @@ -905,9 +905,31 @@ register_unwind_table (struct module *mod) int module_finalize (const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, struct module *mod) { + struct mod_arch_specific *mas = &mod->arch; + DEBUGP("%s: init: entry=%p\n", __func__, mod->init); - if (mod->arch.unwind) + if (mas->unwind) register_unwind_table(mod); + + /* + * ".opd" was already relocated to the final destination. Store + * it's address for use in symbolizer. + */ + mas->opd_addr = (void *)mas->opd->sh_addr; + mas->opd_size = mas->opd->sh_size; + + /* + * Module relocation was already done at this point. Section + * headers are about to be deleted. Wipe out load-time context. + */ + mas->core_plt = NULL; + mas->init_plt = NULL; + mas->got = NULL; + mas->opd = NULL; + mas->unwind = NULL; + mas->gp = 0; + mas->next_got_entry = 0; + return 0; } @@ -926,10 +948,9 @@ module_arch_cleanup (struct module *mod) void *dereference_module_function_descriptor(struct module *mod, void *ptr) { - Elf64_Shdr *opd = mod->arch.opd; + struct mod_arch_specific *mas = &mod->arch; - if (ptr < (void *)opd->sh_addr || - ptr >= (void *)(opd->sh_addr + opd->sh_size)) + if (ptr < mas->opd_addr || ptr >= mas->opd_addr + mas->opd_size) return ptr; return dereference_function_descriptor(ptr); diff --git a/arch/m68k/include/asm/mvme147hw.h b/arch/m68k/include/asm/mvme147hw.h index 257b29184af9..e28eb1c0e0bf 100644 --- a/arch/m68k/include/asm/mvme147hw.h +++ b/arch/m68k/include/asm/mvme147hw.h @@ -66,6 +66,9 @@ struct pcc_regs { #define PCC_INT_ENAB 0x08 #define PCC_TIMER_INT_CLR 0x80 + +#define PCC_TIMER_TIC_EN 0x01 +#define PCC_TIMER_COC_EN 0x02 #define PCC_TIMER_CLR_OVF 0x04 #define PCC_LEVEL_ABORT 0x07 diff --git a/arch/m68k/kernel/sys_m68k.c b/arch/m68k/kernel/sys_m68k.c index 1c235d8f53f3..f55bdcb8e4f1 100644 --- a/arch/m68k/kernel/sys_m68k.c +++ b/arch/m68k/kernel/sys_m68k.c @@ -388,6 +388,8 @@ sys_cacheflush (unsigned long addr, int scope, int cache, unsigned long len) ret = -EPERM; if (!capable(CAP_SYS_ADMIN)) goto out; + + mmap_read_lock(current->mm); } else { struct vm_area_struct *vma; diff --git a/arch/m68k/mvme147/config.c b/arch/m68k/mvme147/config.c index cfdc7f912e14..e1e90c49a496 100644 --- a/arch/m68k/mvme147/config.c +++ b/arch/m68k/mvme147/config.c @@ -114,8 +114,10 @@ static irqreturn_t mvme147_timer_int (int irq, void *dev_id) unsigned long flags; local_irq_save(flags); - m147_pcc->t1_int_cntrl = PCC_TIMER_INT_CLR; - m147_pcc->t1_cntrl = PCC_TIMER_CLR_OVF; + m147_pcc->t1_cntrl = PCC_TIMER_CLR_OVF | PCC_TIMER_COC_EN | + PCC_TIMER_TIC_EN; + m147_pcc->t1_int_cntrl = PCC_INT_ENAB | PCC_TIMER_INT_CLR | + PCC_LEVEL_TIMER1; clk_total += PCC_TIMER_CYCLES; legacy_timer_tick(1); local_irq_restore(flags); @@ -133,10 +135,10 @@ void mvme147_sched_init (void) /* Init the clock with a value */ /* The clock counter increments until 0xFFFF then reloads */ m147_pcc->t1_preload = PCC_TIMER_PRELOAD; - m147_pcc->t1_cntrl = 0x0; /* clear timer */ - m147_pcc->t1_cntrl = 0x3; /* start timer */ - m147_pcc->t1_int_cntrl = PCC_TIMER_INT_CLR; /* clear pending ints */ - m147_pcc->t1_int_cntrl = PCC_INT_ENAB|PCC_LEVEL_TIMER1; + m147_pcc->t1_cntrl = PCC_TIMER_CLR_OVF | PCC_TIMER_COC_EN | + PCC_TIMER_TIC_EN; + m147_pcc->t1_int_cntrl = PCC_INT_ENAB | PCC_TIMER_INT_CLR | + PCC_LEVEL_TIMER1; clocksource_register_hz(&mvme147_clk, PCC_TIMER_CLOCK_FREQ); } diff --git a/arch/m68k/mvme16x/config.c b/arch/m68k/mvme16x/config.c index 30357fe4ba6c..b59593c7cfb9 100644 --- a/arch/m68k/mvme16x/config.c +++ b/arch/m68k/mvme16x/config.c @@ -366,6 +366,7 @@ static u32 clk_total; #define PCCTOVR1_COC_EN 0x02 #define PCCTOVR1_OVR_CLR 0x04 +#define PCCTIC1_INT_LEVEL 6 #define PCCTIC1_INT_CLR 0x08 #define PCCTIC1_INT_EN 0x10 @@ -374,8 +375,8 @@ static irqreturn_t mvme16x_timer_int (int irq, void *dev_id) unsigned long flags; local_irq_save(flags); - out_8(PCCTIC1, in_8(PCCTIC1) | PCCTIC1_INT_CLR); - out_8(PCCTOVR1, PCCTOVR1_OVR_CLR); + out_8(PCCTOVR1, PCCTOVR1_OVR_CLR | PCCTOVR1_TIC_EN | PCCTOVR1_COC_EN); + out_8(PCCTIC1, PCCTIC1_INT_EN | PCCTIC1_INT_CLR | PCCTIC1_INT_LEVEL); clk_total += PCC_TIMER_CYCLES; legacy_timer_tick(1); local_irq_restore(flags); @@ -389,14 +390,15 @@ void mvme16x_sched_init(void) int irq; /* Using PCCchip2 or MC2 chip tick timer 1 */ - out_be32(PCCTCNT1, 0); - out_be32(PCCTCMP1, PCC_TIMER_CYCLES); - out_8(PCCTOVR1, in_8(PCCTOVR1) | PCCTOVR1_TIC_EN | PCCTOVR1_COC_EN); - out_8(PCCTIC1, PCCTIC1_INT_EN | 6); if (request_irq(MVME16x_IRQ_TIMER, mvme16x_timer_int, IRQF_TIMER, "timer", NULL)) panic ("Couldn't register timer int"); + out_be32(PCCTCNT1, 0); + out_be32(PCCTCMP1, PCC_TIMER_CYCLES); + out_8(PCCTOVR1, PCCTOVR1_OVR_CLR | PCCTOVR1_TIC_EN | PCCTOVR1_COC_EN); + out_8(PCCTIC1, PCCTIC1_INT_EN | PCCTIC1_INT_CLR | PCCTIC1_INT_LEVEL); + clocksource_register_hz(&mvme16x_clk, PCC_TIMER_CLOCK_FREQ); if (brdno == 0x0162 || brdno == 0x172) diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index d89efba3d8a4..e89d63cd92d1 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -6,6 +6,7 @@ config MIPS select ARCH_BINFMT_ELF_STATE if MIPS_FP_SUPPORT select ARCH_HAS_FORTIFY_SOURCE select ARCH_HAS_KCOV + select ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE if !EVA select ARCH_HAS_PTE_SPECIAL if !(32BIT && CPU_HAS_RIXI) select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_HAS_UBSAN_SANITIZE_ALL diff --git a/arch/mips/boot/dts/brcm/bcm3368.dtsi b/arch/mips/boot/dts/brcm/bcm3368.dtsi index 69cbef472377..d4b2b430dad0 100644 --- a/arch/mips/boot/dts/brcm/bcm3368.dtsi +++ b/arch/mips/boot/dts/brcm/bcm3368.dtsi @@ -59,7 +59,7 @@ periph_cntl: syscon@fff8c008 { compatible = "syscon"; - reg = <0xfff8c000 0x4>; + reg = <0xfff8c008 0x4>; native-endian; }; diff --git a/arch/mips/boot/dts/brcm/bcm63268.dtsi b/arch/mips/boot/dts/brcm/bcm63268.dtsi index e0021ff9f144..940594436872 100644 --- a/arch/mips/boot/dts/brcm/bcm63268.dtsi +++ b/arch/mips/boot/dts/brcm/bcm63268.dtsi @@ -59,7 +59,7 @@ periph_cntl: syscon@10000008 { compatible = "syscon"; - reg = <0x10000000 0xc>; + reg = <0x10000008 0x4>; native-endian; }; diff --git a/arch/mips/boot/dts/brcm/bcm6358.dtsi b/arch/mips/boot/dts/brcm/bcm6358.dtsi index 9d93e7f5e6fc..d79c88c2fc9c 100644 --- a/arch/mips/boot/dts/brcm/bcm6358.dtsi +++ b/arch/mips/boot/dts/brcm/bcm6358.dtsi @@ -59,7 +59,7 @@ periph_cntl: syscon@fffe0008 { compatible = "syscon"; - reg = <0xfffe0000 0x4>; + reg = <0xfffe0008 0x4>; native-endian; }; diff --git a/arch/mips/boot/dts/brcm/bcm6362.dtsi b/arch/mips/boot/dts/brcm/bcm6362.dtsi index eb10341b75ba..8a21cb761ffd 100644 --- a/arch/mips/boot/dts/brcm/bcm6362.dtsi +++ b/arch/mips/boot/dts/brcm/bcm6362.dtsi @@ -59,7 +59,7 @@ periph_cntl: syscon@10000008 { compatible = "syscon"; - reg = <0x10000000 0xc>; + reg = <0x10000008 0x4>; native-endian; }; diff --git a/arch/mips/boot/dts/brcm/bcm6368.dtsi b/arch/mips/boot/dts/brcm/bcm6368.dtsi index 52c19f40b9cc..8e87867ebc04 100644 --- a/arch/mips/boot/dts/brcm/bcm6368.dtsi +++ b/arch/mips/boot/dts/brcm/bcm6368.dtsi @@ -59,7 +59,7 @@ periph_cntl: syscon@100000008 { compatible = "syscon"; - reg = <0x10000000 0xc>; + reg = <0x10000008 0x4>; native-endian; }; diff --git a/arch/mips/crypto/poly1305-glue.c b/arch/mips/crypto/poly1305-glue.c index fc881b46d911..bc6110fb98e0 100644 --- a/arch/mips/crypto/poly1305-glue.c +++ b/arch/mips/crypto/poly1305-glue.c @@ -17,7 +17,7 @@ asmlinkage void poly1305_init_mips(void *state, const u8 *key); asmlinkage void poly1305_blocks_mips(void *state, const u8 *src, u32 len, u32 hibit); asmlinkage void poly1305_emit_mips(void *state, u8 *digest, const u32 *nonce); -void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 *key) +void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) { poly1305_init_mips(&dctx->h, key); dctx->s[0] = get_unaligned_le32(key + 16); diff --git a/arch/mips/generic/board-boston.its.S b/arch/mips/generic/board-boston.its.S index a7f51f97b910..c45ad2759421 100644 --- a/arch/mips/generic/board-boston.its.S +++ b/arch/mips/generic/board-boston.its.S @@ -1,22 +1,22 @@ / { images { - fdt@boston { + fdt-boston { description = "img,boston Device Tree"; data = /incbin/("boot/dts/img/boston.dtb"); type = "flat_dt"; arch = "mips"; compression = "none"; - hash@0 { + hash { algo = "sha1"; }; }; }; configurations { - conf@boston { + conf-boston { description = "Boston Linux kernel"; - kernel = "kernel@0"; - fdt = "fdt@boston"; + kernel = "kernel"; + fdt = "fdt-boston"; }; }; }; diff --git a/arch/mips/generic/board-jaguar2.its.S b/arch/mips/generic/board-jaguar2.its.S index fb0e589eeff7..c2b8d479b26c 100644 --- a/arch/mips/generic/board-jaguar2.its.S +++ b/arch/mips/generic/board-jaguar2.its.S @@ -1,23 +1,23 @@ /* SPDX-License-Identifier: (GPL-2.0 OR MIT) */ / { images { - fdt@jaguar2_pcb110 { + fdt-jaguar2_pcb110 { description = "MSCC Jaguar2 PCB110 Device Tree"; data = /incbin/("boot/dts/mscc/jaguar2_pcb110.dtb"); type = "flat_dt"; arch = "mips"; compression = "none"; - hash@0 { + hash { algo = "sha1"; }; }; - fdt@jaguar2_pcb111 { + fdt-jaguar2_pcb111 { description = "MSCC Jaguar2 PCB111 Device Tree"; data = /incbin/("boot/dts/mscc/jaguar2_pcb111.dtb"); type = "flat_dt"; arch = "mips"; compression = "none"; - hash@0 { + hash { algo = "sha1"; }; }; @@ -26,14 +26,14 @@ configurations { pcb110 { description = "Jaguar2 Linux kernel"; - kernel = "kernel@0"; - fdt = "fdt@jaguar2_pcb110"; + kernel = "kernel"; + fdt = "fdt-jaguar2_pcb110"; ramdisk = "ramdisk"; }; pcb111 { description = "Jaguar2 Linux kernel"; - kernel = "kernel@0"; - fdt = "fdt@jaguar2_pcb111"; + kernel = "kernel"; + fdt = "fdt-jaguar2_pcb111"; ramdisk = "ramdisk"; }; }; diff --git a/arch/mips/generic/board-luton.its.S b/arch/mips/generic/board-luton.its.S index 39a543f62f25..bd9837c9af97 100644 --- a/arch/mips/generic/board-luton.its.S +++ b/arch/mips/generic/board-luton.its.S @@ -1,13 +1,13 @@ /* SPDX-License-Identifier: (GPL-2.0 OR MIT) */ / { images { - fdt@luton_pcb091 { + fdt-luton_pcb091 { description = "MSCC Luton PCB091 Device Tree"; data = /incbin/("boot/dts/mscc/luton_pcb091.dtb"); type = "flat_dt"; arch = "mips"; compression = "none"; - hash@0 { + hash { algo = "sha1"; }; }; @@ -16,8 +16,8 @@ configurations { pcb091 { description = "Luton Linux kernel"; - kernel = "kernel@0"; - fdt = "fdt@luton_pcb091"; + kernel = "kernel"; + fdt = "fdt-luton_pcb091"; }; }; }; diff --git a/arch/mips/generic/board-ni169445.its.S b/arch/mips/generic/board-ni169445.its.S index e4cb4f95a8cc..0a2e8f7a8526 100644 --- a/arch/mips/generic/board-ni169445.its.S +++ b/arch/mips/generic/board-ni169445.its.S @@ -1,22 +1,22 @@ / { images { - fdt@ni169445 { + fdt-ni169445 { description = "NI 169445 device tree"; data = /incbin/("boot/dts/ni/169445.dtb"); type = "flat_dt"; arch = "mips"; compression = "none"; - hash@0 { + hash { algo = "sha1"; }; }; }; configurations { - conf@ni169445 { + conf-ni169445 { description = "NI 169445 Linux Kernel"; - kernel = "kernel@0"; - fdt = "fdt@ni169445"; + kernel = "kernel"; + fdt = "fdt-ni169445"; }; }; }; diff --git a/arch/mips/generic/board-ocelot.its.S b/arch/mips/generic/board-ocelot.its.S index 3da23988149a..8c7e3a1b68d3 100644 --- a/arch/mips/generic/board-ocelot.its.S +++ b/arch/mips/generic/board-ocelot.its.S @@ -1,40 +1,40 @@ /* SPDX-License-Identifier: (GPL-2.0 OR MIT) */ / { images { - fdt@ocelot_pcb123 { + fdt-ocelot_pcb123 { description = "MSCC Ocelot PCB123 Device Tree"; data = /incbin/("boot/dts/mscc/ocelot_pcb123.dtb"); type = "flat_dt"; arch = "mips"; compression = "none"; - hash@0 { + hash { algo = "sha1"; }; }; - fdt@ocelot_pcb120 { + fdt-ocelot_pcb120 { description = "MSCC Ocelot PCB120 Device Tree"; data = /incbin/("boot/dts/mscc/ocelot_pcb120.dtb"); type = "flat_dt"; arch = "mips"; compression = "none"; - hash@0 { + hash { algo = "sha1"; }; }; }; configurations { - conf@ocelot_pcb123 { + conf-ocelot_pcb123 { description = "Ocelot Linux kernel"; - kernel = "kernel@0"; - fdt = "fdt@ocelot_pcb123"; + kernel = "kernel"; + fdt = "fdt-ocelot_pcb123"; }; - conf@ocelot_pcb120 { + conf-ocelot_pcb120 { description = "Ocelot Linux kernel"; - kernel = "kernel@0"; - fdt = "fdt@ocelot_pcb120"; + kernel = "kernel"; + fdt = "fdt-ocelot_pcb120"; }; }; }; diff --git a/arch/mips/generic/board-serval.its.S b/arch/mips/generic/board-serval.its.S index 4ea4fc9d757f..dde833efe980 100644 --- a/arch/mips/generic/board-serval.its.S +++ b/arch/mips/generic/board-serval.its.S @@ -1,13 +1,13 @@ /* SPDX-License-Identifier: (GPL-2.0 OR MIT) */ / { images { - fdt@serval_pcb105 { + fdt-serval_pcb105 { description = "MSCC Serval PCB105 Device Tree"; data = /incbin/("boot/dts/mscc/serval_pcb105.dtb"); type = "flat_dt"; arch = "mips"; compression = "none"; - hash@0 { + hash { algo = "sha1"; }; }; @@ -16,8 +16,8 @@ configurations { pcb105 { description = "Serval Linux kernel"; - kernel = "kernel@0"; - fdt = "fdt@serval_pcb105"; + kernel = "kernel"; + fdt = "fdt-serval_pcb105"; ramdisk = "ramdisk"; }; }; diff --git a/arch/mips/generic/board-xilfpga.its.S b/arch/mips/generic/board-xilfpga.its.S index a2e773d3f14f..08c1e900eb4e 100644 --- a/arch/mips/generic/board-xilfpga.its.S +++ b/arch/mips/generic/board-xilfpga.its.S @@ -1,22 +1,22 @@ / { images { - fdt@xilfpga { + fdt-xilfpga { description = "MIPSfpga (xilfpga) Device Tree"; data = /incbin/("boot/dts/xilfpga/nexys4ddr.dtb"); type = "flat_dt"; arch = "mips"; compression = "none"; - hash@0 { + hash { algo = "sha1"; }; }; }; configurations { - conf@xilfpga { + conf-xilfpga { description = "MIPSfpga Linux kernel"; - kernel = "kernel@0"; - fdt = "fdt@xilfpga"; + kernel = "kernel"; + fdt = "fdt-xilfpga"; }; }; }; diff --git a/arch/mips/generic/vmlinux.its.S b/arch/mips/generic/vmlinux.its.S index 1a08438fd893..3e254676540f 100644 --- a/arch/mips/generic/vmlinux.its.S +++ b/arch/mips/generic/vmlinux.its.S @@ -6,7 +6,7 @@ #address-cells = ; images { - kernel@0 { + kernel { description = KERNEL_NAME; data = /incbin/(VMLINUX_BINARY); type = "kernel"; @@ -15,18 +15,18 @@ compression = VMLINUX_COMPRESSION; load = /bits/ ADDR_BITS ; entry = /bits/ ADDR_BITS ; - hash@0 { + hash { algo = "sha1"; }; }; }; configurations { - default = "conf@default"; + default = "conf-default"; - conf@default { + conf-default { description = "Generic Linux kernel"; - kernel = "kernel@0"; + kernel = "kernel"; }; }; }; diff --git a/arch/mips/include/asm/asmmacro.h b/arch/mips/include/asm/asmmacro.h index 86f2323ebe6b..ca83ada7015f 100644 --- a/arch/mips/include/asm/asmmacro.h +++ b/arch/mips/include/asm/asmmacro.h @@ -44,8 +44,7 @@ .endm #endif -#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR5) || \ - defined(CONFIG_CPU_MIPSR6) +#ifdef CONFIG_CPU_HAS_DIEI .macro local_irq_enable reg=t0 ei irq_enable_hazard diff --git a/arch/mips/include/asm/div64.h b/arch/mips/include/asm/div64.h index dc5ea5736440..ceece76fc971 100644 --- a/arch/mips/include/asm/div64.h +++ b/arch/mips/include/asm/div64.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2000, 2004 Maciej W. Rozycki + * Copyright (C) 2000, 2004, 2021 Maciej W. Rozycki * Copyright (C) 2003, 07 Ralf Baechle (ralf@linux-mips.org) * * This file is subject to the terms and conditions of the GNU General Public @@ -9,25 +9,18 @@ #ifndef __ASM_DIV64_H #define __ASM_DIV64_H -#include - -#if BITS_PER_LONG == 64 +#include -#include +#if BITS_PER_LONG == 32 /* * No traps on overflows for any of these... */ -#define __div64_32(n, base) \ -({ \ +#define do_div64_32(res, high, low, base) ({ \ unsigned long __cf, __tmp, __tmp2, __i; \ unsigned long __quot32, __mod32; \ - unsigned long __high, __low; \ - unsigned long long __n; \ \ - __high = *__n >> 32; \ - __low = __n; \ __asm__( \ " .set push \n" \ " .set noat \n" \ @@ -51,18 +44,48 @@ " subu %0, %0, %z6 \n" \ " addiu %2, %2, 1 \n" \ "3: \n" \ - " bnez %4, 0b\n\t" \ - " srl %5, %1, 0x1f\n\t" \ + " bnez %4, 0b \n" \ + " srl %5, %1, 0x1f \n" \ " .set pop" \ : "=&r" (__mod32), "=&r" (__tmp), \ "=&r" (__quot32), "=&r" (__cf), \ "=&r" (__i), "=&r" (__tmp2) \ - : "Jr" (base), "0" (__high), "1" (__low)); \ + : "Jr" (base), "0" (high), "1" (low)); \ \ - (__n) = __quot32; \ + (res) = __quot32; \ __mod32; \ }) -#endif /* BITS_PER_LONG == 64 */ +#define __div64_32(n, base) ({ \ + unsigned long __upper, __low, __high, __radix; \ + unsigned long long __quot; \ + unsigned long long __div; \ + unsigned long __mod; \ + \ + __div = (*n); \ + __radix = (base); \ + \ + __high = __div >> 32; \ + __low = __div; \ + \ + if (__high < __radix) { \ + __upper = __high; \ + __high = 0; \ + } else { \ + __upper = __high % __radix; \ + __high /= __radix; \ + } \ + \ + __mod = do_div64_32(__low, __upper, __low, __radix); \ + \ + __quot = __high; \ + __quot = __quot << 32 | __low; \ + (*n) = __quot; \ + __mod; \ +}) + +#endif /* BITS_PER_LONG == 32 */ + +#include #endif /* __ASM_DIV64_H */ diff --git a/arch/mips/include/asm/vdso/gettimeofday.h b/arch/mips/include/asm/vdso/gettimeofday.h index 2203e2d0ae2a..44a45f3fa4b0 100644 --- a/arch/mips/include/asm/vdso/gettimeofday.h +++ b/arch/mips/include/asm/vdso/gettimeofday.h @@ -20,6 +20,12 @@ #define VDSO_HAS_CLOCK_GETRES 1 +#if MIPS_ISA_REV < 6 +#define VDSO_SYSCALL_CLOBBERS "hi", "lo", +#else +#define VDSO_SYSCALL_CLOBBERS +#endif + static __always_inline long gettimeofday_fallback( struct __kernel_old_timeval *_tv, struct timezone *_tz) @@ -35,7 +41,9 @@ static __always_inline long gettimeofday_fallback( : "=r" (ret), "=r" (error) : "r" (tv), "r" (tz), "r" (nr) : "$1", "$3", "$8", "$9", "$10", "$11", "$12", "$13", - "$14", "$15", "$24", "$25", "hi", "lo", "memory"); + "$14", "$15", "$24", "$25", + VDSO_SYSCALL_CLOBBERS + "memory"); return error ? -ret : ret; } @@ -59,7 +67,9 @@ static __always_inline long clock_gettime_fallback( : "=r" (ret), "=r" (error) : "r" (clkid), "r" (ts), "r" (nr) : "$1", "$3", "$8", "$9", "$10", "$11", "$12", "$13", - "$14", "$15", "$24", "$25", "hi", "lo", "memory"); + "$14", "$15", "$24", "$25", + VDSO_SYSCALL_CLOBBERS + "memory"); return error ? -ret : ret; } @@ -83,7 +93,9 @@ static __always_inline int clock_getres_fallback( : "=r" (ret), "=r" (error) : "r" (clkid), "r" (ts), "r" (nr) : "$1", "$3", "$8", "$9", "$10", "$11", "$12", "$13", - "$14", "$15", "$24", "$25", "hi", "lo", "memory"); + "$14", "$15", "$24", "$25", + VDSO_SYSCALL_CLOBBERS + "memory"); return error ? -ret : ret; } @@ -105,7 +117,9 @@ static __always_inline long clock_gettime32_fallback( : "=r" (ret), "=r" (error) : "r" (clkid), "r" (ts), "r" (nr) : "$1", "$3", "$8", "$9", "$10", "$11", "$12", "$13", - "$14", "$15", "$24", "$25", "hi", "lo", "memory"); + "$14", "$15", "$24", "$25", + VDSO_SYSCALL_CLOBBERS + "memory"); return error ? -ret : ret; } @@ -125,7 +139,9 @@ static __always_inline int clock_getres32_fallback( : "=r" (ret), "=r" (error) : "r" (clkid), "r" (ts), "r" (nr) : "$1", "$3", "$8", "$9", "$10", "$11", "$12", "$13", - "$14", "$15", "$24", "$25", "hi", "lo", "memory"); + "$14", "$15", "$24", "$25", + VDSO_SYSCALL_CLOBBERS + "memory"); return error ? -ret : ret; } diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c index b71892064f27..0ef240adefb5 100644 --- a/arch/mips/kernel/cpu-probe.c +++ b/arch/mips/kernel/cpu-probe.c @@ -1752,7 +1752,6 @@ static inline void cpu_probe_loongson(struct cpuinfo_mips *c, unsigned int cpu) set_isa(c, MIPS_CPU_ISA_M64R2); break; } - c->writecombine = _CACHE_UNCACHED_ACCELERATED; c->ases |= (MIPS_ASE_LOONGSON_MMI | MIPS_ASE_LOONGSON_EXT | MIPS_ASE_LOONGSON_EXT2); break; @@ -1782,7 +1781,6 @@ static inline void cpu_probe_loongson(struct cpuinfo_mips *c, unsigned int cpu) * register, we correct it here. */ c->options |= MIPS_CPU_FTLB | MIPS_CPU_TLBINV | MIPS_CPU_LDPTE; - c->writecombine = _CACHE_UNCACHED_ACCELERATED; c->ases |= (MIPS_ASE_LOONGSON_MMI | MIPS_ASE_LOONGSON_CAM | MIPS_ASE_LOONGSON_EXT | MIPS_ASE_LOONGSON_EXT2); c->ases &= ~MIPS_ASE_VZ; /* VZ of Loongson-3A2000/3000 is incomplete */ @@ -1793,7 +1791,6 @@ static inline void cpu_probe_loongson(struct cpuinfo_mips *c, unsigned int cpu) set_elf_platform(cpu, "loongson3a"); set_isa(c, MIPS_CPU_ISA_M64R2); decode_cpucfg(c); - c->writecombine = _CACHE_UNCACHED_ACCELERATED; break; default: panic("Unknown Loongson Processor ID!"); diff --git a/arch/mips/loongson64/init.c b/arch/mips/loongson64/init.c index cfa788bca871..1c664b23c0f9 100644 --- a/arch/mips/loongson64/init.c +++ b/arch/mips/loongson64/init.c @@ -126,7 +126,7 @@ static int __init add_legacy_isa_io(struct fwnode_handle *fwnode, resource_size_ return -ENOMEM; range->fwnode = fwnode; - range->size = size; + range->size = size = round_up(size, PAGE_SIZE); range->hw_start = hw_start; range->flags = LOGIC_PIO_CPU_MMIO; diff --git a/arch/mips/pci/pci-legacy.c b/arch/mips/pci/pci-legacy.c index 39052de915f3..3a909194284a 100644 --- a/arch/mips/pci/pci-legacy.c +++ b/arch/mips/pci/pci-legacy.c @@ -166,8 +166,13 @@ void pci_load_of_ranges(struct pci_controller *hose, struct device_node *node) res = hose->mem_resource; break; } - if (res != NULL) - of_pci_range_to_resource(&range, node, res); + if (res != NULL) { + res->name = node->full_name; + res->flags = range.flags; + res->start = range.cpu_addr; + res->end = range.cpu_addr + range.size - 1; + res->parent = res->child = res->sibling = NULL; + } } } diff --git a/arch/mips/pci/pci-mt7620.c b/arch/mips/pci/pci-mt7620.c index d36061603752..e032932348d6 100644 --- a/arch/mips/pci/pci-mt7620.c +++ b/arch/mips/pci/pci-mt7620.c @@ -30,6 +30,7 @@ #define RALINK_GPIOMODE 0x60 #define PPLL_CFG1 0x9c +#define PPLL_LD BIT(23) #define PPLL_DRV 0xa0 #define PDRV_SW_SET BIT(31) @@ -239,8 +240,8 @@ static int mt7620_pci_hw_init(struct platform_device *pdev) rt_sysc_m32(0, RALINK_PCIE0_CLK_EN, RALINK_CLKCFG1); mdelay(100); - if (!(rt_sysc_r32(PPLL_CFG1) & PDRV_SW_SET)) { - dev_err(&pdev->dev, "MT7620 PPLL unlock\n"); + if (!(rt_sysc_r32(PPLL_CFG1) & PPLL_LD)) { + dev_err(&pdev->dev, "pcie PLL not locked, aborting init\n"); reset_control_assert(rstpcie0); rt_sysc_m32(RALINK_PCIE0_CLK_EN, 0, RALINK_CLKCFG1); return -1; diff --git a/arch/mips/pci/pci-rt2880.c b/arch/mips/pci/pci-rt2880.c index e1f12e398136..f1538d2be89e 100644 --- a/arch/mips/pci/pci-rt2880.c +++ b/arch/mips/pci/pci-rt2880.c @@ -180,7 +180,6 @@ static inline void rt2880_pci_write_u32(unsigned long reg, u32 val) int pcibios_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) { - u16 cmd; int irq = -1; if (dev->bus->number != 0) @@ -188,8 +187,6 @@ int pcibios_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) switch (PCI_SLOT(dev->devfn)) { case 0x00: - rt2880_pci_write_u32(PCI_BASE_ADDRESS_0, 0x08000000); - (void) rt2880_pci_read_u32(PCI_BASE_ADDRESS_0); break; case 0x11: irq = RT288X_CPU_IRQ_PCI; @@ -201,16 +198,6 @@ int pcibios_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) break; } - pci_write_config_byte((struct pci_dev *) dev, - PCI_CACHE_LINE_SIZE, 0x14); - pci_write_config_byte((struct pci_dev *) dev, PCI_LATENCY_TIMER, 0xFF); - pci_read_config_word((struct pci_dev *) dev, PCI_COMMAND, &cmd); - cmd |= PCI_COMMAND_MASTER | PCI_COMMAND_IO | PCI_COMMAND_MEMORY | - PCI_COMMAND_INVALIDATE | PCI_COMMAND_FAST_BACK | - PCI_COMMAND_SERR | PCI_COMMAND_WAIT | PCI_COMMAND_PARITY; - pci_write_config_word((struct pci_dev *) dev, PCI_COMMAND, cmd); - pci_write_config_byte((struct pci_dev *) dev, PCI_INTERRUPT_LINE, - dev->irq); return irq; } @@ -251,6 +238,30 @@ static int rt288x_pci_probe(struct platform_device *pdev) int pcibios_plat_dev_init(struct pci_dev *dev) { + static bool slot0_init; + + /* + * Nobody seems to initialize slot 0, but this platform requires it, so + * do it once when some other slot is being enabled. The PCI subsystem + * should configure other slots properly, so no need to do anything + * special for those. + */ + if (!slot0_init && dev->bus->number == 0) { + u16 cmd; + u32 bar0; + + slot0_init = true; + + pci_bus_write_config_dword(dev->bus, 0, PCI_BASE_ADDRESS_0, + 0x08000000); + pci_bus_read_config_dword(dev->bus, 0, PCI_BASE_ADDRESS_0, + &bar0); + + pci_bus_read_config_word(dev->bus, 0, PCI_COMMAND, &cmd); + cmd |= PCI_COMMAND_MASTER | PCI_COMMAND_IO | PCI_COMMAND_MEMORY; + pci_bus_write_config_word(dev->bus, 0, PCI_COMMAND, cmd); + } + return 0; } diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 386ae12d8523..57c0ab71d51e 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -224,7 +224,7 @@ config PPC select HAVE_LIVEPATCH if HAVE_DYNAMIC_FTRACE_WITH_REGS select HAVE_MOD_ARCH_SPECIFIC select HAVE_NMI if PERF_EVENTS || (PPC64 && PPC_BOOK3S) - select HAVE_HARDLOCKUP_DETECTOR_ARCH if (PPC64 && PPC_BOOK3S) + select HAVE_HARDLOCKUP_DETECTOR_ARCH if PPC64 && PPC_BOOK3S && SMP select HAVE_OPTPROBES if PPC64 select HAVE_PERF_EVENTS select HAVE_PERF_EVENTS_NMI if PPC64 diff --git a/arch/powerpc/Kconfig.debug b/arch/powerpc/Kconfig.debug index ae084357994e..6342f9da4545 100644 --- a/arch/powerpc/Kconfig.debug +++ b/arch/powerpc/Kconfig.debug @@ -353,6 +353,7 @@ config PPC_EARLY_DEBUG_CPM_ADDR config FAIL_IOMMU bool "Fault-injection capability for IOMMU" depends on FAULT_INJECTION + depends on PCI || IBMVIO help Provide fault-injection capability for IOMMU. Each device can be selectively enabled via the fail_iommu property. diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index 058601efbc8a..b703330459b8 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -7,6 +7,7 @@ #ifndef __ASSEMBLY__ #include #include +#include #endif /* @@ -323,7 +324,8 @@ extern unsigned long pci_io_base; #define PHB_IO_END (KERN_IO_START + FULL_IO_SIZE) #define IOREMAP_BASE (PHB_IO_END) #define IOREMAP_START (ioremap_bot) -#define IOREMAP_END (KERN_IO_END) +#define IOREMAP_END (KERN_IO_END - FIXADDR_SIZE) +#define FIXADDR_SIZE SZ_32M /* Advertise special mapping type for AGP */ #define HAVE_PAGE_AGP diff --git a/arch/powerpc/include/asm/book3s/64/radix.h b/arch/powerpc/include/asm/book3s/64/radix.h index c7813dc628fc..59cab558e2f0 100644 --- a/arch/powerpc/include/asm/book3s/64/radix.h +++ b/arch/powerpc/include/asm/book3s/64/radix.h @@ -222,8 +222,10 @@ static inline void radix__set_pte_at(struct mm_struct *mm, unsigned long addr, * from ptesync, it should probably go into update_mmu_cache, rather * than set_pte_at (which is used to set ptes unrelated to faults). * - * Spurious faults to vmalloc region are not tolerated, so there is - * a ptesync in flush_cache_vmap. + * Spurious faults from the kernel memory are not tolerated, so there + * is a ptesync in flush_cache_vmap, and __map_kernel_page() follows + * the pte update sequence from ISA Book III 6.10 Translation Table + * Update Synchronization Requirements. */ } diff --git a/arch/powerpc/include/asm/fixmap.h b/arch/powerpc/include/asm/fixmap.h index 8d03c16a3663..947b5b9c4424 100644 --- a/arch/powerpc/include/asm/fixmap.h +++ b/arch/powerpc/include/asm/fixmap.h @@ -23,12 +23,17 @@ #include #endif +#ifdef CONFIG_PPC64 +#define FIXADDR_TOP (IOREMAP_END + FIXADDR_SIZE) +#else +#define FIXADDR_SIZE 0 #ifdef CONFIG_KASAN #include #define FIXADDR_TOP (KASAN_SHADOW_START - PAGE_SIZE) #else #define FIXADDR_TOP ((unsigned long)(-PAGE_SIZE)) #endif +#endif /* * Here we define all the compile-time 'special' virtual @@ -50,6 +55,7 @@ */ enum fixed_addresses { FIX_HOLE, +#ifdef CONFIG_PPC32 /* reserve the top 128K for early debugging purposes */ FIX_EARLY_DEBUG_TOP = FIX_HOLE, FIX_EARLY_DEBUG_BASE = FIX_EARLY_DEBUG_TOP+(ALIGN(SZ_128K, PAGE_SIZE)/PAGE_SIZE)-1, @@ -72,6 +78,7 @@ enum fixed_addresses { FIX_IMMR_SIZE, #endif /* FIX_PCIE_MCFG, */ +#endif /* CONFIG_PPC32 */ __end_of_permanent_fixed_addresses, #define NR_FIX_BTMAPS (SZ_256K / PAGE_SIZE) @@ -98,6 +105,8 @@ enum fixed_addresses { static inline void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t flags) { + BUILD_BUG_ON(IS_ENABLED(CONFIG_PPC64) && __FIXADDR_SIZE > FIXADDR_SIZE); + if (__builtin_constant_p(idx)) BUILD_BUG_ON(idx >= __end_of_fixed_addresses); else if (WARN_ON(idx >= __end_of_fixed_addresses)) diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h index e8d09a841373..31ed5356590a 100644 --- a/arch/powerpc/include/asm/interrupt.h +++ b/arch/powerpc/include/asm/interrupt.h @@ -138,6 +138,13 @@ static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct inte local_paca->irq_soft_mask = IRQS_ALL_DISABLED; local_paca->irq_happened |= PACA_IRQ_HARD_DIS; + if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && !(regs->msr & MSR_PR) && + regs->nip < (unsigned long)__end_interrupts) { + // Kernel code running below __end_interrupts is + // implicitly soft-masked. + regs->softe = IRQS_ALL_DISABLED; + } + /* Don't do any per-CPU operations until interrupt state is fixed */ #endif /* Allow DEC and PMI to be traced when they are soft-NMI */ diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h index 652ce85f9410..4bc45d3ed8b0 100644 --- a/arch/powerpc/include/asm/mmu_context.h +++ b/arch/powerpc/include/asm/mmu_context.h @@ -263,7 +263,7 @@ extern void arch_exit_mmap(struct mm_struct *mm); static inline void arch_unmap(struct mm_struct *mm, unsigned long start, unsigned long end) { - unsigned long vdso_base = (unsigned long)mm->context.vdso - PAGE_SIZE; + unsigned long vdso_base = (unsigned long)mm->context.vdso; if (start <= vdso_base && vdso_base < end) mm->context.vdso = NULL; diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h index 6cb8aa357191..57cd3892bfe0 100644 --- a/arch/powerpc/include/asm/nohash/64/pgtable.h +++ b/arch/powerpc/include/asm/nohash/64/pgtable.h @@ -6,6 +6,8 @@ * the ppc64 non-hashed page table. */ +#include + #include #include #include @@ -54,7 +56,8 @@ #define PHB_IO_END (KERN_IO_START + FULL_IO_SIZE) #define IOREMAP_BASE (PHB_IO_END) #define IOREMAP_START (ioremap_bot) -#define IOREMAP_END (KERN_VIRT_START + KERN_VIRT_SIZE) +#define IOREMAP_END (KERN_VIRT_START + KERN_VIRT_SIZE - FIXADDR_SIZE) +#define FIXADDR_SIZE SZ_32M /* diff --git a/arch/powerpc/include/asm/reg.h b/arch/powerpc/include/asm/reg.h index da103e92c112..37d0b8c76a59 100644 --- a/arch/powerpc/include/asm/reg.h +++ b/arch/powerpc/include/asm/reg.h @@ -441,6 +441,7 @@ #define LPCR_VRMA_LP1 ASM_CONST(0x0000800000000000) #define LPCR_RMLS 0x1C000000 /* Implementation dependent RMO limit sel */ #define LPCR_RMLS_SH 26 +#define LPCR_HAIL ASM_CONST(0x0000000004000000) /* HV AIL (ISAv3.1) */ #define LPCR_ILE ASM_CONST(0x0000000002000000) /* !HV irqs set MSR:LE */ #define LPCR_AIL ASM_CONST(0x0000000001800000) /* Alternate interrupt location */ #define LPCR_AIL_0 ASM_CONST(0x0000000000000000) /* MMU off exception offset 0x0 */ diff --git a/arch/powerpc/include/asm/smp.h b/arch/powerpc/include/asm/smp.h index 7a13bc20f0a0..47081a9e13ca 100644 --- a/arch/powerpc/include/asm/smp.h +++ b/arch/powerpc/include/asm/smp.h @@ -121,6 +121,11 @@ static inline struct cpumask *cpu_sibling_mask(int cpu) return per_cpu(cpu_sibling_map, cpu); } +static inline struct cpumask *cpu_core_mask(int cpu) +{ + return per_cpu(cpu_core_map, cpu); +} + static inline struct cpumask *cpu_l2_cache_mask(int cpu) { return per_cpu(cpu_l2_cache_map, cpu); diff --git a/arch/powerpc/include/uapi/asm/errno.h b/arch/powerpc/include/uapi/asm/errno.h index cc79856896a1..4ba87de32be0 100644 --- a/arch/powerpc/include/uapi/asm/errno.h +++ b/arch/powerpc/include/uapi/asm/errno.h @@ -2,6 +2,7 @@ #ifndef _ASM_POWERPC_ERRNO_H #define _ASM_POWERPC_ERRNO_H +#undef EDEADLOCK #include #undef EDEADLOCK diff --git a/arch/powerpc/kernel/eeh.c b/arch/powerpc/kernel/eeh.c index cd60bc1c8701..7040e430a124 100644 --- a/arch/powerpc/kernel/eeh.c +++ b/arch/powerpc/kernel/eeh.c @@ -362,14 +362,11 @@ static inline unsigned long eeh_token_to_phys(unsigned long token) pa = pte_pfn(*ptep); /* On radix we can do hugepage mappings for io, so handle that */ - if (hugepage_shift) { - pa <<= hugepage_shift; - pa |= token & ((1ul << hugepage_shift) - 1); - } else { - pa <<= PAGE_SHIFT; - pa |= token & (PAGE_SIZE - 1); - } + if (!hugepage_shift) + hugepage_shift = PAGE_SHIFT; + pa <<= PAGE_SHIFT; + pa |= token & ((1ul << hugepage_shift) - 1); return pa; } diff --git a/arch/powerpc/kernel/fadump.c b/arch/powerpc/kernel/fadump.c index 8482739d42f3..eddf362caedc 100644 --- a/arch/powerpc/kernel/fadump.c +++ b/arch/powerpc/kernel/fadump.c @@ -292,7 +292,7 @@ static void fadump_show_config(void) * that is required for a kernel to boot successfully. * */ -static inline u64 fadump_calculate_reserve_size(void) +static __init u64 fadump_calculate_reserve_size(void) { u64 base, size, bootmem_min; int ret; diff --git a/arch/powerpc/kernel/head_32.h b/arch/powerpc/kernel/head_32.h index 5d4706c14572..cf8ca08295bf 100644 --- a/arch/powerpc/kernel/head_32.h +++ b/arch/powerpc/kernel/head_32.h @@ -261,11 +261,7 @@ lis r1, emergency_ctx@ha #endif lwz r1, emergency_ctx@l(r1) - cmpwi cr1, r1, 0 - bne cr1, 1f - lis r1, init_thread_union@ha - addi r1, r1, init_thread_union@l -1: addi r1, r1, THREAD_SIZE - INT_FRAME_SIZE + addi r1, r1, THREAD_SIZE - INT_FRAME_SIZE EXCEPTION_PROLOG_2 SAVE_NVGPRS(r11) addi r3, r1, STACK_FRAME_OVERHEAD diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c index c475a229a42a..352346e14a08 100644 --- a/arch/powerpc/kernel/interrupt.c +++ b/arch/powerpc/kernel/interrupt.c @@ -34,11 +34,11 @@ notrace long system_call_exception(long r3, long r4, long r5, if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) BUG_ON(irq_soft_mask_return() != IRQS_ALL_DISABLED); + trace_hardirqs_off(); /* finish reconciling */ + CT_WARN_ON(ct_state() == CONTEXT_KERNEL); user_exit_irqoff(); - trace_hardirqs_off(); /* finish reconciling */ - if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x)) BUG_ON(!(regs->msr & MSR_RI)); BUG_ON(!(regs->msr & MSR_PR)); diff --git a/arch/powerpc/kernel/iommu.c b/arch/powerpc/kernel/iommu.c index c00214a4355c..4023f91defa6 100644 --- a/arch/powerpc/kernel/iommu.c +++ b/arch/powerpc/kernel/iommu.c @@ -1096,7 +1096,7 @@ int iommu_take_ownership(struct iommu_table *tbl) spin_lock_irqsave(&tbl->large_pool.lock, flags); for (i = 0; i < tbl->nr_pools; i++) - spin_lock(&tbl->pools[i].lock); + spin_lock_nest_lock(&tbl->pools[i].lock, &tbl->large_pool.lock); iommu_table_release_pages(tbl); @@ -1124,7 +1124,7 @@ void iommu_release_ownership(struct iommu_table *tbl) spin_lock_irqsave(&tbl->large_pool.lock, flags); for (i = 0; i < tbl->nr_pools; i++) - spin_lock(&tbl->pools[i].lock); + spin_lock_nest_lock(&tbl->pools[i].lock, &tbl->large_pool.lock); memset(tbl->it_map, 0, sz); diff --git a/arch/powerpc/kernel/prom.c b/arch/powerpc/kernel/prom.c index 9a4797d1d40d..a8b2d6bfc1ca 100644 --- a/arch/powerpc/kernel/prom.c +++ b/arch/powerpc/kernel/prom.c @@ -267,7 +267,7 @@ static struct feature_property { }; #if defined(CONFIG_44x) && defined(CONFIG_PPC_FPU) -static inline void identical_pvr_fixup(unsigned long node) +static __init void identical_pvr_fixup(unsigned long node) { unsigned int pvr; const char *model = of_get_flat_dt_prop(node, "model", NULL); diff --git a/arch/powerpc/kernel/setup_32.c b/arch/powerpc/kernel/setup_32.c index 8ba49a6bf515..d7c1f92152af 100644 --- a/arch/powerpc/kernel/setup_32.c +++ b/arch/powerpc/kernel/setup_32.c @@ -164,7 +164,7 @@ void __init irqstack_early_init(void) } #ifdef CONFIG_VMAP_STACK -void *emergency_ctx[NR_CPUS] __ro_after_init; +void *emergency_ctx[NR_CPUS] __ro_after_init = {[0] = &init_stack}; void __init emergency_stack_init(void) { diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index 560ed8b975e7..830fee91b2d9 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -232,10 +232,23 @@ static void cpu_ready_for_interrupts(void) * If we are not in hypervisor mode the job is done once for * the whole partition in configure_exceptions(). */ - if (cpu_has_feature(CPU_FTR_HVMODE) && - cpu_has_feature(CPU_FTR_ARCH_207S)) { + if (cpu_has_feature(CPU_FTR_HVMODE)) { unsigned long lpcr = mfspr(SPRN_LPCR); - mtspr(SPRN_LPCR, lpcr | LPCR_AIL_3); + unsigned long new_lpcr = lpcr; + + if (cpu_has_feature(CPU_FTR_ARCH_31)) { + /* P10 DD1 does not have HAIL */ + if (pvr_version_is(PVR_POWER10) && + (mfspr(SPRN_PVR) & 0xf00) == 0x100) + new_lpcr |= LPCR_AIL_3; + else + new_lpcr |= LPCR_HAIL; + } else if (cpu_has_feature(CPU_FTR_ARCH_207S)) { + new_lpcr |= LPCR_AIL_3; + } + + if (new_lpcr != lpcr) + mtspr(SPRN_LPCR, new_lpcr); } /* diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index 5a4d59a1070d..c2473e20f5f5 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -1057,17 +1057,12 @@ void __init smp_prepare_cpus(unsigned int max_cpus) local_memory_node(numa_cpu_lookup_table[cpu])); } #endif - /* - * cpu_core_map is now more updated and exists only since - * its been exported for long. It only will have a snapshot - * of cpu_cpu_mask. - */ - cpumask_copy(per_cpu(cpu_core_map, cpu), cpu_cpu_mask(cpu)); } /* Init the cpumasks so the boot CPU is related to itself */ cpumask_set_cpu(boot_cpuid, cpu_sibling_mask(boot_cpuid)); cpumask_set_cpu(boot_cpuid, cpu_l2_cache_mask(boot_cpuid)); + cpumask_set_cpu(boot_cpuid, cpu_core_mask(boot_cpuid)); if (has_coregroup_support()) cpumask_set_cpu(boot_cpuid, cpu_coregroup_mask(boot_cpuid)); @@ -1408,6 +1403,9 @@ static void remove_cpu_from_masks(int cpu) set_cpus_unrelated(cpu, i, cpu_smallcore_mask); } + for_each_cpu(i, cpu_core_mask(cpu)) + set_cpus_unrelated(cpu, i, cpu_core_mask); + if (has_coregroup_support()) { for_each_cpu(i, cpu_coregroup_mask(cpu)) set_cpus_unrelated(cpu, i, cpu_coregroup_mask); @@ -1468,8 +1466,11 @@ static void update_coregroup_mask(int cpu, cpumask_var_t *mask) static void add_cpu_to_masks(int cpu) { + struct cpumask *(*submask_fn)(int) = cpu_sibling_mask; int first_thread = cpu_first_thread_sibling(cpu); + int chip_id = cpu_to_chip_id(cpu); cpumask_var_t mask; + bool ret; int i; /* @@ -1485,12 +1486,36 @@ static void add_cpu_to_masks(int cpu) add_cpu_to_smallcore_masks(cpu); /* In CPU-hotplug path, hence use GFP_ATOMIC */ - alloc_cpumask_var_node(&mask, GFP_ATOMIC, cpu_to_node(cpu)); + ret = alloc_cpumask_var_node(&mask, GFP_ATOMIC, cpu_to_node(cpu)); update_mask_by_l2(cpu, &mask); if (has_coregroup_support()) update_coregroup_mask(cpu, &mask); + if (chip_id == -1 || !ret) { + cpumask_copy(per_cpu(cpu_core_map, cpu), cpu_cpu_mask(cpu)); + goto out; + } + + if (shared_caches) + submask_fn = cpu_l2_cache_mask; + + /* Update core_mask with all the CPUs that are part of submask */ + or_cpumasks_related(cpu, cpu, submask_fn, cpu_core_mask); + + /* Skip all CPUs already part of current CPU core mask */ + cpumask_andnot(mask, cpu_online_mask, cpu_core_mask(cpu)); + + for_each_cpu(i, mask) { + if (chip_id == cpu_to_chip_id(i)) { + or_cpumasks_related(cpu, i, submask_fn, cpu_core_mask); + cpumask_andnot(mask, mask, submask_fn(i)); + } else { + cpumask_andnot(mask, mask, cpu_core_mask(i)); + } + } + +out: free_cpumask_var(mask); } @@ -1521,6 +1546,9 @@ void start_secondary(void *unused) vdso_getcpu_init(); #endif + set_numa_node(numa_cpu_lookup_table[cpu]); + set_numa_mem(local_memory_node(numa_cpu_lookup_table[cpu])); + /* Update topology CPU masks */ add_cpu_to_masks(cpu); @@ -1539,9 +1567,6 @@ void start_secondary(void *unused) shared_caches = true; } - set_numa_node(numa_cpu_lookup_table[cpu]); - set_numa_mem(local_memory_node(numa_cpu_lookup_table[cpu])); - smp_wmb(); notify_cpu_starting(cpu); set_cpu_online(cpu, true); diff --git a/arch/powerpc/kernel/vdso.c b/arch/powerpc/kernel/vdso.c index e839a906fdf2..b14907209822 100644 --- a/arch/powerpc/kernel/vdso.c +++ b/arch/powerpc/kernel/vdso.c @@ -55,10 +55,10 @@ static int vdso_mremap(const struct vm_special_mapping *sm, struct vm_area_struc { unsigned long new_size = new_vma->vm_end - new_vma->vm_start; - if (new_size != text_size + PAGE_SIZE) + if (new_size != text_size) return -EINVAL; - current->mm->context.vdso = (void __user *)new_vma->vm_start + PAGE_SIZE; + current->mm->context.vdso = (void __user *)new_vma->vm_start; return 0; } @@ -73,6 +73,10 @@ static int vdso64_mremap(const struct vm_special_mapping *sm, struct vm_area_str return vdso_mremap(sm, new_vma, &vdso64_end - &vdso64_start); } +static struct vm_special_mapping vvar_spec __ro_after_init = { + .name = "[vvar]", +}; + static struct vm_special_mapping vdso32_spec __ro_after_init = { .name = "[vdso]", .mremap = vdso32_mremap, @@ -89,11 +93,11 @@ static struct vm_special_mapping vdso64_spec __ro_after_init = { */ static int __arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) { - struct mm_struct *mm = current->mm; + unsigned long vdso_size, vdso_base, mappings_size; struct vm_special_mapping *vdso_spec; + unsigned long vvar_size = PAGE_SIZE; + struct mm_struct *mm = current->mm; struct vm_area_struct *vma; - unsigned long vdso_size; - unsigned long vdso_base; if (is_32bit_task()) { vdso_spec = &vdso32_spec; @@ -110,8 +114,8 @@ static int __arch_setup_additional_pages(struct linux_binprm *bprm, int uses_int vdso_base = 0; } - /* Add a page to the vdso size for the data page */ - vdso_size += PAGE_SIZE; + mappings_size = vdso_size + vvar_size; + mappings_size += (VDSO_ALIGNMENT - 1) & PAGE_MASK; /* * pick a base address for the vDSO in process space. We try to put it @@ -119,9 +123,7 @@ static int __arch_setup_additional_pages(struct linux_binprm *bprm, int uses_int * and end up putting it elsewhere. * Add enough to the size so that the result can be aligned. */ - vdso_base = get_unmapped_area(NULL, vdso_base, - vdso_size + ((VDSO_ALIGNMENT - 1) & PAGE_MASK), - 0, 0); + vdso_base = get_unmapped_area(NULL, vdso_base, mappings_size, 0, 0); if (IS_ERR_VALUE(vdso_base)) return vdso_base; @@ -133,7 +135,13 @@ static int __arch_setup_additional_pages(struct linux_binprm *bprm, int uses_int * install_special_mapping or the perf counter mmap tracking code * will fail to recognise it as a vDSO. */ - mm->context.vdso = (void __user *)vdso_base + PAGE_SIZE; + mm->context.vdso = (void __user *)vdso_base + vvar_size; + + vma = _install_special_mapping(mm, vdso_base, vvar_size, + VM_READ | VM_MAYREAD | VM_IO | + VM_DONTDUMP | VM_PFNMAP, &vvar_spec); + if (IS_ERR(vma)) + return PTR_ERR(vma); /* * our vma flags don't have VM_WRITE so by default, the process isn't @@ -145,9 +153,12 @@ static int __arch_setup_additional_pages(struct linux_binprm *bprm, int uses_int * It's fine to use that for setting breakpoints in the vDSO code * pages though. */ - vma = _install_special_mapping(mm, vdso_base, vdso_size, + vma = _install_special_mapping(mm, vdso_base + vvar_size, vdso_size, VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC, vdso_spec); + if (IS_ERR(vma)) + do_munmap(mm, vdso_base, vvar_size, NULL); + return PTR_ERR_OR_ZERO(vma); } @@ -249,11 +260,22 @@ static struct page ** __init vdso_setup_pages(void *start, void *end) if (!pagelist) panic("%s: Cannot allocate page list for VDSO", __func__); - pagelist[0] = virt_to_page(vdso_data); - for (i = 0; i < pages; i++) - pagelist[i + 1] = virt_to_page(start + i * PAGE_SIZE); + pagelist[i] = virt_to_page(start + i * PAGE_SIZE); + + return pagelist; +} + +static struct page ** __init vvar_setup_pages(void) +{ + struct page **pagelist; + /* .pages is NULL-terminated */ + pagelist = kcalloc(2, sizeof(struct page *), GFP_KERNEL); + if (!pagelist) + panic("%s: Cannot allocate page list for VVAR", __func__); + + pagelist[0] = virt_to_page(vdso_data); return pagelist; } @@ -295,6 +317,8 @@ static int __init vdso_init(void) if (IS_ENABLED(CONFIG_PPC64)) vdso64_spec.pages = vdso_setup_pages(&vdso64_start, &vdso64_end); + vvar_spec.pages = vvar_setup_pages(); + smp_wmb(); return 0; diff --git a/arch/powerpc/kexec/file_load_64.c b/arch/powerpc/kexec/file_load_64.c index 02b9e4d0dc40..a8a7cb71086b 100644 --- a/arch/powerpc/kexec/file_load_64.c +++ b/arch/powerpc/kexec/file_load_64.c @@ -960,6 +960,93 @@ unsigned int kexec_fdt_totalsize_ppc64(struct kimage *image) return fdt_size; } +/** + * add_node_props - Reads node properties from device node structure and add + * them to fdt. + * @fdt: Flattened device tree of the kernel + * @node_offset: offset of the node to add a property at + * @dn: device node pointer + * + * Returns 0 on success, negative errno on error. + */ +static int add_node_props(void *fdt, int node_offset, const struct device_node *dn) +{ + int ret = 0; + struct property *pp; + + if (!dn) + return -EINVAL; + + for_each_property_of_node(dn, pp) { + ret = fdt_setprop(fdt, node_offset, pp->name, pp->value, pp->length); + if (ret < 0) { + pr_err("Unable to add %s property: %s\n", pp->name, fdt_strerror(ret)); + return ret; + } + } + return ret; +} + +/** + * update_cpus_node - Update cpus node of flattened device tree using of_root + * device node. + * @fdt: Flattened device tree of the kernel. + * + * Returns 0 on success, negative errno on error. + */ +static int update_cpus_node(void *fdt) +{ + struct device_node *cpus_node, *dn; + int cpus_offset, cpus_subnode_offset, ret = 0; + + cpus_offset = fdt_path_offset(fdt, "/cpus"); + if (cpus_offset < 0 && cpus_offset != -FDT_ERR_NOTFOUND) { + pr_err("Malformed device tree: error reading /cpus node: %s\n", + fdt_strerror(cpus_offset)); + return cpus_offset; + } + + if (cpus_offset > 0) { + ret = fdt_del_node(fdt, cpus_offset); + if (ret < 0) { + pr_err("Error deleting /cpus node: %s\n", fdt_strerror(ret)); + return -EINVAL; + } + } + + /* Add cpus node to fdt */ + cpus_offset = fdt_add_subnode(fdt, fdt_path_offset(fdt, "/"), "cpus"); + if (cpus_offset < 0) { + pr_err("Error creating /cpus node: %s\n", fdt_strerror(cpus_offset)); + return -EINVAL; + } + + /* Add cpus node properties */ + cpus_node = of_find_node_by_path("/cpus"); + ret = add_node_props(fdt, cpus_offset, cpus_node); + of_node_put(cpus_node); + if (ret < 0) + return ret; + + /* Loop through all subnodes of cpus and add them to fdt */ + for_each_node_by_type(dn, "cpu") { + cpus_subnode_offset = fdt_add_subnode(fdt, cpus_offset, dn->full_name); + if (cpus_subnode_offset < 0) { + pr_err("Unable to add %s subnode: %s\n", dn->full_name, + fdt_strerror(cpus_subnode_offset)); + ret = cpus_subnode_offset; + goto out; + } + + ret = add_node_props(fdt, cpus_subnode_offset, dn); + if (ret < 0) + goto out; + } +out: + of_node_put(dn); + return ret; +} + /** * setup_new_fdt_ppc64 - Update the flattend device-tree of the kernel * being loaded. @@ -1020,6 +1107,11 @@ int setup_new_fdt_ppc64(const struct kimage *image, void *fdt, } } + /* Update cpus nodes information to account hotplug CPUs. */ + ret = update_cpus_node(fdt); + if (ret < 0) + goto out; + /* Update memory reserve map */ ret = get_reserved_memory_ranges(&rmem); if (ret) diff --git a/arch/powerpc/kvm/book3s_64_mmu_host.c b/arch/powerpc/kvm/book3s_64_mmu_host.c index e452158a18d7..c3e31fef0be1 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_host.c +++ b/arch/powerpc/kvm/book3s_64_mmu_host.c @@ -8,6 +8,7 @@ */ #include +#include #include #include @@ -133,6 +134,7 @@ int kvmppc_mmu_map_page(struct kvm_vcpu *vcpu, struct kvmppc_pte *orig_pte, else kvmppc_mmu_flush_icache(pfn); + rflags |= pte_to_hpte_pkey_bits(0, HPTE_USE_KERNEL_KEY); rflags = (rflags & ~HPTE_R_WIMG) | orig_pte->wimg; /* diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index 13bad6bf4c95..208a053c9adf 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -3728,7 +3728,10 @@ static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit, vcpu->arch.dec_expires = dec + tb; vcpu->cpu = -1; vcpu->arch.thread_cpu = -1; + /* Save guest CTRL register, set runlatch to 1 */ vcpu->arch.ctrl = mfspr(SPRN_CTRLF); + if (!(vcpu->arch.ctrl & 1)) + mtspr(SPRN_CTRLT, vcpu->arch.ctrl | 1); vcpu->arch.iamr = mfspr(SPRN_IAMR); vcpu->arch.pspb = mfspr(SPRN_PSPB); diff --git a/arch/powerpc/lib/Makefile b/arch/powerpc/lib/Makefile index d4efc182662a..248f7c9e36fc 100644 --- a/arch/powerpc/lib/Makefile +++ b/arch/powerpc/lib/Makefile @@ -5,6 +5,9 @@ ccflags-$(CONFIG_PPC64) := $(NO_MINIMAL_TOC) +CFLAGS_code-patching.o += -fno-stack-protector +CFLAGS_feature-fixups.o += -fno-stack-protector + CFLAGS_REMOVE_code-patching.o = $(CC_FLAGS_FTRACE) CFLAGS_REMOVE_feature-fixups.o = $(CC_FLAGS_FTRACE) diff --git a/arch/powerpc/lib/feature-fixups.c b/arch/powerpc/lib/feature-fixups.c index 1fd31b4b0e13..0aefa6a4a259 100644 --- a/arch/powerpc/lib/feature-fixups.c +++ b/arch/powerpc/lib/feature-fixups.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -227,11 +228,25 @@ static void do_stf_exit_barrier_fixups(enum stf_barrier_type types) : "unknown"); } +static int __do_stf_barrier_fixups(void *data) +{ + enum stf_barrier_type *types = data; + + do_stf_entry_barrier_fixups(*types); + do_stf_exit_barrier_fixups(*types); + + return 0; +} void do_stf_barrier_fixups(enum stf_barrier_type types) { - do_stf_entry_barrier_fixups(types); - do_stf_exit_barrier_fixups(types); + /* + * The call to the fallback entry flush, and the fallback/sync-ori exit + * flush can not be safely patched in/out while other CPUs are executing + * them. So call __do_stf_barrier_fixups() on one CPU while all other CPUs + * spin in the stop machine core with interrupts hard disabled. + */ + stop_machine(__do_stf_barrier_fixups, &types, NULL); } void do_uaccess_flush_fixups(enum l1d_flush_type types) @@ -284,8 +299,9 @@ void do_uaccess_flush_fixups(enum l1d_flush_type types) : "unknown"); } -void do_entry_flush_fixups(enum l1d_flush_type types) +static int __do_entry_flush_fixups(void *data) { + enum l1d_flush_type types = *(enum l1d_flush_type *)data; unsigned int instrs[3], *dest; long *start, *end; int i; @@ -354,6 +370,19 @@ void do_entry_flush_fixups(enum l1d_flush_type types) : "ori type" : (types & L1D_FLUSH_MTTRIG) ? "mttrig type" : "unknown"); + + return 0; +} + +void do_entry_flush_fixups(enum l1d_flush_type types) +{ + /* + * The call to the fallback flush can not be safely patched in/out while + * other CPUs are executing it. So call __do_entry_flush_fixups() on one + * CPU while all other CPUs spin in the stop machine core with interrupts + * hard disabled. + */ + stop_machine(__do_entry_flush_fixups, &types, NULL); } void do_rfi_flush_fixups(enum l1d_flush_type types) diff --git a/arch/powerpc/mm/book3s64/hash_pgtable.c b/arch/powerpc/mm/book3s64/hash_pgtable.c index 567e0c6b3978..03819c259f0a 100644 --- a/arch/powerpc/mm/book3s64/hash_pgtable.c +++ b/arch/powerpc/mm/book3s64/hash_pgtable.c @@ -428,12 +428,14 @@ static bool hash__change_memory_range(unsigned long start, unsigned long end, void hash__mark_rodata_ro(void) { - unsigned long start, end; + unsigned long start, end, pp; start = (unsigned long)_stext; end = (unsigned long)__init_begin; - WARN_ON(!hash__change_memory_range(start, end, PP_RXXX)); + pp = htab_convert_pte_flags(pgprot_val(PAGE_KERNEL_ROX), HPTE_USE_KERNEL_KEY); + + WARN_ON(!hash__change_memory_range(start, end, pp)); } void hash__mark_initmem_nx(void) diff --git a/arch/powerpc/mm/book3s64/hash_utils.c b/arch/powerpc/mm/book3s64/hash_utils.c index 581b20a2feaf..12de1906e97b 100644 --- a/arch/powerpc/mm/book3s64/hash_utils.c +++ b/arch/powerpc/mm/book3s64/hash_utils.c @@ -338,7 +338,7 @@ int htab_bolt_mapping(unsigned long vstart, unsigned long vend, int htab_remove_mapping(unsigned long vstart, unsigned long vend, int psize, int ssize) { - unsigned long vaddr; + unsigned long vaddr, time_limit; unsigned int step, shift; int rc; int ret = 0; @@ -351,8 +351,19 @@ int htab_remove_mapping(unsigned long vstart, unsigned long vend, /* Unmap the full range specificied */ vaddr = ALIGN_DOWN(vstart, step); + time_limit = jiffies + HZ; + for (;vaddr < vend; vaddr += step) { rc = mmu_hash_ops.hpte_removebolted(vaddr, psize, ssize); + + /* + * For large number of mappings introduce a cond_resched() + * to prevent softlockup warnings. + */ + if (time_after(jiffies, time_limit)) { + cond_resched(); + time_limit = jiffies + HZ; + } if (rc == -ENOENT) { ret = -ENOENT; continue; @@ -1545,10 +1556,10 @@ DEFINE_INTERRUPT_HANDLER_RET(__do_hash_fault) if (user_mode(regs) || (region_id == USER_REGION_ID)) access &= ~_PAGE_PRIVILEGED; - if (regs->trap == 0x400) + if (TRAP(regs) == 0x400) access |= _PAGE_EXEC; - err = hash_page_mm(mm, ea, access, regs->trap, flags); + err = hash_page_mm(mm, ea, access, TRAP(regs), flags); if (unlikely(err < 0)) { // failed to instert a hash PTE due to an hypervisor error if (user_mode(regs)) { diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c index 98f0b243c1ab..39d488a212a0 100644 --- a/arch/powerpc/mm/book3s64/radix_pgtable.c +++ b/arch/powerpc/mm/book3s64/radix_pgtable.c @@ -108,7 +108,7 @@ static int early_map_kernel_page(unsigned long ea, unsigned long pa, set_the_pte: set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags)); - smp_wmb(); + asm volatile("ptesync": : :"memory"); return 0; } @@ -168,7 +168,7 @@ static int __map_kernel_page(unsigned long ea, unsigned long pa, set_the_pte: set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags)); - smp_wmb(); + asm volatile("ptesync": : :"memory"); return 0; } diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c index 4e8ce6d85232..7a59a5c9aa5d 100644 --- a/arch/powerpc/mm/mem.c +++ b/arch/powerpc/mm/mem.c @@ -54,7 +54,6 @@ #include -static DEFINE_MUTEX(linear_mapping_mutex); unsigned long long memory_limit; bool init_mem_is_free; @@ -72,6 +71,7 @@ pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, EXPORT_SYMBOL(phys_mem_access_prot); #ifdef CONFIG_MEMORY_HOTPLUG +static DEFINE_MUTEX(linear_mapping_mutex); #ifdef CONFIG_NUMA int memory_add_physaddr_to_nid(u64 start) diff --git a/arch/powerpc/perf/isa207-common.c b/arch/powerpc/perf/isa207-common.c index e4f577da33d8..8b5eeb6fb2fb 100644 --- a/arch/powerpc/perf/isa207-common.c +++ b/arch/powerpc/perf/isa207-common.c @@ -447,8 +447,8 @@ int isa207_get_constraint(u64 event, unsigned long *maskp, unsigned long *valp, * EBB events are pinned & exclusive, so this should never actually * hit, but we leave it as a fallback in case. */ - mask |= CNST_EBB_VAL(ebb); - value |= CNST_EBB_MASK; + mask |= CNST_EBB_MASK; + value |= CNST_EBB_VAL(ebb); *maskp = mask; *valp = value; diff --git a/arch/powerpc/perf/power10-events-list.h b/arch/powerpc/perf/power10-events-list.h index e45dafe818ed..93be7197d250 100644 --- a/arch/powerpc/perf/power10-events-list.h +++ b/arch/powerpc/perf/power10-events-list.h @@ -75,5 +75,5 @@ EVENT(PM_RUN_INST_CMPL_ALT, 0x00002); * thresh end (TE) */ -EVENT(MEM_LOADS, 0x34340401e0); -EVENT(MEM_STORES, 0x343c0401e0); +EVENT(MEM_LOADS, 0x35340401e0); +EVENT(MEM_STORES, 0x353c0401e0); diff --git a/arch/powerpc/platforms/52xx/lite5200_sleep.S b/arch/powerpc/platforms/52xx/lite5200_sleep.S index 11475c58ea43..afee8b1515a8 100644 --- a/arch/powerpc/platforms/52xx/lite5200_sleep.S +++ b/arch/powerpc/platforms/52xx/lite5200_sleep.S @@ -181,7 +181,7 @@ sram_code: udelay: /* r11 - tb_ticks_per_usec, r12 - usecs, overwrites r13 */ mullw r12, r12, r11 mftb r13 /* start */ - addi r12, r13, r12 /* end */ + add r12, r13, r12 /* end */ 1: mftb r13 /* current */ cmp cr0, r13, r12 diff --git a/arch/powerpc/platforms/powernv/memtrace.c b/arch/powerpc/platforms/powernv/memtrace.c index 019669eb21d2..4ab7c3ef5826 100644 --- a/arch/powerpc/platforms/powernv/memtrace.c +++ b/arch/powerpc/platforms/powernv/memtrace.c @@ -88,8 +88,8 @@ static void memtrace_clear_range(unsigned long start_pfn, * Before we go ahead and use this range as cache inhibited range * flush the cache. */ - flush_dcache_range_chunked(PFN_PHYS(start_pfn), - PFN_PHYS(start_pfn + nr_pages), + flush_dcache_range_chunked((unsigned long)pfn_to_kaddr(start_pfn), + (unsigned long)pfn_to_kaddr(start_pfn + nr_pages), FLUSH_CHUNK_SIZE); } diff --git a/arch/powerpc/platforms/pseries/hotplug-cpu.c b/arch/powerpc/platforms/pseries/hotplug-cpu.c index 12cbffd3c2e3..325f3b220f36 100644 --- a/arch/powerpc/platforms/pseries/hotplug-cpu.c +++ b/arch/powerpc/platforms/pseries/hotplug-cpu.c @@ -47,9 +47,6 @@ static void rtas_stop_self(void) BUG_ON(rtas_stop_self_token == RTAS_UNKNOWN_SERVICE); - printk("cpu %u (hwid %u) Ready to die...\n", - smp_processor_id(), hard_smp_processor_id()); - rtas_call_unlocked(&args, rtas_stop_self_token, 0, 1, NULL); panic("Alas, I survived.\n"); diff --git a/arch/powerpc/platforms/pseries/iommu.c b/arch/powerpc/platforms/pseries/iommu.c index 9fc5217f0c8e..836cbbe0ecc5 100644 --- a/arch/powerpc/platforms/pseries/iommu.c +++ b/arch/powerpc/platforms/pseries/iommu.c @@ -1229,7 +1229,7 @@ static u64 enable_ddw(struct pci_dev *dev, struct device_node *pdn) if (pmem_present) { if (query.largest_available_block >= (1ULL << (MAX_PHYSMEM_BITS - page_shift))) - len = MAX_PHYSMEM_BITS - page_shift; + len = MAX_PHYSMEM_BITS; else dev_info(&dev->dev, "Skipping ibm,pmemory"); } diff --git a/arch/powerpc/platforms/pseries/lpar.c b/arch/powerpc/platforms/pseries/lpar.c index 3805519a6469..cd38bd421f38 100644 --- a/arch/powerpc/platforms/pseries/lpar.c +++ b/arch/powerpc/platforms/pseries/lpar.c @@ -977,11 +977,13 @@ static void pSeries_lpar_hpte_updateboltedpp(unsigned long newpp, slot = pSeries_lpar_hpte_find(vpn, psize, ssize); BUG_ON(slot == -1); - flags = newpp & 7; + flags = newpp & (HPTE_R_PP | HPTE_R_N); if (mmu_has_feature(MMU_FTR_KERNEL_RO)) /* Move pp0 into bit 8 (IBM 55) */ flags |= (newpp & HPTE_R_PP0) >> 55; + flags |= ((newpp & HPTE_R_KEY_HI) >> 48) | (newpp & HPTE_R_KEY_LO); + lpar_rc = plpar_pte_protect(flags, slot, 0); BUG_ON(lpar_rc != H_SUCCESS); diff --git a/arch/powerpc/platforms/pseries/pci_dlpar.c b/arch/powerpc/platforms/pseries/pci_dlpar.c index f9ae17e8a0f4..a8f9140a24fa 100644 --- a/arch/powerpc/platforms/pseries/pci_dlpar.c +++ b/arch/powerpc/platforms/pseries/pci_dlpar.c @@ -50,6 +50,7 @@ EXPORT_SYMBOL_GPL(init_phb_dynamic); int remove_phb_dynamic(struct pci_controller *phb) { struct pci_bus *b = phb->bus; + struct pci_host_bridge *host_bridge = to_pci_host_bridge(b->bridge); struct resource *res; int rc, i; @@ -76,7 +77,8 @@ int remove_phb_dynamic(struct pci_controller *phb) /* Remove the PCI bus and unregister the bridge device from sysfs */ phb->bus = NULL; pci_remove_bus(b); - device_unregister(b->bridge); + host_bridge->bus = NULL; + device_unregister(&host_bridge->dev); /* Now release the IO resource */ if (res->flags & IORESOURCE_IO) diff --git a/arch/powerpc/platforms/pseries/vio.c b/arch/powerpc/platforms/pseries/vio.c index 9cb4fc839fd5..429053d0402a 100644 --- a/arch/powerpc/platforms/pseries/vio.c +++ b/arch/powerpc/platforms/pseries/vio.c @@ -1285,6 +1285,10 @@ static int vio_bus_remove(struct device *dev) int __vio_register_driver(struct vio_driver *viodrv, struct module *owner, const char *mod_name) { + // vio_bus_type is only initialised for pseries + if (!machine_is(pseries)) + return -ENODEV; + pr_debug("%s: driver %s registering\n", __func__, viodrv->name); /* fill in 'struct driver' fields */ diff --git a/arch/powerpc/sysdev/xive/common.c b/arch/powerpc/sysdev/xive/common.c index 595310e056f4..31b657c37735 100644 --- a/arch/powerpc/sysdev/xive/common.c +++ b/arch/powerpc/sysdev/xive/common.c @@ -253,17 +253,20 @@ notrace void xmon_xive_do_dump(int cpu) xmon_printf("\n"); } +static struct irq_data *xive_get_irq_data(u32 hw_irq) +{ + unsigned int irq = irq_find_mapping(xive_irq_domain, hw_irq); + + return irq ? irq_get_irq_data(irq) : NULL; +} + int xmon_xive_get_irq_config(u32 hw_irq, struct irq_data *d) { - struct irq_chip *chip = irq_data_get_irq_chip(d); int rc; u32 target; u8 prio; u32 lirq; - if (!is_xive_irq(chip)) - return -EINVAL; - rc = xive_ops->get_irq_config(hw_irq, &target, &prio, &lirq); if (rc) { xmon_printf("IRQ 0x%08x : no config rc=%d\n", hw_irq, rc); @@ -273,6 +276,9 @@ int xmon_xive_get_irq_config(u32 hw_irq, struct irq_data *d) xmon_printf("IRQ 0x%08x : target=0x%x prio=%02x lirq=0x%x ", hw_irq, target, prio, lirq); + if (!d) + d = xive_get_irq_data(hw_irq); + if (d) { struct xive_irq_data *xd = irq_data_get_irq_handler_data(d); u64 val = xive_esb_read(xd, XIVE_ESB_GET); @@ -1335,17 +1341,14 @@ static int xive_prepare_cpu(unsigned int cpu) xc = per_cpu(xive_cpu, cpu); if (!xc) { - struct device_node *np; - xc = kzalloc_node(sizeof(struct xive_cpu), GFP_KERNEL, cpu_to_node(cpu)); if (!xc) return -ENOMEM; - np = of_get_cpu_node(cpu, NULL); - if (np) - xc->chip_id = of_get_ibm_chip_id(np); - of_node_put(np); xc->hw_ipi = XIVE_BAD_IRQ; + xc->chip_id = XIVE_INVALID_CHIP_ID; + if (xive_ops->prepare_cpu) + xive_ops->prepare_cpu(cpu, xc); per_cpu(xive_cpu, cpu) = xc; } @@ -1599,6 +1602,8 @@ static void xive_debug_show_irq(struct seq_file *m, u32 hw_irq, struct irq_data u32 target; u8 prio; u32 lirq; + struct xive_irq_data *xd; + u64 val; if (!is_xive_irq(chip)) return; @@ -1612,17 +1617,14 @@ static void xive_debug_show_irq(struct seq_file *m, u32 hw_irq, struct irq_data seq_printf(m, "IRQ 0x%08x : target=0x%x prio=%02x lirq=0x%x ", hw_irq, target, prio, lirq); - if (d) { - struct xive_irq_data *xd = irq_data_get_irq_handler_data(d); - u64 val = xive_esb_read(xd, XIVE_ESB_GET); - - seq_printf(m, "flags=%c%c%c PQ=%c%c", - xd->flags & XIVE_IRQ_FLAG_STORE_EOI ? 'S' : ' ', - xd->flags & XIVE_IRQ_FLAG_LSI ? 'L' : ' ', - xd->flags & XIVE_IRQ_FLAG_H_INT_ESB ? 'H' : ' ', - val & XIVE_ESB_VAL_P ? 'P' : '-', - val & XIVE_ESB_VAL_Q ? 'Q' : '-'); - } + xd = irq_data_get_irq_handler_data(d); + val = xive_esb_read(xd, XIVE_ESB_GET); + seq_printf(m, "flags=%c%c%c PQ=%c%c", + xd->flags & XIVE_IRQ_FLAG_STORE_EOI ? 'S' : ' ', + xd->flags & XIVE_IRQ_FLAG_LSI ? 'L' : ' ', + xd->flags & XIVE_IRQ_FLAG_H_INT_ESB ? 'H' : ' ', + val & XIVE_ESB_VAL_P ? 'P' : '-', + val & XIVE_ESB_VAL_Q ? 'Q' : '-'); seq_puts(m, "\n"); } diff --git a/arch/powerpc/sysdev/xive/native.c b/arch/powerpc/sysdev/xive/native.c index 05a800a3104e..57e3f1540435 100644 --- a/arch/powerpc/sysdev/xive/native.c +++ b/arch/powerpc/sysdev/xive/native.c @@ -380,6 +380,11 @@ static void xive_native_update_pending(struct xive_cpu *xc) } } +static void xive_native_prepare_cpu(unsigned int cpu, struct xive_cpu *xc) +{ + xc->chip_id = cpu_to_chip_id(cpu); +} + static void xive_native_setup_cpu(unsigned int cpu, struct xive_cpu *xc) { s64 rc; @@ -462,6 +467,7 @@ static const struct xive_ops xive_native_ops = { .match = xive_native_match, .shutdown = xive_native_shutdown, .update_pending = xive_native_update_pending, + .prepare_cpu = xive_native_prepare_cpu, .setup_cpu = xive_native_setup_cpu, .teardown_cpu = xive_native_teardown_cpu, .sync_source = xive_native_sync_source, diff --git a/arch/powerpc/sysdev/xive/xive-internal.h b/arch/powerpc/sysdev/xive/xive-internal.h index 9cf57c722faa..6478be19b4d3 100644 --- a/arch/powerpc/sysdev/xive/xive-internal.h +++ b/arch/powerpc/sysdev/xive/xive-internal.h @@ -46,6 +46,7 @@ struct xive_ops { u32 *sw_irq); int (*setup_queue)(unsigned int cpu, struct xive_cpu *xc, u8 prio); void (*cleanup_queue)(unsigned int cpu, struct xive_cpu *xc, u8 prio); + void (*prepare_cpu)(unsigned int cpu, struct xive_cpu *xc); void (*setup_cpu)(unsigned int cpu, struct xive_cpu *xc); void (*teardown_cpu)(unsigned int cpu, struct xive_cpu *xc); bool (*match)(struct device_node *np); diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 4515a10c5d22..d9522fc35ca5 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -227,7 +227,7 @@ config ARCH_RV64I bool "RV64I" select 64BIT select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && GCC_VERSION >= 50000 - select HAVE_DYNAMIC_FTRACE if MMU + select HAVE_DYNAMIC_FTRACE if MMU && $(cc-option,-fpatchable-function-entry=8) select HAVE_DYNAMIC_FTRACE_WITH_REGS if HAVE_DYNAMIC_FTRACE select HAVE_FTRACE_MCOUNT_RECORD select HAVE_FUNCTION_GRAPH_TRACER diff --git a/arch/riscv/include/asm/ftrace.h b/arch/riscv/include/asm/ftrace.h index 845002cc2e57..04dad3380041 100644 --- a/arch/riscv/include/asm/ftrace.h +++ b/arch/riscv/include/asm/ftrace.h @@ -13,9 +13,19 @@ #endif #define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR +/* + * Clang prior to 13 had "mcount" instead of "_mcount": + * https://reviews.llvm.org/D98881 + */ +#if defined(CONFIG_CC_IS_GCC) || CONFIG_CLANG_VERSION >= 130000 +#define MCOUNT_NAME _mcount +#else +#define MCOUNT_NAME mcount +#endif + #define ARCH_SUPPORTS_FTRACE_OPS 1 #ifndef __ASSEMBLY__ -void _mcount(void); +void MCOUNT_NAME(void); static inline unsigned long ftrace_call_adjust(unsigned long addr) { return addr; @@ -36,7 +46,7 @@ struct dyn_arch_ftrace { * both auipc and jalr at the same time. */ -#define MCOUNT_ADDR ((unsigned long)_mcount) +#define MCOUNT_ADDR ((unsigned long)MCOUNT_NAME) #define JALR_SIGN_MASK (0x00000800) #define JALR_OFFSET_MASK (0x00000fff) #define AUIPC_OFFSET_MASK (0xfffff000) diff --git a/arch/riscv/kernel/mcount.S b/arch/riscv/kernel/mcount.S index 8a5593ff9ff3..6d462681c9c0 100644 --- a/arch/riscv/kernel/mcount.S +++ b/arch/riscv/kernel/mcount.S @@ -47,8 +47,8 @@ ENTRY(ftrace_stub) #ifdef CONFIG_DYNAMIC_FTRACE - .global _mcount - .set _mcount, ftrace_stub + .global MCOUNT_NAME + .set MCOUNT_NAME, ftrace_stub #endif ret ENDPROC(ftrace_stub) @@ -78,7 +78,7 @@ ENDPROC(return_to_handler) #endif #ifndef CONFIG_DYNAMIC_FTRACE -ENTRY(_mcount) +ENTRY(MCOUNT_NAME) la t4, ftrace_stub #ifdef CONFIG_FUNCTION_GRAPH_TRACER la t0, ftrace_graph_return @@ -124,6 +124,6 @@ do_trace: jalr t5 RESTORE_ABI_STATE ret -ENDPROC(_mcount) +ENDPROC(MCOUNT_NAME) #endif -EXPORT_SYMBOL(_mcount) +EXPORT_SYMBOL(MCOUNT_NAME) diff --git a/arch/riscv/kernel/probes/kprobes.c b/arch/riscv/kernel/probes/kprobes.c index 7e2c78e2ca6b..d71f7c49a721 100644 --- a/arch/riscv/kernel/probes/kprobes.c +++ b/arch/riscv/kernel/probes/kprobes.c @@ -260,8 +260,10 @@ int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int trapnr) if (kcb->kprobe_status == KPROBE_REENTER) restore_previous_kprobe(kcb); - else + else { + kprobes_restore_local_irqflag(kcb, regs); reset_current_kprobe(); + } break; case KPROBE_HIT_ACTIVE: diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c index ea028d9e0d24..d44567490d91 100644 --- a/arch/riscv/kernel/smp.c +++ b/arch/riscv/kernel/smp.c @@ -54,7 +54,7 @@ int riscv_hartid_to_cpuid(int hartid) return i; pr_err("Couldn't find cpu id for hartid [%d]\n", hartid); - return i; + return -ENOENT; } void riscv_cpuid_to_hartid_mask(const struct cpumask *in, struct cpumask *out) diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile index 71a315e73cbe..ca2b40dfd24b 100644 --- a/arch/riscv/kernel/vdso/Makefile +++ b/arch/riscv/kernel/vdso/Makefile @@ -41,11 +41,10 @@ KASAN_SANITIZE := n $(obj)/vdso.o: $(obj)/vdso.so # link rule for the .so file, .lds has to be first -SYSCFLAGS_vdso.so.dbg = $(c_flags) $(obj)/vdso.so.dbg: $(src)/vdso.lds $(obj-vdso) FORCE $(call if_changed,vdsold) -SYSCFLAGS_vdso.so.dbg = -shared -s -Wl,-soname=linux-vdso.so.1 \ - -Wl,--build-id=sha1 -Wl,--hash-style=both +LDFLAGS_vdso.so.dbg = -shared -s -soname=linux-vdso.so.1 \ + --build-id=sha1 --hash-style=both --eh-frame-hdr # We also create a special relocatable object that should mirror the symbol # table and layout of the linked DSO. With ld --just-symbols we can then @@ -60,13 +59,10 @@ $(obj)/%.so: $(obj)/%.so.dbg FORCE # actual build commands # The DSO images are built using a special linker script -# Add -lgcc so rv32 gets static muldi3 and lshrdi3 definitions. # Make sure only to export the intended __vdso_xxx symbol offsets. quiet_cmd_vdsold = VDSOLD $@ - cmd_vdsold = $(CC) $(KBUILD_CFLAGS) $(call cc-option, -no-pie) -nostdlib -nostartfiles $(SYSCFLAGS_$(@F)) \ - -Wl,-T,$(filter-out FORCE,$^) -o $@.tmp && \ - $(CROSS_COMPILE)objcopy \ - $(patsubst %, -G __vdso_%, $(vdso-syms)) $@.tmp $@ && \ + cmd_vdsold = $(LD) $(ld_flags) -T $(filter-out FORCE,$^) -o $@.tmp && \ + $(OBJCOPY) $(patsubst %, -G __vdso_%, $(vdso-syms)) $@.tmp $@ && \ rm $@.tmp # Extracts symbol offsets from the VDSO, converting them into an assembly file diff --git a/arch/s390/crypto/arch_random.c b/arch/s390/crypto/arch_random.c index 7b947728d57e..56007c763902 100644 --- a/arch/s390/crypto/arch_random.c +++ b/arch/s390/crypto/arch_random.c @@ -54,6 +54,10 @@ static DECLARE_DELAYED_WORK(arch_rng_work, arch_rng_refill_buffer); bool s390_arch_random_generate(u8 *buf, unsigned int nbytes) { + /* max hunk is ARCH_RNG_BUF_SIZE */ + if (nbytes > ARCH_RNG_BUF_SIZE) + return false; + /* lock rng buffer */ if (!spin_trylock(&arch_rng_lock)) return false; diff --git a/arch/s390/include/asm/qdio.h b/arch/s390/include/asm/qdio.h index d9215c7106f0..8fc52679543d 100644 --- a/arch/s390/include/asm/qdio.h +++ b/arch/s390/include/asm/qdio.h @@ -246,21 +246,8 @@ struct slsb { u8 val[QDIO_MAX_BUFFERS_PER_Q]; } __attribute__ ((packed, aligned(256))); -/** - * struct qdio_outbuf_state - SBAL related asynchronous operation information - * (for communication with upper layer programs) - * (only required for use with completion queues) - * @user: pointer to upper layer program's state information related to SBAL - * (stored in user1 data of QAOB) - */ -struct qdio_outbuf_state { - void *user; -}; - -#define CHSC_AC1_INITIATE_INPUTQ 0x80 - - /* qdio adapter-characteristics-1 flag */ +#define CHSC_AC1_INITIATE_INPUTQ 0x80 #define AC1_SIGA_INPUT_NEEDED 0x40 /* process input queues */ #define AC1_SIGA_OUTPUT_NEEDED 0x20 /* process output queues */ #define AC1_SIGA_SYNC_NEEDED 0x10 /* ask hypervisor to sync */ @@ -338,7 +325,6 @@ typedef void qdio_handler_t(struct ccw_device *, unsigned int, int, * @int_parm: interruption parameter * @input_sbal_addr_array: per-queue array, each element points to 128 SBALs * @output_sbal_addr_array: per-queue array, each element points to 128 SBALs - * @output_sbal_state_array: no_output_qs * 128 state info (for CQ or NULL) */ struct qdio_initialize { unsigned char q_format; @@ -357,7 +343,6 @@ struct qdio_initialize { unsigned long int_parm; struct qdio_buffer ***input_sbal_addr_array; struct qdio_buffer ***output_sbal_addr_array; - struct qdio_outbuf_state *output_sbal_state_array; }; #define QDIO_STATE_INACTIVE 0x00000002 /* after qdio_cleanup */ @@ -378,9 +363,10 @@ extern int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs, extern int qdio_establish(struct ccw_device *cdev, struct qdio_initialize *init_data); extern int qdio_activate(struct ccw_device *); +extern struct qaob *qdio_allocate_aob(void); extern void qdio_release_aob(struct qaob *); -extern int do_QDIO(struct ccw_device *, unsigned int, int, unsigned int, - unsigned int); +extern int do_QDIO(struct ccw_device *cdev, unsigned int callflags, int q_nr, + unsigned int bufnr, unsigned int count, struct qaob *aob); extern int qdio_start_irq(struct ccw_device *cdev); extern int qdio_stop_irq(struct ccw_device *cdev); extern int qdio_get_next_buffers(struct ccw_device *, int, int *, int *); diff --git a/arch/s390/kernel/dis.c b/arch/s390/kernel/dis.c index a7eab7be4db0..5412efe328f8 100644 --- a/arch/s390/kernel/dis.c +++ b/arch/s390/kernel/dis.c @@ -563,7 +563,7 @@ void show_code(struct pt_regs *regs) void print_fn_code(unsigned char *code, unsigned long len) { - char buffer[64], *ptr; + char buffer[128], *ptr; int opsize, i; while (len) { diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 72134f9f6ff5..5aab59ad5688 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -937,9 +937,9 @@ static int __init setup_hwcaps(void) if (MACHINE_HAS_VX) { elf_hwcap |= HWCAP_S390_VXRS; if (test_facility(134)) - elf_hwcap |= HWCAP_S390_VXRS_EXT; - if (test_facility(135)) elf_hwcap |= HWCAP_S390_VXRS_BCD; + if (test_facility(135)) + elf_hwcap |= HWCAP_S390_VXRS_EXT; if (test_facility(148)) elf_hwcap |= HWCAP_S390_VXRS_EXT2; if (test_facility(152)) diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c index 6d6b57059493..b9f85b2dc053 100644 --- a/arch/s390/kvm/gaccess.c +++ b/arch/s390/kvm/gaccess.c @@ -976,7 +976,9 @@ int kvm_s390_check_low_addr_prot_real(struct kvm_vcpu *vcpu, unsigned long gra) * kvm_s390_shadow_tables - walk the guest page table and create shadow tables * @sg: pointer to the shadow guest address space structure * @saddr: faulting address in the shadow gmap - * @pgt: pointer to the page table address result + * @pgt: pointer to the beginning of the page table for the given address if + * successful (return value 0), or to the first invalid DAT entry in + * case of exceptions (return value > 0) * @fake: pgt references contiguous guest memory block, not a pgtable */ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr, @@ -1034,6 +1036,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr, rfte.val = ptr; goto shadow_r2t; } + *pgt = ptr + vaddr.rfx * 8; rc = gmap_read_table(parent, ptr + vaddr.rfx * 8, &rfte.val); if (rc) return rc; @@ -1060,6 +1063,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr, rste.val = ptr; goto shadow_r3t; } + *pgt = ptr + vaddr.rsx * 8; rc = gmap_read_table(parent, ptr + vaddr.rsx * 8, &rste.val); if (rc) return rc; @@ -1087,6 +1091,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr, rtte.val = ptr; goto shadow_sgt; } + *pgt = ptr + vaddr.rtx * 8; rc = gmap_read_table(parent, ptr + vaddr.rtx * 8, &rtte.val); if (rc) return rc; @@ -1123,6 +1128,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr, ste.val = ptr; goto shadow_pgt; } + *pgt = ptr + vaddr.sx * 8; rc = gmap_read_table(parent, ptr + vaddr.sx * 8, &ste.val); if (rc) return rc; @@ -1157,6 +1163,8 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr, * @vcpu: virtual cpu * @sg: pointer to the shadow guest address space structure * @saddr: faulting address in the shadow gmap + * @datptr: will contain the address of the faulting DAT table entry, or of + * the valid leaf, plus some flags * * Returns: - 0 if the shadow fault was successfully resolved * - > 0 (pgm exception code) on exceptions while faulting @@ -1165,11 +1173,11 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr, * - -ENOMEM if out of memory */ int kvm_s390_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg, - unsigned long saddr) + unsigned long saddr, unsigned long *datptr) { union vaddress vaddr; union page_table_entry pte; - unsigned long pgt; + unsigned long pgt = 0; int dat_protection, fake; int rc; @@ -1191,8 +1199,20 @@ int kvm_s390_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg, pte.val = pgt + vaddr.px * PAGE_SIZE; goto shadow_page; } - if (!rc) - rc = gmap_read_table(sg->parent, pgt + vaddr.px * 8, &pte.val); + + switch (rc) { + case PGM_SEGMENT_TRANSLATION: + case PGM_REGION_THIRD_TRANS: + case PGM_REGION_SECOND_TRANS: + case PGM_REGION_FIRST_TRANS: + pgt |= PEI_NOT_PTE; + break; + case 0: + pgt += vaddr.px * 8; + rc = gmap_read_table(sg->parent, pgt, &pte.val); + } + if (datptr) + *datptr = pgt | dat_protection * PEI_DAT_PROT; if (!rc && pte.i) rc = PGM_PAGE_TRANSLATION; if (!rc && pte.z) diff --git a/arch/s390/kvm/gaccess.h b/arch/s390/kvm/gaccess.h index f4c51756c462..7c72a5e3449f 100644 --- a/arch/s390/kvm/gaccess.h +++ b/arch/s390/kvm/gaccess.h @@ -18,17 +18,14 @@ /** * kvm_s390_real_to_abs - convert guest real address to guest absolute address - * @vcpu - guest virtual cpu + * @prefix - guest prefix * @gra - guest real address * * Returns the guest absolute address that corresponds to the passed guest real - * address @gra of a virtual guest cpu by applying its prefix. + * address @gra of by applying the given prefix. */ -static inline unsigned long kvm_s390_real_to_abs(struct kvm_vcpu *vcpu, - unsigned long gra) +static inline unsigned long _kvm_s390_real_to_abs(u32 prefix, unsigned long gra) { - unsigned long prefix = kvm_s390_get_prefix(vcpu); - if (gra < 2 * PAGE_SIZE) gra += prefix; else if (gra >= prefix && gra < prefix + 2 * PAGE_SIZE) @@ -36,6 +33,43 @@ static inline unsigned long kvm_s390_real_to_abs(struct kvm_vcpu *vcpu, return gra; } +/** + * kvm_s390_real_to_abs - convert guest real address to guest absolute address + * @vcpu - guest virtual cpu + * @gra - guest real address + * + * Returns the guest absolute address that corresponds to the passed guest real + * address @gra of a virtual guest cpu by applying its prefix. + */ +static inline unsigned long kvm_s390_real_to_abs(struct kvm_vcpu *vcpu, + unsigned long gra) +{ + return _kvm_s390_real_to_abs(kvm_s390_get_prefix(vcpu), gra); +} + +/** + * _kvm_s390_logical_to_effective - convert guest logical to effective address + * @psw: psw of the guest + * @ga: guest logical address + * + * Convert a guest logical address to an effective address by applying the + * rules of the addressing mode defined by bits 31 and 32 of the given PSW + * (extendended/basic addressing mode). + * + * Depending on the addressing mode, the upper 40 bits (24 bit addressing + * mode), 33 bits (31 bit addressing mode) or no bits (64 bit addressing + * mode) of @ga will be zeroed and the remaining bits will be returned. + */ +static inline unsigned long _kvm_s390_logical_to_effective(psw_t *psw, + unsigned long ga) +{ + if (psw_bits(*psw).eaba == PSW_BITS_AMODE_64BIT) + return ga; + if (psw_bits(*psw).eaba == PSW_BITS_AMODE_31BIT) + return ga & ((1UL << 31) - 1); + return ga & ((1UL << 24) - 1); +} + /** * kvm_s390_logical_to_effective - convert guest logical to effective address * @vcpu: guest virtual cpu @@ -52,13 +86,7 @@ static inline unsigned long kvm_s390_real_to_abs(struct kvm_vcpu *vcpu, static inline unsigned long kvm_s390_logical_to_effective(struct kvm_vcpu *vcpu, unsigned long ga) { - psw_t *psw = &vcpu->arch.sie_block->gpsw; - - if (psw_bits(*psw).eaba == PSW_BITS_AMODE_64BIT) - return ga; - if (psw_bits(*psw).eaba == PSW_BITS_AMODE_31BIT) - return ga & ((1UL << 31) - 1); - return ga & ((1UL << 24) - 1); + return _kvm_s390_logical_to_effective(&vcpu->arch.sie_block->gpsw, ga); } /* @@ -359,7 +387,11 @@ void ipte_unlock(struct kvm_vcpu *vcpu); int ipte_lock_held(struct kvm_vcpu *vcpu); int kvm_s390_check_low_addr_prot_real(struct kvm_vcpu *vcpu, unsigned long gra); +/* MVPG PEI indication bits */ +#define PEI_DAT_PROT 2 +#define PEI_NOT_PTE 4 + int kvm_s390_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *shadow, - unsigned long saddr); + unsigned long saddr, unsigned long *datptr); #endif /* __KVM_S390_GACCESS_H */ diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 2f09e9d7dc95..24ad447e648c 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -4307,16 +4307,16 @@ static void store_regs_fmt2(struct kvm_vcpu *vcpu) kvm_run->s.regs.bpbc = (vcpu->arch.sie_block->fpf & FPF_BPBC) == FPF_BPBC; kvm_run->s.regs.diag318 = vcpu->arch.diag318_info.val; if (MACHINE_HAS_GS) { + preempt_disable(); __ctl_set_bit(2, 4); if (vcpu->arch.gs_enabled) save_gs_cb(current->thread.gs_cb); - preempt_disable(); current->thread.gs_cb = vcpu->arch.host_gscb; restore_gs_cb(vcpu->arch.host_gscb); - preempt_enable(); if (!vcpu->arch.host_gscb) __ctl_clear_bit(2, 4); vcpu->arch.host_gscb = NULL; + preempt_enable(); } /* SIE will save etoken directly into SDNX and therefore kvm_run */ } diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c index bd803e091918..4002a24bc43a 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/vsie.c @@ -417,11 +417,6 @@ static void unshadow_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) memcpy((void *)((u64)scb_o + 0xc0), (void *)((u64)scb_s + 0xc0), 0xf0 - 0xc0); break; - case ICPT_PARTEXEC: - /* MVPG only */ - memcpy((void *)((u64)scb_o + 0xc0), - (void *)((u64)scb_s + 0xc0), 0xd0 - 0xc0); - break; } if (scb_s->ihcpu != 0xffffU) @@ -620,10 +615,10 @@ static int map_prefix(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) /* with mso/msl, the prefix lies at offset *mso* */ prefix += scb_s->mso; - rc = kvm_s390_shadow_fault(vcpu, vsie_page->gmap, prefix); + rc = kvm_s390_shadow_fault(vcpu, vsie_page->gmap, prefix, NULL); if (!rc && (scb_s->ecb & ECB_TE)) rc = kvm_s390_shadow_fault(vcpu, vsie_page->gmap, - prefix + PAGE_SIZE); + prefix + PAGE_SIZE, NULL); /* * We don't have to mprotect, we will be called for all unshadows. * SIE will detect if protection applies and trigger a validity. @@ -914,7 +909,7 @@ static int handle_fault(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) current->thread.gmap_addr, 1); rc = kvm_s390_shadow_fault(vcpu, vsie_page->gmap, - current->thread.gmap_addr); + current->thread.gmap_addr, NULL); if (rc > 0) { rc = inject_fault(vcpu, rc, current->thread.gmap_addr, @@ -936,7 +931,7 @@ static void handle_last_fault(struct kvm_vcpu *vcpu, { if (vsie_page->fault_addr) kvm_s390_shadow_fault(vcpu, vsie_page->gmap, - vsie_page->fault_addr); + vsie_page->fault_addr, NULL); vsie_page->fault_addr = 0; } @@ -983,6 +978,98 @@ static int handle_stfle(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) return 0; } +/* + * Get a register for a nested guest. + * @vcpu the vcpu of the guest + * @vsie_page the vsie_page for the nested guest + * @reg the register number, the upper 4 bits are ignored. + * returns: the value of the register. + */ +static u64 vsie_get_register(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, u8 reg) +{ + /* no need to validate the parameter and/or perform error handling */ + reg &= 0xf; + switch (reg) { + case 15: + return vsie_page->scb_s.gg15; + case 14: + return vsie_page->scb_s.gg14; + default: + return vcpu->run->s.regs.gprs[reg]; + } +} + +static int vsie_handle_mvpg(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) +{ + struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s; + unsigned long pei_dest, pei_src, src, dest, mask, prefix; + u64 *pei_block = &vsie_page->scb_o->mcic; + int edat, rc_dest, rc_src; + union ctlreg0 cr0; + + cr0.val = vcpu->arch.sie_block->gcr[0]; + edat = cr0.edat && test_kvm_facility(vcpu->kvm, 8); + mask = _kvm_s390_logical_to_effective(&scb_s->gpsw, PAGE_MASK); + prefix = scb_s->prefix << GUEST_PREFIX_SHIFT; + + dest = vsie_get_register(vcpu, vsie_page, scb_s->ipb >> 20) & mask; + dest = _kvm_s390_real_to_abs(prefix, dest) + scb_s->mso; + src = vsie_get_register(vcpu, vsie_page, scb_s->ipb >> 16) & mask; + src = _kvm_s390_real_to_abs(prefix, src) + scb_s->mso; + + rc_dest = kvm_s390_shadow_fault(vcpu, vsie_page->gmap, dest, &pei_dest); + rc_src = kvm_s390_shadow_fault(vcpu, vsie_page->gmap, src, &pei_src); + /* + * Either everything went well, or something non-critical went wrong + * e.g. because of a race. In either case, simply retry. + */ + if (rc_dest == -EAGAIN || rc_src == -EAGAIN || (!rc_dest && !rc_src)) { + retry_vsie_icpt(vsie_page); + return -EAGAIN; + } + /* Something more serious went wrong, propagate the error */ + if (rc_dest < 0) + return rc_dest; + if (rc_src < 0) + return rc_src; + + /* The only possible suppressing exception: just deliver it */ + if (rc_dest == PGM_TRANSLATION_SPEC || rc_src == PGM_TRANSLATION_SPEC) { + clear_vsie_icpt(vsie_page); + rc_dest = kvm_s390_inject_program_int(vcpu, PGM_TRANSLATION_SPEC); + WARN_ON_ONCE(rc_dest); + return 1; + } + + /* + * Forward the PEI intercept to the guest if it was a page fault, or + * also for segment and region table faults if EDAT applies. + */ + if (edat) { + rc_dest = rc_dest == PGM_ASCE_TYPE ? rc_dest : 0; + rc_src = rc_src == PGM_ASCE_TYPE ? rc_src : 0; + } else { + rc_dest = rc_dest != PGM_PAGE_TRANSLATION ? rc_dest : 0; + rc_src = rc_src != PGM_PAGE_TRANSLATION ? rc_src : 0; + } + if (!rc_dest && !rc_src) { + pei_block[0] = pei_dest; + pei_block[1] = pei_src; + return 1; + } + + retry_vsie_icpt(vsie_page); + + /* + * The host has edat, and the guest does not, or it was an ASCE type + * exception. The host needs to inject the appropriate DAT interrupts + * into the guest. + */ + if (rc_dest) + return inject_fault(vcpu, rc_dest, dest, 1); + return inject_fault(vcpu, rc_src, src, 0); +} + /* * Run the vsie on a shadow scb and a shadow gmap, without any further * sanity checks, handling SIE faults. @@ -1071,6 +1158,10 @@ static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) if ((scb_s->ipa & 0xf000) != 0xf000) scb_s->ipa += 0x1000; break; + case ICPT_PARTEXEC: + if (scb_s->ipa == 0xb254) + rc = vsie_handle_mvpg(vcpu, vsie_page); + break; } return rc; } diff --git a/arch/sh/kernel/traps.c b/arch/sh/kernel/traps.c index f5beecdac693..e76b22157099 100644 --- a/arch/sh/kernel/traps.c +++ b/arch/sh/kernel/traps.c @@ -180,7 +180,6 @@ static inline void arch_ftrace_nmi_exit(void) { } BUILD_TRAP_HANDLER(nmi) { - unsigned int cpu = smp_processor_id(); TRAP_HANDLER_DECL; arch_ftrace_nmi_enter(); diff --git a/arch/um/Kconfig.debug b/arch/um/Kconfig.debug index 315d368e63ad..1dfb2959c73b 100644 --- a/arch/um/Kconfig.debug +++ b/arch/um/Kconfig.debug @@ -17,6 +17,7 @@ config GCOV bool "Enable gcov support" depends on DEBUG_INFO depends on !KCOV + depends on !MODULES help This option allows developers to retrieve coverage data from a UML session. diff --git a/arch/um/kernel/Makefile b/arch/um/kernel/Makefile index 5aa882011e04..e698e0c7dbdc 100644 --- a/arch/um/kernel/Makefile +++ b/arch/um/kernel/Makefile @@ -21,7 +21,6 @@ obj-y = config.o exec.o exitcode.o irq.o ksyms.o mem.o \ obj-$(CONFIG_BLK_DEV_INITRD) += initrd.o obj-$(CONFIG_GPROF) += gprof_syms.o -obj-$(CONFIG_GCOV) += gmon_syms.o obj-$(CONFIG_EARLY_PRINTK) += early_printk.o obj-$(CONFIG_STACKTRACE) += stacktrace.o diff --git a/arch/um/kernel/dyn.lds.S b/arch/um/kernel/dyn.lds.S index dacbfabf66d8..2f2a8ce92f1e 100644 --- a/arch/um/kernel/dyn.lds.S +++ b/arch/um/kernel/dyn.lds.S @@ -6,6 +6,12 @@ OUTPUT_ARCH(ELF_ARCH) ENTRY(_start) jiffies = jiffies_64; +VERSION { + { + local: *; + }; +} + SECTIONS { PROVIDE (__executable_start = START); diff --git a/arch/um/kernel/gmon_syms.c b/arch/um/kernel/gmon_syms.c deleted file mode 100644 index 9361a8eb9bf1..000000000000 --- a/arch/um/kernel/gmon_syms.c +++ /dev/null @@ -1,16 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2001 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com) - */ - -#include - -extern void __bb_init_func(void *) __attribute__((weak)); -EXPORT_SYMBOL(__bb_init_func); - -extern void __gcov_init(void *) __attribute__((weak)); -EXPORT_SYMBOL(__gcov_init); -extern void __gcov_merge_add(void *, unsigned int) __attribute__((weak)); -EXPORT_SYMBOL(__gcov_merge_add); -extern void __gcov_exit(void) __attribute__((weak)); -EXPORT_SYMBOL(__gcov_exit); diff --git a/arch/um/kernel/uml.lds.S b/arch/um/kernel/uml.lds.S index 45d957d7004c..7a8e2b123e29 100644 --- a/arch/um/kernel/uml.lds.S +++ b/arch/um/kernel/uml.lds.S @@ -7,6 +7,12 @@ OUTPUT_ARCH(ELF_ARCH) ENTRY(_start) jiffies = jiffies_64; +VERSION { + { + local: *; + }; +} + SECTIONS { /* This must contain the right address - not quite the default ELF one.*/ diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 2792879d398e..861b1b794697 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -571,6 +571,7 @@ config X86_UV depends on X86_EXTENDED_PLATFORM depends on NUMA depends on EFI + depends on KEXEC_CORE depends on X86_X2APIC depends on PCI help @@ -1406,7 +1407,7 @@ config HIGHMEM4G config HIGHMEM64G bool "64GB" - depends on !M486 && !M586 && !M586TSC && !M586MMX && !MGEODE_LX && !MGEODEGX1 && !MCYRIXIII && !MELAN && !MWINCHIPC6 && !WINCHIP3D && !MK6 + depends on !M486SX && !M486 && !M586 && !M586TSC && !M586MMX && !MGEODE_LX && !MGEODEGX1 && !MCYRIXIII && !MELAN && !MWINCHIPC6 && !WINCHIP3D && !MK6 select X86_PAE help Select this if you have a 32-bit processor and more than 4 diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu index 814fe0d349b0..872b9cf598e3 100644 --- a/arch/x86/Kconfig.cpu +++ b/arch/x86/Kconfig.cpu @@ -157,7 +157,7 @@ config MPENTIUM4 config MK6 - bool "K6/K6-II/K6-III" + bool "AMD K6/K6-II/K6-III" depends on X86_32 help Select this for an AMD K6-family processor. Enables use of @@ -165,7 +165,7 @@ config MK6 flags to GCC. config MK7 - bool "Athlon/Duron/K7" + bool "AMD Athlon/Duron/K7" depends on X86_32 help Select this for an AMD Athlon K7-family processor. Enables use of @@ -173,12 +173,98 @@ config MK7 flags to GCC. config MK8 - bool "Opteron/Athlon64/Hammer/K8" + bool "AMD Opteron/Athlon64/Hammer/K8" help Select this for an AMD Opteron or Athlon64 Hammer-family processor. Enables use of some extended instructions, and passes appropriate optimization flags to GCC. +config MK8SSE3 + bool "AMD Opteron/Athlon64/Hammer/K8 with SSE3" + help + Select this for improved AMD Opteron or Athlon64 Hammer-family processors. + Enables use of some extended instructions, and passes appropriate + optimization flags to GCC. + +config MK10 + bool "AMD 61xx/7x50/PhenomX3/X4/II/K10" + help + Select this for an AMD 61xx Eight-Core Magny-Cours, Athlon X2 7x50, + Phenom X3/X4/II, Athlon II X2/X3/X4, or Turion II-family processor. + Enables use of some extended instructions, and passes appropriate + optimization flags to GCC. + +config MBARCELONA + bool "AMD Barcelona" + help + Select this for AMD Family 10h Barcelona processors. + + Enables -march=barcelona + +config MBOBCAT + bool "AMD Bobcat" + help + Select this for AMD Family 14h Bobcat processors. + + Enables -march=btver1 + +config MJAGUAR + bool "AMD Jaguar" + help + Select this for AMD Family 16h Jaguar processors. + + Enables -march=btver2 + +config MBULLDOZER + bool "AMD Bulldozer" + help + Select this for AMD Family 15h Bulldozer processors. + + Enables -march=bdver1 + +config MPILEDRIVER + bool "AMD Piledriver" + help + Select this for AMD Family 15h Piledriver processors. + + Enables -march=bdver2 + +config MSTEAMROLLER + bool "AMD Steamroller" + help + Select this for AMD Family 15h Steamroller processors. + + Enables -march=bdver3 + +config MEXCAVATOR + bool "AMD Excavator" + help + Select this for AMD Family 15h Excavator processors. + + Enables -march=bdver4 + +config MZEN + bool "AMD Zen" + help + Select this for AMD Family 17h Zen processors. + + Enables -march=znver1 + +config MZEN2 + bool "AMD Zen 2" + help + Select this for AMD Family 17h Zen 2 processors. + + Enables -march=znver2 + +config MZEN3 + bool "AMD Zen 3" + depends on GCC_VERSION > 100300 + help + Select this for AMD Family 19h Zen 3 processors. + + Enables -march=znver3 + config MCRUSOE bool "Crusoe" depends on X86_32 @@ -270,7 +356,7 @@ config MPSC in /proc/cpuinfo. Family 15 is an older Xeon, Family 6 a newer one. config MCORE2 - bool "Core 2/newer Xeon" + bool "Intel Core 2" help Select this for Intel Core 2 and newer Core 2 Xeons (Xeon 51xx and @@ -278,6 +364,8 @@ config MCORE2 family in /proc/cpuinfo. Newer ones have 6 and older ones 15 (not a typo) + Enables -march=core2 + config MATOM bool "Intel Atom" help @@ -287,6 +375,182 @@ config MATOM accordingly optimized code. Use a recent GCC with specific Atom support in order to fully benefit from selecting this option. +config MNEHALEM + bool "Intel Nehalem" + select X86_P6_NOP + help + + Select this for 1st Gen Core processors in the Nehalem family. + + Enables -march=nehalem + +config MWESTMERE + bool "Intel Westmere" + select X86_P6_NOP + help + + Select this for the Intel Westmere formerly Nehalem-C family. + + Enables -march=westmere + +config MSILVERMONT + bool "Intel Silvermont" + select X86_P6_NOP + help + + Select this for the Intel Silvermont platform. + + Enables -march=silvermont + +config MGOLDMONT + bool "Intel Goldmont" + select X86_P6_NOP + help + + Select this for the Intel Goldmont platform including Apollo Lake and Denverton. + + Enables -march=goldmont + +config MGOLDMONTPLUS + bool "Intel Goldmont Plus" + select X86_P6_NOP + help + + Select this for the Intel Goldmont Plus platform including Gemini Lake. + + Enables -march=goldmont-plus + +config MSANDYBRIDGE + bool "Intel Sandy Bridge" + select X86_P6_NOP + help + + Select this for 2nd Gen Core processors in the Sandy Bridge family. + + Enables -march=sandybridge + +config MIVYBRIDGE + bool "Intel Ivy Bridge" + select X86_P6_NOP + help + + Select this for 3rd Gen Core processors in the Ivy Bridge family. + + Enables -march=ivybridge + +config MHASWELL + bool "Intel Haswell" + select X86_P6_NOP + help + + Select this for 4th Gen Core processors in the Haswell family. + + Enables -march=haswell + +config MBROADWELL + bool "Intel Broadwell" + select X86_P6_NOP + help + + Select this for 5th Gen Core processors in the Broadwell family. + + Enables -march=broadwell + +config MSKYLAKE + bool "Intel Skylake" + select X86_P6_NOP + help + + Select this for 6th Gen Core processors in the Skylake family. + + Enables -march=skylake + +config MSKYLAKEX + bool "Intel Skylake X" + select X86_P6_NOP + help + + Select this for 6th Gen Core processors in the Skylake X family. + + Enables -march=skylake-avx512 + +config MCANNONLAKE + bool "Intel Cannon Lake" + select X86_P6_NOP + help + + Select this for 8th Gen Core processors + + Enables -march=cannonlake + +config MICELAKE + bool "Intel Ice Lake" + select X86_P6_NOP + help + + Select this for 10th Gen Core processors in the Ice Lake family. + + Enables -march=icelake-client + +config MCASCADELAKE + bool "Intel Cascade Lake" + select X86_P6_NOP + help + + Select this for Xeon processors in the Cascade Lake family. + + Enables -march=cascadelake + +config MCOOPERLAKE + bool "Intel Cooper Lake" + depends on GCC_VERSION > 100100 + select X86_P6_NOP + help + + Select this for Xeon processors in the Cooper Lake family. + + Enables -march=cooperlake + +config MTIGERLAKE + bool "Intel Tiger Lake" + depends on GCC_VERSION > 100100 + select X86_P6_NOP + help + + Select this for third-generation 10 nm process processors in the Tiger Lake family. + + Enables -march=tigerlake + +config MSAPPHIRERAPIDS + bool "Intel Sapphire Rapids" + depends on GCC_VERSION > 110000 + select X86_P6_NOP + help + + Select this for third-generation 10 nm process processors in the Sapphire Rapids family. + + Enables -march=sapphirerapids + +config MROCKETLAKE + bool "Intel Rocket Lake" + depends on GCC_VERSION > 110000 + select X86_P6_NOP + help + + Select this for eleventh-generation processors in the Rocket Lake family. + + Enables -march=rocketlake + +config MALDERLAKE + bool "Intel Alder Lake" + depends on GCC_VERSION > 110000 + select X86_P6_NOP + help + + Select this for twelfth-generation processors in the Alder Lake family. + + Enables -march=alderlake + config GENERIC_CPU bool "Generic-x86-64" depends on X86_64 @@ -294,6 +558,50 @@ config GENERIC_CPU Generic x86-64 CPU. Run equally well on all x86-64 CPUs. +config GENERIC_CPU2 + bool "Generic-x86-64-v2" + depends on GCC_VERSION > 110000 + depends on X86_64 + help + Generic x86-64 CPU. + Run equally well on all x86-64 CPUs with min support of x86-64-v2. + +config GENERIC_CPU3 + bool "Generic-x86-64-v3" + depends on GCC_VERSION > 110000 + depends on X86_64 + help + Generic x86-64-v3 CPU with v3 instructions. + Run equally well on all x86-64 CPUs with min support of x86-64-v3. + +config GENERIC_CPU4 + bool "Generic-x86-64-v4" + depends on GCC_VERSION > 110000 + depends on X86_64 + help + Generic x86-64 CPU with v4 instructions. + Run equally well on all x86-64 CPUs with min support of x86-64-v4. + +config MNATIVE_INTEL + bool "Intel-Native optimizations autodetected by GCC" + help + + GCC 4.2 and above support -march=native, which automatically detects + the optimum settings to use based on your processor. Do NOT use this + for AMD CPUs. Intel Only! + + Enables -march=native + +config MNATIVE_AMD + bool "AMD-Native optimizations autodetected by GCC" + help + + GCC 4.2 and above support -march=native, which automatically detects + the optimum settings to use based on your processor. Do NOT use this + for Intel CPUs. AMD Only! + + Enables -march=native + endchoice config X86_GENERIC @@ -318,7 +626,7 @@ config X86_INTERNODE_CACHE_SHIFT config X86_L1_CACHE_SHIFT int default "7" if MPENTIUM4 || MPSC - default "6" if MK7 || MK8 || MPENTIUMM || MCORE2 || MATOM || MVIAC7 || X86_GENERIC || GENERIC_CPU + default "6" if MK7 || MK8 || MPENTIUMM || MCORE2 || MATOM || MVIAC7 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD || X86_GENERIC || GENERIC_CPU || GENERIC_CPU2 || GENERIC_CPU3 || GENERIC_CPU4 default "4" if MELAN || M486SX || M486 || MGEODEGX1 default "5" if MWINCHIP3D || MWINCHIPC6 || MCRUSOE || MEFFICEON || MCYRIXIII || MK6 || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || M586 || MVIAC3_2 || MGEODE_LX @@ -336,11 +644,11 @@ config X86_ALIGNMENT_16 config X86_INTEL_USERCOPY def_bool y - depends on MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M586MMX || X86_GENERIC || MK8 || MK7 || MEFFICEON || MCORE2 + depends on MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M586MMX || X86_GENERIC || MK8 || MK7 || MEFFICEON || MCORE2 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL config X86_USE_PPRO_CHECKSUM def_bool y - depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MK8 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MCORE2 || MATOM + depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MK8 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MCORE2 || MATOM || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD config X86_USE_3DNOW def_bool y @@ -360,26 +668,26 @@ config X86_USE_3DNOW config X86_P6_NOP def_bool y depends on X86_64 - depends on (MCORE2 || MPENTIUM4 || MPSC) + depends on (MCORE2 || MPENTIUM4 || MPSC || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL) config X86_TSC def_bool y - depends on (MWINCHIP3D || MCRUSOE || MEFFICEON || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || MK8 || MVIAC3_2 || MVIAC7 || MGEODEGX1 || MGEODE_LX || MCORE2 || MATOM) || X86_64 + depends on (MWINCHIP3D || MCRUSOE || MEFFICEON || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || MK8 || MVIAC3_2 || MVIAC7 || MGEODEGX1 || MGEODE_LX || MCORE2 || MATOM || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD) || X86_64 config X86_CMPXCHG64 def_bool y - depends on X86_PAE || X86_64 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586TSC || M586MMX || MATOM || MGEODE_LX || MGEODEGX1 || MK6 || MK7 || MK8 + depends on X86_PAE || X86_64 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586TSC || M586MMX || MATOM || MGEODE_LX || MGEODEGX1 || MK6 || MK7 || MK8 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD # this should be set for all -march=.. options where the compiler # generates cmov. config X86_CMOV def_bool y - depends on (MK8 || MK7 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MCRUSOE || MEFFICEON || X86_64 || MATOM || MGEODE_LX) + depends on (MK8 || MK7 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MCRUSOE || MEFFICEON || X86_64 || MATOM || MGEODE_LX || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD) config X86_MINIMUM_CPU_FAMILY int default "64" if X86_64 - default "6" if X86_32 && (MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MATOM || MCRUSOE || MCORE2 || MK7 || MK8) + default "6" if X86_32 && (MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MATOM || MCRUSOE || MCORE2 || MK7 || MK8 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD) default "5" if X86_32 && X86_CMPXCHG64 default "4" diff --git a/arch/x86/Makefile b/arch/x86/Makefile index 9a85eae37b17..ee0cd507af8b 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -33,6 +33,7 @@ REALMODE_CFLAGS += -ffreestanding REALMODE_CFLAGS += -fno-stack-protector REALMODE_CFLAGS += $(call __cc-option, $(CC), $(REALMODE_CFLAGS), -Wno-address-of-packed-member) REALMODE_CFLAGS += $(call __cc-option, $(CC), $(REALMODE_CFLAGS), $(cc_stack_align4)) +REALMODE_CFLAGS += $(CLANG_FLAGS) export REALMODE_CFLAGS # BITS is used as extension for files which are available in a 32 bit @@ -113,11 +114,48 @@ else # FIXME - should be integrated in Makefile.cpu (Makefile_32.cpu) cflags-$(CONFIG_MK8) += $(call cc-option,-march=k8) cflags-$(CONFIG_MPSC) += $(call cc-option,-march=nocona) - - cflags-$(CONFIG_MCORE2) += \ - $(call cc-option,-march=core2,$(call cc-option,-mtune=generic)) - cflags-$(CONFIG_MATOM) += $(call cc-option,-march=atom) \ - $(call cc-option,-mtune=atom,$(call cc-option,-mtune=generic)) + cflags-$(CONFIG_MK8SSE3) += $(call cc-option,-march=k8-sse3) + cflags-$(CONFIG_MK10) += $(call cc-option,-march=amdfam10) + cflags-$(CONFIG_MBARCELONA) += $(call cc-option,-march=barcelona) + cflags-$(CONFIG_MBOBCAT) += $(call cc-option,-march=btver1) + cflags-$(CONFIG_MJAGUAR) += $(call cc-option,-march=btver2) + cflags-$(CONFIG_MBULLDOZER) += $(call cc-option,-march=bdver1) + cflags-$(CONFIG_MPILEDRIVER) += $(call cc-option,-march=bdver2) + cflags-$(CONFIG_MPILEDRIVER) += $(call cc-option,-mno-tbm) + cflags-$(CONFIG_MSTEAMROLLER) += $(call cc-option,-march=bdver3) + cflags-$(CONFIG_MSTEAMROLLER) += $(call cc-option,-mno-tbm) + cflags-$(CONFIG_MEXCAVATOR) += $(call cc-option,-march=bdver4) + cflags-$(CONFIG_MEXCAVATOR) += $(call cc-option,-mno-tbm) + cflags-$(CONFIG_MZEN) += $(call cc-option,-march=znver1) + cflags-$(CONFIG_MZEN2) += $(call cc-option,-march=znver2) + cflags-$(CONFIG_MZEN3) += $(call cc-option,-march=znver3) + + cflags-$(CONFIG_MNATIVE_INTEL) += $(call cc-option,-march=native) + cflags-$(CONFIG_MNATIVE_AMD) += $(call cc-option,-march=native) + cflags-$(CONFIG_MATOM) += $(call cc-option,-march=bonnell) + cflags-$(CONFIG_MCORE2) += $(call cc-option,-march=core2) + cflags-$(CONFIG_MNEHALEM) += $(call cc-option,-march=nehalem) + cflags-$(CONFIG_MWESTMERE) += $(call cc-option,-march=westmere) + cflags-$(CONFIG_MSILVERMONT) += $(call cc-option,-march=silvermont) + cflags-$(CONFIG_MGOLDMONT) += $(call cc-option,-march=goldmont) + cflags-$(CONFIG_MGOLDMONTPLUS) += $(call cc-option,-march=goldmont-plus) + cflags-$(CONFIG_MSANDYBRIDGE) += $(call cc-option,-march=sandybridge) + cflags-$(CONFIG_MIVYBRIDGE) += $(call cc-option,-march=ivybridge) + cflags-$(CONFIG_MHASWELL) += $(call cc-option,-march=haswell) + cflags-$(CONFIG_MBROADWELL) += $(call cc-option,-march=broadwell) + cflags-$(CONFIG_MSKYLAKE) += $(call cc-option,-march=skylake) + cflags-$(CONFIG_MSKYLAKEX) += $(call cc-option,-march=skylake-avx512) + cflags-$(CONFIG_MCANNONLAKE) += $(call cc-option,-march=cannonlake) + cflags-$(CONFIG_MICELAKE) += $(call cc-option,-march=icelake-client) + cflags-$(CONFIG_MCASCADELAKE) += $(call cc-option,-march=cascadelake) + cflags-$(CONFIG_MCOOPERLAKE) += $(call cc-option,-march=cooperlake) + cflags-$(CONFIG_MTIGERLAKE) += $(call cc-option,-march=tigerlake) + cflags-$(CONFIG_MSAPPHIRERAPIDS) += $(call cc-option,-march=sapphirerapids) + cflags-$(CONFIG_MROCKETLAKE) += $(call cc-option,-march=rocketlake) + cflags-$(CONFIG_MALDERLAKE) += $(call cc-option,-march=alderlake) + cflags-$(CONFIG_GENERIC_CPU2) += $(call cc-option,-march=x86-64-v2) + cflags-$(CONFIG_GENERIC_CPU3) += $(call cc-option,-march=x86-64-v3) + cflags-$(CONFIG_GENERIC_CPU4) += $(call cc-option,-march=x86-64-v4) cflags-$(CONFIG_GENERIC_CPU) += $(call cc-option,-mtune=generic) KBUILD_CFLAGS += $(cflags-y) diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index e0bc3988c3fa..6e5522aebbbd 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -46,6 +46,7 @@ KBUILD_CFLAGS += -D__DISABLE_EXPORTS # Disable relocation relaxation in case the link is not PIE. KBUILD_CFLAGS += $(call as-option,-Wa$(comma)-mrelax-relocations=no) KBUILD_CFLAGS += -include $(srctree)/include/linux/hidden.h +KBUILD_CFLAGS += $(CLANG_FLAGS) # sev-es.c indirectly inludes inat-table.h which is generated during # compilation and stored in $(objtree). Add the directory to the includes so diff --git a/arch/x86/boot/compressed/mem_encrypt.S b/arch/x86/boot/compressed/mem_encrypt.S index aa561795efd1..a6dea4e8a082 100644 --- a/arch/x86/boot/compressed/mem_encrypt.S +++ b/arch/x86/boot/compressed/mem_encrypt.S @@ -23,12 +23,6 @@ SYM_FUNC_START(get_sev_encryption_bit) push %ecx push %edx - /* Check if running under a hypervisor */ - movl $1, %eax - cpuid - bt $31, %ecx /* Check the hypervisor bit */ - jnc .Lno_sev - movl $0x80000000, %eax /* CPUID to check the highest leaf */ cpuid cmpl $0x8000001f, %eax /* See if 0x8000001f is available */ diff --git a/arch/x86/crypto/poly1305_glue.c b/arch/x86/crypto/poly1305_glue.c index 646da46e8d10..1dfb8af48a3c 100644 --- a/arch/x86/crypto/poly1305_glue.c +++ b/arch/x86/crypto/poly1305_glue.c @@ -16,7 +16,7 @@ #include asmlinkage void poly1305_init_x86_64(void *ctx, - const u8 key[POLY1305_KEY_SIZE]); + const u8 key[POLY1305_BLOCK_SIZE]); asmlinkage void poly1305_blocks_x86_64(void *ctx, const u8 *inp, const size_t len, const u32 padbit); asmlinkage void poly1305_emit_x86_64(void *ctx, u8 mac[POLY1305_DIGEST_SIZE], @@ -81,7 +81,7 @@ static void convert_to_base2_64(void *ctx) state->is_base2_26 = 0; } -static void poly1305_simd_init(void *ctx, const u8 key[POLY1305_KEY_SIZE]) +static void poly1305_simd_init(void *ctx, const u8 key[POLY1305_BLOCK_SIZE]) { poly1305_init_x86_64(ctx, key); } @@ -129,7 +129,7 @@ static void poly1305_simd_emit(void *ctx, u8 mac[POLY1305_DIGEST_SIZE], poly1305_emit_avx(ctx, mac, nonce); } -void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 *key) +void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) { poly1305_simd_init(&dctx->h, key); dctx->s[0] = get_unaligned_le32(&key[16]); diff --git a/arch/x86/entry/vdso/vdso2c.h b/arch/x86/entry/vdso/vdso2c.h index 1c7cfac7e64a..5264daa8859f 100644 --- a/arch/x86/entry/vdso/vdso2c.h +++ b/arch/x86/entry/vdso/vdso2c.h @@ -35,7 +35,7 @@ static void BITSFUNC(extract)(const unsigned char *data, size_t data_len, if (offset + len > data_len) fail("section to extract overruns input data"); - fprintf(outfile, "static const unsigned char %s[%lu] = {", name, len); + fprintf(outfile, "static const unsigned char %s[%zu] = {", name, len); BITSFUNC(copy)(outfile, data + offset, len); fprintf(outfile, "\n};\n\n"); } diff --git a/arch/x86/events/amd/iommu.c b/arch/x86/events/amd/iommu.c index be50ef8572cc..6a98a7651621 100644 --- a/arch/x86/events/amd/iommu.c +++ b/arch/x86/events/amd/iommu.c @@ -81,12 +81,12 @@ static struct attribute_group amd_iommu_events_group = { }; struct amd_iommu_event_desc { - struct kobj_attribute attr; + struct device_attribute attr; const char *event; }; -static ssize_t _iommu_event_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf) +static ssize_t _iommu_event_show(struct device *dev, + struct device_attribute *attr, char *buf) { struct amd_iommu_event_desc *event = container_of(attr, struct amd_iommu_event_desc, attr); diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c index 7f014d450bc2..582c0ffb5e98 100644 --- a/arch/x86/events/amd/uncore.c +++ b/arch/x86/events/amd/uncore.c @@ -275,14 +275,14 @@ static struct attribute_group amd_uncore_attr_group = { }; #define DEFINE_UNCORE_FORMAT_ATTR(_var, _name, _format) \ -static ssize_t __uncore_##_var##_show(struct kobject *kobj, \ - struct kobj_attribute *attr, \ +static ssize_t __uncore_##_var##_show(struct device *dev, \ + struct device_attribute *attr, \ char *page) \ { \ BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \ return sprintf(page, _format "\n"); \ } \ -static struct kobj_attribute format_attr_##_var = \ +static struct device_attribute format_attr_##_var = \ __ATTR(_name, 0444, __uncore_##_var##_show, NULL) DEFINE_UNCORE_FORMAT_ATTR(event12, event, "config:0-7,32-35"); diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h index 5eb3bdf36a41..06b0789d61b9 100644 --- a/arch/x86/include/asm/idtentry.h +++ b/arch/x86/include/asm/idtentry.h @@ -588,6 +588,21 @@ DECLARE_IDTENTRY_RAW(X86_TRAP_MC, xenpv_exc_machine_check); #endif /* NMI */ + +#if defined(CONFIG_X86_64) && IS_ENABLED(CONFIG_KVM_INTEL) +/* + * Special NOIST entry point for VMX which invokes this on the kernel + * stack. asm_exc_nmi() requires an IST to work correctly vs. the NMI + * 'executing' marker. + * + * On 32bit this just uses the regular NMI entry point because 32-bit does + * not have ISTs. + */ +DECLARE_IDTENTRY(X86_TRAP_NMI, exc_nmi_noist); +#else +#define asm_exc_nmi_noist asm_exc_nmi +#endif + DECLARE_IDTENTRY_NMI(X86_TRAP_NMI, exc_nmi); #ifdef CONFIG_XEN_PV DECLARE_IDTENTRY_RAW(X86_TRAP_NMI, xenpv_exc_nmi); diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 3768819693e5..eec2dcca2f39 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1753,6 +1753,7 @@ int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low, unsigned long icr, int op_64_bit); void kvm_define_user_return_msr(unsigned index, u32 msr); +int kvm_probe_user_return_msr(u32 msr); int kvm_set_user_return_msr(unsigned index, u64 val, u64 mask); u64 kvm_scale_tsc(struct kvm_vcpu *vcpu, u64 tsc); diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index f1b9ed5efaa9..908bcaea1361 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -804,8 +804,10 @@ DECLARE_PER_CPU(u64, msr_misc_features_shadow); #ifdef CONFIG_CPU_SUP_AMD extern u32 amd_get_nodes_per_socket(void); +extern u32 amd_get_highest_perf(void); #else static inline u32 amd_get_nodes_per_socket(void) { return 0; } +static inline u32 amd_get_highest_perf(void) { return 0; } #endif static inline uint32_t hypervisor_cpuid_base(const char *sig, uint32_t leaves) diff --git a/arch/x86/include/asm/vermagic.h b/arch/x86/include/asm/vermagic.h index 75884d2cdec3..4e6a08d4c7e5 100644 --- a/arch/x86/include/asm/vermagic.h +++ b/arch/x86/include/asm/vermagic.h @@ -17,6 +17,48 @@ #define MODULE_PROC_FAMILY "586MMX " #elif defined CONFIG_MCORE2 #define MODULE_PROC_FAMILY "CORE2 " +#elif defined CONFIG_MNATIVE_INTEL +#define MODULE_PROC_FAMILY "NATIVE_INTEL " +#elif defined CONFIG_MNATIVE_AMD +#define MODULE_PROC_FAMILY "NATIVE_AMD " +#elif defined CONFIG_MNEHALEM +#define MODULE_PROC_FAMILY "NEHALEM " +#elif defined CONFIG_MWESTMERE +#define MODULE_PROC_FAMILY "WESTMERE " +#elif defined CONFIG_MSILVERMONT +#define MODULE_PROC_FAMILY "SILVERMONT " +#elif defined CONFIG_MGOLDMONT +#define MODULE_PROC_FAMILY "GOLDMONT " +#elif defined CONFIG_MGOLDMONTPLUS +#define MODULE_PROC_FAMILY "GOLDMONTPLUS " +#elif defined CONFIG_MSANDYBRIDGE +#define MODULE_PROC_FAMILY "SANDYBRIDGE " +#elif defined CONFIG_MIVYBRIDGE +#define MODULE_PROC_FAMILY "IVYBRIDGE " +#elif defined CONFIG_MHASWELL +#define MODULE_PROC_FAMILY "HASWELL " +#elif defined CONFIG_MBROADWELL +#define MODULE_PROC_FAMILY "BROADWELL " +#elif defined CONFIG_MSKYLAKE +#define MODULE_PROC_FAMILY "SKYLAKE " +#elif defined CONFIG_MSKYLAKEX +#define MODULE_PROC_FAMILY "SKYLAKEX " +#elif defined CONFIG_MCANNONLAKE +#define MODULE_PROC_FAMILY "CANNONLAKE " +#elif defined CONFIG_MICELAKE +#define MODULE_PROC_FAMILY "ICELAKE " +#elif defined CONFIG_MCASCADELAKE +#define MODULE_PROC_FAMILY "CASCADELAKE " +#elif defined CONFIG_MCOOPERLAKE +#define MODULE_PROC_FAMILY "COOPERLAKE " +#elif defined CONFIG_MTIGERLAKE +#define MODULE_PROC_FAMILY "TIGERLAKE " +#elif defined CONFIG_MSAPPHIRERAPIDS +#define MODULE_PROC_FAMILY "SAPPHIRERAPIDS " +#elif defined CONFIG_ROCKETLAKE +#define MODULE_PROC_FAMILY "ROCKETLAKE " +#elif defined CONFIG_MALDERLAKE +#define MODULE_PROC_FAMILY "ALDERLAKE " #elif defined CONFIG_MATOM #define MODULE_PROC_FAMILY "ATOM " #elif defined CONFIG_M686 @@ -35,6 +77,30 @@ #define MODULE_PROC_FAMILY "K7 " #elif defined CONFIG_MK8 #define MODULE_PROC_FAMILY "K8 " +#elif defined CONFIG_MK8SSE3 +#define MODULE_PROC_FAMILY "K8SSE3 " +#elif defined CONFIG_MK10 +#define MODULE_PROC_FAMILY "K10 " +#elif defined CONFIG_MBARCELONA +#define MODULE_PROC_FAMILY "BARCELONA " +#elif defined CONFIG_MBOBCAT +#define MODULE_PROC_FAMILY "BOBCAT " +#elif defined CONFIG_MBULLDOZER +#define MODULE_PROC_FAMILY "BULLDOZER " +#elif defined CONFIG_MPILEDRIVER +#define MODULE_PROC_FAMILY "PILEDRIVER " +#elif defined CONFIG_MSTEAMROLLER +#define MODULE_PROC_FAMILY "STEAMROLLER " +#elif defined CONFIG_MJAGUAR +#define MODULE_PROC_FAMILY "JAGUAR " +#elif defined CONFIG_MEXCAVATOR +#define MODULE_PROC_FAMILY "EXCAVATOR " +#elif defined CONFIG_MZEN +#define MODULE_PROC_FAMILY "ZEN " +#elif defined CONFIG_MZEN2 +#define MODULE_PROC_FAMILY "ZEN2 " +#elif defined CONFIG_MZEN3 +#define MODULE_PROC_FAMILY "ZEN3 " #elif defined CONFIG_MELAN #define MODULE_PROC_FAMILY "ELAN " #elif defined CONFIG_MCRUSOE diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c index 52bc217ca8c3..c9ddd233e32f 100644 --- a/arch/x86/kernel/apic/x2apic_uv_x.c +++ b/arch/x86/kernel/apic/x2apic_uv_x.c @@ -1671,6 +1671,9 @@ static __init int uv_system_init_hubless(void) if (rc < 0) return rc; + /* Set section block size for current node memory */ + set_block_size(); + /* Create user access node */ if (rc >= 0) uv_setup_proc_files(1); diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 347a956f71ca..eedb2b320946 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -1170,3 +1170,19 @@ void set_dr_addr_mask(unsigned long mask, int dr) break; } } + +u32 amd_get_highest_perf(void) +{ + struct cpuinfo_x86 *c = &boot_cpu_data; + + if (c->x86 == 0x17 && ((c->x86_model >= 0x30 && c->x86_model < 0x40) || + (c->x86_model >= 0x70 && c->x86_model < 0x80))) + return 166; + + if (c->x86 == 0x19 && ((c->x86_model >= 0x20 && c->x86_model < 0x30) || + (c->x86_model >= 0x40 && c->x86_model < 0x70))) + return 166; + + return 255; +} +EXPORT_SYMBOL_GPL(amd_get_highest_perf); diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index ab640abe26b6..1e576cc831c1 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1850,7 +1850,7 @@ static inline void setup_getcpu(int cpu) unsigned long cpudata = vdso_encode_cpunode(cpu, early_cpu_to_node(cpu)); struct desc_struct d = { }; - if (boot_cpu_has(X86_FEATURE_RDTSCP)) + if (boot_cpu_has(X86_FEATURE_RDTSCP) || boot_cpu_has(X86_FEATURE_RDPID)) write_rdtscp_aux(cpudata); /* Store CPU and node number in limit. */ diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c index b935e1b5f115..6a6318e9590c 100644 --- a/arch/x86/kernel/cpu/microcode/core.c +++ b/arch/x86/kernel/cpu/microcode/core.c @@ -629,16 +629,16 @@ static ssize_t reload_store(struct device *dev, if (val != 1) return size; - tmp_ret = microcode_ops->request_microcode_fw(bsp, µcode_pdev->dev, true); - if (tmp_ret != UCODE_NEW) - return size; - get_online_cpus(); ret = check_online_cpus(); if (ret) goto put; + tmp_ret = microcode_ops->request_microcode_fw(bsp, µcode_pdev->dev, true); + if (tmp_ret != UCODE_NEW) + goto put; + mutex_lock(µcode_mutex); ret = microcode_reload_late(); mutex_unlock(µcode_mutex); diff --git a/arch/x86/kernel/e820.c b/arch/x86/kernel/e820.c index 22aad412f965..629c4994f165 100644 --- a/arch/x86/kernel/e820.c +++ b/arch/x86/kernel/e820.c @@ -31,8 +31,8 @@ * - inform the user about the firmware's notion of memory layout * via /sys/firmware/memmap * - * - the hibernation code uses it to generate a kernel-independent MD5 - * fingerprint of the physical memory layout of a system. + * - the hibernation code uses it to generate a kernel-independent CRC32 + * checksum of the physical memory layout of a system. * * - 'e820_table_kexec': a slightly modified (by the kernel) firmware version * passed to us by the bootloader - the major difference between diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c index df776cdca327..0bb9fe021bbe 100644 --- a/arch/x86/kernel/kprobes/core.c +++ b/arch/x86/kernel/kprobes/core.c @@ -139,6 +139,8 @@ NOKPROBE_SYMBOL(synthesize_relcall); int can_boost(struct insn *insn, void *addr) { kprobe_opcode_t opcode; + insn_byte_t prefix; + int i; if (search_exception_tables((unsigned long)addr)) return 0; /* Page fault may occur on this address. */ @@ -151,9 +153,14 @@ int can_boost(struct insn *insn, void *addr) if (insn->opcode.nbytes != 1) return 0; - /* Can't boost Address-size override prefix */ - if (unlikely(inat_is_address_size_prefix(insn->attr))) - return 0; + for_each_insn_prefix(insn, i, prefix) { + insn_attr_t attr; + + attr = inat_get_opcode_attribute(prefix); + /* Can't boost Address-size override prefix and CS override prefix */ + if (prefix == 0x2e || inat_is_address_size_prefix(attr)) + return 0; + } opcode = insn->opcode.bytes[0]; @@ -178,8 +185,8 @@ int can_boost(struct insn *insn, void *addr) /* clear and set flags are boostable */ return (opcode == 0xf5 || (0xf7 < opcode && opcode < 0xfe)); default: - /* CS override prefix and call are not boostable */ - return (opcode != 0x2e && opcode != 0x9a); + /* call is not boostable */ + return opcode != 0x9a; } } @@ -448,7 +455,11 @@ static void set_resume_flags(struct kprobe *p, struct insn *insn) break; #endif case 0xff: - opcode = insn->opcode.bytes[1]; + /* + * Since the 0xff is an extended group opcode, the instruction + * is determined by the MOD/RM byte. + */ + opcode = insn->modrm.bytes[0]; if ((opcode & 0x30) == 0x10) { /* * call absolute, indirect diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c index bf250a339655..2ef961cf4cfc 100644 --- a/arch/x86/kernel/nmi.c +++ b/arch/x86/kernel/nmi.c @@ -524,6 +524,16 @@ DEFINE_IDTENTRY_RAW(exc_nmi) mds_user_clear_cpu_buffers(); } +#if defined(CONFIG_X86_64) && IS_ENABLED(CONFIG_KVM_INTEL) +DEFINE_IDTENTRY_RAW(exc_nmi_noist) +{ + exc_nmi(regs); +} +#endif +#if IS_MODULE(CONFIG_KVM_INTEL) +EXPORT_SYMBOL_GPL(asm_exc_nmi_noist); +#endif + void stop_nmi(void) { ignore_nmis++; diff --git a/arch/x86/kernel/sev-es-shared.c b/arch/x86/kernel/sev-es-shared.c index cdc04d091242..387b71669818 100644 --- a/arch/x86/kernel/sev-es-shared.c +++ b/arch/x86/kernel/sev-es-shared.c @@ -186,7 +186,6 @@ void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code) * make it accessible to the hypervisor. * * In particular, check for: - * - Hypervisor CPUID bit * - Availability of CPUID leaf 0x8000001f * - SEV CPUID bit. * @@ -194,10 +193,7 @@ void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code) * can't be checked here. */ - if ((fn == 1 && !(regs->cx & BIT(31)))) - /* Hypervisor bit */ - goto fail; - else if (fn == 0x80000000 && (regs->ax < 0x8000001f)) + if (fn == 0x80000000 && (regs->ax < 0x8000001f)) /* SEV leaf check */ goto fail; else if ((fn == 0x8000001f && !(regs->ax & BIT(1)))) diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index 16703c35a944..363b36bbd791 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -458,29 +458,52 @@ static bool match_smt(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) return false; } +static bool match_die(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) +{ + if (c->phys_proc_id == o->phys_proc_id && + c->cpu_die_id == o->cpu_die_id) + return true; + return false; +} + /* - * Define snc_cpu[] for SNC (Sub-NUMA Cluster) CPUs. + * Unlike the other levels, we do not enforce keeping a + * multicore group inside a NUMA node. If this happens, we will + * discard the MC level of the topology later. + */ +static bool match_pkg(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) +{ + if (c->phys_proc_id == o->phys_proc_id) + return true; + return false; +} + +/* + * Define intel_cod_cpu[] for Intel COD (Cluster-on-Die) CPUs. * - * These are Intel CPUs that enumerate an LLC that is shared by - * multiple NUMA nodes. The LLC on these systems is shared for - * off-package data access but private to the NUMA node (half - * of the package) for on-package access. + * Any Intel CPU that has multiple nodes per package and does not + * match intel_cod_cpu[] has the SNC (Sub-NUMA Cluster) topology. * - * CPUID (the source of the information about the LLC) can only - * enumerate the cache as being shared *or* unshared, but not - * this particular configuration. The CPU in this case enumerates - * the cache to be shared across the entire package (spanning both - * NUMA nodes). + * When in SNC mode, these CPUs enumerate an LLC that is shared + * by multiple NUMA nodes. The LLC is shared for off-package data + * access but private to the NUMA node (half of the package) for + * on-package access. CPUID (the source of the information about + * the LLC) can only enumerate the cache as shared or unshared, + * but not this particular configuration. */ -static const struct x86_cpu_id snc_cpu[] = { - X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, NULL), +static const struct x86_cpu_id intel_cod_cpu[] = { + X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, 0), /* COD */ + X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, 0), /* COD */ + X86_MATCH_INTEL_FAM6_MODEL(ANY, 1), /* SNC */ {} }; static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) { + const struct x86_cpu_id *id = x86_match_cpu(intel_cod_cpu); int cpu1 = c->cpu_index, cpu2 = o->cpu_index; + bool intel_snc = id && id->driver_data; /* Do not match if we do not have a valid APICID for cpu: */ if (per_cpu(cpu_llc_id, cpu1) == BAD_APICID) @@ -495,32 +518,12 @@ static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) * means 'c' does not share the LLC of 'o'. This will be * reflected to userspace. */ - if (!topology_same_node(c, o) && x86_match_cpu(snc_cpu)) + if (match_pkg(c, o) && !topology_same_node(c, o) && intel_snc) return false; return topology_sane(c, o, "llc"); } -/* - * Unlike the other levels, we do not enforce keeping a - * multicore group inside a NUMA node. If this happens, we will - * discard the MC level of the topology later. - */ -static bool match_pkg(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) -{ - if (c->phys_proc_id == o->phys_proc_id) - return true; - return false; -} - -static bool match_die(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) -{ - if ((c->phys_proc_id == o->phys_proc_id) && - (c->cpu_die_id == o->cpu_die_id)) - return true; - return false; -} - #if defined(CONFIG_SCHED_SMT) || defined(CONFIG_SCHED_MC) static inline int x86_sched_itmt_flags(void) @@ -592,14 +595,23 @@ void set_cpu_sibling_map(int cpu) for_each_cpu(i, cpu_sibling_setup_mask) { o = &cpu_data(i); + if (match_pkg(c, o) && !topology_same_node(c, o)) + x86_has_numa_in_package = true; + if ((i == cpu) || (has_smt && match_smt(c, o))) link_mask(topology_sibling_cpumask, cpu, i); if ((i == cpu) || (has_mp && match_llc(c, o))) link_mask(cpu_llc_shared_mask, cpu, i); + if ((i == cpu) || (has_mp && match_die(c, o))) + link_mask(topology_die_cpumask, cpu, i); } + threads = cpumask_weight(topology_sibling_cpumask(cpu)); + if (threads > __max_smt_threads) + __max_smt_threads = threads; + /* * This needs a separate iteration over the cpus because we rely on all * topology_sibling_cpumask links to be set-up. @@ -613,8 +625,7 @@ void set_cpu_sibling_map(int cpu) /* * Does this new cpu bringup a new core? */ - if (cpumask_weight( - topology_sibling_cpumask(cpu)) == 1) { + if (threads == 1) { /* * for each core in package, increment * the booted_cores for this new cpu @@ -631,16 +642,7 @@ void set_cpu_sibling_map(int cpu) } else if (i != cpu && !c->booted_cores) c->booted_cores = cpu_data(i).booted_cores; } - if (match_pkg(c, o) && !topology_same_node(c, o)) - x86_has_numa_in_package = true; - - if ((i == cpu) || (has_mp && match_die(c, o))) - link_mask(topology_die_cpumask, cpu, i); } - - threads = cpumask_weight(topology_sibling_cpumask(cpu)); - if (threads > __max_smt_threads) - __max_smt_threads = threads; } /* maps the cpu to the sched domain representing multi-core */ @@ -2044,7 +2046,7 @@ static bool amd_set_max_freq_ratio(void) return false; } - highest_perf = perf_caps.highest_perf; + highest_perf = amd_get_highest_perf(); nominal_perf = perf_caps.nominal_perf; if (!highest_perf || !nominal_perf) { diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 6bd2f8b830e4..62f795352c02 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -589,7 +589,8 @@ static int __do_cpuid_func_emulated(struct kvm_cpuid_array *array, u32 func) case 7: entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX; entry->eax = 0; - entry->ecx = F(RDPID); + if (kvm_cpu_cap_has(X86_FEATURE_RDTSCP)) + entry->ecx = F(RDPID); ++array->nent; default: break; diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index f7970ba6219f..8fc71e70857d 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -4220,7 +4220,7 @@ static bool valid_cr(int nr) } } -static int check_cr_read(struct x86_emulate_ctxt *ctxt) +static int check_cr_access(struct x86_emulate_ctxt *ctxt) { if (!valid_cr(ctxt->modrm_reg)) return emulate_ud(ctxt); @@ -4228,80 +4228,6 @@ static int check_cr_read(struct x86_emulate_ctxt *ctxt) return X86EMUL_CONTINUE; } -static int check_cr_write(struct x86_emulate_ctxt *ctxt) -{ - u64 new_val = ctxt->src.val64; - int cr = ctxt->modrm_reg; - u64 efer = 0; - - static u64 cr_reserved_bits[] = { - 0xffffffff00000000ULL, - 0, 0, 0, /* CR3 checked later */ - CR4_RESERVED_BITS, - 0, 0, 0, - CR8_RESERVED_BITS, - }; - - if (!valid_cr(cr)) - return emulate_ud(ctxt); - - if (new_val & cr_reserved_bits[cr]) - return emulate_gp(ctxt, 0); - - switch (cr) { - case 0: { - u64 cr4; - if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) || - ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD))) - return emulate_gp(ctxt, 0); - - cr4 = ctxt->ops->get_cr(ctxt, 4); - ctxt->ops->get_msr(ctxt, MSR_EFER, &efer); - - if ((new_val & X86_CR0_PG) && (efer & EFER_LME) && - !(cr4 & X86_CR4_PAE)) - return emulate_gp(ctxt, 0); - - break; - } - case 3: { - u64 rsvd = 0; - - ctxt->ops->get_msr(ctxt, MSR_EFER, &efer); - if (efer & EFER_LMA) { - u64 maxphyaddr; - u32 eax, ebx, ecx, edx; - - eax = 0x80000008; - ecx = 0; - if (ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, - &edx, true)) - maxphyaddr = eax & 0xff; - else - maxphyaddr = 36; - rsvd = rsvd_bits(maxphyaddr, 63); - if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PCIDE) - rsvd &= ~X86_CR3_PCID_NOFLUSH; - } - - if (new_val & rsvd) - return emulate_gp(ctxt, 0); - - break; - } - case 4: { - ctxt->ops->get_msr(ctxt, MSR_EFER, &efer); - - if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE)) - return emulate_gp(ctxt, 0); - - break; - } - } - - return X86EMUL_CONTINUE; -} - static int check_dr7_gd(struct x86_emulate_ctxt *ctxt) { unsigned long dr7; @@ -4576,7 +4502,7 @@ static const struct opcode group8[] = { * from the register case of group9. */ static const struct gprefix pfx_0f_c7_7 = { - N, N, N, II(DstMem | ModRM | Op3264 | EmulateOnUD, em_rdpid, rdtscp), + N, N, N, II(DstMem | ModRM | Op3264 | EmulateOnUD, em_rdpid, rdpid), }; @@ -4841,10 +4767,10 @@ static const struct opcode twobyte_table[256] = { D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 8 * reserved NOP */ D(ImplicitOps | ModRM | SrcMem | NoAccess), /* NOP + 7 * reserved NOP */ /* 0x20 - 0x2F */ - DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read), + DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_access), DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read), IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write, - check_cr_write), + check_cr_access), IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write, check_dr_write), N, N, N, N, diff --git a/arch/x86/kvm/kvm_emulate.h b/arch/x86/kvm/kvm_emulate.h index 0d359115429a..f016838faedd 100644 --- a/arch/x86/kvm/kvm_emulate.h +++ b/arch/x86/kvm/kvm_emulate.h @@ -468,6 +468,7 @@ enum x86_intercept { x86_intercept_clgi, x86_intercept_skinit, x86_intercept_rdtscp, + x86_intercept_rdpid, x86_intercept_icebp, x86_intercept_wbinvd, x86_intercept_monitor, diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index cc369b9ad8f1..fa023f3feb25 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -296,6 +296,10 @@ static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val) atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); } + + /* Check if there are APF page ready requests pending */ + if (enabled) + kvm_make_request(KVM_REQ_APF_READY, apic->vcpu); } static inline void kvm_apic_set_xapic_id(struct kvm_lapic *apic, u8 id) @@ -1909,8 +1913,8 @@ void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu) if (!apic->lapic_timer.hv_timer_in_use) goto out; WARN_ON(rcuwait_active(&vcpu->wait)); - cancel_hv_timer(apic); apic_timer_expired(apic, false); + cancel_hv_timer(apic); if (apic_lvtt_period(apic) && apic->lapic_timer.period) { advance_periodic_target_expiration(apic); @@ -2261,6 +2265,8 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value) if (value & MSR_IA32_APICBASE_ENABLE) { kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); static_branch_slow_dec_deferred(&apic_hw_disabled); + /* Check if there are APF page ready requests pending */ + kvm_make_request(KVM_REQ_APF_READY, vcpu); } else { static_branch_inc(&apic_hw_disabled.key); atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 951dae4e7175..cd0faa187674 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3193,14 +3193,14 @@ void kvm_mmu_free_roots(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, if (mmu->shadow_root_level >= PT64_ROOT_4LEVEL && (mmu->root_level >= PT64_ROOT_4LEVEL || mmu->direct_map)) { mmu_free_root_page(kvm, &mmu->root_hpa, &invalid_list); - } else { + } else if (mmu->pae_root) { for (i = 0; i < 4; ++i) if (mmu->pae_root[i] != 0) mmu_free_root_page(kvm, &mmu->pae_root[i], &invalid_list); - mmu->root_hpa = INVALID_PAGE; } + mmu->root_hpa = INVALID_PAGE; mmu->root_pgd = 0; } @@ -3312,9 +3312,23 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) * the shadow page table may be a PAE or a long mode page table. */ pm_mask = PT_PRESENT_MASK; - if (vcpu->arch.mmu->shadow_root_level == PT64_ROOT_4LEVEL) + if (vcpu->arch.mmu->shadow_root_level == PT64_ROOT_4LEVEL) { pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK; + /* + * Allocate the page for the PDPTEs when shadowing 32-bit NPT + * with 64-bit only when needed. Unlike 32-bit NPT, it doesn't + * need to be in low mem. See also lm_root below. + */ + if (!vcpu->arch.mmu->pae_root) { + WARN_ON_ONCE(!tdp_enabled); + + vcpu->arch.mmu->pae_root = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT); + if (!vcpu->arch.mmu->pae_root) + return -ENOMEM; + } + } + for (i = 0; i < 4; ++i) { MMU_WARN_ON(VALID_PAGE(vcpu->arch.mmu->pae_root[i])); if (vcpu->arch.mmu->root_level == PT32E_ROOT_LEVEL) { @@ -3337,21 +3351,19 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) vcpu->arch.mmu->root_hpa = __pa(vcpu->arch.mmu->pae_root); /* - * If we shadow a 32 bit page table with a long mode page - * table we enter this path. + * When shadowing 32-bit or PAE NPT with 64-bit NPT, the PML4 and PDP + * tables are allocated and initialized at MMU creation as there is no + * equivalent level in the guest's NPT to shadow. Allocate the tables + * on demand, as running a 32-bit L1 VMM is very rare. The PDP is + * handled above (to share logic with PAE), deal with the PML4 here. */ if (vcpu->arch.mmu->shadow_root_level == PT64_ROOT_4LEVEL) { if (vcpu->arch.mmu->lm_root == NULL) { - /* - * The additional page necessary for this is only - * allocated on demand. - */ - u64 *lm_root; lm_root = (void*)get_zeroed_page(GFP_KERNEL_ACCOUNT); - if (lm_root == NULL) - return 1; + if (!lm_root) + return -ENOMEM; lm_root[0] = __pa(vcpu->arch.mmu->pae_root) | pm_mask; @@ -3653,6 +3665,14 @@ static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn, struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); bool async; + /* + * Retry the page fault if the gfn hit a memslot that is being deleted + * or moved. This ensures any existing SPTEs for the old memslot will + * be zapped before KVM inserts a new MMIO SPTE for the gfn. + */ + if (slot && (slot->flags & KVM_MEMSLOT_INVALID)) + return true; + /* Don't expose private memslots to L2. */ if (is_guest_mode(vcpu) && !kvm_is_visible_memslot(slot)) { *pfn = KVM_PFN_NOSLOT; @@ -4615,12 +4635,17 @@ void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, u32 cr0, u32 cr4, u32 efer, struct kvm_mmu *context = &vcpu->arch.guest_mmu; union kvm_mmu_role new_role = kvm_calc_shadow_npt_root_page_role(vcpu); - context->shadow_root_level = new_role.base.level; - __kvm_mmu_new_pgd(vcpu, nested_cr3, new_role.base, false, false); - if (new_role.as_u64 != context->mmu_role.as_u64) + if (new_role.as_u64 != context->mmu_role.as_u64) { shadow_mmu_init_context(vcpu, context, cr0, cr4, efer, new_role); + + /* + * Override the level set by the common init helper, nested TDP + * always uses the host's TDP configuration. + */ + context->shadow_root_level = new_role.base.level; + } } EXPORT_SYMBOL_GPL(kvm_init_shadow_npt_mmu); @@ -5240,9 +5265,11 @@ static int __kvm_mmu_create(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu) * while the PDP table is a per-vCPU construct that's allocated at MMU * creation. When emulating 32-bit mode, cr3 is only 32 bits even on * x86_64. Therefore we need to allocate the PDP table in the first - * 4GB of memory, which happens to fit the DMA32 zone. Except for - * SVM's 32-bit NPT support, TDP paging doesn't use PAE paging and can - * skip allocating the PDP table. + * 4GB of memory, which happens to fit the DMA32 zone. TDP paging + * generally doesn't use PAE paging and can skip allocating the PDP + * table. The main exception, handled here, is SVM's 32-bit NPT. The + * other exception is for shadowing L1's 32-bit or PAE NPT on 64-bit + * KVM; that horror is handled on-demand by mmu_alloc_shadow_roots(). */ if (tdp_enabled && kvm_mmu_get_tdp_level(vcpu) > PT32E_ROOT_LEVEL) return 0; diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 874ea309279f..dbc6214d69de 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -87,7 +87,7 @@ static bool __sev_recycle_asids(int min_asid, int max_asid) return true; } -static int sev_asid_new(struct kvm_sev_info *sev) +static int sev_asid_new(bool es_active) { int pos, min_asid, max_asid; bool retry = true; @@ -98,8 +98,8 @@ static int sev_asid_new(struct kvm_sev_info *sev) * SEV-enabled guests must use asid from min_sev_asid to max_sev_asid. * SEV-ES-enabled guest can use from 1 to min_sev_asid - 1. */ - min_asid = sev->es_active ? 0 : min_sev_asid - 1; - max_asid = sev->es_active ? min_sev_asid - 1 : max_sev_asid; + min_asid = es_active ? 0 : min_sev_asid - 1; + max_asid = es_active ? min_sev_asid - 1 : max_sev_asid; again: pos = find_next_zero_bit(sev_asid_bitmap, max_sev_asid, min_asid); if (pos >= max_asid) { @@ -179,13 +179,17 @@ static void sev_unbind_asid(struct kvm *kvm, unsigned int handle) static int sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp) { struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; + bool es_active = argp->id == KVM_SEV_ES_INIT; int asid, ret; + if (kvm->created_vcpus) + return -EINVAL; + ret = -EBUSY; if (unlikely(sev->active)) return ret; - asid = sev_asid_new(sev); + asid = sev_asid_new(es_active); if (asid < 0) return ret; @@ -194,6 +198,7 @@ static int sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp) goto e_free; sev->active = true; + sev->es_active = es_active; sev->asid = asid; INIT_LIST_HEAD(&sev->regions_list); @@ -204,16 +209,6 @@ static int sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp) return ret; } -static int sev_es_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp) -{ - if (!sev_es) - return -ENOTTY; - - to_kvm_svm(kvm)->sev_info.es_active = true; - - return sev_guest_init(kvm, argp); -} - static int sev_bind_asid(struct kvm *kvm, unsigned int handle, int *error) { struct sev_data_activate *data; @@ -564,6 +559,7 @@ static int sev_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp) { struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; struct sev_data_launch_update_vmsa *vmsa; + struct kvm_vcpu *vcpu; int i, ret; if (!sev_es_guest(kvm)) @@ -573,8 +569,8 @@ static int sev_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp) if (!vmsa) return -ENOMEM; - for (i = 0; i < kvm->created_vcpus; i++) { - struct vcpu_svm *svm = to_svm(kvm->vcpus[i]); + kvm_for_each_vcpu(i, vcpu, kvm) { + struct vcpu_svm *svm = to_svm(vcpu); /* Perform some pre-encryption checks against the VMSA */ ret = sev_es_sync_vmsa(svm); @@ -1127,12 +1123,15 @@ int svm_mem_enc_op(struct kvm *kvm, void __user *argp) mutex_lock(&kvm->lock); switch (sev_cmd.id) { + case KVM_SEV_ES_INIT: + if (!sev_es) { + r = -ENOTTY; + goto out; + } + fallthrough; case KVM_SEV_INIT: r = sev_guest_init(kvm, &sev_cmd); break; - case KVM_SEV_ES_INIT: - r = sev_es_guest_init(kvm, &sev_cmd); - break; case KVM_SEV_LAUNCH_START: r = sev_launch_start(kvm, &sev_cmd); break; @@ -1349,8 +1348,11 @@ void __init sev_hardware_setup(void) goto out; sev_reclaim_asid_bitmap = bitmap_zalloc(max_sev_asid, GFP_KERNEL); - if (!sev_reclaim_asid_bitmap) + if (!sev_reclaim_asid_bitmap) { + bitmap_free(sev_asid_bitmap); + sev_asid_bitmap = NULL; goto out; + } pr_info("SEV supported: %u ASIDs\n", max_sev_asid - min_sev_asid + 1); sev_supported = true; @@ -1666,7 +1668,7 @@ static int sev_es_validate_vmgexit(struct vcpu_svm *svm) return -EINVAL; } -static void pre_sev_es_run(struct vcpu_svm *svm) +void sev_es_unmap_ghcb(struct vcpu_svm *svm) { if (!svm->ghcb) return; @@ -1702,9 +1704,6 @@ void pre_sev_run(struct vcpu_svm *svm, int cpu) struct svm_cpu_data *sd = per_cpu(svm_data, cpu); int asid = sev_get_asid(svm->vcpu.kvm); - /* Perform any SEV-ES pre-run actions */ - pre_sev_es_run(svm); - /* Assign the asid allocated with this SEV guest */ svm->asid = asid; @@ -2104,5 +2103,8 @@ void sev_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector) * the guest will set the CS and RIP. Set SW_EXIT_INFO_2 to a * non-zero value. */ + if (!svm->ghcb) + return; + ghcb_set_sw_exit_info_2(svm->ghcb, 1); } diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 58a45bb139f8..48ee3deab64b 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -564,9 +564,8 @@ static int svm_cpu_init(int cpu) clear_page(page_address(sd->save_area)); if (svm_sev_enabled()) { - sd->sev_vmcbs = kmalloc_array(max_sev_asid + 1, - sizeof(void *), - GFP_KERNEL); + sd->sev_vmcbs = kcalloc(max_sev_asid + 1, sizeof(void *), + GFP_KERNEL); if (!sd->sev_vmcbs) goto free_save_area; } @@ -969,21 +968,6 @@ static __init int svm_hardware_setup(void) kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE); } - if (IS_ENABLED(CONFIG_KVM_AMD_SEV) && sev) { - sev_hardware_setup(); - } else { - sev = false; - sev_es = false; - } - - svm_adjust_mmio_mask(); - - for_each_possible_cpu(cpu) { - r = svm_cpu_init(cpu); - if (r) - goto err; - } - /* * KVM's MMU doesn't support using 2-level paging for itself, and thus * NPT isn't supported if the host is using 2-level paging since host @@ -998,6 +982,21 @@ static __init int svm_hardware_setup(void) kvm_configure_mmu(npt_enabled, get_max_npt_level(), PG_LEVEL_1G); pr_info("kvm: Nested Paging %sabled\n", npt_enabled ? "en" : "dis"); + if (IS_ENABLED(CONFIG_KVM_AMD_SEV) && sev && npt_enabled) { + sev_hardware_setup(); + } else { + sev = false; + sev_es = false; + } + + svm_adjust_mmio_mask(); + + for_each_possible_cpu(cpu) { + r = svm_cpu_init(cpu); + if (r) + goto err; + } + if (nrips) { if (!boot_cpu_has(X86_FEATURE_NRIPS)) nrips = false; @@ -1417,6 +1416,9 @@ static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu) struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu); unsigned int i; + if (sev_es_guest(vcpu->kvm)) + sev_es_unmap_ghcb(svm); + if (svm->guest_state_loaded) return; @@ -1898,7 +1900,7 @@ static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value) static int pf_interception(struct vcpu_svm *svm) { - u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2); + u64 fault_address = svm->vmcb->control.exit_info_2; u64 error_code = svm->vmcb->control.exit_info_1; return kvm_handle_page_fault(&svm->vcpu, error_code, fault_address, @@ -2738,6 +2740,10 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) case MSR_TSC_AUX: if (!boot_cpu_has(X86_FEATURE_RDTSCP)) return 1; + if (!msr_info->host_initiated && + !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) && + !guest_cpuid_has(vcpu, X86_FEATURE_RDPID)) + return 1; msr_info->data = svm->tsc_aux; break; /* @@ -2809,7 +2815,7 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) static int svm_complete_emulated_msr(struct kvm_vcpu *vcpu, int err) { struct vcpu_svm *svm = to_svm(vcpu); - if (!sev_es_guest(svm->vcpu.kvm) || !err) + if (!err || !sev_es_guest(vcpu->kvm) || WARN_ON_ONCE(!svm->ghcb)) return kvm_complete_insn_gp(&svm->vcpu, err); ghcb_set_sw_exit_info_1(svm->ghcb, 1); @@ -2946,6 +2952,11 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr) if (!boot_cpu_has(X86_FEATURE_RDTSCP)) return 1; + if (!msr->host_initiated && + !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) && + !guest_cpuid_has(vcpu, X86_FEATURE_RDPID)) + return 1; + /* * This is rare, so we update the MSR here instead of using * direct_access_msrs. Doing that would require a rdmsr in diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 39e071fdab0c..98da0b91f273 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -571,6 +571,7 @@ void sev_es_init_vmcb(struct vcpu_svm *svm); void sev_es_create_vcpu(struct vcpu_svm *svm); void sev_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector); void sev_es_prepare_guest_switch(struct vcpu_svm *svm, unsigned int cpu); +void sev_es_unmap_ghcb(struct vcpu_svm *svm); /* vmenter.S */ diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index bcca0b80e0d0..4ba2a43e188b 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -619,6 +619,7 @@ static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, } /* KVM unconditionally exposes the FS/GS base MSRs to L1. */ +#ifdef CONFIG_X86_64 nested_vmx_disable_intercept_for_msr(msr_bitmap_l1, msr_bitmap_l0, MSR_FS_BASE, MSR_TYPE_RW); @@ -627,6 +628,7 @@ static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, nested_vmx_disable_intercept_for_msr(msr_bitmap_l1, msr_bitmap_l0, MSR_KERNEL_GS_BASE, MSR_TYPE_RW); +#endif /* * Checking the L0->L1 bitmap is trying to verify two things: @@ -3098,15 +3100,8 @@ static bool nested_get_evmcs_page(struct kvm_vcpu *vcpu) nested_vmx_handle_enlightened_vmptrld(vcpu, false); if (evmptrld_status == EVMPTRLD_VMFAIL || - evmptrld_status == EVMPTRLD_ERROR) { - pr_debug_ratelimited("%s: enlightened vmptrld failed\n", - __func__); - vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; - vcpu->run->internal.suberror = - KVM_INTERNAL_ERROR_EMULATION; - vcpu->run->internal.ndata = 0; + evmptrld_status == EVMPTRLD_ERROR) return false; - } } return true; @@ -3194,8 +3189,16 @@ static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu) static bool vmx_get_nested_state_pages(struct kvm_vcpu *vcpu) { - if (!nested_get_evmcs_page(vcpu)) + if (!nested_get_evmcs_page(vcpu)) { + pr_debug_ratelimited("%s: enlightened vmptrld failed\n", + __func__); + vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + vcpu->run->internal.suberror = + KVM_INTERNAL_ERROR_EMULATION; + vcpu->run->internal.ndata = 0; + return false; + } if (is_guest_mode(vcpu) && !nested_get_vmcs12_pages(vcpu)) return false; @@ -4422,7 +4425,15 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, /* trying to cancel vmlaunch/vmresume is a bug */ WARN_ON_ONCE(vmx->nested.nested_run_pending); - kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); + if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) { + /* + * KVM_REQ_GET_NESTED_STATE_PAGES is also used to map + * Enlightened VMCS after migration and we still need to + * do that when something is forcing L2->L1 exit prior to + * the first L2 run. + */ + (void)nested_get_evmcs_page(vcpu); + } /* Service the TLB flush request for L2 before switching to L1. */ if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu)) @@ -4601,9 +4612,9 @@ int get_vmx_mem_address(struct kvm_vcpu *vcpu, unsigned long exit_qualification, else if (addr_size == 0) off = (gva_t)sign_extend64(off, 15); if (base_is_valid) - off += kvm_register_read(vcpu, base_reg); + off += kvm_register_readl(vcpu, base_reg); if (index_is_valid) - off += kvm_register_read(vcpu, index_reg) << scaling; + off += kvm_register_readl(vcpu, index_reg) << scaling; vmx_get_segment(vcpu, &s, seg_reg); /* @@ -5479,16 +5490,11 @@ static int nested_vmx_eptp_switching(struct kvm_vcpu *vcpu, if (!nested_vmx_check_eptp(vcpu, new_eptp)) return 1; - kvm_mmu_unload(vcpu); mmu->ept_ad = accessed_dirty; mmu->mmu_role.base.ad_disabled = !accessed_dirty; vmcs12->ept_pointer = new_eptp; - /* - * TODO: Check what's the correct approach in case - * mmu reload fails. Currently, we just let the next - * reload potentially fail - */ - kvm_mmu_reload(vcpu); + + kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu); } return 0; @@ -5717,7 +5723,7 @@ static bool nested_vmx_exit_handled_vmcs_access(struct kvm_vcpu *vcpu, /* Decode instruction info and find the field to access */ vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); - field = kvm_register_read(vcpu, (((vmx_instruction_info) >> 28) & 0xf)); + field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf)); /* Out-of-range fields always cause a VM exit from L2 to L1 */ if (field >> 15) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 29b40e092d13..f68ed9a1abcc 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -156,9 +157,11 @@ static u32 vmx_possible_passthrough_msrs[MAX_POSSIBLE_PASSTHROUGH_MSRS] = { MSR_IA32_SPEC_CTRL, MSR_IA32_PRED_CMD, MSR_IA32_TSC, +#ifdef CONFIG_X86_64 MSR_FS_BASE, MSR_GS_BASE, MSR_KERNEL_GS_BASE, +#endif MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP, @@ -1731,7 +1734,8 @@ static void setup_msrs(struct vcpu_vmx *vmx) if (update_transition_efer(vmx)) vmx_setup_uret_msr(vmx, MSR_EFER); - if (guest_cpuid_has(&vmx->vcpu, X86_FEATURE_RDTSCP)) + if (guest_cpuid_has(&vmx->vcpu, X86_FEATURE_RDTSCP) || + guest_cpuid_has(&vmx->vcpu, X86_FEATURE_RDPID)) vmx_setup_uret_msr(vmx, MSR_TSC_AUX); vmx_setup_uret_msr(vmx, MSR_IA32_TSX_CTRL); @@ -1930,7 +1934,8 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) break; case MSR_TSC_AUX: if (!msr_info->host_initiated && - !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP)) + !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) && + !guest_cpuid_has(vcpu, X86_FEATURE_RDPID)) return 1; goto find_uret_msr; case MSR_IA32_DEBUGCTLMSR: @@ -2227,7 +2232,8 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) break; case MSR_TSC_AUX: if (!msr_info->host_initiated && - !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP)) + !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) && + !guest_cpuid_has(vcpu, X86_FEATURE_RDPID)) return 1; /* Check reserved bit, higher 32 bits should be zero */ if ((data >> 32) != 0) @@ -4299,7 +4305,23 @@ static void vmx_compute_secondary_exec_control(struct vcpu_vmx *vmx) xsaves_enabled, false); } - vmx_adjust_sec_exec_feature(vmx, &exec_control, rdtscp, RDTSCP); + /* + * RDPID is also gated by ENABLE_RDTSCP, turn on the control if either + * feature is exposed to the guest. This creates a virtualization hole + * if both are supported in hardware but only one is exposed to the + * guest, but letting the guest execute RDTSCP or RDPID when either one + * is advertised is preferable to emulating the advertised instruction + * in KVM on #UD, and obviously better than incorrectly injecting #UD. + */ + if (cpu_has_vmx_rdtscp()) { + bool rdpid_or_rdtscp_enabled = + guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) || + guest_cpuid_has(vcpu, X86_FEATURE_RDPID); + + vmx_adjust_secondary_exec_control(vmx, &exec_control, + SECONDARY_EXEC_ENABLE_RDTSCP, + rdpid_or_rdtscp_enabled, false); + } vmx_adjust_sec_exec_feature(vmx, &exec_control, invpcid, INVPCID); vmx_adjust_sec_exec_exiting(vmx, &exec_control, rdrand, RDRAND); @@ -5062,12 +5084,12 @@ static int handle_cr(struct kvm_vcpu *vcpu) case 3: WARN_ON_ONCE(enable_unrestricted_guest); val = kvm_read_cr3(vcpu); - kvm_register_write(vcpu, reg, val); + kvm_register_writel(vcpu, reg, val); trace_kvm_cr_read(cr, val); return kvm_skip_emulated_instruction(vcpu); case 8: val = kvm_get_cr8(vcpu); - kvm_register_write(vcpu, reg, val); + kvm_register_writel(vcpu, reg, val); trace_kvm_cr_read(cr, val); return kvm_skip_emulated_instruction(vcpu); } @@ -5140,7 +5162,7 @@ static int handle_dr(struct kvm_vcpu *vcpu) unsigned long val; kvm_get_dr(vcpu, dr, &val); - kvm_register_write(vcpu, reg, val); + kvm_register_writel(vcpu, reg, val); err = 0; } else { err = kvm_set_dr(vcpu, dr, kvm_register_readl(vcpu, reg)); @@ -5792,7 +5814,6 @@ void dump_vmcs(void) u32 vmentry_ctl, vmexit_ctl; u32 cpu_based_exec_ctrl, pin_based_exec_ctrl, secondary_exec_control; unsigned long cr4; - u64 efer; if (!dump_invalid_vmcs) { pr_warn_ratelimited("set kvm_intel.dump_invalid_vmcs=1 to dump internal KVM state.\n"); @@ -5804,7 +5825,6 @@ void dump_vmcs(void) cpu_based_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL); pin_based_exec_ctrl = vmcs_read32(PIN_BASED_VM_EXEC_CONTROL); cr4 = vmcs_readl(GUEST_CR4); - efer = vmcs_read64(GUEST_IA32_EFER); secondary_exec_control = 0; if (cpu_has_secondary_exec_ctrls()) secondary_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL); @@ -5816,9 +5836,7 @@ void dump_vmcs(void) pr_err("CR4: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n", cr4, vmcs_readl(CR4_READ_SHADOW), vmcs_readl(CR4_GUEST_HOST_MASK)); pr_err("CR3 = 0x%016lx\n", vmcs_readl(GUEST_CR3)); - if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT) && - (cr4 & X86_CR4_PAE) && !(efer & EFER_LMA)) - { + if (cpu_has_vmx_ept()) { pr_err("PDPTR0 = 0x%016llx PDPTR1 = 0x%016llx\n", vmcs_read64(GUEST_PDPTR0), vmcs_read64(GUEST_PDPTR1)); pr_err("PDPTR2 = 0x%016llx PDPTR3 = 0x%016llx\n", @@ -5844,7 +5862,8 @@ void dump_vmcs(void) if ((vmexit_ctl & (VM_EXIT_SAVE_IA32_PAT | VM_EXIT_SAVE_IA32_EFER)) || (vmentry_ctl & (VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_IA32_EFER))) pr_err("EFER = 0x%016llx PAT = 0x%016llx\n", - efer, vmcs_read64(GUEST_IA32_PAT)); + vmcs_read64(GUEST_IA32_EFER), + vmcs_read64(GUEST_IA32_PAT)); pr_err("DebugCtl = 0x%016llx DebugExceptions = 0x%016lx\n", vmcs_read64(GUEST_IA32_DEBUGCTL), vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS)); @@ -6395,18 +6414,17 @@ static void vmx_apicv_post_state_restore(struct kvm_vcpu *vcpu) void vmx_do_interrupt_nmi_irqoff(unsigned long entry); -static void handle_interrupt_nmi_irqoff(struct kvm_vcpu *vcpu, u32 intr_info) +static void handle_interrupt_nmi_irqoff(struct kvm_vcpu *vcpu, + unsigned long entry) { - unsigned int vector = intr_info & INTR_INFO_VECTOR_MASK; - gate_desc *desc = (gate_desc *)host_idt_base + vector; - kvm_before_interrupt(vcpu); - vmx_do_interrupt_nmi_irqoff(gate_offset(desc)); + vmx_do_interrupt_nmi_irqoff(entry); kvm_after_interrupt(vcpu); } static void handle_exception_nmi_irqoff(struct vcpu_vmx *vmx) { + const unsigned long nmi_entry = (unsigned long)asm_exc_nmi_noist; u32 intr_info = vmx_get_intr_info(&vmx->vcpu); /* if exit due to PF check for async PF */ @@ -6417,18 +6435,20 @@ static void handle_exception_nmi_irqoff(struct vcpu_vmx *vmx) kvm_machine_check(); /* We need to handle NMIs before interrupts are enabled */ else if (is_nmi(intr_info)) - handle_interrupt_nmi_irqoff(&vmx->vcpu, intr_info); + handle_interrupt_nmi_irqoff(&vmx->vcpu, nmi_entry); } static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu) { u32 intr_info = vmx_get_intr_info(vcpu); + unsigned int vector = intr_info & INTR_INFO_VECTOR_MASK; + gate_desc *desc = (gate_desc *)host_idt_base + vector; if (WARN_ONCE(!is_external_intr(intr_info), "KVM: unexpected VM-Exit interrupt info: 0x%x", intr_info)) return; - handle_interrupt_nmi_irqoff(vcpu, intr_info); + handle_interrupt_nmi_irqoff(vcpu, gate_offset(desc)); } static void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu) @@ -6894,12 +6914,9 @@ static int vmx_create_vcpu(struct kvm_vcpu *vcpu) for (i = 0; i < ARRAY_SIZE(vmx_uret_msrs_list); ++i) { u32 index = vmx_uret_msrs_list[i]; - u32 data_low, data_high; int j = vmx->nr_uret_msrs; - if (rdmsr_safe(index, &data_low, &data_high) < 0) - continue; - if (wrmsr_safe(index, data_low, data_high) < 0) + if (kvm_probe_user_return_msr(index)) continue; vmx->guest_uret_msrs[j].slot = i; @@ -6938,9 +6955,11 @@ static int vmx_create_vcpu(struct kvm_vcpu *vcpu) bitmap_fill(vmx->shadow_msr_intercept.write, MAX_POSSIBLE_PASSTHROUGH_MSRS); vmx_disable_intercept_for_msr(vcpu, MSR_IA32_TSC, MSR_TYPE_R); +#ifdef CONFIG_X86_64 vmx_disable_intercept_for_msr(vcpu, MSR_FS_BASE, MSR_TYPE_RW); vmx_disable_intercept_for_msr(vcpu, MSR_GS_BASE, MSR_TYPE_RW); vmx_disable_intercept_for_msr(vcpu, MSR_KERNEL_GS_BASE, MSR_TYPE_RW); +#endif vmx_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_CS, MSR_TYPE_RW); vmx_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_ESP, MSR_TYPE_RW); vmx_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_EIP, MSR_TYPE_RW); @@ -7330,9 +7349,11 @@ static __init void vmx_set_cpu_caps(void) if (!cpu_has_vmx_xsaves()) kvm_cpu_cap_clear(X86_FEATURE_XSAVES); - /* CPUID 0x80000001 */ - if (!cpu_has_vmx_rdtscp()) + /* CPUID 0x80000001 and 0x7 (RDPID) */ + if (!cpu_has_vmx_rdtscp()) { kvm_cpu_cap_clear(X86_FEATURE_RDTSCP); + kvm_cpu_cap_clear(X86_FEATURE_RDPID); + } if (cpu_has_vmx_waitpkg()) kvm_cpu_cap_check_and_set(X86_FEATURE_WAITPKG); @@ -7388,8 +7409,9 @@ static int vmx_check_intercept(struct kvm_vcpu *vcpu, /* * RDPID causes #UD if disabled through secondary execution controls. * Because it is marked as EmulateOnUD, we need to intercept it here. + * Note, RDPID is hidden behind ENABLE_RDTSCP. */ - case x86_intercept_rdtscp: + case x86_intercept_rdpid: if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_RDTSCP)) { exception->vector = UD_VECTOR; exception->error_code_valid = false; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index ee0dc58ac3a5..87311d39f914 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -335,6 +335,22 @@ static void kvm_on_user_return(struct user_return_notifier *urn) } } +int kvm_probe_user_return_msr(u32 msr) +{ + u64 val; + int ret; + + preempt_disable(); + ret = rdmsrl_safe(msr, &val); + if (ret) + goto out; + ret = wrmsrl_safe(msr, val); +out: + preempt_enable(); + return ret; +} +EXPORT_SYMBOL_GPL(kvm_probe_user_return_msr); + void kvm_define_user_return_msr(unsigned slot, u32 msr) { BUG_ON(slot >= KVM_MAX_NR_USER_RETURN_MSRS); @@ -1072,10 +1088,15 @@ int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) return 0; } - if (is_long_mode(vcpu) && kvm_vcpu_is_illegal_gpa(vcpu, cr3)) + /* + * Do not condition the GPA check on long mode, this helper is used to + * stuff CR3, e.g. for RSM emulation, and there is no guarantee that + * the current vCPU mode is accurate. + */ + if (kvm_vcpu_is_illegal_gpa(vcpu, cr3)) return 1; - else if (is_pae_paging(vcpu) && - !load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3)) + + if (is_pae_paging(vcpu) && !load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3)) return 1; kvm_mmu_new_pgd(vcpu, cr3, skip_tlb_flush, skip_tlb_flush); @@ -5859,7 +5880,8 @@ static void kvm_init_msr_list(void) continue; break; case MSR_TSC_AUX: - if (!kvm_cpu_cap_has(X86_FEATURE_RDTSCP)) + if (!kvm_cpu_cap_has(X86_FEATURE_RDTSCP) && + !kvm_cpu_cap_has(X86_FEATURE_RDPID)) continue; break; case MSR_IA32_UMWAIT_CONTROL: @@ -7959,6 +7981,18 @@ static void pvclock_gtod_update_fn(struct work_struct *work) static DECLARE_WORK(pvclock_gtod_work, pvclock_gtod_update_fn); +/* + * Indirection to move queue_work() out of the tk_core.seq write held + * region to prevent possible deadlocks against time accessors which + * are invoked with work related locks held. + */ +static void pvclock_irq_work_fn(struct irq_work *w) +{ + queue_work(system_long_wq, &pvclock_gtod_work); +} + +static DEFINE_IRQ_WORK(pvclock_irq_work, pvclock_irq_work_fn); + /* * Notification about pvclock gtod data update. */ @@ -7970,13 +8004,14 @@ static int pvclock_gtod_notify(struct notifier_block *nb, unsigned long unused, update_pvclock_gtod(tk); - /* disable master clock if host does not trust, or does not - * use, TSC based clocksource. + /* + * Disable master clock if host does not trust, or does not use, + * TSC based clocksource. Delegate queue_work() to irq_work as + * this is invoked with tk_core.seq write held. */ if (!gtod_is_based_on_tsc(gtod->clock.vclock_mode) && atomic_read(&kvm_guest_has_master_clock) != 0) - queue_work(system_long_wq, &pvclock_gtod_work); - + irq_work_queue(&pvclock_irq_work); return 0; } @@ -8091,6 +8126,8 @@ void kvm_arch_exit(void) cpuhp_remove_state_nocalls(CPUHP_AP_X86_KVM_CLK_ONLINE); #ifdef CONFIG_X86_64 pvclock_gtod_unregister_notifier(&pvclock_gtod_notifier); + irq_work_sync(&pvclock_irq_work); + cancel_work_sync(&pvclock_gtod_work); #endif kvm_x86_ops.hardware_enable = NULL; kvm_mmu_module_exit(); @@ -11020,6 +11057,9 @@ bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu) bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) { + if (vcpu->arch.guest_state_protected) + return true; + return vcpu->arch.preempted_in_kernel; } @@ -11290,7 +11330,7 @@ bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu) if (!kvm_pv_async_pf_enabled(vcpu)) return true; else - return apf_pageready_slot_free(vcpu); + return kvm_lapic_enabled(vcpu) && apf_pageready_slot_free(vcpu); } void kvm_arch_start_assignment(struct kvm *kvm) @@ -11539,7 +11579,7 @@ int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva) fallthrough; case INVPCID_TYPE_ALL_INCL_GLOBAL: - kvm_mmu_unload(vcpu); + kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu); return kvm_skip_emulated_instruction(vcpu); default: diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index ae17250e1efe..7f27bb65a572 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -673,7 +673,7 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu) bool longmode; u64 input, params[6]; - input = (u64)kvm_register_read(vcpu, VCPU_REGS_RAX); + input = (u64)kvm_register_readl(vcpu, VCPU_REGS_RAX); /* Hyper-V hypercalls get bit 31 set in EAX */ if ((input & 0x80000000) && diff --git a/arch/x86/lib/msr-smp.c b/arch/x86/lib/msr-smp.c index 75a0915b0d01..40bbe56bde32 100644 --- a/arch/x86/lib/msr-smp.c +++ b/arch/x86/lib/msr-smp.c @@ -252,7 +252,7 @@ static void __wrmsr_safe_regs_on_cpu(void *info) rv->err = wrmsr_safe_regs(rv->regs); } -int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 *regs) +int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]) { int err; struct msr_regs_info rv; @@ -265,7 +265,7 @@ int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 *regs) } EXPORT_SYMBOL(rdmsr_safe_regs_on_cpu); -int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 *regs) +int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]) { int err; struct msr_regs_info rv; diff --git a/arch/x86/mm/mem_encrypt_identity.c b/arch/x86/mm/mem_encrypt_identity.c index 6c5eb6f3f14f..a19374d26101 100644 --- a/arch/x86/mm/mem_encrypt_identity.c +++ b/arch/x86/mm/mem_encrypt_identity.c @@ -503,14 +503,10 @@ void __init sme_enable(struct boot_params *bp) #define AMD_SME_BIT BIT(0) #define AMD_SEV_BIT BIT(1) - /* - * Set the feature mask (SME or SEV) based on whether we are - * running under a hypervisor. - */ - eax = 1; - ecx = 0; - native_cpuid(&eax, &ebx, &ecx, &edx); - feature_mask = (ecx & BIT(31)) ? AMD_SEV_BIT : AMD_SME_BIT; + + /* Check the SEV MSR whether SEV or SME is enabled */ + sev_status = __rdmsr(MSR_AMD64_SEV); + feature_mask = (sev_status & MSR_AMD64_SEV_ENABLED) ? AMD_SEV_BIT : AMD_SME_BIT; /* * Check for the SME/SEV feature: @@ -530,19 +526,26 @@ void __init sme_enable(struct boot_params *bp) /* Check if memory encryption is enabled */ if (feature_mask == AMD_SME_BIT) { + /* + * No SME if Hypervisor bit is set. This check is here to + * prevent a guest from trying to enable SME. For running as a + * KVM guest the MSR_K8_SYSCFG will be sufficient, but there + * might be other hypervisors which emulate that MSR as non-zero + * or even pass it through to the guest. + * A malicious hypervisor can still trick a guest into this + * path, but there is no way to protect against that. + */ + eax = 1; + ecx = 0; + native_cpuid(&eax, &ebx, &ecx, &edx); + if (ecx & BIT(31)) + return; + /* For SME, check the SYSCFG MSR */ msr = __rdmsr(MSR_K8_SYSCFG); if (!(msr & MSR_K8_SYSCFG_MEM_ENCRYPT)) return; } else { - /* For SEV, check the SEV MSR */ - msr = __rdmsr(MSR_AMD64_SEV); - if (!(msr & MSR_AMD64_SEV_ENABLED)) - return; - - /* Save SEV_STATUS to avoid reading MSR again */ - sev_status = msr; - /* SEV state cannot be controlled by a command line option */ sme_me_mask = me_mask; sev_enabled = true; diff --git a/arch/x86/power/hibernate.c b/arch/x86/power/hibernate.c index cd3914fc9f3d..e94e0050a583 100644 --- a/arch/x86/power/hibernate.c +++ b/arch/x86/power/hibernate.c @@ -13,8 +13,8 @@ #include #include #include - -#include +#include +#include #include #include @@ -54,95 +54,33 @@ int pfn_is_nosave(unsigned long pfn) return pfn >= nosave_begin_pfn && pfn < nosave_end_pfn; } - -#define MD5_DIGEST_SIZE 16 - struct restore_data_record { unsigned long jump_address; unsigned long jump_address_phys; unsigned long cr3; unsigned long magic; - u8 e820_digest[MD5_DIGEST_SIZE]; + unsigned long e820_checksum; }; -#if IS_BUILTIN(CONFIG_CRYPTO_MD5) /** - * get_e820_md5 - calculate md5 according to given e820 table + * compute_e820_crc32 - calculate crc32 of a given e820 table * * @table: the e820 table to be calculated - * @buf: the md5 result to be stored to + * + * Return: the resulting checksum */ -static int get_e820_md5(struct e820_table *table, void *buf) +static inline u32 compute_e820_crc32(struct e820_table *table) { - struct crypto_shash *tfm; - struct shash_desc *desc; - int size; - int ret = 0; - - tfm = crypto_alloc_shash("md5", 0, 0); - if (IS_ERR(tfm)) - return -ENOMEM; - - desc = kmalloc(sizeof(struct shash_desc) + crypto_shash_descsize(tfm), - GFP_KERNEL); - if (!desc) { - ret = -ENOMEM; - goto free_tfm; - } - - desc->tfm = tfm; - - size = offsetof(struct e820_table, entries) + + int size = offsetof(struct e820_table, entries) + sizeof(struct e820_entry) * table->nr_entries; - if (crypto_shash_digest(desc, (u8 *)table, size, buf)) - ret = -EINVAL; - - kfree_sensitive(desc); - -free_tfm: - crypto_free_shash(tfm); - return ret; -} - -static int hibernation_e820_save(void *buf) -{ - return get_e820_md5(e820_table_firmware, buf); -} - -static bool hibernation_e820_mismatch(void *buf) -{ - int ret; - u8 result[MD5_DIGEST_SIZE]; - - memset(result, 0, MD5_DIGEST_SIZE); - /* If there is no digest in suspend kernel, let it go. */ - if (!memcmp(result, buf, MD5_DIGEST_SIZE)) - return false; - - ret = get_e820_md5(e820_table_firmware, result); - if (ret) - return true; - - return memcmp(result, buf, MD5_DIGEST_SIZE) ? true : false; -} -#else -static int hibernation_e820_save(void *buf) -{ - return 0; -} - -static bool hibernation_e820_mismatch(void *buf) -{ - /* If md5 is not builtin for restore kernel, let it go. */ - return false; + return ~crc32_le(~0, (unsigned char const *)table, size); } -#endif #ifdef CONFIG_X86_64 -#define RESTORE_MAGIC 0x23456789ABCDEF01UL +#define RESTORE_MAGIC 0x23456789ABCDEF02UL #else -#define RESTORE_MAGIC 0x12345678UL +#define RESTORE_MAGIC 0x12345679UL #endif /** @@ -179,7 +117,8 @@ int arch_hibernation_header_save(void *addr, unsigned int max_size) */ rdr->cr3 = restore_cr3 & ~CR3_PCID_MASK; - return hibernation_e820_save(rdr->e820_digest); + rdr->e820_checksum = compute_e820_crc32(e820_table_firmware); + return 0; } /** @@ -200,7 +139,7 @@ int arch_hibernation_header_restore(void *addr) jump_address_phys = rdr->jump_address_phys; restore_cr3 = rdr->cr3; - if (hibernation_e820_mismatch(rdr->e820_digest)) { + if (rdr->e820_checksum != compute_e820_crc32(e820_table_firmware)) { pr_crit("Hibernate inconsistent memory map detected!\n"); return -ENODEV; } diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c index b791e2041e49..ede8a0f0e07c 100644 --- a/block/bfq-cgroup.c +++ b/block/bfq-cgroup.c @@ -224,7 +224,7 @@ void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq, { blkg_rwstat_add(&bfqg->stats.queued, op, 1); bfqg_stats_end_empty_time(&bfqg->stats); - if (!(bfqq == ((struct bfq_data *)bfqg->bfqd)->in_service_queue)) + if (!(bfqq == bfqg->bfqd->in_service_queue)) bfqg_stats_set_start_group_wait_time(bfqg, bfqq_group(bfqq)); } @@ -309,8 +309,7 @@ struct bfq_group *bfqq_group(struct bfq_queue *bfqq) { struct bfq_entity *group_entity = bfqq->entity.parent; - return group_entity ? container_of(group_entity, struct bfq_group, - entity) : + return group_entity ? bfq_entity_to_bfqg(group_entity) : bfqq->bfqd->root_group; } @@ -427,6 +426,7 @@ void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg) entity->weight = entity->new_weight; entity->orig_weight = entity->new_weight; + entity->prio_changed = 0; if (bfqq) { bfqq->ioprio = bfqq->new_ioprio; bfqq->ioprio_class = bfqq->new_ioprio_class; @@ -547,6 +547,8 @@ static void bfq_pd_init(struct blkg_policy_data *pd) entity->orig_weight = entity->weight = entity->new_weight = d->weight; entity->my_sched_data = &bfqg->sched_data; + entity->last_bfqq_created = NULL; + bfqg->my_entity = entity; /* * the root_group's will be set to NULL * in bfq_init_queue() @@ -610,8 +612,7 @@ struct bfq_group *bfq_find_set_group(struct bfq_data *bfqd, */ entity = &bfqg->entity; for_each_entity(entity) { - struct bfq_group *curr_bfqg = container_of(entity, - struct bfq_group, entity); + struct bfq_group *curr_bfqg = bfq_entity_to_bfqg(entity); if (curr_bfqg != bfqd->root_group) { parent = bfqg_parent(curr_bfqg); if (!parent) @@ -1431,15 +1432,11 @@ void bfqg_and_blkg_put(struct bfq_group *bfqg) {} struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node) { struct bfq_group *bfqg; - int i; bfqg = kmalloc_node(sizeof(*bfqg), GFP_KERNEL | __GFP_ZERO, node); if (!bfqg) return NULL; - for (i = 0; i < BFQ_IOPRIO_CLASSES; i++) - bfqg->sched_data.service_tree[i] = BFQ_SERVICE_TREE_INIT; - return bfqg; } #endif /* CONFIG_BFQ_GROUP_IOSCHED */ diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c index 95586137194e..8281c5b63c5f 100644 --- a/block/bfq-iosched.c +++ b/block/bfq-iosched.c @@ -372,9 +372,38 @@ struct bfq_queue *bic_to_bfqq(struct bfq_io_cq *bic, bool is_sync) return bic->bfqq[is_sync]; } +static void bfq_put_stable_ref(struct bfq_queue *bfqq); + void bic_set_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq, bool is_sync) { + /* + * If bfqq != NULL, then a non-stable queue merge between + * bic->bfqq and bfqq is happening here. This causes troubles + * in the following case: bic->bfqq has also been scheduled + * for a possible stable merge with bic->stable_merge_bfqq, + * and bic->stable_merge_bfqq == bfqq happens to + * hold. Troubles occur because bfqq may then undergo a split, + * thereby becoming eligible for a stable merge. Yet, if + * bic->stable_merge_bfqq points exactly to bfqq, then bfqq + * would be stably merged with itself. To avoid this anomaly, + * we cancel the stable merge if + * bic->stable_merge_bfqq == bfqq. + */ bic->bfqq[is_sync] = bfqq; + + if (bfqq && bic->stable_merge_bfqq == bfqq) { + /* + * Actually, these same instructions are executed also + * in bfq_setup_cooperator, in case of abort or actual + * execution of a stable merge. We could avoid + * repeating these instructions there too, but if we + * did so, we would nest even more complexity in this + * function. + */ + bfq_put_stable_ref(bic->stable_merge_bfqq); + + bic->stable_merge_bfqq = NULL; + } } struct bfq_data *bic_to_bfqd(struct bfq_io_cq *bic) @@ -1012,7 +1041,7 @@ static void bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_data *bfqd, struct bfq_io_cq *bic, bool bfq_already_existing) { - unsigned int old_wr_coeff = bfqq->wr_coeff; + unsigned int old_wr_coeff = 1; bool busy = bfq_already_existing && bfq_bfqq_busy(bfqq); if (bic->saved_has_short_ttime) @@ -1033,7 +1062,13 @@ bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_data *bfqd, bfqq->ttime = bic->saved_ttime; bfqq->io_start_time = bic->saved_io_start_time; bfqq->tot_idle_time = bic->saved_tot_idle_time; - bfqq->wr_coeff = bic->saved_wr_coeff; + /* + * Restore weight coefficient only if low_latency is on + */ + if (bfqd->low_latency) { + old_wr_coeff = bfqq->wr_coeff; + bfqq->wr_coeff = bic->saved_wr_coeff; + } bfqq->service_from_wr = bic->saved_service_from_wr; bfqq->wr_start_at_switch_to_srt = bic->saved_wr_start_at_switch_to_srt; bfqq->last_wr_start_finish = bic->saved_last_wr_start_finish; @@ -1069,7 +1104,7 @@ bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_data *bfqd, static int bfqq_process_refs(struct bfq_queue *bfqq) { return bfqq->ref - bfqq->allocated - bfqq->entity.on_st_or_in_serv - - (bfqq->weight_counter != NULL); + (bfqq->weight_counter != NULL) - bfqq->stable_ref; } /* Empty burst list and add just bfqq (see comments on bfq_handle_burst) */ @@ -2257,10 +2292,9 @@ static void bfq_remove_request(struct request_queue *q, } -static bool bfq_bio_merge(struct blk_mq_hw_ctx *hctx, struct bio *bio, +static bool bfq_bio_merge(struct request_queue *q, struct bio *bio, unsigned int nr_segs) { - struct request_queue *q = hctx->queue; struct bfq_data *bfqd = q->elevator->elevator_data; struct request *free = NULL; /* @@ -2622,6 +2656,9 @@ static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq, return true; } +static bool idling_boosts_thr_without_issues(struct bfq_data *bfqd, + struct bfq_queue *bfqq); + /* * Attempt to schedule a merge of bfqq with the currently in-service * queue or with a close queue among the scheduled queues. Return @@ -2644,10 +2681,49 @@ static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq, */ static struct bfq_queue * bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq, - void *io_struct, bool request) + void *io_struct, bool request, struct bfq_io_cq *bic) { struct bfq_queue *in_service_bfqq, *new_bfqq; + /* + * Check delayed stable merge for rotational or non-queueing + * devs. For this branch to be executed, bfqq must not be + * currently merged with some other queue (i.e., bfqq->bic + * must be non null). If we considered also merged queues, + * then we should also check whether bfqq has already been + * merged with bic->stable_merge_bfqq. But this would be + * costly and complicated. + */ + if (unlikely(!bfqd->nonrot_with_queueing)) { + if (bic->stable_merge_bfqq && + !bfq_bfqq_just_created(bfqq) && + time_is_after_jiffies(bfqq->split_time + + msecs_to_jiffies(200))) { + struct bfq_queue *stable_merge_bfqq = + bic->stable_merge_bfqq; + int proc_ref = min(bfqq_process_refs(bfqq), + bfqq_process_refs(stable_merge_bfqq)); + + /* deschedule stable merge, because done or aborted here */ + bfq_put_stable_ref(stable_merge_bfqq); + + bic->stable_merge_bfqq = NULL; + + if (!idling_boosts_thr_without_issues(bfqd, bfqq) && + proc_ref > 0) { + /* next function will take at least one ref */ + struct bfq_queue *new_bfqq = + bfq_setup_merge(bfqq, stable_merge_bfqq); + + bic->stably_merged = true; + if (new_bfqq && new_bfqq->bic) + new_bfqq->bic->stably_merged = true; + return new_bfqq; + } else + return NULL; + } + } + /* * Do not perform queue merging if the device is non * rotational and performs internal queueing. In fact, such a @@ -2789,6 +2865,17 @@ static void bfq_bfqq_save_state(struct bfq_queue *bfqq) } } + +static void +bfq_reassign_last_bfqq(struct bfq_queue *cur_bfqq, struct bfq_queue *new_bfqq) +{ + if (cur_bfqq->entity.parent && + cur_bfqq->entity.parent->last_bfqq_created == cur_bfqq) + cur_bfqq->entity.parent->last_bfqq_created = new_bfqq; + else if (cur_bfqq->bfqd && cur_bfqq->bfqd->last_bfqq_created == cur_bfqq) + cur_bfqq->bfqd->last_bfqq_created = new_bfqq; +} + void bfq_release_process_ref(struct bfq_data *bfqd, struct bfq_queue *bfqq) { /* @@ -2806,6 +2893,8 @@ void bfq_release_process_ref(struct bfq_data *bfqd, struct bfq_queue *bfqq) bfqq != bfqd->in_service_queue) bfq_del_bfqq_busy(bfqd, bfqq, false); + bfq_reassign_last_bfqq(bfqq, NULL); + bfq_put_queue(bfqq); } @@ -2822,6 +2911,29 @@ bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic, bfq_mark_bfqq_IO_bound(new_bfqq); bfq_clear_bfqq_IO_bound(bfqq); + /* + * The processes associated with bfqq are cooperators of the + * processes associated with new_bfqq. So, if bfqq has a + * waker, then assume that all these processes will be happy + * to let bfqq's waker freely inject I/O when they have no + * I/O. + */ + if (bfqq->waker_bfqq && !new_bfqq->waker_bfqq && + bfqq->waker_bfqq != new_bfqq) { + new_bfqq->waker_bfqq = bfqq->waker_bfqq; + new_bfqq->tentative_waker_bfqq = NULL; + + /* + * If the waker queue disappears, then + * new_bfqq->waker_bfqq must be reset. So insert + * new_bfqq into the woken_list of the waker. See + * bfq_check_waker for details. + */ + hlist_add_head(&new_bfqq->woken_list_node, + &new_bfqq->waker_bfqq->woken_list); + + } + /* * If bfqq is weight-raised, then let new_bfqq inherit * weight-raising. To reduce false positives, neglect the case @@ -2879,6 +2991,9 @@ bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic, */ new_bfqq->pid = -1; bfqq->bic = NULL; + + bfq_reassign_last_bfqq(bfqq, new_bfqq); + bfq_release_process_ref(bfqd, bfqq); } @@ -2906,7 +3021,7 @@ static bool bfq_allow_bio_merge(struct request_queue *q, struct request *rq, * We take advantage of this function to perform an early merge * of the queues of possible cooperating processes. */ - new_bfqq = bfq_setup_cooperator(bfqd, bfqq, bio, false); + new_bfqq = bfq_setup_cooperator(bfqd, bfqq, bio, false, bfqd->bio_bic); if (new_bfqq) { /* * bic still points to bfqq, then it has not yet been @@ -4491,9 +4606,15 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd) bfq_bfqq_busy(bfqq->bic->bfqq[0]) && bfqq->bic->bfqq[0]->next_rq ? bfqq->bic->bfqq[0] : NULL; + struct bfq_queue *blocked_bfqq = + !hlist_empty(&bfqq->woken_list) ? + container_of(bfqq->woken_list.first, + struct bfq_queue, + woken_list_node) + : NULL; /* - * The next three mutually-exclusive ifs decide + * The next four mutually-exclusive ifs decide * whether to try injection, and choose the queue to * pick an I/O request from. * @@ -4526,7 +4647,15 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd) * next bfqq's I/O is brought forward dramatically, * for it is not blocked for milliseconds. * - * The third if checks whether bfqq is a queue for + * The third if checks whether there is a queue woken + * by bfqq, and currently with pending I/O. Such a + * woken queue does not steal bandwidth from bfqq, + * because it remains soon without I/O if bfqq is not + * served. So there is virtually no risk of loss of + * bandwidth for bfqq if this woken queue has I/O + * dispatched while bfqq is waiting for new I/O. + * + * The fourth if checks whether bfqq is a queue for * which it is better to avoid injection. It is so if * bfqq delivers more throughput when served without * any further I/O from other queues in the middle, or @@ -4546,11 +4675,11 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd) * bfq_update_has_short_ttime(), it is rather likely * that, if I/O is being plugged for bfqq and the * waker queue has pending I/O requests that are - * blocking bfqq's I/O, then the third alternative + * blocking bfqq's I/O, then the fourth alternative * above lets the waker queue get served before the * I/O-plugging timeout fires. So one may deem the * second alternative superfluous. It is not, because - * the third alternative may be way less effective in + * the fourth alternative may be way less effective in * case of a synchronization. For two main * reasons. First, throughput may be low because the * inject limit may be too low to guarantee the same @@ -4559,7 +4688,7 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd) * guarantees (the second alternative unconditionally * injects a pending I/O request of the waker queue * for each bfq_dispatch_request()). Second, with the - * third alternative, the duration of the plugging, + * fourth alternative, the duration of the plugging, * i.e., the time before bfqq finally receives new I/O, * may not be minimized, because the waker queue may * happen to be served only after other queues. @@ -4577,6 +4706,14 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd) bfq_bfqq_budget_left(bfqq->waker_bfqq) ) bfqq = bfqq->waker_bfqq; + else if (blocked_bfqq && + bfq_bfqq_busy(blocked_bfqq) && + blocked_bfqq->next_rq && + bfq_serv_to_charge(blocked_bfqq->next_rq, + blocked_bfqq) <= + bfq_bfqq_budget_left(blocked_bfqq) + ) + bfqq = blocked_bfqq; else if (!idling_boosts_thr_without_issues(bfqd, bfqq) && (bfqq->wr_coeff == 1 || bfqd->wr_busy_queues > 1 || !bfq_bfqq_has_short_ttime(bfqq))) @@ -4808,6 +4945,17 @@ static struct request *__bfq_dispatch_request(struct blk_mq_hw_ctx *hctx) if (!bfqq) goto exit; + /* + * Here, the IO depth of queues belong to CLASS_IDLE is limited + * to 1, so that it can avoid introducing a larger tail latency + * under a device with a larger IO depth. Although limiting the + * IO depth may reduce the performance of idle_class, it is + * generally not a big problem, because idle_class usually + * does not have strict performance requirements. + */ + if (bfq_class_idle(bfqq) && bfqq->dispatched) + goto exit; + rq = bfq_dispatch_rq_from_bfqq(bfqd, bfqq); if (rq) { @@ -4983,6 +5131,12 @@ void bfq_put_queue(struct bfq_queue *bfqq) bfqg_and_blkg_put(bfqg); } +static void bfq_put_stable_ref(struct bfq_queue *bfqq) +{ + bfqq->stable_ref--; + bfq_put_queue(bfqq); +} + static void bfq_put_cooperator(struct bfq_queue *bfqq) { struct bfq_queue *__bfqq, *next; @@ -5027,7 +5181,14 @@ static void bfq_exit_icq_bfqq(struct bfq_io_cq *bic, bool is_sync) if (bfqq && bfqd) { unsigned long flags; - spin_lock_irqsave(&bfqd->lock, flags); + /* bfq_exit_icq is usually called with ioc->lock held, which is + * inverse order from elsewhere, which may grab ioc->lock + * under bfqd->lock if we merge requests and drop the last ioc + * refcount. Since exit_icq is either called with a refcount, + * or with queue quiesced, use a differnet lock class to + * silence lockdep + */ + spin_lock_irqsave_nested(&bfqd->lock, flags, 1); bfqq->bic = NULL; bfq_exit_bfqq(bfqd, bfqq); bic_set_bfqq(bic, NULL, is_sync); @@ -5039,6 +5200,24 @@ static void bfq_exit_icq(struct io_cq *icq) { struct bfq_io_cq *bic = icq_to_bic(icq); + if (bic->stable_merge_bfqq) { + struct bfq_data *bfqd = bic->stable_merge_bfqq->bfqd; + + /* + * bfqd is NULL if scheduler already exited, and in + * that case this is the last time bfqq is accessed. + */ + if (bfqd) { + unsigned long flags; + + spin_lock_irqsave(&bfqd->lock, flags); + bfq_put_stable_ref(bic->stable_merge_bfqq); + spin_unlock_irqrestore(&bfqd->lock, flags); + } else { + bfq_put_stable_ref(bic->stable_merge_bfqq); + } + } + bfq_exit_icq_bfqq(bic, true); bfq_exit_icq_bfqq(bic, false); } @@ -5099,7 +5278,8 @@ bfq_set_next_ioprio_data(struct bfq_queue *bfqq, struct bfq_io_cq *bic) static struct bfq_queue *bfq_get_queue(struct bfq_data *bfqd, struct bio *bio, bool is_sync, - struct bfq_io_cq *bic); + struct bfq_io_cq *bic, + bool respawn); static void bfq_check_ioprio_change(struct bfq_io_cq *bic, struct bio *bio) { @@ -5119,7 +5299,7 @@ static void bfq_check_ioprio_change(struct bfq_io_cq *bic, struct bio *bio) bfqq = bic_to_bfqq(bic, false); if (bfqq) { bfq_release_process_ref(bfqd, bfqq); - bfqq = bfq_get_queue(bfqd, bio, BLK_RW_ASYNC, bic); + bfqq = bfq_get_queue(bfqd, bio, BLK_RW_ASYNC, bic, true); bic_set_bfqq(bic, bfqq, false); } @@ -5162,6 +5342,8 @@ static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, /* set end request to minus infinity from now */ bfqq->ttime.last_end_request = now_ns + 1; + bfqq->creation_time = jiffies; + bfqq->io_start_time = now_ns; bfq_mark_bfqq_IO_bound(bfqq); @@ -5211,9 +5393,156 @@ static struct bfq_queue **bfq_async_queue_prio(struct bfq_data *bfqd, } } +static struct bfq_queue * +bfq_do_early_stable_merge(struct bfq_data *bfqd, struct bfq_queue *bfqq, + struct bfq_io_cq *bic, + struct bfq_queue *last_bfqq_created) +{ + struct bfq_queue *new_bfqq = + bfq_setup_merge(bfqq, last_bfqq_created); + + if (!new_bfqq) + return bfqq; + + if (new_bfqq->bic) + new_bfqq->bic->stably_merged = true; + bic->stably_merged = true; + + /* + * Reusing merge functions. This implies that + * bfqq->bic must be set too, for + * bfq_merge_bfqqs to correctly save bfqq's + * state before killing it. + */ + bfqq->bic = bic; + bfq_merge_bfqqs(bfqd, bic, bfqq, new_bfqq); + + return new_bfqq; +} + +/* + * Many throughput-sensitive workloads are made of several parallel + * I/O flows, with all flows generated by the same application, or + * more generically by the same task (e.g., system boot). The most + * counterproductive action with these workloads is plugging I/O + * dispatch when one of the bfq_queues associated with these flows + * remains temporarily empty. + * + * To avoid this plugging, BFQ has been using a burst-handling + * mechanism for years now. This mechanism has proven effective for + * throughput, and not detrimental for service guarantees. The + * following function pushes this mechanism a little bit further, + * basing on the following two facts. + * + * First, all the I/O flows of a the same application or task + * contribute to the execution/completion of that common application + * or task. So the performance figures that matter are total + * throughput of the flows and task-wide I/O latency. In particular, + * these flows do not need to be protected from each other, in terms + * of individual bandwidth or latency. + * + * Second, the above fact holds regardless of the number of flows. + * + * Putting these two facts together, this commits merges stably the + * bfq_queues associated with these I/O flows, i.e., with the + * processes that generate these IO/ flows, regardless of how many the + * involved processes are. + * + * To decide whether a set of bfq_queues is actually associated with + * the I/O flows of a common application or task, and to merge these + * queues stably, this function operates as follows: given a bfq_queue, + * say Q2, currently being created, and the last bfq_queue, say Q1, + * created before Q2, Q2 is merged stably with Q1 if + * - very little time has elapsed since when Q1 was created + * - Q2 has the same ioprio as Q1 + * - Q2 belongs to the same group as Q1 + * + * Merging bfq_queues also reduces scheduling overhead. A fio test + * with ten random readers on /dev/nullb shows a throughput boost of + * 40%, with a quadcore. Since BFQ's execution time amounts to ~50% of + * the total per-request processing time, the above throughput boost + * implies that BFQ's overhead is reduced by more than 50%. + * + * This new mechanism most certainly obsoletes the current + * burst-handling heuristics. We keep those heuristics for the moment. + */ +static struct bfq_queue *bfq_do_or_sched_stable_merge(struct bfq_data *bfqd, + struct bfq_queue *bfqq, + struct bfq_io_cq *bic) +{ + struct bfq_queue **source_bfqq = bfqq->entity.parent ? + &bfqq->entity.parent->last_bfqq_created : + &bfqd->last_bfqq_created; + + struct bfq_queue *last_bfqq_created = *source_bfqq; + + /* + * If last_bfqq_created has not been set yet, then init it. If + * it has been set already, but too long ago, then move it + * forward to bfqq. Finally, move also if bfqq belongs to a + * different group than last_bfqq_created, or if bfqq has a + * different ioprio or ioprio_class. If none of these + * conditions holds true, then try an early stable merge or + * schedule a delayed stable merge. + * + * A delayed merge is scheduled (instead of performing an + * early merge), in case bfqq might soon prove to be more + * throughput-beneficial if not merged. Currently this is + * possible only if bfqd is rotational with no queueing. For + * such a drive, not merging bfqq is better for throughput if + * bfqq happens to contain sequential I/O. So, we wait a + * little bit for enough I/O to flow through bfqq. After that, + * if such an I/O is sequential, then the merge is + * canceled. Otherwise the merge is finally performed. + */ + if (!last_bfqq_created || + time_before(last_bfqq_created->creation_time + + bfqd->bfq_burst_interval, + bfqq->creation_time) || + bfqq->entity.parent != last_bfqq_created->entity.parent || + bfqq->ioprio != last_bfqq_created->ioprio || + bfqq->ioprio_class != last_bfqq_created->ioprio_class) + *source_bfqq = bfqq; + else if (time_after_eq(last_bfqq_created->creation_time + + bfqd->bfq_burst_interval, + bfqq->creation_time)) { + if (likely(bfqd->nonrot_with_queueing)) + /* + * With this type of drive, leaving + * bfqq alone may provide no + * throughput benefits compared with + * merging bfqq. So merge bfqq now. + */ + bfqq = bfq_do_early_stable_merge(bfqd, bfqq, + bic, + last_bfqq_created); + else { /* schedule tentative stable merge */ + /* + * get reference on last_bfqq_created, + * to prevent it from being freed, + * until we decide whether to merge + */ + last_bfqq_created->ref++; + /* + * need to keep track of stable refs, to + * compute process refs correctly + */ + last_bfqq_created->stable_ref++; + /* + * Record the bfqq to merge to. + */ + bic->stable_merge_bfqq = last_bfqq_created; + } + } + + return bfqq; +} + + static struct bfq_queue *bfq_get_queue(struct bfq_data *bfqd, struct bio *bio, bool is_sync, - struct bfq_io_cq *bic) + struct bfq_io_cq *bic, + bool respawn) { const int ioprio = IOPRIO_PRIO_DATA(bic->ioprio); const int ioprio_class = IOPRIO_PRIO_CLASS(bic->ioprio); @@ -5271,7 +5600,10 @@ static struct bfq_queue *bfq_get_queue(struct bfq_data *bfqd, out: bfqq->ref++; /* get a process reference to this queue */ - bfq_log_bfqq(bfqd, bfqq, "get_queue, at end: %p, %d", bfqq, bfqq->ref); + + if (bfqq != &bfqd->oom_bfqq && is_sync && !respawn) + bfqq = bfq_do_or_sched_stable_merge(bfqd, bfqq, bic); + rcu_read_unlock(); return bfqq; } @@ -5521,7 +5853,8 @@ static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq, static bool __bfq_insert_request(struct bfq_data *bfqd, struct request *rq) { struct bfq_queue *bfqq = RQ_BFQQ(rq), - *new_bfqq = bfq_setup_cooperator(bfqd, bfqq, rq, true); + *new_bfqq = bfq_setup_cooperator(bfqd, bfqq, rq, true, + RQ_BIC(rq)); bool waiting, idle_timer_disabled = false; if (new_bfqq) { @@ -5627,7 +5960,48 @@ static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, spin_lock_irq(&bfqd->lock); bfqq = bfq_init_rq(rq); - if (!bfqq || at_head || blk_rq_is_passthrough(rq)) { + + /* + * Reqs with at_head or passthrough flags set are to be put + * directly into dispatch list. Additional case for putting rq + * directly into the dispatch queue: the only active + * bfq_queues are bfqq and either its waker bfq_queue or one + * of its woken bfq_queues. The rationale behind this + * additional condition is as follows: + * - consider a bfq_queue, say Q1, detected as a waker of + * another bfq_queue, say Q2 + * - by definition of a waker, Q1 blocks the I/O of Q2, i.e., + * some I/O of Q1 needs to be completed for new I/O of Q2 + * to arrive. A notable example of waker is journald + * - so, Q1 and Q2 are in any respect the queues of two + * cooperating processes (or of two cooperating sets of + * processes): the goal of Q1's I/O is doing what needs to + * be done so that new Q2's I/O can finally be + * issued. Therefore, if the service of Q1's I/O is delayed, + * then Q2's I/O is delayed too. Conversely, if Q2's I/O is + * delayed, the goal of Q1's I/O is hindered. + * - as a consequence, if some I/O of Q1/Q2 arrives while + * Q2/Q1 is the only queue in service, there is absolutely + * no point in delaying the service of such an I/O. The + * only possible result is a throughput loss + * - so, when the above condition holds, the best option is to + * have the new I/O dispatched as soon as possible + * - the most effective and efficient way to attain the above + * goal is to put the new I/O directly in the dispatch + * list + * - as an additional restriction, Q1 and Q2 must be the only + * busy queues for this commit to put the I/O of Q2/Q1 in + * the dispatch list. This is necessary, because, if also + * other queues are waiting for service, then putting new + * I/O directly in the dispatch list may evidently cause a + * violation of service guarantees for the other queues + */ + if (!bfqq || + (bfqq != bfqd->in_service_queue && + bfqd->in_service_queue != NULL && + bfq_tot_busy_queues(bfqd) == 1 + bfq_bfqq_busy(bfqq) && + (bfqq->waker_bfqq == bfqd->in_service_queue || + bfqd->in_service_queue->waker_bfqq == bfqq)) || at_head) { if (at_head) list_add(&rq->queuelist, &bfqd->dispatch); else @@ -5767,7 +6141,17 @@ static void bfq_completed_request(struct bfq_queue *bfqq, struct bfq_data *bfqd) 1UL<<(BFQ_RATE_SHIFT - 10)) bfq_update_rate_reset(bfqd, NULL); bfqd->last_completion = now_ns; - bfqd->last_completed_rq_bfqq = bfqq; + /* + * Shared queues are likely to receive I/O at a high + * rate. This may deceptively let them be considered as wakers + * of other queues. But a false waker will unjustly steal + * bandwidth to its supposedly woken queue. So considering + * also shared queues in the waking mechanism may cause more + * control troubles than throughput benefits. Then do not set + * last_completed_rq_bfqq to bfqq if bfqq is a shared queue. + */ + if (!bfq_bfqq_coop(bfqq)) + bfqd->last_completed_rq_bfqq = bfqq; /* * If we are waiting to discover whether the request pattern @@ -6124,7 +6508,7 @@ static struct bfq_queue *bfq_get_bfqq_handle_split(struct bfq_data *bfqd, if (bfqq) bfq_put_queue(bfqq); - bfqq = bfq_get_queue(bfqd, bio, is_sync, bic); + bfqq = bfq_get_queue(bfqd, bio, is_sync, bic, split); bic_set_bfqq(bic, bfqq, is_sync); if (split && is_sync) { @@ -6245,8 +6629,9 @@ static struct bfq_queue *bfq_init_rq(struct request *rq) if (likely(!new_queue)) { /* If the queue was seeky for too long, break it apart. */ - if (bfq_bfqq_coop(bfqq) && bfq_bfqq_split_coop(bfqq)) { - bfq_log_bfqq(bfqd, bfqq, "breaking apart bfqq"); + if (bfq_bfqq_coop(bfqq) && bfq_bfqq_split_coop(bfqq) && + !bic->stably_merged) { + struct bfq_queue *old_bfqq = bfqq; /* Update bic before losing reference to bfqq */ if (bfq_bfqq_in_large_burst(bfqq)) @@ -6255,11 +6640,24 @@ static struct bfq_queue *bfq_init_rq(struct request *rq) bfqq = bfq_split_bfqq(bic, bfqq); split = true; - if (!bfqq) + if (!bfqq) { bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio, true, is_sync, NULL); - else + bfqq->waker_bfqq = old_bfqq->waker_bfqq; + bfqq->tentative_waker_bfqq = NULL; + + /* + * If the waker queue disappears, then + * new_bfqq->waker_bfqq must be + * reset. So insert new_bfqq into the + * woken_list of the waker. See + * bfq_check_waker for details. + */ + if (bfqq->waker_bfqq) + hlist_add_head(&bfqq->woken_list_node, + &bfqq->waker_bfqq->woken_list); + } else bfqq_already_existing = true; } } @@ -6526,9 +6924,11 @@ static void bfq_init_root_group(struct bfq_group *root_group, root_group->bfqd = bfqd; #endif root_group->rq_pos_tree = RB_ROOT; - for (i = 0; i < BFQ_IOPRIO_CLASSES; i++) + for (i = 0; i < BFQ_IOPRIO_CLASSES; i++) { root_group->sched_data.service_tree[i] = BFQ_SERVICE_TREE_INIT; - root_group->sched_data.bfq_class_idle_last_service = jiffies; + root_group->sched_data.bfq_class_last_service[i] = jiffies; + } + root_group->sched_data.class_timeout_last_check = jiffies; } static int bfq_init_queue(struct request_queue *q, struct elevator_type *e) diff --git a/block/bfq-iosched.h b/block/bfq-iosched.h index b8e793c34ff1..c8c8a36f98a5 100644 --- a/block/bfq-iosched.h +++ b/block/bfq-iosched.h @@ -13,7 +13,7 @@ #include "blk-cgroup-rwstat.h" #define BFQ_IOPRIO_CLASSES 3 -#define BFQ_CL_IDLE_TIMEOUT (HZ/5) +#define BFQ_CLASS_TIMEOUT (HZ/5) #define BFQ_MIN_WEIGHT 1 #define BFQ_MAX_WEIGHT 1000 @@ -22,7 +22,6 @@ #define BFQ_DEFAULT_QUEUE_IOPRIO 4 #define BFQ_WEIGHT_LEGACY_DFL 100 -#define BFQ_DEFAULT_GRP_IOPRIO 0 #define BFQ_DEFAULT_GRP_CLASS IOPRIO_CLASS_BE #define MAX_PID_STR_LENGTH 12 @@ -97,9 +96,12 @@ struct bfq_sched_data { struct bfq_entity *next_in_service; /* array of service trees, one per ioprio_class */ struct bfq_service_tree service_tree[BFQ_IOPRIO_CLASSES]; - /* last time CLASS_IDLE was served */ - unsigned long bfq_class_idle_last_service; - + /* last time the class was served */ + unsigned long bfq_class_last_service[BFQ_IOPRIO_CLASSES]; + /* last time class timeout was checked */ + unsigned long class_timeout_last_check; + /* next index to check class timeout */ + unsigned int next_class_index; }; /** @@ -197,6 +199,9 @@ struct bfq_entity { /* flag, set if the entity is counted in groups_with_pending_reqs */ bool in_groups_with_pending_reqs; + + /* last child queue of entity created (for non-leaf entities) */ + struct bfq_queue *last_bfqq_created; }; struct bfq_group; @@ -230,6 +235,8 @@ struct bfq_ttime { struct bfq_queue { /* reference counter */ int ref; + /* counter of references from other queues for delayed stable merge */ + int stable_ref; /* parent bfq_data */ struct bfq_data *bfqd; @@ -365,6 +372,8 @@ struct bfq_queue { unsigned long first_IO_time; /* time of first I/O for this queue */ + unsigned long creation_time; /* when this queue is created */ + /* max service rate measured so far */ u32 max_service_rate; @@ -454,6 +463,11 @@ struct bfq_io_cq { u64 saved_last_serv_time_ns; unsigned int saved_inject_limit; unsigned long saved_decrease_time_jif; + + /* candidate queue for a stable merge (due to close creation time) */ + struct bfq_queue *stable_merge_bfqq; + + bool stably_merged; /* non splittable if true */ }; /** @@ -578,6 +592,9 @@ struct bfq_data { /* bfqq owning the last completed rq */ struct bfq_queue *last_completed_rq_bfqq; + /* last bfqq created, among those in the root group */ + struct bfq_queue *last_bfqq_created; + /* time of last transition from empty to non-empty (ns) */ u64 last_empty_occupied_ns; @@ -914,7 +931,7 @@ struct bfq_group { struct bfq_entity entity; struct bfq_sched_data sched_data; - void *bfqd; + struct bfq_data *bfqd; struct bfq_queue *async_bfqq[2][IOPRIO_BE_NR]; struct bfq_queue *async_idle_bfqq; @@ -941,6 +958,7 @@ struct bfq_group { #endif struct bfq_queue *bfq_entity_to_bfqq(struct bfq_entity *entity); +struct bfq_group *bfq_entity_to_bfqg(struct bfq_entity *entity); /* --------------- main algorithm interface ----------------- */ diff --git a/block/bfq-wf2q.c b/block/bfq-wf2q.c index 070e34a7feb1..7e48ed5b7be5 100644 --- a/block/bfq-wf2q.c +++ b/block/bfq-wf2q.c @@ -149,7 +149,7 @@ struct bfq_group *bfq_bfqq_to_bfqg(struct bfq_queue *bfqq) if (!group_entity) group_entity = &bfqq->bfqd->root_group->entity; - return container_of(group_entity, struct bfq_group, entity); + return bfq_entity_to_bfqg(group_entity); } /* @@ -208,7 +208,7 @@ static bool bfq_no_longer_next_in_service(struct bfq_entity *entity) if (bfq_entity_to_bfqq(entity)) return true; - bfqg = container_of(entity, struct bfq_group, entity); + bfqg = bfq_entity_to_bfqg(entity); /* * The field active_entities does not always contain the @@ -266,6 +266,15 @@ struct bfq_queue *bfq_entity_to_bfqq(struct bfq_entity *entity) return bfqq; } +struct bfq_group *bfq_entity_to_bfqg(struct bfq_entity *entity) +{ + struct bfq_group *bfqg = NULL; + + if (entity->my_sched_data) + bfqg = container_of(entity, struct bfq_group, entity); + + return bfqg; +} /** * bfq_delta - map service into the virtual time domain. @@ -489,7 +498,7 @@ static void bfq_active_insert(struct bfq_service_tree *st, #ifdef CONFIG_BFQ_GROUP_IOSCHED sd = entity->sched_data; bfqg = container_of(sd, struct bfq_group, sched_data); - bfqd = (struct bfq_data *)bfqg->bfqd; + bfqd = bfqg->bfqd; #endif if (bfqq) list_add(&bfqq->bfqq_list, &bfqq->bfqd->active_list); @@ -518,8 +527,9 @@ unsigned short bfq_ioprio_to_weight(int ioprio) */ static unsigned short bfq_weight_to_ioprio(int weight) { - return max_t(int, 0, - IOPRIO_BE_NR * BFQ_WEIGHT_CONVERSION_COEFF - weight); + int ioprio = IOPRIO_BE_NR - weight / BFQ_WEIGHT_CONVERSION_COEFF; + + return ioprio < 0 ? 0 : min_t(int, ioprio, IOPRIO_BE_NR - 1); } static void bfq_get_entity(struct bfq_entity *entity) @@ -588,7 +598,7 @@ static void bfq_active_extract(struct bfq_service_tree *st, #ifdef CONFIG_BFQ_GROUP_IOSCHED sd = entity->sched_data; bfqg = container_of(sd, struct bfq_group, sched_data); - bfqd = (struct bfq_data *)bfqg->bfqd; + bfqd = bfqg->bfqd; #endif if (bfqq) list_del(&bfqq->bfqq_list); @@ -734,7 +744,7 @@ __bfq_entity_update_weight_prio(struct bfq_service_tree *old_st, else { sd = entity->my_sched_data; bfqg = container_of(sd, struct bfq_group, sched_data); - bfqd = (struct bfq_data *)bfqg->bfqd; + bfqd = bfqg->bfqd; } #endif @@ -872,7 +882,7 @@ void bfq_bfqq_charge_time(struct bfq_data *bfqd, struct bfq_queue *bfqq, unsigned long time_ms) { struct bfq_entity *entity = &bfqq->entity; - unsigned long timeout_ms = jiffies_to_msecs(bfq_timeout); + unsigned long timeout_ms = jiffies_to_msecs(bfqd->bfq_timeout); unsigned long bounded_time_ms = min(time_ms, timeout_ms); int serv_to_charge_for_time = (bfqd->bfq_max_budget * bounded_time_ms) / timeout_ms; @@ -1001,8 +1011,7 @@ static void __bfq_activate_entity(struct bfq_entity *entity, #ifdef CONFIG_BFQ_GROUP_IOSCHED if (!bfq_entity_to_bfqq(entity)) { /* bfq_group */ - struct bfq_group *bfqg = - container_of(entity, struct bfq_group, entity); + struct bfq_group *bfqg = bfq_entity_to_bfqg(entity); struct bfq_data *bfqd = bfqg->bfqd; if (!entity->in_groups_with_pending_reqs) { @@ -1160,6 +1169,7 @@ bool __bfq_deactivate_entity(struct bfq_entity *entity, bool ins_into_idle_tree) { struct bfq_sched_data *sd = entity->sched_data; struct bfq_service_tree *st; + int idx = bfq_class_idx(entity); bool is_in_service; if (!entity->on_st_or_in_serv) /* @@ -1199,6 +1209,7 @@ bool __bfq_deactivate_entity(struct bfq_entity *entity, bool ins_into_idle_tree) else bfq_idle_insert(st, entity); + sd->bfq_class_last_service[idx] = jiffies; return true; } @@ -1427,6 +1438,45 @@ __bfq_lookup_next_entity(struct bfq_service_tree *st, bool in_service) return entity; } +static int bfq_select_next_class(struct bfq_sched_data *sd) +{ + struct bfq_service_tree *st = sd->service_tree; + unsigned long last_check, last_serve; + int i, class_idx, next_class = 0; + bool found = false; + + /* + * we needed to guarantee a minimum bandwidth for each class (if + * there is some active entity in this class). This should also + * mitigate priority-inversion problems in case a low priority + * task is holding file system resources. + */ + last_check = sd->class_timeout_last_check; + if (time_is_after_jiffies(last_check + BFQ_CLASS_TIMEOUT)) + return next_class; + + sd->class_timeout_last_check = jiffies; + for (i = 0; i < BFQ_IOPRIO_CLASSES; i++) { + class_idx = (sd->next_class_index + i) % BFQ_IOPRIO_CLASSES; + last_serve = sd->bfq_class_last_service[class_idx]; + + if (time_is_after_jiffies(last_serve + BFQ_CLASS_TIMEOUT)) + continue; + + if (!RB_EMPTY_ROOT(&(st + class_idx)->active)) { + if (found) + continue; + + next_class = class_idx++; + class_idx %= BFQ_IOPRIO_CLASSES; + sd->next_class_index = class_idx; + found = true; + } + sd->bfq_class_last_service[class_idx] = jiffies; + } + return next_class; +} + /** * bfq_lookup_next_entity - return the first eligible entity in @sd. * @sd: the sched_data. @@ -1440,24 +1490,8 @@ static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd, bool expiration) { struct bfq_service_tree *st = sd->service_tree; - struct bfq_service_tree *idle_class_st = st + (BFQ_IOPRIO_CLASSES - 1); struct bfq_entity *entity = NULL; - int class_idx = 0; - - /* - * Choose from idle class, if needed to guarantee a minimum - * bandwidth to this class (and if there is some active entity - * in idle class). This should also mitigate - * priority-inversion problems in case a low priority task is - * holding file system resources. - */ - if (time_is_before_jiffies(sd->bfq_class_idle_last_service + - BFQ_CL_IDLE_TIMEOUT)) { - if (!RB_EMPTY_ROOT(&idle_class_st->active)) - class_idx = BFQ_IOPRIO_CLASSES - 1; - /* About to be served if backlogged, or not yet backlogged */ - sd->bfq_class_idle_last_service = jiffies; - } + int class_idx = bfq_select_next_class(sd); /* * Find the next entity to serve for the highest-priority @@ -1706,4 +1740,12 @@ void bfq_add_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq) if (bfqq->wr_coeff > 1) bfqd->wr_busy_queues++; + + /* Move bfqq to the head of the woken list of its waker */ + if (!hlist_unhashed(&bfqq->woken_list_node) && + &bfqq->woken_list_node != bfqq->waker_bfqq->woken_list.first) { + hlist_del_init(&bfqq->woken_list_node); + hlist_add_head(&bfqq->woken_list_node, + &bfqq->waker_bfqq->woken_list); + } } diff --git a/block/blk-iocost.c b/block/blk-iocost.c index 98d656bdb42b..4fbc875f7cb2 100644 --- a/block/blk-iocost.c +++ b/block/blk-iocost.c @@ -1073,7 +1073,17 @@ static void __propagate_weights(struct ioc_gq *iocg, u32 active, u32 inuse, lockdep_assert_held(&ioc->lock); - inuse = clamp_t(u32, inuse, 1, active); + /* + * For an active leaf node, its inuse shouldn't be zero or exceed + * @active. An active internal node's inuse is solely determined by the + * inuse to active ratio of its children regardless of @inuse. + */ + if (list_empty(&iocg->active_list) && iocg->child_active_sum) { + inuse = DIV64_U64_ROUND_UP(active * iocg->child_inuse_sum, + iocg->child_active_sum); + } else { + inuse = clamp_t(u32, inuse, 1, active); + } iocg->last_inuse = iocg->inuse; if (save) @@ -1090,7 +1100,7 @@ static void __propagate_weights(struct ioc_gq *iocg, u32 active, u32 inuse, /* update the level sums */ parent->child_active_sum += (s32)(active - child->active); parent->child_inuse_sum += (s32)(inuse - child->inuse); - /* apply the udpates */ + /* apply the updates */ child->active = active; child->inuse = inuse; diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c index e1e997af89a0..fdeb9773b55c 100644 --- a/block/blk-mq-sched.c +++ b/block/blk-mq-sched.c @@ -348,14 +348,16 @@ bool __blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio, unsigned int nr_segs) { struct elevator_queue *e = q->elevator; - struct blk_mq_ctx *ctx = blk_mq_get_ctx(q); - struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, bio->bi_opf, ctx); + struct blk_mq_ctx *ctx; + struct blk_mq_hw_ctx *hctx; bool ret = false; enum hctx_type type; if (e && e->type->ops.bio_merge) - return e->type->ops.bio_merge(hctx, bio, nr_segs); + return e->type->ops.bio_merge(q, bio, nr_segs); + ctx = blk_mq_get_ctx(q); + hctx = blk_mq_map_queue(q, bio->bi_opf, ctx); type = hctx->type; if (!(hctx->flags & BLK_MQ_F_SHOULD_MERGE) || list_empty_careful(&ctx->rq_lists[type])) diff --git a/block/blk-mq.c b/block/blk-mq.c index d4d7c1caa439..9b439175f636 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -361,11 +361,12 @@ static struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data) if (e) { /* - * Flush requests are special and go directly to the + * Flush/passthrough requests are special and go directly to the * dispatch list. Don't include reserved tags in the * limiting, as it isn't useful. */ if (!op_is_flush(data->cmd_flags) && + !blk_op_is_passthrough(data->cmd_flags) && e->type->ops.limit_depth && !(data->flags & BLK_MQ_REQ_RESERVED)) e->type->ops.limit_depth(data->cmd_flags, data); @@ -2216,8 +2217,9 @@ blk_qc_t blk_mq_submit_bio(struct bio *bio) /* Bypass scheduler for flush requests */ blk_insert_flush(rq); blk_mq_run_hw_queue(data.hctx, true); - } else if (plug && (q->nr_hw_queues == 1 || q->mq_ops->commit_rqs || - !blk_queue_nonrot(q))) { + } else if (plug && (q->nr_hw_queues == 1 || + blk_mq_is_sbitmap_shared(rq->mq_hctx->flags) || + q->mq_ops->commit_rqs || !blk_queue_nonrot(q))) { /* * Use plugging if we have a ->commit_rqs() hook as well, as * we know the driver uses bd->last in a smart fashion. @@ -3269,10 +3271,12 @@ EXPORT_SYMBOL(blk_mq_init_allocated_queue); /* tags can _not_ be used after returning from blk_mq_exit_queue */ void blk_mq_exit_queue(struct request_queue *q) { - struct blk_mq_tag_set *set = q->tag_set; + struct blk_mq_tag_set *set = q->tag_set; - blk_mq_del_queue_tag_set(q); + /* Checks hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED. */ blk_mq_exit_hw_queues(q, set, set->nr_hw_queues); + /* May clear BLK_MQ_F_TAG_QUEUE_SHARED in hctx->flags. */ + blk_mq_del_queue_tag_set(q); } static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set) diff --git a/block/kyber-iosched.c b/block/kyber-iosched.c index 33d34d69cade..79b69d7046d6 100644 --- a/block/kyber-iosched.c +++ b/block/kyber-iosched.c @@ -560,11 +560,12 @@ static void kyber_limit_depth(unsigned int op, struct blk_mq_alloc_data *data) } } -static bool kyber_bio_merge(struct blk_mq_hw_ctx *hctx, struct bio *bio, +static bool kyber_bio_merge(struct request_queue *q, struct bio *bio, unsigned int nr_segs) { + struct blk_mq_ctx *ctx = blk_mq_get_ctx(q); + struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, bio->bi_opf, ctx); struct kyber_hctx_data *khd = hctx->sched_data; - struct blk_mq_ctx *ctx = blk_mq_get_ctx(hctx->queue); struct kyber_ctx_queue *kcq = &khd->kcqs[ctx->index_hw[hctx->type]]; unsigned int sched_domain = kyber_sched_domain(bio->bi_opf); struct list_head *rq_list = &kcq->rq_list[sched_domain]; diff --git a/block/mq-deadline.c b/block/mq-deadline.c index f3631a287466..8eea2cbf2bf4 100644 --- a/block/mq-deadline.c +++ b/block/mq-deadline.c @@ -461,10 +461,9 @@ static int dd_request_merge(struct request_queue *q, struct request **rq, return ELEVATOR_NO_MERGE; } -static bool dd_bio_merge(struct blk_mq_hw_ctx *hctx, struct bio *bio, +static bool dd_bio_merge(struct request_queue *q, struct bio *bio, unsigned int nr_segs) { - struct request_queue *q = hctx->queue; struct deadline_data *dd = q->elevator->elevator_data; struct request *free = NULL; bool ret; @@ -500,11 +499,8 @@ static void dd_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, trace_block_rq_insert(rq); - if (at_head || blk_rq_is_passthrough(rq)) { - if (at_head) - list_add(&rq->queuelist, &dd->dispatch); - else - list_add_tail(&rq->queuelist, &dd->dispatch); + if (at_head) { + list_add(&rq->queuelist, &dd->dispatch); } else { deadline_add_rq_rb(dd, rq); diff --git a/crypto/api.c b/crypto/api.c index ed08cbd5b9d3..c4eda56cff89 100644 --- a/crypto/api.c +++ b/crypto/api.c @@ -562,7 +562,7 @@ void crypto_destroy_tfm(void *mem, struct crypto_tfm *tfm) { struct crypto_alg *alg; - if (unlikely(!mem)) + if (IS_ERR_OR_NULL(mem)) return; alg = tfm->__crt_alg; diff --git a/crypto/async_tx/async_xor.c b/crypto/async_tx/async_xor.c index a057ecb1288d..6cd7f7025df4 100644 --- a/crypto/async_tx/async_xor.c +++ b/crypto/async_tx/async_xor.c @@ -233,6 +233,7 @@ async_xor_offs(struct page *dest, unsigned int offset, if (submit->flags & ASYNC_TX_XOR_DROP_DST) { src_cnt--; src_list++; + src_offs++; } /* wait for any prerequisite operations */ diff --git a/crypto/rng.c b/crypto/rng.c index a888d84b524a..fea082b25fe4 100644 --- a/crypto/rng.c +++ b/crypto/rng.c @@ -34,22 +34,18 @@ int crypto_rng_reset(struct crypto_rng *tfm, const u8 *seed, unsigned int slen) u8 *buf = NULL; int err; - crypto_stats_get(alg); if (!seed && slen) { buf = kmalloc(slen, GFP_KERNEL); - if (!buf) { - crypto_alg_put(alg); + if (!buf) return -ENOMEM; - } err = get_random_bytes_wait(buf, slen); - if (err) { - crypto_alg_put(alg); + if (err) goto out; - } seed = buf; } + crypto_stats_get(alg); err = crypto_rng_alg(tfm)->seed(tfm, seed, slen); crypto_stats_rng_seed(alg, err); out: diff --git a/crypto/zstd.c b/crypto/zstd.c index 1a3309f066f7..154a969c83a8 100644 --- a/crypto/zstd.c +++ b/crypto/zstd.c @@ -18,22 +18,22 @@ #define ZSTD_DEF_LEVEL 3 struct zstd_ctx { - ZSTD_CCtx *cctx; - ZSTD_DCtx *dctx; + zstd_cctx *cctx; + zstd_dctx *dctx; void *cwksp; void *dwksp; }; -static ZSTD_parameters zstd_params(void) +static zstd_parameters zstd_params(void) { - return ZSTD_getParams(ZSTD_DEF_LEVEL, 0, 0); + return zstd_get_params(ZSTD_DEF_LEVEL, 0); } static int zstd_comp_init(struct zstd_ctx *ctx) { int ret = 0; - const ZSTD_parameters params = zstd_params(); - const size_t wksp_size = ZSTD_CCtxWorkspaceBound(params.cParams); + const zstd_parameters params = zstd_params(); + const size_t wksp_size = zstd_cctx_workspace_bound(¶ms.cParams); ctx->cwksp = vzalloc(wksp_size); if (!ctx->cwksp) { @@ -41,7 +41,7 @@ static int zstd_comp_init(struct zstd_ctx *ctx) goto out; } - ctx->cctx = ZSTD_initCCtx(ctx->cwksp, wksp_size); + ctx->cctx = zstd_init_cctx(ctx->cwksp, wksp_size); if (!ctx->cctx) { ret = -EINVAL; goto out_free; @@ -56,7 +56,7 @@ static int zstd_comp_init(struct zstd_ctx *ctx) static int zstd_decomp_init(struct zstd_ctx *ctx) { int ret = 0; - const size_t wksp_size = ZSTD_DCtxWorkspaceBound(); + const size_t wksp_size = zstd_dctx_workspace_bound(); ctx->dwksp = vzalloc(wksp_size); if (!ctx->dwksp) { @@ -64,7 +64,7 @@ static int zstd_decomp_init(struct zstd_ctx *ctx) goto out; } - ctx->dctx = ZSTD_initDCtx(ctx->dwksp, wksp_size); + ctx->dctx = zstd_init_dctx(ctx->dwksp, wksp_size); if (!ctx->dctx) { ret = -EINVAL; goto out_free; @@ -152,10 +152,10 @@ static int __zstd_compress(const u8 *src, unsigned int slen, { size_t out_len; struct zstd_ctx *zctx = ctx; - const ZSTD_parameters params = zstd_params(); + const zstd_parameters params = zstd_params(); - out_len = ZSTD_compressCCtx(zctx->cctx, dst, *dlen, src, slen, params); - if (ZSTD_isError(out_len)) + out_len = zstd_compress_cctx(zctx->cctx, dst, *dlen, src, slen, ¶ms); + if (zstd_is_error(out_len)) return -EINVAL; *dlen = out_len; return 0; @@ -182,8 +182,8 @@ static int __zstd_decompress(const u8 *src, unsigned int slen, size_t out_len; struct zstd_ctx *zctx = ctx; - out_len = ZSTD_decompressDCtx(zctx->dctx, dst, *dlen, src, slen); - if (ZSTD_isError(out_len)) + out_len = zstd_decompress_dctx(zctx->dctx, dst, *dlen, src, slen); + if (zstd_is_error(out_len)) return -EINVAL; *dlen = out_len; return 0; diff --git a/drivers/acpi/arm64/gtdt.c b/drivers/acpi/arm64/gtdt.c index f2d0e5915dab..0a0a982f9c28 100644 --- a/drivers/acpi/arm64/gtdt.c +++ b/drivers/acpi/arm64/gtdt.c @@ -329,7 +329,7 @@ static int __init gtdt_import_sbsa_gwdt(struct acpi_gtdt_watchdog *wd, int index) { struct platform_device *pdev; - int irq = map_gt_gsi(wd->timer_interrupt, wd->timer_flags); + int irq; /* * According to SBSA specification the size of refresh and control @@ -338,7 +338,7 @@ static int __init gtdt_import_sbsa_gwdt(struct acpi_gtdt_watchdog *wd, struct resource res[] = { DEFINE_RES_MEM(wd->control_frame_address, SZ_4K), DEFINE_RES_MEM(wd->refresh_frame_address, SZ_4K), - DEFINE_RES_IRQ(irq), + {}, }; int nr_res = ARRAY_SIZE(res); @@ -348,10 +348,11 @@ static int __init gtdt_import_sbsa_gwdt(struct acpi_gtdt_watchdog *wd, if (!(wd->refresh_frame_address && wd->control_frame_address)) { pr_err(FW_BUG "failed to get the Watchdog base address.\n"); - acpi_unregister_gsi(wd->timer_interrupt); return -EINVAL; } + irq = map_gt_gsi(wd->timer_interrupt, wd->timer_flags); + res[2] = (struct resource)DEFINE_RES_IRQ(irq); if (irq <= 0) { pr_warn("failed to map the Watchdog interrupt.\n"); nr_res--; @@ -364,7 +365,8 @@ static int __init gtdt_import_sbsa_gwdt(struct acpi_gtdt_watchdog *wd, */ pdev = platform_device_register_simple("sbsa-gwdt", index, res, nr_res); if (IS_ERR(pdev)) { - acpi_unregister_gsi(wd->timer_interrupt); + if (irq > 0) + acpi_unregister_gsi(wd->timer_interrupt); return PTR_ERR(pdev); } diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index 69057fcd2c04..a5e6fd0bafa1 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -119,23 +119,15 @@ static DEFINE_PER_CPU(struct cpc_desc *, cpc_desc_ptr); */ #define NUM_RETRIES 500ULL -struct cppc_attr { - struct attribute attr; - ssize_t (*show)(struct kobject *kobj, - struct attribute *attr, char *buf); - ssize_t (*store)(struct kobject *kobj, - struct attribute *attr, const char *c, ssize_t count); -}; - #define define_one_cppc_ro(_name) \ -static struct cppc_attr _name = \ +static struct kobj_attribute _name = \ __ATTR(_name, 0444, show_##_name, NULL) #define to_cpc_desc(a) container_of(a, struct cpc_desc, kobj) #define show_cppc_data(access_fn, struct_name, member_name) \ static ssize_t show_##member_name(struct kobject *kobj, \ - struct attribute *attr, char *buf) \ + struct kobj_attribute *attr, char *buf) \ { \ struct cpc_desc *cpc_ptr = to_cpc_desc(kobj); \ struct struct_name st_name = {0}; \ @@ -161,7 +153,7 @@ show_cppc_data(cppc_get_perf_ctrs, cppc_perf_fb_ctrs, reference_perf); show_cppc_data(cppc_get_perf_ctrs, cppc_perf_fb_ctrs, wraparound_time); static ssize_t show_feedback_ctrs(struct kobject *kobj, - struct attribute *attr, char *buf) + struct kobj_attribute *attr, char *buf) { struct cpc_desc *cpc_ptr = to_cpc_desc(kobj); struct cppc_perf_fb_ctrs fb_ctrs = {0}; diff --git a/drivers/acpi/custom_method.c b/drivers/acpi/custom_method.c index 7b54dc95d36b..4058e0241091 100644 --- a/drivers/acpi/custom_method.c +++ b/drivers/acpi/custom_method.c @@ -42,6 +42,8 @@ static ssize_t cm_write(struct file *file, const char __user * user_buf, sizeof(struct acpi_table_header))) return -EFAULT; uncopied_bytes = max_size = table.length; + /* make sure the buf is not allocated */ + kfree(buf); buf = kzalloc(max_size, GFP_KERNEL); if (!buf) return -ENOMEM; @@ -55,6 +57,7 @@ static ssize_t cm_write(struct file *file, const char __user * user_buf, (*ppos + count < count) || (count > uncopied_bytes)) { kfree(buf); + buf = NULL; return -EINVAL; } @@ -76,7 +79,6 @@ static ssize_t cm_write(struct file *file, const char __user * user_buf, add_taint(TAINT_OVERRIDDEN_ACPI_TABLE, LOCKDEP_NOW_UNRELIABLE); } - kfree(buf); return count; } diff --git a/drivers/acpi/device_pm.c b/drivers/acpi/device_pm.c index 096153761ebc..58876248b192 100644 --- a/drivers/acpi/device_pm.c +++ b/drivers/acpi/device_pm.c @@ -1310,6 +1310,7 @@ int acpi_dev_pm_attach(struct device *dev, bool power_on) {"PNP0C0B", }, /* Generic ACPI fan */ {"INT3404", }, /* Fan */ {"INTC1044", }, /* Fan for Tiger Lake generation */ + {"INTC1048", }, /* Fan for Alder Lake generation */ {} }; struct acpi_device *adev = ACPI_COMPANION(dev); diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c index 6efe7edd7b1e..345777bf7af9 100644 --- a/drivers/acpi/scan.c +++ b/drivers/acpi/scan.c @@ -701,6 +701,7 @@ int acpi_device_add(struct acpi_device *device, result = acpi_device_set_name(device, acpi_device_bus_id); if (result) { + kfree_const(acpi_device_bus_id->bus_id); kfree(acpi_device_bus_id); goto err_unlock; } diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index 00ba8e5a1ccc..33192a8f687d 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -1772,6 +1772,11 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) hpriv->flags |= AHCI_HFLAG_NO_DEVSLP; #ifdef CONFIG_ARM64 + if (pdev->vendor == PCI_VENDOR_ID_HUAWEI && + pdev->device == 0xa235 && + pdev->revision < 0x30) + hpriv->flags |= AHCI_HFLAG_NO_SXS; + if (pdev->vendor == 0x177d && pdev->device == 0xa01c) hpriv->irq_handler = ahci_thunderx_irq_handler; #endif diff --git a/drivers/ata/ahci.h b/drivers/ata/ahci.h index 98b8baa47dc5..d1f284f0c83d 100644 --- a/drivers/ata/ahci.h +++ b/drivers/ata/ahci.h @@ -242,6 +242,7 @@ enum { suspend/resume */ AHCI_HFLAG_IGN_NOTSUPP_POWER_ON = (1 << 27), /* ignore -EOPNOTSUPP from phy_power_on() */ + AHCI_HFLAG_NO_SXS = (1 << 28), /* SXS not supported */ /* ap->flags bits */ diff --git a/drivers/ata/ahci_brcm.c b/drivers/ata/ahci_brcm.c index 5b32df5d33ad..6e9c5ade4c2e 100644 --- a/drivers/ata/ahci_brcm.c +++ b/drivers/ata/ahci_brcm.c @@ -86,7 +86,8 @@ struct brcm_ahci_priv { u32 port_mask; u32 quirks; enum brcm_ahci_version version; - struct reset_control *rcdev; + struct reset_control *rcdev_rescal; + struct reset_control *rcdev_ahci; }; static inline u32 brcm_sata_readreg(void __iomem *addr) @@ -352,8 +353,8 @@ static int brcm_ahci_suspend(struct device *dev) else ret = 0; - if (priv->version != BRCM_SATA_BCM7216) - reset_control_assert(priv->rcdev); + reset_control_assert(priv->rcdev_ahci); + reset_control_rearm(priv->rcdev_rescal); return ret; } @@ -365,10 +366,10 @@ static int __maybe_unused brcm_ahci_resume(struct device *dev) struct brcm_ahci_priv *priv = hpriv->plat_data; int ret = 0; - if (priv->version == BRCM_SATA_BCM7216) - ret = reset_control_reset(priv->rcdev); - else - ret = reset_control_deassert(priv->rcdev); + ret = reset_control_deassert(priv->rcdev_ahci); + if (ret) + return ret; + ret = reset_control_reset(priv->rcdev_rescal); if (ret) return ret; @@ -434,7 +435,6 @@ static int brcm_ahci_probe(struct platform_device *pdev) { const struct of_device_id *of_id; struct device *dev = &pdev->dev; - const char *reset_name = NULL; struct brcm_ahci_priv *priv; struct ahci_host_priv *hpriv; struct resource *res; @@ -456,15 +456,15 @@ static int brcm_ahci_probe(struct platform_device *pdev) if (IS_ERR(priv->top_ctrl)) return PTR_ERR(priv->top_ctrl); - /* Reset is optional depending on platform and named differently */ - if (priv->version == BRCM_SATA_BCM7216) - reset_name = "rescal"; - else - reset_name = "ahci"; - - priv->rcdev = devm_reset_control_get_optional(&pdev->dev, reset_name); - if (IS_ERR(priv->rcdev)) - return PTR_ERR(priv->rcdev); + if (priv->version == BRCM_SATA_BCM7216) { + priv->rcdev_rescal = devm_reset_control_get_optional_shared( + &pdev->dev, "rescal"); + if (IS_ERR(priv->rcdev_rescal)) + return PTR_ERR(priv->rcdev_rescal); + } + priv->rcdev_ahci = devm_reset_control_get_optional(&pdev->dev, "ahci"); + if (IS_ERR(priv->rcdev_ahci)) + return PTR_ERR(priv->rcdev_ahci); hpriv = ahci_platform_get_resources(pdev, 0); if (IS_ERR(hpriv)) @@ -485,10 +485,10 @@ static int brcm_ahci_probe(struct platform_device *pdev) break; } - if (priv->version == BRCM_SATA_BCM7216) - ret = reset_control_reset(priv->rcdev); - else - ret = reset_control_deassert(priv->rcdev); + ret = reset_control_reset(priv->rcdev_rescal); + if (ret) + return ret; + ret = reset_control_deassert(priv->rcdev_ahci); if (ret) return ret; @@ -539,8 +539,8 @@ static int brcm_ahci_probe(struct platform_device *pdev) out_disable_clks: ahci_platform_disable_clks(hpriv); out_reset: - if (priv->version != BRCM_SATA_BCM7216) - reset_control_assert(priv->rcdev); + reset_control_assert(priv->rcdev_ahci); + reset_control_rearm(priv->rcdev_rescal); return ret; } diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c index ea5bf5f4cbed..fec2e9754aed 100644 --- a/drivers/ata/libahci.c +++ b/drivers/ata/libahci.c @@ -493,6 +493,11 @@ void ahci_save_initial_config(struct device *dev, struct ahci_host_priv *hpriv) cap |= HOST_CAP_ALPM; } + if ((cap & HOST_CAP_SXS) && (hpriv->flags & AHCI_HFLAG_NO_SXS)) { + dev_info(dev, "controller does not support SXS, disabling CAP_SXS\n"); + cap &= ~HOST_CAP_SXS; + } + if (hpriv->force_port_map && port_map != hpriv->force_port_map) { dev_info(dev, "forcing port_map 0x%x -> 0x%x\n", port_map, hpriv->force_port_map); diff --git a/drivers/ata/libahci_platform.c b/drivers/ata/libahci_platform.c index de638dafce21..b2f552088291 100644 --- a/drivers/ata/libahci_platform.c +++ b/drivers/ata/libahci_platform.c @@ -582,11 +582,13 @@ int ahci_platform_init_host(struct platform_device *pdev, int i, irq, n_ports, rc; irq = platform_get_irq(pdev, 0); - if (irq <= 0) { + if (irq < 0) { if (irq != -EPROBE_DEFER) dev_err(dev, "no irq\n"); return irq; } + if (!irq) + return -EINVAL; hpriv->irq = irq; diff --git a/drivers/ata/pata_arasan_cf.c b/drivers/ata/pata_arasan_cf.c index e9cf31f38450..63f39440a9b4 100644 --- a/drivers/ata/pata_arasan_cf.c +++ b/drivers/ata/pata_arasan_cf.c @@ -818,12 +818,19 @@ static int arasan_cf_probe(struct platform_device *pdev) else quirk = CF_BROKEN_UDMA; /* as it is on spear1340 */ - /* if irq is 0, support only PIO */ - acdev->irq = platform_get_irq(pdev, 0); - if (acdev->irq) + /* + * If there's an error getting IRQ (or we do get IRQ0), + * support only PIO + */ + ret = platform_get_irq(pdev, 0); + if (ret > 0) { + acdev->irq = ret; irq_handler = arasan_cf_interrupt; - else + } else if (ret == -EPROBE_DEFER) { + return ret; + } else { quirk |= CF_BROKEN_MWDMA | CF_BROKEN_UDMA; + } acdev->pbase = res->start; acdev->vbase = devm_ioremap(&pdev->dev, res->start, diff --git a/drivers/ata/pata_ixp4xx_cf.c b/drivers/ata/pata_ixp4xx_cf.c index d1644a8ef9fa..abc0e87ca1a8 100644 --- a/drivers/ata/pata_ixp4xx_cf.c +++ b/drivers/ata/pata_ixp4xx_cf.c @@ -165,8 +165,12 @@ static int ixp4xx_pata_probe(struct platform_device *pdev) return -ENOMEM; irq = platform_get_irq(pdev, 0); - if (irq) + if (irq > 0) irq_set_irq_type(irq, IRQ_TYPE_EDGE_RISING); + else if (irq < 0) + return irq; + else + return -EINVAL; /* Setup expansion bus chip selects */ *data->cs0_cfg = data->cs0_bits; diff --git a/drivers/ata/sata_mv.c b/drivers/ata/sata_mv.c index 664ef658a955..b62446ea5f40 100644 --- a/drivers/ata/sata_mv.c +++ b/drivers/ata/sata_mv.c @@ -4097,6 +4097,10 @@ static int mv_platform_probe(struct platform_device *pdev) n_ports = mv_platform_data->n_ports; irq = platform_get_irq(pdev, 0); } + if (irq < 0) + return irq; + if (!irq) + return -EINVAL; host = ata_host_alloc_pinfo(&pdev->dev, ppi, n_ports); hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL); diff --git a/drivers/base/devtmpfs.c b/drivers/base/devtmpfs.c index 653c8c6ac7a7..aedeb2dc1a18 100644 --- a/drivers/base/devtmpfs.c +++ b/drivers/base/devtmpfs.c @@ -419,7 +419,6 @@ static int __init devtmpfs_setup(void *p) init_chroot("."); out: *(int *)p = err; - complete(&setup_done); return err; } @@ -432,6 +431,7 @@ static int __ref devtmpfsd(void *p) { int err = devtmpfs_setup(p); + complete(&setup_done); if (err) return err; devtmpfs_work_loop(); diff --git a/drivers/base/firmware_loader/main.c b/drivers/base/firmware_loader/main.c index 78355095e00d..4fdb8219cd08 100644 --- a/drivers/base/firmware_loader/main.c +++ b/drivers/base/firmware_loader/main.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -504,6 +505,7 @@ fw_get_filesystem_firmware(struct device *device, struct fw_priv *fw_priv, if (!path) return -ENOMEM; + wait_for_initramfs(); for (i = 0; i < ARRAY_SIZE(fw_path); i++) { size_t file_size = 0; size_t *file_size_ptr = NULL; diff --git a/drivers/base/node.c b/drivers/base/node.c index f449dbb2c746..2c36f61d30bc 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -268,21 +268,20 @@ static void node_init_cache_dev(struct node *node) if (!dev) return; + device_initialize(dev); dev->parent = &node->dev; dev->release = node_cache_release; if (dev_set_name(dev, "memory_side_cache")) - goto free_dev; + goto put_device; - if (device_register(dev)) - goto free_name; + if (device_add(dev)) + goto put_device; pm_runtime_no_callbacks(dev); node->cache_dev = dev; return; -free_name: - kfree_const(dev->kobj.name); -free_dev: - kfree(dev); +put_device: + put_device(dev); } /** @@ -319,25 +318,24 @@ void node_add_cache(unsigned int nid, struct node_cache_attrs *cache_attrs) return; dev = &info->dev; + device_initialize(dev); dev->parent = node->cache_dev; dev->release = node_cacheinfo_release; dev->groups = cache_groups; if (dev_set_name(dev, "index%d", cache_attrs->level)) - goto free_cache; + goto put_device; info->cache_attrs = *cache_attrs; - if (device_register(dev)) { + if (device_add(dev)) { dev_warn(&node->dev, "failed to add cache level:%d\n", cache_attrs->level); - goto free_name; + goto put_device; } pm_runtime_no_callbacks(dev); list_add_tail(&info->node, &node->cache_attrs); return; -free_name: - kfree_const(dev->kobj.name); -free_cache: - kfree(info); +put_device: + put_device(dev); } static void node_remove_caches(struct node *node) diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c index fe1dad68aee4..ae011f2bc537 100644 --- a/drivers/base/power/runtime.c +++ b/drivers/base/power/runtime.c @@ -1637,6 +1637,7 @@ void pm_runtime_init(struct device *dev) dev->power.request_pending = false; dev->power.request = RPM_REQ_NONE; dev->power.deferred_resume = false; + dev->power.needs_force_resume = 0; INIT_WORK(&dev->power.work, pm_runtime_work); dev->power.timer_expires = 0; @@ -1804,10 +1805,12 @@ int pm_runtime_force_suspend(struct device *dev) * its parent, but set its status to RPM_SUSPENDED anyway in case this * function will be called again for it in the meantime. */ - if (pm_runtime_need_not_resume(dev)) + if (pm_runtime_need_not_resume(dev)) { pm_runtime_set_suspended(dev); - else + } else { __update_runtime_status(dev, RPM_SUSPENDED); + dev->power.needs_force_resume = 1; + } return 0; @@ -1834,7 +1837,7 @@ int pm_runtime_force_resume(struct device *dev) int (*callback)(struct device *); int ret = 0; - if (!pm_runtime_status_suspended(dev) || pm_runtime_need_not_resume(dev)) + if (!pm_runtime_status_suspended(dev) || !dev->power.needs_force_resume) goto out; /* @@ -1853,6 +1856,7 @@ int pm_runtime_force_resume(struct device *dev) pm_runtime_mark_last_busy(dev); out: + dev->power.needs_force_resume = 0; pm_runtime_enable(dev); return ret; } diff --git a/drivers/base/regmap/regmap-debugfs.c b/drivers/base/regmap/regmap-debugfs.c index ff2ee87987c7..211a335a608d 100644 --- a/drivers/base/regmap/regmap-debugfs.c +++ b/drivers/base/regmap/regmap-debugfs.c @@ -660,6 +660,7 @@ void regmap_debugfs_exit(struct regmap *map) regmap_debugfs_free_dump_cache(map); mutex_unlock(&map->cache_lock); kfree(map->debugfs_name); + map->debugfs_name = NULL; } else { struct regmap_debugfs_node *node, *tmp; diff --git a/drivers/base/swnode.c b/drivers/base/swnode.c index fa3719ef80e4..88310ac9ce90 100644 --- a/drivers/base/swnode.c +++ b/drivers/base/swnode.c @@ -1032,6 +1032,7 @@ int device_add_software_node(struct device *dev, const struct software_node *nod } set_secondary_fwnode(dev, &swnode->fwnode); + software_node_notify(dev, KOBJ_ADD); return 0; } @@ -1105,8 +1106,8 @@ int software_node_notify(struct device *dev, unsigned long action) switch (action) { case KOBJ_ADD: - ret = sysfs_create_link(&dev->kobj, &swnode->kobj, - "software_node"); + ret = sysfs_create_link_nowarn(&dev->kobj, &swnode->kobj, + "software_node"); if (ret) break; diff --git a/drivers/block/ataflop.c b/drivers/block/ataflop.c index 104b713f4055..d601e49f80e0 100644 --- a/drivers/block/ataflop.c +++ b/drivers/block/ataflop.c @@ -729,8 +729,12 @@ static int do_format(int drive, int type, struct atari_format_descr *desc) unsigned long flags; int ret; - if (type) + if (type) { type--; + if (type >= NUM_DISK_MINORS || + minor2disktype[type].drive_types > DriveType) + return -EINVAL; + } q = unit[drive].disk[type]->queue; blk_mq_freeze_queue(q); @@ -742,11 +746,6 @@ static int do_format(int drive, int type, struct atari_format_descr *desc) local_irq_restore(flags); if (type) { - if (type >= NUM_DISK_MINORS || - minor2disktype[type].drive_types > DriveType) { - ret = -EINVAL; - goto out; - } type = minor2disktype[type].index; UDT = &atari_disk_type[type]; } @@ -2002,7 +2001,10 @@ static void ataflop_probe(dev_t dev) int drive = MINOR(dev) & 3; int type = MINOR(dev) >> 2; - if (drive >= FD_MAX_UNITS || type > NUM_DISK_MINORS) + if (type) + type--; + + if (drive >= FD_MAX_UNITS || type >= NUM_DISK_MINORS) return; mutex_lock(&ataflop_probe_lock); if (!unit[drive].disk[type]) { diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 4ff71b579cfc..974da561b8e5 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -1980,7 +1980,8 @@ static void nbd_disconnect_and_put(struct nbd_device *nbd) * config ref and try to destroy the workqueue from inside the work * queue. */ - flush_workqueue(nbd->recv_workq); + if (nbd->recv_workq) + flush_workqueue(nbd->recv_workq); if (test_and_clear_bit(NBD_RT_HAS_CONFIG_REF, &nbd->config->runtime_flags)) nbd_config_put(nbd); diff --git a/drivers/block/null_blk/zoned.c b/drivers/block/null_blk/zoned.c index bfcab1c782b5..dae54dd1aeac 100644 --- a/drivers/block/null_blk/zoned.c +++ b/drivers/block/null_blk/zoned.c @@ -180,6 +180,7 @@ int null_register_zoned_dev(struct nullb *nullb) void null_free_zoned_dev(struct nullb_device *dev) { kvfree(dev->zones); + dev->zones = NULL; } int null_report_zones(struct gendisk *disk, sector_t sector, diff --git a/drivers/block/rnbd/rnbd-clt-sysfs.c b/drivers/block/rnbd/rnbd-clt-sysfs.c index d4aa6bfc9555..49ad400a5225 100644 --- a/drivers/block/rnbd/rnbd-clt-sysfs.c +++ b/drivers/block/rnbd/rnbd-clt-sysfs.c @@ -432,10 +432,14 @@ void rnbd_clt_remove_dev_symlink(struct rnbd_clt_dev *dev) * i.e. rnbd_clt_unmap_dev_store() leading to a sysfs warning because * of sysfs link already was removed already. */ - if (dev->blk_symlink_name && try_module_get(THIS_MODULE)) { - sysfs_remove_link(rnbd_devs_kobj, dev->blk_symlink_name); + if (dev->blk_symlink_name) { + if (try_module_get(THIS_MODULE)) { + sysfs_remove_link(rnbd_devs_kobj, dev->blk_symlink_name); + module_put(THIS_MODULE); + } + /* It should be freed always. */ kfree(dev->blk_symlink_name); - module_put(THIS_MODULE); + dev->blk_symlink_name = NULL; } } @@ -479,11 +483,7 @@ static int rnbd_clt_get_path_name(struct rnbd_clt_dev *dev, char *buf, while ((s = strchr(pathname, '/'))) s[0] = '!'; - ret = snprintf(buf, len, "%s", pathname); - if (ret >= len) - return -ENAMETOOLONG; - - ret = snprintf(buf, len, "%s@%s", buf, dev->sess->sessname); + ret = snprintf(buf, len, "%s@%s", pathname, dev->sess->sessname); if (ret >= len) return -ENAMETOOLONG; diff --git a/drivers/block/rnbd/rnbd-clt.c b/drivers/block/rnbd/rnbd-clt.c index 45a470076652..5ab7319ff2ea 100644 --- a/drivers/block/rnbd/rnbd-clt.c +++ b/drivers/block/rnbd/rnbd-clt.c @@ -693,7 +693,11 @@ static void remap_devs(struct rnbd_clt_session *sess) return; } - rtrs_clt_query(sess->rtrs, &attrs); + err = rtrs_clt_query(sess->rtrs, &attrs); + if (err) { + pr_err("rtrs_clt_query(\"%s\"): %d\n", sess->sessname, err); + return; + } mutex_lock(&sess->lock); sess->max_io_size = attrs.max_io_size; @@ -1234,7 +1238,11 @@ find_and_get_or_create_sess(const char *sessname, err = PTR_ERR(sess->rtrs); goto wake_up_and_put; } - rtrs_clt_query(sess->rtrs, &attrs); + + err = rtrs_clt_query(sess->rtrs, &attrs); + if (err) + goto close_rtrs; + sess->max_io_size = attrs.max_io_size; sess->queue_depth = attrs.queue_depth; diff --git a/drivers/block/rnbd/rnbd-clt.h b/drivers/block/rnbd/rnbd-clt.h index 537d499dad3b..73d980840531 100644 --- a/drivers/block/rnbd/rnbd-clt.h +++ b/drivers/block/rnbd/rnbd-clt.h @@ -87,7 +87,7 @@ struct rnbd_clt_session { DECLARE_BITMAP(cpu_queues_bm, NR_CPUS); int __percpu *cpu_rr; /* per-cpu var for CPU round-robin */ atomic_t busy; - int queue_depth; + size_t queue_depth; u32 max_io_size; struct blk_mq_tag_set tag_set; struct mutex lock; /* protects state and devs_list */ diff --git a/drivers/block/rnbd/rnbd-srv.c b/drivers/block/rnbd/rnbd-srv.c index a6a68d44f517..677770f32843 100644 --- a/drivers/block/rnbd/rnbd-srv.c +++ b/drivers/block/rnbd/rnbd-srv.c @@ -341,7 +341,9 @@ void rnbd_srv_sess_dev_force_close(struct rnbd_srv_sess_dev *sess_dev) struct rnbd_srv_session *sess = sess_dev->sess; sess_dev->keep_id = true; - mutex_lock(&sess->lock); + /* It is already started to close by client's close message. */ + if (!mutex_trylock(&sess->lock)) + return; rnbd_srv_destroy_dev_session_sysfs(sess_dev); mutex_unlock(&sess->lock); } diff --git a/drivers/block/xen-blkback/common.h b/drivers/block/xen-blkback/common.h index b0c71d3a81a0..bda5c815e441 100644 --- a/drivers/block/xen-blkback/common.h +++ b/drivers/block/xen-blkback/common.h @@ -313,6 +313,7 @@ struct xen_blkif { struct work_struct free_work; unsigned int nr_ring_pages; + bool multi_ref; /* All rings for this device. */ struct xen_blkif_ring *rings; unsigned int nr_rings; diff --git a/drivers/block/xen-blkback/xenbus.c b/drivers/block/xen-blkback/xenbus.c index c2aaf690352c..125b22205d38 100644 --- a/drivers/block/xen-blkback/xenbus.c +++ b/drivers/block/xen-blkback/xenbus.c @@ -998,14 +998,17 @@ static int read_per_ring_refs(struct xen_blkif_ring *ring, const char *dir) for (i = 0; i < nr_grefs; i++) { char ring_ref_name[RINGREF_NAME_LEN]; - snprintf(ring_ref_name, RINGREF_NAME_LEN, "ring-ref%u", i); + if (blkif->multi_ref) + snprintf(ring_ref_name, RINGREF_NAME_LEN, "ring-ref%u", i); + else { + WARN_ON(i != 0); + snprintf(ring_ref_name, RINGREF_NAME_LEN, "ring-ref"); + } + err = xenbus_scanf(XBT_NIL, dir, ring_ref_name, "%u", &ring_ref[i]); if (err != 1) { - if (nr_grefs == 1) - break; - err = -EINVAL; xenbus_dev_fatal(dev, err, "reading %s/%s", dir, ring_ref_name); @@ -1013,18 +1016,6 @@ static int read_per_ring_refs(struct xen_blkif_ring *ring, const char *dir) } } - if (err != 1) { - WARN_ON(nr_grefs != 1); - - err = xenbus_scanf(XBT_NIL, dir, "ring-ref", "%u", - &ring_ref[0]); - if (err != 1) { - err = -EINVAL; - xenbus_dev_fatal(dev, err, "reading %s/ring-ref", dir); - return err; - } - } - err = -ENOMEM; for (i = 0; i < nr_grefs * XEN_BLKIF_REQS_PER_PAGE; i++) { req = kzalloc(sizeof(*req), GFP_KERNEL); @@ -1129,10 +1120,15 @@ static int connect_ring(struct backend_info *be) blkif->nr_rings, blkif->blk_protocol, protocol, blkif->vbd.feature_gnt_persistent ? "persistent grants" : ""); - ring_page_order = xenbus_read_unsigned(dev->otherend, - "ring-page-order", 0); - - if (ring_page_order > xen_blkif_max_ring_order) { + err = xenbus_scanf(XBT_NIL, dev->otherend, "ring-page-order", "%u", + &ring_page_order); + if (err != 1) { + blkif->nr_ring_pages = 1; + blkif->multi_ref = false; + } else if (ring_page_order <= xen_blkif_max_ring_order) { + blkif->nr_ring_pages = 1 << ring_page_order; + blkif->multi_ref = true; + } else { err = -EINVAL; xenbus_dev_fatal(dev, err, "requested ring page order %d exceed max:%d", @@ -1141,8 +1137,6 @@ static int connect_ring(struct backend_info *be) return err; } - blkif->nr_ring_pages = 1 << ring_page_order; - if (blkif->nr_rings == 1) return read_per_ring_refs(&blkif->rings[0], dev->otherend); else { diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c index 5cbfbd948f67..4a901508e48e 100644 --- a/drivers/bluetooth/btusb.c +++ b/drivers/bluetooth/btusb.c @@ -399,7 +399,9 @@ static const struct usb_device_id blacklist_table[] = { /* MediaTek Bluetooth devices */ { USB_VENDOR_AND_INTERFACE_INFO(0x0e8d, 0xe0, 0x01, 0x01), - .driver_info = BTUSB_MEDIATEK }, + .driver_info = BTUSB_MEDIATEK | + BTUSB_WIDEBAND_SPEECH | + BTUSB_VALID_LE_STATES }, /* Additional MediaTek MT7615E Bluetooth devices */ { USB_DEVICE(0x13d3, 0x3560), .driver_info = BTUSB_MEDIATEK}, diff --git a/drivers/bus/mhi/core/boot.c b/drivers/bus/mhi/core/boot.c index c2546bf229fb..08c28740dc4e 100644 --- a/drivers/bus/mhi/core/boot.c +++ b/drivers/bus/mhi/core/boot.c @@ -389,7 +389,6 @@ static void mhi_firmware_copy(struct mhi_controller *mhi_cntrl, void mhi_fw_load_handler(struct mhi_controller *mhi_cntrl) { const struct firmware *firmware = NULL; - struct image_info *image_info; struct device *dev = &mhi_cntrl->mhi_dev->dev; const char *fw_name; void *buf; @@ -491,44 +490,42 @@ void mhi_fw_load_handler(struct mhi_controller *mhi_cntrl) fw_load_ee_pthru: /* Transitioning into MHI RESET->READY state */ ret = mhi_ready_state_transition(mhi_cntrl); - - if (!mhi_cntrl->fbc_download) - return; - if (ret) { dev_err(dev, "MHI did not enter READY state\n"); goto error_ready_state; } - /* Wait for the SBL event */ - ret = wait_event_timeout(mhi_cntrl->state_event, - mhi_cntrl->ee == MHI_EE_SBL || - MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state), - msecs_to_jiffies(mhi_cntrl->timeout_ms)); + dev_info(dev, "Wait for device to enter SBL or Mission mode\n"); + return; - if (!ret || MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) { - dev_err(dev, "MHI did not enter SBL\n"); - goto error_ready_state; +error_ready_state: + if (mhi_cntrl->fbc_download) { + mhi_free_bhie_table(mhi_cntrl, mhi_cntrl->fbc_image); + mhi_cntrl->fbc_image = NULL; } - /* Start full firmware image download */ - image_info = mhi_cntrl->fbc_image; +error_fw_load: + mhi_cntrl->pm_state = MHI_PM_FW_DL_ERR; + wake_up_all(&mhi_cntrl->state_event); +} + +int mhi_download_amss_image(struct mhi_controller *mhi_cntrl) +{ + struct image_info *image_info = mhi_cntrl->fbc_image; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + int ret; + + if (!image_info) + return -EIO; + ret = mhi_fw_load_bhie(mhi_cntrl, /* Vector table is the last entry */ &image_info->mhi_buf[image_info->entries - 1]); if (ret) { - dev_err(dev, "MHI did not load image over BHIe, ret: %d\n", - ret); - goto error_fw_load; + dev_err(dev, "MHI did not load AMSS, ret:%d\n", ret); + mhi_cntrl->pm_state = MHI_PM_FW_DL_ERR; + wake_up_all(&mhi_cntrl->state_event); } - return; - -error_ready_state: - mhi_free_bhie_table(mhi_cntrl, mhi_cntrl->fbc_image); - mhi_cntrl->fbc_image = NULL; - -error_fw_load: - mhi_cntrl->pm_state = MHI_PM_FW_DL_ERR; - wake_up_all(&mhi_cntrl->state_event); + return ret; } diff --git a/drivers/bus/mhi/core/init.c b/drivers/bus/mhi/core/init.c index be4eebb0971b..08b7f4a06bfc 100644 --- a/drivers/bus/mhi/core/init.c +++ b/drivers/bus/mhi/core/init.c @@ -508,8 +508,6 @@ int mhi_init_mmio(struct mhi_controller *mhi_cntrl) /* Setup wake db */ mhi_cntrl->wake_db = base + val + (8 * MHI_DEV_WAKE_DB); - mhi_write_reg(mhi_cntrl, mhi_cntrl->wake_db, 4, 0); - mhi_write_reg(mhi_cntrl, mhi_cntrl->wake_db, 0, 0); mhi_cntrl->wake_set = false; /* Setup channel db address for each channel in tre_ring */ @@ -552,6 +550,7 @@ void mhi_deinit_chan_ctxt(struct mhi_controller *mhi_cntrl, struct mhi_ring *buf_ring; struct mhi_ring *tre_ring; struct mhi_chan_ctxt *chan_ctxt; + u32 tmp; buf_ring = &mhi_chan->buf_ring; tre_ring = &mhi_chan->tre_ring; @@ -565,7 +564,19 @@ void mhi_deinit_chan_ctxt(struct mhi_controller *mhi_cntrl, vfree(buf_ring->base); buf_ring->base = tre_ring->base = NULL; + tre_ring->ctxt_wp = NULL; chan_ctxt->rbase = 0; + chan_ctxt->rlen = 0; + chan_ctxt->rp = 0; + chan_ctxt->wp = 0; + + tmp = chan_ctxt->chcfg; + tmp &= ~CHAN_CTX_CHSTATE_MASK; + tmp |= (MHI_CH_STATE_DISABLED << CHAN_CTX_CHSTATE_SHIFT); + chan_ctxt->chcfg = tmp; + + /* Update to all cores */ + smp_wmb(); } int mhi_init_chan_ctxt(struct mhi_controller *mhi_cntrl, @@ -863,12 +874,10 @@ int mhi_register_controller(struct mhi_controller *mhi_cntrl, u32 soc_info; int ret, i; - if (!mhi_cntrl) - return -EINVAL; - - if (!mhi_cntrl->runtime_get || !mhi_cntrl->runtime_put || + if (!mhi_cntrl || !mhi_cntrl->cntrl_dev || !mhi_cntrl->regs || + !mhi_cntrl->runtime_get || !mhi_cntrl->runtime_put || !mhi_cntrl->status_cb || !mhi_cntrl->read_reg || - !mhi_cntrl->write_reg || !mhi_cntrl->nr_irqs) + !mhi_cntrl->write_reg || !mhi_cntrl->nr_irqs || !mhi_cntrl->irq) return -EINVAL; ret = parse_config(mhi_cntrl, config); @@ -890,8 +899,7 @@ int mhi_register_controller(struct mhi_controller *mhi_cntrl, INIT_WORK(&mhi_cntrl->st_worker, mhi_pm_st_worker); init_waitqueue_head(&mhi_cntrl->state_event); - mhi_cntrl->hiprio_wq = alloc_ordered_workqueue - ("mhi_hiprio_wq", WQ_MEM_RECLAIM | WQ_HIGHPRI); + mhi_cntrl->hiprio_wq = alloc_ordered_workqueue("mhi_hiprio_wq", WQ_HIGHPRI); if (!mhi_cntrl->hiprio_wq) { dev_err(mhi_cntrl->cntrl_dev, "Failed to allocate workqueue\n"); ret = -ENOMEM; @@ -1296,7 +1304,8 @@ static int mhi_driver_remove(struct device *dev) mutex_lock(&mhi_chan->mutex); - if (ch_state[dir] == MHI_CH_STATE_ENABLED && + if ((ch_state[dir] == MHI_CH_STATE_ENABLED || + ch_state[dir] == MHI_CH_STATE_STOP) && !mhi_chan->offload_ch) mhi_deinit_chan_ctxt(mhi_cntrl, mhi_chan); diff --git a/drivers/bus/mhi/core/internal.h b/drivers/bus/mhi/core/internal.h index 6f80ec30c0cd..6f37439e5247 100644 --- a/drivers/bus/mhi/core/internal.h +++ b/drivers/bus/mhi/core/internal.h @@ -619,6 +619,7 @@ int mhi_pm_m3_transition(struct mhi_controller *mhi_cntrl); int __mhi_device_get_sync(struct mhi_controller *mhi_cntrl); int mhi_send_cmd(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan, enum mhi_cmd_type cmd); +int mhi_download_amss_image(struct mhi_controller *mhi_cntrl); static inline bool mhi_is_active(struct mhi_controller *mhi_cntrl) { return (mhi_cntrl->dev_state >= MHI_STATE_M0 && diff --git a/drivers/bus/mhi/core/main.c b/drivers/bus/mhi/core/main.c index 4e0131b94056..61c37b23dd71 100644 --- a/drivers/bus/mhi/core/main.c +++ b/drivers/bus/mhi/core/main.c @@ -242,10 +242,17 @@ static void mhi_del_ring_element(struct mhi_controller *mhi_cntrl, smp_wmb(); } +static bool is_valid_ring_ptr(struct mhi_ring *ring, dma_addr_t addr) +{ + return addr >= ring->iommu_base && addr < ring->iommu_base + ring->len; +} + int mhi_destroy_device(struct device *dev, void *data) { + struct mhi_chan *ul_chan, *dl_chan; struct mhi_device *mhi_dev; struct mhi_controller *mhi_cntrl; + enum mhi_ee_type ee = MHI_EE_MAX; if (dev->bus != &mhi_bus_type) return 0; @@ -257,6 +264,17 @@ int mhi_destroy_device(struct device *dev, void *data) if (mhi_dev->dev_type == MHI_DEVICE_CONTROLLER) return 0; + ul_chan = mhi_dev->ul_chan; + dl_chan = mhi_dev->dl_chan; + + /* + * If execution environment is specified, remove only those devices that + * started in them based on ee_mask for the channels as we move on to a + * different execution environment + */ + if (data) + ee = *(enum mhi_ee_type *)data; + /* * For the suspend and resume case, this function will get called * without mhi_unregister_controller(). Hence, we need to drop the @@ -264,11 +282,19 @@ int mhi_destroy_device(struct device *dev, void *data) * be sure that there will be no instances of mhi_dev left after * this. */ - if (mhi_dev->ul_chan) - put_device(&mhi_dev->ul_chan->mhi_dev->dev); + if (ul_chan) { + if (ee != MHI_EE_MAX && !(ul_chan->ee_mask & BIT(ee))) + return 0; - if (mhi_dev->dl_chan) - put_device(&mhi_dev->dl_chan->mhi_dev->dev); + put_device(&ul_chan->mhi_dev->dev); + } + + if (dl_chan) { + if (ee != MHI_EE_MAX && !(dl_chan->ee_mask & BIT(ee))) + return 0; + + put_device(&dl_chan->mhi_dev->dev); + } dev_dbg(&mhi_cntrl->mhi_dev->dev, "destroy device for chan:%s\n", mhi_dev->name); @@ -383,7 +409,16 @@ irqreturn_t mhi_irq_handler(int irq_number, void *dev) struct mhi_event_ctxt *er_ctxt = &mhi_cntrl->mhi_ctxt->er_ctxt[mhi_event->er_index]; struct mhi_ring *ev_ring = &mhi_event->ring; - void *dev_rp = mhi_to_virtual(ev_ring, er_ctxt->rp); + dma_addr_t ptr = er_ctxt->rp; + void *dev_rp; + + if (!is_valid_ring_ptr(ev_ring, ptr)) { + dev_err(&mhi_cntrl->mhi_dev->dev, + "Event ring rp points outside of the event ring\n"); + return IRQ_HANDLED; + } + + dev_rp = mhi_to_virtual(ev_ring, ptr); /* Only proceed if event ring has pending events */ if (ev_ring->rp == dev_rp) @@ -409,7 +444,7 @@ irqreturn_t mhi_intvec_threaded_handler(int irq_number, void *priv) struct device *dev = &mhi_cntrl->mhi_dev->dev; enum mhi_state state = MHI_STATE_MAX; enum mhi_pm_state pm_state = 0; - enum mhi_ee_type ee = 0; + enum mhi_ee_type ee = MHI_EE_MAX; write_lock_irq(&mhi_cntrl->pm_lock); if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) { @@ -418,8 +453,7 @@ irqreturn_t mhi_intvec_threaded_handler(int irq_number, void *priv) } state = mhi_get_mhi_state(mhi_cntrl); - ee = mhi_cntrl->ee; - mhi_cntrl->ee = mhi_get_exec_env(mhi_cntrl); + ee = mhi_get_exec_env(mhi_cntrl); dev_dbg(dev, "local ee:%s device ee:%s dev_state:%s\n", TO_MHI_EXEC_STR(mhi_cntrl->ee), TO_MHI_EXEC_STR(ee), TO_MHI_STATE_STR(state)); @@ -431,27 +465,30 @@ irqreturn_t mhi_intvec_threaded_handler(int irq_number, void *priv) } write_unlock_irq(&mhi_cntrl->pm_lock); - /* If device supports RDDM don't bother processing SYS error */ - if (mhi_cntrl->rddm_image) { - /* host may be performing a device power down already */ - if (!mhi_is_active(mhi_cntrl)) - goto exit_intvec; + if (pm_state != MHI_PM_SYS_ERR_DETECT || ee == mhi_cntrl->ee) + goto exit_intvec; - if (mhi_cntrl->ee == MHI_EE_RDDM && mhi_cntrl->ee != ee) { + switch (ee) { + case MHI_EE_RDDM: + /* proceed if power down is not already in progress */ + if (mhi_cntrl->rddm_image && mhi_is_active(mhi_cntrl)) { mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_EE_RDDM); + mhi_cntrl->ee = ee; wake_up_all(&mhi_cntrl->state_event); } - goto exit_intvec; - } - - if (pm_state == MHI_PM_SYS_ERR_DETECT) { + break; + case MHI_EE_PBL: + case MHI_EE_EDL: + case MHI_EE_PTHRU: + mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_FATAL_ERROR); + mhi_cntrl->ee = ee; wake_up_all(&mhi_cntrl->state_event); - - /* For fatal errors, we let controller decide next step */ - if (MHI_IN_PBL(ee)) - mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_FATAL_ERROR); - else - mhi_pm_sys_err_handler(mhi_cntrl); + mhi_pm_sys_err_handler(mhi_cntrl); + break; + default: + wake_up_all(&mhi_cntrl->state_event); + mhi_pm_sys_err_handler(mhi_cntrl); + break; } exit_intvec: @@ -536,6 +573,11 @@ static int parse_xfer_event(struct mhi_controller *mhi_cntrl, struct mhi_buf_info *buf_info; u16 xfer_len; + if (!is_valid_ring_ptr(tre_ring, ptr)) { + dev_err(&mhi_cntrl->mhi_dev->dev, + "Event element points outside of the tre ring\n"); + break; + } /* Get the TRB this event points to */ ev_tre = mhi_to_virtual(tre_ring, ptr); @@ -570,8 +612,11 @@ static int parse_xfer_event(struct mhi_controller *mhi_cntrl, /* notify client */ mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result); - if (mhi_chan->dir == DMA_TO_DEVICE) + if (mhi_chan->dir == DMA_TO_DEVICE) { atomic_dec(&mhi_cntrl->pending_pkts); + /* Release the reference got from mhi_queue() */ + mhi_cntrl->runtime_put(mhi_cntrl); + } /* * Recycle the buffer if buffer is pre-allocated, @@ -695,6 +740,12 @@ static void mhi_process_cmd_completion(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan; u32 chan; + if (!is_valid_ring_ptr(mhi_ring, ptr)) { + dev_err(&mhi_cntrl->mhi_dev->dev, + "Event element points outside of the cmd ring\n"); + return; + } + cmd_pkt = mhi_to_virtual(mhi_ring, ptr); chan = MHI_TRE_GET_CMD_CHID(cmd_pkt); @@ -719,6 +770,7 @@ int mhi_process_ctrl_ev_ring(struct mhi_controller *mhi_cntrl, struct device *dev = &mhi_cntrl->mhi_dev->dev; u32 chan; int count = 0; + dma_addr_t ptr = er_ctxt->rp; /* * This is a quick check to avoid unnecessary event processing @@ -728,7 +780,13 @@ int mhi_process_ctrl_ev_ring(struct mhi_controller *mhi_cntrl, if (unlikely(MHI_EVENT_ACCESS_INVALID(mhi_cntrl->pm_state))) return -EIO; - dev_rp = mhi_to_virtual(ev_ring, er_ctxt->rp); + if (!is_valid_ring_ptr(ev_ring, ptr)) { + dev_err(&mhi_cntrl->mhi_dev->dev, + "Event ring rp points outside of the event ring\n"); + return -EIO; + } + + dev_rp = mhi_to_virtual(ev_ring, ptr); local_rp = ev_ring->rp; while (dev_rp != local_rp) { @@ -834,6 +892,8 @@ int mhi_process_ctrl_ev_ring(struct mhi_controller *mhi_cntrl, */ if (chan < mhi_cntrl->max_chan) { mhi_chan = &mhi_cntrl->mhi_chan[chan]; + if (!mhi_chan->configured) + break; parse_xfer_event(mhi_cntrl, local_rp, mhi_chan); event_quota--; } @@ -845,7 +905,15 @@ int mhi_process_ctrl_ev_ring(struct mhi_controller *mhi_cntrl, mhi_recycle_ev_ring_element(mhi_cntrl, ev_ring); local_rp = ev_ring->rp; - dev_rp = mhi_to_virtual(ev_ring, er_ctxt->rp); + + ptr = er_ctxt->rp; + if (!is_valid_ring_ptr(ev_ring, ptr)) { + dev_err(&mhi_cntrl->mhi_dev->dev, + "Event ring rp points outside of the event ring\n"); + return -EIO; + } + + dev_rp = mhi_to_virtual(ev_ring, ptr); count++; } @@ -868,11 +936,18 @@ int mhi_process_data_event_ring(struct mhi_controller *mhi_cntrl, int count = 0; u32 chan; struct mhi_chan *mhi_chan; + dma_addr_t ptr = er_ctxt->rp; if (unlikely(MHI_EVENT_ACCESS_INVALID(mhi_cntrl->pm_state))) return -EIO; - dev_rp = mhi_to_virtual(ev_ring, er_ctxt->rp); + if (!is_valid_ring_ptr(ev_ring, ptr)) { + dev_err(&mhi_cntrl->mhi_dev->dev, + "Event ring rp points outside of the event ring\n"); + return -EIO; + } + + dev_rp = mhi_to_virtual(ev_ring, ptr); local_rp = ev_ring->rp; while (dev_rp != local_rp && event_quota > 0) { @@ -886,7 +961,8 @@ int mhi_process_data_event_ring(struct mhi_controller *mhi_cntrl, * Only process the event ring elements whose channel * ID is within the maximum supported range. */ - if (chan < mhi_cntrl->max_chan) { + if (chan < mhi_cntrl->max_chan && + mhi_cntrl->mhi_chan[chan].configured) { mhi_chan = &mhi_cntrl->mhi_chan[chan]; if (likely(type == MHI_PKT_TYPE_TX_EVENT)) { @@ -900,7 +976,15 @@ int mhi_process_data_event_ring(struct mhi_controller *mhi_cntrl, mhi_recycle_ev_ring_element(mhi_cntrl, ev_ring); local_rp = ev_ring->rp; - dev_rp = mhi_to_virtual(ev_ring, er_ctxt->rp); + + ptr = er_ctxt->rp; + if (!is_valid_ring_ptr(ev_ring, ptr)) { + dev_err(&mhi_cntrl->mhi_dev->dev, + "Event ring rp points outside of the event ring\n"); + return -EIO; + } + + dev_rp = mhi_to_virtual(ev_ring, ptr); count++; } read_lock_bh(&mhi_cntrl->pm_lock); @@ -1004,9 +1088,11 @@ static int mhi_queue(struct mhi_device *mhi_dev, struct mhi_buf_info *buf_info, if (unlikely(ret)) goto exit_unlock; - /* trigger M3 exit if necessary */ - if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state)) - mhi_trigger_resume(mhi_cntrl); + /* Packet is queued, take a usage ref to exit M3 if necessary + * for host->device buffer, balanced put is done on buffer completion + * for device->host buffer, balanced put is after ringing the DB + */ + mhi_cntrl->runtime_get(mhi_cntrl); /* Assert dev_wake (to exit/prevent M1/M2)*/ mhi_cntrl->wake_toggle(mhi_cntrl); @@ -1014,12 +1100,11 @@ static int mhi_queue(struct mhi_device *mhi_dev, struct mhi_buf_info *buf_info, if (mhi_chan->dir == DMA_TO_DEVICE) atomic_inc(&mhi_cntrl->pending_pkts); - if (unlikely(!MHI_DB_ACCESS_VALID(mhi_cntrl))) { - ret = -EIO; - goto exit_unlock; - } + if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) + mhi_ring_chan_db(mhi_cntrl, mhi_chan); - mhi_ring_chan_db(mhi_cntrl, mhi_chan); + if (dir == DMA_FROM_DEVICE) + mhi_cntrl->runtime_put(mhi_cntrl); exit_unlock: read_unlock_irqrestore(&mhi_cntrl->pm_lock, flags); @@ -1365,6 +1450,7 @@ static void mhi_mark_stale_events(struct mhi_controller *mhi_cntrl, struct mhi_ring *ev_ring; struct device *dev = &mhi_cntrl->mhi_dev->dev; unsigned long flags; + dma_addr_t ptr; dev_dbg(dev, "Marking all events for chan: %d as stale\n", chan); @@ -1372,7 +1458,15 @@ static void mhi_mark_stale_events(struct mhi_controller *mhi_cntrl, /* mark all stale events related to channel as STALE event */ spin_lock_irqsave(&mhi_event->lock, flags); - dev_rp = mhi_to_virtual(ev_ring, er_ctxt->rp); + + ptr = er_ctxt->rp; + if (!is_valid_ring_ptr(ev_ring, ptr)) { + dev_err(&mhi_cntrl->mhi_dev->dev, + "Event ring rp points outside of the event ring\n"); + dev_rp = ev_ring->rp; + } else { + dev_rp = mhi_to_virtual(ev_ring, ptr); + } local_rp = ev_ring->rp; while (dev_rp != local_rp) { @@ -1403,8 +1497,11 @@ static void mhi_reset_data_chan(struct mhi_controller *mhi_cntrl, while (tre_ring->rp != tre_ring->wp) { struct mhi_buf_info *buf_info = buf_ring->rp; - if (mhi_chan->dir == DMA_TO_DEVICE) + if (mhi_chan->dir == DMA_TO_DEVICE) { atomic_dec(&mhi_cntrl->pending_pkts); + /* Release the reference got from mhi_queue() */ + mhi_cntrl->runtime_put(mhi_cntrl); + } if (!buf_info->pre_mapped) mhi_cntrl->unmap_single(mhi_cntrl, buf_info); diff --git a/drivers/bus/mhi/core/pm.c b/drivers/bus/mhi/core/pm.c index 681960c72d2a..87d3b73bcade 100644 --- a/drivers/bus/mhi/core/pm.c +++ b/drivers/bus/mhi/core/pm.c @@ -377,24 +377,28 @@ static int mhi_pm_mission_mode_transition(struct mhi_controller *mhi_cntrl) { struct mhi_event *mhi_event; struct device *dev = &mhi_cntrl->mhi_dev->dev; + enum mhi_ee_type ee = MHI_EE_MAX, current_ee = mhi_cntrl->ee; int i, ret; dev_dbg(dev, "Processing Mission Mode transition\n"); write_lock_irq(&mhi_cntrl->pm_lock); if (MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) - mhi_cntrl->ee = mhi_get_exec_env(mhi_cntrl); + ee = mhi_get_exec_env(mhi_cntrl); - if (!MHI_IN_MISSION_MODE(mhi_cntrl->ee)) { + if (!MHI_IN_MISSION_MODE(ee)) { mhi_cntrl->pm_state = MHI_PM_LD_ERR_FATAL_DETECT; write_unlock_irq(&mhi_cntrl->pm_lock); wake_up_all(&mhi_cntrl->state_event); return -EIO; } + mhi_cntrl->ee = ee; write_unlock_irq(&mhi_cntrl->pm_lock); wake_up_all(&mhi_cntrl->state_event); + device_for_each_child(&mhi_cntrl->mhi_dev->dev, ¤t_ee, + mhi_destroy_device); mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_EE_MISSION_MODE); /* Force MHI to be in M0 state before continuing */ @@ -755,6 +759,8 @@ void mhi_pm_st_worker(struct work_struct *work) * either SBL or AMSS states */ mhi_create_devices(mhi_cntrl); + if (mhi_cntrl->fbc_download) + mhi_download_amss_image(mhi_cntrl); break; case DEV_ST_TRANSITION_MISSION_MODE: mhi_pm_mission_mode_transition(mhi_cntrl); @@ -1092,7 +1098,7 @@ int mhi_async_power_up(struct mhi_controller *mhi_cntrl) &val) || !val, msecs_to_jiffies(mhi_cntrl->timeout_ms)); - if (ret) { + if (!ret) { ret = -EIO; dev_info(dev, "Failed to reset MHI due to syserr state\n"); goto error_bhi_offset; diff --git a/drivers/bus/mhi/pci_generic.c b/drivers/bus/mhi/pci_generic.c index 20673a4b4a3c..ef549c695b55 100644 --- a/drivers/bus/mhi/pci_generic.c +++ b/drivers/bus/mhi/pci_generic.c @@ -230,6 +230,21 @@ static void mhi_pci_status_cb(struct mhi_controller *mhi_cntrl, } } +static void mhi_pci_wake_get_nop(struct mhi_controller *mhi_cntrl, bool force) +{ + /* no-op */ +} + +static void mhi_pci_wake_put_nop(struct mhi_controller *mhi_cntrl, bool override) +{ + /* no-op */ +} + +static void mhi_pci_wake_toggle_nop(struct mhi_controller *mhi_cntrl) +{ + /* no-op */ +} + static bool mhi_pci_is_alive(struct mhi_controller *mhi_cntrl) { struct pci_dev *pdev = to_pci_dev(mhi_cntrl->cntrl_dev); @@ -433,6 +448,9 @@ static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) mhi_cntrl->status_cb = mhi_pci_status_cb; mhi_cntrl->runtime_get = mhi_pci_runtime_get; mhi_cntrl->runtime_put = mhi_pci_runtime_put; + mhi_cntrl->wake_get = mhi_pci_wake_get_nop; + mhi_cntrl->wake_put = mhi_pci_wake_put_nop; + mhi_cntrl->wake_toggle = mhi_pci_wake_toggle_nop; err = mhi_pci_claim(mhi_cntrl, info->bar_num, DMA_BIT_MASK(info->dma_data_width)); if (err) @@ -498,6 +516,12 @@ static void mhi_pci_remove(struct pci_dev *pdev) mhi_unregister_controller(mhi_cntrl); } +static void mhi_pci_shutdown(struct pci_dev *pdev) +{ + mhi_pci_remove(pdev); + pci_set_power_state(pdev, PCI_D3hot); +} + static void mhi_pci_reset_prepare(struct pci_dev *pdev) { struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev); @@ -668,6 +692,7 @@ static struct pci_driver mhi_pci_driver = { .id_table = mhi_pci_id_table, .probe = mhi_pci_probe, .remove = mhi_pci_remove, + .shutdown = mhi_pci_shutdown, .err_handler = &mhi_pci_err_handler, .driver.pm = &mhi_pci_pm_ops }; diff --git a/drivers/bus/qcom-ebi2.c b/drivers/bus/qcom-ebi2.c index 03ddcf426887..0b8f53a688b8 100644 --- a/drivers/bus/qcom-ebi2.c +++ b/drivers/bus/qcom-ebi2.c @@ -353,8 +353,10 @@ static int qcom_ebi2_probe(struct platform_device *pdev) /* Figure out the chipselect */ ret = of_property_read_u32(child, "reg", &csindex); - if (ret) + if (ret) { + of_node_put(child); return ret; + } if (csindex > 5) { dev_err(dev, diff --git a/drivers/bus/ti-sysc.c b/drivers/bus/ti-sysc.c index 3d74f237f005..68145e326eb9 100644 --- a/drivers/bus/ti-sysc.c +++ b/drivers/bus/ti-sysc.c @@ -635,6 +635,51 @@ static int sysc_parse_and_check_child_range(struct sysc *ddata) return 0; } +/* Interconnect instances to probe before l4_per instances */ +static struct resource early_bus_ranges[] = { + /* am3/4 l4_wkup */ + { .start = 0x44c00000, .end = 0x44c00000 + 0x300000, }, + /* omap4/5 and dra7 l4_cfg */ + { .start = 0x4a000000, .end = 0x4a000000 + 0x300000, }, + /* omap4 l4_wkup */ + { .start = 0x4a300000, .end = 0x4a300000 + 0x30000, }, + /* omap5 and dra7 l4_wkup without dra7 dcan segment */ + { .start = 0x4ae00000, .end = 0x4ae00000 + 0x30000, }, +}; + +static atomic_t sysc_defer = ATOMIC_INIT(10); + +/** + * sysc_defer_non_critical - defer non_critical interconnect probing + * @ddata: device driver data + * + * We want to probe l4_cfg and l4_wkup interconnect instances before any + * l4_per instances as l4_per instances depend on resources on l4_cfg and + * l4_wkup interconnects. + */ +static int sysc_defer_non_critical(struct sysc *ddata) +{ + struct resource *res; + int i; + + if (!atomic_read(&sysc_defer)) + return 0; + + for (i = 0; i < ARRAY_SIZE(early_bus_ranges); i++) { + res = &early_bus_ranges[i]; + if (ddata->module_pa >= res->start && + ddata->module_pa <= res->end) { + atomic_set(&sysc_defer, 0); + + return 0; + } + } + + atomic_dec_if_positive(&sysc_defer); + + return -EPROBE_DEFER; +} + static struct device_node *stdout_path; static void sysc_init_stdout_path(struct sysc *ddata) @@ -856,15 +901,19 @@ static int sysc_map_and_check_registers(struct sysc *ddata) struct device_node *np = ddata->dev->of_node; int error; - if (!of_get_property(np, "reg", NULL)) - return 0; - error = sysc_parse_and_check_child_range(ddata); if (error) return error; + error = sysc_defer_non_critical(ddata); + if (error) + return error; + sysc_check_children(ddata); + if (!of_get_property(np, "reg", NULL)) + return 0; + error = sysc_parse_registers(ddata); if (error) return error; diff --git a/drivers/char/random.c b/drivers/char/random.c index 0fe9e200e4c8..5d6acfecd919 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -819,7 +819,7 @@ static bool __init crng_init_try_arch_early(struct crng_state *crng) static void __maybe_unused crng_initialize_secondary(struct crng_state *crng) { - memcpy(&crng->state[0], "expand 32-byte k", 16); + chacha_init_consts(crng->state); _get_random_bytes(&crng->state[4], sizeof(__u32) * 12); crng_init_try_arch(crng); crng->init_time = jiffies - CRNG_RESEED_INTERVAL - 1; @@ -827,7 +827,7 @@ static void __maybe_unused crng_initialize_secondary(struct crng_state *crng) static void __init crng_initialize_primary(struct crng_state *crng) { - memcpy(&crng->state[0], "expand 32-byte k", 16); + chacha_init_consts(crng->state); _extract_entropy(&input_pool, &crng->state[4], sizeof(__u32) * 12, 0); if (crng_init_try_arch_early(crng) && trust_cpu) { invalidate_batched_entropy(); diff --git a/drivers/char/tpm/eventlog/acpi.c b/drivers/char/tpm/eventlog/acpi.c index 3633ed70f48f..1b18ce5ebab1 100644 --- a/drivers/char/tpm/eventlog/acpi.c +++ b/drivers/char/tpm/eventlog/acpi.c @@ -41,6 +41,27 @@ struct acpi_tcpa { }; }; +/* Check that the given log is indeed a TPM2 log. */ +static bool tpm_is_tpm2_log(void *bios_event_log, u64 len) +{ + struct tcg_efi_specid_event_head *efispecid; + struct tcg_pcr_event *event_header; + int n; + + if (len < sizeof(*event_header)) + return false; + len -= sizeof(*event_header); + event_header = bios_event_log; + + if (len < sizeof(*efispecid)) + return false; + efispecid = (struct tcg_efi_specid_event_head *)event_header->event; + + n = memcmp(efispecid->signature, TCG_SPECID_SIG, + sizeof(TCG_SPECID_SIG)); + return n == 0; +} + /* read binary bios log */ int tpm_read_log_acpi(struct tpm_chip *chip) { @@ -52,6 +73,7 @@ int tpm_read_log_acpi(struct tpm_chip *chip) struct acpi_table_tpm2 *tbl; struct acpi_tpm2_phy *tpm2_phy; int format; + int ret; log = &chip->log; @@ -112,6 +134,7 @@ int tpm_read_log_acpi(struct tpm_chip *chip) log->bios_event_log_end = log->bios_event_log + len; + ret = -EIO; virt = acpi_os_map_iomem(start, len); if (!virt) goto err; @@ -119,11 +142,19 @@ int tpm_read_log_acpi(struct tpm_chip *chip) memcpy_fromio(log->bios_event_log, virt, len); acpi_os_unmap_iomem(virt, len); + + if (chip->flags & TPM_CHIP_FLAG_TPM2 && + !tpm_is_tpm2_log(log->bios_event_log, len)) { + /* try EFI log next */ + ret = -ENODEV; + goto err; + } + return format; err: kfree(log->bios_event_log); log->bios_event_log = NULL; - return -EIO; + return ret; } diff --git a/drivers/char/tpm/eventlog/common.c b/drivers/char/tpm/eventlog/common.c index 7460f230bae4..8512ec76d526 100644 --- a/drivers/char/tpm/eventlog/common.c +++ b/drivers/char/tpm/eventlog/common.c @@ -107,6 +107,9 @@ void tpm_bios_log_setup(struct tpm_chip *chip) int log_version; int rc = 0; + if (chip->flags & TPM_CHIP_FLAG_VIRTUAL) + return; + rc = tpm_read_log(chip); if (rc < 0) return; diff --git a/drivers/char/tpm/eventlog/efi.c b/drivers/char/tpm/eventlog/efi.c index 35229e5143ca..e6cb9d525e30 100644 --- a/drivers/char/tpm/eventlog/efi.c +++ b/drivers/char/tpm/eventlog/efi.c @@ -17,6 +17,7 @@ int tpm_read_log_efi(struct tpm_chip *chip) { struct efi_tcg2_final_events_table *final_tbl = NULL; + int final_events_log_size = efi_tpm_final_log_size; struct linux_efi_tpm_eventlog *log_tbl; struct tpm_bios_log *log; u32 log_size; @@ -66,12 +67,12 @@ int tpm_read_log_efi(struct tpm_chip *chip) ret = tpm_log_version; if (efi.tpm_final_log == EFI_INVALID_TABLE_ADDR || - efi_tpm_final_log_size == 0 || + final_events_log_size == 0 || tpm_log_version != EFI_TCG2_EVENT_LOG_FORMAT_TCG_2) goto out; final_tbl = memremap(efi.tpm_final_log, - sizeof(*final_tbl) + efi_tpm_final_log_size, + sizeof(*final_tbl) + final_events_log_size, MEMREMAP_WB); if (!final_tbl) { pr_err("Could not map UEFI TPM final log\n"); @@ -80,10 +81,18 @@ int tpm_read_log_efi(struct tpm_chip *chip) goto out; } - efi_tpm_final_log_size -= log_tbl->final_events_preboot_size; + /* + * The 'final events log' size excludes the 'final events preboot log' + * at its beginning. + */ + final_events_log_size -= log_tbl->final_events_preboot_size; + /* + * Allocate memory for the 'combined log' where we will append the + * 'final events log' to. + */ tmp = krealloc(log->bios_event_log, - log_size + efi_tpm_final_log_size, + log_size + final_events_log_size, GFP_KERNEL); if (!tmp) { kfree(log->bios_event_log); @@ -94,15 +103,19 @@ int tpm_read_log_efi(struct tpm_chip *chip) log->bios_event_log = tmp; /* - * Copy any of the final events log that didn't also end up in the - * main log. Events can be logged in both if events are generated + * Append any of the 'final events log' that didn't also end up in the + * 'main log'. Events can be logged in both if events are generated * between GetEventLog() and ExitBootServices(). */ memcpy((void *)log->bios_event_log + log_size, final_tbl->events + log_tbl->final_events_preboot_size, - efi_tpm_final_log_size); + final_events_log_size); + /* + * The size of the 'combined log' is the size of the 'main log' plus + * the size of the 'final events log'. + */ log->bios_event_log_end = log->bios_event_log + - log_size + efi_tpm_final_log_size; + log_size + final_events_log_size; out: memunmap(final_tbl); diff --git a/drivers/char/tpm/tpm2-cmd.c b/drivers/char/tpm/tpm2-cmd.c index eff1f12d981a..c84d23951219 100644 --- a/drivers/char/tpm/tpm2-cmd.c +++ b/drivers/char/tpm/tpm2-cmd.c @@ -656,6 +656,7 @@ int tpm2_get_cc_attrs_tbl(struct tpm_chip *chip) if (nr_commands != be32_to_cpup((__be32 *)&buf.data[TPM_HEADER_SIZE + 5])) { + rc = -EFAULT; tpm_buf_destroy(&buf); goto out; } diff --git a/drivers/char/tpm/tpm_tis_core.c b/drivers/char/tpm/tpm_tis_core.c index a2e0395cbe61..55b9d3965ae1 100644 --- a/drivers/char/tpm/tpm_tis_core.c +++ b/drivers/char/tpm/tpm_tis_core.c @@ -709,16 +709,14 @@ static int tpm_tis_gen_interrupt(struct tpm_chip *chip) cap_t cap; int ret; - /* TPM 2.0 */ - if (chip->flags & TPM_CHIP_FLAG_TPM2) - return tpm2_get_tpm_pt(chip, 0x100, &cap2, desc); - - /* TPM 1.2 */ ret = request_locality(chip, 0); if (ret < 0) return ret; - ret = tpm1_getcap(chip, TPM_CAP_PROP_TIS_TIMEOUT, &cap, desc, 0); + if (chip->flags & TPM_CHIP_FLAG_TPM2) + ret = tpm2_get_tpm_pt(chip, 0x100, &cap2, desc); + else + ret = tpm1_getcap(chip, TPM_CAP_PROP_TIS_TIMEOUT, &cap, desc, 0); release_locality(chip, 0); @@ -1127,12 +1125,20 @@ int tpm_tis_resume(struct device *dev) if (ret) return ret; - /* TPM 1.2 requires self-test on resume. This function actually returns + /* + * TPM 1.2 requires self-test on resume. This function actually returns * an error code but for unknown reason it isn't handled. */ - if (!(chip->flags & TPM_CHIP_FLAG_TPM2)) + if (!(chip->flags & TPM_CHIP_FLAG_TPM2)) { + ret = request_locality(chip, 0); + if (ret < 0) + return ret; + tpm1_do_selftest(chip); + release_locality(chip, 0); + } + return 0; } EXPORT_SYMBOL_GPL(tpm_tis_resume); diff --git a/drivers/char/tpm/tpm_tis_i2c_cr50.c b/drivers/char/tpm/tpm_tis_i2c_cr50.c index ec9a65e7887d..f19c227d20f4 100644 --- a/drivers/char/tpm/tpm_tis_i2c_cr50.c +++ b/drivers/char/tpm/tpm_tis_i2c_cr50.c @@ -483,6 +483,7 @@ static int tpm_cr50_i2c_tis_recv(struct tpm_chip *chip, u8 *buf, size_t buf_len) expected = be32_to_cpup((__be32 *)(buf + 2)); if (expected > buf_len) { dev_err(&chip->dev, "Buffer too small to receive i2c data\n"); + rc = -E2BIG; goto out_err; } diff --git a/drivers/char/ttyprintk.c b/drivers/char/ttyprintk.c index 6a0059e508e3..93f5d11c830b 100644 --- a/drivers/char/ttyprintk.c +++ b/drivers/char/ttyprintk.c @@ -158,12 +158,23 @@ static int tpk_ioctl(struct tty_struct *tty, return 0; } +/* + * TTY operations hangup function. + */ +static void tpk_hangup(struct tty_struct *tty) +{ + struct ttyprintk_port *tpkp = tty->driver_data; + + tty_port_hangup(&tpkp->port); +} + static const struct tty_operations ttyprintk_ops = { .open = tpk_open, .close = tpk_close, .write = tpk_write, .write_room = tpk_write_room, .ioctl = tpk_ioctl, + .hangup = tpk_hangup, }; static const struct tty_port_operations null_ops = { }; diff --git a/drivers/clk/clk-ast2600.c b/drivers/clk/clk-ast2600.c index a55b37fc2c8b..bc3be5f3eae1 100644 --- a/drivers/clk/clk-ast2600.c +++ b/drivers/clk/clk-ast2600.c @@ -61,10 +61,10 @@ static void __iomem *scu_g6_base; static const struct aspeed_gate_data aspeed_g6_gates[] = { /* clk rst name parent flags */ [ASPEED_CLK_GATE_MCLK] = { 0, -1, "mclk-gate", "mpll", CLK_IS_CRITICAL }, /* SDRAM */ - [ASPEED_CLK_GATE_ECLK] = { 1, -1, "eclk-gate", "eclk", 0 }, /* Video Engine */ + [ASPEED_CLK_GATE_ECLK] = { 1, 6, "eclk-gate", "eclk", 0 }, /* Video Engine */ [ASPEED_CLK_GATE_GCLK] = { 2, 7, "gclk-gate", NULL, 0 }, /* 2D engine */ /* vclk parent - dclk/d1clk/hclk/mclk */ - [ASPEED_CLK_GATE_VCLK] = { 3, 6, "vclk-gate", NULL, 0 }, /* Video Capture */ + [ASPEED_CLK_GATE_VCLK] = { 3, -1, "vclk-gate", NULL, 0 }, /* Video Capture */ [ASPEED_CLK_GATE_BCLK] = { 4, 8, "bclk-gate", "bclk", 0 }, /* PCIe/PCI */ /* From dpll */ [ASPEED_CLK_GATE_DCLK] = { 5, -1, "dclk-gate", NULL, CLK_IS_CRITICAL }, /* DAC */ diff --git a/drivers/clk/imx/clk-imx25.c b/drivers/clk/imx/clk-imx25.c index a66cabfbf94f..66192fe0a898 100644 --- a/drivers/clk/imx/clk-imx25.c +++ b/drivers/clk/imx/clk-imx25.c @@ -73,16 +73,6 @@ enum mx25_clks { static struct clk *clk[clk_max]; -static struct clk ** const uart_clks[] __initconst = { - &clk[uart_ipg_per], - &clk[uart1_ipg], - &clk[uart2_ipg], - &clk[uart3_ipg], - &clk[uart4_ipg], - &clk[uart5_ipg], - NULL -}; - static int __init __mx25_clocks_init(void __iomem *ccm_base) { BUG_ON(!ccm_base); @@ -228,7 +218,7 @@ static int __init __mx25_clocks_init(void __iomem *ccm_base) */ clk_set_parent(clk[cko_sel], clk[ipg]); - imx_register_uart_clocks(uart_clks); + imx_register_uart_clocks(6); return 0; } diff --git a/drivers/clk/imx/clk-imx27.c b/drivers/clk/imx/clk-imx27.c index 5585ded8b8c6..56a5fc402b10 100644 --- a/drivers/clk/imx/clk-imx27.c +++ b/drivers/clk/imx/clk-imx27.c @@ -49,17 +49,6 @@ static const char *ssi_sel_clks[] = { "spll_gate", "mpll", }; static struct clk *clk[IMX27_CLK_MAX]; static struct clk_onecell_data clk_data; -static struct clk ** const uart_clks[] __initconst = { - &clk[IMX27_CLK_PER1_GATE], - &clk[IMX27_CLK_UART1_IPG_GATE], - &clk[IMX27_CLK_UART2_IPG_GATE], - &clk[IMX27_CLK_UART3_IPG_GATE], - &clk[IMX27_CLK_UART4_IPG_GATE], - &clk[IMX27_CLK_UART5_IPG_GATE], - &clk[IMX27_CLK_UART6_IPG_GATE], - NULL -}; - static void __init _mx27_clocks_init(unsigned long fref) { BUG_ON(!ccm); @@ -176,7 +165,7 @@ static void __init _mx27_clocks_init(unsigned long fref) clk_prepare_enable(clk[IMX27_CLK_EMI_AHB_GATE]); - imx_register_uart_clocks(uart_clks); + imx_register_uart_clocks(7); imx_print_silicon_rev("i.MX27", mx27_revision()); } diff --git a/drivers/clk/imx/clk-imx35.c b/drivers/clk/imx/clk-imx35.c index c1df03665c09..0fe5ac210156 100644 --- a/drivers/clk/imx/clk-imx35.c +++ b/drivers/clk/imx/clk-imx35.c @@ -82,14 +82,6 @@ enum mx35_clks { static struct clk *clk[clk_max]; -static struct clk ** const uart_clks[] __initconst = { - &clk[ipg], - &clk[uart1_gate], - &clk[uart2_gate], - &clk[uart3_gate], - NULL -}; - static void __init _mx35_clocks_init(void) { void __iomem *base; @@ -243,7 +235,7 @@ static void __init _mx35_clocks_init(void) */ clk_prepare_enable(clk[scc_gate]); - imx_register_uart_clocks(uart_clks); + imx_register_uart_clocks(4); imx_print_silicon_rev("i.MX35", mx35_revision()); } diff --git a/drivers/clk/imx/clk-imx5.c b/drivers/clk/imx/clk-imx5.c index 01e079b81026..e4493846454d 100644 --- a/drivers/clk/imx/clk-imx5.c +++ b/drivers/clk/imx/clk-imx5.c @@ -128,30 +128,6 @@ static const char *ieee1588_sels[] = { "pll3_sw", "pll4_sw", "dummy" /* usbphy2_ static struct clk *clk[IMX5_CLK_END]; static struct clk_onecell_data clk_data; -static struct clk ** const uart_clks_mx51[] __initconst = { - &clk[IMX5_CLK_UART1_IPG_GATE], - &clk[IMX5_CLK_UART1_PER_GATE], - &clk[IMX5_CLK_UART2_IPG_GATE], - &clk[IMX5_CLK_UART2_PER_GATE], - &clk[IMX5_CLK_UART3_IPG_GATE], - &clk[IMX5_CLK_UART3_PER_GATE], - NULL -}; - -static struct clk ** const uart_clks_mx50_mx53[] __initconst = { - &clk[IMX5_CLK_UART1_IPG_GATE], - &clk[IMX5_CLK_UART1_PER_GATE], - &clk[IMX5_CLK_UART2_IPG_GATE], - &clk[IMX5_CLK_UART2_PER_GATE], - &clk[IMX5_CLK_UART3_IPG_GATE], - &clk[IMX5_CLK_UART3_PER_GATE], - &clk[IMX5_CLK_UART4_IPG_GATE], - &clk[IMX5_CLK_UART4_PER_GATE], - &clk[IMX5_CLK_UART5_IPG_GATE], - &clk[IMX5_CLK_UART5_PER_GATE], - NULL -}; - static void __init mx5_clocks_common_init(void __iomem *ccm_base) { clk[IMX5_CLK_DUMMY] = imx_clk_fixed("dummy", 0); @@ -382,7 +358,7 @@ static void __init mx50_clocks_init(struct device_node *np) r = clk_round_rate(clk[IMX5_CLK_USBOH3_PER_GATE], 54000000); clk_set_rate(clk[IMX5_CLK_USBOH3_PER_GATE], r); - imx_register_uart_clocks(uart_clks_mx50_mx53); + imx_register_uart_clocks(5); } CLK_OF_DECLARE(imx50_ccm, "fsl,imx50-ccm", mx50_clocks_init); @@ -488,7 +464,7 @@ static void __init mx51_clocks_init(struct device_node *np) val |= 1 << 23; writel(val, MXC_CCM_CLPCR); - imx_register_uart_clocks(uart_clks_mx51); + imx_register_uart_clocks(3); } CLK_OF_DECLARE(imx51_ccm, "fsl,imx51-ccm", mx51_clocks_init); @@ -633,6 +609,6 @@ static void __init mx53_clocks_init(struct device_node *np) r = clk_round_rate(clk[IMX5_CLK_USBOH3_PER_GATE], 54000000); clk_set_rate(clk[IMX5_CLK_USBOH3_PER_GATE], r); - imx_register_uart_clocks(uart_clks_mx50_mx53); + imx_register_uart_clocks(5); } CLK_OF_DECLARE(imx53_ccm, "fsl,imx53-ccm", mx53_clocks_init); diff --git a/drivers/clk/imx/clk-imx6q.c b/drivers/clk/imx/clk-imx6q.c index 521d6136d22c..496900de0b0b 100644 --- a/drivers/clk/imx/clk-imx6q.c +++ b/drivers/clk/imx/clk-imx6q.c @@ -140,13 +140,6 @@ static inline int clk_on_imx6dl(void) return of_machine_is_compatible("fsl,imx6dl"); } -static const int uart_clk_ids[] __initconst = { - IMX6QDL_CLK_UART_IPG, - IMX6QDL_CLK_UART_SERIAL, -}; - -static struct clk **uart_clks[ARRAY_SIZE(uart_clk_ids) + 1] __initdata; - static int ldb_di_sel_by_clock_id(int clock_id) { switch (clock_id) { @@ -440,7 +433,6 @@ static void __init imx6q_clocks_init(struct device_node *ccm_node) struct device_node *np; void __iomem *anatop_base, *base; int ret; - int i; clk_hw_data = kzalloc(struct_size(clk_hw_data, hws, IMX6QDL_CLK_END), GFP_KERNEL); @@ -982,12 +974,6 @@ static void __init imx6q_clocks_init(struct device_node *ccm_node) hws[IMX6QDL_CLK_PLL3_USB_OTG]->clk); } - for (i = 0; i < ARRAY_SIZE(uart_clk_ids); i++) { - int index = uart_clk_ids[i]; - - uart_clks[i] = &hws[index]->clk; - } - - imx_register_uart_clocks(uart_clks); + imx_register_uart_clocks(1); } CLK_OF_DECLARE(imx6q, "fsl,imx6q-ccm", imx6q_clocks_init); diff --git a/drivers/clk/imx/clk-imx6sl.c b/drivers/clk/imx/clk-imx6sl.c index 29eab05c9068..277365970320 100644 --- a/drivers/clk/imx/clk-imx6sl.c +++ b/drivers/clk/imx/clk-imx6sl.c @@ -179,19 +179,11 @@ void imx6sl_set_wait_clk(bool enter) imx6sl_enable_pll_arm(false); } -static const int uart_clk_ids[] __initconst = { - IMX6SL_CLK_UART, - IMX6SL_CLK_UART_SERIAL, -}; - -static struct clk **uart_clks[ARRAY_SIZE(uart_clk_ids) + 1] __initdata; - static void __init imx6sl_clocks_init(struct device_node *ccm_node) { struct device_node *np; void __iomem *base; int ret; - int i; clk_hw_data = kzalloc(struct_size(clk_hw_data, hws, IMX6SL_CLK_END), GFP_KERNEL); @@ -448,12 +440,6 @@ static void __init imx6sl_clocks_init(struct device_node *ccm_node) clk_set_parent(hws[IMX6SL_CLK_LCDIF_AXI_SEL]->clk, hws[IMX6SL_CLK_PLL2_PFD2]->clk); - for (i = 0; i < ARRAY_SIZE(uart_clk_ids); i++) { - int index = uart_clk_ids[i]; - - uart_clks[i] = &hws[index]->clk; - } - - imx_register_uart_clocks(uart_clks); + imx_register_uart_clocks(2); } CLK_OF_DECLARE(imx6sl, "fsl,imx6sl-ccm", imx6sl_clocks_init); diff --git a/drivers/clk/imx/clk-imx6sll.c b/drivers/clk/imx/clk-imx6sll.c index 8e8288bda4d0..31d777f30039 100644 --- a/drivers/clk/imx/clk-imx6sll.c +++ b/drivers/clk/imx/clk-imx6sll.c @@ -76,26 +76,10 @@ static u32 share_count_ssi1; static u32 share_count_ssi2; static u32 share_count_ssi3; -static const int uart_clk_ids[] __initconst = { - IMX6SLL_CLK_UART1_IPG, - IMX6SLL_CLK_UART1_SERIAL, - IMX6SLL_CLK_UART2_IPG, - IMX6SLL_CLK_UART2_SERIAL, - IMX6SLL_CLK_UART3_IPG, - IMX6SLL_CLK_UART3_SERIAL, - IMX6SLL_CLK_UART4_IPG, - IMX6SLL_CLK_UART4_SERIAL, - IMX6SLL_CLK_UART5_IPG, - IMX6SLL_CLK_UART5_SERIAL, -}; - -static struct clk **uart_clks[ARRAY_SIZE(uart_clk_ids) + 1] __initdata; - static void __init imx6sll_clocks_init(struct device_node *ccm_node) { struct device_node *np; void __iomem *base; - int i; clk_hw_data = kzalloc(struct_size(clk_hw_data, hws, IMX6SLL_CLK_END), GFP_KERNEL); @@ -356,13 +340,7 @@ static void __init imx6sll_clocks_init(struct device_node *ccm_node) of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_hw_data); - for (i = 0; i < ARRAY_SIZE(uart_clk_ids); i++) { - int index = uart_clk_ids[i]; - - uart_clks[i] = &hws[index]->clk; - } - - imx_register_uart_clocks(uart_clks); + imx_register_uart_clocks(5); /* Lower the AHB clock rate before changing the clock source. */ clk_set_rate(hws[IMX6SLL_CLK_AHB]->clk, 99000000); diff --git a/drivers/clk/imx/clk-imx6sx.c b/drivers/clk/imx/clk-imx6sx.c index 20dcce526d07..fc1bd23d4583 100644 --- a/drivers/clk/imx/clk-imx6sx.c +++ b/drivers/clk/imx/clk-imx6sx.c @@ -117,18 +117,10 @@ static u32 share_count_ssi3; static u32 share_count_sai1; static u32 share_count_sai2; -static const int uart_clk_ids[] __initconst = { - IMX6SX_CLK_UART_IPG, - IMX6SX_CLK_UART_SERIAL, -}; - -static struct clk **uart_clks[ARRAY_SIZE(uart_clk_ids) + 1] __initdata; - static void __init imx6sx_clocks_init(struct device_node *ccm_node) { struct device_node *np; void __iomem *base; - int i; clk_hw_data = kzalloc(struct_size(clk_hw_data, hws, IMX6SX_CLK_CLK_END), GFP_KERNEL); @@ -556,12 +548,6 @@ static void __init imx6sx_clocks_init(struct device_node *ccm_node) clk_set_parent(hws[IMX6SX_CLK_QSPI1_SEL]->clk, hws[IMX6SX_CLK_PLL2_BUS]->clk); clk_set_parent(hws[IMX6SX_CLK_QSPI2_SEL]->clk, hws[IMX6SX_CLK_PLL2_BUS]->clk); - for (i = 0; i < ARRAY_SIZE(uart_clk_ids); i++) { - int index = uart_clk_ids[i]; - - uart_clks[i] = &hws[index]->clk; - } - - imx_register_uart_clocks(uart_clks); + imx_register_uart_clocks(2); } CLK_OF_DECLARE(imx6sx, "fsl,imx6sx-ccm", imx6sx_clocks_init); diff --git a/drivers/clk/imx/clk-imx7d.c b/drivers/clk/imx/clk-imx7d.c index 22d24a6a05e7..c4e0f1c07192 100644 --- a/drivers/clk/imx/clk-imx7d.c +++ b/drivers/clk/imx/clk-imx7d.c @@ -377,23 +377,10 @@ static const char *pll_video_bypass_sel[] = { "pll_video_main", "pll_video_main_ static struct clk_hw **hws; static struct clk_hw_onecell_data *clk_hw_data; -static const int uart_clk_ids[] __initconst = { - IMX7D_UART1_ROOT_CLK, - IMX7D_UART2_ROOT_CLK, - IMX7D_UART3_ROOT_CLK, - IMX7D_UART4_ROOT_CLK, - IMX7D_UART5_ROOT_CLK, - IMX7D_UART6_ROOT_CLK, - IMX7D_UART7_ROOT_CLK, -}; - -static struct clk **uart_clks[ARRAY_SIZE(uart_clk_ids) + 1] __initdata; - static void __init imx7d_clocks_init(struct device_node *ccm_node) { struct device_node *np; void __iomem *base; - int i; clk_hw_data = kzalloc(struct_size(clk_hw_data, hws, IMX7D_CLK_END), GFP_KERNEL); @@ -897,14 +884,7 @@ static void __init imx7d_clocks_init(struct device_node *ccm_node) hws[IMX7D_USB1_MAIN_480M_CLK] = imx_clk_hw_fixed_factor("pll_usb1_main_clk", "osc", 20, 1); hws[IMX7D_USB_MAIN_480M_CLK] = imx_clk_hw_fixed_factor("pll_usb_main_clk", "osc", 20, 1); - for (i = 0; i < ARRAY_SIZE(uart_clk_ids); i++) { - int index = uart_clk_ids[i]; - - uart_clks[i] = &hws[index]->clk; - } - - - imx_register_uart_clocks(uart_clks); + imx_register_uart_clocks(7); } CLK_OF_DECLARE(imx7d, "fsl,imx7d-ccm", imx7d_clocks_init); diff --git a/drivers/clk/imx/clk-imx7ulp.c b/drivers/clk/imx/clk-imx7ulp.c index 634c0b6636b0..779e09105da7 100644 --- a/drivers/clk/imx/clk-imx7ulp.c +++ b/drivers/clk/imx/clk-imx7ulp.c @@ -43,19 +43,6 @@ static const struct clk_div_table ulp_div_table[] = { { /* sentinel */ }, }; -static const int pcc2_uart_clk_ids[] __initconst = { - IMX7ULP_CLK_LPUART4, - IMX7ULP_CLK_LPUART5, -}; - -static const int pcc3_uart_clk_ids[] __initconst = { - IMX7ULP_CLK_LPUART6, - IMX7ULP_CLK_LPUART7, -}; - -static struct clk **pcc2_uart_clks[ARRAY_SIZE(pcc2_uart_clk_ids) + 1] __initdata; -static struct clk **pcc3_uart_clks[ARRAY_SIZE(pcc3_uart_clk_ids) + 1] __initdata; - static void __init imx7ulp_clk_scg1_init(struct device_node *np) { struct clk_hw_onecell_data *clk_data; @@ -150,7 +137,6 @@ static void __init imx7ulp_clk_pcc2_init(struct device_node *np) struct clk_hw_onecell_data *clk_data; struct clk_hw **hws; void __iomem *base; - int i; clk_data = kzalloc(struct_size(clk_data, hws, IMX7ULP_CLK_PCC2_END), GFP_KERNEL); @@ -190,13 +176,7 @@ static void __init imx7ulp_clk_pcc2_init(struct device_node *np) of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_data); - for (i = 0; i < ARRAY_SIZE(pcc2_uart_clk_ids); i++) { - int index = pcc2_uart_clk_ids[i]; - - pcc2_uart_clks[i] = &hws[index]->clk; - } - - imx_register_uart_clocks(pcc2_uart_clks); + imx_register_uart_clocks(2); } CLK_OF_DECLARE(imx7ulp_clk_pcc2, "fsl,imx7ulp-pcc2", imx7ulp_clk_pcc2_init); @@ -205,7 +185,6 @@ static void __init imx7ulp_clk_pcc3_init(struct device_node *np) struct clk_hw_onecell_data *clk_data; struct clk_hw **hws; void __iomem *base; - int i; clk_data = kzalloc(struct_size(clk_data, hws, IMX7ULP_CLK_PCC3_END), GFP_KERNEL); @@ -244,13 +223,7 @@ static void __init imx7ulp_clk_pcc3_init(struct device_node *np) of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_data); - for (i = 0; i < ARRAY_SIZE(pcc3_uart_clk_ids); i++) { - int index = pcc3_uart_clk_ids[i]; - - pcc3_uart_clks[i] = &hws[index]->clk; - } - - imx_register_uart_clocks(pcc3_uart_clks); + imx_register_uart_clocks(7); } CLK_OF_DECLARE(imx7ulp_clk_pcc3, "fsl,imx7ulp-pcc3", imx7ulp_clk_pcc3_init); diff --git a/drivers/clk/imx/clk-imx8mm.c b/drivers/clk/imx/clk-imx8mm.c index 6a01eec36dd0..f1919fafb124 100644 --- a/drivers/clk/imx/clk-imx8mm.c +++ b/drivers/clk/imx/clk-imx8mm.c @@ -296,20 +296,12 @@ static const char * const clkout_sels[] = {"audio_pll1_out", "audio_pll2_out", " static struct clk_hw_onecell_data *clk_hw_data; static struct clk_hw **hws; -static const int uart_clk_ids[] = { - IMX8MM_CLK_UART1_ROOT, - IMX8MM_CLK_UART2_ROOT, - IMX8MM_CLK_UART3_ROOT, - IMX8MM_CLK_UART4_ROOT, -}; -static struct clk **uart_hws[ARRAY_SIZE(uart_clk_ids) + 1]; - static int imx8mm_clocks_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; void __iomem *base; - int ret, i; + int ret; clk_hw_data = kzalloc(struct_size(clk_hw_data, hws, IMX8MM_CLK_END), GFP_KERNEL); @@ -634,13 +626,7 @@ static int imx8mm_clocks_probe(struct platform_device *pdev) goto unregister_hws; } - for (i = 0; i < ARRAY_SIZE(uart_clk_ids); i++) { - int index = uart_clk_ids[i]; - - uart_hws[i] = &hws[index]->clk; - } - - imx_register_uart_clocks(uart_hws); + imx_register_uart_clocks(4); return 0; diff --git a/drivers/clk/imx/clk-imx8mn.c b/drivers/clk/imx/clk-imx8mn.c index 324c5fd0aa04..88f6630cd472 100644 --- a/drivers/clk/imx/clk-imx8mn.c +++ b/drivers/clk/imx/clk-imx8mn.c @@ -289,20 +289,12 @@ static const char * const clkout_sels[] = {"audio_pll1_out", "audio_pll2_out", " static struct clk_hw_onecell_data *clk_hw_data; static struct clk_hw **hws; -static const int uart_clk_ids[] = { - IMX8MN_CLK_UART1_ROOT, - IMX8MN_CLK_UART2_ROOT, - IMX8MN_CLK_UART3_ROOT, - IMX8MN_CLK_UART4_ROOT, -}; -static struct clk **uart_hws[ARRAY_SIZE(uart_clk_ids) + 1]; - static int imx8mn_clocks_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; void __iomem *base; - int ret, i; + int ret; clk_hw_data = kzalloc(struct_size(clk_hw_data, hws, IMX8MN_CLK_END), GFP_KERNEL); @@ -585,13 +577,7 @@ static int imx8mn_clocks_probe(struct platform_device *pdev) goto unregister_hws; } - for (i = 0; i < ARRAY_SIZE(uart_clk_ids); i++) { - int index = uart_clk_ids[i]; - - uart_hws[i] = &hws[index]->clk; - } - - imx_register_uart_clocks(uart_hws); + imx_register_uart_clocks(4); return 0; diff --git a/drivers/clk/imx/clk-imx8mp.c b/drivers/clk/imx/clk-imx8mp.c index 2f4e1d674e1c..3e6557e7d559 100644 --- a/drivers/clk/imx/clk-imx8mp.c +++ b/drivers/clk/imx/clk-imx8mp.c @@ -414,20 +414,11 @@ static const char * const imx8mp_dram_core_sels[] = {"dram_pll_out", "dram_alt_r static struct clk_hw **hws; static struct clk_hw_onecell_data *clk_hw_data; -static const int uart_clk_ids[] = { - IMX8MP_CLK_UART1_ROOT, - IMX8MP_CLK_UART2_ROOT, - IMX8MP_CLK_UART3_ROOT, - IMX8MP_CLK_UART4_ROOT, -}; -static struct clk **uart_clks[ARRAY_SIZE(uart_clk_ids) + 1]; - static int imx8mp_clocks_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct device_node *np; void __iomem *anatop_base, *ccm_base; - int i; np = of_find_compatible_node(NULL, NULL, "fsl,imx8mp-anatop"); anatop_base = of_iomap(np, 0); @@ -737,13 +728,7 @@ static int imx8mp_clocks_probe(struct platform_device *pdev) of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_hw_data); - for (i = 0; i < ARRAY_SIZE(uart_clk_ids); i++) { - int index = uart_clk_ids[i]; - - uart_clks[i] = &hws[index]->clk; - } - - imx_register_uart_clocks(uart_clks); + imx_register_uart_clocks(4); return 0; } diff --git a/drivers/clk/imx/clk-imx8mq.c b/drivers/clk/imx/clk-imx8mq.c index 4dd4ae9d022b..3e1a10d3f55c 100644 --- a/drivers/clk/imx/clk-imx8mq.c +++ b/drivers/clk/imx/clk-imx8mq.c @@ -281,20 +281,12 @@ static const char * const pllout_monitor_sels[] = {"osc_25m", "osc_27m", "dummy" static struct clk_hw_onecell_data *clk_hw_data; static struct clk_hw **hws; -static const int uart_clk_ids[] = { - IMX8MQ_CLK_UART1_ROOT, - IMX8MQ_CLK_UART2_ROOT, - IMX8MQ_CLK_UART3_ROOT, - IMX8MQ_CLK_UART4_ROOT, -}; -static struct clk **uart_hws[ARRAY_SIZE(uart_clk_ids) + 1]; - static int imx8mq_clocks_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; void __iomem *base; - int err, i; + int err; clk_hw_data = kzalloc(struct_size(clk_hw_data, hws, IMX8MQ_CLK_END), GFP_KERNEL); @@ -629,13 +621,7 @@ static int imx8mq_clocks_probe(struct platform_device *pdev) goto unregister_hws; } - for (i = 0; i < ARRAY_SIZE(uart_clk_ids); i++) { - int index = uart_clk_ids[i]; - - uart_hws[i] = &hws[index]->clk; - } - - imx_register_uart_clocks(uart_hws); + imx_register_uart_clocks(4); return 0; diff --git a/drivers/clk/imx/clk.c b/drivers/clk/imx/clk.c index 47882c51cb85..7cc669934253 100644 --- a/drivers/clk/imx/clk.c +++ b/drivers/clk/imx/clk.c @@ -147,8 +147,10 @@ void imx_cscmr1_fixup(u32 *val) } #ifndef MODULE -static int imx_keep_uart_clocks; -static struct clk ** const *imx_uart_clocks; + +static bool imx_keep_uart_clocks; +static int imx_enabled_uart_clocks; +static struct clk **imx_uart_clocks; static int __init imx_keep_uart_clocks_param(char *str) { @@ -161,24 +163,45 @@ __setup_param("earlycon", imx_keep_uart_earlycon, __setup_param("earlyprintk", imx_keep_uart_earlyprintk, imx_keep_uart_clocks_param, 0); -void imx_register_uart_clocks(struct clk ** const clks[]) +void imx_register_uart_clocks(unsigned int clk_count) { + imx_enabled_uart_clocks = 0; + +/* i.MX boards use device trees now. For build tests without CONFIG_OF, do nothing */ +#ifdef CONFIG_OF if (imx_keep_uart_clocks) { int i; - imx_uart_clocks = clks; - for (i = 0; imx_uart_clocks[i]; i++) - clk_prepare_enable(*imx_uart_clocks[i]); + imx_uart_clocks = kcalloc(clk_count, sizeof(struct clk *), GFP_KERNEL); + + if (!of_stdout) + return; + + for (i = 0; i < clk_count; i++) { + imx_uart_clocks[imx_enabled_uart_clocks] = of_clk_get(of_stdout, i); + + /* Stop if there are no more of_stdout references */ + if (IS_ERR(imx_uart_clocks[imx_enabled_uart_clocks])) + return; + + /* Only enable the clock if it's not NULL */ + if (imx_uart_clocks[imx_enabled_uart_clocks]) + clk_prepare_enable(imx_uart_clocks[imx_enabled_uart_clocks++]); + } } +#endif } static int __init imx_clk_disable_uart(void) { - if (imx_keep_uart_clocks && imx_uart_clocks) { + if (imx_keep_uart_clocks && imx_enabled_uart_clocks) { int i; - for (i = 0; imx_uart_clocks[i]; i++) - clk_disable_unprepare(*imx_uart_clocks[i]); + for (i = 0; i < imx_enabled_uart_clocks; i++) { + clk_disable_unprepare(imx_uart_clocks[i]); + clk_put(imx_uart_clocks[i]); + } + kfree(imx_uart_clocks); } return 0; diff --git a/drivers/clk/imx/clk.h b/drivers/clk/imx/clk.h index 4f04c8287286..7571603bee23 100644 --- a/drivers/clk/imx/clk.h +++ b/drivers/clk/imx/clk.h @@ -11,9 +11,9 @@ extern spinlock_t imx_ccm_lock; void imx_check_clocks(struct clk *clks[], unsigned int count); void imx_check_clk_hws(struct clk_hw *clks[], unsigned int count); #ifndef MODULE -void imx_register_uart_clocks(struct clk ** const clks[]); +void imx_register_uart_clocks(unsigned int clk_count); #else -static inline void imx_register_uart_clocks(struct clk ** const clks[]) +static inline void imx_register_uart_clocks(unsigned int clk_count) { } #endif diff --git a/drivers/clk/mvebu/armada-37xx-periph.c b/drivers/clk/mvebu/armada-37xx-periph.c index f5746f9ea929..32ac6b6b7530 100644 --- a/drivers/clk/mvebu/armada-37xx-periph.c +++ b/drivers/clk/mvebu/armada-37xx-periph.c @@ -84,6 +84,7 @@ struct clk_pm_cpu { void __iomem *reg_div; u8 shift_div; struct regmap *nb_pm_base; + unsigned long l1_expiration; }; #define to_clk_double_div(_hw) container_of(_hw, struct clk_double_div, hw) @@ -440,33 +441,6 @@ static u8 clk_pm_cpu_get_parent(struct clk_hw *hw) return val; } -static int clk_pm_cpu_set_parent(struct clk_hw *hw, u8 index) -{ - struct clk_pm_cpu *pm_cpu = to_clk_pm_cpu(hw); - struct regmap *base = pm_cpu->nb_pm_base; - int load_level; - - /* - * We set the clock parent only if the DVFS is available but - * not enabled. - */ - if (IS_ERR(base) || armada_3700_pm_dvfs_is_enabled(base)) - return -EINVAL; - - /* Set the parent clock for all the load level */ - for (load_level = 0; load_level < LOAD_LEVEL_NR; load_level++) { - unsigned int reg, mask, val, - offset = ARMADA_37XX_NB_TBG_SEL_OFF; - - armada_3700_pm_dvfs_update_regs(load_level, ®, &offset); - - val = index << offset; - mask = ARMADA_37XX_NB_TBG_SEL_MASK << offset; - regmap_update_bits(base, reg, mask, val); - } - return 0; -} - static unsigned long clk_pm_cpu_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) { @@ -514,8 +488,10 @@ static long clk_pm_cpu_round_rate(struct clk_hw *hw, unsigned long rate, } /* - * Switching the CPU from the L2 or L3 frequencies (300 and 200 Mhz - * respectively) to L0 frequency (1.2 Ghz) requires a significant + * Workaround when base CPU frequnecy is 1000 or 1200 MHz + * + * Switching the CPU from the L2 or L3 frequencies (250/300 or 200 MHz + * respectively) to L0 frequency (1/1.2 GHz) requires a significant * amount of time to let VDD stabilize to the appropriate * voltage. This amount of time is large enough that it cannot be * covered by the hardware countdown register. Due to this, the CPU @@ -525,26 +501,56 @@ static long clk_pm_cpu_round_rate(struct clk_hw *hw, unsigned long rate, * To work around this problem, we prevent switching directly from the * L2/L3 frequencies to the L0 frequency, and instead switch to the L1 * frequency in-between. The sequence therefore becomes: - * 1. First switch from L2/L3(200/300MHz) to L1(600MHZ) + * 1. First switch from L2/L3 (200/250/300 MHz) to L1 (500/600 MHz) * 2. Sleep 20ms for stabling VDD voltage - * 3. Then switch from L1(600MHZ) to L0(1200Mhz). + * 3. Then switch from L1 (500/600 MHz) to L0 (1000/1200 MHz). */ -static void clk_pm_cpu_set_rate_wa(unsigned long rate, struct regmap *base) +static void clk_pm_cpu_set_rate_wa(struct clk_pm_cpu *pm_cpu, + unsigned int new_level, unsigned long rate, + struct regmap *base) { unsigned int cur_level; - if (rate != 1200 * 1000 * 1000) - return; - regmap_read(base, ARMADA_37XX_NB_CPU_LOAD, &cur_level); cur_level &= ARMADA_37XX_NB_CPU_LOAD_MASK; - if (cur_level <= ARMADA_37XX_DVFS_LOAD_1) + + if (cur_level == new_level) + return; + + /* + * System wants to go to L1 on its own. If we are going from L2/L3, + * remember when 20ms will expire. If from L0, set the value so that + * next switch to L0 won't have to wait. + */ + if (new_level == ARMADA_37XX_DVFS_LOAD_1) { + if (cur_level == ARMADA_37XX_DVFS_LOAD_0) + pm_cpu->l1_expiration = jiffies; + else + pm_cpu->l1_expiration = jiffies + msecs_to_jiffies(20); return; + } + + /* + * If we are setting to L2/L3, just invalidate L1 expiration time, + * sleeping is not needed. + */ + if (rate < 1000*1000*1000) + goto invalidate_l1_exp; + + /* + * We are going to L0 with rate >= 1GHz. Check whether we have been at + * L1 for long enough time. If not, go to L1 for 20ms. + */ + if (pm_cpu->l1_expiration && jiffies >= pm_cpu->l1_expiration) + goto invalidate_l1_exp; regmap_update_bits(base, ARMADA_37XX_NB_CPU_LOAD, ARMADA_37XX_NB_CPU_LOAD_MASK, ARMADA_37XX_DVFS_LOAD_1); msleep(20); + +invalidate_l1_exp: + pm_cpu->l1_expiration = 0; } static int clk_pm_cpu_set_rate(struct clk_hw *hw, unsigned long rate, @@ -578,7 +584,9 @@ static int clk_pm_cpu_set_rate(struct clk_hw *hw, unsigned long rate, reg = ARMADA_37XX_NB_CPU_LOAD; mask = ARMADA_37XX_NB_CPU_LOAD_MASK; - clk_pm_cpu_set_rate_wa(rate, base); + /* Apply workaround when base CPU frequency is 1000 or 1200 MHz */ + if (parent_rate >= 1000*1000*1000) + clk_pm_cpu_set_rate_wa(pm_cpu, load_level, rate, base); regmap_update_bits(base, reg, mask, load_level); @@ -592,7 +600,6 @@ static int clk_pm_cpu_set_rate(struct clk_hw *hw, unsigned long rate, static const struct clk_ops clk_pm_cpu_ops = { .get_parent = clk_pm_cpu_get_parent, - .set_parent = clk_pm_cpu_set_parent, .round_rate = clk_pm_cpu_round_rate, .set_rate = clk_pm_cpu_set_rate, .recalc_rate = clk_pm_cpu_recalc_rate, diff --git a/drivers/clk/qcom/a53-pll.c b/drivers/clk/qcom/a53-pll.c index 45cfc57bff92..af6ac17c7dae 100644 --- a/drivers/clk/qcom/a53-pll.c +++ b/drivers/clk/qcom/a53-pll.c @@ -93,6 +93,7 @@ static const struct of_device_id qcom_a53pll_match_table[] = { { .compatible = "qcom,msm8916-a53pll" }, { } }; +MODULE_DEVICE_TABLE(of, qcom_a53pll_match_table); static struct platform_driver qcom_a53pll_driver = { .probe = qcom_a53pll_probe, diff --git a/drivers/clk/qcom/a7-pll.c b/drivers/clk/qcom/a7-pll.c index e171d3caf2cf..c4a53e5db229 100644 --- a/drivers/clk/qcom/a7-pll.c +++ b/drivers/clk/qcom/a7-pll.c @@ -86,6 +86,7 @@ static const struct of_device_id qcom_a7pll_match_table[] = { { .compatible = "qcom,sdx55-a7pll" }, { } }; +MODULE_DEVICE_TABLE(of, qcom_a7pll_match_table); static struct platform_driver qcom_a7pll_driver = { .probe = qcom_a7pll_probe, diff --git a/drivers/clk/qcom/apss-ipq-pll.c b/drivers/clk/qcom/apss-ipq-pll.c index 30be87fb222a..bef7899ad0d6 100644 --- a/drivers/clk/qcom/apss-ipq-pll.c +++ b/drivers/clk/qcom/apss-ipq-pll.c @@ -81,6 +81,7 @@ static const struct of_device_id apss_ipq_pll_match_table[] = { { .compatible = "qcom,ipq6018-a53pll" }, { } }; +MODULE_DEVICE_TABLE(of, apss_ipq_pll_match_table); static struct platform_driver apss_ipq_pll_driver = { .probe = apss_ipq_pll_probe, diff --git a/drivers/clk/samsung/clk-exynos7.c b/drivers/clk/samsung/clk-exynos7.c index 87ee1bad9a9a..4a5d2a914bd6 100644 --- a/drivers/clk/samsung/clk-exynos7.c +++ b/drivers/clk/samsung/clk-exynos7.c @@ -537,8 +537,13 @@ static const struct samsung_gate_clock top1_gate_clks[] __initconst = { GATE(CLK_ACLK_FSYS0_200, "aclk_fsys0_200", "dout_aclk_fsys0_200", ENABLE_ACLK_TOP13, 28, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0), + /* + * This clock is required for the CMU_FSYS1 registers access, keep it + * enabled permanently until proper runtime PM support is added. + */ GATE(CLK_ACLK_FSYS1_200, "aclk_fsys1_200", "dout_aclk_fsys1_200", - ENABLE_ACLK_TOP13, 24, CLK_SET_RATE_PARENT, 0), + ENABLE_ACLK_TOP13, 24, CLK_SET_RATE_PARENT | + CLK_IS_CRITICAL, 0), GATE(CLK_SCLK_PHY_FSYS1_26M, "sclk_phy_fsys1_26m", "dout_sclk_phy_fsys1_26m", ENABLE_SCLK_TOP1_FSYS11, diff --git a/drivers/clk/socfpga/clk-gate-a10.c b/drivers/clk/socfpga/clk-gate-a10.c index cd5df9103614..d62778884208 100644 --- a/drivers/clk/socfpga/clk-gate-a10.c +++ b/drivers/clk/socfpga/clk-gate-a10.c @@ -146,6 +146,7 @@ static void __init __socfpga_gate_init(struct device_node *node, if (IS_ERR(socfpga_clk->sys_mgr_base_addr)) { pr_err("%s: failed to find altr,sys-mgr regmap!\n", __func__); + kfree(socfpga_clk); return; } } diff --git a/drivers/clk/uniphier/clk-uniphier-mux.c b/drivers/clk/uniphier/clk-uniphier-mux.c index 462c84321b2d..1998e9d4cfc0 100644 --- a/drivers/clk/uniphier/clk-uniphier-mux.c +++ b/drivers/clk/uniphier/clk-uniphier-mux.c @@ -31,10 +31,10 @@ static int uniphier_clk_mux_set_parent(struct clk_hw *hw, u8 index) static u8 uniphier_clk_mux_get_parent(struct clk_hw *hw) { struct uniphier_clk_mux *mux = to_uniphier_clk_mux(hw); - int num_parents = clk_hw_get_num_parents(hw); + unsigned int num_parents = clk_hw_get_num_parents(hw); int ret; unsigned int val; - u8 i; + unsigned int i; ret = regmap_read(mux->regmap, mux->reg, &val); if (ret) diff --git a/drivers/clk/zynqmp/pll.c b/drivers/clk/zynqmp/pll.c index 92f449ed38e5..abe6afbf3407 100644 --- a/drivers/clk/zynqmp/pll.c +++ b/drivers/clk/zynqmp/pll.c @@ -14,10 +14,12 @@ * struct zynqmp_pll - PLL clock * @hw: Handle between common and hardware-specific interfaces * @clk_id: PLL clock ID + * @set_pll_mode: Whether an IOCTL_SET_PLL_FRAC_MODE request be sent to ATF */ struct zynqmp_pll { struct clk_hw hw; u32 clk_id; + bool set_pll_mode; }; #define to_zynqmp_pll(_hw) container_of(_hw, struct zynqmp_pll, hw) @@ -81,6 +83,8 @@ static inline void zynqmp_pll_set_mode(struct clk_hw *hw, bool on) if (ret) pr_warn_once("%s() PLL set frac mode failed for %s, ret = %d\n", __func__, clk_name, ret); + else + clk->set_pll_mode = true; } /** @@ -100,9 +104,7 @@ static long zynqmp_pll_round_rate(struct clk_hw *hw, unsigned long rate, /* Enable the fractional mode if needed */ rate_div = (rate * FRAC_DIV) / *prate; f = rate_div % FRAC_DIV; - zynqmp_pll_set_mode(hw, !!f); - - if (zynqmp_pll_get_mode(hw) == PLL_MODE_FRAC) { + if (f) { if (rate > PS_PLL_VCO_MAX) { fbdiv = rate / PS_PLL_VCO_MAX; rate = rate / (fbdiv + 1); @@ -173,10 +175,12 @@ static int zynqmp_pll_set_rate(struct clk_hw *hw, unsigned long rate, long rate_div, frac, m, f; int ret; - if (zynqmp_pll_get_mode(hw) == PLL_MODE_FRAC) { - rate_div = (rate * FRAC_DIV) / parent_rate; + rate_div = (rate * FRAC_DIV) / parent_rate; + f = rate_div % FRAC_DIV; + zynqmp_pll_set_mode(hw, !!f); + + if (f) { m = rate_div / FRAC_DIV; - f = rate_div % FRAC_DIV; m = clamp_t(u32, m, (PLL_FBDIV_MIN), (PLL_FBDIV_MAX)); rate = parent_rate * m; frac = (parent_rate * f) / FRAC_DIV; @@ -240,9 +244,15 @@ static int zynqmp_pll_enable(struct clk_hw *hw) u32 clk_id = clk->clk_id; int ret; - if (zynqmp_pll_is_enabled(hw)) + /* + * Don't skip enabling clock if there is an IOCTL_SET_PLL_FRAC_MODE request + * that has been sent to ATF. + */ + if (zynqmp_pll_is_enabled(hw) && (!clk->set_pll_mode)) return 0; + clk->set_pll_mode = false; + ret = zynqmp_pm_clock_enable(clk_id); if (ret) pr_warn_once("%s() clock enable failed for %s, ret = %d\n", diff --git a/drivers/clocksource/dw_apb_timer_of.c b/drivers/clocksource/dw_apb_timer_of.c index 42e7e43b8fcd..b1e2b697b21b 100644 --- a/drivers/clocksource/dw_apb_timer_of.c +++ b/drivers/clocksource/dw_apb_timer_of.c @@ -52,18 +52,34 @@ static int __init timer_get_base_and_rate(struct device_node *np, return 0; timer_clk = of_clk_get_by_name(np, "timer"); - if (IS_ERR(timer_clk)) - return PTR_ERR(timer_clk); + if (IS_ERR(timer_clk)) { + ret = PTR_ERR(timer_clk); + goto out_pclk_disable; + } ret = clk_prepare_enable(timer_clk); if (ret) - return ret; + goto out_timer_clk_put; *rate = clk_get_rate(timer_clk); - if (!(*rate)) - return -EINVAL; + if (!(*rate)) { + ret = -EINVAL; + goto out_timer_clk_disable; + } return 0; + +out_timer_clk_disable: + clk_disable_unprepare(timer_clk); +out_timer_clk_put: + clk_put(timer_clk); +out_pclk_disable: + if (!IS_ERR(pclk)) { + clk_disable_unprepare(pclk); + clk_put(pclk); + } + iounmap(*base); + return ret; } static int __init add_clockevent(struct device_node *event_timer) diff --git a/drivers/clocksource/ingenic-ost.c b/drivers/clocksource/ingenic-ost.c index 029efc2731b4..6af2470136bd 100644 --- a/drivers/clocksource/ingenic-ost.c +++ b/drivers/clocksource/ingenic-ost.c @@ -88,9 +88,9 @@ static int __init ingenic_ost_probe(struct platform_device *pdev) return PTR_ERR(ost->regs); map = device_node_to_regmap(dev->parent->of_node); - if (!map) { + if (IS_ERR(map)) { dev_err(dev, "regmap not found"); - return -EINVAL; + return PTR_ERR(map); } ost->clk = devm_clk_get(dev, "ost"); diff --git a/drivers/clocksource/timer-ti-dm-systimer.c b/drivers/clocksource/timer-ti-dm-systimer.c index 33b3e8aa2cc5..b6f97960d8ee 100644 --- a/drivers/clocksource/timer-ti-dm-systimer.c +++ b/drivers/clocksource/timer-ti-dm-systimer.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -449,13 +450,13 @@ static int dmtimer_set_next_event(unsigned long cycles, struct dmtimer_systimer *t = &clkevt->t; void __iomem *pend = t->base + t->pend; - writel_relaxed(0xffffffff - cycles, t->base + t->counter); while (readl_relaxed(pend) & WP_TCRR) cpu_relax(); + writel_relaxed(0xffffffff - cycles, t->base + t->counter); - writel_relaxed(OMAP_TIMER_CTRL_ST, t->base + t->ctrl); while (readl_relaxed(pend) & WP_TCLR) cpu_relax(); + writel_relaxed(OMAP_TIMER_CTRL_ST, t->base + t->ctrl); return 0; } @@ -490,18 +491,18 @@ static int dmtimer_set_periodic(struct clock_event_device *evt) dmtimer_clockevent_shutdown(evt); /* Looks like we need to first set the load value separately */ - writel_relaxed(clkevt->period, t->base + t->load); while (readl_relaxed(pend) & WP_TLDR) cpu_relax(); + writel_relaxed(clkevt->period, t->base + t->load); - writel_relaxed(clkevt->period, t->base + t->counter); while (readl_relaxed(pend) & WP_TCRR) cpu_relax(); + writel_relaxed(clkevt->period, t->base + t->counter); - writel_relaxed(OMAP_TIMER_CTRL_AR | OMAP_TIMER_CTRL_ST, - t->base + t->ctrl); while (readl_relaxed(pend) & WP_TCLR) cpu_relax(); + writel_relaxed(OMAP_TIMER_CTRL_AR | OMAP_TIMER_CTRL_ST, + t->base + t->ctrl); return 0; } @@ -530,17 +531,17 @@ static void omap_clockevent_unidle(struct clock_event_device *evt) writel_relaxed(OMAP_TIMER_INT_OVERFLOW, t->base + t->wakeup); } -static int __init dmtimer_clockevent_init(struct device_node *np) +static int __init dmtimer_clkevt_init_common(struct dmtimer_clockevent *clkevt, + struct device_node *np, + unsigned int features, + const struct cpumask *cpumask, + const char *name, + int rating) { - struct dmtimer_clockevent *clkevt; struct clock_event_device *dev; struct dmtimer_systimer *t; int error; - clkevt = kzalloc(sizeof(*clkevt), GFP_KERNEL); - if (!clkevt) - return -ENOMEM; - t = &clkevt->t; dev = &clkevt->dev; @@ -548,24 +549,23 @@ static int __init dmtimer_clockevent_init(struct device_node *np) * We mostly use cpuidle_coupled with ARM local timers for runtime, * so there's probably no use for CLOCK_EVT_FEAT_DYNIRQ here. */ - dev->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT; - dev->rating = 300; + dev->features = features; + dev->rating = rating; dev->set_next_event = dmtimer_set_next_event; dev->set_state_shutdown = dmtimer_clockevent_shutdown; dev->set_state_periodic = dmtimer_set_periodic; dev->set_state_oneshot = dmtimer_clockevent_shutdown; + dev->set_state_oneshot_stopped = dmtimer_clockevent_shutdown; dev->tick_resume = dmtimer_clockevent_shutdown; - dev->cpumask = cpu_possible_mask; + dev->cpumask = cpumask; dev->irq = irq_of_parse_and_map(np, 0); - if (!dev->irq) { - error = -ENXIO; - goto err_out_free; - } + if (!dev->irq) + return -ENXIO; error = dmtimer_systimer_setup(np, &clkevt->t); if (error) - goto err_out_free; + return error; clkevt->period = 0xffffffff - DIV_ROUND_CLOSEST(t->rate, HZ); @@ -577,38 +577,132 @@ static int __init dmtimer_clockevent_init(struct device_node *np) writel_relaxed(OMAP_TIMER_CTRL_POSTED, t->base + t->ifctrl); error = request_irq(dev->irq, dmtimer_clockevent_interrupt, - IRQF_TIMER, "clockevent", clkevt); + IRQF_TIMER, name, clkevt); if (error) goto err_out_unmap; writel_relaxed(OMAP_TIMER_INT_OVERFLOW, t->base + t->irq_ena); writel_relaxed(OMAP_TIMER_INT_OVERFLOW, t->base + t->wakeup); - pr_info("TI gptimer clockevent: %s%lu Hz at %pOF\n", - of_find_property(np, "ti,timer-alwon", NULL) ? + pr_info("TI gptimer %s: %s%lu Hz at %pOF\n", + name, of_find_property(np, "ti,timer-alwon", NULL) ? "always-on " : "", t->rate, np->parent); - clockevents_config_and_register(dev, t->rate, - 3, /* Timer internal resynch latency */ + return 0; + +err_out_unmap: + iounmap(t->base); + + return error; +} + +static int __init dmtimer_clockevent_init(struct device_node *np) +{ + struct dmtimer_clockevent *clkevt; + int error; + + clkevt = kzalloc(sizeof(*clkevt), GFP_KERNEL); + if (!clkevt) + return -ENOMEM; + + error = dmtimer_clkevt_init_common(clkevt, np, + CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_ONESHOT, + cpu_possible_mask, "clockevent", + 300); + if (error) + goto err_out_free; + + clockevents_config_and_register(&clkevt->dev, clkevt->t.rate, + 3, /* Timer internal resync latency */ 0xffffffff); if (of_machine_is_compatible("ti,am33xx") || of_machine_is_compatible("ti,am43")) { - dev->suspend = omap_clockevent_idle; - dev->resume = omap_clockevent_unidle; + clkevt->dev.suspend = omap_clockevent_idle; + clkevt->dev.resume = omap_clockevent_unidle; } return 0; -err_out_unmap: - iounmap(t->base); - err_out_free: kfree(clkevt); return error; } +/* Dmtimer as percpu timer. See dra7 ARM architected timer wrap erratum i940 */ +static DEFINE_PER_CPU(struct dmtimer_clockevent, dmtimer_percpu_timer); + +static int __init dmtimer_percpu_timer_init(struct device_node *np, int cpu) +{ + struct dmtimer_clockevent *clkevt; + int error; + + if (!cpu_possible(cpu)) + return -EINVAL; + + if (!of_property_read_bool(np->parent, "ti,no-reset-on-init") || + !of_property_read_bool(np->parent, "ti,no-idle")) + pr_warn("Incomplete dtb for percpu dmtimer %pOF\n", np->parent); + + clkevt = per_cpu_ptr(&dmtimer_percpu_timer, cpu); + + error = dmtimer_clkevt_init_common(clkevt, np, CLOCK_EVT_FEAT_ONESHOT, + cpumask_of(cpu), "percpu-dmtimer", + 500); + if (error) + return error; + + return 0; +} + +/* See TRM for timer internal resynch latency */ +static int omap_dmtimer_starting_cpu(unsigned int cpu) +{ + struct dmtimer_clockevent *clkevt = per_cpu_ptr(&dmtimer_percpu_timer, cpu); + struct clock_event_device *dev = &clkevt->dev; + struct dmtimer_systimer *t = &clkevt->t; + + clockevents_config_and_register(dev, t->rate, 3, ULONG_MAX); + irq_force_affinity(dev->irq, cpumask_of(cpu)); + + return 0; +} + +static int __init dmtimer_percpu_timer_startup(void) +{ + struct dmtimer_clockevent *clkevt = per_cpu_ptr(&dmtimer_percpu_timer, 0); + struct dmtimer_systimer *t = &clkevt->t; + + if (t->sysc) { + cpuhp_setup_state(CPUHP_AP_TI_GP_TIMER_STARTING, + "clockevents/omap/gptimer:starting", + omap_dmtimer_starting_cpu, NULL); + } + + return 0; +} +subsys_initcall(dmtimer_percpu_timer_startup); + +static int __init dmtimer_percpu_quirk_init(struct device_node *np, u32 pa) +{ + struct device_node *arm_timer; + + arm_timer = of_find_compatible_node(NULL, NULL, "arm,armv7-timer"); + if (of_device_is_available(arm_timer)) { + pr_warn_once("ARM architected timer wrap issue i940 detected\n"); + return 0; + } + + if (pa == 0x48034000) /* dra7 dmtimer3 */ + return dmtimer_percpu_timer_init(np, 0); + else if (pa == 0x48036000) /* dra7 dmtimer4 */ + return dmtimer_percpu_timer_init(np, 1); + + return 0; +} + /* Clocksource */ static struct dmtimer_clocksource * to_dmtimer_clocksource(struct clocksource *cs) @@ -742,6 +836,9 @@ static int __init dmtimer_systimer_init(struct device_node *np) if (clockevent == pa) return dmtimer_clockevent_init(np); + if (of_machine_is_compatible("ti,dra7")) + return dmtimer_percpu_quirk_init(np, pa); + return 0; } diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c index d1bbc16fba4b..7e7450453714 100644 --- a/drivers/cpufreq/acpi-cpufreq.c +++ b/drivers/cpufreq/acpi-cpufreq.c @@ -646,7 +646,11 @@ static u64 get_max_boost_ratio(unsigned int cpu) return 0; } - highest_perf = perf_caps.highest_perf; + if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) + highest_perf = amd_get_highest_perf(); + else + highest_perf = perf_caps.highest_perf; + nominal_perf = perf_caps.nominal_perf; if (!highest_perf || !nominal_perf) { diff --git a/drivers/cpufreq/armada-37xx-cpufreq.c b/drivers/cpufreq/armada-37xx-cpufreq.c index b4af4094309b..e4782f562e7a 100644 --- a/drivers/cpufreq/armada-37xx-cpufreq.c +++ b/drivers/cpufreq/armada-37xx-cpufreq.c @@ -25,6 +25,10 @@ #include "cpufreq-dt.h" +/* Clk register set */ +#define ARMADA_37XX_CLK_TBG_SEL 0 +#define ARMADA_37XX_CLK_TBG_SEL_CPU_OFF 22 + /* Power management in North Bridge register set */ #define ARMADA_37XX_NB_L0L1 0x18 #define ARMADA_37XX_NB_L2L3 0x1C @@ -69,6 +73,8 @@ #define LOAD_LEVEL_NR 4 #define MIN_VOLT_MV 1000 +#define MIN_VOLT_MV_FOR_L1_1000MHZ 1108 +#define MIN_VOLT_MV_FOR_L1_1200MHZ 1155 /* AVS value for the corresponding voltage (in mV) */ static int avs_map[] = { @@ -120,10 +126,15 @@ static struct armada_37xx_dvfs *armada_37xx_cpu_freq_info_get(u32 freq) * will be configured then the DVFS will be enabled. */ static void __init armada37xx_cpufreq_dvfs_setup(struct regmap *base, - struct clk *clk, u8 *divider) + struct regmap *clk_base, u8 *divider) { + u32 cpu_tbg_sel; int load_lvl; - struct clk *parent; + + /* Determine to which TBG clock is CPU connected */ + regmap_read(clk_base, ARMADA_37XX_CLK_TBG_SEL, &cpu_tbg_sel); + cpu_tbg_sel >>= ARMADA_37XX_CLK_TBG_SEL_CPU_OFF; + cpu_tbg_sel &= ARMADA_37XX_NB_TBG_SEL_MASK; for (load_lvl = 0; load_lvl < LOAD_LEVEL_NR; load_lvl++) { unsigned int reg, mask, val, offset = 0; @@ -142,6 +153,11 @@ static void __init armada37xx_cpufreq_dvfs_setup(struct regmap *base, mask = (ARMADA_37XX_NB_CLK_SEL_MASK << ARMADA_37XX_NB_CLK_SEL_OFF); + /* Set TBG index, for all levels we use the same TBG */ + val = cpu_tbg_sel << ARMADA_37XX_NB_TBG_SEL_OFF; + mask = (ARMADA_37XX_NB_TBG_SEL_MASK + << ARMADA_37XX_NB_TBG_SEL_OFF); + /* * Set cpu divider based on the pre-computed array in * order to have balanced step. @@ -160,14 +176,6 @@ static void __init armada37xx_cpufreq_dvfs_setup(struct regmap *base, regmap_update_bits(base, reg, mask, val); } - - /* - * Set cpu clock source, for all the level we keep the same - * clock source that the one already configured. For this one - * we need to use the clock framework - */ - parent = clk_get_parent(clk); - clk_set_parent(clk, parent); } /* @@ -202,6 +210,8 @@ static u32 armada_37xx_avs_val_match(int target_vm) * - L2 & L3 voltage should be about 150mv smaller than L0 voltage. * This function calculates L1 & L2 & L3 AVS values dynamically based * on L0 voltage and fill all AVS values to the AVS value table. + * When base CPU frequency is 1000 or 1200 MHz then there is additional + * minimal avs value for load L1. */ static void __init armada37xx_cpufreq_avs_configure(struct regmap *base, struct armada_37xx_dvfs *dvfs) @@ -233,6 +243,19 @@ static void __init armada37xx_cpufreq_avs_configure(struct regmap *base, for (load_level = 1; load_level < LOAD_LEVEL_NR; load_level++) dvfs->avs[load_level] = avs_min; + /* + * Set the avs values for load L0 and L1 when base CPU frequency + * is 1000/1200 MHz to its typical initial values according to + * the Armada 3700 Hardware Specifications. + */ + if (dvfs->cpu_freq_max >= 1000*1000*1000) { + if (dvfs->cpu_freq_max >= 1200*1000*1000) + avs_min = armada_37xx_avs_val_match(MIN_VOLT_MV_FOR_L1_1200MHZ); + else + avs_min = armada_37xx_avs_val_match(MIN_VOLT_MV_FOR_L1_1000MHZ); + dvfs->avs[0] = dvfs->avs[1] = avs_min; + } + return; } @@ -252,6 +275,26 @@ static void __init armada37xx_cpufreq_avs_configure(struct regmap *base, target_vm = avs_map[l0_vdd_min] - 150; target_vm = target_vm > MIN_VOLT_MV ? target_vm : MIN_VOLT_MV; dvfs->avs[2] = dvfs->avs[3] = armada_37xx_avs_val_match(target_vm); + + /* + * Fix the avs value for load L1 when base CPU frequency is 1000/1200 MHz, + * otherwise the CPU gets stuck when switching from load L1 to load L0. + * Also ensure that avs value for load L1 is not higher than for L0. + */ + if (dvfs->cpu_freq_max >= 1000*1000*1000) { + u32 avs_min_l1; + + if (dvfs->cpu_freq_max >= 1200*1000*1000) + avs_min_l1 = armada_37xx_avs_val_match(MIN_VOLT_MV_FOR_L1_1200MHZ); + else + avs_min_l1 = armada_37xx_avs_val_match(MIN_VOLT_MV_FOR_L1_1000MHZ); + + if (avs_min_l1 > dvfs->avs[0]) + avs_min_l1 = dvfs->avs[0]; + + if (dvfs->avs[1] < avs_min_l1) + dvfs->avs[1] = avs_min_l1; + } } static void __init armada37xx_cpufreq_avs_setup(struct regmap *base, @@ -358,11 +401,16 @@ static int __init armada37xx_cpufreq_driver_init(void) struct platform_device *pdev; unsigned long freq; unsigned int cur_frequency, base_frequency; - struct regmap *nb_pm_base, *avs_base; + struct regmap *nb_clk_base, *nb_pm_base, *avs_base; struct device *cpu_dev; int load_lvl, ret; struct clk *clk, *parent; + nb_clk_base = + syscon_regmap_lookup_by_compatible("marvell,armada-3700-periph-clock-nb"); + if (IS_ERR(nb_clk_base)) + return -ENODEV; + nb_pm_base = syscon_regmap_lookup_by_compatible("marvell,armada-3700-nb-pm"); @@ -421,7 +469,7 @@ static int __init armada37xx_cpufreq_driver_init(void) return -EINVAL; } - dvfs = armada_37xx_cpu_freq_info_get(cur_frequency); + dvfs = armada_37xx_cpu_freq_info_get(base_frequency); if (!dvfs) { clk_put(clk); return -EINVAL; @@ -439,7 +487,7 @@ static int __init armada37xx_cpufreq_driver_init(void) armada37xx_cpufreq_avs_configure(avs_base, dvfs); armada37xx_cpufreq_avs_setup(avs_base, dvfs); - armada37xx_cpufreq_dvfs_setup(nb_pm_base, clk, dvfs->divider); + armada37xx_cpufreq_dvfs_setup(nb_pm_base, nb_clk_base, dvfs->divider); clk_put(clk); for (load_lvl = ARMADA_37XX_DVFS_LOAD_0; load_lvl < LOAD_LEVEL_NR; @@ -473,7 +521,7 @@ static int __init armada37xx_cpufreq_driver_init(void) remove_opp: /* clean-up the already added opp before leaving */ while (load_lvl-- > ARMADA_37XX_DVFS_LOAD_0) { - freq = cur_frequency / dvfs->divider[load_lvl]; + freq = base_frequency / dvfs->divider[load_lvl]; dev_pm_opp_remove(cpu_dev, freq); } diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 5175ae3cac44..34196c107de6 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -3054,6 +3054,14 @@ static const struct x86_cpu_id hwp_support_ids[] __initconst = { {} }; +static bool intel_pstate_hwp_is_enabled(void) +{ + u64 value; + + rdmsrl(MSR_PM_ENABLE, value); + return !!(value & 0x1); +} + static int __init intel_pstate_init(void) { const struct x86_cpu_id *id; @@ -3072,8 +3080,12 @@ static int __init intel_pstate_init(void) * Avoid enabling HWP for processors without EPP support, * because that means incomplete HWP implementation which is a * corner case and supporting it is generally problematic. + * + * If HWP is enabled already, though, there is no choice but to + * deal with it. */ - if (!no_hwp && boot_cpu_has(X86_FEATURE_HWP_EPP)) { + if ((!no_hwp && boot_cpu_has(X86_FEATURE_HWP_EPP)) || + intel_pstate_hwp_is_enabled()) { hwp_active++; hwp_mode_bdw = id->driver_data; intel_pstate.attr = hwp_cpufreq_attrs; diff --git a/drivers/cpuidle/Kconfig.arm b/drivers/cpuidle/Kconfig.arm index 0844fadc4be8..334f83e56120 100644 --- a/drivers/cpuidle/Kconfig.arm +++ b/drivers/cpuidle/Kconfig.arm @@ -107,7 +107,7 @@ config ARM_TEGRA_CPUIDLE config ARM_QCOM_SPM_CPUIDLE bool "CPU Idle Driver for Qualcomm Subsystem Power Manager (SPM)" - depends on (ARCH_QCOM || COMPILE_TEST) && !ARM64 + depends on (ARCH_QCOM || COMPILE_TEST) && !ARM64 && MMU select ARM_CPU_SUSPEND select CPU_IDLE_MULTIPLE_DRIVERS select DT_IDLE_STATES diff --git a/drivers/cpuidle/cpuidle-tegra.c b/drivers/cpuidle/cpuidle-tegra.c index 191966dc8d02..29c5e83500d3 100644 --- a/drivers/cpuidle/cpuidle-tegra.c +++ b/drivers/cpuidle/cpuidle-tegra.c @@ -135,13 +135,13 @@ static int tegra_cpuidle_c7_enter(void) { int err; - if (tegra_cpuidle_using_firmware()) { - err = call_firmware_op(prepare_idle, TF_PM_MODE_LP2_NOFLUSH_L2); - if (err) - return err; + err = call_firmware_op(prepare_idle, TF_PM_MODE_LP2_NOFLUSH_L2); + if (err && err != -ENOSYS) + return err; - return call_firmware_op(do_idle, 0); - } + err = call_firmware_op(do_idle, 0); + if (err != -ENOSYS) + return err; return cpu_suspend(0, tegra30_pm_secondary_cpu_suspend); } diff --git a/drivers/crypto/allwinner/Kconfig b/drivers/crypto/allwinner/Kconfig index 856fb2045656..b8e75210a0e3 100644 --- a/drivers/crypto/allwinner/Kconfig +++ b/drivers/crypto/allwinner/Kconfig @@ -71,10 +71,10 @@ config CRYPTO_DEV_SUN8I_CE_DEBUG config CRYPTO_DEV_SUN8I_CE_HASH bool "Enable support for hash on sun8i-ce" depends on CRYPTO_DEV_SUN8I_CE - select MD5 - select SHA1 - select SHA256 - select SHA512 + select CRYPTO_MD5 + select CRYPTO_SHA1 + select CRYPTO_SHA256 + select CRYPTO_SHA512 help Say y to enable support for hash algorithms. @@ -132,8 +132,8 @@ config CRYPTO_DEV_SUN8I_SS_PRNG config CRYPTO_DEV_SUN8I_SS_HASH bool "Enable support for hash on sun8i-ss" depends on CRYPTO_DEV_SUN8I_SS - select MD5 - select SHA1 - select SHA256 + select CRYPTO_MD5 + select CRYPTO_SHA1 + select CRYPTO_SHA256 help Say y to enable support for hash algorithms. diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-cipher.c b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-cipher.c index c2e6f5ed1d79..dec79fa3ebaf 100644 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-cipher.c +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-cipher.c @@ -561,7 +561,7 @@ int sun4i_ss_cipher_init(struct crypto_tfm *tfm) sizeof(struct sun4i_cipher_req_ctx) + crypto_skcipher_reqsize(op->fallback_tfm)); - err = pm_runtime_get_sync(op->ss->dev); + err = pm_runtime_resume_and_get(op->ss->dev); if (err < 0) goto error_pm; diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c index 709905ec4680..02a2d34845f2 100644 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c @@ -459,7 +459,7 @@ static int sun4i_ss_probe(struct platform_device *pdev) * this info could be useful */ - err = pm_runtime_get_sync(ss->dev); + err = pm_runtime_resume_and_get(ss->dev); if (err < 0) goto error_pm; diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-hash.c b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-hash.c index c1b4585e9bbc..d28292762b32 100644 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-hash.c +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-hash.c @@ -27,7 +27,7 @@ int sun4i_hash_crainit(struct crypto_tfm *tfm) algt = container_of(alg, struct sun4i_ss_alg_template, alg.hash); op->ss = algt->ss; - err = pm_runtime_get_sync(op->ss->dev); + err = pm_runtime_resume_and_get(op->ss->dev); if (err < 0) return err; diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-prng.c b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-prng.c index 443160a114bb..491fcb7b81b4 100644 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-prng.c +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-prng.c @@ -29,7 +29,7 @@ int sun4i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src, algt = container_of(alg, struct sun4i_ss_alg_template, alg.rng); ss = algt->ss; - err = pm_runtime_get_sync(ss->dev); + err = pm_runtime_resume_and_get(ss->dev); if (err < 0) return err; diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c index 158422ff5695..00194d1d9ae6 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c @@ -932,7 +932,7 @@ static int sun8i_ce_probe(struct platform_device *pdev) if (err) goto error_alg; - err = pm_runtime_get_sync(ce->dev); + err = pm_runtime_resume_and_get(ce->dev); if (err < 0) goto error_alg; diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c index ed2a69f82e1c..7c355bc2fb06 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c @@ -351,7 +351,7 @@ int sun8i_ss_cipher_init(struct crypto_tfm *tfm) op->enginectx.op.prepare_request = NULL; op->enginectx.op.unprepare_request = NULL; - err = pm_runtime_get_sync(op->ss->dev); + err = pm_runtime_resume_and_get(op->ss->dev); if (err < 0) { dev_err(op->ss->dev, "pm error %d\n", err); goto error_pm; diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c index e0ddc684798d..80e89066dbd1 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c @@ -753,7 +753,7 @@ static int sun8i_ss_probe(struct platform_device *pdev) if (err) goto error_alg; - err = pm_runtime_get_sync(ss->dev); + err = pm_runtime_resume_and_get(ss->dev); if (err < 0) goto error_alg; diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c index 11cbcbc83a7b..64446b86c927 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c @@ -348,8 +348,10 @@ int sun8i_ss_hash_run(struct crypto_engine *engine, void *breq) bf = (__le32 *)pad; result = kzalloc(digestsize, GFP_KERNEL | GFP_DMA); - if (!result) + if (!result) { + kfree(pad); return -ENOMEM; + } for (i = 0; i < MAX_SG; i++) { rctx->t_dst[i].addr = 0; @@ -435,11 +437,10 @@ int sun8i_ss_hash_run(struct crypto_engine *engine, void *breq) dma_unmap_sg(ss->dev, areq->src, nr_sgs, DMA_TO_DEVICE); dma_unmap_single(ss->dev, addr_res, digestsize, DMA_FROM_DEVICE); - kfree(pad); - memcpy(areq->result, result, algt->alg.hash.halg.digestsize); - kfree(result); theend: + kfree(pad); + kfree(result); crypto_finalize_hash_request(engine, breq, err); return 0; } diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-prng.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-prng.c index 08a1473b2145..3191527928e4 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-prng.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-prng.c @@ -103,7 +103,8 @@ int sun8i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src, dma_iv = dma_map_single(ss->dev, ctx->seed, ctx->slen, DMA_TO_DEVICE); if (dma_mapping_error(ss->dev, dma_iv)) { dev_err(ss->dev, "Cannot DMA MAP IV\n"); - return -EFAULT; + err = -EFAULT; + goto err_free; } dma_dst = dma_map_single(ss->dev, d, todo, DMA_FROM_DEVICE); @@ -167,6 +168,7 @@ int sun8i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src, memcpy(ctx->seed, d + dlen, ctx->slen); } memzero_explicit(d, todo); +err_free: kfree(d); return err; diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index cb9b4c4e371e..3e0d1d6922ba 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -150,6 +150,9 @@ static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret) sev = psp->sev_data; + if (data && WARN_ON_ONCE(!virt_addr_valid(data))) + return -EINVAL; + /* Get the physical address of the command buffer */ phys_lsb = data ? lower_32_bits(__psp_pa(data)) : 0; phys_msb = data ? upper_32_bits(__psp_pa(data)) : 0; @@ -987,7 +990,7 @@ int sev_dev_init(struct psp_device *psp) if (!sev->vdata) { ret = -ENODEV; dev_err(dev, "sev: missing driver data\n"); - goto e_err; + goto e_sev; } psp_set_sev_irq_handler(psp, sev_irq_handler, sev); @@ -1002,6 +1005,8 @@ int sev_dev_init(struct psp_device *psp) e_irq: psp_clear_sev_irq_handler(psp); +e_sev: + devm_kfree(dev, sev); e_err: psp->sev_data = NULL; diff --git a/drivers/crypto/ccp/tee-dev.c b/drivers/crypto/ccp/tee-dev.c index 5e697a90ea7f..bcb81fef4211 100644 --- a/drivers/crypto/ccp/tee-dev.c +++ b/drivers/crypto/ccp/tee-dev.c @@ -36,6 +36,7 @@ static int tee_alloc_ring(struct psp_tee_device *tee, int ring_size) if (!start_addr) return -ENOMEM; + memset(start_addr, 0x0, ring_size); rb_mgr->ring_start = start_addr; rb_mgr->ring_size = ring_size; rb_mgr->ring_pa = __psp_pa(start_addr); @@ -244,41 +245,54 @@ static int tee_submit_cmd(struct psp_tee_device *tee, enum tee_cmd_id cmd_id, void *buf, size_t len, struct tee_ring_cmd **resp) { struct tee_ring_cmd *cmd; - u32 rptr, wptr; int nloop = 1000, ret = 0; + u32 rptr; *resp = NULL; mutex_lock(&tee->rb_mgr.mutex); - wptr = tee->rb_mgr.wptr; - - /* Check if ring buffer is full */ + /* Loop until empty entry found in ring buffer */ do { + /* Get pointer to ring buffer command entry */ + cmd = (struct tee_ring_cmd *) + (tee->rb_mgr.ring_start + tee->rb_mgr.wptr); + rptr = ioread32(tee->io_regs + tee->vdata->ring_rptr_reg); - if (!(wptr + sizeof(struct tee_ring_cmd) == rptr)) + /* Check if ring buffer is full or command entry is waiting + * for response from TEE + */ + if (!(tee->rb_mgr.wptr + sizeof(struct tee_ring_cmd) == rptr || + cmd->flag == CMD_WAITING_FOR_RESPONSE)) break; - dev_info(tee->dev, "tee: ring buffer full. rptr = %u wptr = %u\n", - rptr, wptr); + dev_dbg(tee->dev, "tee: ring buffer full. rptr = %u wptr = %u\n", + rptr, tee->rb_mgr.wptr); - /* Wait if ring buffer is full */ + /* Wait if ring buffer is full or TEE is processing data */ mutex_unlock(&tee->rb_mgr.mutex); schedule_timeout_interruptible(msecs_to_jiffies(10)); mutex_lock(&tee->rb_mgr.mutex); } while (--nloop); - if (!nloop && (wptr + sizeof(struct tee_ring_cmd) == rptr)) { - dev_err(tee->dev, "tee: ring buffer full. rptr = %u wptr = %u\n", - rptr, wptr); + if (!nloop && + (tee->rb_mgr.wptr + sizeof(struct tee_ring_cmd) == rptr || + cmd->flag == CMD_WAITING_FOR_RESPONSE)) { + dev_err(tee->dev, "tee: ring buffer full. rptr = %u wptr = %u response flag %u\n", + rptr, tee->rb_mgr.wptr, cmd->flag); ret = -EBUSY; goto unlock; } - /* Pointer to empty data entry in ring buffer */ - cmd = (struct tee_ring_cmd *)(tee->rb_mgr.ring_start + wptr); + /* Do not submit command if PSP got disabled while processing any + * command in another thread + */ + if (psp_dead) { + ret = -EBUSY; + goto unlock; + } /* Write command data into ring buffer */ cmd->cmd_id = cmd_id; @@ -286,6 +300,9 @@ static int tee_submit_cmd(struct psp_tee_device *tee, enum tee_cmd_id cmd_id, memset(&cmd->buf[0], 0, sizeof(cmd->buf)); memcpy(&cmd->buf[0], buf, len); + /* Indicate driver is waiting for response */ + cmd->flag = CMD_WAITING_FOR_RESPONSE; + /* Update local copy of write pointer */ tee->rb_mgr.wptr += sizeof(struct tee_ring_cmd); if (tee->rb_mgr.wptr >= tee->rb_mgr.ring_size) @@ -353,12 +370,16 @@ int psp_tee_process_cmd(enum tee_cmd_id cmd_id, void *buf, size_t len, return ret; ret = tee_wait_cmd_completion(tee, resp, TEE_DEFAULT_TIMEOUT); - if (ret) + if (ret) { + resp->flag = CMD_RESPONSE_TIMEDOUT; return ret; + } memcpy(buf, &resp->buf[0], len); *status = resp->status; + resp->flag = CMD_RESPONSE_COPIED; + return 0; } EXPORT_SYMBOL(psp_tee_process_cmd); diff --git a/drivers/crypto/ccp/tee-dev.h b/drivers/crypto/ccp/tee-dev.h index f09960112115..49d26158b71e 100644 --- a/drivers/crypto/ccp/tee-dev.h +++ b/drivers/crypto/ccp/tee-dev.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: MIT */ /* - * Copyright 2019 Advanced Micro Devices, Inc. + * Copyright (C) 2019,2021 Advanced Micro Devices, Inc. * * Author: Rijo Thomas * Author: Devaraj Rangasamy @@ -18,7 +18,7 @@ #include #define TEE_DEFAULT_TIMEOUT 10 -#define MAX_BUFFER_SIZE 992 +#define MAX_BUFFER_SIZE 988 /** * enum tee_ring_cmd_id - TEE interface commands for ring buffer configuration @@ -81,6 +81,20 @@ enum tee_cmd_state { TEE_CMD_STATE_COMPLETED, }; +/** + * enum cmd_resp_state - TEE command's response status maintained by driver + * @CMD_RESPONSE_INVALID: initial state when no command is written to ring + * @CMD_WAITING_FOR_RESPONSE: driver waiting for response from TEE + * @CMD_RESPONSE_TIMEDOUT: failed to get response from TEE + * @CMD_RESPONSE_COPIED: driver has copied response from TEE + */ +enum cmd_resp_state { + CMD_RESPONSE_INVALID, + CMD_WAITING_FOR_RESPONSE, + CMD_RESPONSE_TIMEDOUT, + CMD_RESPONSE_COPIED, +}; + /** * struct tee_ring_cmd - Structure of the command buffer in TEE ring * @cmd_id: refers to &enum tee_cmd_id. Command id for the ring buffer @@ -91,6 +105,7 @@ enum tee_cmd_state { * @pdata: private data (currently unused) * @res1: reserved region * @buf: TEE command specific buffer + * @flag: refers to &enum cmd_resp_state */ struct tee_ring_cmd { u32 cmd_id; @@ -100,6 +115,7 @@ struct tee_ring_cmd { u64 pdata; u32 res1[2]; u8 buf[MAX_BUFFER_SIZE]; + u32 flag; /* Total size: 1024 bytes */ } __packed; diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index f5a336634daa..405ff957b837 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -769,13 +769,14 @@ static inline void create_wreq(struct chcr_context *ctx, struct uld_ctx *u_ctx = ULD_CTX(ctx); unsigned int tx_channel_id, rx_channel_id; unsigned int txqidx = 0, rxqidx = 0; - unsigned int qid, fid; + unsigned int qid, fid, portno; get_qidxs(req, &txqidx, &rxqidx); qid = u_ctx->lldi.rxq_ids[rxqidx]; fid = u_ctx->lldi.rxq_ids[0]; + portno = rxqidx / ctx->rxq_perchan; tx_channel_id = txqidx / ctx->txq_perchan; - rx_channel_id = rxqidx / ctx->rxq_perchan; + rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[portno]); chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE; @@ -806,6 +807,7 @@ static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(wrparam->req); struct chcr_context *ctx = c_ctx(tfm); + struct uld_ctx *u_ctx = ULD_CTX(ctx); struct ablk_ctx *ablkctx = ABLK_CTX(ctx); struct sk_buff *skb = NULL; struct chcr_wr *chcr_req; @@ -822,6 +824,7 @@ static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam) struct adapter *adap = padap(ctx->dev); unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan; + rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]); nents = sg_nents_xlen(reqctx->dstsg, wrparam->bytes, CHCR_DST_SG_SIZE, reqctx->dst_ofst); dst_size = get_space_for_phys_dsgl(nents); @@ -1580,6 +1583,7 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req, int error = 0; unsigned int rx_channel_id = req_ctx->rxqidx / ctx->rxq_perchan; + rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]); transhdr_len = HASH_TRANSHDR_SIZE(param->kctx_len); req_ctx->hctx_wr.imm = (transhdr_len + param->bfr_len + param->sg_len) <= SGE_MAX_WR_LEN; @@ -2438,6 +2442,7 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, { struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct chcr_context *ctx = a_ctx(tfm); + struct uld_ctx *u_ctx = ULD_CTX(ctx); struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); @@ -2457,6 +2462,7 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, struct adapter *adap = padap(ctx->dev); unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan; + rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]); if (req->cryptlen == 0) return NULL; @@ -2710,9 +2716,11 @@ void chcr_add_aead_dst_ent(struct aead_request *req, struct dsgl_walk dsgl_walk; unsigned int authsize = crypto_aead_authsize(tfm); struct chcr_context *ctx = a_ctx(tfm); + struct uld_ctx *u_ctx = ULD_CTX(ctx); u32 temp; unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan; + rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]); dsgl_walk_init(&dsgl_walk, phys_cpl); dsgl_walk_add_page(&dsgl_walk, IV + reqctx->b0_len, reqctx->iv_dma); temp = req->assoclen + req->cryptlen + @@ -2752,9 +2760,11 @@ void chcr_add_cipher_dst_ent(struct skcipher_request *req, struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(wrparam->req); struct chcr_context *ctx = c_ctx(tfm); + struct uld_ctx *u_ctx = ULD_CTX(ctx); struct dsgl_walk dsgl_walk; unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan; + rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]); dsgl_walk_init(&dsgl_walk, phys_cpl); dsgl_walk_add_sg(&dsgl_walk, reqctx->dstsg, wrparam->bytes, reqctx->dst_ofst); @@ -2958,6 +2968,7 @@ static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, { struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct chcr_context *ctx = a_ctx(tfm); + struct uld_ctx *u_ctx = ULD_CTX(ctx); struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM; @@ -2967,6 +2978,8 @@ static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, unsigned int tag_offset = 0, auth_offset = 0; unsigned int assoclen; + rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]); + if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) assoclen = req->assoclen - 8; else @@ -3127,6 +3140,7 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, { struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct chcr_context *ctx = a_ctx(tfm); + struct uld_ctx *u_ctx = ULD_CTX(ctx); struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); struct sk_buff *skb = NULL; @@ -3143,6 +3157,7 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, struct adapter *adap = padap(ctx->dev); unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan; + rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]); if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) assoclen = req->assoclen - 8; diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 2eaa516b3231..8adcbb327126 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -546,7 +546,7 @@ static int sec_skcipher_init(struct crypto_skcipher *tfm) crypto_skcipher_set_reqsize(tfm, sizeof(struct sec_req)); ctx->c_ctx.ivsize = crypto_skcipher_ivsize(tfm); if (ctx->c_ctx.ivsize > SEC_IV_SIZE) { - dev_err(SEC_CTX_DEV(ctx), "get error skcipher iv size!\n"); + pr_err("get error skcipher iv size!\n"); return -EINVAL; } diff --git a/drivers/crypto/keembay/keembay-ocs-aes-core.c b/drivers/crypto/keembay/keembay-ocs-aes-core.c index b6b25d994af3..2ef312866338 100644 --- a/drivers/crypto/keembay/keembay-ocs-aes-core.c +++ b/drivers/crypto/keembay/keembay-ocs-aes-core.c @@ -1649,8 +1649,10 @@ static int kmb_ocs_aes_probe(struct platform_device *pdev) /* Initialize crypto engine */ aes_dev->engine = crypto_engine_alloc_init(dev, true); - if (!aes_dev->engine) + if (!aes_dev->engine) { + rc = -ENOMEM; goto list_del; + } rc = crypto_engine_start(aes_dev->engine); if (rc) { diff --git a/drivers/crypto/keembay/keembay-ocs-hcu-core.c b/drivers/crypto/keembay/keembay-ocs-hcu-core.c index c4b97b4160e9..322c51a6936f 100644 --- a/drivers/crypto/keembay/keembay-ocs-hcu-core.c +++ b/drivers/crypto/keembay/keembay-ocs-hcu-core.c @@ -1220,8 +1220,10 @@ static int kmb_ocs_hcu_probe(struct platform_device *pdev) /* Initialize crypto engine */ hcu_dev->engine = crypto_engine_alloc_init(dev, 1); - if (!hcu_dev->engine) + if (!hcu_dev->engine) { + rc = -ENOMEM; goto list_del; + } rc = crypto_engine_start(hcu_dev->engine); if (rc) { diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index a45bdcf3026d..0dd4c6b157de 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -103,9 +103,8 @@ static int omap_aes_hw_init(struct omap_aes_dev *dd) dd->err = 0; } - err = pm_runtime_get_sync(dd->dev); + err = pm_runtime_resume_and_get(dd->dev); if (err < 0) { - pm_runtime_put_noidle(dd->dev); dev_err(dd->dev, "failed to get sync: %d\n", err); return err; } @@ -1134,7 +1133,7 @@ static int omap_aes_probe(struct platform_device *pdev) pm_runtime_set_autosuspend_delay(dev, DEFAULT_AUTOSUSPEND_DELAY); pm_runtime_enable(dev); - err = pm_runtime_get_sync(dev); + err = pm_runtime_resume_and_get(dev); if (err < 0) { dev_err(dev, "%s: failed to get_sync(%d)\n", __func__, err); @@ -1303,7 +1302,7 @@ static int omap_aes_suspend(struct device *dev) static int omap_aes_resume(struct device *dev) { - pm_runtime_get_sync(dev); + pm_runtime_resume_and_get(dev); return 0; } #endif diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c b/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c index 1d1532e8fb6d..067ca5e17d38 100644 --- a/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c +++ b/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c @@ -184,12 +184,12 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (ret) goto out_err_free_reg; - set_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status); - ret = adf_dev_init(accel_dev); if (ret) goto out_err_dev_shutdown; + set_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status); + ret = adf_dev_start(accel_dev); if (ret) goto out_err_dev_stop; diff --git a/drivers/crypto/qat/qat_c62xvf/adf_drv.c b/drivers/crypto/qat/qat_c62xvf/adf_drv.c index 04742a6d91ca..51ea88c0b17d 100644 --- a/drivers/crypto/qat/qat_c62xvf/adf_drv.c +++ b/drivers/crypto/qat/qat_c62xvf/adf_drv.c @@ -184,12 +184,12 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (ret) goto out_err_free_reg; - set_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status); - ret = adf_dev_init(accel_dev); if (ret) goto out_err_dev_shutdown; + set_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status); + ret = adf_dev_start(accel_dev); if (ret) goto out_err_dev_stop; diff --git a/drivers/crypto/qat/qat_common/adf_isr.c b/drivers/crypto/qat/qat_common/adf_isr.c index c45853463530..e3ad5587be49 100644 --- a/drivers/crypto/qat/qat_common/adf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_isr.c @@ -291,19 +291,32 @@ int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev) ret = adf_isr_alloc_msix_entry_table(accel_dev); if (ret) - return ret; - if (adf_enable_msix(accel_dev)) goto err_out; - if (adf_setup_bh(accel_dev)) - goto err_out; + ret = adf_enable_msix(accel_dev); + if (ret) + goto err_free_msix_table; - if (adf_request_irqs(accel_dev)) - goto err_out; + ret = adf_setup_bh(accel_dev); + if (ret) + goto err_disable_msix; + + ret = adf_request_irqs(accel_dev); + if (ret) + goto err_cleanup_bh; return 0; + +err_cleanup_bh: + adf_cleanup_bh(accel_dev); + +err_disable_msix: + adf_disable_msix(&accel_dev->accel_pci_dev); + +err_free_msix_table: + adf_isr_free_msix_entry_table(accel_dev); + err_out: - adf_isr_resource_free(accel_dev); - return -EFAULT; + return ret; } EXPORT_SYMBOL_GPL(adf_isr_resource_alloc); diff --git a/drivers/crypto/qat/qat_common/adf_transport.c b/drivers/crypto/qat/qat_common/adf_transport.c index 888c1e047295..8ba28409fb74 100644 --- a/drivers/crypto/qat/qat_common/adf_transport.c +++ b/drivers/crypto/qat/qat_common/adf_transport.c @@ -172,6 +172,7 @@ static int adf_init_ring(struct adf_etr_ring_data *ring) dev_err(&GET_DEV(accel_dev), "Ring address not aligned\n"); dma_free_coherent(&GET_DEV(accel_dev), ring_size_bytes, ring->base_addr, ring->dma_addr); + ring->base_addr = NULL; return -EFAULT; } diff --git a/drivers/crypto/qat/qat_common/adf_vf_isr.c b/drivers/crypto/qat/qat_common/adf_vf_isr.c index 38d316a42ba6..888388acb6bd 100644 --- a/drivers/crypto/qat/qat_common/adf_vf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_vf_isr.c @@ -261,17 +261,26 @@ int adf_vf_isr_resource_alloc(struct adf_accel_dev *accel_dev) goto err_out; if (adf_setup_pf2vf_bh(accel_dev)) - goto err_out; + goto err_disable_msi; if (adf_setup_bh(accel_dev)) - goto err_out; + goto err_cleanup_pf2vf_bh; if (adf_request_msi_irq(accel_dev)) - goto err_out; + goto err_cleanup_bh; return 0; + +err_cleanup_bh: + adf_cleanup_bh(accel_dev); + +err_cleanup_pf2vf_bh: + adf_cleanup_pf2vf_bh(accel_dev); + +err_disable_msi: + adf_disable_msi(accel_dev); + err_out: - adf_vf_isr_resource_free(accel_dev); return -EFAULT; } EXPORT_SYMBOL_GPL(adf_vf_isr_resource_alloc); diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index ff78c73c47e3..ea1c6899290d 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -719,7 +719,7 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, struct qat_alg_buf_list *bufl; struct qat_alg_buf_list *buflout = NULL; dma_addr_t blp; - dma_addr_t bloutp = 0; + dma_addr_t bloutp; struct scatterlist *sg; size_t sz_out, sz = struct_size(bufl, bufers, n + 1); @@ -731,6 +731,9 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, if (unlikely(!bufl)) return -ENOMEM; + for_each_sg(sgl, sg, n, i) + bufl->bufers[i].addr = DMA_MAPPING_ERROR; + blp = dma_map_single(dev, bufl, sz, DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, blp))) goto err_in; @@ -764,10 +767,14 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, dev_to_node(&GET_DEV(inst->accel_dev))); if (unlikely(!buflout)) goto err_in; + + bufers = buflout->bufers; + for_each_sg(sglout, sg, n, i) + bufers[i].addr = DMA_MAPPING_ERROR; + bloutp = dma_map_single(dev, buflout, sz_out, DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, bloutp))) goto err_out; - bufers = buflout->bufers; for_each_sg(sglout, sg, n, i) { int y = sg_nctr; diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c b/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c index c972554a755e..29999da716cc 100644 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c @@ -184,12 +184,12 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (ret) goto out_err_free_reg; - set_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status); - ret = adf_dev_init(accel_dev); if (ret) goto out_err_dev_shutdown; + set_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status); + ret = adf_dev_start(accel_dev); if (ret) goto out_err_dev_stop; diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c index f300b0a5958a..b0f0502a5bb0 100644 --- a/drivers/crypto/sa2ul.c +++ b/drivers/crypto/sa2ul.c @@ -1146,8 +1146,10 @@ static int sa_run(struct sa_req *req) mapped_sg->sgt.sgl = src; mapped_sg->sgt.orig_nents = src_nents; ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0); - if (ret) + if (ret) { + kfree(rxd); return ret; + } mapped_sg->dir = dir_src; mapped_sg->mapped = true; @@ -1155,8 +1157,10 @@ static int sa_run(struct sa_req *req) mapped_sg->sgt.sgl = req->src; mapped_sg->sgt.orig_nents = sg_nents; ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0); - if (ret) + if (ret) { + kfree(rxd); return ret; + } mapped_sg->dir = dir_src; mapped_sg->mapped = true; @@ -2350,7 +2354,7 @@ static int sa_ul_probe(struct platform_device *pdev) dev_set_drvdata(sa_k3_dev, dev_data); pm_runtime_enable(dev); - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0) { dev_err(&pdev->dev, "%s: failed to get sync: %d\n", __func__, ret); diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c index 2a4793176c71..7389a0536ff0 100644 --- a/drivers/crypto/stm32/stm32-cryp.c +++ b/drivers/crypto/stm32/stm32-cryp.c @@ -542,7 +542,7 @@ static int stm32_cryp_hw_init(struct stm32_cryp *cryp) int ret; u32 cfg, hw_mode; - pm_runtime_get_sync(cryp->dev); + pm_runtime_resume_and_get(cryp->dev); /* Disable interrupt */ stm32_cryp_write(cryp, CRYP_IMSCR, 0); @@ -2043,7 +2043,7 @@ static int stm32_cryp_remove(struct platform_device *pdev) if (!cryp) return -ENODEV; - ret = pm_runtime_get_sync(cryp->dev); + ret = pm_runtime_resume_and_get(cryp->dev); if (ret < 0) return ret; diff --git a/drivers/crypto/stm32/stm32-hash.c b/drivers/crypto/stm32/stm32-hash.c index 7ac0573ef663..389de9e3302d 100644 --- a/drivers/crypto/stm32/stm32-hash.c +++ b/drivers/crypto/stm32/stm32-hash.c @@ -813,7 +813,7 @@ static void stm32_hash_finish_req(struct ahash_request *req, int err) static int stm32_hash_hw_init(struct stm32_hash_dev *hdev, struct stm32_hash_request_ctx *rctx) { - pm_runtime_get_sync(hdev->dev); + pm_runtime_resume_and_get(hdev->dev); if (!(HASH_FLAGS_INIT & hdev->flags)) { stm32_hash_write(hdev, HASH_CR, HASH_CR_INIT); @@ -962,7 +962,7 @@ static int stm32_hash_export(struct ahash_request *req, void *out) u32 *preg; unsigned int i; - pm_runtime_get_sync(hdev->dev); + pm_runtime_resume_and_get(hdev->dev); while ((stm32_hash_read(hdev, HASH_SR) & HASH_SR_BUSY)) cpu_relax(); @@ -1000,7 +1000,7 @@ static int stm32_hash_import(struct ahash_request *req, const void *in) preg = rctx->hw_context; - pm_runtime_get_sync(hdev->dev); + pm_runtime_resume_and_get(hdev->dev); stm32_hash_write(hdev, HASH_IMR, *preg++); stm32_hash_write(hdev, HASH_STR, *preg++); @@ -1566,7 +1566,7 @@ static int stm32_hash_remove(struct platform_device *pdev) if (!hdev) return -ENODEV; - ret = pm_runtime_get_sync(hdev->dev); + ret = pm_runtime_resume_and_get(hdev->dev); if (ret < 0) return ret; diff --git a/drivers/devfreq/devfreq.c b/drivers/devfreq/devfreq.c index bf3047896e41..59ba59bea0f5 100644 --- a/drivers/devfreq/devfreq.c +++ b/drivers/devfreq/devfreq.c @@ -387,7 +387,7 @@ static int devfreq_set_target(struct devfreq *devfreq, unsigned long new_freq, devfreq->previous_freq = new_freq; if (devfreq->suspend_freq) - devfreq->resume_freq = cur_freq; + devfreq->resume_freq = new_freq; return err; } @@ -821,7 +821,8 @@ struct devfreq *devfreq_add_device(struct device *dev, if (devfreq->profile->timer < 0 || devfreq->profile->timer >= DEVFREQ_TIMER_NUM) { - goto err_out; + mutex_unlock(&devfreq->lock); + goto err_dev; } if (!devfreq->profile->max_state && !devfreq->profile->freq_table) { diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index 08d71dafa001..58c8cc8fe0e1 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -937,22 +937,21 @@ int dw_edma_remove(struct dw_edma_chip *chip) /* Power management */ pm_runtime_disable(dev); + /* Deregister eDMA device */ + dma_async_device_unregister(&dw->wr_edma); list_for_each_entry_safe(chan, _chan, &dw->wr_edma.channels, vc.chan.device_node) { - list_del(&chan->vc.chan.device_node); tasklet_kill(&chan->vc.task); + list_del(&chan->vc.chan.device_node); } + dma_async_device_unregister(&dw->rd_edma); list_for_each_entry_safe(chan, _chan, &dw->rd_edma.channels, vc.chan.device_node) { - list_del(&chan->vc.chan.device_node); tasklet_kill(&chan->vc.task); + list_del(&chan->vc.chan.device_node); } - /* Deregister eDMA device */ - dma_async_device_unregister(&dw->wr_edma); - dma_async_device_unregister(&dw->rd_edma); - /* Turn debugfs off */ dw_edma_v0_core_debugfs_off(); diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c index 0db9b82ed8cf..1d8a3876b745 100644 --- a/drivers/dma/idxd/cdev.c +++ b/drivers/dma/idxd/cdev.c @@ -39,15 +39,15 @@ struct idxd_user_context { struct iommu_sva *sva; }; -enum idxd_cdev_cleanup { - CDEV_NORMAL = 0, - CDEV_FAILED, -}; - static void idxd_cdev_dev_release(struct device *dev) { - dev_dbg(dev, "releasing cdev device\n"); - kfree(dev); + struct idxd_cdev *idxd_cdev = container_of(dev, struct idxd_cdev, dev); + struct idxd_cdev_context *cdev_ctx; + struct idxd_wq *wq = idxd_cdev->wq; + + cdev_ctx = &ictx[wq->idxd->type]; + ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor); + kfree(idxd_cdev); } static struct device_type idxd_cdev_device_type = { @@ -62,14 +62,11 @@ static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode) return container_of(cdev, struct idxd_cdev, cdev); } -static inline struct idxd_wq *idxd_cdev_wq(struct idxd_cdev *idxd_cdev) -{ - return container_of(idxd_cdev, struct idxd_wq, idxd_cdev); -} - static inline struct idxd_wq *inode_wq(struct inode *inode) { - return idxd_cdev_wq(inode_idxd_cdev(inode)); + struct idxd_cdev *idxd_cdev = inode_idxd_cdev(inode); + + return idxd_cdev->wq; } static int idxd_cdev_open(struct inode *inode, struct file *filp) @@ -220,11 +217,10 @@ static __poll_t idxd_cdev_poll(struct file *filp, struct idxd_user_context *ctx = filp->private_data; struct idxd_wq *wq = ctx->wq; struct idxd_device *idxd = wq->idxd; - struct idxd_cdev *idxd_cdev = &wq->idxd_cdev; unsigned long flags; __poll_t out = 0; - poll_wait(filp, &idxd_cdev->err_queue, wait); + poll_wait(filp, &wq->err_queue, wait); spin_lock_irqsave(&idxd->dev_lock, flags); if (idxd->sw_err.valid) out = EPOLLIN | EPOLLRDNORM; @@ -246,98 +242,67 @@ int idxd_cdev_get_major(struct idxd_device *idxd) return MAJOR(ictx[idxd->type].devt); } -static int idxd_wq_cdev_dev_setup(struct idxd_wq *wq) +int idxd_wq_add_cdev(struct idxd_wq *wq) { struct idxd_device *idxd = wq->idxd; - struct idxd_cdev *idxd_cdev = &wq->idxd_cdev; - struct idxd_cdev_context *cdev_ctx; + struct idxd_cdev *idxd_cdev; + struct cdev *cdev; struct device *dev; - int minor, rc; + struct idxd_cdev_context *cdev_ctx; + int rc, minor; - idxd_cdev->dev = kzalloc(sizeof(*idxd_cdev->dev), GFP_KERNEL); - if (!idxd_cdev->dev) + idxd_cdev = kzalloc(sizeof(*idxd_cdev), GFP_KERNEL); + if (!idxd_cdev) return -ENOMEM; - dev = idxd_cdev->dev; - dev->parent = &idxd->pdev->dev; - dev_set_name(dev, "%s/wq%u.%u", idxd_get_dev_name(idxd), - idxd->id, wq->id); - dev->bus = idxd_get_bus_type(idxd); - + idxd_cdev->wq = wq; + cdev = &idxd_cdev->cdev; + dev = &idxd_cdev->dev; cdev_ctx = &ictx[wq->idxd->type]; minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL); if (minor < 0) { - rc = minor; - kfree(dev); - goto ida_err; - } - - dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor); - dev->type = &idxd_cdev_device_type; - rc = device_register(dev); - if (rc < 0) { - dev_err(&idxd->pdev->dev, "device register failed\n"); - goto dev_reg_err; + kfree(idxd_cdev); + return minor; } idxd_cdev->minor = minor; - return 0; - - dev_reg_err: - ida_simple_remove(&cdev_ctx->minor_ida, MINOR(dev->devt)); - put_device(dev); - ida_err: - idxd_cdev->dev = NULL; - return rc; -} - -static void idxd_wq_cdev_cleanup(struct idxd_wq *wq, - enum idxd_cdev_cleanup cdev_state) -{ - struct idxd_cdev *idxd_cdev = &wq->idxd_cdev; - struct idxd_cdev_context *cdev_ctx; - - cdev_ctx = &ictx[wq->idxd->type]; - if (cdev_state == CDEV_NORMAL) - cdev_del(&idxd_cdev->cdev); - device_unregister(idxd_cdev->dev); - /* - * The device_type->release() will be called on the device and free - * the allocated struct device. We can just forget it. - */ - ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor); - idxd_cdev->dev = NULL; - idxd_cdev->minor = -1; -} - -int idxd_wq_add_cdev(struct idxd_wq *wq) -{ - struct idxd_cdev *idxd_cdev = &wq->idxd_cdev; - struct cdev *cdev = &idxd_cdev->cdev; - struct device *dev; - int rc; + device_initialize(dev); + dev->parent = &wq->conf_dev; + dev->bus = idxd_get_bus_type(idxd); + dev->type = &idxd_cdev_device_type; + dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor); - rc = idxd_wq_cdev_dev_setup(wq); + rc = dev_set_name(dev, "%s/wq%u.%u", idxd_get_dev_name(idxd), + idxd->id, wq->id); if (rc < 0) - return rc; + goto err; - dev = idxd_cdev->dev; + wq->idxd_cdev = idxd_cdev; cdev_init(cdev, &idxd_cdev_fops); - cdev_set_parent(cdev, &dev->kobj); - rc = cdev_add(cdev, dev->devt, 1); + rc = cdev_device_add(cdev, dev); if (rc) { dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc); - idxd_wq_cdev_cleanup(wq, CDEV_FAILED); - return rc; + goto err; } - init_waitqueue_head(&idxd_cdev->err_queue); return 0; + + err: + put_device(dev); + wq->idxd_cdev = NULL; + return rc; } void idxd_wq_del_cdev(struct idxd_wq *wq) { - idxd_wq_cdev_cleanup(wq, CDEV_NORMAL); + struct idxd_cdev *idxd_cdev; + struct idxd_cdev_context *cdev_ctx; + + cdev_ctx = &ictx[wq->idxd->type]; + idxd_cdev = wq->idxd_cdev; + wq->idxd_cdev = NULL; + cdev_device_del(&idxd_cdev->cdev, &idxd_cdev->dev); + put_device(&idxd_cdev->dev); } int idxd_cdev_register(void) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 31c819544a22..4fef57717049 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -19,7 +19,7 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand, /* Interrupt control bits */ void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id) { - struct irq_data *data = irq_get_irq_data(idxd->msix_entries[vec_id].vector); + struct irq_data *data = irq_get_irq_data(idxd->irq_entries[vec_id].vector); pci_msi_mask_irq(data); } @@ -36,7 +36,7 @@ void idxd_mask_msix_vectors(struct idxd_device *idxd) void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id) { - struct irq_data *data = irq_get_irq_data(idxd->msix_entries[vec_id].vector); + struct irq_data *data = irq_get_irq_data(idxd->irq_entries[vec_id].vector); pci_msi_unmask_irq(data); } @@ -186,8 +186,6 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq) desc->id = i; desc->wq = wq; desc->cpu = -1; - dma_async_tx_descriptor_init(&desc->txd, &wq->dma_chan); - desc->txd.tx_submit = idxd_dma_tx_submit; } return 0; @@ -451,7 +449,8 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand, if (idxd_device_is_halted(idxd)) { dev_warn(&idxd->pdev->dev, "Device is HALTED!\n"); - *status = IDXD_CMDSTS_HW_ERR; + if (status) + *status = IDXD_CMDSTS_HW_ERR; return; } @@ -521,7 +520,7 @@ void idxd_device_wqs_clear_state(struct idxd_device *idxd) lockdep_assert_held(&idxd->dev_lock); for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = &idxd->wqs[i]; + struct idxd_wq *wq = idxd->wqs[i]; if (wq->state == IDXD_WQ_ENABLED) { idxd_wq_disable_cleanup(wq); @@ -660,7 +659,7 @@ static int idxd_groups_config_write(struct idxd_device *idxd) ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET)); for (i = 0; i < idxd->max_groups; i++) { - struct idxd_group *group = &idxd->groups[i]; + struct idxd_group *group = idxd->groups[i]; idxd_group_config_write(group); } @@ -739,7 +738,7 @@ static int idxd_wqs_config_write(struct idxd_device *idxd) int i, rc; for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = &idxd->wqs[i]; + struct idxd_wq *wq = idxd->wqs[i]; rc = idxd_wq_config_write(wq); if (rc < 0) @@ -755,7 +754,7 @@ static void idxd_group_flags_setup(struct idxd_device *idxd) /* TC-A 0 and TC-B 1 should be defaults */ for (i = 0; i < idxd->max_groups; i++) { - struct idxd_group *group = &idxd->groups[i]; + struct idxd_group *group = idxd->groups[i]; if (group->tc_a == -1) group->tc_a = group->grpcfg.flags.tc_a = 0; @@ -782,12 +781,12 @@ static int idxd_engines_setup(struct idxd_device *idxd) struct idxd_group *group; for (i = 0; i < idxd->max_groups; i++) { - group = &idxd->groups[i]; + group = idxd->groups[i]; group->grpcfg.engines = 0; } for (i = 0; i < idxd->max_engines; i++) { - eng = &idxd->engines[i]; + eng = idxd->engines[i]; group = eng->group; if (!group) @@ -811,13 +810,13 @@ static int idxd_wqs_setup(struct idxd_device *idxd) struct device *dev = &idxd->pdev->dev; for (i = 0; i < idxd->max_groups; i++) { - group = &idxd->groups[i]; + group = idxd->groups[i]; for (j = 0; j < 4; j++) group->grpcfg.wqs[j] = 0; } for (i = 0; i < idxd->max_wqs; i++) { - wq = &idxd->wqs[i]; + wq = idxd->wqs[i]; group = wq->group; if (!wq->group) diff --git a/drivers/dma/idxd/dma.c b/drivers/dma/idxd/dma.c index a15e50126434..77439b645044 100644 --- a/drivers/dma/idxd/dma.c +++ b/drivers/dma/idxd/dma.c @@ -14,7 +14,10 @@ static inline struct idxd_wq *to_idxd_wq(struct dma_chan *c) { - return container_of(c, struct idxd_wq, dma_chan); + struct idxd_dma_chan *idxd_chan; + + idxd_chan = container_of(c, struct idxd_dma_chan, chan); + return idxd_chan->wq; } void idxd_dma_complete_txd(struct idxd_desc *desc, @@ -135,7 +138,7 @@ static void idxd_dma_issue_pending(struct dma_chan *dma_chan) { } -dma_cookie_t idxd_dma_tx_submit(struct dma_async_tx_descriptor *tx) +static dma_cookie_t idxd_dma_tx_submit(struct dma_async_tx_descriptor *tx) { struct dma_chan *c = tx->chan; struct idxd_wq *wq = to_idxd_wq(c); @@ -156,14 +159,25 @@ dma_cookie_t idxd_dma_tx_submit(struct dma_async_tx_descriptor *tx) static void idxd_dma_release(struct dma_device *device) { + struct idxd_dma_dev *idxd_dma = container_of(device, struct idxd_dma_dev, dma); + + kfree(idxd_dma); } int idxd_register_dma_device(struct idxd_device *idxd) { - struct dma_device *dma = &idxd->dma_dev; + struct idxd_dma_dev *idxd_dma; + struct dma_device *dma; + struct device *dev = &idxd->pdev->dev; + int rc; + idxd_dma = kzalloc_node(sizeof(*idxd_dma), GFP_KERNEL, dev_to_node(dev)); + if (!idxd_dma) + return -ENOMEM; + + dma = &idxd_dma->dma; INIT_LIST_HEAD(&dma->channels); - dma->dev = &idxd->pdev->dev; + dma->dev = dev; dma_cap_set(DMA_PRIVATE, dma->cap_mask); dma_cap_set(DMA_COMPLETION_NO_ORDER, dma->cap_mask); @@ -179,35 +193,72 @@ int idxd_register_dma_device(struct idxd_device *idxd) dma->device_alloc_chan_resources = idxd_dma_alloc_chan_resources; dma->device_free_chan_resources = idxd_dma_free_chan_resources; - return dma_async_device_register(&idxd->dma_dev); + rc = dma_async_device_register(dma); + if (rc < 0) { + kfree(idxd_dma); + return rc; + } + + idxd_dma->idxd = idxd; + /* + * This pointer is protected by the refs taken by the dma_chan. It will remain valid + * as long as there are outstanding channels. + */ + idxd->idxd_dma = idxd_dma; + return 0; } void idxd_unregister_dma_device(struct idxd_device *idxd) { - dma_async_device_unregister(&idxd->dma_dev); + dma_async_device_unregister(&idxd->idxd_dma->dma); } int idxd_register_dma_channel(struct idxd_wq *wq) { struct idxd_device *idxd = wq->idxd; - struct dma_device *dma = &idxd->dma_dev; - struct dma_chan *chan = &wq->dma_chan; - int rc; + struct dma_device *dma = &idxd->idxd_dma->dma; + struct device *dev = &idxd->pdev->dev; + struct idxd_dma_chan *idxd_chan; + struct dma_chan *chan; + int rc, i; + + idxd_chan = kzalloc_node(sizeof(*idxd_chan), GFP_KERNEL, dev_to_node(dev)); + if (!idxd_chan) + return -ENOMEM; - memset(&wq->dma_chan, 0, sizeof(struct dma_chan)); + chan = &idxd_chan->chan; chan->device = dma; list_add_tail(&chan->device_node, &dma->channels); + + for (i = 0; i < wq->num_descs; i++) { + struct idxd_desc *desc = wq->descs[i]; + + dma_async_tx_descriptor_init(&desc->txd, chan); + desc->txd.tx_submit = idxd_dma_tx_submit; + } + rc = dma_async_device_channel_register(dma, chan); - if (rc < 0) + if (rc < 0) { + kfree(idxd_chan); return rc; + } + + wq->idxd_chan = idxd_chan; + idxd_chan->wq = wq; + get_device(&wq->conf_dev); return 0; } void idxd_unregister_dma_channel(struct idxd_wq *wq) { - struct dma_chan *chan = &wq->dma_chan; + struct idxd_dma_chan *idxd_chan = wq->idxd_chan; + struct dma_chan *chan = &idxd_chan->chan; + struct idxd_dma_dev *idxd_dma = wq->idxd->idxd_dma; - dma_async_device_channel_unregister(&wq->idxd->dma_dev, chan); + dma_async_device_channel_unregister(&idxd_dma->dma, chan); list_del(&chan->device_node); + kfree(wq->idxd_chan); + wq->idxd_chan = NULL; + put_device(&wq->conf_dev); } diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 76014c14f473..89daf746d121 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -8,12 +8,16 @@ #include #include #include +#include #include "registers.h" #define IDXD_DRIVER_VERSION "1.00" extern struct kmem_cache *idxd_desc_pool; +struct idxd_device; +struct idxd_wq; + #define IDXD_REG_TIMEOUT 50 #define IDXD_DRAIN_TIMEOUT 5000 @@ -33,6 +37,7 @@ struct idxd_device_driver { struct idxd_irq_entry { struct idxd_device *idxd; int id; + int vector; struct llist_head pending_llist; struct list_head work_list; /* @@ -75,10 +80,10 @@ enum idxd_wq_type { }; struct idxd_cdev { + struct idxd_wq *wq; struct cdev cdev; - struct device *dev; + struct device dev; int minor; - struct wait_queue_head err_queue; }; #define IDXD_ALLOCATED_BATCH_SIZE 128U @@ -96,10 +101,16 @@ enum idxd_complete_type { IDXD_COMPLETE_DEV_FAIL, }; +struct idxd_dma_chan { + struct dma_chan chan; + struct idxd_wq *wq; +}; + struct idxd_wq { void __iomem *portal; struct device conf_dev; - struct idxd_cdev idxd_cdev; + struct idxd_cdev *idxd_cdev; + struct wait_queue_head err_queue; struct idxd_device *idxd; int id; enum idxd_wq_type type; @@ -125,7 +136,7 @@ struct idxd_wq { int compls_size; struct idxd_desc **descs; struct sbitmap_queue sbq; - struct dma_chan dma_chan; + struct idxd_dma_chan *idxd_chan; char name[WQ_NAME_SIZE + 1]; u64 max_xfer_bytes; u32 max_batch_size; @@ -162,6 +173,11 @@ enum idxd_device_flag { IDXD_FLAG_PASID_ENABLED, }; +struct idxd_dma_dev { + struct idxd_device *idxd; + struct dma_device dma; +}; + struct idxd_device { enum idxd_type type; struct device conf_dev; @@ -178,9 +194,9 @@ struct idxd_device { spinlock_t dev_lock; /* spinlock for device */ struct completion *cmd_done; - struct idxd_group *groups; - struct idxd_wq *wqs; - struct idxd_engine *engines; + struct idxd_group **groups; + struct idxd_wq **wqs; + struct idxd_engine **engines; struct iommu_sva *sva; unsigned int pasid; @@ -206,11 +222,10 @@ struct idxd_device { union sw_err_reg sw_err; wait_queue_head_t cmd_waitq; - struct msix_entry *msix_entries; int num_wq_irqs; struct idxd_irq_entry *irq_entries; - struct dma_device dma_dev; + struct idxd_dma_dev *idxd_dma; struct workqueue_struct *wq; struct work_struct work; }; @@ -242,6 +257,43 @@ extern struct bus_type dsa_bus_type; extern struct bus_type iax_bus_type; extern bool support_enqcmd; +extern struct device_type dsa_device_type; +extern struct device_type iax_device_type; +extern struct device_type idxd_wq_device_type; +extern struct device_type idxd_engine_device_type; +extern struct device_type idxd_group_device_type; + +static inline bool is_dsa_dev(struct device *dev) +{ + return dev->type == &dsa_device_type; +} + +static inline bool is_iax_dev(struct device *dev) +{ + return dev->type == &iax_device_type; +} + +static inline bool is_idxd_dev(struct device *dev) +{ + return is_dsa_dev(dev) || is_iax_dev(dev); +} + +static inline bool is_idxd_wq_dev(struct device *dev) +{ + return dev->type == &idxd_wq_device_type; +} + +static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq) +{ + if (wq->type == IDXD_WQT_KERNEL && strcmp(wq->name, "dmaengine") == 0) + return true; + return false; +} + +static inline bool is_idxd_wq_cdev(struct idxd_wq *wq) +{ + return wq->type == IDXD_WQT_USER; +} static inline bool wq_dedicated(struct idxd_wq *wq) { @@ -279,18 +331,6 @@ static inline int idxd_get_wq_portal_full_offset(int wq_id, return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot); } -static inline void idxd_set_type(struct idxd_device *idxd) -{ - struct pci_dev *pdev = idxd->pdev; - - if (pdev->device == PCI_DEVICE_ID_INTEL_DSA_SPR0) - idxd->type = IDXD_TYPE_DSA; - else if (pdev->device == PCI_DEVICE_ID_INTEL_IAX_SPR0) - idxd->type = IDXD_TYPE_IAX; - else - idxd->type = IDXD_TYPE_UNKNOWN; -} - static inline void idxd_wq_get(struct idxd_wq *wq) { wq->client_count++; @@ -306,14 +346,16 @@ static inline int idxd_wq_refcount(struct idxd_wq *wq) return wq->client_count; }; +struct ida *idxd_ida(struct idxd_device *idxd); const char *idxd_get_dev_name(struct idxd_device *idxd); int idxd_register_bus_type(void); void idxd_unregister_bus_type(void); -int idxd_setup_sysfs(struct idxd_device *idxd); -void idxd_cleanup_sysfs(struct idxd_device *idxd); +int idxd_register_devices(struct idxd_device *idxd); +void idxd_unregister_devices(struct idxd_device *idxd); int idxd_register_driver(void); void idxd_unregister_driver(void); struct bus_type *idxd_get_bus_type(struct idxd_device *idxd); +struct device_type *idxd_get_device_type(struct idxd_device *idxd); /* device interrupt control */ void idxd_msix_perm_setup(struct idxd_device *idxd); @@ -363,7 +405,6 @@ void idxd_unregister_dma_channel(struct idxd_wq *wq); void idxd_parse_completion_status(u8 status, enum dmaengine_tx_result *res); void idxd_dma_complete_txd(struct idxd_desc *desc, enum idxd_complete_type comp_type); -dma_cookie_t idxd_dma_tx_submit(struct dma_async_tx_descriptor *tx); /* cdev */ int idxd_cdev_register(void); diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 6584b0ec07d5..07cf7977a045 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -34,8 +34,7 @@ MODULE_PARM_DESC(sva, "Toggle SVA support on/off"); bool support_enqcmd; -static struct idr idxd_idrs[IDXD_TYPE_MAX]; -static DEFINE_MUTEX(idxd_idr_lock); +static struct ida idxd_idas[IDXD_TYPE_MAX]; static struct pci_device_id idxd_pci_tbl[] = { /* DSA ver 1.0 platforms */ @@ -52,6 +51,11 @@ static char *idxd_name[] = { "iax" }; +struct ida *idxd_ida(struct idxd_device *idxd) +{ + return &idxd_idas[idxd->type]; +} + const char *idxd_get_dev_name(struct idxd_device *idxd) { return idxd_name[idxd->type]; @@ -61,7 +65,6 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) { struct pci_dev *pdev = idxd->pdev; struct device *dev = &pdev->dev; - struct msix_entry *msix; struct idxd_irq_entry *irq_entry; int i, msixcnt; int rc = 0; @@ -69,23 +72,13 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) msixcnt = pci_msix_vec_count(pdev); if (msixcnt < 0) { dev_err(dev, "Not MSI-X interrupt capable.\n"); - goto err_no_irq; - } - - idxd->msix_entries = devm_kzalloc(dev, sizeof(struct msix_entry) * - msixcnt, GFP_KERNEL); - if (!idxd->msix_entries) { - rc = -ENOMEM; - goto err_no_irq; + return -ENOSPC; } - for (i = 0; i < msixcnt; i++) - idxd->msix_entries[i].entry = i; - - rc = pci_enable_msix_exact(pdev, idxd->msix_entries, msixcnt); - if (rc) { - dev_err(dev, "Failed enabling %d MSIX entries.\n", msixcnt); - goto err_no_irq; + rc = pci_alloc_irq_vectors(pdev, msixcnt, msixcnt, PCI_IRQ_MSIX); + if (rc != msixcnt) { + dev_err(dev, "Failed enabling %d MSIX entries: %d\n", msixcnt, rc); + return -ENOSPC; } dev_dbg(dev, "Enabled %d msix vectors\n", msixcnt); @@ -93,119 +86,236 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) * We implement 1 completion list per MSI-X entry except for * entry 0, which is for errors and others. */ - idxd->irq_entries = devm_kcalloc(dev, msixcnt, - sizeof(struct idxd_irq_entry), - GFP_KERNEL); + idxd->irq_entries = kcalloc_node(msixcnt, sizeof(struct idxd_irq_entry), + GFP_KERNEL, dev_to_node(dev)); if (!idxd->irq_entries) { rc = -ENOMEM; - goto err_no_irq; + goto err_irq_entries; } for (i = 0; i < msixcnt; i++) { idxd->irq_entries[i].id = i; idxd->irq_entries[i].idxd = idxd; + idxd->irq_entries[i].vector = pci_irq_vector(pdev, i); spin_lock_init(&idxd->irq_entries[i].list_lock); } - msix = &idxd->msix_entries[0]; irq_entry = &idxd->irq_entries[0]; - rc = devm_request_threaded_irq(dev, msix->vector, idxd_irq_handler, - idxd_misc_thread, 0, "idxd-misc", - irq_entry); + rc = request_threaded_irq(irq_entry->vector, idxd_irq_handler, idxd_misc_thread, + 0, "idxd-misc", irq_entry); if (rc < 0) { dev_err(dev, "Failed to allocate misc interrupt.\n"); - goto err_no_irq; + goto err_misc_irq; } - dev_dbg(dev, "Allocated idxd-misc handler on msix vector %d\n", - msix->vector); + dev_dbg(dev, "Allocated idxd-misc handler on msix vector %d\n", irq_entry->vector); /* first MSI-X entry is not for wq interrupts */ idxd->num_wq_irqs = msixcnt - 1; for (i = 1; i < msixcnt; i++) { - msix = &idxd->msix_entries[i]; irq_entry = &idxd->irq_entries[i]; init_llist_head(&idxd->irq_entries[i].pending_llist); INIT_LIST_HEAD(&idxd->irq_entries[i].work_list); - rc = devm_request_threaded_irq(dev, msix->vector, - idxd_irq_handler, - idxd_wq_thread, 0, - "idxd-portal", irq_entry); + rc = request_threaded_irq(irq_entry->vector, idxd_irq_handler, + idxd_wq_thread, 0, "idxd-portal", irq_entry); if (rc < 0) { - dev_err(dev, "Failed to allocate irq %d.\n", - msix->vector); - goto err_no_irq; + dev_err(dev, "Failed to allocate irq %d.\n", irq_entry->vector); + goto err_wq_irqs; } - dev_dbg(dev, "Allocated idxd-msix %d for vector %d\n", - i, msix->vector); + dev_dbg(dev, "Allocated idxd-msix %d for vector %d\n", i, irq_entry->vector); } idxd_unmask_error_interrupts(idxd); idxd_msix_perm_setup(idxd); return 0; - err_no_irq: + err_wq_irqs: + while (--i >= 0) { + irq_entry = &idxd->irq_entries[i]; + free_irq(irq_entry->vector, irq_entry); + } + err_misc_irq: /* Disable error interrupt generation */ idxd_mask_error_interrupts(idxd); - pci_disable_msix(pdev); + err_irq_entries: + pci_free_irq_vectors(pdev); dev_err(dev, "No usable interrupts\n"); return rc; } -static int idxd_setup_internals(struct idxd_device *idxd) +static int idxd_setup_wqs(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; - int i; - - init_waitqueue_head(&idxd->cmd_waitq); - idxd->groups = devm_kcalloc(dev, idxd->max_groups, - sizeof(struct idxd_group), GFP_KERNEL); - if (!idxd->groups) - return -ENOMEM; - - for (i = 0; i < idxd->max_groups; i++) { - idxd->groups[i].idxd = idxd; - idxd->groups[i].id = i; - idxd->groups[i].tc_a = -1; - idxd->groups[i].tc_b = -1; - } + struct idxd_wq *wq; + int i, rc; - idxd->wqs = devm_kcalloc(dev, idxd->max_wqs, sizeof(struct idxd_wq), - GFP_KERNEL); + idxd->wqs = kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *), + GFP_KERNEL, dev_to_node(dev)); if (!idxd->wqs) return -ENOMEM; - idxd->engines = devm_kcalloc(dev, idxd->max_engines, - sizeof(struct idxd_engine), GFP_KERNEL); - if (!idxd->engines) - return -ENOMEM; - for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = &idxd->wqs[i]; + wq = kzalloc_node(sizeof(*wq), GFP_KERNEL, dev_to_node(dev)); + if (!wq) { + rc = -ENOMEM; + goto err; + } wq->id = i; wq->idxd = idxd; + device_initialize(&wq->conf_dev); + wq->conf_dev.parent = &idxd->conf_dev; + wq->conf_dev.bus = idxd_get_bus_type(idxd); + wq->conf_dev.type = &idxd_wq_device_type; + rc = dev_set_name(&wq->conf_dev, "wq%d.%d", idxd->id, wq->id); + if (rc < 0) { + put_device(&wq->conf_dev); + goto err; + } + mutex_init(&wq->wq_lock); - wq->idxd_cdev.minor = -1; + init_waitqueue_head(&wq->err_queue); wq->max_xfer_bytes = idxd->max_xfer_bytes; wq->max_batch_size = idxd->max_batch_size; - wq->wqcfg = devm_kzalloc(dev, idxd->wqcfg_size, GFP_KERNEL); - if (!wq->wqcfg) - return -ENOMEM; + wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev)); + if (!wq->wqcfg) { + put_device(&wq->conf_dev); + rc = -ENOMEM; + goto err; + } + idxd->wqs[i] = wq; } + return 0; + + err: + while (--i >= 0) + put_device(&idxd->wqs[i]->conf_dev); + return rc; +} + +static int idxd_setup_engines(struct idxd_device *idxd) +{ + struct idxd_engine *engine; + struct device *dev = &idxd->pdev->dev; + int i, rc; + + idxd->engines = kcalloc_node(idxd->max_engines, sizeof(struct idxd_engine *), + GFP_KERNEL, dev_to_node(dev)); + if (!idxd->engines) + return -ENOMEM; + for (i = 0; i < idxd->max_engines; i++) { - idxd->engines[i].idxd = idxd; - idxd->engines[i].id = i; + engine = kzalloc_node(sizeof(*engine), GFP_KERNEL, dev_to_node(dev)); + if (!engine) { + rc = -ENOMEM; + goto err; + } + + engine->id = i; + engine->idxd = idxd; + device_initialize(&engine->conf_dev); + engine->conf_dev.parent = &idxd->conf_dev; + engine->conf_dev.type = &idxd_engine_device_type; + rc = dev_set_name(&engine->conf_dev, "engine%d.%d", idxd->id, engine->id); + if (rc < 0) { + put_device(&engine->conf_dev); + goto err; + } + + idxd->engines[i] = engine; } - idxd->wq = create_workqueue(dev_name(dev)); - if (!idxd->wq) + return 0; + + err: + while (--i >= 0) + put_device(&idxd->engines[i]->conf_dev); + return rc; +} + +static int idxd_setup_groups(struct idxd_device *idxd) +{ + struct device *dev = &idxd->pdev->dev; + struct idxd_group *group; + int i, rc; + + idxd->groups = kcalloc_node(idxd->max_groups, sizeof(struct idxd_group *), + GFP_KERNEL, dev_to_node(dev)); + if (!idxd->groups) return -ENOMEM; + for (i = 0; i < idxd->max_groups; i++) { + group = kzalloc_node(sizeof(*group), GFP_KERNEL, dev_to_node(dev)); + if (!group) { + rc = -ENOMEM; + goto err; + } + + group->id = i; + group->idxd = idxd; + device_initialize(&group->conf_dev); + group->conf_dev.parent = &idxd->conf_dev; + group->conf_dev.bus = idxd_get_bus_type(idxd); + group->conf_dev.type = &idxd_group_device_type; + rc = dev_set_name(&group->conf_dev, "group%d.%d", idxd->id, group->id); + if (rc < 0) { + put_device(&group->conf_dev); + goto err; + } + + idxd->groups[i] = group; + group->tc_a = -1; + group->tc_b = -1; + } + + return 0; + + err: + while (--i >= 0) + put_device(&idxd->groups[i]->conf_dev); + return rc; +} + +static int idxd_setup_internals(struct idxd_device *idxd) +{ + struct device *dev = &idxd->pdev->dev; + int rc, i; + + init_waitqueue_head(&idxd->cmd_waitq); + + rc = idxd_setup_wqs(idxd); + if (rc < 0) + return rc; + + rc = idxd_setup_engines(idxd); + if (rc < 0) + goto err_engine; + + rc = idxd_setup_groups(idxd); + if (rc < 0) + goto err_group; + + idxd->wq = create_workqueue(dev_name(dev)); + if (!idxd->wq) { + rc = -ENOMEM; + goto err_wkq_create; + } + return 0; + + err_wkq_create: + for (i = 0; i < idxd->max_groups; i++) + put_device(&idxd->groups[i]->conf_dev); + err_group: + for (i = 0; i < idxd->max_engines; i++) + put_device(&idxd->engines[i]->conf_dev); + err_engine: + for (i = 0; i < idxd->max_wqs; i++) + put_device(&idxd->wqs[i]->conf_dev); + return rc; } static void idxd_read_table_offsets(struct idxd_device *idxd) @@ -275,16 +385,44 @@ static void idxd_read_caps(struct idxd_device *idxd) } } +static inline void idxd_set_type(struct idxd_device *idxd) +{ + struct pci_dev *pdev = idxd->pdev; + + if (pdev->device == PCI_DEVICE_ID_INTEL_DSA_SPR0) + idxd->type = IDXD_TYPE_DSA; + else if (pdev->device == PCI_DEVICE_ID_INTEL_IAX_SPR0) + idxd->type = IDXD_TYPE_IAX; + else + idxd->type = IDXD_TYPE_UNKNOWN; +} + static struct idxd_device *idxd_alloc(struct pci_dev *pdev) { struct device *dev = &pdev->dev; struct idxd_device *idxd; + int rc; - idxd = devm_kzalloc(dev, sizeof(struct idxd_device), GFP_KERNEL); + idxd = kzalloc_node(sizeof(*idxd), GFP_KERNEL, dev_to_node(dev)); if (!idxd) return NULL; idxd->pdev = pdev; + idxd_set_type(idxd); + idxd->id = ida_alloc(idxd_ida(idxd), GFP_KERNEL); + if (idxd->id < 0) + return NULL; + + device_initialize(&idxd->conf_dev); + idxd->conf_dev.parent = dev; + idxd->conf_dev.bus = idxd_get_bus_type(idxd); + idxd->conf_dev.type = idxd_get_device_type(idxd); + rc = dev_set_name(&idxd->conf_dev, "%s%d", idxd_get_dev_name(idxd), idxd->id); + if (rc < 0) { + put_device(&idxd->conf_dev); + return NULL; + } + spin_lock_init(&idxd->dev_lock); return idxd; @@ -352,31 +490,20 @@ static int idxd_probe(struct idxd_device *idxd) rc = idxd_setup_internals(idxd); if (rc) - goto err_setup; + goto err; rc = idxd_setup_interrupts(idxd); if (rc) - goto err_setup; + goto err; dev_dbg(dev, "IDXD interrupt setup complete.\n"); - mutex_lock(&idxd_idr_lock); - idxd->id = idr_alloc(&idxd_idrs[idxd->type], idxd, 0, 0, GFP_KERNEL); - mutex_unlock(&idxd_idr_lock); - if (idxd->id < 0) { - rc = -ENOMEM; - goto err_idr_fail; - } - idxd->major = idxd_cdev_get_major(idxd); dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id); return 0; - err_idr_fail: - idxd_mask_error_interrupts(idxd); - idxd_mask_msix_vectors(idxd); - err_setup: + err: if (device_pasid_enabled(idxd)) idxd_disable_system_pasid(idxd); return rc; @@ -396,34 +523,37 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) struct idxd_device *idxd; int rc; - rc = pcim_enable_device(pdev); + rc = pci_enable_device(pdev); if (rc) return rc; dev_dbg(dev, "Alloc IDXD context\n"); idxd = idxd_alloc(pdev); - if (!idxd) - return -ENOMEM; + if (!idxd) { + rc = -ENOMEM; + goto err_idxd_alloc; + } dev_dbg(dev, "Mapping BARs\n"); - idxd->reg_base = pcim_iomap(pdev, IDXD_MMIO_BAR, 0); - if (!idxd->reg_base) - return -ENOMEM; + idxd->reg_base = pci_iomap(pdev, IDXD_MMIO_BAR, 0); + if (!idxd->reg_base) { + rc = -ENOMEM; + goto err_iomap; + } dev_dbg(dev, "Set DMA masks\n"); rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); if (rc) rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); if (rc) - return rc; + goto err; rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); if (rc) rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); if (rc) - return rc; + goto err; - idxd_set_type(idxd); idxd_type_init(idxd); @@ -435,13 +565,13 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) rc = idxd_probe(idxd); if (rc) { dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n"); - return -ENODEV; + goto err; } - rc = idxd_setup_sysfs(idxd); + rc = idxd_register_devices(idxd); if (rc) { dev_err(dev, "IDXD sysfs setup failed\n"); - return -ENODEV; + goto err; } idxd->state = IDXD_DEV_CONF_READY; @@ -450,6 +580,14 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) idxd->hw.version); return 0; + + err: + pci_iounmap(pdev, idxd->reg_base); + err_iomap: + put_device(&idxd->conf_dev); + err_idxd_alloc: + pci_disable_device(pdev); + return rc; } static void idxd_flush_pending_llist(struct idxd_irq_entry *ie) @@ -495,7 +633,8 @@ static void idxd_shutdown(struct pci_dev *pdev) for (i = 0; i < msixcnt; i++) { irq_entry = &idxd->irq_entries[i]; - synchronize_irq(idxd->msix_entries[i].vector); + synchronize_irq(irq_entry->vector); + free_irq(irq_entry->vector, irq_entry); if (i == 0) continue; idxd_flush_pending_llist(irq_entry); @@ -503,6 +642,9 @@ static void idxd_shutdown(struct pci_dev *pdev) } idxd_msix_perm_clear(idxd); + pci_free_irq_vectors(pdev); + pci_iounmap(pdev, idxd->reg_base); + pci_disable_device(pdev); destroy_workqueue(idxd->wq); } @@ -511,13 +653,10 @@ static void idxd_remove(struct pci_dev *pdev) struct idxd_device *idxd = pci_get_drvdata(pdev); dev_dbg(&pdev->dev, "%s called\n", __func__); - idxd_cleanup_sysfs(idxd); idxd_shutdown(pdev); if (device_pasid_enabled(idxd)) idxd_disable_system_pasid(idxd); - mutex_lock(&idxd_idr_lock); - idr_remove(&idxd_idrs[idxd->type], idxd->id); - mutex_unlock(&idxd_idr_lock); + idxd_unregister_devices(idxd); } static struct pci_driver idxd_pci_driver = { @@ -547,7 +686,7 @@ static int __init idxd_init_module(void) support_enqcmd = true; for (i = 0; i < IDXD_TYPE_MAX; i++) - idr_init(&idxd_idrs[i]); + ida_init(&idxd_idas[i]); err = idxd_register_bus_type(); if (err < 0) diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c index f1463fc58112..fc0781e3f36d 100644 --- a/drivers/dma/idxd/irq.c +++ b/drivers/dma/idxd/irq.c @@ -45,7 +45,7 @@ static void idxd_device_reinit(struct work_struct *work) goto out; for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = &idxd->wqs[i]; + struct idxd_wq *wq = idxd->wqs[i]; if (wq->state == IDXD_WQ_ENABLED) { rc = idxd_wq_enable(wq); @@ -130,18 +130,18 @@ static int process_misc_interrupts(struct idxd_device *idxd, u32 cause) if (idxd->sw_err.valid && idxd->sw_err.wq_idx_valid) { int id = idxd->sw_err.wq_idx; - struct idxd_wq *wq = &idxd->wqs[id]; + struct idxd_wq *wq = idxd->wqs[id]; if (wq->type == IDXD_WQT_USER) - wake_up_interruptible(&wq->idxd_cdev.err_queue); + wake_up_interruptible(&wq->err_queue); } else { int i; for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = &idxd->wqs[i]; + struct idxd_wq *wq = idxd->wqs[i]; if (wq->type == IDXD_WQT_USER) - wake_up_interruptible(&wq->idxd_cdev.err_queue); + wake_up_interruptible(&wq->err_queue); } } diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 18bf4d148989..9586b55abce5 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -16,69 +16,6 @@ static char *idxd_wq_type_names[] = { [IDXD_WQT_USER] = "user", }; -static void idxd_conf_device_release(struct device *dev) -{ - dev_dbg(dev, "%s for %s\n", __func__, dev_name(dev)); -} - -static struct device_type idxd_group_device_type = { - .name = "group", - .release = idxd_conf_device_release, -}; - -static struct device_type idxd_wq_device_type = { - .name = "wq", - .release = idxd_conf_device_release, -}; - -static struct device_type idxd_engine_device_type = { - .name = "engine", - .release = idxd_conf_device_release, -}; - -static struct device_type dsa_device_type = { - .name = "dsa", - .release = idxd_conf_device_release, -}; - -static struct device_type iax_device_type = { - .name = "iax", - .release = idxd_conf_device_release, -}; - -static inline bool is_dsa_dev(struct device *dev) -{ - return dev ? dev->type == &dsa_device_type : false; -} - -static inline bool is_iax_dev(struct device *dev) -{ - return dev ? dev->type == &iax_device_type : false; -} - -static inline bool is_idxd_dev(struct device *dev) -{ - return is_dsa_dev(dev) || is_iax_dev(dev); -} - -static inline bool is_idxd_wq_dev(struct device *dev) -{ - return dev ? dev->type == &idxd_wq_device_type : false; -} - -static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq) -{ - if (wq->type == IDXD_WQT_KERNEL && - strcmp(wq->name, "dmaengine") == 0) - return true; - return false; -} - -static inline bool is_idxd_wq_cdev(struct idxd_wq *wq) -{ - return wq->type == IDXD_WQT_USER; -} - static int idxd_config_bus_match(struct device *dev, struct device_driver *drv) { @@ -322,7 +259,7 @@ static int idxd_config_bus_remove(struct device *dev) dev_dbg(dev, "%s removing dev %s\n", __func__, dev_name(&idxd->conf_dev)); for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = &idxd->wqs[i]; + struct idxd_wq *wq = idxd->wqs[i]; if (wq->state == IDXD_WQ_DISABLED) continue; @@ -334,7 +271,7 @@ static int idxd_config_bus_remove(struct device *dev) idxd_unregister_dma_device(idxd); rc = idxd_device_disable(idxd); for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = &idxd->wqs[i]; + struct idxd_wq *wq = idxd->wqs[i]; mutex_lock(&wq->wq_lock); idxd_wq_disable_cleanup(wq); @@ -405,7 +342,7 @@ struct bus_type *idxd_get_bus_type(struct idxd_device *idxd) return idxd_bus_types[idxd->type]; } -static struct device_type *idxd_get_device_type(struct idxd_device *idxd) +struct device_type *idxd_get_device_type(struct idxd_device *idxd) { if (idxd->type == IDXD_TYPE_DSA) return &dsa_device_type; @@ -488,7 +425,7 @@ static ssize_t engine_group_id_store(struct device *dev, if (prevg) prevg->num_engines--; - engine->group = &idxd->groups[id]; + engine->group = idxd->groups[id]; engine->group->num_engines++; return count; @@ -512,6 +449,19 @@ static const struct attribute_group *idxd_engine_attribute_groups[] = { NULL, }; +static void idxd_conf_engine_release(struct device *dev) +{ + struct idxd_engine *engine = container_of(dev, struct idxd_engine, conf_dev); + + kfree(engine); +} + +struct device_type idxd_engine_device_type = { + .name = "engine", + .release = idxd_conf_engine_release, + .groups = idxd_engine_attribute_groups, +}; + /* Group attributes */ static void idxd_set_free_tokens(struct idxd_device *idxd) @@ -519,7 +469,7 @@ static void idxd_set_free_tokens(struct idxd_device *idxd) int i, tokens; for (i = 0, tokens = 0; i < idxd->max_groups; i++) { - struct idxd_group *g = &idxd->groups[i]; + struct idxd_group *g = idxd->groups[i]; tokens += g->tokens_reserved; } @@ -674,7 +624,7 @@ static ssize_t group_engines_show(struct device *dev, struct idxd_device *idxd = group->idxd; for (i = 0; i < idxd->max_engines; i++) { - struct idxd_engine *engine = &idxd->engines[i]; + struct idxd_engine *engine = idxd->engines[i]; if (!engine->group) continue; @@ -703,7 +653,7 @@ static ssize_t group_work_queues_show(struct device *dev, struct idxd_device *idxd = group->idxd; for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = &idxd->wqs[i]; + struct idxd_wq *wq = idxd->wqs[i]; if (!wq->group) continue; @@ -824,6 +774,19 @@ static const struct attribute_group *idxd_group_attribute_groups[] = { NULL, }; +static void idxd_conf_group_release(struct device *dev) +{ + struct idxd_group *group = container_of(dev, struct idxd_group, conf_dev); + + kfree(group); +} + +struct device_type idxd_group_device_type = { + .name = "group", + .release = idxd_conf_group_release, + .groups = idxd_group_attribute_groups, +}; + /* IDXD work queue attribs */ static ssize_t wq_clients_show(struct device *dev, struct device_attribute *attr, char *buf) @@ -896,7 +859,7 @@ static ssize_t wq_group_id_store(struct device *dev, return count; } - group = &idxd->groups[id]; + group = idxd->groups[id]; prevg = wq->group; if (prevg) @@ -960,7 +923,7 @@ static int total_claimed_wq_size(struct idxd_device *idxd) int wq_size = 0; for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = &idxd->wqs[i]; + struct idxd_wq *wq = idxd->wqs[i]; wq_size += wq->size; } @@ -1206,8 +1169,16 @@ static ssize_t wq_cdev_minor_show(struct device *dev, struct device_attribute *attr, char *buf) { struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + int minor = -1; - return sprintf(buf, "%d\n", wq->idxd_cdev.minor); + mutex_lock(&wq->wq_lock); + if (wq->idxd_cdev) + minor = wq->idxd_cdev->minor; + mutex_unlock(&wq->wq_lock); + + if (minor == -1) + return -ENXIO; + return sysfs_emit(buf, "%d\n", minor); } static struct device_attribute dev_attr_wq_cdev_minor = @@ -1356,6 +1327,20 @@ static const struct attribute_group *idxd_wq_attribute_groups[] = { NULL, }; +static void idxd_conf_wq_release(struct device *dev) +{ + struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + + kfree(wq->wqcfg); + kfree(wq); +} + +struct device_type idxd_wq_device_type = { + .name = "wq", + .release = idxd_conf_wq_release, + .groups = idxd_wq_attribute_groups, +}; + /* IDXD device attribs */ static ssize_t version_show(struct device *dev, struct device_attribute *attr, char *buf) @@ -1486,7 +1471,7 @@ static ssize_t clients_show(struct device *dev, spin_lock_irqsave(&idxd->dev_lock, flags); for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = &idxd->wqs[i]; + struct idxd_wq *wq = idxd->wqs[i]; count += wq->client_count; } @@ -1644,183 +1629,160 @@ static const struct attribute_group *idxd_attribute_groups[] = { NULL, }; -static int idxd_setup_engine_sysfs(struct idxd_device *idxd) +static void idxd_conf_device_release(struct device *dev) { - struct device *dev = &idxd->pdev->dev; - int i, rc; + struct idxd_device *idxd = container_of(dev, struct idxd_device, conf_dev); + + kfree(idxd->groups); + kfree(idxd->wqs); + kfree(idxd->engines); + kfree(idxd->irq_entries); + ida_free(idxd_ida(idxd), idxd->id); + kfree(idxd); +} + +struct device_type dsa_device_type = { + .name = "dsa", + .release = idxd_conf_device_release, + .groups = idxd_attribute_groups, +}; + +struct device_type iax_device_type = { + .name = "iax", + .release = idxd_conf_device_release, + .groups = idxd_attribute_groups, +}; + +static int idxd_register_engine_devices(struct idxd_device *idxd) +{ + int i, j, rc; for (i = 0; i < idxd->max_engines; i++) { - struct idxd_engine *engine = &idxd->engines[i]; - - engine->conf_dev.parent = &idxd->conf_dev; - dev_set_name(&engine->conf_dev, "engine%d.%d", - idxd->id, engine->id); - engine->conf_dev.bus = idxd_get_bus_type(idxd); - engine->conf_dev.groups = idxd_engine_attribute_groups; - engine->conf_dev.type = &idxd_engine_device_type; - dev_dbg(dev, "Engine device register: %s\n", - dev_name(&engine->conf_dev)); - rc = device_register(&engine->conf_dev); - if (rc < 0) { - put_device(&engine->conf_dev); + struct idxd_engine *engine = idxd->engines[i]; + + rc = device_add(&engine->conf_dev); + if (rc < 0) goto cleanup; - } } return 0; cleanup: - while (i--) { - struct idxd_engine *engine = &idxd->engines[i]; + j = i - 1; + for (; i < idxd->max_engines; i++) + put_device(&idxd->engines[i]->conf_dev); - device_unregister(&engine->conf_dev); - } + while (j--) + device_unregister(&idxd->engines[j]->conf_dev); return rc; } -static int idxd_setup_group_sysfs(struct idxd_device *idxd) +static int idxd_register_group_devices(struct idxd_device *idxd) { - struct device *dev = &idxd->pdev->dev; - int i, rc; + int i, j, rc; for (i = 0; i < idxd->max_groups; i++) { - struct idxd_group *group = &idxd->groups[i]; - - group->conf_dev.parent = &idxd->conf_dev; - dev_set_name(&group->conf_dev, "group%d.%d", - idxd->id, group->id); - group->conf_dev.bus = idxd_get_bus_type(idxd); - group->conf_dev.groups = idxd_group_attribute_groups; - group->conf_dev.type = &idxd_group_device_type; - dev_dbg(dev, "Group device register: %s\n", - dev_name(&group->conf_dev)); - rc = device_register(&group->conf_dev); - if (rc < 0) { - put_device(&group->conf_dev); + struct idxd_group *group = idxd->groups[i]; + + rc = device_add(&group->conf_dev); + if (rc < 0) goto cleanup; - } } return 0; cleanup: - while (i--) { - struct idxd_group *group = &idxd->groups[i]; + j = i - 1; + for (; i < idxd->max_groups; i++) + put_device(&idxd->groups[i]->conf_dev); - device_unregister(&group->conf_dev); - } + while (j--) + device_unregister(&idxd->groups[j]->conf_dev); return rc; } -static int idxd_setup_wq_sysfs(struct idxd_device *idxd) +static int idxd_register_wq_devices(struct idxd_device *idxd) { - struct device *dev = &idxd->pdev->dev; - int i, rc; + int i, rc, j; for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = &idxd->wqs[i]; - - wq->conf_dev.parent = &idxd->conf_dev; - dev_set_name(&wq->conf_dev, "wq%d.%d", idxd->id, wq->id); - wq->conf_dev.bus = idxd_get_bus_type(idxd); - wq->conf_dev.groups = idxd_wq_attribute_groups; - wq->conf_dev.type = &idxd_wq_device_type; - dev_dbg(dev, "WQ device register: %s\n", - dev_name(&wq->conf_dev)); - rc = device_register(&wq->conf_dev); - if (rc < 0) { - put_device(&wq->conf_dev); + struct idxd_wq *wq = idxd->wqs[i]; + + rc = device_add(&wq->conf_dev); + if (rc < 0) goto cleanup; - } } return 0; cleanup: - while (i--) { - struct idxd_wq *wq = &idxd->wqs[i]; + j = i - 1; + for (; i < idxd->max_wqs; i++) + put_device(&idxd->wqs[i]->conf_dev); - device_unregister(&wq->conf_dev); - } + while (j--) + device_unregister(&idxd->wqs[j]->conf_dev); return rc; } -static int idxd_setup_device_sysfs(struct idxd_device *idxd) +int idxd_register_devices(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; - int rc; - char devname[IDXD_NAME_SIZE]; - - sprintf(devname, "%s%d", idxd_get_dev_name(idxd), idxd->id); - idxd->conf_dev.parent = dev; - dev_set_name(&idxd->conf_dev, "%s", devname); - idxd->conf_dev.bus = idxd_get_bus_type(idxd); - idxd->conf_dev.groups = idxd_attribute_groups; - idxd->conf_dev.type = idxd_get_device_type(idxd); - - dev_dbg(dev, "IDXD device register: %s\n", dev_name(&idxd->conf_dev)); - rc = device_register(&idxd->conf_dev); - if (rc < 0) { - put_device(&idxd->conf_dev); - return rc; - } + int rc, i; - return 0; -} - -int idxd_setup_sysfs(struct idxd_device *idxd) -{ - struct device *dev = &idxd->pdev->dev; - int rc; - - rc = idxd_setup_device_sysfs(idxd); - if (rc < 0) { - dev_dbg(dev, "Device sysfs registering failed: %d\n", rc); + rc = device_add(&idxd->conf_dev); + if (rc < 0) return rc; - } - rc = idxd_setup_wq_sysfs(idxd); + rc = idxd_register_wq_devices(idxd); if (rc < 0) { - /* unregister conf dev */ - dev_dbg(dev, "Work Queue sysfs registering failed: %d\n", rc); - return rc; + dev_dbg(dev, "WQ devices registering failed: %d\n", rc); + goto err_wq; } - rc = idxd_setup_group_sysfs(idxd); + rc = idxd_register_engine_devices(idxd); if (rc < 0) { - /* unregister conf dev */ - dev_dbg(dev, "Group sysfs registering failed: %d\n", rc); - return rc; + dev_dbg(dev, "Engine devices registering failed: %d\n", rc); + goto err_engine; } - rc = idxd_setup_engine_sysfs(idxd); + rc = idxd_register_group_devices(idxd); if (rc < 0) { - /* unregister conf dev */ - dev_dbg(dev, "Engine sysfs registering failed: %d\n", rc); - return rc; + dev_dbg(dev, "Group device registering failed: %d\n", rc); + goto err_group; } return 0; + + err_group: + for (i = 0; i < idxd->max_engines; i++) + device_unregister(&idxd->engines[i]->conf_dev); + err_engine: + for (i = 0; i < idxd->max_wqs; i++) + device_unregister(&idxd->wqs[i]->conf_dev); + err_wq: + device_del(&idxd->conf_dev); + return rc; } -void idxd_cleanup_sysfs(struct idxd_device *idxd) +void idxd_unregister_devices(struct idxd_device *idxd) { int i; for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = &idxd->wqs[i]; + struct idxd_wq *wq = idxd->wqs[i]; device_unregister(&wq->conf_dev); } for (i = 0; i < idxd->max_engines; i++) { - struct idxd_engine *engine = &idxd->engines[i]; + struct idxd_engine *engine = idxd->engines[i]; device_unregister(&engine->conf_dev); } for (i = 0; i < idxd->max_groups; i++) { - struct idxd_group *group = &idxd->groups[i]; + struct idxd_group *group = idxd->groups[i]; device_unregister(&group->conf_dev); } diff --git a/drivers/extcon/extcon-arizona.c b/drivers/extcon/extcon-arizona.c index aae82db542a5..76aacbac5869 100644 --- a/drivers/extcon/extcon-arizona.c +++ b/drivers/extcon/extcon-arizona.c @@ -601,7 +601,7 @@ static irqreturn_t arizona_hpdet_irq(int irq, void *data) struct arizona *arizona = info->arizona; int id_gpio = arizona->pdata.hpdet_id_gpio; unsigned int report = EXTCON_JACK_HEADPHONE; - int ret, reading; + int ret, reading, state; bool mic = false; mutex_lock(&info->lock); @@ -614,12 +614,11 @@ static irqreturn_t arizona_hpdet_irq(int irq, void *data) } /* If the cable was removed while measuring ignore the result */ - ret = extcon_get_state(info->edev, EXTCON_MECHANICAL); - if (ret < 0) { - dev_err(arizona->dev, "Failed to check cable state: %d\n", - ret); + state = extcon_get_state(info->edev, EXTCON_MECHANICAL); + if (state < 0) { + dev_err(arizona->dev, "Failed to check cable state: %d\n", state); goto out; - } else if (!ret) { + } else if (!state) { dev_dbg(arizona->dev, "Ignoring HPDET for removed cable\n"); goto done; } @@ -667,7 +666,7 @@ static irqreturn_t arizona_hpdet_irq(int irq, void *data) gpio_set_value_cansleep(id_gpio, 0); /* If we have a mic then reenable MICDET */ - if (mic || info->mic) + if (state && (mic || info->mic)) arizona_start_mic(info); if (info->hpdet_active) { @@ -675,7 +674,9 @@ static irqreturn_t arizona_hpdet_irq(int irq, void *data) info->hpdet_active = false; } - info->hpdet_done = true; + /* Do not set hp_det done when the cable has been unplugged */ + if (state) + info->hpdet_done = true; out: mutex_unlock(&info->lock); @@ -1759,25 +1760,6 @@ static int arizona_extcon_remove(struct platform_device *pdev) bool change; int ret; - ret = regmap_update_bits_check(arizona->regmap, ARIZONA_MIC_DETECT_1, - ARIZONA_MICD_ENA, 0, - &change); - if (ret < 0) { - dev_err(&pdev->dev, "Failed to disable micd on remove: %d\n", - ret); - } else if (change) { - regulator_disable(info->micvdd); - pm_runtime_put(info->dev); - } - - gpiod_put(info->micd_pol_gpio); - - pm_runtime_disable(&pdev->dev); - - regmap_update_bits(arizona->regmap, - ARIZONA_MICD_CLAMP_CONTROL, - ARIZONA_MICD_CLAMP_MODE_MASK, 0); - if (info->micd_clamp) { jack_irq_rise = ARIZONA_IRQ_MICD_CLAMP_RISE; jack_irq_fall = ARIZONA_IRQ_MICD_CLAMP_FALL; @@ -1793,10 +1775,31 @@ static int arizona_extcon_remove(struct platform_device *pdev) arizona_free_irq(arizona, jack_irq_rise, info); arizona_free_irq(arizona, jack_irq_fall, info); cancel_delayed_work_sync(&info->hpdet_work); + cancel_delayed_work_sync(&info->micd_detect_work); + cancel_delayed_work_sync(&info->micd_timeout_work); + + ret = regmap_update_bits_check(arizona->regmap, ARIZONA_MIC_DETECT_1, + ARIZONA_MICD_ENA, 0, + &change); + if (ret < 0) { + dev_err(&pdev->dev, "Failed to disable micd on remove: %d\n", + ret); + } else if (change) { + regulator_disable(info->micvdd); + pm_runtime_put(info->dev); + } + + regmap_update_bits(arizona->regmap, + ARIZONA_MICD_CLAMP_CONTROL, + ARIZONA_MICD_CLAMP_MODE_MASK, 0); regmap_update_bits(arizona->regmap, ARIZONA_JACK_DETECT_ANALOGUE, ARIZONA_JD1_ENA, 0); arizona_clk32k_disable(arizona); + gpiod_put(info->micd_pol_gpio); + + pm_runtime_disable(&pdev->dev); + return 0; } diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig index 3f14dffb9669..5dd19dbd67a3 100644 --- a/drivers/firmware/Kconfig +++ b/drivers/firmware/Kconfig @@ -237,6 +237,7 @@ config INTEL_STRATIX10_RSU config QCOM_SCM bool depends on ARM || ARM64 + depends on HAVE_ARM_SMCCC select RESET_CONTROLLER config QCOM_SCM_DOWNLOAD_MODE_DEFAULT diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile index c23466e05e60..d0537573501e 100644 --- a/drivers/firmware/efi/libstub/Makefile +++ b/drivers/firmware/efi/libstub/Makefile @@ -13,7 +13,8 @@ cflags-$(CONFIG_X86) += -m$(BITS) -D__KERNEL__ \ -Wno-pointer-sign \ $(call cc-disable-warning, address-of-packed-member) \ $(call cc-disable-warning, gnu) \ - -fno-asynchronous-unwind-tables + -fno-asynchronous-unwind-tables \ + $(CLANG_FLAGS) # arm64 uses the full KBUILD_CFLAGS so it's necessary to explicitly # disable the stackleak plugin diff --git a/drivers/firmware/qcom_scm-smc.c b/drivers/firmware/qcom_scm-smc.c index 497c13ba98d6..d111833364ba 100644 --- a/drivers/firmware/qcom_scm-smc.c +++ b/drivers/firmware/qcom_scm-smc.c @@ -77,8 +77,10 @@ static void __scm_smc_do(const struct arm_smccc_args *smc, } while (res->a0 == QCOM_SCM_V2_EBUSY); } -int scm_smc_call(struct device *dev, const struct qcom_scm_desc *desc, - struct qcom_scm_res *res, bool atomic) + +int __scm_smc_call(struct device *dev, const struct qcom_scm_desc *desc, + enum qcom_scm_convention qcom_convention, + struct qcom_scm_res *res, bool atomic) { int arglen = desc->arginfo & 0xf; int i; @@ -87,9 +89,8 @@ int scm_smc_call(struct device *dev, const struct qcom_scm_desc *desc, size_t alloc_len; gfp_t flag = atomic ? GFP_ATOMIC : GFP_KERNEL; u32 smccc_call_type = atomic ? ARM_SMCCC_FAST_CALL : ARM_SMCCC_STD_CALL; - u32 qcom_smccc_convention = - (qcom_scm_convention == SMC_CONVENTION_ARM_32) ? - ARM_SMCCC_SMC_32 : ARM_SMCCC_SMC_64; + u32 qcom_smccc_convention = (qcom_convention == SMC_CONVENTION_ARM_32) ? + ARM_SMCCC_SMC_32 : ARM_SMCCC_SMC_64; struct arm_smccc_res smc_res; struct arm_smccc_args smc = {0}; @@ -148,4 +149,5 @@ int scm_smc_call(struct device *dev, const struct qcom_scm_desc *desc, } return (long)smc_res.a0 ? qcom_scm_remap_error(smc_res.a0) : 0; + } diff --git a/drivers/firmware/qcom_scm.c b/drivers/firmware/qcom_scm.c index f57779fc7ee9..9ac84b5d6ce0 100644 --- a/drivers/firmware/qcom_scm.c +++ b/drivers/firmware/qcom_scm.c @@ -113,14 +113,10 @@ static void qcom_scm_clk_disable(void) clk_disable_unprepare(__scm->bus_clk); } -static int __qcom_scm_is_call_available(struct device *dev, u32 svc_id, - u32 cmd_id); +enum qcom_scm_convention qcom_scm_convention = SMC_CONVENTION_UNKNOWN; +static DEFINE_SPINLOCK(scm_query_lock); -enum qcom_scm_convention qcom_scm_convention; -static bool has_queried __read_mostly; -static DEFINE_SPINLOCK(query_lock); - -static void __query_convention(void) +static enum qcom_scm_convention __get_convention(void) { unsigned long flags; struct qcom_scm_desc desc = { @@ -133,36 +129,50 @@ static void __query_convention(void) .owner = ARM_SMCCC_OWNER_SIP, }; struct qcom_scm_res res; + enum qcom_scm_convention probed_convention; int ret; + bool forced = false; - spin_lock_irqsave(&query_lock, flags); - if (has_queried) - goto out; + if (likely(qcom_scm_convention != SMC_CONVENTION_UNKNOWN)) + return qcom_scm_convention; - qcom_scm_convention = SMC_CONVENTION_ARM_64; - // Device isn't required as there is only one argument - no device - // needed to dma_map_single to secure world - ret = scm_smc_call(NULL, &desc, &res, true); + /* + * Device isn't required as there is only one argument - no device + * needed to dma_map_single to secure world + */ + probed_convention = SMC_CONVENTION_ARM_64; + ret = __scm_smc_call(NULL, &desc, probed_convention, &res, true); if (!ret && res.result[0] == 1) - goto out; + goto found; + + /* + * Some SC7180 firmwares didn't implement the + * QCOM_SCM_INFO_IS_CALL_AVAIL call, so we fallback to forcing ARM_64 + * calling conventions on these firmwares. Luckily we don't make any + * early calls into the firmware on these SoCs so the device pointer + * will be valid here to check if the compatible matches. + */ + if (of_device_is_compatible(__scm ? __scm->dev->of_node : NULL, "qcom,scm-sc7180")) { + forced = true; + goto found; + } - qcom_scm_convention = SMC_CONVENTION_ARM_32; - ret = scm_smc_call(NULL, &desc, &res, true); + probed_convention = SMC_CONVENTION_ARM_32; + ret = __scm_smc_call(NULL, &desc, probed_convention, &res, true); if (!ret && res.result[0] == 1) - goto out; - - qcom_scm_convention = SMC_CONVENTION_LEGACY; -out: - has_queried = true; - spin_unlock_irqrestore(&query_lock, flags); - pr_info("qcom_scm: convention: %s\n", - qcom_scm_convention_names[qcom_scm_convention]); -} + goto found; + + probed_convention = SMC_CONVENTION_LEGACY; +found: + spin_lock_irqsave(&scm_query_lock, flags); + if (probed_convention != qcom_scm_convention) { + qcom_scm_convention = probed_convention; + pr_info("qcom_scm: convention: %s%s\n", + qcom_scm_convention_names[qcom_scm_convention], + forced ? " (forced)" : ""); + } + spin_unlock_irqrestore(&scm_query_lock, flags); -static inline enum qcom_scm_convention __get_convention(void) -{ - if (unlikely(!has_queried)) - __query_convention(); return qcom_scm_convention; } @@ -219,8 +229,8 @@ static int qcom_scm_call_atomic(struct device *dev, } } -static int __qcom_scm_is_call_available(struct device *dev, u32 svc_id, - u32 cmd_id) +static bool __qcom_scm_is_call_available(struct device *dev, u32 svc_id, + u32 cmd_id) { int ret; struct qcom_scm_desc desc = { @@ -247,7 +257,7 @@ static int __qcom_scm_is_call_available(struct device *dev, u32 svc_id, ret = qcom_scm_call(dev, &desc, &res); - return ret ? : res.result[0]; + return ret ? false : !!res.result[0]; } /** @@ -585,9 +595,8 @@ bool qcom_scm_pas_supported(u32 peripheral) }; struct qcom_scm_res res; - ret = __qcom_scm_is_call_available(__scm->dev, QCOM_SCM_SVC_PIL, - QCOM_SCM_PIL_PAS_IS_SUPPORTED); - if (ret <= 0) + if (!__qcom_scm_is_call_available(__scm->dev, QCOM_SCM_SVC_PIL, + QCOM_SCM_PIL_PAS_IS_SUPPORTED)) return false; ret = qcom_scm_call(__scm->dev, &desc, &res); @@ -1060,17 +1069,18 @@ EXPORT_SYMBOL(qcom_scm_ice_set_key); */ bool qcom_scm_hdcp_available(void) { + bool avail; int ret = qcom_scm_clk_enable(); if (ret) return ret; - ret = __qcom_scm_is_call_available(__scm->dev, QCOM_SCM_SVC_HDCP, + avail = __qcom_scm_is_call_available(__scm->dev, QCOM_SCM_SVC_HDCP, QCOM_SCM_HDCP_INVOKE); qcom_scm_clk_disable(); - return ret > 0; + return avail; } EXPORT_SYMBOL(qcom_scm_hdcp_available); @@ -1242,7 +1252,7 @@ static int qcom_scm_probe(struct platform_device *pdev) __scm = scm; __scm->dev = &pdev->dev; - __query_convention(); + __get_convention(); /* * If requested enable "download mode", from this point on warmboot diff --git a/drivers/firmware/qcom_scm.h b/drivers/firmware/qcom_scm.h index 95cd1ac30ab0..632fe3142462 100644 --- a/drivers/firmware/qcom_scm.h +++ b/drivers/firmware/qcom_scm.h @@ -61,8 +61,11 @@ struct qcom_scm_res { }; #define SCM_SMC_FNID(s, c) ((((s) & 0xFF) << 8) | ((c) & 0xFF)) -extern int scm_smc_call(struct device *dev, const struct qcom_scm_desc *desc, - struct qcom_scm_res *res, bool atomic); +extern int __scm_smc_call(struct device *dev, const struct qcom_scm_desc *desc, + enum qcom_scm_convention qcom_convention, + struct qcom_scm_res *res, bool atomic); +#define scm_smc_call(dev, desc, res, atomic) \ + __scm_smc_call((dev), (desc), qcom_scm_convention, (res), (atomic)) #define SCM_LEGACY_FNID(s, c) (((s) << 10) | ((c) & 0x3ff)) extern int scm_legacy_call_atomic(struct device *dev, diff --git a/drivers/firmware/xilinx/zynqmp.c b/drivers/firmware/xilinx/zynqmp.c index 7eb9958662dd..83082e2f2e44 100644 --- a/drivers/firmware/xilinx/zynqmp.c +++ b/drivers/firmware/xilinx/zynqmp.c @@ -2,7 +2,7 @@ /* * Xilinx Zynq MPSoC Firmware layer * - * Copyright (C) 2014-2020 Xilinx, Inc. + * Copyright (C) 2014-2021 Xilinx, Inc. * * Michal Simek * Davorin Mista @@ -1280,12 +1280,13 @@ static int zynqmp_firmware_probe(struct platform_device *pdev) static int zynqmp_firmware_remove(struct platform_device *pdev) { struct pm_api_feature_data *feature_data; + struct hlist_node *tmp; int i; mfd_remove_devices(&pdev->dev); zynqmp_pm_api_debugfs_exit(); - hash_for_each(pm_api_features_map, i, feature_data, hentry) { + hash_for_each_safe(pm_api_features_map, i, tmp, feature_data, hentry) { hash_del(&feature_data->hentry); kfree(feature_data); } diff --git a/drivers/fpga/dfl-pci.c b/drivers/fpga/dfl-pci.c index 04e47e266f26..b44523ea8c91 100644 --- a/drivers/fpga/dfl-pci.c +++ b/drivers/fpga/dfl-pci.c @@ -69,14 +69,16 @@ static void cci_pci_free_irq(struct pci_dev *pcidev) } /* PCI Device ID */ -#define PCIE_DEVICE_ID_PF_INT_5_X 0xBCBD -#define PCIE_DEVICE_ID_PF_INT_6_X 0xBCC0 -#define PCIE_DEVICE_ID_PF_DSC_1_X 0x09C4 -#define PCIE_DEVICE_ID_INTEL_PAC_N3000 0x0B30 +#define PCIE_DEVICE_ID_PF_INT_5_X 0xBCBD +#define PCIE_DEVICE_ID_PF_INT_6_X 0xBCC0 +#define PCIE_DEVICE_ID_PF_DSC_1_X 0x09C4 +#define PCIE_DEVICE_ID_INTEL_PAC_N3000 0x0B30 +#define PCIE_DEVICE_ID_INTEL_PAC_D5005 0x0B2B /* VF Device */ -#define PCIE_DEVICE_ID_VF_INT_5_X 0xBCBF -#define PCIE_DEVICE_ID_VF_INT_6_X 0xBCC1 -#define PCIE_DEVICE_ID_VF_DSC_1_X 0x09C5 +#define PCIE_DEVICE_ID_VF_INT_5_X 0xBCBF +#define PCIE_DEVICE_ID_VF_INT_6_X 0xBCC1 +#define PCIE_DEVICE_ID_VF_DSC_1_X 0x09C5 +#define PCIE_DEVICE_ID_INTEL_PAC_D5005_VF 0x0B2C static struct pci_device_id cci_pcie_id_tbl[] = { {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_PF_INT_5_X),}, @@ -86,6 +88,8 @@ static struct pci_device_id cci_pcie_id_tbl[] = { {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_PF_DSC_1_X),}, {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_VF_DSC_1_X),}, {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_INTEL_PAC_N3000),}, + {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_INTEL_PAC_D5005),}, + {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_INTEL_PAC_D5005_VF),}, {0,} }; MODULE_DEVICE_TABLE(pci, cci_pcie_id_tbl); diff --git a/drivers/fpga/xilinx-spi.c b/drivers/fpga/xilinx-spi.c index 27defa98092d..fee4d0abf6bf 100644 --- a/drivers/fpga/xilinx-spi.c +++ b/drivers/fpga/xilinx-spi.c @@ -233,25 +233,19 @@ static int xilinx_spi_probe(struct spi_device *spi) /* PROGRAM_B is active low */ conf->prog_b = devm_gpiod_get(&spi->dev, "prog_b", GPIOD_OUT_LOW); - if (IS_ERR(conf->prog_b)) { - dev_err(&spi->dev, "Failed to get PROGRAM_B gpio: %ld\n", - PTR_ERR(conf->prog_b)); - return PTR_ERR(conf->prog_b); - } + if (IS_ERR(conf->prog_b)) + return dev_err_probe(&spi->dev, PTR_ERR(conf->prog_b), + "Failed to get PROGRAM_B gpio\n"); conf->init_b = devm_gpiod_get_optional(&spi->dev, "init-b", GPIOD_IN); - if (IS_ERR(conf->init_b)) { - dev_err(&spi->dev, "Failed to get INIT_B gpio: %ld\n", - PTR_ERR(conf->init_b)); - return PTR_ERR(conf->init_b); - } + if (IS_ERR(conf->init_b)) + return dev_err_probe(&spi->dev, PTR_ERR(conf->init_b), + "Failed to get INIT_B gpio\n"); conf->done = devm_gpiod_get(&spi->dev, "done", GPIOD_IN); - if (IS_ERR(conf->done)) { - dev_err(&spi->dev, "Failed to get DONE gpio: %ld\n", - PTR_ERR(conf->done)); - return PTR_ERR(conf->done); - } + if (IS_ERR(conf->done)) + return dev_err_probe(&spi->dev, PTR_ERR(conf->done), + "Failed to get DONE gpio\n"); mgr = devm_fpga_mgr_create(&spi->dev, "Xilinx Slave Serial FPGA Manager", diff --git a/drivers/gpio/gpiolib-acpi.c b/drivers/gpio/gpiolib-acpi.c index 1aacd2a5a1fd..174839f3772f 100644 --- a/drivers/gpio/gpiolib-acpi.c +++ b/drivers/gpio/gpiolib-acpi.c @@ -1438,6 +1438,20 @@ static const struct dmi_system_id gpiolib_acpi_quirks[] __initconst = { .no_edge_events_on_boot = true, }, }, + { + /* + * The Dell Venue 10 Pro 5055, with Bay Trail SoC + TI PMIC uses an + * external embedded-controller connected via I2C + an ACPI GPIO + * event handler on INT33FFC:02 pin 12, causing spurious wakeups. + */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_MATCH(DMI_PRODUCT_NAME, "Venue 10 Pro 5055"), + }, + .driver_data = &(struct acpi_gpiolib_dmi_quirk) { + .ignore_wake = "INT33FC:02@12", + }, + }, { /* * HP X2 10 models with Cherry Trail SoC + TI PMIC use an diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 8a5a8ff5d362..5eee251e3335 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -3613,6 +3613,7 @@ void amdgpu_device_fini(struct amdgpu_device *adev) { dev_info(adev->dev, "amdgpu: finishing device.\n"); flush_delayed_work(&adev->delayed_init_work); + ttm_bo_lock_delayed_workqueue(&adev->mman.bdev); adev->shutdown = true; kfree(adev->pci_state); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c index f753e04fee99..a2ac44cc2a6d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c @@ -1355,7 +1355,7 @@ int amdgpu_display_suspend_helper(struct amdgpu_device *adev) } } } - return r; + return 0; } int amdgpu_display_resume_helper(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c index d56f4023ebb3..7e8e46c39dbd 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c @@ -533,6 +533,8 @@ void amdgpu_fence_driver_fini(struct amdgpu_device *adev) if (!ring || !ring->fence_drv.initialized) continue; + if (!ring->no_scheduler) + drm_sched_fini(&ring->sched); r = amdgpu_fence_wait_empty(ring); if (r) { /* no need to trigger GPU reset as we are unloading */ @@ -541,8 +543,7 @@ void amdgpu_fence_driver_fini(struct amdgpu_device *adev) if (ring->fence_drv.irq_src) amdgpu_irq_put(adev, ring->fence_drv.irq_src, ring->fence_drv.irq_type); - if (!ring->no_scheduler) - drm_sched_fini(&ring->sched); + del_timer_sync(&ring->fence_drv.fallback_timer); for (j = 0; j <= ring->fence_drv.num_fences_mask; ++j) dma_fence_put(ring->fence_drv.fences[j]); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c index 7645223ea0ef..97c11aa47ad0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c @@ -77,6 +77,8 @@ int amdgpu_ib_get(struct amdgpu_device *adev, struct amdgpu_vm *vm, } ib->ptr = amdgpu_sa_bo_cpu_addr(ib->sa_bo); + /* flush the cache before commit the IB */ + ib->flags = AMDGPU_IB_FLAG_EMIT_MEM_SYNC; if (!vm) ib->gpu_addr = amdgpu_sa_bo_gpu_addr(ib->sa_bo); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c index 94b069630db3..b4971e90b98c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c @@ -215,7 +215,11 @@ static int amdgpu_vmid_grab_idle(struct amdgpu_vm *vm, /* Check if we have an idle VMID */ i = 0; list_for_each_entry((*idle), &id_mgr->ids_lru, list) { - fences[i] = amdgpu_sync_peek_fence(&(*idle)->active, ring); + /* Don't use per engine and per process VMID at the same time */ + struct amdgpu_ring *r = adev->vm_manager.concurrent_flush ? + NULL : ring; + + fences[i] = amdgpu_sync_peek_fence(&(*idle)->active, r); if (!fences[i]) break; ++i; @@ -281,7 +285,7 @@ static int amdgpu_vmid_grab_reserved(struct amdgpu_vm *vm, if (updates && (*id)->flushed_updates && updates->context == (*id)->flushed_updates->context && !dma_fence_is_later(updates, (*id)->flushed_updates)) - updates = NULL; + updates = NULL; if ((*id)->owner != vm->immediate.fence_context || job->vm_pd_addr != (*id)->pd_gpu_addr || @@ -290,6 +294,10 @@ static int amdgpu_vmid_grab_reserved(struct amdgpu_vm *vm, !dma_fence_is_signaled((*id)->last_flush))) { struct dma_fence *tmp; + /* Don't use per engine and per process VMID at the same time */ + if (adev->vm_manager.concurrent_flush) + ring = NULL; + /* to prevent one context starved by another context */ (*id)->pd_gpu_addr = 0; tmp = amdgpu_sync_peek_fence(&(*id)->active, ring); @@ -365,12 +373,7 @@ static int amdgpu_vmid_grab_used(struct amdgpu_vm *vm, if (updates && (!flushed || dma_fence_is_later(updates, flushed))) needs_flush = true; - /* Concurrent flushes are only possible starting with Vega10 and - * are broken on Navi10 and Navi14. - */ - if (needs_flush && (adev->asic_type < CHIP_VEGA10 || - adev->asic_type == CHIP_NAVI10 || - adev->asic_type == CHIP_NAVI14)) + if (needs_flush && !adev->vm_manager.concurrent_flush) continue; /* Good, we can use this VMID. Remember this submission as diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c index afbbec82a289..9be945d8e72f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c @@ -535,7 +535,7 @@ void amdgpu_irq_gpu_reset_resume_helper(struct amdgpu_device *adev) for (j = 0; j < AMDGPU_MAX_IRQ_SRC_ID; ++j) { struct amdgpu_irq_src *src = adev->irq.client[i].sources[j]; - if (!src) + if (!src || !src->funcs || !src->funcs->set) continue; for (k = 0; k < src->num_types; k++) amdgpu_irq_update(adev, src, k); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.c index 19c0a3655228..82e9ecf84352 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.c @@ -519,8 +519,10 @@ static int init_pmu_entry_by_type_and_add(struct amdgpu_pmu_entry *pmu_entry, pmu_entry->pmu.attr_groups = kmemdup(attr_groups, sizeof(attr_groups), GFP_KERNEL); - if (!pmu_entry->pmu.attr_groups) + if (!pmu_entry->pmu.attr_groups) { + ret = -ENOMEM; goto err_attr_group; + } snprintf(pmu_name, PMU_NAME_SIZE, "%s_%d", pmu_entry->pmu_file_prefix, adev_to_drm(pmu_entry->adev)->primary->index); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 5efa331e3ee8..383c178cf074 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -942,7 +942,7 @@ static void amdgpu_ttm_tt_unpin_userptr(struct ttm_bo_device *bdev, DMA_BIDIRECTIONAL : DMA_TO_DEVICE; /* double check that we don't free the table twice */ - if (!ttm->sg->sgl) + if (!ttm->sg || !ttm->sg->sgl) return; /* unmap the pages mapped to the device */ @@ -1162,13 +1162,13 @@ static void amdgpu_ttm_backend_unbind(struct ttm_bo_device *bdev, struct amdgpu_ttm_tt *gtt = (void *)ttm; int r; - if (!gtt->bound) - return; - /* if the pages have userptr pinning then clear that first */ if (gtt->userptr) amdgpu_ttm_tt_unpin_userptr(bdev, ttm); + if (!gtt->bound) + return; + if (gtt->offset == AMDGPU_BO_INVALID_OFFSET) return; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c index e2ed4689118a..c6dbc0801604 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c @@ -259,7 +259,7 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev) if ((adev->asic_type == CHIP_POLARIS10 || adev->asic_type == CHIP_POLARIS11) && (adev->uvd.fw_version < FW_1_66_16)) - DRM_ERROR("POLARIS10/11 UVD firmware version %hu.%hu is too old.\n", + DRM_ERROR("POLARIS10/11 UVD firmware version %u.%u is too old.\n", version_major, version_minor); } else { unsigned int enc_major, enc_minor, dec_minor; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 326dae31b675..a566bbe26bdd 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -92,13 +92,13 @@ struct amdgpu_prt_cb { static inline void amdgpu_vm_eviction_lock(struct amdgpu_vm *vm) { mutex_lock(&vm->eviction_lock); - vm->saved_flags = memalloc_nofs_save(); + vm->saved_flags = memalloc_noreclaim_save(); } static inline int amdgpu_vm_eviction_trylock(struct amdgpu_vm *vm) { if (mutex_trylock(&vm->eviction_lock)) { - vm->saved_flags = memalloc_nofs_save(); + vm->saved_flags = memalloc_noreclaim_save(); return 1; } return 0; @@ -106,7 +106,7 @@ static inline int amdgpu_vm_eviction_trylock(struct amdgpu_vm *vm) static inline void amdgpu_vm_eviction_unlock(struct amdgpu_vm *vm) { - memalloc_nofs_restore(vm->saved_flags); + memalloc_noreclaim_restore(vm->saved_flags); mutex_unlock(&vm->eviction_lock); } @@ -3147,6 +3147,12 @@ void amdgpu_vm_manager_init(struct amdgpu_device *adev) { unsigned i; + /* Concurrent flushes are only possible starting with Vega10 and + * are broken on Navi10 and Navi14. + */ + adev->vm_manager.concurrent_flush = !(adev->asic_type < CHIP_VEGA10 || + adev->asic_type == CHIP_NAVI10 || + adev->asic_type == CHIP_NAVI14); amdgpu_vmid_mgr_init(adev); adev->vm_manager.fence_context = diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index 976a12e5a8b9..4e140288159c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h @@ -331,6 +331,7 @@ struct amdgpu_vm_manager { /* Handling of VMIDs */ struct amdgpu_vmid_mgr id_mgr[AMDGPU_MAX_VMHUBS]; unsigned int first_kfd_vmid; + bool concurrent_flush; /* Handling of VM fences */ u64 fence_context; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c index 659b385b27b5..4d3a24fdeb9c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c @@ -468,15 +468,22 @@ int amdgpu_xgmi_update_topology(struct amdgpu_hive_info *hive, struct amdgpu_dev } +/* + * NOTE psp_xgmi_node_info.num_hops layout is as follows: + * num_hops[7:6] = link type (0 = xGMI2, 1 = xGMI3, 2/3 = reserved) + * num_hops[5:3] = reserved + * num_hops[2:0] = number of hops + */ int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev, struct amdgpu_device *peer_adev) { struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info; + uint8_t num_hops_mask = 0x7; int i; for (i = 0 ; i < top->num_nodes; ++i) if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id) - return top->nodes[i].num_hops; + return top->nodes[i].num_hops & num_hops_mask; return -EINVAL; } diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c index 2d832fc23119..421d6069c509 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c @@ -59,6 +59,7 @@ MODULE_FIRMWARE("amdgpu/tonga_mc.bin"); MODULE_FIRMWARE("amdgpu/polaris11_mc.bin"); MODULE_FIRMWARE("amdgpu/polaris10_mc.bin"); MODULE_FIRMWARE("amdgpu/polaris12_mc.bin"); +MODULE_FIRMWARE("amdgpu/polaris12_32_mc.bin"); MODULE_FIRMWARE("amdgpu/polaris11_k_mc.bin"); MODULE_FIRMWARE("amdgpu/polaris10_k_mc.bin"); MODULE_FIRMWARE("amdgpu/polaris12_k_mc.bin"); @@ -243,10 +244,16 @@ static int gmc_v8_0_init_microcode(struct amdgpu_device *adev) chip_name = "polaris10"; break; case CHIP_POLARIS12: - if (ASICID_IS_P23(adev->pdev->device, adev->pdev->revision)) + if (ASICID_IS_P23(adev->pdev->device, adev->pdev->revision)) { chip_name = "polaris12_k"; - else - chip_name = "polaris12"; + } else { + WREG32(mmMC_SEQ_IO_DEBUG_INDEX, ixMC_IO_DEBUG_UP_159); + /* Polaris12 32bit ASIC needs a special MC firmware */ + if (RREG32(mmMC_SEQ_IO_DEBUG_DATA) == 0x05b4dc40) + chip_name = "polaris12_32"; + else + chip_name = "polaris12"; + } break; case CHIP_FIJI: case CHIP_CARRIZO: diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c index def583916294..9b844e9fb16f 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c @@ -584,6 +584,10 @@ static void vcn_v3_0_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( VCN, inst_idx, mmUVD_VCPU_NONCACHE_SIZE0), AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)), 0, indirect); + + /* VCN global tiling registers */ + WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( + UVD, 0, mmUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect); } static void vcn_v3_0_disable_static_power_gating(struct amdgpu_device *adev, int inst) diff --git a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c index 88626d83e07b..ca8efa5c6978 100644 --- a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c @@ -220,10 +220,8 @@ static int vega10_ih_enable_ring(struct amdgpu_device *adev, tmp = vega10_ih_rb_cntl(ih, tmp); if (ih == &adev->irq.ih) tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RPTR_REARM, !!adev->irq.msi_enabled); - if (ih == &adev->irq.ih1) { - tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 0); + if (ih == &adev->irq.ih1) tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_FULL_DRAIN_ENABLE, 1); - } if (amdgpu_sriov_vf(adev)) { if (psp_reg_program(&adev->psp, ih_regs->psp_reg_id, tmp)) { dev_err(adev->dev, "PSP program IH_RB_CNTL failed!\n"); @@ -265,7 +263,6 @@ static int vega10_ih_irq_init(struct amdgpu_device *adev) u32 ih_chicken; int ret; int i; - u32 tmp; /* disable irqs */ ret = vega10_ih_toggle_interrupts(adev, false); @@ -291,15 +288,6 @@ static int vega10_ih_irq_init(struct amdgpu_device *adev) } } - tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL); - tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL, - CLIENT18_IS_STORM_CLIENT, 1); - WREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL, tmp); - - tmp = RREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL); - tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1); - WREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL, tmp); - pci_set_master(adev->pdev); /* enable interrupts */ @@ -345,11 +333,17 @@ static u32 vega10_ih_get_wptr(struct amdgpu_device *adev, u32 wptr, tmp; struct amdgpu_ih_regs *ih_regs; - wptr = le32_to_cpu(*ih->wptr_cpu); - ih_regs = &ih->ih_regs; + if (ih == &adev->irq.ih) { + /* Only ring0 supports writeback. On other rings fall back + * to register-based code with overflow checking below. + */ + wptr = le32_to_cpu(*ih->wptr_cpu); - if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) - goto out; + if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) + goto out; + } + + ih_regs = &ih->ih_regs; /* Double check that the overflow wasn't already cleared. */ wptr = RREG32_NO_KIQ(ih_regs->ih_rb_wptr); @@ -440,15 +434,11 @@ static int vega10_ih_self_irq(struct amdgpu_device *adev, struct amdgpu_irq_src *source, struct amdgpu_iv_entry *entry) { - uint32_t wptr = cpu_to_le32(entry->src_data[0]); - switch (entry->ring_id) { case 1: - *adev->irq.ih1.wptr_cpu = wptr; schedule_work(&adev->irq.ih1_work); break; case 2: - *adev->irq.ih2.wptr_cpu = wptr; schedule_work(&adev->irq.ih2_work); break; default: break; diff --git a/drivers/gpu/drm/amd/amdgpu/vega20_ih.c b/drivers/gpu/drm/amd/amdgpu/vega20_ih.c index 5a3c867d5881..86dcf448e0c2 100644 --- a/drivers/gpu/drm/amd/amdgpu/vega20_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/vega20_ih.c @@ -104,6 +104,8 @@ static int vega20_ih_toggle_ring_interrupts(struct amdgpu_device *adev, tmp = RREG32(ih_regs->ih_rb_cntl); tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_ENABLE, (enable ? 1 : 0)); + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_GPU_TS_ENABLE, 1); + /* enable_intr field is only valid in ring0 */ if (ih == &adev->irq.ih) tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, ENABLE_INTR, (enable ? 1 : 0)); @@ -220,10 +222,8 @@ static int vega20_ih_enable_ring(struct amdgpu_device *adev, tmp = vega20_ih_rb_cntl(ih, tmp); if (ih == &adev->irq.ih) tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RPTR_REARM, !!adev->irq.msi_enabled); - if (ih == &adev->irq.ih1) { - tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 0); + if (ih == &adev->irq.ih1) tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_FULL_DRAIN_ENABLE, 1); - } if (amdgpu_sriov_vf(adev)) { if (psp_reg_program(&adev->psp, ih_regs->psp_reg_id, tmp)) { dev_err(adev->dev, "PSP program IH_RB_CNTL failed!\n"); @@ -297,7 +297,6 @@ static int vega20_ih_irq_init(struct amdgpu_device *adev) u32 ih_chicken; int ret; int i; - u32 tmp; /* disable irqs */ ret = vega20_ih_toggle_interrupts(adev, false); @@ -326,15 +325,6 @@ static int vega20_ih_irq_init(struct amdgpu_device *adev) } } - tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL); - tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL, - CLIENT18_IS_STORM_CLIENT, 1); - WREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL, tmp); - - tmp = RREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL); - tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1); - WREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL, tmp); - pci_set_master(adev->pdev); /* enable interrupts */ @@ -380,11 +370,17 @@ static u32 vega20_ih_get_wptr(struct amdgpu_device *adev, u32 wptr, tmp; struct amdgpu_ih_regs *ih_regs; - wptr = le32_to_cpu(*ih->wptr_cpu); - ih_regs = &ih->ih_regs; + if (ih == &adev->irq.ih) { + /* Only ring0 supports writeback. On other rings fall back + * to register-based code with overflow checking below. + */ + wptr = le32_to_cpu(*ih->wptr_cpu); - if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) - goto out; + if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) + goto out; + } + + ih_regs = &ih->ih_regs; /* Double check that the overflow wasn't already cleared. */ wptr = RREG32_NO_KIQ(ih_regs->ih_rb_wptr); @@ -476,15 +472,11 @@ static int vega20_ih_self_irq(struct amdgpu_device *adev, struct amdgpu_irq_src *source, struct amdgpu_iv_entry *entry) { - uint32_t wptr = cpu_to_le32(entry->src_data[0]); - switch (entry->ring_id) { case 1: - *adev->irq.ih1.wptr_cpu = wptr; schedule_work(&adev->irq.ih1_work); break; case 2: - *adev->irq.ih2.wptr_cpu = wptr; schedule_work(&adev->irq.ih2_work); break; default: break; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c b/drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c index 511712c2e382..673d5e34f213 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c @@ -33,6 +33,11 @@ static int kfd_debugfs_open(struct inode *inode, struct file *file) return single_open(file, show, NULL); } +static int kfd_debugfs_hang_hws_read(struct seq_file *m, void *data) +{ + seq_printf(m, "echo gpu_id > hang_hws\n"); + return 0; +} static ssize_t kfd_debugfs_hang_hws_write(struct file *file, const char __user *user_buf, size_t size, loff_t *ppos) @@ -94,7 +99,7 @@ void kfd_debugfs_init(void) debugfs_create_file("rls", S_IFREG | 0444, debugfs_root, kfd_debugfs_rls_by_device, &kfd_debugfs_fops); debugfs_create_file("hang_hws", S_IFREG | 0200, debugfs_root, - NULL, &kfd_debugfs_hang_hws_fops); + kfd_debugfs_hang_hws_read, &kfd_debugfs_hang_hws_fops); } void kfd_debugfs_fini(void) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index 4598a9a58125..a4266c4bca13 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -1128,6 +1128,9 @@ static int set_sched_resources(struct device_queue_manager *dqm) static int initialize_cpsch(struct device_queue_manager *dqm) { + uint64_t num_sdma_queues; + uint64_t num_xgmi_sdma_queues; + pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm)); mutex_init(&dqm->lock_hidden); @@ -1136,8 +1139,18 @@ static int initialize_cpsch(struct device_queue_manager *dqm) dqm->active_cp_queue_count = 0; dqm->gws_queue_count = 0; dqm->active_runlist = false; - dqm->sdma_bitmap = ~0ULL >> (64 - get_num_sdma_queues(dqm)); - dqm->xgmi_sdma_bitmap = ~0ULL >> (64 - get_num_xgmi_sdma_queues(dqm)); + + num_sdma_queues = get_num_sdma_queues(dqm); + if (num_sdma_queues >= BITS_PER_TYPE(dqm->sdma_bitmap)) + dqm->sdma_bitmap = ULLONG_MAX; + else + dqm->sdma_bitmap = (BIT_ULL(num_sdma_queues) - 1); + + num_xgmi_sdma_queues = get_num_xgmi_sdma_queues(dqm); + if (num_xgmi_sdma_queues >= BITS_PER_TYPE(dqm->xgmi_sdma_bitmap)) + dqm->xgmi_sdma_bitmap = ULLONG_MAX; + else + dqm->xgmi_sdma_bitmap = (BIT_ULL(num_xgmi_sdma_queues) - 1); INIT_WORK(&dqm->hw_exception_work, kfd_process_hw_exception); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c b/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c index 66bbca61e3ef..9318936aa805 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c @@ -20,6 +20,10 @@ * OTHER DEALINGS IN THE SOFTWARE. */ +#include + +#if IS_REACHABLE(CONFIG_AMD_IOMMU_V2) + #include #include #include @@ -355,3 +359,5 @@ int kfd_iommu_add_perf_counters(struct kfd_topology_device *kdev) return 0; } + +#endif diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_iommu.h b/drivers/gpu/drm/amd/amdkfd/kfd_iommu.h index dd23d9fdf6a8..afd420b01a0c 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_iommu.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_iommu.h @@ -23,7 +23,9 @@ #ifndef __KFD_IOMMU_H__ #define __KFD_IOMMU_H__ -#if defined(CONFIG_AMD_IOMMU_V2_MODULE) || defined(CONFIG_AMD_IOMMU_V2) +#include + +#if IS_REACHABLE(CONFIG_AMD_IOMMU_V2) #define KFD_SUPPORT_IOMMU_V2 @@ -46,6 +48,9 @@ static inline int kfd_iommu_check_device(struct kfd_dev *kfd) } static inline int kfd_iommu_device_init(struct kfd_dev *kfd) { +#if IS_MODULE(CONFIG_AMD_IOMMU_V2) + WARN_ONCE(1, "iommu_v2 module is not usable by built-in KFD"); +#endif return 0; } @@ -73,6 +78,6 @@ static inline int kfd_iommu_add_perf_counters(struct kfd_topology_device *kdev) return 0; } -#endif /* defined(CONFIG_AMD_IOMMU_V2) */ +#endif /* IS_REACHABLE(CONFIG_AMD_IOMMU_V2) */ #endif /* __KFD_IOMMU_H__ */ diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index d699a5cf6c11..b63f55ea8758 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -1191,6 +1191,15 @@ static void amdgpu_dm_fini(struct amdgpu_device *adev) if (adev->dm.dc) dc_deinit_callbacks(adev->dm.dc); #endif + +#if defined(CONFIG_DRM_AMD_DC_DCN) + if (adev->dm.vblank_workqueue) { + adev->dm.vblank_workqueue->dm = NULL; + kfree(adev->dm.vblank_workqueue); + adev->dm.vblank_workqueue = NULL; + } +#endif + if (adev->dm.dc->ctx->dmub_srv) { dc_dmub_srv_destroy(&adev->dm.dc->ctx->dmub_srv); adev->dm.dc->ctx->dmub_srv = NULL; @@ -3841,6 +3850,23 @@ static int fill_dc_scaling_info(const struct drm_plane_state *state, scaling_info->src_rect.x = state->src_x >> 16; scaling_info->src_rect.y = state->src_y >> 16; + /* + * For reasons we don't (yet) fully understand a non-zero + * src_y coordinate into an NV12 buffer can cause a + * system hang. To avoid hangs (and maybe be overly cautious) + * let's reject both non-zero src_x and src_y. + * + * We currently know of only one use-case to reproduce a + * scenario with non-zero src_x and src_y for NV12, which + * is to gesture the YouTube Android app into full screen + * on ChromeOS. + */ + if (state->fb && + state->fb->format->format == DRM_FORMAT_NV12 && + (scaling_info->src_rect.x != 0 || + scaling_info->src_rect.y != 0)) + return -EINVAL; + scaling_info->src_rect.width = state->src_w >> 16; if (scaling_info->src_rect.width == 0) return -EINVAL; @@ -5863,6 +5889,15 @@ create_validate_stream_for_sink(struct amdgpu_dm_connector *aconnector, } while (stream == NULL && requested_bpc >= 6); + if (dc_result == DC_FAIL_ENC_VALIDATE && !aconnector->force_yuv420_output) { + DRM_DEBUG_KMS("Retry forcing YCbCr420 encoding\n"); + + aconnector->force_yuv420_output = true; + stream = create_validate_stream_for_sink(aconnector, drm_mode, + dm_state, old_stream); + aconnector->force_yuv420_output = false; + } + return stream; } @@ -7417,10 +7452,6 @@ static int get_cursor_position(struct drm_plane *plane, struct drm_crtc *crtc, int x, y; int xorigin = 0, yorigin = 0; - position->enable = false; - position->x = 0; - position->y = 0; - if (!crtc || !plane->state->fb) return 0; @@ -7467,7 +7498,7 @@ static void handle_cursor_update(struct drm_plane *plane, struct dm_crtc_state *crtc_state = crtc ? to_dm_crtc_state(crtc->state) : NULL; struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); uint64_t address = afb ? afb->address : 0; - struct dc_cursor_position position; + struct dc_cursor_position position = {0}; struct dc_cursor_attributes attributes; int ret; @@ -9264,7 +9295,8 @@ static int dm_check_crtc_cursor(struct drm_atomic_state *state, new_cursor_state = drm_atomic_get_new_plane_state(state, crtc->cursor); new_primary_state = drm_atomic_get_new_plane_state(state, crtc->primary); - if (!new_cursor_state || !new_primary_state || !new_cursor_state->fb) { + if (!new_cursor_state || !new_primary_state || + !new_cursor_state->fb || !new_primary_state->fb) { return 0; } @@ -9312,6 +9344,53 @@ static int add_affected_mst_dsc_crtcs(struct drm_atomic_state *state, struct drm } #endif +static int validate_overlay(struct drm_atomic_state *state) +{ + int i; + struct drm_plane *plane; + struct drm_plane_state *old_plane_state, *new_plane_state; + struct drm_plane_state *primary_state, *overlay_state = NULL; + + /* Check if primary plane is contained inside overlay */ + for_each_oldnew_plane_in_state_reverse(state, plane, old_plane_state, new_plane_state, i) { + if (plane->type == DRM_PLANE_TYPE_OVERLAY) { + if (drm_atomic_plane_disabling(plane->state, new_plane_state)) + return 0; + + overlay_state = new_plane_state; + continue; + } + } + + /* check if we're making changes to the overlay plane */ + if (!overlay_state) + return 0; + + /* check if overlay plane is enabled */ + if (!overlay_state->crtc) + return 0; + + /* find the primary plane for the CRTC that the overlay is enabled on */ + primary_state = drm_atomic_get_plane_state(state, overlay_state->crtc->primary); + if (IS_ERR(primary_state)) + return PTR_ERR(primary_state); + + /* check if primary plane is enabled */ + if (!primary_state->crtc) + return 0; + + /* Perform the bounds check to ensure the overlay plane covers the primary */ + if (primary_state->crtc_x < overlay_state->crtc_x || + primary_state->crtc_y < overlay_state->crtc_y || + primary_state->crtc_x + primary_state->crtc_w > overlay_state->crtc_x + overlay_state->crtc_w || + primary_state->crtc_y + primary_state->crtc_h > overlay_state->crtc_y + overlay_state->crtc_h) { + DRM_DEBUG_ATOMIC("Overlay plane is enabled with hardware cursor but does not fully cover primary plane\n"); + return -EINVAL; + } + + return 0; +} + /** * amdgpu_dm_atomic_check() - Atomic check implementation for AMDgpu DM. * @dev: The DRM device @@ -9383,7 +9462,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev, } #if defined(CONFIG_DRM_AMD_DC_DCN) - if (adev->asic_type >= CHIP_NAVI10) { + if (dc_resource_is_dsc_encoding_supported(dc)) { for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { if (drm_atomic_crtc_needs_modeset(new_crtc_state)) { ret = add_affected_mst_dsc_crtcs(state, crtc); @@ -9486,6 +9565,10 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev, goto fail; } + ret = validate_overlay(state); + if (ret) + goto fail; + /* Add new/modified planes */ for_each_oldnew_plane_in_state_reverse(state, plane, old_plane_state, new_plane_state, i) { ret = dm_update_plane_state(dc, state, plane, diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h index 8bfe901cf237..52cc81705280 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h @@ -68,18 +68,6 @@ struct common_irq_params { enum dc_irq_source irq_src; }; -/** - * struct irq_list_head - Linked-list for low context IRQ handlers. - * - * @head: The list_head within &struct handler_data - * @work: A work_struct containing the deferred handler work - */ -struct irq_list_head { - struct list_head head; - /* In case this interrupt needs post-processing, 'work' will be queued*/ - struct work_struct work; -}; - /** * struct dm_compressor_info - Buffer info used by frame buffer compression * @cpu_addr: MMIO cpu addr @@ -293,7 +281,7 @@ struct amdgpu_display_manager { * Note that handlers are called in the same order as they were * registered (FIFO). */ - struct irq_list_head irq_handler_list_low_tab[DAL_IRQ_SOURCES_NUMBER]; + struct list_head irq_handler_list_low_tab[DAL_IRQ_SOURCES_NUMBER]; /** * @irq_handler_list_high_tab: diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c index 360952129b6d..29139b34dbe2 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c @@ -150,7 +150,7 @@ static int parse_write_buffer_into_params(char *wr_buf, uint32_t wr_buf_size, * * --- to get dp configuration * - * cat link_settings + * cat /sys/kernel/debug/dri/0/DP-x/link_settings * * It will list current, verified, reported, preferred dp configuration. * current -- for current video mode @@ -163,7 +163,7 @@ static int parse_write_buffer_into_params(char *wr_buf, uint32_t wr_buf_size, * echo > link_settings * * for example, to force to 2 lane, 2.7GHz, - * echo 4 0xa > link_settings + * echo 4 0xa > /sys/kernel/debug/dri/0/DP-x/link_settings * * spread_spectrum could not be changed dynamically. * @@ -171,7 +171,7 @@ static int parse_write_buffer_into_params(char *wr_buf, uint32_t wr_buf_size, * done. please check link settings after force operation to see if HW get * programming. * - * cat link_settings + * cat /sys/kernel/debug/dri/0/DP-x/link_settings * * check current and preferred settings. * @@ -255,7 +255,7 @@ static ssize_t dp_link_settings_write(struct file *f, const char __user *buf, int max_param_num = 2; uint8_t param_nums = 0; long param[2]; - bool valid_input = false; + bool valid_input = true; if (size == 0) return -EINVAL; @@ -282,9 +282,9 @@ static ssize_t dp_link_settings_write(struct file *f, const char __user *buf, case LANE_COUNT_ONE: case LANE_COUNT_TWO: case LANE_COUNT_FOUR: - valid_input = true; break; default: + valid_input = false; break; } @@ -294,9 +294,9 @@ static ssize_t dp_link_settings_write(struct file *f, const char __user *buf, case LINK_RATE_RBR2: case LINK_RATE_HIGH2: case LINK_RATE_HIGH3: - valid_input = true; break; default: + valid_input = false; break; } @@ -310,10 +310,11 @@ static ssize_t dp_link_settings_write(struct file *f, const char __user *buf, * spread spectrum will not be changed */ prefer_link_settings.link_spread = link->cur_link_settings.link_spread; + prefer_link_settings.use_link_rate_set = false; prefer_link_settings.lane_count = param[0]; prefer_link_settings.link_rate = param[1]; - dc_link_set_preferred_link_settings(dc, &prefer_link_settings, link); + dc_link_set_preferred_training_settings(dc, &prefer_link_settings, NULL, link, true); kfree(wr_buf); return size; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c index 0cdbfcd475ec..71a15f68514b 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c @@ -644,6 +644,7 @@ struct hdcp_workqueue *hdcp_create_workqueue(struct amdgpu_device *adev, struct /* File created at /sys/class/drm/card0/device/hdcp_srm*/ hdcp_work[0].attr = data_attr; + sysfs_bin_attr_init(&hdcp_work[0].attr); if (sysfs_create_bin_file(&adev->dev->kobj, &hdcp_work[0].attr)) DRM_WARN("Failed to create device file hdcp_srm"); diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c index e0000c180ed1..8ce10d0973c5 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c @@ -82,6 +82,7 @@ struct amdgpu_dm_irq_handler_data { struct amdgpu_display_manager *dm; /* DAL irq source which registered for this interrupt. */ enum dc_irq_source irq_source; + struct work_struct work; }; #define DM_IRQ_TABLE_LOCK(adev, flags) \ @@ -111,20 +112,10 @@ static void init_handler_common_data(struct amdgpu_dm_irq_handler_data *hcd, */ static void dm_irq_work_func(struct work_struct *work) { - struct irq_list_head *irq_list_head = - container_of(work, struct irq_list_head, work); - struct list_head *handler_list = &irq_list_head->head; - struct amdgpu_dm_irq_handler_data *handler_data; - - list_for_each_entry(handler_data, handler_list, list) { - DRM_DEBUG_KMS("DM_IRQ: work_func: for dal_src=%d\n", - handler_data->irq_source); + struct amdgpu_dm_irq_handler_data *handler_data = + container_of(work, struct amdgpu_dm_irq_handler_data, work); - DRM_DEBUG_KMS("DM_IRQ: schedule_work: for dal_src=%d\n", - handler_data->irq_source); - - handler_data->handler(handler_data->handler_arg); - } + handler_data->handler(handler_data->handler_arg); /* Call a DAL subcomponent which registered for interrupt notification * at INTERRUPT_LOW_IRQ_CONTEXT. @@ -156,7 +147,7 @@ static struct list_head *remove_irq_handler(struct amdgpu_device *adev, break; case INTERRUPT_LOW_IRQ_CONTEXT: default: - hnd_list = &adev->dm.irq_handler_list_low_tab[irq_source].head; + hnd_list = &adev->dm.irq_handler_list_low_tab[irq_source]; break; } @@ -290,7 +281,8 @@ void *amdgpu_dm_irq_register_interrupt(struct amdgpu_device *adev, break; case INTERRUPT_LOW_IRQ_CONTEXT: default: - hnd_list = &adev->dm.irq_handler_list_low_tab[irq_source].head; + hnd_list = &adev->dm.irq_handler_list_low_tab[irq_source]; + INIT_WORK(&handler_data->work, dm_irq_work_func); break; } @@ -372,7 +364,7 @@ void amdgpu_dm_irq_unregister_interrupt(struct amdgpu_device *adev, int amdgpu_dm_irq_init(struct amdgpu_device *adev) { int src; - struct irq_list_head *lh; + struct list_head *lh; DRM_DEBUG_KMS("DM_IRQ\n"); @@ -381,9 +373,7 @@ int amdgpu_dm_irq_init(struct amdgpu_device *adev) for (src = 0; src < DAL_IRQ_SOURCES_NUMBER; src++) { /* low context handler list init */ lh = &adev->dm.irq_handler_list_low_tab[src]; - INIT_LIST_HEAD(&lh->head); - INIT_WORK(&lh->work, dm_irq_work_func); - + INIT_LIST_HEAD(lh); /* high context handler init */ INIT_LIST_HEAD(&adev->dm.irq_handler_list_high_tab[src]); } @@ -400,8 +390,11 @@ int amdgpu_dm_irq_init(struct amdgpu_device *adev) void amdgpu_dm_irq_fini(struct amdgpu_device *adev) { int src; - struct irq_list_head *lh; + struct list_head *lh; + struct list_head *entry, *tmp; + struct amdgpu_dm_irq_handler_data *handler; unsigned long irq_table_flags; + DRM_DEBUG_KMS("DM_IRQ: releasing resources.\n"); for (src = 0; src < DAL_IRQ_SOURCES_NUMBER; src++) { DM_IRQ_TABLE_LOCK(adev, irq_table_flags); @@ -410,7 +403,16 @@ void amdgpu_dm_irq_fini(struct amdgpu_device *adev) * (because no code can schedule a new one). */ lh = &adev->dm.irq_handler_list_low_tab[src]; DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags); - flush_work(&lh->work); + + if (!list_empty(lh)) { + list_for_each_safe(entry, tmp, lh) { + handler = list_entry( + entry, + struct amdgpu_dm_irq_handler_data, + list); + flush_work(&handler->work); + } + } } } @@ -420,6 +422,8 @@ int amdgpu_dm_irq_suspend(struct amdgpu_device *adev) struct list_head *hnd_list_h; struct list_head *hnd_list_l; unsigned long irq_table_flags; + struct list_head *entry, *tmp; + struct amdgpu_dm_irq_handler_data *handler; DM_IRQ_TABLE_LOCK(adev, irq_table_flags); @@ -430,14 +434,22 @@ int amdgpu_dm_irq_suspend(struct amdgpu_device *adev) * will be disabled from manage_dm_interrupts on disable CRTC. */ for (src = DC_IRQ_SOURCE_HPD1; src <= DC_IRQ_SOURCE_HPD6RX; src++) { - hnd_list_l = &adev->dm.irq_handler_list_low_tab[src].head; + hnd_list_l = &adev->dm.irq_handler_list_low_tab[src]; hnd_list_h = &adev->dm.irq_handler_list_high_tab[src]; if (!list_empty(hnd_list_l) || !list_empty(hnd_list_h)) dc_interrupt_set(adev->dm.dc, src, false); DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags); - flush_work(&adev->dm.irq_handler_list_low_tab[src].work); + if (!list_empty(hnd_list_l)) { + list_for_each_safe (entry, tmp, hnd_list_l) { + handler = list_entry( + entry, + struct amdgpu_dm_irq_handler_data, + list); + flush_work(&handler->work); + } + } DM_IRQ_TABLE_LOCK(adev, irq_table_flags); } @@ -457,7 +469,7 @@ int amdgpu_dm_irq_resume_early(struct amdgpu_device *adev) /* re-enable short pulse interrupts HW interrupt */ for (src = DC_IRQ_SOURCE_HPD1RX; src <= DC_IRQ_SOURCE_HPD6RX; src++) { - hnd_list_l = &adev->dm.irq_handler_list_low_tab[src].head; + hnd_list_l = &adev->dm.irq_handler_list_low_tab[src]; hnd_list_h = &adev->dm.irq_handler_list_high_tab[src]; if (!list_empty(hnd_list_l) || !list_empty(hnd_list_h)) dc_interrupt_set(adev->dm.dc, src, true); @@ -483,7 +495,7 @@ int amdgpu_dm_irq_resume_late(struct amdgpu_device *adev) * will be enabled from manage_dm_interrupts on enable CRTC. */ for (src = DC_IRQ_SOURCE_HPD1; src <= DC_IRQ_SOURCE_HPD6; src++) { - hnd_list_l = &adev->dm.irq_handler_list_low_tab[src].head; + hnd_list_l = &adev->dm.irq_handler_list_low_tab[src]; hnd_list_h = &adev->dm.irq_handler_list_high_tab[src]; if (!list_empty(hnd_list_l) || !list_empty(hnd_list_h)) dc_interrupt_set(adev->dm.dc, src, true); @@ -500,22 +512,53 @@ int amdgpu_dm_irq_resume_late(struct amdgpu_device *adev) static void amdgpu_dm_irq_schedule_work(struct amdgpu_device *adev, enum dc_irq_source irq_source) { - unsigned long irq_table_flags; - struct work_struct *work = NULL; + struct list_head *handler_list = &adev->dm.irq_handler_list_low_tab[irq_source]; + struct amdgpu_dm_irq_handler_data *handler_data; + bool work_queued = false; - DM_IRQ_TABLE_LOCK(adev, irq_table_flags); + if (list_empty(handler_list)) + return; + + list_for_each_entry (handler_data, handler_list, list) { + if (!queue_work(system_highpri_wq, &handler_data->work)) { + continue; + } else { + work_queued = true; + break; + } + } - if (!list_empty(&adev->dm.irq_handler_list_low_tab[irq_source].head)) - work = &adev->dm.irq_handler_list_low_tab[irq_source].work; + if (!work_queued) { + struct amdgpu_dm_irq_handler_data *handler_data_add; + /*get the amdgpu_dm_irq_handler_data of first item pointed by handler_list*/ + handler_data = container_of(handler_list->next, struct amdgpu_dm_irq_handler_data, list); - DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags); + /*allocate a new amdgpu_dm_irq_handler_data*/ + handler_data_add = kzalloc(sizeof(*handler_data), GFP_KERNEL); + if (!handler_data_add) { + DRM_ERROR("DM_IRQ: failed to allocate irq handler!\n"); + return; + } - if (work) { - if (!schedule_work(work)) - DRM_INFO("amdgpu_dm_irq_schedule_work FAILED src %d\n", - irq_source); - } + /*copy new amdgpu_dm_irq_handler_data members from handler_data*/ + handler_data_add->handler = handler_data->handler; + handler_data_add->handler_arg = handler_data->handler_arg; + handler_data_add->dm = handler_data->dm; + handler_data_add->irq_source = irq_source; + list_add_tail(&handler_data_add->list, handler_list); + + INIT_WORK(&handler_data_add->work, dm_irq_work_func); + + if (queue_work(system_highpri_wq, &handler_data_add->work)) + DRM_DEBUG("Queued work for handling interrupt from " + "display for IRQ source %d\n", + irq_source); + else + DRM_ERROR("Failed to queue work for handling interrupt " + "from display for IRQ source %d\n", + irq_source); + } } /* diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c index 995ffbbf64e7..1ee27f2f28f1 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c @@ -217,6 +217,9 @@ void dc_destroy_clk_mgr(struct clk_mgr *clk_mgr_base) if (ASICREV_IS_SIENNA_CICHLID_P(clk_mgr_base->ctx->asic_id.hw_internal_rev)) { dcn3_clk_mgr_destroy(clk_mgr); } + if (ASICREV_IS_DIMGREY_CAVEFISH_P(clk_mgr_base->ctx->asic_id.hw_internal_rev)) { + dcn3_clk_mgr_destroy(clk_mgr); + } break; case FAMILY_VGH: diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c index c7e5a64e06af..81ea5d3a1947 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c @@ -252,6 +252,7 @@ static void dcn3_update_clocks(struct clk_mgr *clk_mgr_base, bool force_reset = false; bool update_uclk = false; bool p_state_change_support; + int total_plane_count; if (dc->work_arounds.skip_clock_update || !clk_mgr->smu_present) return; @@ -292,7 +293,8 @@ static void dcn3_update_clocks(struct clk_mgr *clk_mgr_base, clk_mgr_base->clks.socclk_khz = new_clocks->socclk_khz; clk_mgr_base->clks.prev_p_state_change_support = clk_mgr_base->clks.p_state_change_support; - p_state_change_support = new_clocks->p_state_change_support || (display_count == 0); + total_plane_count = clk_mgr_helper_get_active_plane_cnt(dc, context); + p_state_change_support = new_clocks->p_state_change_support || (total_plane_count == 0); if (should_update_pstate_support(safe_to_lower, p_state_change_support, clk_mgr_base->clks.p_state_change_support)) { clk_mgr_base->clks.p_state_change_support = p_state_change_support; diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index 8f8a13c7cf73..4781279024a9 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -2398,7 +2398,8 @@ static void commit_planes_do_stream_update(struct dc *dc, if (pipe_ctx->stream_res.audio && !dc->debug.az_endpoint_mute_only) pipe_ctx->stream_res.audio->funcs->az_disable(pipe_ctx->stream_res.audio); - dc->hwss.optimize_bandwidth(dc, dc->current_state); + dc->optimized_required = true; + } else { if (dc->optimize_seamless_boot_streams == 0) dc->hwss.prepare_bandwidth(dc, dc->current_state); @@ -2545,6 +2546,10 @@ static void commit_planes_for_stream(struct dc *dc, plane_state->triplebuffer_flips = true; } } + if (update_type == UPDATE_TYPE_FULL) { + /* force vsync flip when reconfiguring pipes to prevent underflow */ + plane_state->flip_immediate = false; + } } } diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link.c b/drivers/gpu/drm/amd/display/dc/core/dc_link.c index bd0101013ec8..440bf0a0e12a 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link.c @@ -1603,6 +1603,7 @@ static bool dc_link_construct(struct dc_link *link, link->psr_settings.psr_version = DC_PSR_VERSION_UNSUPPORTED; DC_LOG_DC("BIOS object table - %s finished successfully.\n", __func__); + kfree(info); return true; device_tag_fail: link->link_enc->funcs->destroy(&link->link_enc); diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_abm.c b/drivers/gpu/drm/amd/display/dc/dce/dce_abm.c index 4e87e70237e3..874b132fe1d7 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_abm.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_abm.c @@ -283,7 +283,7 @@ struct abm *dce_abm_create( const struct dce_abm_shift *abm_shift, const struct dce_abm_mask *abm_mask) { - struct dce_abm *abm_dce = kzalloc(sizeof(*abm_dce), GFP_KERNEL); + struct dce_abm *abm_dce = kzalloc(sizeof(*abm_dce), GFP_ATOMIC); if (abm_dce == NULL) { BREAK_TO_DEBUGGER(); diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.h b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.h index 277484cf853e..d4be5954d7aa 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.h +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.h @@ -99,7 +99,6 @@ struct dce110_aux_registers { AUX_SF(AUX_SW_CONTROL, AUX_SW_GO, mask_sh),\ AUX_SF(AUX_SW_DATA, AUX_SW_AUTOINCREMENT_DISABLE, mask_sh),\ AUX_SF(AUX_SW_DATA, AUX_SW_DATA_RW, mask_sh),\ - AUX_SF(AUX_SW_DATA, AUX_SW_AUTOINCREMENT_DISABLE, mask_sh),\ AUX_SF(AUX_SW_DATA, AUX_SW_INDEX, mask_sh),\ AUX_SF(AUX_SW_DATA, AUX_SW_DATA, mask_sh),\ AUX_SF(AUX_SW_STATUS, AUX_SW_REPLY_BYTE_COUNT, mask_sh),\ diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c index ddc789daf3b1..09d4cb5c97b6 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c @@ -1049,7 +1049,7 @@ struct dmcu *dcn10_dmcu_create( const struct dce_dmcu_shift *dmcu_shift, const struct dce_dmcu_mask *dmcu_mask) { - struct dce_dmcu *dmcu_dce = kzalloc(sizeof(*dmcu_dce), GFP_KERNEL); + struct dce_dmcu *dmcu_dce = kzalloc(sizeof(*dmcu_dce), GFP_ATOMIC); if (dmcu_dce == NULL) { BREAK_TO_DEBUGGER(); @@ -1070,7 +1070,7 @@ struct dmcu *dcn20_dmcu_create( const struct dce_dmcu_shift *dmcu_shift, const struct dce_dmcu_mask *dmcu_mask) { - struct dce_dmcu *dmcu_dce = kzalloc(sizeof(*dmcu_dce), GFP_KERNEL); + struct dce_dmcu *dmcu_dce = kzalloc(sizeof(*dmcu_dce), GFP_ATOMIC); if (dmcu_dce == NULL) { BREAK_TO_DEBUGGER(); @@ -1091,7 +1091,7 @@ struct dmcu *dcn21_dmcu_create( const struct dce_dmcu_shift *dmcu_shift, const struct dce_dmcu_mask *dmcu_mask) { - struct dce_dmcu *dmcu_dce = kzalloc(sizeof(*dmcu_dce), GFP_KERNEL); + struct dce_dmcu *dmcu_dce = kzalloc(sizeof(*dmcu_dce), GFP_ATOMIC); if (dmcu_dce == NULL) { BREAK_TO_DEBUGGER(); diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c b/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c index 69e34bef274c..febccb35ddad 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c @@ -81,13 +81,18 @@ static void dmub_psr_get_state(struct dmub_psr *dmub, enum dc_psr_state *state) { struct dmub_srv *srv = dmub->ctx->dmub_srv->dmub; uint32_t raw_state; + enum dmub_status status = DMUB_STATUS_INVALID; // Send gpint command and wait for ack - dmub_srv_send_gpint_command(srv, DMUB_GPINT__GET_PSR_STATE, 0, 30); - - dmub_srv_get_gpint_response(srv, &raw_state); - - *state = convert_psr_state(raw_state); + status = dmub_srv_send_gpint_command(srv, DMUB_GPINT__GET_PSR_STATE, 0, 30); + + if (status == DMUB_STATUS_OK) { + // GPINT was executed, get response + dmub_srv_get_gpint_response(srv, &raw_state); + *state = convert_psr_state(raw_state); + } else + // Return invalid state when GPINT times out + *state = 0xFF; } /* diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.c index 62cc2651e00c..8774406120fc 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.c @@ -112,7 +112,7 @@ struct dccg *dccg2_create( const struct dccg_shift *dccg_shift, const struct dccg_mask *dccg_mask) { - struct dcn_dccg *dccg_dcn = kzalloc(sizeof(*dccg_dcn), GFP_KERNEL); + struct dcn_dccg *dccg_dcn = kzalloc(sizeof(*dccg_dcn), GFP_ATOMIC); struct dccg *base; if (dccg_dcn == NULL) { diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c index bec7059f6d5d..a1318c31bcfa 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c @@ -1,5 +1,5 @@ /* - * Copyright 2012-17 Advanced Micro Devices, Inc. + * Copyright 2012-2021 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -181,11 +181,14 @@ void hubp2_vready_at_or_After_vsync(struct hubp *hubp, else Set HUBP_VREADY_AT_OR_AFTER_VSYNC = 0 */ - if ((pipe_dest->vstartup_start - (pipe_dest->vready_offset+pipe_dest->vupdate_width - + pipe_dest->vupdate_offset) / pipe_dest->htotal) <= pipe_dest->vblank_end) { - value = 1; - } else - value = 0; + if (pipe_dest->htotal != 0) { + if ((pipe_dest->vstartup_start - (pipe_dest->vready_offset+pipe_dest->vupdate_width + + pipe_dest->vupdate_offset) / pipe_dest->htotal) <= pipe_dest->vblank_end) { + value = 1; + } else + value = 0; + } + REG_UPDATE(DCHUBP_CNTL, HUBP_VREADY_AT_OR_AFTER_VSYNC, value); } diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c index 2c2dbfcd8957..bfbc23b76cd5 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c @@ -1104,7 +1104,7 @@ struct dpp *dcn20_dpp_create( uint32_t inst) { struct dcn20_dpp *dpp = - kzalloc(sizeof(struct dcn20_dpp), GFP_KERNEL); + kzalloc(sizeof(struct dcn20_dpp), GFP_ATOMIC); if (!dpp) return NULL; @@ -1122,7 +1122,7 @@ struct input_pixel_processor *dcn20_ipp_create( struct dc_context *ctx, uint32_t inst) { struct dcn10_ipp *ipp = - kzalloc(sizeof(struct dcn10_ipp), GFP_KERNEL); + kzalloc(sizeof(struct dcn10_ipp), GFP_ATOMIC); if (!ipp) { BREAK_TO_DEBUGGER(); @@ -1139,7 +1139,7 @@ struct output_pixel_processor *dcn20_opp_create( struct dc_context *ctx, uint32_t inst) { struct dcn20_opp *opp = - kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL); + kzalloc(sizeof(struct dcn20_opp), GFP_ATOMIC); if (!opp) { BREAK_TO_DEBUGGER(); @@ -1156,7 +1156,7 @@ struct dce_aux *dcn20_aux_engine_create( uint32_t inst) { struct aux_engine_dce110 *aux_engine = - kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL); + kzalloc(sizeof(struct aux_engine_dce110), GFP_ATOMIC); if (!aux_engine) return NULL; @@ -1194,7 +1194,7 @@ struct dce_i2c_hw *dcn20_i2c_hw_create( uint32_t inst) { struct dce_i2c_hw *dce_i2c_hw = - kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL); + kzalloc(sizeof(struct dce_i2c_hw), GFP_ATOMIC); if (!dce_i2c_hw) return NULL; @@ -1207,7 +1207,7 @@ struct dce_i2c_hw *dcn20_i2c_hw_create( struct mpc *dcn20_mpc_create(struct dc_context *ctx) { struct dcn20_mpc *mpc20 = kzalloc(sizeof(struct dcn20_mpc), - GFP_KERNEL); + GFP_ATOMIC); if (!mpc20) return NULL; @@ -1225,7 +1225,7 @@ struct hubbub *dcn20_hubbub_create(struct dc_context *ctx) { int i; struct dcn20_hubbub *hubbub = kzalloc(sizeof(struct dcn20_hubbub), - GFP_KERNEL); + GFP_ATOMIC); if (!hubbub) return NULL; @@ -1253,7 +1253,7 @@ struct timing_generator *dcn20_timing_generator_create( uint32_t instance) { struct optc *tgn10 = - kzalloc(sizeof(struct optc), GFP_KERNEL); + kzalloc(sizeof(struct optc), GFP_ATOMIC); if (!tgn10) return NULL; @@ -1332,7 +1332,7 @@ static struct clock_source *dcn20_clock_source_create( bool dp_clk_src) { struct dce110_clk_src *clk_src = - kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL); + kzalloc(sizeof(struct dce110_clk_src), GFP_ATOMIC); if (!clk_src) return NULL; @@ -1438,7 +1438,7 @@ struct display_stream_compressor *dcn20_dsc_create( struct dc_context *ctx, uint32_t inst) { struct dcn20_dsc *dsc = - kzalloc(sizeof(struct dcn20_dsc), GFP_KERNEL); + kzalloc(sizeof(struct dcn20_dsc), GFP_ATOMIC); if (!dsc) { BREAK_TO_DEBUGGER(); @@ -1572,7 +1572,7 @@ struct hubp *dcn20_hubp_create( uint32_t inst) { struct dcn20_hubp *hubp2 = - kzalloc(sizeof(struct dcn20_hubp), GFP_KERNEL); + kzalloc(sizeof(struct dcn20_hubp), GFP_ATOMIC); if (!hubp2) return NULL; @@ -3390,7 +3390,7 @@ bool dcn20_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool) static struct pp_smu_funcs *dcn20_pp_smu_create(struct dc_context *ctx) { - struct pp_smu_funcs *pp_smu = kzalloc(sizeof(*pp_smu), GFP_KERNEL); + struct pp_smu_funcs *pp_smu = kzalloc(sizeof(*pp_smu), GFP_ATOMIC); if (!pp_smu) return pp_smu; @@ -4034,7 +4034,7 @@ struct resource_pool *dcn20_create_resource_pool( struct dc *dc) { struct dcn20_resource_pool *pool = - kzalloc(sizeof(struct dcn20_resource_pool), GFP_KERNEL); + kzalloc(sizeof(struct dcn20_resource_pool), GFP_ATOMIC); if (!pool) return NULL; diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c index 06dc1e2e8383..07c8d2e2c09c 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c @@ -848,7 +848,7 @@ bool dcn30_apply_idle_power_optimizations(struct dc *dc, bool enable) cmd.mall.cursor_copy_src.quad_part = cursor_attr.address.quad_part; cmd.mall.cursor_copy_dst.quad_part = - plane->address.grph.cursor_cache_addr.quad_part; + (plane->address.grph.cursor_cache_addr.quad_part + 2047) & ~2047; cmd.mall.cursor_width = cursor_attr.width; cmd.mall.cursor_height = cursor_attr.height; cmd.mall.cursor_pitch = cursor_attr.pitch; @@ -858,8 +858,7 @@ bool dcn30_apply_idle_power_optimizations(struct dc *dc, bool enable) dc_dmub_srv_wait_idle(dc->ctx->dmub_srv); /* Use copied cursor, and it's okay to not switch back */ - cursor_attr.address.quad_part = - plane->address.grph.cursor_cache_addr.quad_part; + cursor_attr.address.quad_part = cmd.mall.cursor_copy_dst.quad_part; dc_stream_set_cursor_attributes(stream, &cursor_attr); } diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mpc.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mpc.c index 3e6f76096119..a7598356f37d 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mpc.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mpc.c @@ -143,16 +143,18 @@ static void mpc3_power_on_ogam_lut( { struct dcn30_mpc *mpc30 = TO_DCN30_MPC(mpc); - if (mpc->ctx->dc->debug.enable_mem_low_power.bits.mpc) { - // Force power on - REG_UPDATE(MPCC_MEM_PWR_CTRL[mpcc_id], MPCC_OGAM_MEM_PWR_DIS, power_on == true ? 1:0); - // Wait for confirmation when powering on - if (power_on) - REG_WAIT(MPCC_MEM_PWR_CTRL[mpcc_id], MPCC_OGAM_MEM_PWR_STATE, 0, 10, 10); - } else { - REG_SET(MPCC_MEM_PWR_CTRL[mpcc_id], 0, - MPCC_OGAM_MEM_PWR_FORCE, power_on == true ? 0 : 1); - } + /* + * Powering on: force memory active so the LUT can be updated. + * Powering off: allow entering memory low power mode + * + * Memory low power mode is controlled during MPC OGAM LUT init. + */ + REG_UPDATE(MPCC_MEM_PWR_CTRL[mpcc_id], + MPCC_OGAM_MEM_PWR_DIS, power_on != 0); + + /* Wait for memory to be powered on - we won't be able to write to it otherwise. */ + if (power_on) + REG_WAIT(MPCC_MEM_PWR_CTRL[mpcc_id], MPCC_OGAM_MEM_PWR_STATE, 0, 10, 10); } static void mpc3_configure_ogam_lut( @@ -1427,7 +1429,7 @@ const struct mpc_funcs dcn30_mpc_funcs = { .acquire_rmu = mpcc3_acquire_rmu, .program_3dlut = mpc3_program_3dlut, .release_rmu = mpcc3_release_rmu, - .power_on_mpc_mem_pwr = mpc20_power_on_ogam_lut, + .power_on_mpc_mem_pwr = mpc3_power_on_ogam_lut, .get_mpc_out_mux = mpc1_get_mpc_out_mux, }; diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c index fb7f1dea3c46..71e2d5e02571 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c @@ -181,7 +181,7 @@ struct _vcs_dpi_soc_bounding_box_st dcn3_0_soc = { }, .min_dcfclk = 500.0, /* TODO: set this to actual min DCFCLK */ .num_states = 1, - .sr_exit_time_us = 12, + .sr_exit_time_us = 15.5, .sr_enter_plus_exit_time_us = 20, .urgent_latency_us = 4.0, .urgent_latency_pixel_data_only_us = 4.0, diff --git a/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c b/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c index 4b659b63f75b..d03b1975e417 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c @@ -164,7 +164,7 @@ struct _vcs_dpi_soc_bounding_box_st dcn3_02_soc = { .min_dcfclk = 500.0, /* TODO: set this to actual min DCFCLK */ .num_states = 1, - .sr_exit_time_us = 12, + .sr_exit_time_us = 15.5, .sr_enter_plus_exit_time_us = 20, .urgent_latency_us = 4.0, .urgent_latency_pixel_data_only_us = 4.0, diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.c index 0f3f510fd83b..9729cf292e84 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.c @@ -3437,6 +3437,7 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l mode_lib->vba.DCCEnabledInAnyPlane = true; } } + mode_lib->vba.UrgentLatency = mode_lib->vba.UrgentLatencyPixelDataOnly; for (i = 0; i <= mode_lib->vba.soc.num_states; i++) { locals->FabricAndDRAMBandwidthPerState[i] = dml_min( mode_lib->vba.DRAMSpeedPerState[i] * mode_lib->vba.NumberOfChannels diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c index 210c96cd5b03..51098c2c9854 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c @@ -3544,6 +3544,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode mode_lib->vba.DCCEnabledInAnyPlane = true; } } + mode_lib->vba.UrgentLatency = mode_lib->vba.UrgentLatencyPixelDataOnly; for (i = 0; i <= mode_lib->vba.soc.num_states; i++) { locals->FabricAndDRAMBandwidthPerState[i] = dml_min( mode_lib->vba.DRAMSpeedPerState[i] * mode_lib->vba.NumberOfChannels diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.c index 72423dc425dc..799bae229e67 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.c @@ -293,13 +293,31 @@ static void handle_det_buf_split(struct display_mode_lib *mode_lib, if (surf_linear) { log2_swath_height_l = 0; log2_swath_height_c = 0; - } else if (!surf_vert) { - log2_swath_height_l = dml_log2(rq_param->misc.rq_l.blk256_height) - req128_l; - log2_swath_height_c = dml_log2(rq_param->misc.rq_c.blk256_height) - req128_c; } else { - log2_swath_height_l = dml_log2(rq_param->misc.rq_l.blk256_width) - req128_l; - log2_swath_height_c = dml_log2(rq_param->misc.rq_c.blk256_width) - req128_c; + unsigned int swath_height_l; + unsigned int swath_height_c; + + if (!surf_vert) { + swath_height_l = rq_param->misc.rq_l.blk256_height; + swath_height_c = rq_param->misc.rq_c.blk256_height; + } else { + swath_height_l = rq_param->misc.rq_l.blk256_width; + swath_height_c = rq_param->misc.rq_c.blk256_width; + } + + if (swath_height_l > 0) + log2_swath_height_l = dml_log2(swath_height_l); + + if (req128_l && log2_swath_height_l > 0) + log2_swath_height_l -= 1; + + if (swath_height_c > 0) + log2_swath_height_c = dml_log2(swath_height_c); + + if (req128_c && log2_swath_height_c > 0) + log2_swath_height_c -= 1; } + rq_param->dlg.rq_l.swath_height = 1 << log2_swath_height_l; rq_param->dlg.rq_c.swath_height = 1 << log2_swath_height_c; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20v2.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20v2.c index 9c78446c3a9d..6a6d5970d1d5 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20v2.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20v2.c @@ -293,13 +293,31 @@ static void handle_det_buf_split(struct display_mode_lib *mode_lib, if (surf_linear) { log2_swath_height_l = 0; log2_swath_height_c = 0; - } else if (!surf_vert) { - log2_swath_height_l = dml_log2(rq_param->misc.rq_l.blk256_height) - req128_l; - log2_swath_height_c = dml_log2(rq_param->misc.rq_c.blk256_height) - req128_c; } else { - log2_swath_height_l = dml_log2(rq_param->misc.rq_l.blk256_width) - req128_l; - log2_swath_height_c = dml_log2(rq_param->misc.rq_c.blk256_width) - req128_c; + unsigned int swath_height_l; + unsigned int swath_height_c; + + if (!surf_vert) { + swath_height_l = rq_param->misc.rq_l.blk256_height; + swath_height_c = rq_param->misc.rq_c.blk256_height; + } else { + swath_height_l = rq_param->misc.rq_l.blk256_width; + swath_height_c = rq_param->misc.rq_c.blk256_width; + } + + if (swath_height_l > 0) + log2_swath_height_l = dml_log2(swath_height_l); + + if (req128_l && log2_swath_height_l > 0) + log2_swath_height_l -= 1; + + if (swath_height_c > 0) + log2_swath_height_c = dml_log2(swath_height_c); + + if (req128_c && log2_swath_height_c > 0) + log2_swath_height_c -= 1; } + rq_param->dlg.rq_l.swath_height = 1 << log2_swath_height_l; rq_param->dlg.rq_c.swath_height = 1 << log2_swath_height_c; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_rq_dlg_calc_21.c b/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_rq_dlg_calc_21.c index edd41d358291..dc1c81a6e377 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_rq_dlg_calc_21.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_rq_dlg_calc_21.c @@ -277,13 +277,31 @@ static void handle_det_buf_split( if (surf_linear) { log2_swath_height_l = 0; log2_swath_height_c = 0; - } else if (!surf_vert) { - log2_swath_height_l = dml_log2(rq_param->misc.rq_l.blk256_height) - req128_l; - log2_swath_height_c = dml_log2(rq_param->misc.rq_c.blk256_height) - req128_c; } else { - log2_swath_height_l = dml_log2(rq_param->misc.rq_l.blk256_width) - req128_l; - log2_swath_height_c = dml_log2(rq_param->misc.rq_c.blk256_width) - req128_c; + unsigned int swath_height_l; + unsigned int swath_height_c; + + if (!surf_vert) { + swath_height_l = rq_param->misc.rq_l.blk256_height; + swath_height_c = rq_param->misc.rq_c.blk256_height; + } else { + swath_height_l = rq_param->misc.rq_l.blk256_width; + swath_height_c = rq_param->misc.rq_c.blk256_width; + } + + if (swath_height_l > 0) + log2_swath_height_l = dml_log2(swath_height_l); + + if (req128_l && log2_swath_height_l > 0) + log2_swath_height_l -= 1; + + if (swath_height_c > 0) + log2_swath_height_c = dml_log2(swath_height_c); + + if (req128_c && log2_swath_height_c > 0) + log2_swath_height_c -= 1; } + rq_param->dlg.rq_l.swath_height = 1 << log2_swath_height_l; rq_param->dlg.rq_c.swath_height = 1 << log2_swath_height_c; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c index 0f14f205ebe5..04601a767a8f 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c @@ -237,13 +237,31 @@ static void handle_det_buf_split(struct display_mode_lib *mode_lib, if (surf_linear) { log2_swath_height_l = 0; log2_swath_height_c = 0; - } else if (!surf_vert) { - log2_swath_height_l = dml_log2(rq_param->misc.rq_l.blk256_height) - req128_l; - log2_swath_height_c = dml_log2(rq_param->misc.rq_c.blk256_height) - req128_c; } else { - log2_swath_height_l = dml_log2(rq_param->misc.rq_l.blk256_width) - req128_l; - log2_swath_height_c = dml_log2(rq_param->misc.rq_c.blk256_width) - req128_c; + unsigned int swath_height_l; + unsigned int swath_height_c; + + if (!surf_vert) { + swath_height_l = rq_param->misc.rq_l.blk256_height; + swath_height_c = rq_param->misc.rq_c.blk256_height; + } else { + swath_height_l = rq_param->misc.rq_l.blk256_width; + swath_height_c = rq_param->misc.rq_c.blk256_width; + } + + if (swath_height_l > 0) + log2_swath_height_l = dml_log2(swath_height_l); + + if (req128_l && log2_swath_height_l > 0) + log2_swath_height_l -= 1; + + if (swath_height_c > 0) + log2_swath_height_c = dml_log2(swath_height_c); + + if (req128_c && log2_swath_height_c > 0) + log2_swath_height_c -= 1; } + rq_param->dlg.rq_l.swath_height = 1 << log2_swath_height_l; rq_param->dlg.rq_c.swath_height = 1 << log2_swath_height_c; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dml1_display_rq_dlg_calc.c b/drivers/gpu/drm/amd/display/dc/dml/dml1_display_rq_dlg_calc.c index 4c3e9cc30167..414da64f5734 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dml1_display_rq_dlg_calc.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dml1_display_rq_dlg_calc.c @@ -344,13 +344,31 @@ static void handle_det_buf_split( if (surf_linear) { log2_swath_height_l = 0; log2_swath_height_c = 0; - } else if (!surf_vert) { - log2_swath_height_l = dml_log2(rq_param->misc.rq_l.blk256_height) - req128_l; - log2_swath_height_c = dml_log2(rq_param->misc.rq_c.blk256_height) - req128_c; } else { - log2_swath_height_l = dml_log2(rq_param->misc.rq_l.blk256_width) - req128_l; - log2_swath_height_c = dml_log2(rq_param->misc.rq_c.blk256_width) - req128_c; + unsigned int swath_height_l; + unsigned int swath_height_c; + + if (!surf_vert) { + swath_height_l = rq_param->misc.rq_l.blk256_height; + swath_height_c = rq_param->misc.rq_c.blk256_height; + } else { + swath_height_l = rq_param->misc.rq_l.blk256_width; + swath_height_c = rq_param->misc.rq_c.blk256_width; + } + + if (swath_height_l > 0) + log2_swath_height_l = dml_log2(swath_height_l); + + if (req128_l && log2_swath_height_l > 0) + log2_swath_height_l -= 1; + + if (swath_height_c > 0) + log2_swath_height_c = dml_log2(swath_height_c); + + if (req128_c && log2_swath_height_c > 0) + log2_swath_height_c -= 1; } + rq_param->dlg.rq_l.swath_height = 1 << log2_swath_height_l; rq_param->dlg.rq_c.swath_height = 1 << log2_swath_height_c; diff --git a/drivers/gpu/drm/amd/display/dc/hdcp/hdcp_msg.c b/drivers/gpu/drm/amd/display/dc/hdcp/hdcp_msg.c index 5e384a8a83dc..51855a2624cf 100644 --- a/drivers/gpu/drm/amd/display/dc/hdcp/hdcp_msg.c +++ b/drivers/gpu/drm/amd/display/dc/hdcp/hdcp_msg.c @@ -39,7 +39,7 @@ #define HDCP14_KSV_SIZE 5 #define HDCP14_MAX_KSV_FIFO_SIZE 127*HDCP14_KSV_SIZE -static const bool hdcp_cmd_is_read[] = { +static const bool hdcp_cmd_is_read[HDCP_MESSAGE_ID_MAX] = { [HDCP_MESSAGE_ID_READ_BKSV] = true, [HDCP_MESSAGE_ID_READ_RI_R0] = true, [HDCP_MESSAGE_ID_READ_PJ] = true, @@ -75,7 +75,7 @@ static const bool hdcp_cmd_is_read[] = { [HDCP_MESSAGE_ID_WRITE_CONTENT_STREAM_TYPE] = false }; -static const uint8_t hdcp_i2c_offsets[] = { +static const uint8_t hdcp_i2c_offsets[HDCP_MESSAGE_ID_MAX] = { [HDCP_MESSAGE_ID_READ_BKSV] = 0x0, [HDCP_MESSAGE_ID_READ_RI_R0] = 0x8, [HDCP_MESSAGE_ID_READ_PJ] = 0xA, @@ -106,7 +106,8 @@ static const uint8_t hdcp_i2c_offsets[] = { [HDCP_MESSAGE_ID_WRITE_REPEATER_AUTH_SEND_ACK] = 0x60, [HDCP_MESSAGE_ID_WRITE_REPEATER_AUTH_STREAM_MANAGE] = 0x60, [HDCP_MESSAGE_ID_READ_REPEATER_AUTH_STREAM_READY] = 0x80, - [HDCP_MESSAGE_ID_READ_RXSTATUS] = 0x70 + [HDCP_MESSAGE_ID_READ_RXSTATUS] = 0x70, + [HDCP_MESSAGE_ID_WRITE_CONTENT_STREAM_TYPE] = 0x0, }; struct protection_properties { @@ -184,7 +185,7 @@ static const struct protection_properties hdmi_14_protection = { .process_transaction = hdmi_14_process_transaction }; -static const uint32_t hdcp_dpcd_addrs[] = { +static const uint32_t hdcp_dpcd_addrs[HDCP_MESSAGE_ID_MAX] = { [HDCP_MESSAGE_ID_READ_BKSV] = 0x68000, [HDCP_MESSAGE_ID_READ_RI_R0] = 0x68005, [HDCP_MESSAGE_ID_READ_PJ] = 0xFFFFFFFF, diff --git a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_psp.c b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_psp.c index 904ce9b88088..afbe8856468a 100644 --- a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_psp.c +++ b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_psp.c @@ -791,6 +791,8 @@ enum mod_hdcp_status mod_hdcp_hdcp2_validate_rx_id_list(struct mod_hdcp *hdcp) TA_HDCP2_MSG_AUTHENTICATION_STATUS__RECEIVERID_REVOKED) { hdcp->connection.is_hdcp2_revoked = 1; status = MOD_HDCP_STATUS_HDCP2_RX_ID_LIST_REVOKED; + } else { + status = MOD_HDCP_STATUS_HDCP2_VALIDATE_RX_ID_LIST_FAILURE; } } mutex_unlock(&psp->hdcp_context.mutex); diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.c index ed05a30d1139..e2a56a7f3d7a 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.c @@ -1526,20 +1526,6 @@ static int smu10_set_fine_grain_clk_vol(struct pp_hwmgr *hwmgr, smu10_data->gfx_actual_soft_min_freq = min_freq; smu10_data->gfx_actual_soft_max_freq = max_freq; - - ret = smum_send_msg_to_smc_with_parameter(hwmgr, - PPSMC_MSG_SetHardMinGfxClk, - min_freq, - NULL); - if (ret) - return ret; - - ret = smum_send_msg_to_smc_with_parameter(hwmgr, - PPSMC_MSG_SetSoftMaxGfxClk, - max_freq, - NULL); - if (ret) - return ret; } else if (type == PP_OD_COMMIT_DPM_TABLE) { if (size != 0) { pr_err("Input parameter number not correct\n"); diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c index 599ec9726601..959143eff651 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c @@ -5160,7 +5160,7 @@ static int vega10_set_power_profile_mode(struct pp_hwmgr *hwmgr, long *input, ui out: smum_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_SetWorkloadMask, - 1 << power_profile_mode, + (!power_profile_mode) ? 0 : 1 << (power_profile_mode - 1), NULL); hwmgr->power_profile_mode = power_profile_mode; diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c index cd905e41080e..ec0037a21331 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c @@ -279,35 +279,25 @@ static void smu_set_user_clk_dependencies(struct smu_context *smu, enum smu_clk_ if (smu->adev->in_suspend) return; - /* - * mclk, fclk and socclk are interdependent - * on each other - */ if (clk == SMU_MCLK) { - /* reset clock dependency */ smu->user_dpm_profile.clk_dependency = 0; - /* set mclk dependent clocks(fclk and socclk) */ smu->user_dpm_profile.clk_dependency = BIT(SMU_FCLK) | BIT(SMU_SOCCLK); } else if (clk == SMU_FCLK) { - /* give priority to mclk, if mclk dependent clocks are set */ + /* MCLK takes precedence over FCLK */ if (smu->user_dpm_profile.clk_dependency == (BIT(SMU_FCLK) | BIT(SMU_SOCCLK))) return; - /* reset clock dependency */ smu->user_dpm_profile.clk_dependency = 0; - /* set fclk dependent clocks(mclk and socclk) */ smu->user_dpm_profile.clk_dependency = BIT(SMU_MCLK) | BIT(SMU_SOCCLK); } else if (clk == SMU_SOCCLK) { - /* give priority to mclk, if mclk dependent clocks are set */ + /* MCLK takes precedence over SOCCLK */ if (smu->user_dpm_profile.clk_dependency == (BIT(SMU_FCLK) | BIT(SMU_SOCCLK))) return; - /* reset clock dependency */ smu->user_dpm_profile.clk_dependency = 0; - /* set socclk dependent clocks(mclk and fclk) */ smu->user_dpm_profile.clk_dependency = BIT(SMU_MCLK) | BIT(SMU_FCLK); } else - /* add clk dependencies here, if any */ + /* Add clk dependencies here, if any */ return; } @@ -331,7 +321,7 @@ static void smu_restore_dpm_user_profile(struct smu_context *smu) return; /* Enable restore flag */ - smu->user_dpm_profile.flags = SMU_DPM_USER_PROFILE_RESTORE; + smu->user_dpm_profile.flags |= SMU_DPM_USER_PROFILE_RESTORE; /* set the user dpm power limit */ if (smu->user_dpm_profile.power_limit) { @@ -354,8 +344,8 @@ static void smu_restore_dpm_user_profile(struct smu_context *smu) ret = smu_force_clk_levels(smu, clk_type, smu->user_dpm_profile.clk_mask[clk_type]); if (ret) - dev_err(smu->adev->dev, "Failed to set clock type = %d\n", - clk_type); + dev_err(smu->adev->dev, + "Failed to set clock type = %d\n", clk_type); } } } @@ -1777,7 +1767,7 @@ int smu_force_clk_levels(struct smu_context *smu, if (smu->ppt_funcs && smu->ppt_funcs->force_clk_levels) { ret = smu->ppt_funcs->force_clk_levels(smu, clk_type, mask); - if (!ret && smu->user_dpm_profile.flags != SMU_DPM_USER_PROFILE_RESTORE) { + if (!ret && !(smu->user_dpm_profile.flags & SMU_DPM_USER_PROFILE_RESTORE)) { smu->user_dpm_profile.clk_mask[clk_type] = mask; smu_set_user_clk_dependencies(smu, clk_type); } @@ -2034,7 +2024,7 @@ int smu_set_fan_speed_rpm(struct smu_context *smu, uint32_t speed) if (smu->ppt_funcs->set_fan_speed_percent) { percent = speed * 100 / smu->fan_max_rpm; ret = smu->ppt_funcs->set_fan_speed_percent(smu, percent); - if (!ret && smu->user_dpm_profile.flags != SMU_DPM_USER_PROFILE_RESTORE) + if (!ret && !(smu->user_dpm_profile.flags & SMU_DPM_USER_PROFILE_RESTORE)) smu->user_dpm_profile.fan_speed_percent = percent; } @@ -2096,6 +2086,7 @@ int smu_set_power_limit(struct smu_context *smu, uint32_t limit) dev_err(smu->adev->dev, "New power limit (%d) is over the max allowed %d\n", limit, smu->max_power_limit); + ret = -EINVAL; goto out; } @@ -2104,7 +2095,7 @@ int smu_set_power_limit(struct smu_context *smu, uint32_t limit) if (smu->ppt_funcs->set_power_limit) { ret = smu->ppt_funcs->set_power_limit(smu, limit); - if (!ret && smu->user_dpm_profile.flags != SMU_DPM_USER_PROFILE_RESTORE) + if (!ret && !(smu->user_dpm_profile.flags & SMU_DPM_USER_PROFILE_RESTORE)) smu->user_dpm_profile.power_limit = limit; } @@ -2285,7 +2276,7 @@ int smu_set_fan_control_mode(struct smu_context *smu, int value) if (smu->ppt_funcs->set_fan_control_mode) { ret = smu->ppt_funcs->set_fan_control_mode(smu, value); - if (!ret && smu->user_dpm_profile.flags != SMU_DPM_USER_PROFILE_RESTORE) + if (!ret && !(smu->user_dpm_profile.flags & SMU_DPM_USER_PROFILE_RESTORE)) smu->user_dpm_profile.fan_mode = value; } @@ -2293,7 +2284,7 @@ int smu_set_fan_control_mode(struct smu_context *smu, int value) /* reset user dpm fan speed */ if (!ret && value != AMD_FAN_CTRL_MANUAL && - smu->user_dpm_profile.flags != SMU_DPM_USER_PROFILE_RESTORE) + !(smu->user_dpm_profile.flags & SMU_DPM_USER_PROFILE_RESTORE)) smu->user_dpm_profile.fan_speed_percent = 0; return ret; @@ -2335,7 +2326,7 @@ int smu_set_fan_speed_percent(struct smu_context *smu, uint32_t speed) if (speed > 100) speed = 100; ret = smu->ppt_funcs->set_fan_speed_percent(smu, speed); - if (!ret && smu->user_dpm_profile.flags != SMU_DPM_USER_PROFILE_RESTORE) + if (!ret && !(smu->user_dpm_profile.flags & SMU_DPM_USER_PROFILE_RESTORE)) smu->user_dpm_profile.fan_speed_percent = speed; } diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c index 6e641f1513d8..fbff3df72e6c 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c @@ -1110,7 +1110,6 @@ static int navi10_force_clk_levels(struct smu_context *smu, case SMU_SOCCLK: case SMU_MCLK: case SMU_UCLK: - case SMU_DCEFCLK: case SMU_FCLK: /* There is only 2 levels for fine grained DPM */ if (navi10_is_support_fine_grained_dpm(smu, clk_type)) { @@ -1130,6 +1129,10 @@ static int navi10_force_clk_levels(struct smu_context *smu, if (ret) return size; break; + case SMU_DCEFCLK: + dev_info(smu->adev->dev,"Setting DCEFCLK min/max dpm level is not supported!\n"); + break; + default: break; } diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c index af73e1430af5..61438940c26e 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c @@ -1127,7 +1127,6 @@ static int sienna_cichlid_force_clk_levels(struct smu_context *smu, case SMU_SOCCLK: case SMU_MCLK: case SMU_UCLK: - case SMU_DCEFCLK: case SMU_FCLK: /* There is only 2 levels for fine grained DPM */ if (sienna_cichlid_is_support_fine_grained_dpm(smu, clk_type)) { @@ -1147,6 +1146,9 @@ static int sienna_cichlid_force_clk_levels(struct smu_context *smu, if (ret) goto forec_level_out; break; + case SMU_DCEFCLK: + dev_info(smu->adev->dev,"Setting DCEFCLK min/max dpm level is not supported!\n"); + break; default: break; } diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c index 101eaa20db9b..a80f551771b9 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c @@ -1462,7 +1462,6 @@ static int vangogh_od_edit_dpm_table(struct smu_context *smu, enum PP_OD_DPM_TAB long input[], uint32_t size) { int ret = 0; - int i; struct smu_dpm_context *smu_dpm_ctx = &(smu->smu_dpm); if (!(smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL)) { @@ -1535,43 +1534,6 @@ static int vangogh_od_edit_dpm_table(struct smu_context *smu, enum PP_OD_DPM_TAB smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; smu->cpu_actual_soft_min_freq = smu->cpu_default_soft_min_freq; smu->cpu_actual_soft_max_freq = smu->cpu_default_soft_max_freq; - - ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetHardMinGfxClk, - smu->gfx_actual_hard_min_freq, NULL); - if (ret) { - dev_err(smu->adev->dev, "Restore the default hard min sclk failed!"); - return ret; - } - - ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetSoftMaxGfxClk, - smu->gfx_actual_soft_max_freq, NULL); - if (ret) { - dev_err(smu->adev->dev, "Restore the default soft max sclk failed!"); - return ret; - } - - if (smu->adev->pm.fw_version < 0x43f1b00) { - dev_warn(smu->adev->dev, "CPUSoftMax/CPUSoftMin are not supported, please update SBIOS!\n"); - break; - } - - for (i = 0; i < smu->cpu_core_num; i++) { - ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetSoftMinCclk, - (i << 20) | smu->cpu_actual_soft_min_freq, - NULL); - if (ret) { - dev_err(smu->adev->dev, "Set hard min cclk failed!"); - return ret; - } - - ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetSoftMaxCclk, - (i << 20) | smu->cpu_actual_soft_max_freq, - NULL); - if (ret) { - dev_err(smu->adev->dev, "Set soft max cclk failed!"); - return ret; - } - } } break; case PP_OD_COMMIT_DPM_TABLE: diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c index 5493388fcb10..dbe6d0caddb7 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c @@ -389,24 +389,6 @@ static int renoir_od_edit_dpm_table(struct smu_context *smu, } smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; - - ret = smu_cmn_send_smc_msg_with_param(smu, - SMU_MSG_SetHardMinGfxClk, - smu->gfx_actual_hard_min_freq, - NULL); - if (ret) { - dev_err(smu->adev->dev, "Restore the default hard min sclk failed!"); - return ret; - } - - ret = smu_cmn_send_smc_msg_with_param(smu, - SMU_MSG_SetSoftMaxGfxClk, - smu->gfx_actual_soft_max_freq, - NULL); - if (ret) { - dev_err(smu->adev->dev, "Restore the default soft max sclk failed!"); - return ret; - } break; case PP_OD_COMMIT_DPM_TABLE: if (size != 0) { diff --git a/drivers/gpu/drm/arm/display/include/malidp_utils.h b/drivers/gpu/drm/arm/display/include/malidp_utils.h index 3bc383d5bf73..49a1d7f3539c 100644 --- a/drivers/gpu/drm/arm/display/include/malidp_utils.h +++ b/drivers/gpu/drm/arm/display/include/malidp_utils.h @@ -13,9 +13,6 @@ #define has_bit(nr, mask) (BIT(nr) & (mask)) #define has_bits(bits, mask) (((bits) & (mask)) == (bits)) -#define dp_for_each_set_bit(bit, mask) \ - for_each_set_bit((bit), ((unsigned long *)&(mask)), sizeof(mask) * 8) - #define dp_wait_cond(__cond, __tries, __min_range, __max_range) \ ({ \ int num_tries = __tries; \ diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_pipeline.c b/drivers/gpu/drm/arm/display/komeda/komeda_pipeline.c index 719a79728e24..06c595378dda 100644 --- a/drivers/gpu/drm/arm/display/komeda/komeda_pipeline.c +++ b/drivers/gpu/drm/arm/display/komeda/komeda_pipeline.c @@ -46,8 +46,9 @@ void komeda_pipeline_destroy(struct komeda_dev *mdev, { struct komeda_component *c; int i; + unsigned long avail_comps = pipe->avail_comps; - dp_for_each_set_bit(i, pipe->avail_comps) { + for_each_set_bit(i, &avail_comps, 32) { c = komeda_pipeline_get_component(pipe, i); komeda_component_destroy(mdev, c); } @@ -247,6 +248,7 @@ static void komeda_pipeline_dump(struct komeda_pipeline *pipe) { struct komeda_component *c; int id; + unsigned long avail_comps = pipe->avail_comps; DRM_INFO("Pipeline-%d: n_layers: %d, n_scalers: %d, output: %s.\n", pipe->id, pipe->n_layers, pipe->n_scalers, @@ -258,7 +260,7 @@ static void komeda_pipeline_dump(struct komeda_pipeline *pipe) pipe->of_output_links[1] ? pipe->of_output_links[1]->full_name : "none"); - dp_for_each_set_bit(id, pipe->avail_comps) { + for_each_set_bit(id, &avail_comps, 32) { c = komeda_pipeline_get_component(pipe, id); komeda_component_dump(c); @@ -270,8 +272,9 @@ static void komeda_component_verify_inputs(struct komeda_component *c) struct komeda_pipeline *pipe = c->pipeline; struct komeda_component *input; int id; + unsigned long supported_inputs = c->supported_inputs; - dp_for_each_set_bit(id, c->supported_inputs) { + for_each_set_bit(id, &supported_inputs, 32) { input = komeda_pipeline_get_component(pipe, id); if (!input) { c->supported_inputs &= ~(BIT(id)); @@ -302,8 +305,9 @@ static void komeda_pipeline_assemble(struct komeda_pipeline *pipe) struct komeda_component *c; struct komeda_layer *layer; int i, id; + unsigned long avail_comps = pipe->avail_comps; - dp_for_each_set_bit(id, pipe->avail_comps) { + for_each_set_bit(id, &avail_comps, 32) { c = komeda_pipeline_get_component(pipe, id); komeda_component_verify_inputs(c); } @@ -355,13 +359,15 @@ void komeda_pipeline_dump_register(struct komeda_pipeline *pipe, { struct komeda_component *c; u32 id; + unsigned long avail_comps; seq_printf(sf, "\n======== Pipeline-%d ==========\n", pipe->id); if (pipe->funcs && pipe->funcs->dump_register) pipe->funcs->dump_register(pipe, sf); - dp_for_each_set_bit(id, pipe->avail_comps) { + avail_comps = pipe->avail_comps; + for_each_set_bit(id, &avail_comps, 32) { c = komeda_pipeline_get_component(pipe, id); seq_printf(sf, "\n------%s------\n", c->name); diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_pipeline_state.c b/drivers/gpu/drm/arm/display/komeda/komeda_pipeline_state.c index 5c085116de3f..e672b9cffee3 100644 --- a/drivers/gpu/drm/arm/display/komeda/komeda_pipeline_state.c +++ b/drivers/gpu/drm/arm/display/komeda/komeda_pipeline_state.c @@ -1231,14 +1231,15 @@ komeda_pipeline_unbound_components(struct komeda_pipeline *pipe, struct komeda_pipeline_state *old = priv_to_pipe_st(pipe->obj.state); struct komeda_component_state *c_st; struct komeda_component *c; - u32 disabling_comps, id; + u32 id; + unsigned long disabling_comps; WARN_ON(!old); disabling_comps = (~new->active_comps) & old->active_comps; /* unbound all disabling component */ - dp_for_each_set_bit(id, disabling_comps) { + for_each_set_bit(id, &disabling_comps, 32) { c = komeda_pipeline_get_component(pipe, id); c_st = komeda_component_get_state_and_set_user(c, drm_st, NULL, new->crtc); @@ -1286,7 +1287,8 @@ bool komeda_pipeline_disable(struct komeda_pipeline *pipe, struct komeda_pipeline_state *old; struct komeda_component *c; struct komeda_component_state *c_st; - u32 id, disabling_comps = 0; + u32 id; + unsigned long disabling_comps; old = komeda_pipeline_get_old_state(pipe, old_state); @@ -1296,10 +1298,10 @@ bool komeda_pipeline_disable(struct komeda_pipeline *pipe, disabling_comps = old->active_comps & pipe->standalone_disabled_comps; - DRM_DEBUG_ATOMIC("PIPE%d: active_comps: 0x%x, disabling_comps: 0x%x.\n", + DRM_DEBUG_ATOMIC("PIPE%d: active_comps: 0x%x, disabling_comps: 0x%lx.\n", pipe->id, old->active_comps, disabling_comps); - dp_for_each_set_bit(id, disabling_comps) { + for_each_set_bit(id, &disabling_comps, 32) { c = komeda_pipeline_get_component(pipe, id); c_st = priv_to_comp_st(c->obj.state); @@ -1330,16 +1332,17 @@ void komeda_pipeline_update(struct komeda_pipeline *pipe, struct komeda_pipeline_state *new = priv_to_pipe_st(pipe->obj.state); struct komeda_pipeline_state *old; struct komeda_component *c; - u32 id, changed_comps = 0; + u32 id; + unsigned long changed_comps; old = komeda_pipeline_get_old_state(pipe, old_state); changed_comps = new->active_comps | old->active_comps; - DRM_DEBUG_ATOMIC("PIPE%d: active_comps: 0x%x, changed: 0x%x.\n", + DRM_DEBUG_ATOMIC("PIPE%d: active_comps: 0x%x, changed: 0x%lx.\n", pipe->id, new->active_comps, changed_comps); - dp_for_each_set_bit(id, changed_comps) { + for_each_set_bit(id, &changed_comps, 32) { c = komeda_pipeline_get_component(pipe, id); if (new->active_comps & BIT(c->id)) diff --git a/drivers/gpu/drm/ast/ast_drv.c b/drivers/gpu/drm/ast/ast_drv.c index ea8164e7a6dc..01837bea18c2 100644 --- a/drivers/gpu/drm/ast/ast_drv.c +++ b/drivers/gpu/drm/ast/ast_drv.c @@ -30,6 +30,7 @@ #include #include +#include #include #include #include @@ -138,6 +139,7 @@ static void ast_pci_remove(struct pci_dev *pdev) struct drm_device *dev = pci_get_drvdata(pdev); drm_dev_unregister(dev); + drm_atomic_helper_shutdown(dev); } static int ast_drm_freeze(struct drm_device *dev) diff --git a/drivers/gpu/drm/ast/ast_mode.c b/drivers/gpu/drm/ast/ast_mode.c index 988b270fea5e..758c69aa7232 100644 --- a/drivers/gpu/drm/ast/ast_mode.c +++ b/drivers/gpu/drm/ast/ast_mode.c @@ -688,7 +688,7 @@ ast_cursor_plane_helper_atomic_update(struct drm_plane *plane, unsigned int offset_x, offset_y; offset_x = AST_MAX_HWC_WIDTH - fb->width; - offset_y = AST_MAX_HWC_WIDTH - fb->height; + offset_y = AST_MAX_HWC_HEIGHT - fb->height; if (state->fb != old_state->fb) { /* A new cursor image was installed. */ diff --git a/drivers/gpu/drm/bridge/Kconfig b/drivers/gpu/drm/bridge/Kconfig index e4110d6ca7b3..bc60fc4728d7 100644 --- a/drivers/gpu/drm/bridge/Kconfig +++ b/drivers/gpu/drm/bridge/Kconfig @@ -67,6 +67,7 @@ config DRM_LONTIUM_LT9611UXC depends on OF select DRM_PANEL_BRIDGE select DRM_KMS_HELPER + select DRM_MIPI_DSI select REGMAP_I2C help Driver for Lontium LT9611UXC DSI to HDMI bridge @@ -151,6 +152,7 @@ config DRM_SII902X tristate "Silicon Image sii902x RGB/HDMI bridge" depends on OF select DRM_KMS_HELPER + select DRM_MIPI_DSI select REGMAP_I2C select I2C_MUX select SND_SOC_HDMI_CODEC if SND_SOC @@ -200,6 +202,7 @@ config DRM_TOSHIBA_TC358767 tristate "Toshiba TC358767 eDP bridge" depends on OF select DRM_KMS_HELPER + select DRM_MIPI_DSI select REGMAP_I2C select DRM_PANEL help diff --git a/drivers/gpu/drm/bridge/analogix/Kconfig b/drivers/gpu/drm/bridge/analogix/Kconfig index 024ea2a570e7..9160fd80dd70 100644 --- a/drivers/gpu/drm/bridge/analogix/Kconfig +++ b/drivers/gpu/drm/bridge/analogix/Kconfig @@ -30,6 +30,7 @@ config DRM_ANALOGIX_ANX7625 tristate "Analogix Anx7625 MIPI to DP interface support" depends on DRM depends on OF + select DRM_MIPI_DSI help ANX7625 is an ultra-low power 4K mobile HD transmitter designed for portable devices. It converts MIPI/DPI to diff --git a/drivers/gpu/drm/bridge/panel.c b/drivers/gpu/drm/bridge/panel.c index 0ddc37551194..c916f4b8907e 100644 --- a/drivers/gpu/drm/bridge/panel.c +++ b/drivers/gpu/drm/bridge/panel.c @@ -87,6 +87,18 @@ static int panel_bridge_attach(struct drm_bridge *bridge, static void panel_bridge_detach(struct drm_bridge *bridge) { + struct panel_bridge *panel_bridge = drm_bridge_to_panel_bridge(bridge); + struct drm_connector *connector = &panel_bridge->connector; + + /* + * Cleanup the connector if we know it was initialized. + * + * FIXME: This wouldn't be needed if the panel_bridge structure was + * allocated with drmm_kzalloc(). This might be tricky since the + * drm_device pointer can only be retrieved when the bridge is attached. + */ + if (connector->dev) + drm_connector_cleanup(connector); } static void panel_bridge_pre_enable(struct drm_bridge *bridge) diff --git a/drivers/gpu/drm/drm_dp_mst_topology.c b/drivers/gpu/drm/drm_dp_mst_topology.c index 309afe61afdd..9c75c8815056 100644 --- a/drivers/gpu/drm/drm_dp_mst_topology.c +++ b/drivers/gpu/drm/drm_dp_mst_topology.c @@ -1154,6 +1154,7 @@ static void build_clear_payload_id_table(struct drm_dp_sideband_msg_tx *msg) req.req_type = DP_CLEAR_PAYLOAD_ID_TABLE; drm_dp_encode_sideband_req(&req, msg); + msg->path_msg = true; } static int build_enum_path_resources(struct drm_dp_sideband_msg_tx *msg, @@ -2824,15 +2825,21 @@ static int set_hdr_from_dst_qlock(struct drm_dp_sideband_msg_hdr *hdr, req_type = txmsg->msg[0] & 0x7f; if (req_type == DP_CONNECTION_STATUS_NOTIFY || - req_type == DP_RESOURCE_STATUS_NOTIFY) + req_type == DP_RESOURCE_STATUS_NOTIFY || + req_type == DP_CLEAR_PAYLOAD_ID_TABLE) hdr->broadcast = 1; else hdr->broadcast = 0; hdr->path_msg = txmsg->path_msg; - hdr->lct = mstb->lct; - hdr->lcr = mstb->lct - 1; - if (mstb->lct > 1) - memcpy(hdr->rad, mstb->rad, mstb->lct / 2); + if (hdr->broadcast) { + hdr->lct = 1; + hdr->lcr = 6; + } else { + hdr->lct = mstb->lct; + hdr->lcr = mstb->lct - 1; + } + + memcpy(hdr->rad, mstb->rad, hdr->lct / 2); return 0; } diff --git a/drivers/gpu/drm/drm_panel_orientation_quirks.c b/drivers/gpu/drm/drm_panel_orientation_quirks.c index 58f5dc2f6dd5..f6bdec7fa925 100644 --- a/drivers/gpu/drm/drm_panel_orientation_quirks.c +++ b/drivers/gpu/drm/drm_panel_orientation_quirks.c @@ -84,6 +84,13 @@ static const struct drm_dmi_panel_orientation_data itworks_tw891 = { .orientation = DRM_MODE_PANEL_ORIENTATION_RIGHT_UP, }; +static const struct drm_dmi_panel_orientation_data onegx1_pro = { + .width = 1200, + .height = 1920, + .bios_dates = (const char * const []){ "12/17/2020", NULL }, + .orientation = DRM_MODE_PANEL_ORIENTATION_RIGHT_UP, +}; + static const struct drm_dmi_panel_orientation_data lcd720x1280_rightside_up = { .width = 720, .height = 1280, @@ -211,6 +218,13 @@ static const struct dmi_system_id orientation_data[] = { DMI_EXACT_MATCH(DMI_PRODUCT_VERSION, "Lenovo ideapad D330-10IGM"), }, .driver_data = (void *)&lcd1200x1920_rightside_up, + }, { /* OneGX1 Pro */ + .matches = { + DMI_EXACT_MATCH(DMI_SYS_VENDOR, "SYSTEM_MANUFACTURER"), + DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "SYSTEM_PRODUCT_NAME"), + DMI_EXACT_MATCH(DMI_PRODUCT_VERSION, "Default string"), + }, + .driver_data = (void *)&onegx1_pro, }, { /* VIOS LTH17 */ .matches = { DMI_EXACT_MATCH(DMI_SYS_VENDOR, "VIOS"), diff --git a/drivers/gpu/drm/drm_probe_helper.c b/drivers/gpu/drm/drm_probe_helper.c index ad59a51eab6d..e7e1ee2aa352 100644 --- a/drivers/gpu/drm/drm_probe_helper.c +++ b/drivers/gpu/drm/drm_probe_helper.c @@ -624,6 +624,7 @@ static void output_poll_execute(struct work_struct *work) struct drm_connector_list_iter conn_iter; enum drm_connector_status old_status; bool repoll = false, changed; + u64 old_epoch_counter; if (!dev->mode_config.poll_enabled) return; @@ -660,8 +661,9 @@ static void output_poll_execute(struct work_struct *work) repoll = true; + old_epoch_counter = connector->epoch_counter; connector->status = drm_helper_probe_detect(connector, NULL, false); - if (old_status != connector->status) { + if (old_epoch_counter != connector->epoch_counter) { const char *old, *new; /* @@ -690,6 +692,9 @@ static void output_poll_execute(struct work_struct *work) connector->base.id, connector->name, old, new); + DRM_DEBUG_KMS("[CONNECTOR:%d:%s] epoch counter %llu -> %llu\n", + connector->base.id, connector->name, + old_epoch_counter, connector->epoch_counter); changed = true; } diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c index 775d89b6c3fc..97a785aa8839 100644 --- a/drivers/gpu/drm/i915/display/intel_dp.c +++ b/drivers/gpu/drm/i915/display/intel_dp.c @@ -1174,44 +1174,6 @@ intel_dp_compute_link_config_wide(struct intel_dp *intel_dp, return -EINVAL; } -/* Optimize link config in order: max bpp, min lanes, min clock */ -static int -intel_dp_compute_link_config_fast(struct intel_dp *intel_dp, - struct intel_crtc_state *pipe_config, - const struct link_config_limits *limits) -{ - const struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode; - int bpp, clock, lane_count; - int mode_rate, link_clock, link_avail; - - for (bpp = limits->max_bpp; bpp >= limits->min_bpp; bpp -= 2 * 3) { - int output_bpp = intel_dp_output_bpp(pipe_config->output_format, bpp); - - mode_rate = intel_dp_link_required(adjusted_mode->crtc_clock, - output_bpp); - - for (lane_count = limits->min_lane_count; - lane_count <= limits->max_lane_count; - lane_count <<= 1) { - for (clock = limits->min_clock; clock <= limits->max_clock; clock++) { - link_clock = intel_dp->common_rates[clock]; - link_avail = intel_dp_max_data_rate(link_clock, - lane_count); - - if (mode_rate <= link_avail) { - pipe_config->lane_count = lane_count; - pipe_config->pipe_bpp = bpp; - pipe_config->port_clock = link_clock; - - return 0; - } - } - } - } - - return -EINVAL; -} - static int intel_dp_dsc_compute_bpp(struct intel_dp *intel_dp, u8 dsc_max_bpc) { int i, num_bpc; @@ -1461,22 +1423,11 @@ intel_dp_compute_link_config(struct intel_encoder *encoder, intel_dp_can_bigjoiner(intel_dp)) pipe_config->bigjoiner = true; - if (intel_dp_is_edp(intel_dp)) - /* - * Optimize for fast and narrow. eDP 1.3 section 3.3 and eDP 1.4 - * section A.1: "It is recommended that the minimum number of - * lanes be used, using the minimum link rate allowed for that - * lane configuration." - * - * Note that we fall back to the max clock and lane count for eDP - * panels that fail with the fast optimal settings (see - * intel_dp->use_max_params), in which case the fast vs. wide - * choice doesn't matter. - */ - ret = intel_dp_compute_link_config_fast(intel_dp, pipe_config, &limits); - else - /* Optimize for slow and wide. */ - ret = intel_dp_compute_link_config_wide(intel_dp, pipe_config, &limits); + /* + * Optimize for slow and wide for everything, because there are some + * eDP 1.3 and 1.4 panels don't work well with fast and narrow. + */ + ret = intel_dp_compute_link_config_wide(intel_dp, pipe_config, &limits); /* enable compression if the mode doesn't fit available BW */ drm_dbg_kms(&i915->drm, "Force DSC en = %d\n", intel_dp->force_dsc_en); @@ -4537,7 +4488,18 @@ intel_dp_check_mst_status(struct intel_dp *intel_dp) drm_WARN_ON_ONCE(&i915->drm, intel_dp->active_mst_links < 0); for (;;) { - u8 esi[DP_DPRX_ESI_LEN] = {}; + /* + * The +2 is because DP_DPRX_ESI_LEN is 14, but we then + * pass in "esi+10" to drm_dp_channel_eq_ok(), which + * takes a 6-byte array. So we actually need 16 bytes + * here. + * + * Somebody who knows what the limits actually are + * should check this, but for now this is at least + * harmless and avoids a valid compiler warning about + * using more of the array than we have allocated. + */ + u8 esi[DP_DPRX_ESI_LEN+2] = {}; bool handled; int retry; diff --git a/drivers/gpu/drm/i915/display/intel_overlay.c b/drivers/gpu/drm/i915/display/intel_overlay.c index f455040fa989..7cbc81da80b7 100644 --- a/drivers/gpu/drm/i915/display/intel_overlay.c +++ b/drivers/gpu/drm/i915/display/intel_overlay.c @@ -383,7 +383,7 @@ static void intel_overlay_off_tail(struct intel_overlay *overlay) i830_overlay_clock_gating(dev_priv, true); } -static void +__i915_active_call static void intel_overlay_last_flip_retire(struct i915_active *active) { struct intel_overlay *overlay = diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/i915_gem_mman.c index ec28a6cde49b..0b2434e29d00 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.c @@ -189,7 +189,7 @@ compute_partial_view(const struct drm_i915_gem_object *obj, struct i915_ggtt_view view; if (i915_gem_object_is_tiled(obj)) - chunk = roundup(chunk, tile_row_pages(obj)); + chunk = roundup(chunk, tile_row_pages(obj) ?: 1); view.type = I915_GGTT_VIEW_PARTIAL; view.partial.offset = rounddown(page_offset, chunk); diff --git a/drivers/gpu/drm/i915/gt/gen7_renderclear.c b/drivers/gpu/drm/i915/gt/gen7_renderclear.c index de575fdb033f..21f08e53889c 100644 --- a/drivers/gpu/drm/i915/gt/gen7_renderclear.c +++ b/drivers/gpu/drm/i915/gt/gen7_renderclear.c @@ -397,7 +397,10 @@ static void emit_batch(struct i915_vma * const vma, gen7_emit_pipeline_invalidate(&cmds); batch_add(&cmds, MI_LOAD_REGISTER_IMM(2)); batch_add(&cmds, i915_mmio_reg_offset(CACHE_MODE_0_GEN7)); - batch_add(&cmds, 0xffff0000); + batch_add(&cmds, 0xffff0000 | + ((IS_IVB_GT1(i915) || IS_VALLEYVIEW(i915)) ? + HIZ_RAW_STALL_OPT_DISABLE : + 0)); batch_add(&cmds, i915_mmio_reg_offset(CACHE_MODE_1)); batch_add(&cmds, 0xffff0000 | PIXEL_SUBSPAN_COLLECT_OPT_DISABLE); gen7_emit_pipeline_invalidate(&cmds); diff --git a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c index 755522ced60d..3ae16945bd43 100644 --- a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c @@ -630,7 +630,6 @@ static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt) err = pin_pt_dma(vm, pde->pt.base); if (err) { - i915_gem_object_put(pde->pt.base); free_pd(vm, pde); return err; } diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c b/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c index 67de2b189598..4b09490c20c0 100644 --- a/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c +++ b/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c @@ -670,8 +670,8 @@ static void detect_bit_6_swizzle(struct i915_ggtt *ggtt) * banks of memory are paired and unswizzled on the * uneven portion, so leave that as unknown. */ - if (intel_uncore_read(uncore, C0DRB3) == - intel_uncore_read(uncore, C1DRB3)) { + if (intel_uncore_read16(uncore, C0DRB3) == + intel_uncore_read16(uncore, C1DRB3)) { swizzle_x = I915_BIT_6_SWIZZLE_9_10; swizzle_y = I915_BIT_6_SWIZZLE_9; } diff --git a/drivers/gpu/drm/i915/gvt/gvt.c b/drivers/gpu/drm/i915/gvt/gvt.c index d1d8ee4a5f16..57578bf28d77 100644 --- a/drivers/gpu/drm/i915/gvt/gvt.c +++ b/drivers/gpu/drm/i915/gvt/gvt.c @@ -126,7 +126,7 @@ static bool intel_get_gvt_attrs(struct attribute_group ***intel_vgpu_type_groups return true; } -static bool intel_gvt_init_vgpu_type_groups(struct intel_gvt *gvt) +static int intel_gvt_init_vgpu_type_groups(struct intel_gvt *gvt) { int i, j; struct intel_vgpu_type *type; @@ -144,7 +144,7 @@ static bool intel_gvt_init_vgpu_type_groups(struct intel_gvt *gvt) gvt_vgpu_type_groups[i] = group; } - return true; + return 0; unwind: for (j = 0; j < i; j++) { @@ -152,7 +152,7 @@ static bool intel_gvt_init_vgpu_type_groups(struct intel_gvt *gvt) kfree(group); } - return false; + return -ENOMEM; } static void intel_gvt_cleanup_vgpu_type_groups(struct intel_gvt *gvt) @@ -360,7 +360,7 @@ int intel_gvt_init_device(struct drm_i915_private *i915) goto out_clean_thread; ret = intel_gvt_init_vgpu_type_groups(gvt); - if (ret == false) { + if (ret) { gvt_err("failed to init vgpu type groups: %d\n", ret); goto out_clean_types; } diff --git a/drivers/gpu/drm/i915/i915_active.c b/drivers/gpu/drm/i915/i915_active.c index 3bc616cc1ad2..ea660e541c90 100644 --- a/drivers/gpu/drm/i915/i915_active.c +++ b/drivers/gpu/drm/i915/i915_active.c @@ -1156,7 +1156,8 @@ static int auto_active(struct i915_active *ref) return 0; } -static void auto_retire(struct i915_active *ref) +__i915_active_call static void +auto_retire(struct i915_active *ref) { i915_active_put(ref); } diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index 8e9cb44e66e5..4ecb813c9bc7 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -1049,6 +1049,8 @@ static void intel_shutdown_encoders(struct drm_i915_private *dev_priv) void i915_driver_shutdown(struct drm_i915_private *i915) { disable_rpm_wakeref_asserts(&i915->runtime_pm); + intel_runtime_pm_disable(&i915->runtime_pm); + intel_power_domains_disable(i915); i915_gem_suspend(i915); @@ -1064,7 +1066,15 @@ void i915_driver_shutdown(struct drm_i915_private *i915) intel_suspend_encoders(i915); intel_shutdown_encoders(i915); + /* + * The only requirement is to reboot with display DC states disabled, + * for now leaving all display power wells in the INIT power domain + * enabled matching the driver reload sequence. + */ + intel_power_domains_driver_remove(i915); enable_rpm_wakeref_asserts(&i915->runtime_pm); + + intel_runtime_pm_driver_release(&i915->runtime_pm); } static bool suspend_to_idle(struct drm_i915_private *dev_priv) diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index 4b4d8d034782..4ba20f959a71 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -2993,7 +2993,7 @@ int ilk_wm_max_level(const struct drm_i915_private *dev_priv) static void intel_print_wm_latency(struct drm_i915_private *dev_priv, const char *name, - const u16 wm[8]) + const u16 wm[]) { int level, max_level = ilk_wm_max_level(dev_priv); diff --git a/drivers/gpu/drm/ingenic/ingenic-drm-drv.c b/drivers/gpu/drm/ingenic/ingenic-drm-drv.c index 7bb31fbee29d..fd8870edde0e 100644 --- a/drivers/gpu/drm/ingenic/ingenic-drm-drv.c +++ b/drivers/gpu/drm/ingenic/ingenic-drm-drv.c @@ -554,7 +554,7 @@ static void ingenic_drm_plane_atomic_update(struct drm_plane *plane, height = state->src_h >> 16; cpp = state->fb->format->cpp[0]; - if (priv->soc_info->has_osd && plane->type == DRM_PLANE_TYPE_OVERLAY) + if (!priv->soc_info->has_osd || plane->type == DRM_PLANE_TYPE_OVERLAY) hwdesc = &priv->dma_hwdescs->hwdesc_f0; else hwdesc = &priv->dma_hwdescs->hwdesc_f1; @@ -826,6 +826,7 @@ static int ingenic_drm_bind(struct device *dev, bool has_components) const struct jz_soc_info *soc_info; struct ingenic_drm *priv; struct clk *parent_clk; + struct drm_plane *primary; struct drm_bridge *bridge; struct drm_panel *panel; struct drm_encoder *encoder; @@ -940,9 +941,11 @@ static int ingenic_drm_bind(struct device *dev, bool has_components) if (soc_info->has_osd) priv->ipu_plane = drm_plane_from_index(drm, 0); - drm_plane_helper_add(&priv->f1, &ingenic_drm_plane_helper_funcs); + primary = priv->soc_info->has_osd ? &priv->f1 : &priv->f0; - ret = drm_universal_plane_init(drm, &priv->f1, 1, + drm_plane_helper_add(primary, &ingenic_drm_plane_helper_funcs); + + ret = drm_universal_plane_init(drm, primary, 1, &ingenic_drm_primary_plane_funcs, priv->soc_info->formats_f1, priv->soc_info->num_formats_f1, @@ -954,7 +957,7 @@ static int ingenic_drm_bind(struct device *dev, bool has_components) drm_crtc_helper_add(&priv->crtc, &ingenic_drm_crtc_helper_funcs); - ret = drm_crtc_init_with_planes(drm, &priv->crtc, &priv->f1, + ret = drm_crtc_init_with_planes(drm, &priv->crtc, primary, NULL, &ingenic_drm_crtc_funcs, NULL); if (ret) { dev_err(dev, "Failed to init CRTC: %i\n", ret); diff --git a/drivers/gpu/drm/mcde/mcde_dsi.c b/drivers/gpu/drm/mcde/mcde_dsi.c index 2314c8122992..b3fd3501c412 100644 --- a/drivers/gpu/drm/mcde/mcde_dsi.c +++ b/drivers/gpu/drm/mcde/mcde_dsi.c @@ -760,7 +760,7 @@ static void mcde_dsi_start(struct mcde_dsi *d) DSI_MCTL_MAIN_DATA_CTL_BTA_EN | DSI_MCTL_MAIN_DATA_CTL_READ_EN | DSI_MCTL_MAIN_DATA_CTL_REG_TE_EN; - if (d->mdsi->mode_flags & MIPI_DSI_MODE_EOT_PACKET) + if (!(d->mdsi->mode_flags & MIPI_DSI_MODE_EOT_PACKET)) val |= DSI_MCTL_MAIN_DATA_CTL_HOST_EOT_GEN; writel(val, d->regs + DSI_MCTL_MAIN_DATA_CTL); diff --git a/drivers/gpu/drm/mediatek/mtk_hdmi.c b/drivers/gpu/drm/mediatek/mtk_hdmi.c index 8ee55f9e2954..7fb358167f8d 100644 --- a/drivers/gpu/drm/mediatek/mtk_hdmi.c +++ b/drivers/gpu/drm/mediatek/mtk_hdmi.c @@ -153,7 +153,7 @@ struct mtk_hdmi_conf { struct mtk_hdmi { struct drm_bridge bridge; struct drm_bridge *next_bridge; - struct drm_connector conn; + struct drm_connector *curr_conn;/* current connector (only valid when 'enabled') */ struct device *dev; const struct mtk_hdmi_conf *conf; struct phy *phy; @@ -186,11 +186,6 @@ static inline struct mtk_hdmi *hdmi_ctx_from_bridge(struct drm_bridge *b) return container_of(b, struct mtk_hdmi, bridge); } -static inline struct mtk_hdmi *hdmi_ctx_from_conn(struct drm_connector *c) -{ - return container_of(c, struct mtk_hdmi, conn); -} - static u32 mtk_hdmi_read(struct mtk_hdmi *hdmi, u32 offset) { return readl(hdmi->regs + offset); @@ -974,7 +969,7 @@ static int mtk_hdmi_setup_avi_infoframe(struct mtk_hdmi *hdmi, ssize_t err; err = drm_hdmi_avi_infoframe_from_display_mode(&frame, - &hdmi->conn, mode); + hdmi->curr_conn, mode); if (err < 0) { dev_err(hdmi->dev, "Failed to get AVI infoframe from mode: %zd\n", err); @@ -1054,7 +1049,7 @@ static int mtk_hdmi_setup_vendor_specific_infoframe(struct mtk_hdmi *hdmi, ssize_t err; err = drm_hdmi_vendor_infoframe_from_display_mode(&frame, - &hdmi->conn, mode); + hdmi->curr_conn, mode); if (err) { dev_err(hdmi->dev, "Failed to get vendor infoframe from mode: %zd\n", err); @@ -1201,48 +1196,16 @@ mtk_hdmi_update_plugged_status(struct mtk_hdmi *hdmi) connector_status_connected : connector_status_disconnected; } -static enum drm_connector_status hdmi_conn_detect(struct drm_connector *conn, - bool force) +static enum drm_connector_status mtk_hdmi_detect(struct mtk_hdmi *hdmi) { - struct mtk_hdmi *hdmi = hdmi_ctx_from_conn(conn); return mtk_hdmi_update_plugged_status(hdmi); } -static void hdmi_conn_destroy(struct drm_connector *conn) -{ - struct mtk_hdmi *hdmi = hdmi_ctx_from_conn(conn); - - mtk_cec_set_hpd_event(hdmi->cec_dev, NULL, NULL); - - drm_connector_cleanup(conn); -} - -static int mtk_hdmi_conn_get_modes(struct drm_connector *conn) -{ - struct mtk_hdmi *hdmi = hdmi_ctx_from_conn(conn); - struct edid *edid; - int ret; - - if (!hdmi->ddc_adpt) - return -ENODEV; - - edid = drm_get_edid(conn, hdmi->ddc_adpt); - if (!edid) - return -ENODEV; - - hdmi->dvi_mode = !drm_detect_monitor_audio(edid); - - drm_connector_update_edid_property(conn, edid); - - ret = drm_add_edid_modes(conn, edid); - kfree(edid); - return ret; -} - -static int mtk_hdmi_conn_mode_valid(struct drm_connector *conn, - struct drm_display_mode *mode) +static int mtk_hdmi_bridge_mode_valid(struct drm_bridge *bridge, + const struct drm_display_info *info, + const struct drm_display_mode *mode) { - struct mtk_hdmi *hdmi = hdmi_ctx_from_conn(conn); + struct mtk_hdmi *hdmi = hdmi_ctx_from_bridge(bridge); struct drm_bridge *next_bridge; dev_dbg(hdmi->dev, "xres=%d, yres=%d, refresh=%d, intl=%d clock=%d\n", @@ -1267,74 +1230,57 @@ static int mtk_hdmi_conn_mode_valid(struct drm_connector *conn, return drm_mode_validate_size(mode, 0x1fff, 0x1fff); } -static struct drm_encoder *mtk_hdmi_conn_best_enc(struct drm_connector *conn) -{ - struct mtk_hdmi *hdmi = hdmi_ctx_from_conn(conn); - - return hdmi->bridge.encoder; -} - -static const struct drm_connector_funcs mtk_hdmi_connector_funcs = { - .detect = hdmi_conn_detect, - .fill_modes = drm_helper_probe_single_connector_modes, - .destroy = hdmi_conn_destroy, - .reset = drm_atomic_helper_connector_reset, - .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, - .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, -}; - -static const struct drm_connector_helper_funcs - mtk_hdmi_connector_helper_funcs = { - .get_modes = mtk_hdmi_conn_get_modes, - .mode_valid = mtk_hdmi_conn_mode_valid, - .best_encoder = mtk_hdmi_conn_best_enc, -}; - static void mtk_hdmi_hpd_event(bool hpd, struct device *dev) { struct mtk_hdmi *hdmi = dev_get_drvdata(dev); - if (hdmi && hdmi->bridge.encoder && hdmi->bridge.encoder->dev) + if (hdmi && hdmi->bridge.encoder && hdmi->bridge.encoder->dev) { + static enum drm_connector_status status; + + status = mtk_hdmi_detect(hdmi); drm_helper_hpd_irq_event(hdmi->bridge.encoder->dev); + drm_bridge_hpd_notify(&hdmi->bridge, status); + } } /* * Bridge callbacks */ +static enum drm_connector_status mtk_hdmi_bridge_detect(struct drm_bridge *bridge) +{ + struct mtk_hdmi *hdmi = hdmi_ctx_from_bridge(bridge); + + return mtk_hdmi_detect(hdmi); +} + +static struct edid *mtk_hdmi_bridge_get_edid(struct drm_bridge *bridge, + struct drm_connector *connector) +{ + struct mtk_hdmi *hdmi = hdmi_ctx_from_bridge(bridge); + struct edid *edid; + + if (!hdmi->ddc_adpt) + return NULL; + edid = drm_get_edid(connector, hdmi->ddc_adpt); + if (!edid) + return NULL; + hdmi->dvi_mode = !drm_detect_monitor_audio(edid); + return edid; +} + static int mtk_hdmi_bridge_attach(struct drm_bridge *bridge, enum drm_bridge_attach_flags flags) { struct mtk_hdmi *hdmi = hdmi_ctx_from_bridge(bridge); int ret; - if (flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR) { - DRM_ERROR("Fix bridge driver to make connector optional!"); + if (!(flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR)) { + DRM_ERROR("%s: The flag DRM_BRIDGE_ATTACH_NO_CONNECTOR must be supplied\n", + __func__); return -EINVAL; } - ret = drm_connector_init_with_ddc(bridge->encoder->dev, &hdmi->conn, - &mtk_hdmi_connector_funcs, - DRM_MODE_CONNECTOR_HDMIA, - hdmi->ddc_adpt); - if (ret) { - dev_err(hdmi->dev, "Failed to initialize connector: %d\n", ret); - return ret; - } - drm_connector_helper_add(&hdmi->conn, &mtk_hdmi_connector_helper_funcs); - - hdmi->conn.polled = DRM_CONNECTOR_POLL_HPD; - hdmi->conn.interlace_allowed = true; - hdmi->conn.doublescan_allowed = false; - - ret = drm_connector_attach_encoder(&hdmi->conn, - bridge->encoder); - if (ret) { - dev_err(hdmi->dev, - "Failed to attach connector to encoder: %d\n", ret); - return ret; - } - if (hdmi->next_bridge) { ret = drm_bridge_attach(bridge->encoder, hdmi->next_bridge, bridge, flags); @@ -1357,7 +1303,8 @@ static bool mtk_hdmi_bridge_mode_fixup(struct drm_bridge *bridge, return true; } -static void mtk_hdmi_bridge_disable(struct drm_bridge *bridge) +static void mtk_hdmi_bridge_atomic_disable(struct drm_bridge *bridge, + struct drm_bridge_state *old_bridge_state) { struct mtk_hdmi *hdmi = hdmi_ctx_from_bridge(bridge); @@ -1368,10 +1315,13 @@ static void mtk_hdmi_bridge_disable(struct drm_bridge *bridge) clk_disable_unprepare(hdmi->clk[MTK_HDMI_CLK_HDMI_PIXEL]); clk_disable_unprepare(hdmi->clk[MTK_HDMI_CLK_HDMI_PLL]); + hdmi->curr_conn = NULL; + hdmi->enabled = false; } -static void mtk_hdmi_bridge_post_disable(struct drm_bridge *bridge) +static void mtk_hdmi_bridge_atomic_post_disable(struct drm_bridge *bridge, + struct drm_bridge_state *old_state) { struct mtk_hdmi *hdmi = hdmi_ctx_from_bridge(bridge); @@ -1406,7 +1356,8 @@ static void mtk_hdmi_bridge_mode_set(struct drm_bridge *bridge, drm_mode_copy(&hdmi->mode, adjusted_mode); } -static void mtk_hdmi_bridge_pre_enable(struct drm_bridge *bridge) +static void mtk_hdmi_bridge_atomic_pre_enable(struct drm_bridge *bridge, + struct drm_bridge_state *old_state) { struct mtk_hdmi *hdmi = hdmi_ctx_from_bridge(bridge); @@ -1426,10 +1377,16 @@ static void mtk_hdmi_send_infoframe(struct mtk_hdmi *hdmi, mtk_hdmi_setup_vendor_specific_infoframe(hdmi, mode); } -static void mtk_hdmi_bridge_enable(struct drm_bridge *bridge) +static void mtk_hdmi_bridge_atomic_enable(struct drm_bridge *bridge, + struct drm_bridge_state *old_state) { + struct drm_atomic_state *state = old_state->base.state; struct mtk_hdmi *hdmi = hdmi_ctx_from_bridge(bridge); + /* Retrieve the connector through the atomic state. */ + hdmi->curr_conn = drm_atomic_get_new_connector_for_encoder(state, + bridge->encoder); + mtk_hdmi_output_set_display_mode(hdmi, &hdmi->mode); clk_prepare_enable(hdmi->clk[MTK_HDMI_CLK_HDMI_PLL]); clk_prepare_enable(hdmi->clk[MTK_HDMI_CLK_HDMI_PIXEL]); @@ -1440,13 +1397,19 @@ static void mtk_hdmi_bridge_enable(struct drm_bridge *bridge) } static const struct drm_bridge_funcs mtk_hdmi_bridge_funcs = { + .mode_valid = mtk_hdmi_bridge_mode_valid, + .atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_bridge_destroy_state, + .atomic_reset = drm_atomic_helper_bridge_reset, .attach = mtk_hdmi_bridge_attach, .mode_fixup = mtk_hdmi_bridge_mode_fixup, - .disable = mtk_hdmi_bridge_disable, - .post_disable = mtk_hdmi_bridge_post_disable, + .atomic_disable = mtk_hdmi_bridge_atomic_disable, + .atomic_post_disable = mtk_hdmi_bridge_atomic_post_disable, .mode_set = mtk_hdmi_bridge_mode_set, - .pre_enable = mtk_hdmi_bridge_pre_enable, - .enable = mtk_hdmi_bridge_enable, + .atomic_pre_enable = mtk_hdmi_bridge_atomic_pre_enable, + .atomic_enable = mtk_hdmi_bridge_atomic_enable, + .detect = mtk_hdmi_bridge_detect, + .get_edid = mtk_hdmi_bridge_get_edid, }; static int mtk_hdmi_dt_parse_pdata(struct mtk_hdmi *hdmi, @@ -1662,8 +1625,10 @@ static int mtk_hdmi_audio_get_eld(struct device *dev, void *data, uint8_t *buf, { struct mtk_hdmi *hdmi = dev_get_drvdata(dev); - memcpy(buf, hdmi->conn.eld, min(sizeof(hdmi->conn.eld), len)); - + if (hdmi->enabled) + memcpy(buf, hdmi->curr_conn->eld, min(sizeof(hdmi->curr_conn->eld), len)); + else + memset(buf, 0, len); return 0; } @@ -1755,6 +1720,9 @@ static int mtk_drm_hdmi_probe(struct platform_device *pdev) hdmi->bridge.funcs = &mtk_hdmi_bridge_funcs; hdmi->bridge.of_node = pdev->dev.of_node; + hdmi->bridge.ops = DRM_BRIDGE_OP_DETECT | DRM_BRIDGE_OP_EDID + | DRM_BRIDGE_OP_HPD; + hdmi->bridge.type = DRM_MODE_CONNECTOR_HDMIA; drm_bridge_add(&hdmi->bridge); ret = mtk_hdmi_clk_enable_audio(hdmi); diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c index 91cf46f84025..3d55e153fa9c 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c @@ -246,7 +246,7 @@ static int a6xx_gmu_hfi_start(struct a6xx_gmu *gmu) } struct a6xx_gmu_oob_bits { - int set, ack, set_new, ack_new; + int set, ack, set_new, ack_new, clear, clear_new; const char *name; }; @@ -260,6 +260,8 @@ static const struct a6xx_gmu_oob_bits a6xx_gmu_oob_bits[] = { .ack = 24, .set_new = 30, .ack_new = 31, + .clear = 24, + .clear_new = 31, }, [GMU_OOB_PERFCOUNTER_SET] = { @@ -268,18 +270,22 @@ static const struct a6xx_gmu_oob_bits a6xx_gmu_oob_bits[] = { .ack = 25, .set_new = 28, .ack_new = 30, + .clear = 25, + .clear_new = 29, }, [GMU_OOB_BOOT_SLUMBER] = { .name = "BOOT_SLUMBER", .set = 22, .ack = 30, + .clear = 30, }, [GMU_OOB_DCVS_SET] = { .name = "GPU_DCVS", .set = 23, .ack = 31, + .clear = 31, }, }; @@ -335,9 +341,9 @@ void a6xx_gmu_clear_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state) return; if (gmu->legacy) - bit = a6xx_gmu_oob_bits[state].ack; + bit = a6xx_gmu_oob_bits[state].clear; else - bit = a6xx_gmu_oob_bits[state].ack_new; + bit = a6xx_gmu_oob_bits[state].clear_new; gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, 1 << bit); } diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c index d553f62f4eeb..b4d8e1b01ee4 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c @@ -1153,10 +1153,6 @@ static void a6xx_llc_slices_init(struct platform_device *pdev, { struct device_node *phandle; - a6xx_gpu->llc_mmio = msm_ioremap(pdev, "cx_mem", "gpu_cx"); - if (IS_ERR(a6xx_gpu->llc_mmio)) - return; - /* * There is a different programming path for targets with an mmu500 * attached, so detect if that is the case @@ -1166,6 +1162,11 @@ static void a6xx_llc_slices_init(struct platform_device *pdev, of_device_is_compatible(phandle, "arm,mmu-500")); of_node_put(phandle); + if (a6xx_gpu->have_mmu500) + a6xx_gpu->llc_mmio = NULL; + else + a6xx_gpu->llc_mmio = msm_ioremap(pdev, "cx_mem", "gpu_cx"); + a6xx_gpu->llc_slice = llcc_slice_getd(LLCC_GPU); a6xx_gpu->htw_llc_slice = llcc_slice_getd(LLCC_GPUHTW); diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c index 189f3533525c..e4444452759c 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c @@ -22,7 +22,7 @@ (VIG_MASK | BIT(DPU_SSPP_QOS_8LVL) | BIT(DPU_SSPP_SCALER_QSEED4)) #define VIG_SM8250_MASK \ - (VIG_MASK | BIT(DPU_SSPP_SCALER_QSEED3LITE)) + (VIG_MASK | BIT(DPU_SSPP_QOS_8LVL) | BIT(DPU_SSPP_SCALER_QSEED3LITE)) #define DMA_SDM845_MASK \ (BIT(DPU_SSPP_SRC) | BIT(DPU_SSPP_QOS) | BIT(DPU_SSPP_QOS_8LVL) |\ diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_cmd_encoder.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_cmd_encoder.c index ff2c1d583c79..0392d4dfe270 100644 --- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_cmd_encoder.c +++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_cmd_encoder.c @@ -20,7 +20,7 @@ static int pingpong_tearcheck_setup(struct drm_encoder *encoder, { struct mdp5_kms *mdp5_kms = get_kms(encoder); struct device *dev = encoder->dev->dev; - u32 total_lines_x100, vclks_line, cfg; + u32 total_lines, vclks_line, cfg; long vsync_clk_speed; struct mdp5_hw_mixer *mixer = mdp5_crtc_get_mixer(encoder->crtc); int pp_id = mixer->pp; @@ -30,8 +30,8 @@ static int pingpong_tearcheck_setup(struct drm_encoder *encoder, return -EINVAL; } - total_lines_x100 = mode->vtotal * drm_mode_vrefresh(mode); - if (!total_lines_x100) { + total_lines = mode->vtotal * drm_mode_vrefresh(mode); + if (!total_lines) { DRM_DEV_ERROR(dev, "%s: vtotal(%d) or vrefresh(%d) is 0\n", __func__, mode->vtotal, drm_mode_vrefresh(mode)); return -EINVAL; @@ -43,15 +43,23 @@ static int pingpong_tearcheck_setup(struct drm_encoder *encoder, vsync_clk_speed); return -EINVAL; } - vclks_line = vsync_clk_speed * 100 / total_lines_x100; + vclks_line = vsync_clk_speed / total_lines; cfg = MDP5_PP_SYNC_CONFIG_VSYNC_COUNTER_EN | MDP5_PP_SYNC_CONFIG_VSYNC_IN_EN; cfg |= MDP5_PP_SYNC_CONFIG_VSYNC_COUNT(vclks_line); + /* + * Tearcheck emits a blanking signal every vclks_line * vtotal * 2 ticks on + * the vsync_clk equating to roughly half the desired panel refresh rate. + * This is only necessary as stability fallback if interrupts from the + * panel arrive too late or not at all, but is currently used by default + * because these panel interrupts are not wired up yet. + */ mdp5_write(mdp5_kms, REG_MDP5_PP_SYNC_CONFIG_VSYNC(pp_id), cfg); mdp5_write(mdp5_kms, - REG_MDP5_PP_SYNC_CONFIG_HEIGHT(pp_id), 0xfff0); + REG_MDP5_PP_SYNC_CONFIG_HEIGHT(pp_id), (2 * mode->vtotal)); + mdp5_write(mdp5_kms, REG_MDP5_PP_VSYNC_INIT_VAL(pp_id), mode->vdisplay); mdp5_write(mdp5_kms, REG_MDP5_PP_RD_PTR_IRQ(pp_id), mode->vdisplay + 1); diff --git a/drivers/gpu/drm/msm/dp/dp_audio.c b/drivers/gpu/drm/msm/dp/dp_audio.c index 82a8673ab8da..d7e4a39a904e 100644 --- a/drivers/gpu/drm/msm/dp/dp_audio.c +++ b/drivers/gpu/drm/msm/dp/dp_audio.c @@ -527,6 +527,7 @@ int dp_audio_hw_params(struct device *dev, dp_audio_setup_acr(audio); dp_audio_safe_to_exit_level(audio); dp_audio_enable(audio, true); + dp_display_signal_audio_start(dp_display); dp_display->audio_enabled = true; end: diff --git a/drivers/gpu/drm/msm/dp/dp_display.c b/drivers/gpu/drm/msm/dp/dp_display.c index 5a39da6e1eaf..1784e119269b 100644 --- a/drivers/gpu/drm/msm/dp/dp_display.c +++ b/drivers/gpu/drm/msm/dp/dp_display.c @@ -178,6 +178,15 @@ static int dp_del_event(struct dp_display_private *dp_priv, u32 event) return 0; } +void dp_display_signal_audio_start(struct msm_dp *dp_display) +{ + struct dp_display_private *dp; + + dp = container_of(dp_display, struct dp_display_private, dp_display); + + reinit_completion(&dp->audio_comp); +} + void dp_display_signal_audio_complete(struct msm_dp *dp_display) { struct dp_display_private *dp; @@ -586,10 +595,8 @@ static int dp_connect_pending_timeout(struct dp_display_private *dp, u32 data) mutex_lock(&dp->event_mutex); state = dp->hpd_state; - if (state == ST_CONNECT_PENDING) { - dp_display_enable(dp, 0); + if (state == ST_CONNECT_PENDING) dp->hpd_state = ST_CONNECTED; - } mutex_unlock(&dp->event_mutex); @@ -651,7 +658,6 @@ static int dp_hpd_unplug_handle(struct dp_display_private *dp, u32 data) dp_add_event(dp, EV_DISCONNECT_PENDING_TIMEOUT, 0, DP_TIMEOUT_5_SECOND); /* signal the disconnect event early to ensure proper teardown */ - reinit_completion(&dp->audio_comp); dp_display_handle_plugged_change(g_dp_display, false); dp_catalog_hpd_config_intr(dp->catalog, DP_DP_HPD_PLUG_INT_MASK | @@ -669,10 +675,8 @@ static int dp_disconnect_pending_timeout(struct dp_display_private *dp, u32 data mutex_lock(&dp->event_mutex); state = dp->hpd_state; - if (state == ST_DISCONNECT_PENDING) { - dp_display_disable(dp, 0); + if (state == ST_DISCONNECT_PENDING) dp->hpd_state = ST_DISCONNECTED; - } mutex_unlock(&dp->event_mutex); @@ -898,7 +902,6 @@ static int dp_display_disable(struct dp_display_private *dp, u32 data) /* wait only if audio was enabled */ if (dp_display->audio_enabled) { /* signal the disconnect event */ - reinit_completion(&dp->audio_comp); dp_display_handle_plugged_change(dp_display, false); if (!wait_for_completion_timeout(&dp->audio_comp, HZ * 5)) @@ -1272,7 +1275,12 @@ static int dp_pm_resume(struct device *dev) status = dp_catalog_link_is_connected(dp->catalog); - if (status) + /* + * can not declared display is connected unless + * HDMI cable is plugged in and sink_count of + * dongle become 1 + */ + if (status && dp->link->sink_count) dp->dp_display.is_connected = true; else dp->dp_display.is_connected = false; diff --git a/drivers/gpu/drm/msm/dp/dp_display.h b/drivers/gpu/drm/msm/dp/dp_display.h index 6092ba1ed85e..5173c89eedf7 100644 --- a/drivers/gpu/drm/msm/dp/dp_display.h +++ b/drivers/gpu/drm/msm/dp/dp_display.h @@ -34,6 +34,7 @@ int dp_display_get_modes(struct msm_dp *dp_display, int dp_display_request_irq(struct msm_dp *dp_display); bool dp_display_check_video_test(struct msm_dp *dp_display); int dp_display_get_test_bpp(struct msm_dp *dp_display); +void dp_display_signal_audio_start(struct msm_dp *dp_display); void dp_display_signal_audio_complete(struct msm_dp *dp_display); #endif /* _DP_DISPLAY_H_ */ diff --git a/drivers/gpu/drm/msm/dp/dp_hpd.c b/drivers/gpu/drm/msm/dp/dp_hpd.c index 5b8fe32022b5..e1c90fa47411 100644 --- a/drivers/gpu/drm/msm/dp/dp_hpd.c +++ b/drivers/gpu/drm/msm/dp/dp_hpd.c @@ -34,8 +34,8 @@ int dp_hpd_connect(struct dp_usbpd *dp_usbpd, bool hpd) dp_usbpd->hpd_high = hpd; - if (!hpd_priv->dp_cb && !hpd_priv->dp_cb->configure - && !hpd_priv->dp_cb->disconnect) { + if (!hpd_priv->dp_cb || !hpd_priv->dp_cb->configure + || !hpd_priv->dp_cb->disconnect) { pr_err("hpd dp_cb not initialized\n"); return -EINVAL; } diff --git a/drivers/gpu/drm/msm/msm_debugfs.c b/drivers/gpu/drm/msm/msm_debugfs.c index 85ad0babc326..d611cc8e54a4 100644 --- a/drivers/gpu/drm/msm/msm_debugfs.c +++ b/drivers/gpu/drm/msm/msm_debugfs.c @@ -111,23 +111,15 @@ static const struct file_operations msm_gpu_fops = { static int msm_gem_show(struct drm_device *dev, struct seq_file *m) { struct msm_drm_private *priv = dev->dev_private; - struct msm_gpu *gpu = priv->gpu; int ret; - ret = mutex_lock_interruptible(&priv->mm_lock); + ret = mutex_lock_interruptible(&priv->obj_lock); if (ret) return ret; - if (gpu) { - seq_printf(m, "Active Objects (%s):\n", gpu->name); - msm_gem_describe_objects(&gpu->active_list, m); - } - - seq_printf(m, "Inactive Objects:\n"); - msm_gem_describe_objects(&priv->inactive_dontneed, m); - msm_gem_describe_objects(&priv->inactive_willneed, m); + msm_gem_describe_objects(&priv->objects, m); - mutex_unlock(&priv->mm_lock); + mutex_unlock(&priv->obj_lock); return 0; } diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c index 196907689c82..18ea1c66de71 100644 --- a/drivers/gpu/drm/msm/msm_drv.c +++ b/drivers/gpu/drm/msm/msm_drv.c @@ -446,6 +446,9 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv) priv->wq = alloc_ordered_workqueue("msm", 0); + INIT_LIST_HEAD(&priv->objects); + mutex_init(&priv->obj_lock); + INIT_LIST_HEAD(&priv->inactive_willneed); INIT_LIST_HEAD(&priv->inactive_dontneed); mutex_init(&priv->mm_lock); diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h index 591c47a654e8..6b58e49754cb 100644 --- a/drivers/gpu/drm/msm/msm_drv.h +++ b/drivers/gpu/drm/msm/msm_drv.h @@ -174,7 +174,14 @@ struct msm_drm_private { struct msm_rd_state *hangrd; /* debugfs to dump hanging submits */ struct msm_perf_state *perf; - /* + /** + * List of all GEM objects (mainly for debugfs, protected by obj_lock + * (acquire before per GEM object lock) + */ + struct list_head objects; + struct mutex obj_lock; + + /** * Lists of inactive GEM objects. Every bo is either in one of the * inactive lists (depending on whether or not it is shrinkable) or * gpu->active_list (for the gpu it is active on[1]) diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c index f091c1e164fa..aeba3eb8ce46 100644 --- a/drivers/gpu/drm/msm/msm_gem.c +++ b/drivers/gpu/drm/msm/msm_gem.c @@ -951,7 +951,7 @@ void msm_gem_describe_objects(struct list_head *list, struct seq_file *m) size_t size = 0; seq_puts(m, " flags id ref offset kaddr size madv name\n"); - list_for_each_entry(msm_obj, list, mm_list) { + list_for_each_entry(msm_obj, list, node) { struct drm_gem_object *obj = &msm_obj->base; seq_puts(m, " "); msm_gem_describe(obj, m); @@ -970,6 +970,10 @@ void msm_gem_free_object(struct drm_gem_object *obj) struct drm_device *dev = obj->dev; struct msm_drm_private *priv = dev->dev_private; + mutex_lock(&priv->obj_lock); + list_del(&msm_obj->node); + mutex_unlock(&priv->obj_lock); + mutex_lock(&priv->mm_lock); list_del(&msm_obj->mm_list); mutex_unlock(&priv->mm_lock); @@ -1157,6 +1161,10 @@ static struct drm_gem_object *_msm_gem_new(struct drm_device *dev, list_add_tail(&msm_obj->mm_list, &priv->inactive_willneed); mutex_unlock(&priv->mm_lock); + mutex_lock(&priv->obj_lock); + list_add_tail(&msm_obj->node, &priv->objects); + mutex_unlock(&priv->obj_lock); + return obj; fail: @@ -1227,6 +1235,10 @@ struct drm_gem_object *msm_gem_import(struct drm_device *dev, list_add_tail(&msm_obj->mm_list, &priv->inactive_willneed); mutex_unlock(&priv->mm_lock); + mutex_lock(&priv->obj_lock); + list_add_tail(&msm_obj->node, &priv->objects); + mutex_unlock(&priv->obj_lock); + return obj; fail: diff --git a/drivers/gpu/drm/msm/msm_gem.h b/drivers/gpu/drm/msm/msm_gem.h index b3a0a880cbab..99d4c0e9465e 100644 --- a/drivers/gpu/drm/msm/msm_gem.h +++ b/drivers/gpu/drm/msm/msm_gem.h @@ -55,8 +55,16 @@ struct msm_gem_object { */ uint8_t vmap_count; - /* And object is either: - * inactive - on priv->inactive_list + /** + * Node in list of all objects (mainly for debugfs, protected by + * priv->obj_lock + */ + struct list_head node; + + /** + * An object is either: + * inactive - on priv->inactive_dontneed or priv->inactive_willneed + * (depending on purgability status) * active - on one one of the gpu's active_list.. well, at * least for now we don't have (I don't think) hw sync between * 2d and 3d one devices which have both, meaning we need to diff --git a/drivers/gpu/drm/omapdrm/dss/dsi.c b/drivers/gpu/drm/omapdrm/dss/dsi.c index b31d750c425a..5f1722b040f4 100644 --- a/drivers/gpu/drm/omapdrm/dss/dsi.c +++ b/drivers/gpu/drm/omapdrm/dss/dsi.c @@ -4327,7 +4327,8 @@ static int omap_dsi_register_te_irq(struct dsi_data *dsi, irq_set_status_flags(te_irq, IRQ_NOAUTOEN); err = request_threaded_irq(te_irq, NULL, omap_dsi_te_irq_handler, - IRQF_TRIGGER_RISING, "TE", dsi); + IRQF_TRIGGER_RISING | IRQF_ONESHOT, + "TE", dsi); if (err) { dev_err(dsi->dev, "request irq failed with %d\n", err); gpiod_put(dsi->te_gpio); diff --git a/drivers/gpu/drm/panel/panel-novatek-nt35510.c b/drivers/gpu/drm/panel/panel-novatek-nt35510.c index b9a0e56f33e2..ef70140c5b09 100644 --- a/drivers/gpu/drm/panel/panel-novatek-nt35510.c +++ b/drivers/gpu/drm/panel/panel-novatek-nt35510.c @@ -898,8 +898,7 @@ static int nt35510_probe(struct mipi_dsi_device *dsi) */ dsi->hs_rate = 349440000; dsi->lp_rate = 9600000; - dsi->mode_flags = MIPI_DSI_CLOCK_NON_CONTINUOUS | - MIPI_DSI_MODE_EOT_PACKET; + dsi->mode_flags = MIPI_DSI_CLOCK_NON_CONTINUOUS; /* * Every new incarnation of this display must have a unique diff --git a/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c b/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c index 4aac0d1573dd..70560cac53a9 100644 --- a/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c +++ b/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c @@ -184,9 +184,7 @@ static int s6d16d0_probe(struct mipi_dsi_device *dsi) * As we only send commands we do not need to be continuously * clocked. */ - dsi->mode_flags = - MIPI_DSI_CLOCK_NON_CONTINUOUS | - MIPI_DSI_MODE_EOT_PACKET; + dsi->mode_flags = MIPI_DSI_CLOCK_NON_CONTINUOUS; s6->supply = devm_regulator_get(dev, "vdd1"); if (IS_ERR(s6->supply)) diff --git a/drivers/gpu/drm/panel/panel-samsung-s6e63m0-dsi.c b/drivers/gpu/drm/panel/panel-samsung-s6e63m0-dsi.c index eec74c10ddda..9c3563c61e8c 100644 --- a/drivers/gpu/drm/panel/panel-samsung-s6e63m0-dsi.c +++ b/drivers/gpu/drm/panel/panel-samsung-s6e63m0-dsi.c @@ -97,7 +97,6 @@ static int s6e63m0_dsi_probe(struct mipi_dsi_device *dsi) dsi->hs_rate = 349440000; dsi->lp_rate = 9600000; dsi->mode_flags = MIPI_DSI_MODE_VIDEO | - MIPI_DSI_MODE_EOT_PACKET | MIPI_DSI_MODE_VIDEO_BURST; ret = s6e63m0_probe(dev, s6e63m0_dsi_dcs_read, s6e63m0_dsi_dcs_write, diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c index 4e2dad314c79..e8b1a0e873ea 100644 --- a/drivers/gpu/drm/panel/panel-simple.c +++ b/drivers/gpu/drm/panel/panel-simple.c @@ -406,7 +406,7 @@ static int panel_simple_prepare(struct drm_panel *panel) if (IS_ERR(p->hpd_gpio)) { err = panel_simple_get_hpd_gpio(panel->dev, p, false); if (err) - return err; + goto error; } err = readx_poll_timeout(gpiod_get_value_cansleep, p->hpd_gpio, @@ -418,13 +418,20 @@ static int panel_simple_prepare(struct drm_panel *panel) if (err) { dev_err(panel->dev, "error waiting for hpd GPIO: %d\n", err); - return err; + goto error; } } p->prepared_time = ktime_get(); return 0; + +error: + gpiod_set_value_cansleep(p->enable_gpio, 0); + regulator_disable(p->supply); + p->unprepared_time = ktime_get(); + + return err; } static int panel_simple_enable(struct drm_panel *panel) diff --git a/drivers/gpu/drm/panel/panel-sony-acx424akp.c b/drivers/gpu/drm/panel/panel-sony-acx424akp.c index 065efae213f5..95659a4d15e9 100644 --- a/drivers/gpu/drm/panel/panel-sony-acx424akp.c +++ b/drivers/gpu/drm/panel/panel-sony-acx424akp.c @@ -449,8 +449,7 @@ static int acx424akp_probe(struct mipi_dsi_device *dsi) MIPI_DSI_MODE_VIDEO_BURST; else dsi->mode_flags = - MIPI_DSI_CLOCK_NON_CONTINUOUS | - MIPI_DSI_MODE_EOT_PACKET; + MIPI_DSI_CLOCK_NON_CONTINUOUS; acx->supply = devm_regulator_get(dev, "vddi"); if (IS_ERR(acx->supply)) diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c index 7c1b3481b785..21e552d1ac71 100644 --- a/drivers/gpu/drm/panfrost/panfrost_mmu.c +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c @@ -488,8 +488,14 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, } bo->base.pages = pages; bo->base.pages_use_count = 1; - } else + } else { pages = bo->base.pages; + if (pages[page_offset]) { + /* Pages are already mapped, bail out. */ + mutex_unlock(&bo->base.pages_lock); + goto out; + } + } mapping = bo->base.base.filp->f_mapping; mapping_set_unevictable(mapping); @@ -522,6 +528,7 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, dev_dbg(pfdev->dev, "mapped page fault @ AS%d %llx", as, addr); +out: panfrost_gem_mapping_put(bomapping); return 0; @@ -593,6 +600,8 @@ static irqreturn_t panfrost_mmu_irq_handler_thread(int irq, void *data) access_type = (fault_status >> 8) & 0x3; source_id = (fault_status >> 16); + mmu_write(pfdev, MMU_INT_CLEAR, mask); + /* Page fault only */ ret = -1; if ((status & mask) == BIT(i) && (exception_type & 0xF8) == 0xC0) @@ -616,8 +625,6 @@ static irqreturn_t panfrost_mmu_irq_handler_thread(int irq, void *data) access_type, access_type_name(pfdev, fault_status), source_id); - mmu_write(pfdev, MMU_INT_CLEAR, mask); - status &= ~mask; } diff --git a/drivers/gpu/drm/qxl/qxl_cmd.c b/drivers/gpu/drm/qxl/qxl_cmd.c index 54e3c3a97440..741cc983daf1 100644 --- a/drivers/gpu/drm/qxl/qxl_cmd.c +++ b/drivers/gpu/drm/qxl/qxl_cmd.c @@ -268,7 +268,7 @@ int qxl_alloc_bo_reserved(struct qxl_device *qdev, int ret; ret = qxl_bo_create(qdev, size, false /* not kernel - device */, - false, QXL_GEM_DOMAIN_VRAM, NULL, &bo); + false, QXL_GEM_DOMAIN_VRAM, 0, NULL, &bo); if (ret) { DRM_ERROR("failed to allocate VRAM BO\n"); return ret; diff --git a/drivers/gpu/drm/qxl/qxl_display.c b/drivers/gpu/drm/qxl/qxl_display.c index 10738e04c09b..3f432ec8e771 100644 --- a/drivers/gpu/drm/qxl/qxl_display.c +++ b/drivers/gpu/drm/qxl/qxl_display.c @@ -798,8 +798,8 @@ static int qxl_plane_prepare_fb(struct drm_plane *plane, qdev->dumb_shadow_bo = NULL; } qxl_bo_create(qdev, surf.height * surf.stride, - true, true, QXL_GEM_DOMAIN_SURFACE, &surf, - &qdev->dumb_shadow_bo); + true, true, QXL_GEM_DOMAIN_SURFACE, 0, + &surf, &qdev->dumb_shadow_bo); } if (user_bo->shadow != qdev->dumb_shadow_bo) { if (user_bo->shadow) { @@ -1228,6 +1228,10 @@ int qxl_modeset_init(struct qxl_device *qdev) void qxl_modeset_fini(struct qxl_device *qdev) { + if (qdev->dumb_shadow_bo) { + drm_gem_object_put(&qdev->dumb_shadow_bo->tbo.base); + qdev->dumb_shadow_bo = NULL; + } qxl_destroy_monitors_object(qdev); drm_mode_config_cleanup(&qdev->ddev); } diff --git a/drivers/gpu/drm/qxl/qxl_gem.c b/drivers/gpu/drm/qxl/qxl_gem.c index 48e096285b4c..a08da0bd9098 100644 --- a/drivers/gpu/drm/qxl/qxl_gem.c +++ b/drivers/gpu/drm/qxl/qxl_gem.c @@ -55,7 +55,7 @@ int qxl_gem_object_create(struct qxl_device *qdev, int size, /* At least align on page size */ if (alignment < PAGE_SIZE) alignment = PAGE_SIZE; - r = qxl_bo_create(qdev, size, kernel, false, initial_domain, surf, &qbo); + r = qxl_bo_create(qdev, size, kernel, false, initial_domain, 0, surf, &qbo); if (r) { if (r != -ERESTARTSYS) DRM_ERROR( diff --git a/drivers/gpu/drm/qxl/qxl_object.c b/drivers/gpu/drm/qxl/qxl_object.c index ceebc5881f68..a5806667697a 100644 --- a/drivers/gpu/drm/qxl/qxl_object.c +++ b/drivers/gpu/drm/qxl/qxl_object.c @@ -103,8 +103,8 @@ static const struct drm_gem_object_funcs qxl_object_funcs = { .print_info = drm_gem_ttm_print_info, }; -int qxl_bo_create(struct qxl_device *qdev, - unsigned long size, bool kernel, bool pinned, u32 domain, +int qxl_bo_create(struct qxl_device *qdev, unsigned long size, + bool kernel, bool pinned, u32 domain, u32 priority, struct qxl_surface *surf, struct qxl_bo **bo_ptr) { @@ -137,6 +137,7 @@ int qxl_bo_create(struct qxl_device *qdev, qxl_ttm_placement_from_domain(bo, domain); + bo->tbo.priority = priority; r = ttm_bo_init_reserved(&qdev->mman.bdev, &bo->tbo, size, type, &bo->placement, 0, &ctx, size, NULL, NULL, &qxl_ttm_bo_destroy); diff --git a/drivers/gpu/drm/qxl/qxl_object.h b/drivers/gpu/drm/qxl/qxl_object.h index e60a8f88e226..dc1659e717f1 100644 --- a/drivers/gpu/drm/qxl/qxl_object.h +++ b/drivers/gpu/drm/qxl/qxl_object.h @@ -61,6 +61,7 @@ static inline u64 qxl_bo_mmap_offset(struct qxl_bo *bo) extern int qxl_bo_create(struct qxl_device *qdev, unsigned long size, bool kernel, bool pinned, u32 domain, + u32 priority, struct qxl_surface *surf, struct qxl_bo **bo_ptr); extern int qxl_bo_kmap(struct qxl_bo *bo, struct dma_buf_map *map); diff --git a/drivers/gpu/drm/qxl/qxl_release.c b/drivers/gpu/drm/qxl/qxl_release.c index b372455e2729..801ce77b1dac 100644 --- a/drivers/gpu/drm/qxl/qxl_release.c +++ b/drivers/gpu/drm/qxl/qxl_release.c @@ -199,11 +199,12 @@ qxl_release_free(struct qxl_device *qdev, } static int qxl_release_bo_alloc(struct qxl_device *qdev, - struct qxl_bo **bo) + struct qxl_bo **bo, + u32 priority) { /* pin releases bo's they are too messy to evict */ return qxl_bo_create(qdev, PAGE_SIZE, false, true, - QXL_GEM_DOMAIN_VRAM, NULL, bo); + QXL_GEM_DOMAIN_VRAM, priority, NULL, bo); } int qxl_release_list_add(struct qxl_release *release, struct qxl_bo *bo) @@ -326,13 +327,18 @@ int qxl_alloc_release_reserved(struct qxl_device *qdev, unsigned long size, int ret = 0; union qxl_release_info *info; int cur_idx; + u32 priority; - if (type == QXL_RELEASE_DRAWABLE) + if (type == QXL_RELEASE_DRAWABLE) { cur_idx = 0; - else if (type == QXL_RELEASE_SURFACE_CMD) + priority = 0; + } else if (type == QXL_RELEASE_SURFACE_CMD) { cur_idx = 1; - else if (type == QXL_RELEASE_CURSOR_CMD) + priority = 1; + } else if (type == QXL_RELEASE_CURSOR_CMD) { cur_idx = 2; + priority = 1; + } else { DRM_ERROR("got illegal type: %d\n", type); return -EINVAL; @@ -352,7 +358,7 @@ int qxl_alloc_release_reserved(struct qxl_device *qdev, unsigned long size, qdev->current_release_bo[cur_idx] = NULL; } if (!qdev->current_release_bo[cur_idx]) { - ret = qxl_release_bo_alloc(qdev, &qdev->current_release_bo[cur_idx]); + ret = qxl_release_bo_alloc(qdev, &qdev->current_release_bo[cur_idx], priority); if (ret) { mutex_unlock(&qdev->release_mutex); if (free_bo) { diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h index 3effc8c71494..ea44423376c4 100644 --- a/drivers/gpu/drm/radeon/radeon.h +++ b/drivers/gpu/drm/radeon/radeon.h @@ -1558,6 +1558,7 @@ struct radeon_dpm { void *priv; u32 new_active_crtcs; int new_active_crtc_count; + int high_pixelclock_count; u32 current_active_crtcs; int current_active_crtc_count; bool single_display; diff --git a/drivers/gpu/drm/radeon/radeon_atombios.c b/drivers/gpu/drm/radeon/radeon_atombios.c index 42301b4e56f5..28c4413f4dc8 100644 --- a/drivers/gpu/drm/radeon/radeon_atombios.c +++ b/drivers/gpu/drm/radeon/radeon_atombios.c @@ -2120,11 +2120,14 @@ static int radeon_atombios_parse_power_table_1_3(struct radeon_device *rdev) return state_index; /* last mode is usually default, array is low to high */ for (i = 0; i < num_modes; i++) { - rdev->pm.power_state[state_index].clock_info = - kcalloc(1, sizeof(struct radeon_pm_clock_info), - GFP_KERNEL); + /* avoid memory leaks from invalid modes or unknown frev. */ + if (!rdev->pm.power_state[state_index].clock_info) { + rdev->pm.power_state[state_index].clock_info = + kzalloc(sizeof(struct radeon_pm_clock_info), + GFP_KERNEL); + } if (!rdev->pm.power_state[state_index].clock_info) - return state_index; + goto out; rdev->pm.power_state[state_index].num_clock_modes = 1; rdev->pm.power_state[state_index].clock_info[0].voltage.type = VOLTAGE_NONE; switch (frev) { @@ -2243,17 +2246,24 @@ static int radeon_atombios_parse_power_table_1_3(struct radeon_device *rdev) break; } } +out: + /* free any unused clock_info allocation. */ + if (state_index && state_index < num_modes) { + kfree(rdev->pm.power_state[state_index].clock_info); + rdev->pm.power_state[state_index].clock_info = NULL; + } + /* last mode is usually default */ - if (rdev->pm.default_power_state_index == -1) { + if (state_index && rdev->pm.default_power_state_index == -1) { rdev->pm.power_state[state_index - 1].type = POWER_STATE_TYPE_DEFAULT; rdev->pm.default_power_state_index = state_index - 1; rdev->pm.power_state[state_index - 1].default_clock_mode = &rdev->pm.power_state[state_index - 1].clock_info[0]; - rdev->pm.power_state[state_index].flags &= + rdev->pm.power_state[state_index - 1].flags &= ~RADEON_PM_STATE_SINGLE_DISPLAY_ONLY; - rdev->pm.power_state[state_index].misc = 0; - rdev->pm.power_state[state_index].misc2 = 0; + rdev->pm.power_state[state_index - 1].misc = 0; + rdev->pm.power_state[state_index - 1].misc2 = 0; } return state_index; } diff --git a/drivers/gpu/drm/radeon/radeon_dp_mst.c b/drivers/gpu/drm/radeon/radeon_dp_mst.c index 2c32186c4acd..4e4c937c36c6 100644 --- a/drivers/gpu/drm/radeon/radeon_dp_mst.c +++ b/drivers/gpu/drm/radeon/radeon_dp_mst.c @@ -242,6 +242,9 @@ radeon_dp_mst_detect(struct drm_connector *connector, to_radeon_connector(connector); struct radeon_connector *master = radeon_connector->mst_port; + if (drm_connector_is_unregistered(connector)) + return connector_status_disconnected; + return drm_dp_mst_detect_port(connector, ctx, &master->mst_mgr, radeon_connector->port); } diff --git a/drivers/gpu/drm/radeon/radeon_kms.c b/drivers/gpu/drm/radeon/radeon_kms.c index 2479d6ab7a36..58876bb4ef2a 100644 --- a/drivers/gpu/drm/radeon/radeon_kms.c +++ b/drivers/gpu/drm/radeon/radeon_kms.c @@ -518,6 +518,7 @@ int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) *value = rdev->config.si.backend_enable_mask; } else { DRM_DEBUG_KMS("BACKEND_ENABLED_MASK is si+ only!\n"); + return -EINVAL; } break; case RADEON_INFO_MAX_SCLK: diff --git a/drivers/gpu/drm/radeon/radeon_object.c b/drivers/gpu/drm/radeon/radeon_object.c index 9b81786782de..499ce55e34cc 100644 --- a/drivers/gpu/drm/radeon/radeon_object.c +++ b/drivers/gpu/drm/radeon/radeon_object.c @@ -384,6 +384,8 @@ int radeon_bo_evict_vram(struct radeon_device *rdev) } #endif man = ttm_manager_type(bdev, TTM_PL_VRAM); + if (!man) + return 0; return ttm_resource_manager_evict_all(bdev, man); } diff --git a/drivers/gpu/drm/radeon/radeon_pm.c b/drivers/gpu/drm/radeon/radeon_pm.c index 1995dad59dd0..2db4a8b1542d 100644 --- a/drivers/gpu/drm/radeon/radeon_pm.c +++ b/drivers/gpu/drm/radeon/radeon_pm.c @@ -1775,6 +1775,7 @@ static void radeon_pm_compute_clocks_dpm(struct radeon_device *rdev) struct drm_device *ddev = rdev->ddev; struct drm_crtc *crtc; struct radeon_crtc *radeon_crtc; + struct radeon_connector *radeon_connector; if (!rdev->pm.dpm_enabled) return; @@ -1784,6 +1785,7 @@ static void radeon_pm_compute_clocks_dpm(struct radeon_device *rdev) /* update active crtc counts */ rdev->pm.dpm.new_active_crtcs = 0; rdev->pm.dpm.new_active_crtc_count = 0; + rdev->pm.dpm.high_pixelclock_count = 0; if (rdev->num_crtc && rdev->mode_info.mode_config_initialized) { list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) { @@ -1791,6 +1793,12 @@ static void radeon_pm_compute_clocks_dpm(struct radeon_device *rdev) if (crtc->enabled) { rdev->pm.dpm.new_active_crtcs |= (1 << radeon_crtc->crtc_id); rdev->pm.dpm.new_active_crtc_count++; + if (!radeon_crtc->connector) + continue; + + radeon_connector = to_radeon_connector(radeon_crtc->connector); + if (radeon_connector->pixelclock_for_modeset > 297000) + rdev->pm.dpm.high_pixelclock_count++; } } } diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c index 78893bea85ae..c0258d213a72 100644 --- a/drivers/gpu/drm/radeon/radeon_ttm.c +++ b/drivers/gpu/drm/radeon/radeon_ttm.c @@ -485,13 +485,14 @@ static void radeon_ttm_backend_unbind(struct ttm_bo_device *bdev, struct ttm_tt struct radeon_ttm_tt *gtt = (void *)ttm; struct radeon_device *rdev = radeon_get_rdev(bdev); + if (gtt->userptr) + radeon_ttm_tt_unpin_userptr(bdev, ttm); + if (!gtt->bound) return; radeon_gart_unbind(rdev, gtt->offset, ttm->num_pages); - if (gtt->userptr) - radeon_ttm_tt_unpin_userptr(bdev, ttm); gtt->bound = false; } diff --git a/drivers/gpu/drm/radeon/si_dpm.c b/drivers/gpu/drm/radeon/si_dpm.c index 91bfc4762767..43b63705d073 100644 --- a/drivers/gpu/drm/radeon/si_dpm.c +++ b/drivers/gpu/drm/radeon/si_dpm.c @@ -2979,6 +2979,9 @@ static void si_apply_state_adjust_rules(struct radeon_device *rdev, (rdev->pdev->device == 0x6605)) { max_sclk = 75000; } + + if (rdev->pm.dpm.high_pixelclock_count > 1) + disable_sclk_switching = true; } if (rps->vce_active) { diff --git a/drivers/gpu/drm/stm/ltdc.c b/drivers/gpu/drm/stm/ltdc.c index 7812094f93d6..6f3b523e16e8 100644 --- a/drivers/gpu/drm/stm/ltdc.c +++ b/drivers/gpu/drm/stm/ltdc.c @@ -525,13 +525,42 @@ static void ltdc_crtc_mode_set_nofb(struct drm_crtc *crtc) { struct ltdc_device *ldev = crtc_to_ltdc(crtc); struct drm_device *ddev = crtc->dev; + struct drm_connector_list_iter iter; + struct drm_connector *connector = NULL; + struct drm_encoder *encoder = NULL; + struct drm_bridge *bridge = NULL; struct drm_display_mode *mode = &crtc->state->adjusted_mode; struct videomode vm; u32 hsync, vsync, accum_hbp, accum_vbp, accum_act_w, accum_act_h; u32 total_width, total_height; + u32 bus_flags = 0; u32 val; int ret; + /* get encoder from crtc */ + drm_for_each_encoder(encoder, ddev) + if (encoder->crtc == crtc) + break; + + if (encoder) { + /* get bridge from encoder */ + list_for_each_entry(bridge, &encoder->bridge_chain, chain_node) + if (bridge->encoder == encoder) + break; + + /* Get the connector from encoder */ + drm_connector_list_iter_begin(ddev, &iter); + drm_for_each_connector_iter(connector, &iter) + if (connector->encoder == encoder) + break; + drm_connector_list_iter_end(&iter); + } + + if (bridge && bridge->timings) + bus_flags = bridge->timings->input_bus_flags; + else if (connector) + bus_flags = connector->display_info.bus_flags; + if (!pm_runtime_active(ddev->dev)) { ret = pm_runtime_get_sync(ddev->dev); if (ret) { @@ -567,10 +596,10 @@ static void ltdc_crtc_mode_set_nofb(struct drm_crtc *crtc) if (vm.flags & DISPLAY_FLAGS_VSYNC_HIGH) val |= GCR_VSPOL; - if (vm.flags & DISPLAY_FLAGS_DE_LOW) + if (bus_flags & DRM_BUS_FLAG_DE_LOW) val |= GCR_DEPOL; - if (vm.flags & DISPLAY_FLAGS_PIXDATA_NEGEDGE) + if (bus_flags & DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE) val |= GCR_PCPOL; reg_update_bits(ldev->regs, LTDC_GCR, diff --git a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c index 30213708fc99..d99afd19ca08 100644 --- a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c +++ b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c @@ -515,6 +515,15 @@ static void tilcdc_crtc_off(struct drm_crtc *crtc, bool shutdown) drm_crtc_vblank_off(crtc); + spin_lock_irq(&crtc->dev->event_lock); + + if (crtc->state->event) { + drm_crtc_send_vblank_event(crtc, crtc->state->event); + crtc->state->event = NULL; + } + + spin_unlock_irq(&crtc->dev->event_lock); + tilcdc_crtc_disable_irqs(dev); pm_runtime_put_sync(dev->dev); diff --git a/drivers/gpu/drm/virtio/virtgpu_ioctl.c b/drivers/gpu/drm/virtio/virtgpu_ioctl.c index 23eb6d772e40..669f2ee39515 100644 --- a/drivers/gpu/drm/virtio/virtgpu_ioctl.c +++ b/drivers/gpu/drm/virtio/virtgpu_ioctl.c @@ -174,7 +174,7 @@ static int virtio_gpu_execbuffer_ioctl(struct drm_device *dev, void *data, if (!sync_file) { dma_fence_put(&out_fence->f); ret = -ENOMEM; - goto out_memdup; + goto out_unresv; } exbuf->fence_fd = out_fence_fd; diff --git a/drivers/gpu/drm/virtio/virtgpu_object.c b/drivers/gpu/drm/virtio/virtgpu_object.c index d69a5b6da553..4ff1ec28e630 100644 --- a/drivers/gpu/drm/virtio/virtgpu_object.c +++ b/drivers/gpu/drm/virtio/virtgpu_object.c @@ -248,6 +248,7 @@ int virtio_gpu_object_create(struct virtio_gpu_device *vgdev, ret = virtio_gpu_object_shmem_init(vgdev, bo, &ents, &nents); if (ret != 0) { + virtio_gpu_array_put_free(objs); virtio_gpu_free_object(&shmem_obj->base); return ret; } diff --git a/drivers/gpu/drm/vkms/vkms_crtc.c b/drivers/gpu/drm/vkms/vkms_crtc.c index 0443b7deeaef..758d8a98d96b 100644 --- a/drivers/gpu/drm/vkms/vkms_crtc.c +++ b/drivers/gpu/drm/vkms/vkms_crtc.c @@ -18,7 +18,8 @@ static enum hrtimer_restart vkms_vblank_simulate(struct hrtimer *timer) ret_overrun = hrtimer_forward_now(&output->vblank_hrtimer, output->period_ns); - WARN_ON(ret_overrun != 1); + if (ret_overrun != 1) + pr_warn("%s: vblank timer overrun\n", __func__); spin_lock(&output->lock); ret = drm_crtc_handle_vblank(crtc); diff --git a/drivers/gpu/drm/xlnx/zynqmp_dp.c b/drivers/gpu/drm/xlnx/zynqmp_dp.c index 99158ee67d02..59d1fb017da0 100644 --- a/drivers/gpu/drm/xlnx/zynqmp_dp.c +++ b/drivers/gpu/drm/xlnx/zynqmp_dp.c @@ -866,7 +866,7 @@ static int zynqmp_dp_train(struct zynqmp_dp *dp) return ret; zynqmp_dp_write(dp, ZYNQMP_DP_SCRAMBLING_DISABLE, 1); - memset(dp->train_set, 0, 4); + memset(dp->train_set, 0, sizeof(dp->train_set)); ret = zynqmp_dp_link_train_cr(dp); if (ret) return ret; diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h index 67fd8a2f5aba..ba338973e968 100644 --- a/drivers/hid/hid-ids.h +++ b/drivers/hid/hid-ids.h @@ -946,6 +946,7 @@ #define USB_DEVICE_ID_ORTEK_IHOME_IMAC_A210S 0x8003 #define USB_VENDOR_ID_PLANTRONICS 0x047f +#define USB_DEVICE_ID_PLANTRONICS_BLACKWIRE_3220_SERIES 0xc056 #define USB_VENDOR_ID_PANASONIC 0x04da #define USB_DEVICE_ID_PANABOARD_UBT780 0x1044 diff --git a/drivers/hid/hid-lenovo.c b/drivers/hid/hid-lenovo.c index c6c8e20f3e8d..0ff03fed9770 100644 --- a/drivers/hid/hid-lenovo.c +++ b/drivers/hid/hid-lenovo.c @@ -33,6 +33,9 @@ #include "hid-ids.h" +/* Userspace expects F20 for mic-mute KEY_MICMUTE does not work */ +#define LENOVO_KEY_MICMUTE KEY_F20 + struct lenovo_drvdata { u8 led_report[3]; /* Must be first for proper alignment */ int led_state; @@ -62,8 +65,8 @@ struct lenovo_drvdata { #define TP10UBKBD_LED_OFF 1 #define TP10UBKBD_LED_ON 2 -static void lenovo_led_set_tp10ubkbd(struct hid_device *hdev, u8 led_code, - enum led_brightness value) +static int lenovo_led_set_tp10ubkbd(struct hid_device *hdev, u8 led_code, + enum led_brightness value) { struct lenovo_drvdata *data = hid_get_drvdata(hdev); int ret; @@ -75,10 +78,18 @@ static void lenovo_led_set_tp10ubkbd(struct hid_device *hdev, u8 led_code, data->led_report[2] = value ? TP10UBKBD_LED_ON : TP10UBKBD_LED_OFF; ret = hid_hw_raw_request(hdev, data->led_report[0], data->led_report, 3, HID_OUTPUT_REPORT, HID_REQ_SET_REPORT); - if (ret) - hid_err(hdev, "Set LED output report error: %d\n", ret); + if (ret != 3) { + if (ret != -ENODEV) + hid_err(hdev, "Set LED output report error: %d\n", ret); + + ret = ret < 0 ? ret : -EIO; + } else { + ret = 0; + } mutex_unlock(&data->led_report_mutex); + + return ret; } static void lenovo_tp10ubkbd_sync_fn_lock(struct work_struct *work) @@ -126,7 +137,7 @@ static int lenovo_input_mapping_tpkbd(struct hid_device *hdev, if (usage->hid == (HID_UP_BUTTON | 0x0010)) { /* This sub-device contains trackpoint, mark it */ hid_set_drvdata(hdev, (void *)1); - map_key_clear(KEY_MICMUTE); + map_key_clear(LENOVO_KEY_MICMUTE); return 1; } return 0; @@ -141,7 +152,7 @@ static int lenovo_input_mapping_cptkbd(struct hid_device *hdev, (usage->hid & HID_USAGE_PAGE) == HID_UP_LNVENDOR) { switch (usage->hid & HID_USAGE) { case 0x00f1: /* Fn-F4: Mic mute */ - map_key_clear(KEY_MICMUTE); + map_key_clear(LENOVO_KEY_MICMUTE); return 1; case 0x00f2: /* Fn-F5: Brightness down */ map_key_clear(KEY_BRIGHTNESSDOWN); @@ -231,7 +242,7 @@ static int lenovo_input_mapping_tp10_ultrabook_kbd(struct hid_device *hdev, map_key_clear(KEY_FN_ESC); return 1; case 9: /* Fn-F4: Mic mute */ - map_key_clear(KEY_MICMUTE); + map_key_clear(LENOVO_KEY_MICMUTE); return 1; case 10: /* Fn-F7: Control panel */ map_key_clear(KEY_CONFIG); @@ -349,7 +360,7 @@ static ssize_t attr_fn_lock_store(struct device *dev, { struct hid_device *hdev = to_hid_device(dev); struct lenovo_drvdata *data = hid_get_drvdata(hdev); - int value; + int value, ret; if (kstrtoint(buf, 10, &value)) return -EINVAL; @@ -364,7 +375,9 @@ static ssize_t attr_fn_lock_store(struct device *dev, lenovo_features_set_cptkbd(hdev); break; case USB_DEVICE_ID_LENOVO_TP10UBKBD: - lenovo_led_set_tp10ubkbd(hdev, TP10UBKBD_FN_LOCK_LED, value); + ret = lenovo_led_set_tp10ubkbd(hdev, TP10UBKBD_FN_LOCK_LED, value); + if (ret) + return ret; break; } @@ -498,6 +511,9 @@ static int lenovo_event_cptkbd(struct hid_device *hdev, static int lenovo_event(struct hid_device *hdev, struct hid_field *field, struct hid_usage *usage, __s32 value) { + if (!hid_get_drvdata(hdev)) + return 0; + switch (hdev->product) { case USB_DEVICE_ID_LENOVO_CUSBKBD: case USB_DEVICE_ID_LENOVO_CBTKBD: @@ -777,7 +793,7 @@ static enum led_brightness lenovo_led_brightness_get( : LED_OFF; } -static void lenovo_led_brightness_set(struct led_classdev *led_cdev, +static int lenovo_led_brightness_set(struct led_classdev *led_cdev, enum led_brightness value) { struct device *dev = led_cdev->dev->parent; @@ -785,6 +801,7 @@ static void lenovo_led_brightness_set(struct led_classdev *led_cdev, struct lenovo_drvdata *data_pointer = hid_get_drvdata(hdev); u8 tp10ubkbd_led[] = { TP10UBKBD_MUTE_LED, TP10UBKBD_MICMUTE_LED }; int led_nr = 0; + int ret = 0; if (led_cdev == &data_pointer->led_micmute) led_nr = 1; @@ -799,9 +816,11 @@ static void lenovo_led_brightness_set(struct led_classdev *led_cdev, lenovo_led_set_tpkbd(hdev); break; case USB_DEVICE_ID_LENOVO_TP10UBKBD: - lenovo_led_set_tp10ubkbd(hdev, tp10ubkbd_led[led_nr], value); + ret = lenovo_led_set_tp10ubkbd(hdev, tp10ubkbd_led[led_nr], value); break; } + + return ret; } static int lenovo_register_leds(struct hid_device *hdev) @@ -822,7 +841,8 @@ static int lenovo_register_leds(struct hid_device *hdev) data->led_mute.name = name_mute; data->led_mute.brightness_get = lenovo_led_brightness_get; - data->led_mute.brightness_set = lenovo_led_brightness_set; + data->led_mute.brightness_set_blocking = lenovo_led_brightness_set; + data->led_mute.flags = LED_HW_PLUGGABLE; data->led_mute.dev = &hdev->dev; ret = led_classdev_register(&hdev->dev, &data->led_mute); if (ret < 0) @@ -830,7 +850,8 @@ static int lenovo_register_leds(struct hid_device *hdev) data->led_micmute.name = name_micm; data->led_micmute.brightness_get = lenovo_led_brightness_get; - data->led_micmute.brightness_set = lenovo_led_brightness_set; + data->led_micmute.brightness_set_blocking = lenovo_led_brightness_set; + data->led_micmute.flags = LED_HW_PLUGGABLE; data->led_micmute.dev = &hdev->dev; ret = led_classdev_register(&hdev->dev, &data->led_micmute); if (ret < 0) { diff --git a/drivers/hid/hid-plantronics.c b/drivers/hid/hid-plantronics.c index 85b685efc12f..e81b7cec2d12 100644 --- a/drivers/hid/hid-plantronics.c +++ b/drivers/hid/hid-plantronics.c @@ -13,6 +13,7 @@ #include #include +#include #define PLT_HID_1_0_PAGE 0xffa00000 #define PLT_HID_2_0_PAGE 0xffa20000 @@ -36,6 +37,16 @@ #define PLT_ALLOW_CONSUMER (field->application == HID_CP_CONSUMERCONTROL && \ (usage->hid & HID_USAGE_PAGE) == HID_UP_CONSUMER) +#define PLT_QUIRK_DOUBLE_VOLUME_KEYS BIT(0) + +#define PLT_DOUBLE_KEY_TIMEOUT 5 /* ms */ + +struct plt_drv_data { + unsigned long device_type; + unsigned long last_volume_key_ts; + u32 quirks; +}; + static int plantronics_input_mapping(struct hid_device *hdev, struct hid_input *hi, struct hid_field *field, @@ -43,7 +54,8 @@ static int plantronics_input_mapping(struct hid_device *hdev, unsigned long **bit, int *max) { unsigned short mapped_key; - unsigned long plt_type = (unsigned long)hid_get_drvdata(hdev); + struct plt_drv_data *drv_data = hid_get_drvdata(hdev); + unsigned long plt_type = drv_data->device_type; /* special case for PTT products */ if (field->application == HID_GD_JOYSTICK) @@ -105,6 +117,30 @@ static int plantronics_input_mapping(struct hid_device *hdev, return 1; } +static int plantronics_event(struct hid_device *hdev, struct hid_field *field, + struct hid_usage *usage, __s32 value) +{ + struct plt_drv_data *drv_data = hid_get_drvdata(hdev); + + if (drv_data->quirks & PLT_QUIRK_DOUBLE_VOLUME_KEYS) { + unsigned long prev_ts, cur_ts; + + /* Usages are filtered in plantronics_usages. */ + + if (!value) /* Handle key presses only. */ + return 0; + + prev_ts = drv_data->last_volume_key_ts; + cur_ts = jiffies; + if (jiffies_to_msecs(cur_ts - prev_ts) <= PLT_DOUBLE_KEY_TIMEOUT) + return 1; /* Ignore the repeated key. */ + + drv_data->last_volume_key_ts = cur_ts; + } + + return 0; +} + static unsigned long plantronics_device_type(struct hid_device *hdev) { unsigned i, col_page; @@ -133,15 +169,24 @@ static unsigned long plantronics_device_type(struct hid_device *hdev) static int plantronics_probe(struct hid_device *hdev, const struct hid_device_id *id) { + struct plt_drv_data *drv_data; int ret; + drv_data = devm_kzalloc(&hdev->dev, sizeof(*drv_data), GFP_KERNEL); + if (!drv_data) + return -ENOMEM; + ret = hid_parse(hdev); if (ret) { hid_err(hdev, "parse failed\n"); goto err; } - hid_set_drvdata(hdev, (void *)plantronics_device_type(hdev)); + drv_data->device_type = plantronics_device_type(hdev); + drv_data->quirks = id->driver_data; + drv_data->last_volume_key_ts = jiffies - msecs_to_jiffies(PLT_DOUBLE_KEY_TIMEOUT); + + hid_set_drvdata(hdev, drv_data); ret = hid_hw_start(hdev, HID_CONNECT_DEFAULT | HID_CONNECT_HIDINPUT_FORCE | HID_CONNECT_HIDDEV_FORCE); @@ -153,15 +198,26 @@ static int plantronics_probe(struct hid_device *hdev, } static const struct hid_device_id plantronics_devices[] = { + { HID_USB_DEVICE(USB_VENDOR_ID_PLANTRONICS, + USB_DEVICE_ID_PLANTRONICS_BLACKWIRE_3220_SERIES), + .driver_data = PLT_QUIRK_DOUBLE_VOLUME_KEYS }, { HID_USB_DEVICE(USB_VENDOR_ID_PLANTRONICS, HID_ANY_ID) }, { } }; MODULE_DEVICE_TABLE(hid, plantronics_devices); +static const struct hid_usage_id plantronics_usages[] = { + { HID_CP_VOLUMEUP, EV_KEY, HID_ANY_ID }, + { HID_CP_VOLUMEDOWN, EV_KEY, HID_ANY_ID }, + { HID_TERMINATOR, HID_TERMINATOR, HID_TERMINATOR } +}; + static struct hid_driver plantronics_driver = { .name = "plantronics", .id_table = plantronics_devices, + .usage_table = plantronics_usages, .input_mapping = plantronics_input_mapping, + .event = plantronics_event, .probe = plantronics_probe, }; module_hid_driver(plantronics_driver); diff --git a/drivers/hsi/hsi_core.c b/drivers/hsi/hsi_core.c index c3fb5beb846e..ec90713564e3 100644 --- a/drivers/hsi/hsi_core.c +++ b/drivers/hsi/hsi_core.c @@ -210,8 +210,6 @@ static void hsi_add_client_from_dt(struct hsi_port *port, if (err) goto err; - dev_set_name(&cl->device, "%s", name); - err = hsi_of_property_parse_mode(client, "hsi-mode", &mode); if (err) { err = hsi_of_property_parse_mode(client, "hsi-rx-mode", @@ -293,6 +291,7 @@ static void hsi_add_client_from_dt(struct hsi_port *port, cl->device.release = hsi_client_release; cl->device.of_node = client; + dev_set_name(&cl->device, "%s", name); if (device_register(&cl->device) < 0) { pr_err("hsi: failed to register client: %s\n", name); put_device(&cl->device); diff --git a/drivers/hv/channel.c b/drivers/hv/channel.c index 0bd202de7960..945e41f5e3a8 100644 --- a/drivers/hv/channel.c +++ b/drivers/hv/channel.c @@ -653,7 +653,7 @@ static int __vmbus_open(struct vmbus_channel *newchannel, if (newchannel->rescind) { err = -ENODEV; - goto error_free_info; + goto error_clean_msglist; } err = vmbus_post_msg(open_msg, diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c index f0ed730e2e4e..ecebf1235fd5 100644 --- a/drivers/hv/channel_mgmt.c +++ b/drivers/hv/channel_mgmt.c @@ -756,6 +756,12 @@ static void init_vp_index(struct vmbus_channel *channel) free_cpumask_var(available_mask); } +#define UNLOAD_DELAY_UNIT_MS 10 /* 10 milliseconds */ +#define UNLOAD_WAIT_MS (100*1000) /* 100 seconds */ +#define UNLOAD_WAIT_LOOPS (UNLOAD_WAIT_MS/UNLOAD_DELAY_UNIT_MS) +#define UNLOAD_MSG_MS (5*1000) /* Every 5 seconds */ +#define UNLOAD_MSG_LOOPS (UNLOAD_MSG_MS/UNLOAD_DELAY_UNIT_MS) + static void vmbus_wait_for_unload(void) { int cpu; @@ -773,12 +779,17 @@ static void vmbus_wait_for_unload(void) * vmbus_connection.unload_event. If not, the last thing we can do is * read message pages for all CPUs directly. * - * Wait no more than 10 seconds so that the panic path can't get - * hung forever in case the response message isn't seen. + * Wait up to 100 seconds since an Azure host must writeback any dirty + * data in its disk cache before the VMbus UNLOAD request will + * complete. This flushing has been empirically observed to take up + * to 50 seconds in cases with a lot of dirty data, so allow additional + * leeway and for inaccuracies in mdelay(). But eventually time out so + * that the panic path can't get hung forever in case the response + * message isn't seen. */ - for (i = 0; i < 1000; i++) { + for (i = 1; i <= UNLOAD_WAIT_LOOPS; i++) { if (completion_done(&vmbus_connection.unload_event)) - break; + goto completed; for_each_online_cpu(cpu) { struct hv_per_cpu_context *hv_cpu @@ -801,9 +812,18 @@ static void vmbus_wait_for_unload(void) vmbus_signal_eom(msg, message_type); } - mdelay(10); + /* + * Give a notice periodically so someone watching the + * serial output won't think it is completely hung. + */ + if (!(i % UNLOAD_MSG_LOOPS)) + pr_notice("Waiting for VMBus UNLOAD to complete\n"); + + mdelay(UNLOAD_DELAY_UNIT_MS); } + pr_err("Continuing even though VMBus UNLOAD did not complete\n"); +completed: /* * We're crashing and already got the UNLOAD_RESPONSE, cleanup all * maybe-pending messages on all CPUs to be able to receive new diff --git a/drivers/hv/ring_buffer.c b/drivers/hv/ring_buffer.c index 35833d4d1a1d..ecd82ebfd5bc 100644 --- a/drivers/hv/ring_buffer.c +++ b/drivers/hv/ring_buffer.c @@ -313,7 +313,6 @@ int hv_ringbuffer_write(struct vmbus_channel *channel, rqst_id = vmbus_next_request_id(&channel->requestor, requestid); if (rqst_id == VMBUS_RQST_ERROR) { spin_unlock_irqrestore(&outring_info->ring_lock, flags); - pr_err("No request id available\n"); return -EAGAIN; } } diff --git a/drivers/hwmon/ltc2992.c b/drivers/hwmon/ltc2992.c index 4382105bf142..2a4bed0ab226 100644 --- a/drivers/hwmon/ltc2992.c +++ b/drivers/hwmon/ltc2992.c @@ -900,11 +900,15 @@ static int ltc2992_parse_dt(struct ltc2992_state *st) fwnode_for_each_available_child_node(fwnode, child) { ret = fwnode_property_read_u32(child, "reg", &addr); - if (ret < 0) + if (ret < 0) { + fwnode_handle_put(child); return ret; + } - if (addr > 1) + if (addr > 1) { + fwnode_handle_put(child); return -EINVAL; + } ret = fwnode_property_read_u32(child, "shunt-resistor-micro-ohms", &val); if (!ret) diff --git a/drivers/hwmon/occ/common.c b/drivers/hwmon/occ/common.c index 7a5e539b567b..580e63d7daa0 100644 --- a/drivers/hwmon/occ/common.c +++ b/drivers/hwmon/occ/common.c @@ -217,9 +217,9 @@ int occ_update_response(struct occ *occ) return rc; /* limit the maximum rate of polling the OCC */ - if (time_after(jiffies, occ->last_update + OCC_UPDATE_FREQUENCY)) { + if (time_after(jiffies, occ->next_update)) { rc = occ_poll(occ); - occ->last_update = jiffies; + occ->next_update = jiffies + OCC_UPDATE_FREQUENCY; } else { rc = occ->last_error; } @@ -1164,6 +1164,7 @@ int occ_setup(struct occ *occ, const char *name) return rc; } + occ->next_update = jiffies + OCC_UPDATE_FREQUENCY; occ_parse_poll_response(occ); rc = occ_setup_sensor_attrs(occ); diff --git a/drivers/hwmon/occ/common.h b/drivers/hwmon/occ/common.h index 67e6968b8978..e6df719770e8 100644 --- a/drivers/hwmon/occ/common.h +++ b/drivers/hwmon/occ/common.h @@ -99,7 +99,7 @@ struct occ { u8 poll_cmd_data; /* to perform OCC poll command */ int (*send_cmd)(struct occ *occ, u8 *cmd); - unsigned long last_update; + unsigned long next_update; struct mutex lock; /* lock OCC access */ struct device *hwmon; diff --git a/drivers/hwmon/pmbus/pxe1610.c b/drivers/hwmon/pmbus/pxe1610.c index da27ce34ee3f..eb4a06003b7f 100644 --- a/drivers/hwmon/pmbus/pxe1610.c +++ b/drivers/hwmon/pmbus/pxe1610.c @@ -41,6 +41,15 @@ static int pxe1610_identify(struct i2c_client *client, info->vrm_version[i] = vr13; break; default: + /* + * If prior pages are available limit operation + * to them + */ + if (i != 0) { + info->pages = i; + return 0; + } + return -ENODEV; } } diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c index 0f603b4094f2..a706ba11b93e 100644 --- a/drivers/hwtracing/coresight/coresight-etm-perf.c +++ b/drivers/hwtracing/coresight/coresight-etm-perf.c @@ -52,7 +52,7 @@ static ssize_t format_attr_contextid_show(struct device *dev, { int pid_fmt = ETM_OPT_CTXTID; -#if defined(CONFIG_CORESIGHT_SOURCE_ETM4X) +#if IS_ENABLED(CONFIG_CORESIGHT_SOURCE_ETM4X) pid_fmt = is_kernel_in_hyp_mode() ? ETM_OPT_CTXTID2 : ETM_OPT_CTXTID; #endif return sprintf(page, "config:%d\n", pid_fmt); diff --git a/drivers/hwtracing/coresight/coresight-platform.c b/drivers/hwtracing/coresight/coresight-platform.c index 3629b7885aca..c594f45319fc 100644 --- a/drivers/hwtracing/coresight/coresight-platform.c +++ b/drivers/hwtracing/coresight/coresight-platform.c @@ -90,6 +90,12 @@ static void of_coresight_get_ports_legacy(const struct device_node *node, struct of_endpoint endpoint; int in = 0, out = 0; + /* + * Avoid warnings in of_graph_get_next_endpoint() + * if the device doesn't have any graph connections + */ + if (!of_graph_is_present(node)) + return; do { ep = of_graph_get_next_endpoint(node, ep); if (!ep) diff --git a/drivers/hwtracing/intel_th/gth.c b/drivers/hwtracing/intel_th/gth.c index f72803a02391..28509b02a0b5 100644 --- a/drivers/hwtracing/intel_th/gth.c +++ b/drivers/hwtracing/intel_th/gth.c @@ -543,7 +543,7 @@ static void intel_th_gth_disable(struct intel_th_device *thdev, output->active = false; for_each_set_bit(master, gth->output[output->port].master, - TH_CONFIGURABLE_MASTERS) { + TH_CONFIGURABLE_MASTERS + 1) { gth_master_set(gth, master, -1); } spin_unlock(>h->gth_lock); @@ -697,7 +697,7 @@ static void intel_th_gth_unassign(struct intel_th_device *thdev, othdev->output.port = -1; othdev->output.active = false; gth->output[port].output = NULL; - for (master = 0; master <= TH_CONFIGURABLE_MASTERS; master++) + for (master = 0; master < TH_CONFIGURABLE_MASTERS + 1; master++) if (gth->master[master] == port) gth->master[master] = -1; spin_unlock(>h->gth_lock); diff --git a/drivers/hwtracing/intel_th/pci.c b/drivers/hwtracing/intel_th/pci.c index 251e75c9ba9d..817cdb29bbd8 100644 --- a/drivers/hwtracing/intel_th/pci.c +++ b/drivers/hwtracing/intel_th/pci.c @@ -273,11 +273,21 @@ static const struct pci_device_id intel_th_pci_id_table[] = { PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x51a6), .driver_data = (kernel_ulong_t)&intel_th_2x, }, + { + /* Alder Lake-M */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x54a6), + .driver_data = (kernel_ulong_t)&intel_th_2x, + }, { /* Alder Lake CPU */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x466f), .driver_data = (kernel_ulong_t)&intel_th_2x, }, + { + /* Rocket Lake CPU */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x4c19), + .driver_data = (kernel_ulong_t)&intel_th_2x, + }, { 0 }, }; diff --git a/drivers/i2c/busses/i2c-cadence.c b/drivers/i2c/busses/i2c-cadence.c index e4b7f2a951ad..c1bbc4caeb5c 100644 --- a/drivers/i2c/busses/i2c-cadence.c +++ b/drivers/i2c/busses/i2c-cadence.c @@ -789,7 +789,7 @@ static int cdns_i2c_master_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, bool change_role = false; #endif - ret = pm_runtime_get_sync(id->dev); + ret = pm_runtime_resume_and_get(id->dev); if (ret < 0) return ret; @@ -911,7 +911,7 @@ static int cdns_reg_slave(struct i2c_client *slave) if (slave->flags & I2C_CLIENT_TEN) return -EAFNOSUPPORT; - ret = pm_runtime_get_sync(id->dev); + ret = pm_runtime_resume_and_get(id->dev); if (ret < 0) return ret; @@ -1200,7 +1200,10 @@ static int cdns_i2c_probe(struct platform_device *pdev) if (IS_ERR(id->membase)) return PTR_ERR(id->membase); - id->irq = platform_get_irq(pdev, 0); + ret = platform_get_irq(pdev, 0); + if (ret < 0) + return ret; + id->irq = ret; id->adap.owner = THIS_MODULE; id->adap.dev.of_node = pdev->dev.of_node; diff --git a/drivers/i2c/busses/i2c-emev2.c b/drivers/i2c/busses/i2c-emev2.c index a08554c1a570..bdff0e6345d9 100644 --- a/drivers/i2c/busses/i2c-emev2.c +++ b/drivers/i2c/busses/i2c-emev2.c @@ -395,7 +395,10 @@ static int em_i2c_probe(struct platform_device *pdev) em_i2c_reset(&priv->adap); - priv->irq = platform_get_irq(pdev, 0); + ret = platform_get_irq(pdev, 0); + if (ret < 0) + goto err_clk; + priv->irq = ret; ret = devm_request_irq(&pdev->dev, priv->irq, em_i2c_irq_handler, 0, "em_i2c", priv); if (ret) diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c index 4acee6f9e5a3..99d446763530 100644 --- a/drivers/i2c/busses/i2c-i801.c +++ b/drivers/i2c/busses/i2c-i801.c @@ -73,6 +73,7 @@ * Comet Lake-V (PCH) 0xa3a3 32 hard yes yes yes * Alder Lake-S (PCH) 0x7aa3 32 hard yes yes yes * Alder Lake-P (PCH) 0x51a3 32 hard yes yes yes + * Alder Lake-M (PCH) 0x54a3 32 hard yes yes yes * * Features supported by this driver: * Software PEC no @@ -230,6 +231,7 @@ #define PCI_DEVICE_ID_INTEL_ELKHART_LAKE_SMBUS 0x4b23 #define PCI_DEVICE_ID_INTEL_JASPER_LAKE_SMBUS 0x4da3 #define PCI_DEVICE_ID_INTEL_ALDER_LAKE_P_SMBUS 0x51a3 +#define PCI_DEVICE_ID_INTEL_ALDER_LAKE_M_SMBUS 0x54a3 #define PCI_DEVICE_ID_INTEL_BROXTON_SMBUS 0x5ad4 #define PCI_DEVICE_ID_INTEL_ALDER_LAKE_S_SMBUS 0x7aa3 #define PCI_DEVICE_ID_INTEL_LYNXPOINT_SMBUS 0x8c22 @@ -1087,6 +1089,7 @@ static const struct pci_device_id i801_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_JASPER_LAKE_SMBUS) }, { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ALDER_LAKE_S_SMBUS) }, { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ALDER_LAKE_P_SMBUS) }, + { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ALDER_LAKE_M_SMBUS) }, { 0, } }; @@ -1771,6 +1774,7 @@ static int i801_probe(struct pci_dev *dev, const struct pci_device_id *id) case PCI_DEVICE_ID_INTEL_EBG_SMBUS: case PCI_DEVICE_ID_INTEL_ALDER_LAKE_S_SMBUS: case PCI_DEVICE_ID_INTEL_ALDER_LAKE_P_SMBUS: + case PCI_DEVICE_ID_INTEL_ALDER_LAKE_M_SMBUS: priv->features |= FEATURE_BLOCK_PROC; priv->features |= FEATURE_I2C_BLOCK_READ; priv->features |= FEATURE_IRQ; diff --git a/drivers/i2c/busses/i2c-img-scb.c b/drivers/i2c/busses/i2c-img-scb.c index 98a89301ed2a..8e987945ed45 100644 --- a/drivers/i2c/busses/i2c-img-scb.c +++ b/drivers/i2c/busses/i2c-img-scb.c @@ -1057,7 +1057,7 @@ static int img_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, atomic = true; } - ret = pm_runtime_get_sync(adap->dev.parent); + ret = pm_runtime_resume_and_get(adap->dev.parent); if (ret < 0) return ret; @@ -1158,7 +1158,7 @@ static int img_i2c_init(struct img_i2c *i2c) u32 rev; int ret; - ret = pm_runtime_get_sync(i2c->adap.dev.parent); + ret = pm_runtime_resume_and_get(i2c->adap.dev.parent); if (ret < 0) return ret; diff --git a/drivers/i2c/busses/i2c-imx-lpi2c.c b/drivers/i2c/busses/i2c-imx-lpi2c.c index 9db6ccded5e9..8b9ba055c418 100644 --- a/drivers/i2c/busses/i2c-imx-lpi2c.c +++ b/drivers/i2c/busses/i2c-imx-lpi2c.c @@ -259,7 +259,7 @@ static int lpi2c_imx_master_enable(struct lpi2c_imx_struct *lpi2c_imx) unsigned int temp; int ret; - ret = pm_runtime_get_sync(lpi2c_imx->adapter.dev.parent); + ret = pm_runtime_resume_and_get(lpi2c_imx->adapter.dev.parent); if (ret < 0) return ret; diff --git a/drivers/i2c/busses/i2c-imx.c b/drivers/i2c/busses/i2c-imx.c index b80fdc1f0092..dc5ca71906db 100644 --- a/drivers/i2c/busses/i2c-imx.c +++ b/drivers/i2c/busses/i2c-imx.c @@ -801,7 +801,7 @@ static int i2c_imx_reg_slave(struct i2c_client *client) i2c_imx->last_slave_event = I2C_SLAVE_STOP; /* Resume */ - ret = pm_runtime_get_sync(i2c_imx->adapter.dev.parent); + ret = pm_runtime_resume_and_get(i2c_imx->adapter.dev.parent); if (ret < 0) { dev_err(&i2c_imx->adapter.dev, "failed to resume i2c controller"); return ret; @@ -1253,7 +1253,7 @@ static int i2c_imx_xfer(struct i2c_adapter *adapter, struct imx_i2c_struct *i2c_imx = i2c_get_adapdata(adapter); int result; - result = pm_runtime_get_sync(i2c_imx->adapter.dev.parent); + result = pm_runtime_resume_and_get(i2c_imx->adapter.dev.parent); if (result < 0) return result; @@ -1496,7 +1496,7 @@ static int i2c_imx_remove(struct platform_device *pdev) struct imx_i2c_struct *i2c_imx = platform_get_drvdata(pdev); int irq, ret; - ret = pm_runtime_get_sync(&pdev->dev); + ret = pm_runtime_resume_and_get(&pdev->dev); if (ret < 0) return ret; diff --git a/drivers/i2c/busses/i2c-jz4780.c b/drivers/i2c/busses/i2c-jz4780.c index 55177eb21d7b..baa7319eee53 100644 --- a/drivers/i2c/busses/i2c-jz4780.c +++ b/drivers/i2c/busses/i2c-jz4780.c @@ -825,7 +825,10 @@ static int jz4780_i2c_probe(struct platform_device *pdev) jz4780_i2c_writew(i2c, JZ4780_I2C_INTM, 0x0); - i2c->irq = platform_get_irq(pdev, 0); + ret = platform_get_irq(pdev, 0); + if (ret < 0) + goto err; + i2c->irq = ret; ret = devm_request_irq(&pdev->dev, i2c->irq, jz4780_i2c_irq, 0, dev_name(&pdev->dev), i2c); if (ret) diff --git a/drivers/i2c/busses/i2c-mlxbf.c b/drivers/i2c/busses/i2c-mlxbf.c index 2fb0532d8a16..ab261d762dea 100644 --- a/drivers/i2c/busses/i2c-mlxbf.c +++ b/drivers/i2c/busses/i2c-mlxbf.c @@ -2376,6 +2376,8 @@ static int mlxbf_i2c_probe(struct platform_device *pdev) mlxbf_i2c_init_slave(pdev, priv); irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; ret = devm_request_irq(dev, irq, mlxbf_smbus_irq, IRQF_ONESHOT | IRQF_SHARED | IRQF_PROBE_SHARED, dev_name(dev), priv); diff --git a/drivers/i2c/busses/i2c-mt65xx.c b/drivers/i2c/busses/i2c-mt65xx.c index 2ffd2f354d0a..bf25acba2ed5 100644 --- a/drivers/i2c/busses/i2c-mt65xx.c +++ b/drivers/i2c/busses/i2c-mt65xx.c @@ -479,7 +479,7 @@ static void mtk_i2c_init_hw(struct mtk_i2c *i2c) { u16 control_reg; - if (i2c->dev_comp->dma_sync) { + if (i2c->dev_comp->apdma_sync) { writel(I2C_DMA_WARM_RST, i2c->pdmabase + OFFSET_RST); udelay(10); writel(I2C_DMA_CLR_FLAG, i2c->pdmabase + OFFSET_RST); @@ -564,7 +564,7 @@ static const struct i2c_spec_values *mtk_i2c_get_spec(unsigned int speed) static int mtk_i2c_max_step_cnt(unsigned int target_speed) { - if (target_speed > I2C_MAX_FAST_MODE_FREQ) + if (target_speed > I2C_MAX_FAST_MODE_PLUS_FREQ) return MAX_HS_STEP_CNT_DIV; else return MAX_STEP_CNT_DIV; @@ -635,7 +635,7 @@ static int mtk_i2c_check_ac_timing(struct mtk_i2c *i2c, if (sda_min > sda_max) return -3; - if (check_speed > I2C_MAX_FAST_MODE_FREQ) { + if (check_speed > I2C_MAX_FAST_MODE_PLUS_FREQ) { if (i2c->dev_comp->ltiming_adjust) { i2c->ac_timing.hs = I2C_TIME_DEFAULT_VALUE | (sample_cnt << 12) | (high_cnt << 8); @@ -850,7 +850,7 @@ static int mtk_i2c_do_transfer(struct mtk_i2c *i2c, struct i2c_msg *msgs, control_reg = mtk_i2c_readw(i2c, OFFSET_CONTROL) & ~(I2C_CONTROL_DIR_CHANGE | I2C_CONTROL_RS); - if ((i2c->speed_hz > I2C_MAX_FAST_MODE_FREQ) || (left_num >= 1)) + if ((i2c->speed_hz > I2C_MAX_FAST_MODE_PLUS_FREQ) || (left_num >= 1)) control_reg |= I2C_CONTROL_RS; if (i2c->op == I2C_MASTER_WRRD) @@ -1067,7 +1067,8 @@ static int mtk_i2c_transfer(struct i2c_adapter *adap, } } - if (i2c->auto_restart && num >= 2 && i2c->speed_hz > I2C_MAX_FAST_MODE_FREQ) + if (i2c->auto_restart && num >= 2 && + i2c->speed_hz > I2C_MAX_FAST_MODE_PLUS_FREQ) /* ignore the first restart irq after the master code, * otherwise the first transfer will be discarded. */ diff --git a/drivers/i2c/busses/i2c-omap.c b/drivers/i2c/busses/i2c-omap.c index 12ac4212aded..d4f6c6d60683 100644 --- a/drivers/i2c/busses/i2c-omap.c +++ b/drivers/i2c/busses/i2c-omap.c @@ -1404,9 +1404,9 @@ omap_i2c_probe(struct platform_device *pdev) pm_runtime_set_autosuspend_delay(omap->dev, OMAP_I2C_PM_TIMEOUT); pm_runtime_use_autosuspend(omap->dev); - r = pm_runtime_get_sync(omap->dev); + r = pm_runtime_resume_and_get(omap->dev); if (r < 0) - goto err_free_mem; + goto err_disable_pm; /* * Read the Rev hi bit-[15:14] ie scheme this is 1 indicates ver2. @@ -1513,8 +1513,8 @@ omap_i2c_probe(struct platform_device *pdev) omap_i2c_write_reg(omap, OMAP_I2C_CON_REG, 0); pm_runtime_dont_use_autosuspend(omap->dev); pm_runtime_put_sync(omap->dev); +err_disable_pm: pm_runtime_disable(&pdev->dev); -err_free_mem: return r; } @@ -1525,7 +1525,7 @@ static int omap_i2c_remove(struct platform_device *pdev) int ret; i2c_del_adapter(&omap->adapter); - ret = pm_runtime_get_sync(&pdev->dev); + ret = pm_runtime_resume_and_get(&pdev->dev); if (ret < 0) return ret; diff --git a/drivers/i2c/busses/i2c-rcar.c b/drivers/i2c/busses/i2c-rcar.c index 12f6d452c0f7..8722ca23f889 100644 --- a/drivers/i2c/busses/i2c-rcar.c +++ b/drivers/i2c/busses/i2c-rcar.c @@ -1027,7 +1027,10 @@ static int rcar_i2c_probe(struct platform_device *pdev) if (of_property_read_bool(dev->of_node, "smbus")) priv->flags |= ID_P_HOST_NOTIFY; - priv->irq = platform_get_irq(pdev, 0); + ret = platform_get_irq(pdev, 0); + if (ret < 0) + goto out_pm_disable; + priv->irq = ret; ret = devm_request_irq(dev, priv->irq, irqhandler, irqflags, dev_name(dev), priv); if (ret < 0) { dev_err(dev, "cannot get irq %d\n", priv->irq); diff --git a/drivers/i2c/busses/i2c-sh7760.c b/drivers/i2c/busses/i2c-sh7760.c index c2005c789d2b..319d1fa617c8 100644 --- a/drivers/i2c/busses/i2c-sh7760.c +++ b/drivers/i2c/busses/i2c-sh7760.c @@ -471,7 +471,10 @@ static int sh7760_i2c_probe(struct platform_device *pdev) goto out2; } - id->irq = platform_get_irq(pdev, 0); + ret = platform_get_irq(pdev, 0); + if (ret < 0) + goto out3; + id->irq = ret; id->adap.nr = pdev->id; id->adap.algo = &sh7760_i2c_algo; diff --git a/drivers/i2c/busses/i2c-sprd.c b/drivers/i2c/busses/i2c-sprd.c index 2917fecf6c80..8ead7e021008 100644 --- a/drivers/i2c/busses/i2c-sprd.c +++ b/drivers/i2c/busses/i2c-sprd.c @@ -290,7 +290,7 @@ static int sprd_i2c_master_xfer(struct i2c_adapter *i2c_adap, struct sprd_i2c *i2c_dev = i2c_adap->algo_data; int im, ret; - ret = pm_runtime_get_sync(i2c_dev->dev); + ret = pm_runtime_resume_and_get(i2c_dev->dev); if (ret < 0) return ret; @@ -576,7 +576,7 @@ static int sprd_i2c_remove(struct platform_device *pdev) struct sprd_i2c *i2c_dev = platform_get_drvdata(pdev); int ret; - ret = pm_runtime_get_sync(i2c_dev->dev); + ret = pm_runtime_resume_and_get(i2c_dev->dev); if (ret < 0) return ret; diff --git a/drivers/i2c/busses/i2c-stm32f7.c b/drivers/i2c/busses/i2c-stm32f7.c index c62c815b88eb..318abfa7926b 100644 --- a/drivers/i2c/busses/i2c-stm32f7.c +++ b/drivers/i2c/busses/i2c-stm32f7.c @@ -1652,7 +1652,7 @@ static int stm32f7_i2c_xfer(struct i2c_adapter *i2c_adap, i2c_dev->msg_id = 0; f7_msg->smbus = false; - ret = pm_runtime_get_sync(i2c_dev->dev); + ret = pm_runtime_resume_and_get(i2c_dev->dev); if (ret < 0) return ret; @@ -1698,7 +1698,7 @@ static int stm32f7_i2c_smbus_xfer(struct i2c_adapter *adapter, u16 addr, f7_msg->read_write = read_write; f7_msg->smbus = true; - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0) return ret; @@ -1799,7 +1799,7 @@ static int stm32f7_i2c_reg_slave(struct i2c_client *slave) if (ret) return ret; - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0) return ret; @@ -1880,7 +1880,7 @@ static int stm32f7_i2c_unreg_slave(struct i2c_client *slave) WARN_ON(!i2c_dev->slave[id]); - ret = pm_runtime_get_sync(i2c_dev->dev); + ret = pm_runtime_resume_and_get(i2c_dev->dev); if (ret < 0) return ret; @@ -2273,7 +2273,7 @@ static int stm32f7_i2c_regs_backup(struct stm32f7_i2c_dev *i2c_dev) int ret; struct stm32f7_i2c_regs *backup_regs = &i2c_dev->backup_regs; - ret = pm_runtime_get_sync(i2c_dev->dev); + ret = pm_runtime_resume_and_get(i2c_dev->dev); if (ret < 0) return ret; @@ -2295,7 +2295,7 @@ static int stm32f7_i2c_regs_restore(struct stm32f7_i2c_dev *i2c_dev) int ret; struct stm32f7_i2c_regs *backup_regs = &i2c_dev->backup_regs; - ret = pm_runtime_get_sync(i2c_dev->dev); + ret = pm_runtime_resume_and_get(i2c_dev->dev); if (ret < 0) return ret; diff --git a/drivers/i2c/busses/i2c-xiic.c b/drivers/i2c/busses/i2c-xiic.c index 087b2951942e..2a8568b97c14 100644 --- a/drivers/i2c/busses/i2c-xiic.c +++ b/drivers/i2c/busses/i2c-xiic.c @@ -706,7 +706,7 @@ static int xiic_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num) dev_dbg(adap->dev.parent, "%s entry SR: 0x%x\n", __func__, xiic_getreg8(i2c, XIIC_SR_REG_OFFSET)); - err = pm_runtime_get_sync(i2c->dev); + err = pm_runtime_resume_and_get(i2c->dev); if (err < 0) return err; @@ -873,7 +873,7 @@ static int xiic_i2c_remove(struct platform_device *pdev) /* remove adapter & data */ i2c_del_adapter(&i2c->adap); - ret = pm_runtime_get_sync(i2c->dev); + ret = pm_runtime_resume_and_get(i2c->dev); if (ret < 0) return ret; diff --git a/drivers/i2c/i2c-core-base.c b/drivers/i2c/i2c-core-base.c index f21362355973..0ff6d41d1d7d 100644 --- a/drivers/i2c/i2c-core-base.c +++ b/drivers/i2c/i2c-core-base.c @@ -1388,7 +1388,7 @@ int i2c_handle_smbus_host_notify(struct i2c_adapter *adap, unsigned short addr) if (irq <= 0) return -ENXIO; - generic_handle_irq(irq); + generic_dispatch_irq(irq); return 0; } diff --git a/drivers/i2c/i2c-dev.c b/drivers/i2c/i2c-dev.c index 6ceb11cc4be1..6ef38a8ee95c 100644 --- a/drivers/i2c/i2c-dev.c +++ b/drivers/i2c/i2c-dev.c @@ -440,8 +440,13 @@ static long i2cdev_ioctl(struct file *file, unsigned int cmd, unsigned long arg) sizeof(rdwr_arg))) return -EFAULT; - /* Put an arbitrary limit on the number of messages that can - * be sent at once */ + if (!rdwr_arg.msgs || rdwr_arg.nmsgs == 0) + return -EINVAL; + + /* + * Put an arbitrary limit on the number of messages that can + * be sent at once + */ if (rdwr_arg.nmsgs > I2C_RDWR_IOCTL_MAX_MSGS) return -EINVAL; diff --git a/drivers/i3c/master.c b/drivers/i3c/master.c index f8e9b7305c13..e2e12a5585e5 100644 --- a/drivers/i3c/master.c +++ b/drivers/i3c/master.c @@ -2535,7 +2535,7 @@ int i3c_master_register(struct i3c_master_controller *master, ret = i3c_master_bus_init(master); if (ret) - goto err_destroy_wq; + goto err_put_dev; ret = device_add(&master->dev); if (ret) @@ -2566,9 +2566,6 @@ int i3c_master_register(struct i3c_master_controller *master, err_cleanup_bus: i3c_master_bus_cleanup(master); -err_destroy_wq: - destroy_workqueue(master->wq); - err_put_dev: put_device(&master->dev); diff --git a/drivers/iio/accel/Kconfig b/drivers/iio/accel/Kconfig index 2e0c62c39155..8acf277b8b25 100644 --- a/drivers/iio/accel/Kconfig +++ b/drivers/iio/accel/Kconfig @@ -211,7 +211,6 @@ config DMARD10 config HID_SENSOR_ACCEL_3D depends on HID_SENSOR_HUB select IIO_BUFFER - select IIO_TRIGGERED_BUFFER select HID_SENSOR_IIO_COMMON select HID_SENSOR_IIO_TRIGGER tristate "HID Accelerometers 3D" diff --git a/drivers/iio/accel/adis16201.c b/drivers/iio/accel/adis16201.c index 3633a4e302c6..fe225990de24 100644 --- a/drivers/iio/accel/adis16201.c +++ b/drivers/iio/accel/adis16201.c @@ -215,7 +215,7 @@ static const struct iio_chan_spec adis16201_channels[] = { ADIS_AUX_ADC_CHAN(ADIS16201_AUX_ADC_REG, ADIS16201_SCAN_AUX_ADC, 0, 12), ADIS_INCLI_CHAN(X, ADIS16201_XINCL_OUT_REG, ADIS16201_SCAN_INCLI_X, BIT(IIO_CHAN_INFO_CALIBBIAS), 0, 14), - ADIS_INCLI_CHAN(X, ADIS16201_YINCL_OUT_REG, ADIS16201_SCAN_INCLI_Y, + ADIS_INCLI_CHAN(Y, ADIS16201_YINCL_OUT_REG, ADIS16201_SCAN_INCLI_Y, BIT(IIO_CHAN_INFO_CALIBBIAS), 0, 14), IIO_CHAN_SOFT_TIMESTAMP(7) }; diff --git a/drivers/iio/adc/Kconfig b/drivers/iio/adc/Kconfig index e0667c4b3c08..dda0f1e37ec1 100644 --- a/drivers/iio/adc/Kconfig +++ b/drivers/iio/adc/Kconfig @@ -249,7 +249,7 @@ config AD799X config AD9467 tristate "Analog Devices AD9467 High Speed ADC driver" depends on SPI - select ADI_AXI_ADC + depends on ADI_AXI_ADC help Say yes here to build support for Analog Devices: * AD9467 16-Bit, 200 MSPS/250 MSPS Analog-to-Digital Converter diff --git a/drivers/iio/adc/ad7476.c b/drivers/iio/adc/ad7476.c index 17402714b387..9e9ff07cf972 100644 --- a/drivers/iio/adc/ad7476.c +++ b/drivers/iio/adc/ad7476.c @@ -321,25 +321,15 @@ static int ad7476_probe(struct spi_device *spi) spi_message_init(&st->msg); spi_message_add_tail(&st->xfer, &st->msg); - ret = iio_triggered_buffer_setup(indio_dev, NULL, - &ad7476_trigger_handler, NULL); + ret = devm_iio_triggered_buffer_setup(&spi->dev, indio_dev, NULL, + &ad7476_trigger_handler, NULL); if (ret) - goto error_disable_reg; + return ret; if (st->chip_info->reset) st->chip_info->reset(st); - ret = iio_device_register(indio_dev); - if (ret) - goto error_ring_unregister; - return 0; - -error_ring_unregister: - iio_triggered_buffer_cleanup(indio_dev); -error_disable_reg: - regulator_disable(st->reg); - - return ret; + return devm_iio_device_register(&spi->dev, indio_dev); } static const struct spi_device_id ad7476_id[] = { diff --git a/drivers/iio/common/hid-sensors/Kconfig b/drivers/iio/common/hid-sensors/Kconfig index 24d492567336..2a3dd3b907be 100644 --- a/drivers/iio/common/hid-sensors/Kconfig +++ b/drivers/iio/common/hid-sensors/Kconfig @@ -19,6 +19,7 @@ config HID_SENSOR_IIO_TRIGGER tristate "Common module (trigger) for all HID Sensor IIO drivers" depends on HID_SENSOR_HUB && HID_SENSOR_IIO_COMMON && IIO_BUFFER select IIO_TRIGGER + select IIO_TRIGGERED_BUFFER help Say yes here to build trigger support for HID sensors. Triggers will be send if all requested attributes were read. diff --git a/drivers/iio/gyro/Kconfig b/drivers/iio/gyro/Kconfig index 5824f2edf975..20b5ac7ab66a 100644 --- a/drivers/iio/gyro/Kconfig +++ b/drivers/iio/gyro/Kconfig @@ -111,7 +111,6 @@ config FXAS21002C_SPI config HID_SENSOR_GYRO_3D depends on HID_SENSOR_HUB select IIO_BUFFER - select IIO_TRIGGERED_BUFFER select HID_SENSOR_IIO_COMMON select HID_SENSOR_IIO_TRIGGER tristate "HID Gyroscope 3D" diff --git a/drivers/iio/gyro/mpu3050-core.c b/drivers/iio/gyro/mpu3050-core.c index ac90be03332a..f17a93519535 100644 --- a/drivers/iio/gyro/mpu3050-core.c +++ b/drivers/iio/gyro/mpu3050-core.c @@ -272,7 +272,16 @@ static int mpu3050_read_raw(struct iio_dev *indio_dev, case IIO_CHAN_INFO_OFFSET: switch (chan->type) { case IIO_TEMP: - /* The temperature scaling is (x+23000)/280 Celsius */ + /* + * The temperature scaling is (x+23000)/280 Celsius + * for the "best fit straight line" temperature range + * of -30C..85C. The 23000 includes room temperature + * offset of +35C, 280 is the precision scale and x is + * the 16-bit signed integer reported by hardware. + * + * Temperature value itself represents temperature of + * the sensor die. + */ *val = 23000; return IIO_VAL_INT; default: @@ -329,7 +338,7 @@ static int mpu3050_read_raw(struct iio_dev *indio_dev, goto out_read_raw_unlock; } - *val = be16_to_cpu(raw_val); + *val = (s16)be16_to_cpu(raw_val); ret = IIO_VAL_INT; goto out_read_raw_unlock; diff --git a/drivers/iio/humidity/Kconfig b/drivers/iio/humidity/Kconfig index 6549fcf6db69..2de5494e7c22 100644 --- a/drivers/iio/humidity/Kconfig +++ b/drivers/iio/humidity/Kconfig @@ -52,7 +52,6 @@ config HID_SENSOR_HUMIDITY tristate "HID Environmental humidity sensor" depends on HID_SENSOR_HUB select IIO_BUFFER - select IIO_TRIGGERED_BUFFER select HID_SENSOR_IIO_COMMON select HID_SENSOR_IIO_TRIGGER help diff --git a/drivers/iio/imu/adis16480.c b/drivers/iio/imu/adis16480.c index dfe86c589325..c41b8ef1e250 100644 --- a/drivers/iio/imu/adis16480.c +++ b/drivers/iio/imu/adis16480.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -17,6 +18,7 @@ #include #include #include +#include #include #include @@ -170,6 +172,11 @@ static const char * const adis16480_int_pin_names[4] = { [ADIS16480_PIN_DIO4] = "DIO4", }; +static bool low_rate_allow; +module_param(low_rate_allow, bool, 0444); +MODULE_PARM_DESC(low_rate_allow, + "Allow IMU rates below the minimum advisable when external clk is used in PPS mode (default: N)"); + #ifdef CONFIG_DEBUG_FS static ssize_t adis16480_show_firmware_revision(struct file *file, @@ -312,7 +319,8 @@ static int adis16480_debugfs_init(struct iio_dev *indio_dev) static int adis16480_set_freq(struct iio_dev *indio_dev, int val, int val2) { struct adis16480 *st = iio_priv(indio_dev); - unsigned int t, reg; + unsigned int t, sample_rate = st->clk_freq; + int ret; if (val < 0 || val2 < 0) return -EINVAL; @@ -321,28 +329,65 @@ static int adis16480_set_freq(struct iio_dev *indio_dev, int val, int val2) if (t == 0) return -EINVAL; + mutex_lock(&st->adis.state_lock); /* - * When using PPS mode, the rate of data collection is equal to the - * product of the external clock frequency and the scale factor in the - * SYNC_SCALE register. - * When using sync mode, or internal clock, the output data rate is - * equal with the clock frequency divided by DEC_RATE + 1. + * When using PPS mode, the input clock needs to be scaled so that we have an IMU + * sample rate between (optimally) 4000 and 4250. After this, we can use the + * decimation filter to lower the sampling rate in order to get what the user wants. + * Optimally, the user sample rate is a multiple of both the IMU sample rate and + * the input clock. Hence, calculating the sync_scale dynamically gives us better + * chances of achieving a perfect/integer value for DEC_RATE. The math here is: + * 1. lcm of the input clock and the desired output rate. + * 2. get the highest multiple of the previous result lower than the adis max rate. + * 3. The last result becomes the IMU sample rate. Use that to calculate SYNC_SCALE + * and DEC_RATE (to get the user output rate) */ if (st->clk_mode == ADIS16480_CLK_PPS) { - t = t / st->clk_freq; - reg = ADIS16495_REG_SYNC_SCALE; - } else { - t = st->clk_freq / t; - reg = ADIS16480_REG_DEC_RATE; + unsigned long scaled_rate = lcm(st->clk_freq, t); + int sync_scale; + + /* + * If lcm is bigger than the IMU maximum sampling rate there's no perfect + * solution. In this case, we get the highest multiple of the input clock + * lower than the IMU max sample rate. + */ + if (scaled_rate > st->chip_info->int_clk) + scaled_rate = st->chip_info->int_clk / st->clk_freq * st->clk_freq; + else + scaled_rate = st->chip_info->int_clk / scaled_rate * scaled_rate; + + /* + * This is not an hard requirement but it's not advised to run the IMU + * with a sample rate lower than 4000Hz due to possible undersampling + * issues. However, there are users that might really want to take the risk. + * Hence, we provide a module parameter for them. If set, we allow sample + * rates lower than 4KHz. By default, we won't allow this and we just roundup + * the rate to the next multiple of the input clock bigger than 4KHz. This + * is done like this as in some cases (when DEC_RATE is 0) might give + * us the closest value to the one desired by the user... + */ + if (scaled_rate < 4000000 && !low_rate_allow) + scaled_rate = roundup(4000000, st->clk_freq); + + sync_scale = scaled_rate / st->clk_freq; + ret = __adis_write_reg_16(&st->adis, ADIS16495_REG_SYNC_SCALE, sync_scale); + if (ret) + goto error; + + sample_rate = scaled_rate; } + t = DIV_ROUND_CLOSEST(sample_rate, t); + if (t) + t--; + if (t > st->chip_info->max_dec_rate) t = st->chip_info->max_dec_rate; - if ((t != 0) && (st->clk_mode != ADIS16480_CLK_PPS)) - t--; - - return adis_write_reg_16(&st->adis, reg, t); + ret = __adis_write_reg_16(&st->adis, ADIS16480_REG_DEC_RATE, t); +error: + mutex_unlock(&st->adis.state_lock); + return ret; } static int adis16480_get_freq(struct iio_dev *indio_dev, int *val, int *val2) @@ -350,34 +395,35 @@ static int adis16480_get_freq(struct iio_dev *indio_dev, int *val, int *val2) struct adis16480 *st = iio_priv(indio_dev); uint16_t t; int ret; - unsigned int freq; - unsigned int reg; + unsigned int freq, sample_rate = st->clk_freq; - if (st->clk_mode == ADIS16480_CLK_PPS) - reg = ADIS16495_REG_SYNC_SCALE; - else - reg = ADIS16480_REG_DEC_RATE; + mutex_lock(&st->adis.state_lock); + + if (st->clk_mode == ADIS16480_CLK_PPS) { + u16 sync_scale; + + ret = __adis_read_reg_16(&st->adis, ADIS16495_REG_SYNC_SCALE, &sync_scale); + if (ret) + goto error; - ret = adis_read_reg_16(&st->adis, reg, &t); + sample_rate = st->clk_freq * sync_scale; + } + + ret = __adis_read_reg_16(&st->adis, ADIS16480_REG_DEC_RATE, &t); if (ret) - return ret; + goto error; - /* - * When using PPS mode, the rate of data collection is equal to the - * product of the external clock frequency and the scale factor in the - * SYNC_SCALE register. - * When using sync mode, or internal clock, the output data rate is - * equal with the clock frequency divided by DEC_RATE + 1. - */ - if (st->clk_mode == ADIS16480_CLK_PPS) - freq = st->clk_freq * t; - else - freq = st->clk_freq / (t + 1); + mutex_unlock(&st->adis.state_lock); + + freq = DIV_ROUND_CLOSEST(sample_rate, (t + 1)); *val = freq / 1000; *val2 = (freq % 1000) * 1000; return IIO_VAL_INT_PLUS_MICRO; +error: + mutex_unlock(&st->adis.state_lock); + return ret; } enum { @@ -1278,6 +1324,20 @@ static int adis16480_probe(struct spi_device *spi) st->clk_freq = clk_get_rate(st->ext_clk); st->clk_freq *= 1000; /* micro */ + if (st->clk_mode == ADIS16480_CLK_PPS) { + u16 sync_scale; + + /* + * In PPS mode, the IMU sample rate is the clk_freq * sync_scale. Hence, + * default the IMU sample rate to the highest multiple of the input clock + * lower than the IMU max sample rate. The internal sample rate is the + * max... + */ + sync_scale = st->chip_info->int_clk / st->clk_freq; + ret = __adis_write_reg_16(&st->adis, ADIS16495_REG_SYNC_SCALE, sync_scale); + if (ret) + return ret; + } } else { st->clk_freq = st->chip_info->int_clk; } diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c index 453c51c79655..69ab94ab7297 100644 --- a/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c +++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c @@ -731,12 +731,16 @@ inv_mpu6050_read_raw(struct iio_dev *indio_dev, } } -static int inv_mpu6050_write_gyro_scale(struct inv_mpu6050_state *st, int val) +static int inv_mpu6050_write_gyro_scale(struct inv_mpu6050_state *st, int val, + int val2) { int result, i; + if (val != 0) + return -EINVAL; + for (i = 0; i < ARRAY_SIZE(gyro_scale_6050); ++i) { - if (gyro_scale_6050[i] == val) { + if (gyro_scale_6050[i] == val2) { result = inv_mpu6050_set_gyro_fsr(st, i); if (result) return result; @@ -767,13 +771,17 @@ static int inv_write_raw_get_fmt(struct iio_dev *indio_dev, return -EINVAL; } -static int inv_mpu6050_write_accel_scale(struct inv_mpu6050_state *st, int val) +static int inv_mpu6050_write_accel_scale(struct inv_mpu6050_state *st, int val, + int val2) { int result, i; u8 d; + if (val != 0) + return -EINVAL; + for (i = 0; i < ARRAY_SIZE(accel_scale); ++i) { - if (accel_scale[i] == val) { + if (accel_scale[i] == val2) { d = (i << INV_MPU6050_ACCL_CONFIG_FSR_SHIFT); result = regmap_write(st->map, st->reg->accl_config, d); if (result) @@ -814,10 +822,10 @@ static int inv_mpu6050_write_raw(struct iio_dev *indio_dev, case IIO_CHAN_INFO_SCALE: switch (chan->type) { case IIO_ANGL_VEL: - result = inv_mpu6050_write_gyro_scale(st, val2); + result = inv_mpu6050_write_gyro_scale(st, val, val2); break; case IIO_ACCEL: - result = inv_mpu6050_write_accel_scale(st, val2); + result = inv_mpu6050_write_accel_scale(st, val, val2); break; default: result = -EINVAL; diff --git a/drivers/iio/industrialio-core.c b/drivers/iio/industrialio-core.c index 7db761afa578..36f3a900878d 100644 --- a/drivers/iio/industrialio-core.c +++ b/drivers/iio/industrialio-core.c @@ -1734,7 +1734,6 @@ static long iio_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) if (!indio_dev->info) goto out_unlock; - ret = -EINVAL; list_for_each_entry(h, &iio_dev_opaque->ioctl_handlers, entry) { ret = h->ioctl(indio_dev, filp, cmd, arg); if (ret != IIO_IOCTL_UNHANDLED) @@ -1742,7 +1741,7 @@ static long iio_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) } if (ret == IIO_IOCTL_UNHANDLED) - ret = -EINVAL; + ret = -ENODEV; out_unlock: mutex_unlock(&indio_dev->info_exist_lock); @@ -1864,9 +1863,6 @@ EXPORT_SYMBOL(__iio_device_register); **/ void iio_device_unregister(struct iio_dev *indio_dev) { - struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev); - struct iio_ioctl_handler *h, *t; - cdev_device_del(&indio_dev->chrdev, &indio_dev->dev); mutex_lock(&indio_dev->info_exist_lock); @@ -1877,9 +1873,6 @@ void iio_device_unregister(struct iio_dev *indio_dev) indio_dev->info = NULL; - list_for_each_entry_safe(h, t, &iio_dev_opaque->ioctl_handlers, entry) - list_del(&h->entry); - iio_device_wakeup_eventset(indio_dev); iio_buffer_wakeup_poll(indio_dev); diff --git a/drivers/iio/light/Kconfig b/drivers/iio/light/Kconfig index 33ad4dd0b5c7..917f9becf9c7 100644 --- a/drivers/iio/light/Kconfig +++ b/drivers/iio/light/Kconfig @@ -256,7 +256,6 @@ config ISL29125 config HID_SENSOR_ALS depends on HID_SENSOR_HUB select IIO_BUFFER - select IIO_TRIGGERED_BUFFER select HID_SENSOR_IIO_COMMON select HID_SENSOR_IIO_TRIGGER tristate "HID ALS" @@ -270,7 +269,6 @@ config HID_SENSOR_ALS config HID_SENSOR_PROX depends on HID_SENSOR_HUB select IIO_BUFFER - select IIO_TRIGGERED_BUFFER select HID_SENSOR_IIO_COMMON select HID_SENSOR_IIO_TRIGGER tristate "HID PROX" diff --git a/drivers/iio/light/gp2ap002.c b/drivers/iio/light/gp2ap002.c index 7ba7aa59437c..040d8429a6e0 100644 --- a/drivers/iio/light/gp2ap002.c +++ b/drivers/iio/light/gp2ap002.c @@ -583,7 +583,7 @@ static int gp2ap002_probe(struct i2c_client *client, "gp2ap002", indio_dev); if (ret) { dev_err(dev, "unable to request IRQ\n"); - goto out_disable_vio; + goto out_put_pm; } gp2ap002->irq = client->irq; @@ -613,8 +613,9 @@ static int gp2ap002_probe(struct i2c_client *client, return 0; -out_disable_pm: +out_put_pm: pm_runtime_put_noidle(dev); +out_disable_pm: pm_runtime_disable(dev); out_disable_vio: regulator_disable(gp2ap002->vio); diff --git a/drivers/iio/light/tsl2583.c b/drivers/iio/light/tsl2583.c index 0f787bfc88fc..c9d8f07a6fcd 100644 --- a/drivers/iio/light/tsl2583.c +++ b/drivers/iio/light/tsl2583.c @@ -341,6 +341,14 @@ static int tsl2583_als_calibrate(struct iio_dev *indio_dev) return lux_val; } + /* Avoid division by zero of lux_value later on */ + if (lux_val == 0) { + dev_err(&chip->client->dev, + "%s: lux_val of 0 will produce out of range trim_value\n", + __func__); + return -ENODATA; + } + gain_trim_val = (unsigned int)(((chip->als_settings.als_cal_target) * chip->als_settings.als_gain_trim) / lux_val); if ((gain_trim_val < 250) || (gain_trim_val > 4000)) { diff --git a/drivers/iio/magnetometer/Kconfig b/drivers/iio/magnetometer/Kconfig index 5d4ffd66032e..74ad5701c6c2 100644 --- a/drivers/iio/magnetometer/Kconfig +++ b/drivers/iio/magnetometer/Kconfig @@ -95,7 +95,6 @@ config MAG3110 config HID_SENSOR_MAGNETOMETER_3D depends on HID_SENSOR_HUB select IIO_BUFFER - select IIO_TRIGGERED_BUFFER select HID_SENSOR_IIO_COMMON select HID_SENSOR_IIO_TRIGGER tristate "HID Magenetometer 3D" diff --git a/drivers/iio/magnetometer/yamaha-yas530.c b/drivers/iio/magnetometer/yamaha-yas530.c index d46f23d82b3d..2f2f8cb3c26c 100644 --- a/drivers/iio/magnetometer/yamaha-yas530.c +++ b/drivers/iio/magnetometer/yamaha-yas530.c @@ -32,13 +32,14 @@ #include #include #include -#include #include #include #include #include +#include + /* This register map covers YAS530 and YAS532 but differs in YAS 537 and YAS539 */ #define YAS5XX_DEVICE_ID 0x80 #define YAS5XX_ACTUATE_INIT_COIL 0x81 @@ -887,6 +888,7 @@ static int yas5xx_probe(struct i2c_client *i2c, strncpy(yas5xx->name, "yas532", sizeof(yas5xx->name)); break; default: + ret = -ENODEV; dev_err(dev, "unhandled device ID %02x\n", yas5xx->devid); goto assert_reset; } diff --git a/drivers/iio/orientation/Kconfig b/drivers/iio/orientation/Kconfig index a505583cc2fd..396cbbb867f4 100644 --- a/drivers/iio/orientation/Kconfig +++ b/drivers/iio/orientation/Kconfig @@ -9,7 +9,6 @@ menu "Inclinometer sensors" config HID_SENSOR_INCLINOMETER_3D depends on HID_SENSOR_HUB select IIO_BUFFER - select IIO_TRIGGERED_BUFFER select HID_SENSOR_IIO_COMMON select HID_SENSOR_IIO_TRIGGER tristate "HID Inclinometer 3D" @@ -20,7 +19,6 @@ config HID_SENSOR_INCLINOMETER_3D config HID_SENSOR_DEVICE_ROTATION depends on HID_SENSOR_HUB select IIO_BUFFER - select IIO_TRIGGERED_BUFFER select HID_SENSOR_IIO_COMMON select HID_SENSOR_IIO_TRIGGER tristate "HID Device Rotation" diff --git a/drivers/iio/orientation/hid-sensor-rotation.c b/drivers/iio/orientation/hid-sensor-rotation.c index 18e4ef060096..c087d8f72a54 100644 --- a/drivers/iio/orientation/hid-sensor-rotation.c +++ b/drivers/iio/orientation/hid-sensor-rotation.c @@ -21,7 +21,7 @@ struct dev_rot_state { struct hid_sensor_common common_attributes; struct hid_sensor_hub_attribute_info quaternion; struct { - u32 sampled_vals[4] __aligned(16); + s32 sampled_vals[4] __aligned(16); u64 timestamp __aligned(8); } scan; int scale_pre_decml; @@ -170,8 +170,15 @@ static int dev_rot_capture_sample(struct hid_sensor_hub_device *hsdev, struct dev_rot_state *rot_state = iio_priv(indio_dev); if (usage_id == HID_USAGE_SENSOR_ORIENT_QUATERNION) { - memcpy(&rot_state->scan.sampled_vals, raw_data, - sizeof(rot_state->scan.sampled_vals)); + if (raw_len / 4 == sizeof(s16)) { + rot_state->scan.sampled_vals[0] = ((s16 *)raw_data)[0]; + rot_state->scan.sampled_vals[1] = ((s16 *)raw_data)[1]; + rot_state->scan.sampled_vals[2] = ((s16 *)raw_data)[2]; + rot_state->scan.sampled_vals[3] = ((s16 *)raw_data)[3]; + } else { + memcpy(&rot_state->scan.sampled_vals, raw_data, + sizeof(rot_state->scan.sampled_vals)); + } dev_dbg(&indio_dev->dev, "Recd Quat len:%zu::%zu\n", raw_len, sizeof(rot_state->scan.sampled_vals)); diff --git a/drivers/iio/pressure/Kconfig b/drivers/iio/pressure/Kconfig index 689b978db4f9..fc0d3cfca418 100644 --- a/drivers/iio/pressure/Kconfig +++ b/drivers/iio/pressure/Kconfig @@ -79,7 +79,6 @@ config DPS310 config HID_SENSOR_PRESS depends on HID_SENSOR_HUB select IIO_BUFFER - select IIO_TRIGGERED_BUFFER select HID_SENSOR_IIO_COMMON select HID_SENSOR_IIO_TRIGGER tristate "HID PRESS" diff --git a/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c b/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c index c685f10b5ae4..cc206bfa09c7 100644 --- a/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c +++ b/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c @@ -160,6 +160,7 @@ static int lidar_get_measurement(struct lidar_data *data, u16 *reg) ret = lidar_write_control(data, LIDAR_REG_CONTROL_ACQUIRE); if (ret < 0) { dev_err(&client->dev, "cannot send start measurement command"); + pm_runtime_put_noidle(&client->dev); return ret; } diff --git a/drivers/iio/proximity/sx9310.c b/drivers/iio/proximity/sx9310.c index 37fd0b65a014..ea82cfaf7f42 100644 --- a/drivers/iio/proximity/sx9310.c +++ b/drivers/iio/proximity/sx9310.c @@ -763,7 +763,11 @@ static int sx9310_write_far_debounce(struct sx9310_data *data, int val) int ret; unsigned int regval; - val = ilog2(val); + if (val > 0) + val = ilog2(val); + if (!FIELD_FIT(SX9310_REG_PROX_CTRL10_FAR_DEBOUNCE_MASK, val)) + return -EINVAL; + regval = FIELD_PREP(SX9310_REG_PROX_CTRL10_FAR_DEBOUNCE_MASK, val); mutex_lock(&data->mutex); @@ -780,7 +784,11 @@ static int sx9310_write_close_debounce(struct sx9310_data *data, int val) int ret; unsigned int regval; - val = ilog2(val); + if (val > 0) + val = ilog2(val); + if (!FIELD_FIT(SX9310_REG_PROX_CTRL10_CLOSE_DEBOUNCE_MASK, val)) + return -EINVAL; + regval = FIELD_PREP(SX9310_REG_PROX_CTRL10_CLOSE_DEBOUNCE_MASK, val); mutex_lock(&data->mutex); @@ -1213,17 +1221,17 @@ static int sx9310_init_compensation(struct iio_dev *indio_dev) } static const struct sx9310_reg_default * -sx9310_get_default_reg(struct sx9310_data *data, int i, +sx9310_get_default_reg(struct sx9310_data *data, int idx, struct sx9310_reg_default *reg_def) { - int ret; const struct device_node *np = data->client->dev.of_node; - u32 combined[SX9310_NUM_CHANNELS] = { 4, 4, 4, 4 }; + u32 combined[SX9310_NUM_CHANNELS]; + u32 start = 0, raw = 0, pos = 0; unsigned long comb_mask = 0; + int ret, i, count; const char *res; - u32 start = 0, raw = 0, pos = 0; - memcpy(reg_def, &sx9310_default_regs[i], sizeof(*reg_def)); + memcpy(reg_def, &sx9310_default_regs[idx], sizeof(*reg_def)); if (!np) return reg_def; @@ -1234,15 +1242,31 @@ sx9310_get_default_reg(struct sx9310_data *data, int i, reg_def->def |= SX9310_REG_PROX_CTRL2_SHIELDEN_GROUND; } - reg_def->def &= ~SX9310_REG_PROX_CTRL2_COMBMODE_MASK; - of_property_read_u32_array(np, "semtech,combined-sensors", - combined, ARRAY_SIZE(combined)); - for (i = 0; i < ARRAY_SIZE(combined); i++) { - if (combined[i] <= SX9310_NUM_CHANNELS) - comb_mask |= BIT(combined[i]); + count = of_property_count_elems_of_size(np, "semtech,combined-sensors", + sizeof(u32)); + if (count > 0 && count <= ARRAY_SIZE(combined)) { + ret = of_property_read_u32_array(np, "semtech,combined-sensors", + combined, count); + if (ret) + break; + } else { + /* + * Either the property does not exist in the DT or the + * number of entries is incorrect. + */ + break; } + for (i = 0; i < count; i++) { + if (combined[i] >= SX9310_NUM_CHANNELS) { + /* Invalid sensor (invalid DT). */ + break; + } + comb_mask |= BIT(combined[i]); + } + if (i < count) + break; - comb_mask &= 0xf; + reg_def->def &= ~SX9310_REG_PROX_CTRL2_COMBMODE_MASK; if (comb_mask == (BIT(3) | BIT(2) | BIT(1) | BIT(0))) reg_def->def |= SX9310_REG_PROX_CTRL2_COMBMODE_CS0_CS1_CS2_CS3; else if (comb_mask == (BIT(1) | BIT(2))) diff --git a/drivers/iio/temperature/Kconfig b/drivers/iio/temperature/Kconfig index f1f2a1499c9e..4df60082c1fa 100644 --- a/drivers/iio/temperature/Kconfig +++ b/drivers/iio/temperature/Kconfig @@ -45,7 +45,6 @@ config HID_SENSOR_TEMP tristate "HID Environmental temperature sensor" depends on HID_SENSOR_HUB select IIO_BUFFER - select IIO_TRIGGERED_BUFFER select HID_SENSOR_IIO_COMMON select HID_SENSOR_IIO_TRIGGER help diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c index 3d194bb60840..6adbaea358ae 100644 --- a/drivers/infiniband/core/cm.c +++ b/drivers/infiniband/core/cm.c @@ -2138,7 +2138,8 @@ static int cm_req_handler(struct cm_work *work) goto destroy; } - cm_process_routed_req(req_msg, work->mad_recv_wc->wc); + if (cm_id_priv->av.ah_attr.type != RDMA_AH_ATTR_TYPE_ROCE) + cm_process_routed_req(req_msg, work->mad_recv_wc->wc); memset(&work->path[0], 0, sizeof(work->path[0])); if (cm_req_has_alt_path(req_msg)) diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index 94096511599f..6ac07911a17b 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -463,7 +463,6 @@ static void _cma_attach_to_dev(struct rdma_id_private *id_priv, id_priv->id.route.addr.dev_addr.transport = rdma_node_get_transport(cma_dev->device->node_type); list_add_tail(&id_priv->list, &cma_dev->id_list); - rdma_restrack_add(&id_priv->res); trace_cm_id_attach(id_priv, cma_dev->device); } @@ -700,6 +699,7 @@ static int cma_ib_acquire_dev(struct rdma_id_private *id_priv, mutex_lock(&lock); cma_attach_to_dev(id_priv, listen_id_priv->cma_dev); mutex_unlock(&lock); + rdma_restrack_add(&id_priv->res); return 0; } @@ -754,8 +754,10 @@ static int cma_iw_acquire_dev(struct rdma_id_private *id_priv, } out: - if (!ret) + if (!ret) { cma_attach_to_dev(id_priv, cma_dev); + rdma_restrack_add(&id_priv->res); + } mutex_unlock(&lock); return ret; @@ -816,6 +818,7 @@ static int cma_resolve_ib_dev(struct rdma_id_private *id_priv) found: cma_attach_to_dev(id_priv, cma_dev); + rdma_restrack_add(&id_priv->res); mutex_unlock(&lock); addr = (struct sockaddr_ib *)cma_src_addr(id_priv); memcpy(&addr->sib_addr, &sgid, sizeof(sgid)); @@ -2529,6 +2532,7 @@ static int cma_listen_on_dev(struct rdma_id_private *id_priv, rdma_addr_size(cma_src_addr(id_priv))); _cma_attach_to_dev(dev_id_priv, cma_dev); + rdma_restrack_add(&dev_id_priv->res); cma_id_get(id_priv); dev_id_priv->internal_id = 1; dev_id_priv->afonly = id_priv->afonly; @@ -3169,6 +3173,7 @@ static int cma_bind_loopback(struct rdma_id_private *id_priv) ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey); id_priv->id.port_num = p; cma_attach_to_dev(id_priv, cma_dev); + rdma_restrack_add(&id_priv->res); cma_set_loopback(cma_src_addr(id_priv)); out: mutex_unlock(&lock); @@ -3201,6 +3206,7 @@ static void addr_handler(int status, struct sockaddr *src_addr, if (status) pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to acquire device. status %d\n", status); + rdma_restrack_add(&id_priv->res); } else if (status) { pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to resolve IP. status %d\n", status); } @@ -3812,6 +3818,8 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr) if (ret) goto err2; + if (!cma_any_addr(addr)) + rdma_restrack_add(&id_priv->res); return 0; err2: if (id_priv->cma_dev) diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c index 995d4633b0a1..d4d4959c2434 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c @@ -2784,6 +2784,7 @@ static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq, dev_err(&cq->hwq.pdev->dev, "FP: CQ Processed terminal reported rq_cons_idx 0x%x exceeds max 0x%x\n", cqe_cons, rq->max_wqe); + rc = -EINVAL; goto done; } diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c index fa7878336100..3ca47004b752 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_res.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c @@ -854,6 +854,7 @@ static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res *res, unmap_io: pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem); + dpit->dbr_bar_reg_iomem = NULL; return -ENOMEM; } diff --git a/drivers/infiniband/hw/cxgb4/resource.c b/drivers/infiniband/hw/cxgb4/resource.c index 5c95c789f302..e800e8e8bed5 100644 --- a/drivers/infiniband/hw/cxgb4/resource.c +++ b/drivers/infiniband/hw/cxgb4/resource.c @@ -216,7 +216,7 @@ u32 c4iw_get_qpid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx) goto out; entry->qid = qid; list_add_tail(&entry->entry, &uctx->cqids); - for (i = qid; i & rdev->qpmask; i++) { + for (i = qid + 1; i & rdev->qpmask; i++) { entry = kmalloc(sizeof(*entry), GFP_KERNEL); if (!entry) goto out; diff --git a/drivers/infiniband/hw/hfi1/firmware.c b/drivers/infiniband/hw/hfi1/firmware.c index 0e83d4b61e46..2cf102b5abd4 100644 --- a/drivers/infiniband/hw/hfi1/firmware.c +++ b/drivers/infiniband/hw/hfi1/firmware.c @@ -1916,6 +1916,7 @@ int parse_platform_config(struct hfi1_devdata *dd) dd_dev_err(dd, "%s: Failed CRC check at offset %ld\n", __func__, (ptr - (u32 *)dd->platform_config.data)); + ret = -EINVAL; goto bail; } /* Jump the CRC DWORD */ diff --git a/drivers/infiniband/hw/hfi1/ipoib.h b/drivers/infiniband/hw/hfi1/ipoib.h index f650cac9d424..d30c23b6527a 100644 --- a/drivers/infiniband/hw/hfi1/ipoib.h +++ b/drivers/infiniband/hw/hfi1/ipoib.h @@ -52,8 +52,9 @@ union hfi1_ipoib_flow { * @producer_lock: producer sync lock * @consumer_lock: consumer sync lock */ +struct ipoib_txreq; struct hfi1_ipoib_circ_buf { - void **items; + struct ipoib_txreq **items; unsigned long head; unsigned long tail; unsigned long max_items; diff --git a/drivers/infiniband/hw/hfi1/ipoib_tx.c b/drivers/infiniband/hw/hfi1/ipoib_tx.c index edd4eeac8dd1..cdc26ee3cf52 100644 --- a/drivers/infiniband/hw/hfi1/ipoib_tx.c +++ b/drivers/infiniband/hw/hfi1/ipoib_tx.c @@ -702,14 +702,14 @@ int hfi1_ipoib_txreq_init(struct hfi1_ipoib_dev_priv *priv) priv->tx_napis = kcalloc_node(dev->num_tx_queues, sizeof(struct napi_struct), - GFP_ATOMIC, + GFP_KERNEL, priv->dd->node); if (!priv->tx_napis) goto free_txreq_cache; priv->txqs = kcalloc_node(dev->num_tx_queues, sizeof(struct hfi1_ipoib_txq), - GFP_ATOMIC, + GFP_KERNEL, priv->dd->node); if (!priv->txqs) goto free_tx_napis; @@ -741,9 +741,9 @@ int hfi1_ipoib_txreq_init(struct hfi1_ipoib_dev_priv *priv) priv->dd->node); txq->tx_ring.items = - vzalloc_node(array_size(tx_ring_size, - sizeof(struct ipoib_txreq)), - priv->dd->node); + kcalloc_node(tx_ring_size, + sizeof(struct ipoib_txreq *), + GFP_KERNEL, priv->dd->node); if (!txq->tx_ring.items) goto free_txqs; @@ -764,7 +764,7 @@ int hfi1_ipoib_txreq_init(struct hfi1_ipoib_dev_priv *priv) struct hfi1_ipoib_txq *txq = &priv->txqs[i]; netif_napi_del(txq->napi); - vfree(txq->tx_ring.items); + kfree(txq->tx_ring.items); } kfree(priv->txqs); @@ -817,7 +817,7 @@ void hfi1_ipoib_txreq_deinit(struct hfi1_ipoib_dev_priv *priv) hfi1_ipoib_drain_tx_list(txq); netif_napi_del(txq->napi); (void)hfi1_ipoib_drain_tx_ring(txq, txq->tx_ring.max_items); - vfree(txq->tx_ring.items); + kfree(txq->tx_ring.items); } kfree(priv->txqs); diff --git a/drivers/infiniband/hw/hfi1/mmu_rb.c b/drivers/infiniband/hw/hfi1/mmu_rb.c index f3fb28e3d5d7..d213f65d4cdd 100644 --- a/drivers/infiniband/hw/hfi1/mmu_rb.c +++ b/drivers/infiniband/hw/hfi1/mmu_rb.c @@ -89,7 +89,7 @@ int hfi1_mmu_rb_register(void *ops_arg, struct mmu_rb_handler *h; int ret; - h = kmalloc(sizeof(*h), GFP_KERNEL); + h = kzalloc(sizeof(*h), GFP_KERNEL); if (!h) return -ENOMEM; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index ce26f97b2ca2..ad3cee54140e 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -5068,6 +5068,7 @@ static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, qp_attr->cur_qp_state = qp_attr->qp_state; qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt; qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge; + qp_attr->cap.max_inline_data = hr_qp->max_inline_data; if (!ibqp->uobject) { qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt; diff --git a/drivers/infiniband/hw/i40iw/i40iw_pble.c b/drivers/infiniband/hw/i40iw/i40iw_pble.c index 53e5cd1a2bd6..146a4148219b 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_pble.c +++ b/drivers/infiniband/hw/i40iw/i40iw_pble.c @@ -393,12 +393,9 @@ static enum i40iw_status_code add_pble_pool(struct i40iw_sc_dev *dev, i40iw_debug(dev, I40IW_DEBUG_PBLE, "next_fpm_addr = %llx chunk_size[%u] = 0x%x\n", pble_rsrc->next_fpm_addr, chunk->size, chunk->size); pble_rsrc->unallocated_pble -= (chunk->size >> 3); - list_add(&chunk->list, &pble_rsrc->pinfo.clist); sd_reg_val = (sd_entry_type == I40IW_SD_TYPE_PAGED) ? sd_entry->u.pd_table.pd_page_addr.pa : sd_entry->u.bp.addr.pa; - if (sd_entry->valid) - return 0; - if (dev->is_pf) { + if (dev->is_pf && !sd_entry->valid) { ret_code = i40iw_hmc_sd_one(dev, hmc_info->hmc_fn_id, sd_reg_val, idx->sd_idx, sd_entry->entry_type, true); @@ -409,6 +406,7 @@ static enum i40iw_status_code add_pble_pool(struct i40iw_sc_dev *dev, } sd_entry->valid = true; + list_add(&chunk->list, &pble_rsrc->pinfo.clist); return 0; error: kfree(chunk); diff --git a/drivers/infiniband/hw/mlx5/fs.c b/drivers/infiniband/hw/mlx5/fs.c index 25da0b05b4e2..f0af3f1ae039 100644 --- a/drivers/infiniband/hw/mlx5/fs.c +++ b/drivers/infiniband/hw/mlx5/fs.c @@ -1528,8 +1528,8 @@ static struct mlx5_ib_flow_handler *raw_fs_rule_add( dst_num++; } - handler = _create_raw_flow_rule(dev, ft_prio, dst, fs_matcher, - flow_context, flow_act, + handler = _create_raw_flow_rule(dev, ft_prio, dst_num ? dst : NULL, + fs_matcher, flow_context, flow_act, cmd_in, inlen, dst_num); if (IS_ERR(handler)) { @@ -1885,8 +1885,9 @@ static int get_dests(struct uverbs_attr_bundle *attrs, else *dest_id = mqp->raw_packet_qp.rq.tirn; *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; - } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS || - fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) { + } else if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS || + fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) && + !(*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)) { *dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT; } diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 0d69a697d75f..4be7bccefaa4 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -499,7 +499,7 @@ static int mlx5_query_port_roce(struct ib_device *device, u8 port_num, translate_eth_proto_oper(eth_prot_oper, &props->active_speed, &props->active_width, ext); - if (!dev->is_rep && mlx5_is_roce_enabled(mdev)) { + if (!dev->is_rep && dev->mdev->roce.roce_en) { u16 qkey_viol_cntr; props->port_cap_flags |= IB_PORT_CM_SUP; @@ -4174,7 +4174,7 @@ static int mlx5_ib_roce_init(struct mlx5_ib_dev *dev) /* Register only for native ports */ err = mlx5_add_netdev_notifier(dev, port_num); - if (err || dev->is_rep || !mlx5_is_roce_enabled(mdev)) + if (err || dev->is_rep || !mlx5_is_roce_init_enabled(mdev)) /* * We don't enable ETH interface for * 1. IB representors @@ -4711,7 +4711,7 @@ static int mlx5r_probe(struct auxiliary_device *adev, dev->mdev = mdev; dev->num_ports = num_ports; - if (ll == IB_LINK_LAYER_ETHERNET && !mlx5_is_roce_enabled(mdev)) + if (ll == IB_LINK_LAYER_ETHERNET && !mlx5_is_roce_init_enabled(mdev)) profile = &raw_eth_profile; else profile = &pf_profile; diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 88cc26e008fc..b085c02b53d0 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -547,11 +547,6 @@ static inline const struct mlx5_umr_wr *umr_wr(const struct ib_send_wr *wr) return container_of(wr, struct mlx5_umr_wr, wr); } -struct mlx5_shared_mr_info { - int mr_id; - struct ib_umem *umem; -}; - enum mlx5_ib_cq_pr_flags { MLX5_IB_CQ_PR_FLAGS_CQE_128_PAD = 1 << 0, }; @@ -654,47 +649,69 @@ struct mlx5_ib_dm { atomic64_add(value, &((mr)->odp_stats.counter_name)) struct mlx5_ib_mr { - struct ib_mr ibmr; - void *descs; - dma_addr_t desc_map; - int ndescs; - int data_length; - int meta_ndescs; - int meta_length; - int max_descs; - int desc_size; - int access_mode; - unsigned int page_shift; - struct mlx5_core_mkey mmkey; - struct ib_umem *umem; - struct mlx5_shared_mr_info *smr_info; - struct list_head list; - struct mlx5_cache_ent *cache_ent; - u32 out[MLX5_ST_SZ_DW(create_mkey_out)]; - struct mlx5_core_sig_ctx *sig; - void *descs_alloc; - int access_flags; /* Needed for rereg MR */ - - struct mlx5_ib_mr *parent; - /* Needed for IB_MR_TYPE_INTEGRITY */ - struct mlx5_ib_mr *pi_mr; - struct mlx5_ib_mr *klm_mr; - struct mlx5_ib_mr *mtt_mr; - u64 data_iova; - u64 pi_iova; - - /* For ODP and implicit */ - struct xarray implicit_children; - union { - struct list_head elm; - struct work_struct work; - } odp_destroy; - struct ib_odp_counters odp_stats; - bool is_odp_implicit; + struct ib_mr ibmr; + struct mlx5_core_mkey mmkey; - struct mlx5_async_work cb_work; + /* User MR data */ + struct mlx5_cache_ent *cache_ent; + struct ib_umem *umem; + + /* This is zero'd when the MR is allocated */ + struct { + /* Used only while the MR is in the cache */ + struct { + u32 out[MLX5_ST_SZ_DW(create_mkey_out)]; + struct mlx5_async_work cb_work; + /* Cache list element */ + struct list_head list; + }; + + /* Used only by kernel MRs (umem == NULL) */ + struct { + void *descs; + void *descs_alloc; + dma_addr_t desc_map; + int max_descs; + int ndescs; + int desc_size; + int access_mode; + + /* For Kernel IB_MR_TYPE_INTEGRITY */ + struct mlx5_core_sig_ctx *sig; + struct mlx5_ib_mr *pi_mr; + struct mlx5_ib_mr *klm_mr; + struct mlx5_ib_mr *mtt_mr; + u64 data_iova; + u64 pi_iova; + int meta_ndescs; + int meta_length; + int data_length; + }; + + /* Used only by User MRs (umem != NULL) */ + struct { + unsigned int page_shift; + /* Current access_flags */ + int access_flags; + + /* For User ODP */ + struct mlx5_ib_mr *parent; + struct xarray implicit_children; + union { + struct work_struct work; + } odp_destroy; + struct ib_odp_counters odp_stats; + bool is_odp_implicit; + }; + }; }; +/* Zero the fields in the mr that are variant depending on usage */ +static inline void mlx5_clear_mr(struct mlx5_ib_mr *mr) +{ + memset(mr->out, 0, sizeof(*mr) - offsetof(struct mlx5_ib_mr, out)); +} + static inline bool is_odp_mr(struct mlx5_ib_mr *mr) { return IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) && mr->umem && diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index db05b0e0a8d7..ea8f068a6da3 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -590,6 +590,8 @@ struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, ent->available_mrs--; queue_adjust_cache_locked(ent); spin_unlock_irq(&ent->lock); + + mlx5_clear_mr(mr); } mr->access_flags = access_flags; return mr; @@ -615,16 +617,14 @@ static struct mlx5_ib_mr *get_cache_mr(struct mlx5_cache_ent *req_ent) ent->available_mrs--; queue_adjust_cache_locked(ent); spin_unlock_irq(&ent->lock); - break; + mlx5_clear_mr(mr); + return mr; } queue_adjust_cache_locked(ent); spin_unlock_irq(&ent->lock); } - - if (!mr) - req_ent->miss++; - - return mr; + req_ent->miss++; + return NULL; } static void detach_mr_from_cache(struct mlx5_ib_mr *mr) @@ -993,8 +993,6 @@ static struct mlx5_ib_mr *alloc_cacheable_mr(struct ib_pd *pd, mr->ibmr.pd = pd; mr->umem = umem; - mr->access_flags = access_flags; - mr->desc_size = sizeof(struct mlx5_mtt); mr->mmkey.iova = iova; mr->mmkey.size = umem->length; mr->mmkey.pd = to_mpd(pd)->pdn; diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index b103555b1f5d..d98755e78362 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -227,7 +227,6 @@ static void free_implicit_child_mr(struct mlx5_ib_mr *mr, bool need_imr_xlt) dma_fence_odp_mr(mr); - mr->parent = NULL; mlx5_mr_cache_free(mr_to_mdev(mr), mr); ib_umem_odp_release(odp); } diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index f5a52a6fae43..843f9e7fe96f 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -3146,6 +3146,19 @@ enum { MLX5_PATH_FLAG_COUNTER = 1 << 2, }; +static int mlx5_to_ib_rate_map(u8 rate) +{ + static const int rates[] = { IB_RATE_PORT_CURRENT, IB_RATE_56_GBPS, + IB_RATE_25_GBPS, IB_RATE_100_GBPS, + IB_RATE_200_GBPS, IB_RATE_50_GBPS, + IB_RATE_400_GBPS }; + + if (rate < ARRAY_SIZE(rates)) + return rates[rate]; + + return rate - MLX5_STAT_RATE_OFFSET; +} + static int ib_to_mlx5_rate_map(u8 rate) { switch (rate) { @@ -4485,7 +4498,7 @@ static void to_rdma_ah_attr(struct mlx5_ib_dev *ibdev, rdma_ah_set_path_bits(ah_attr, MLX5_GET(ads, path, mlid)); static_rate = MLX5_GET(ads, path, stat_rate); - rdma_ah_set_static_rate(ah_attr, static_rate ? static_rate - 5 : 0); + rdma_ah_set_static_rate(ah_attr, mlx5_to_ib_rate_map(static_rate)); if (MLX5_GET(ads, path, grh) || ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE) { rdma_ah_set_grh(ah_attr, NULL, MLX5_GET(ads, path, flow_label), diff --git a/drivers/infiniband/hw/qedr/qedr_iw_cm.c b/drivers/infiniband/hw/qedr/qedr_iw_cm.c index c4bc58736e48..1715fbe0719d 100644 --- a/drivers/infiniband/hw/qedr/qedr_iw_cm.c +++ b/drivers/infiniband/hw/qedr/qedr_iw_cm.c @@ -636,8 +636,10 @@ int qedr_iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) memcpy(in_params.local_mac_addr, dev->ndev->dev_addr, ETH_ALEN); if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_CONNECT, - &qp->iwarp_cm_flags)) + &qp->iwarp_cm_flags)) { + rc = -ENODEV; goto err; /* QP already being destroyed */ + } rc = dev->ops->iwarp_connect(dev->rdma_ctx, &in_params, &out_params); if (rc) { diff --git a/drivers/infiniband/sw/rxe/rxe_av.c b/drivers/infiniband/sw/rxe/rxe_av.c index df0d173d6acb..da2e867a1ed9 100644 --- a/drivers/infiniband/sw/rxe/rxe_av.c +++ b/drivers/infiniband/sw/rxe/rxe_av.c @@ -88,7 +88,7 @@ void rxe_av_fill_ip_info(struct rxe_av *av, struct rdma_ah_attr *attr) type = RXE_NETWORK_TYPE_IPV4; break; case RDMA_NETWORK_IPV6: - type = RXE_NETWORK_TYPE_IPV4; + type = RXE_NETWORK_TYPE_IPV6; break; default: /* not reached - checked in rxe_av_chk_attr */ diff --git a/drivers/infiniband/sw/siw/siw_mem.c b/drivers/infiniband/sw/siw/siw_mem.c index 34a910cf0edb..61c17db70d65 100644 --- a/drivers/infiniband/sw/siw/siw_mem.c +++ b/drivers/infiniband/sw/siw/siw_mem.c @@ -106,8 +106,6 @@ int siw_mr_add_mem(struct siw_mr *mr, struct ib_pd *pd, void *mem_obj, mem->perms = rights & IWARP_ACCESS_MASK; kref_init(&mem->ref); - mr->mem = mem; - get_random_bytes(&next, 4); next &= 0x00ffffff; @@ -116,6 +114,8 @@ int siw_mr_add_mem(struct siw_mr *mr, struct ib_pd *pd, void *mem_obj, kfree(mem); return -ENOMEM; } + + mr->mem = mem; /* Set the STag index part */ mem->stag = id << 8; mr->base_mr.lkey = mr->base_mr.rkey = mem->stag; diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c index 7305ed8976c2..18266f07c58d 100644 --- a/drivers/infiniband/ulp/isert/ib_isert.c +++ b/drivers/infiniband/ulp/isert/ib_isert.c @@ -438,23 +438,23 @@ isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) isert_init_conn(isert_conn); isert_conn->cm_id = cma_id; - ret = isert_alloc_login_buf(isert_conn, cma_id->device); - if (ret) - goto out; - device = isert_device_get(cma_id); if (IS_ERR(device)) { ret = PTR_ERR(device); - goto out_rsp_dma_map; + goto out; } isert_conn->device = device; + ret = isert_alloc_login_buf(isert_conn, cma_id->device); + if (ret) + goto out_conn_dev; + isert_set_nego_params(isert_conn, &event->param.conn); isert_conn->qp = isert_create_qp(isert_conn, cma_id); if (IS_ERR(isert_conn->qp)) { ret = PTR_ERR(isert_conn->qp); - goto out_conn_dev; + goto out_rsp_dma_map; } ret = isert_login_post_recv(isert_conn); @@ -473,10 +473,10 @@ isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) out_destroy_qp: isert_destroy_qp(isert_conn); -out_conn_dev: - isert_device_put(device); out_rsp_dma_map: isert_free_login_buf(isert_conn); +out_conn_dev: + isert_device_put(device); out: kfree(isert_conn); rdma_reject(cma_id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED); diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c index 6734329cca33..959ba0462ef0 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c @@ -2784,8 +2784,8 @@ int rtrs_clt_remove_path_from_sysfs(struct rtrs_clt_sess *sess, } while (!changed && old_state != RTRS_CLT_DEAD); if (likely(changed)) { - rtrs_clt_destroy_sess_files(sess, sysfs_self); rtrs_clt_remove_path_from_arr(sess); + rtrs_clt_destroy_sess_files(sess, sysfs_self); kobject_put(&sess->kobj); } diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c index 6be60aa5ffe2..7f0420ad9057 100644 --- a/drivers/infiniband/ulp/srpt/ib_srpt.c +++ b/drivers/infiniband/ulp/srpt/ib_srpt.c @@ -2378,6 +2378,7 @@ static int srpt_cm_req_recv(struct srpt_device *const sdev, pr_info("rejected SRP_LOGIN_REQ because target %s_%d is not enabled\n", dev_name(&sdev->device->dev), port_num); mutex_unlock(&sport->mutex); + ret = -EINVAL; goto reject; } diff --git a/drivers/input/touchscreen/elants_i2c.c b/drivers/input/touchscreen/elants_i2c.c index 5f7706febcb0..17540bdb1eaf 100644 --- a/drivers/input/touchscreen/elants_i2c.c +++ b/drivers/input/touchscreen/elants_i2c.c @@ -38,6 +38,7 @@ #include #include #include +#include #include /* Device, Driver information */ @@ -1334,6 +1335,40 @@ static void elants_i2c_power_off(void *_data) } } +#ifdef CONFIG_ACPI +static const struct acpi_device_id i2c_hid_ids[] = { + {"ACPI0C50", 0 }, + {"PNP0C50", 0 }, + { }, +}; + +static const guid_t i2c_hid_guid = + GUID_INIT(0x3CDFF6F7, 0x4267, 0x4555, + 0xAD, 0x05, 0xB3, 0x0A, 0x3D, 0x89, 0x38, 0xDE); + +static bool elants_acpi_is_hid_device(struct device *dev) +{ + acpi_handle handle = ACPI_HANDLE(dev); + union acpi_object *obj; + + if (acpi_match_device_ids(ACPI_COMPANION(dev), i2c_hid_ids)) + return false; + + obj = acpi_evaluate_dsm_typed(handle, &i2c_hid_guid, 1, 1, NULL, ACPI_TYPE_INTEGER); + if (obj) { + ACPI_FREE(obj); + return true; + } + + return false; +} +#else +static bool elants_acpi_is_hid_device(struct device *dev) +{ + return false; +} +#endif + static int elants_i2c_probe(struct i2c_client *client, const struct i2c_device_id *id) { @@ -1342,9 +1377,14 @@ static int elants_i2c_probe(struct i2c_client *client, unsigned long irqflags; int error; + /* Don't bind to i2c-hid compatible devices, these are handled by the i2c-hid drv. */ + if (elants_acpi_is_hid_device(&client->dev)) { + dev_warn(&client->dev, "This device appears to be an I2C-HID device, not binding\n"); + return -ENODEV; + } + if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) { - dev_err(&client->dev, - "%s: i2c check functionality error\n", DEVICE_NAME); + dev_err(&client->dev, "I2C check functionality error\n"); return -ENXIO; } diff --git a/drivers/input/touchscreen/ili210x.c b/drivers/input/touchscreen/ili210x.c index d8fccf048bf4..30576a5f2f04 100644 --- a/drivers/input/touchscreen/ili210x.c +++ b/drivers/input/touchscreen/ili210x.c @@ -87,7 +87,7 @@ static bool ili210x_touchdata_to_coords(const u8 *touchdata, unsigned int *x, unsigned int *y, unsigned int *z) { - if (touchdata[0] & BIT(finger)) + if (!(touchdata[0] & BIT(finger))) return false; *x = get_unaligned_be16(touchdata + 1 + (finger * 4) + 0); diff --git a/drivers/input/touchscreen/silead.c b/drivers/input/touchscreen/silead.c index 8fa2f3b7cfd8..e8b6c3137420 100644 --- a/drivers/input/touchscreen/silead.c +++ b/drivers/input/touchscreen/silead.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -335,10 +336,8 @@ static int silead_ts_get_id(struct i2c_client *client) error = i2c_smbus_read_i2c_block_data(client, SILEAD_REG_ID, sizeof(chip_id), (u8 *)&chip_id); - if (error < 0) { - dev_err(&client->dev, "Chip ID read error %d\n", error); + if (error < 0) return error; - } data->chip_id = le32_to_cpu(chip_id); dev_info(&client->dev, "Silead chip ID: 0x%8X", data->chip_id); @@ -351,12 +350,49 @@ static int silead_ts_setup(struct i2c_client *client) int error; u32 status; + /* + * Some buggy BIOS-es bring up the chip in a stuck state where it + * blocks the I2C bus. The following steps are necessary to + * unstuck the chip / bus: + * 1. Turn off the Silead chip. + * 2. Try to do an I2C transfer with the chip, this will fail in + * response to which the I2C-bus-driver will call: + * i2c_recover_bus() which will unstuck the I2C-bus. Note the + * unstuck-ing of the I2C bus only works if we first drop the + * chip off the bus by turning it off. + * 3. Turn the chip back on. + * + * On the x86/ACPI systems were this problem is seen, step 1. and + * 3. require making ACPI calls and dealing with ACPI Power + * Resources. The workaround below runtime-suspends the chip to + * turn it off, leaving it up to the ACPI subsystem to deal with + * this. + */ + + if (device_property_read_bool(&client->dev, + "silead,stuck-controller-bug")) { + pm_runtime_set_active(&client->dev); + pm_runtime_enable(&client->dev); + pm_runtime_allow(&client->dev); + + pm_runtime_suspend(&client->dev); + + dev_warn(&client->dev, FW_BUG "Stuck I2C bus: please ignore the next 'controller timed out' error\n"); + silead_ts_get_id(client); + + /* The forbid will also resume the device */ + pm_runtime_forbid(&client->dev); + pm_runtime_disable(&client->dev); + } + silead_ts_set_power(client, SILEAD_POWER_OFF); silead_ts_set_power(client, SILEAD_POWER_ON); error = silead_ts_get_id(client); - if (error) + if (error) { + dev_err(&client->dev, "Chip ID read error %d\n", error); return error; + } error = silead_ts_init(client); if (error) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 321f5906e6ed..df7b19ff0a9e 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include @@ -257,8 +256,6 @@ static enum iommu_init_state init_state = IOMMU_START_STATE; static int amd_iommu_enable_interrupts(void); static int __init iommu_go_to_state(enum iommu_init_state state); static void init_device_table_dma(void); -static int iommu_pc_get_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, - u8 fxn, u64 *value, bool is_write); static bool amd_iommu_pre_enabled = true; @@ -1717,53 +1714,16 @@ static int __init init_iommu_all(struct acpi_table_header *table) return 0; } -static void __init init_iommu_perf_ctr(struct amd_iommu *iommu) +static void init_iommu_perf_ctr(struct amd_iommu *iommu) { - int retry; + u64 val; struct pci_dev *pdev = iommu->dev; - u64 val = 0xabcd, val2 = 0, save_reg, save_src; if (!iommu_feature(iommu, FEATURE_PC)) return; amd_iommu_pc_present = true; - /* save the value to restore, if writable */ - if (iommu_pc_get_set_reg(iommu, 0, 0, 0, &save_reg, false) || - iommu_pc_get_set_reg(iommu, 0, 0, 8, &save_src, false)) - goto pc_false; - - /* - * Disable power gating by programing the performance counter - * source to 20 (i.e. counts the reads and writes from/to IOMMU - * Reserved Register [MMIO Offset 1FF8h] that are ignored.), - * which never get incremented during this init phase. - * (Note: The event is also deprecated.) - */ - val = 20; - if (iommu_pc_get_set_reg(iommu, 0, 0, 8, &val, true)) - goto pc_false; - - /* Check if the performance counters can be written to */ - val = 0xabcd; - for (retry = 5; retry; retry--) { - if (iommu_pc_get_set_reg(iommu, 0, 0, 0, &val, true) || - iommu_pc_get_set_reg(iommu, 0, 0, 0, &val2, false) || - val2) - break; - - /* Wait about 20 msec for power gating to disable and retry. */ - msleep(20); - } - - /* restore */ - if (iommu_pc_get_set_reg(iommu, 0, 0, 0, &save_reg, true) || - iommu_pc_get_set_reg(iommu, 0, 0, 8, &save_src, true)) - goto pc_false; - - if (val != val2) - goto pc_false; - pci_info(pdev, "IOMMU performance counters supported\n"); val = readl(iommu->mmio_base + MMIO_CNTR_CONF_OFFSET); @@ -1771,11 +1731,6 @@ static void __init init_iommu_perf_ctr(struct amd_iommu *iommu) iommu->max_counters = (u8) ((val >> 7) & 0xf); return; - -pc_false: - pci_err(pdev, "Unable to read/write to IOMMU perf counter.\n"); - amd_iommu_pc_present = false; - return; } static ssize_t amd_iommu_show_cap(struct device *dev, @@ -1837,7 +1792,7 @@ static void __init late_iommu_features_init(struct amd_iommu *iommu) * IVHD and MMIO conflict. */ if (features != iommu->features) - pr_warn(FW_WARN "EFR mismatch. Use IVHD EFR (%#llx : %#llx\n).", + pr_warn(FW_WARN "EFR mismatch. Use IVHD EFR (%#llx : %#llx).\n", features, iommu->features); } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 8594b4a83043..941ba5484731 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2305,6 +2305,9 @@ static void arm_smmu_iotlb_sync(struct iommu_domain *domain, { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + if (!gather->pgsize) + return; + arm_smmu_tlb_inv_range_domain(gather->start, gather->end - gather->start + 1, gather->pgsize, true, smmu_domain); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index f985817c967a..230b6f6b3901 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -115,7 +115,7 @@ #define GERROR_PRIQ_ABT_ERR (1 << 3) #define GERROR_EVTQ_ABT_ERR (1 << 2) #define GERROR_CMDQ_ERR (1 << 0) -#define GERROR_ERR_MASK 0xfd +#define GERROR_ERR_MASK 0x1fd #define ARM_SMMU_GERRORN 0x64 diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index af765c813cc8..fdd095e1fa52 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -52,6 +52,17 @@ struct iommu_dma_cookie { }; static DEFINE_STATIC_KEY_FALSE(iommu_deferred_attach_enabled); +bool iommu_dma_forcedac __read_mostly; + +static int __init iommu_dma_forcedac_setup(char *str) +{ + int ret = kstrtobool(str, &iommu_dma_forcedac); + + if (!ret && iommu_dma_forcedac) + pr_info("Forcing DAC for PCI devices\n"); + return ret; +} +early_param("iommu.forcedac", iommu_dma_forcedac_setup); void iommu_dma_free_cpu_cached_iovas(unsigned int cpu, struct iommu_domain *domain) @@ -444,7 +455,7 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain, dma_limit = min(dma_limit, (u64)domain->geometry.aperture_end); /* Try to get PCI devices a SAC address */ - if (dma_limit > DMA_BIT_MASK(32) && dev_is_pci(dev)) + if (dma_limit > DMA_BIT_MASK(32) && !iommu_dma_forcedac && dev_is_pci(dev)) iova = alloc_iova_fast(iovad, iova_len, DMA_BIT_MASK(32) >> shift, false); diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index ee0932307d64..7e551da6c1fb 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -360,7 +360,6 @@ int intel_iommu_enabled = 0; EXPORT_SYMBOL_GPL(intel_iommu_enabled); static int dmar_map_gfx = 1; -static int dmar_forcedac; static int intel_iommu_strict; static int intel_iommu_superpage = 1; static int iommu_identity_mapping; @@ -451,8 +450,8 @@ static int __init intel_iommu_setup(char *str) dmar_map_gfx = 0; pr_info("Disable GFX device mapping\n"); } else if (!strncmp(str, "forcedac", 8)) { - pr_info("Forcing DAC for PCI devices\n"); - dmar_forcedac = 1; + pr_warn("intel_iommu=forcedac deprecated; use iommu.forcedac instead\n"); + iommu_dma_forcedac = true; } else if (!strncmp(str, "strict", 6)) { pr_info("Disable batched IOTLB flush\n"); intel_iommu_strict = 1; @@ -658,7 +657,14 @@ static int domain_update_iommu_snooping(struct intel_iommu *skip) rcu_read_lock(); for_each_active_iommu(iommu, drhd) { if (iommu != skip) { - if (!ecap_sc_support(iommu->ecap)) { + /* + * If the hardware is operating in the scalable mode, + * the snooping control is always supported since we + * always set PASID-table-entry.PGSNP bit if the domain + * is managed outside (UNMANAGED). + */ + if (!sm_supported(iommu) && + !ecap_sc_support(iommu->ecap)) { ret = 0; break; } @@ -1340,6 +1346,11 @@ static void iommu_set_root_entry(struct intel_iommu *iommu) readl, (sts & DMA_GSTS_RTPS), sts); raw_spin_unlock_irqrestore(&iommu->register_lock, flag); + + iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL); + if (sm_supported(iommu)) + qi_flush_pasid_cache(iommu, 0, QI_PC_GLOBAL, 0); + iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); } void iommu_flush_write_buffer(struct intel_iommu *iommu) @@ -2289,6 +2300,41 @@ static inline int hardware_largepage_caps(struct dmar_domain *domain, return level; } +/* + * Ensure that old small page tables are removed to make room for superpage(s). + * We're going to add new large pages, so make sure we don't remove their parent + * tables. The IOTLB/devTLBs should be flushed if any PDE/PTEs are cleared. + */ +static void switch_to_super_page(struct dmar_domain *domain, + unsigned long start_pfn, + unsigned long end_pfn, int level) +{ + unsigned long lvl_pages = lvl_to_nr_pages(level); + struct dma_pte *pte = NULL; + int i; + + while (start_pfn <= end_pfn) { + if (!pte) + pte = pfn_to_dma_pte(domain, start_pfn, &level); + + if (dma_pte_present(pte)) { + dma_pte_free_pagetable(domain, start_pfn, + start_pfn + lvl_pages - 1, + level + 1); + + for_each_domain_iommu(i, domain) + iommu_flush_iotlb_psi(g_iommus[i], domain, + start_pfn, lvl_pages, + 0, 0); + } + + pte++; + start_pfn += lvl_pages; + if (first_pte_in_page(pte)) + pte = NULL; + } +} + static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, unsigned long phys_pfn, unsigned long nr_pages, int prot) @@ -2305,8 +2351,9 @@ __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, return -EINVAL; attr = prot & (DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP); + attr |= DMA_FL_PTE_PRESENT; if (domain_use_first_level(domain)) { - attr |= DMA_FL_PTE_PRESENT | DMA_FL_PTE_XD | DMA_FL_PTE_US; + attr |= DMA_FL_PTE_XD | DMA_FL_PTE_US; if (domain->domain.type == IOMMU_DOMAIN_DMA) { attr |= DMA_FL_PTE_ACCESS; @@ -2329,22 +2376,11 @@ __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, return -ENOMEM; /* It is large page*/ if (largepage_lvl > 1) { - unsigned long nr_superpages, end_pfn; + unsigned long end_pfn; pteval |= DMA_PTE_LARGE_PAGE; - lvl_pages = lvl_to_nr_pages(largepage_lvl); - - nr_superpages = nr_pages / lvl_pages; - end_pfn = iov_pfn + nr_superpages * lvl_pages - 1; - - /* - * Ensure that old small page tables are - * removed to make room for superpage(s). - * We're adding new large pages, so make sure - * we don't remove their parent tables. - */ - dma_pte_free_pagetable(domain, iov_pfn, end_pfn, - largepage_lvl + 1); + end_pfn = ((iov_pfn + nr_pages) & level_mask(largepage_lvl)) - 1; + switch_to_super_page(domain, iov_pfn, end_pfn, largepage_lvl); } else { pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE; } @@ -2422,6 +2458,10 @@ static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn (((u16)bus) << 8) | devfn, DMA_CCMD_MASK_NOBIT, DMA_CCMD_DEVICE_INVL); + + if (sm_supported(iommu)) + qi_flush_pasid_cache(iommu, did_old, QI_PC_ALL_PASIDS, 0); + iommu->flush.flush_iotlb(iommu, did_old, 0, @@ -2505,6 +2545,9 @@ static int domain_setup_first_level(struct intel_iommu *iommu, flags |= (level == 5) ? PASID_FLAG_FL5LP : 0; + if (domain->domain.type == IOMMU_DOMAIN_UNMANAGED) + flags |= PASID_FLAG_PAGE_SNOOP; + return intel_pasid_setup_first_level(iommu, dev, (pgd_t *)pgd, pasid, domain->iommu_did[iommu->seq_id], flags); @@ -3267,8 +3310,6 @@ static int __init init_dmars(void) register_pasid_allocator(iommu); #endif iommu_set_root_entry(iommu); - iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL); - iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); } #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA @@ -3458,12 +3499,7 @@ static int init_iommu_hw(void) } iommu_flush_write_buffer(iommu); - iommu_set_root_entry(iommu); - - iommu->flush.flush_context(iommu, 0, 0, 0, - DMA_CCMD_GLOBAL_INVL); - iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); iommu_enable_translation(iommu); iommu_disable_protect_mem_regions(iommu); } @@ -3846,8 +3882,6 @@ static int intel_iommu_add(struct dmar_drhd_unit *dmaru) goto disable_iommu; iommu_set_root_entry(iommu); - iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL); - iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); iommu_enable_translation(iommu); iommu_disable_protect_mem_regions(iommu); diff --git a/drivers/iommu/intel/irq_remapping.c b/drivers/iommu/intel/irq_remapping.c index 611ef5243cb6..5c16ebe037a1 100644 --- a/drivers/iommu/intel/irq_remapping.c +++ b/drivers/iommu/intel/irq_remapping.c @@ -736,7 +736,7 @@ static int __init intel_prepare_irq_remapping(void) return -ENODEV; if (intel_cap_audit(CAP_AUDIT_STATIC_IRQR, NULL)) - goto error; + return -ENODEV; if (!dmar_ir_support()) return -ENODEV; diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index f26cb6195b2c..5093d317ff1a 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -411,6 +411,16 @@ static inline void pasid_set_page_snoop(struct pasid_entry *pe, bool value) pasid_set_bits(&pe->val[1], 1 << 23, value << 23); } +/* + * Setup the Page Snoop (PGSNP) field (Bit 88) of a scalable mode + * PASID entry. + */ +static inline void +pasid_set_pgsnp(struct pasid_entry *pe) +{ + pasid_set_bits(&pe->val[1], 1ULL << 24, 1ULL << 24); +} + /* * Setup the First Level Page table Pointer field (Bit 140~191) * of a scalable mode PASID entry. @@ -565,6 +575,9 @@ int intel_pasid_setup_first_level(struct intel_iommu *iommu, } } + if (flags & PASID_FLAG_PAGE_SNOOP) + pasid_set_pgsnp(pte); + pasid_set_domain_id(pte, did); pasid_set_address_width(pte, iommu->agaw); pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap)); @@ -643,6 +656,9 @@ int intel_pasid_setup_second_level(struct intel_iommu *iommu, pasid_set_fault_enable(pte); pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap)); + if (domain->domain.type == IOMMU_DOMAIN_UNMANAGED) + pasid_set_pgsnp(pte); + /* * Since it is a second level only translation setup, we should * set SRE bit as well (addresses are expected to be GPAs). diff --git a/drivers/iommu/intel/pasid.h b/drivers/iommu/intel/pasid.h index 444c0bec221a..086ebd697319 100644 --- a/drivers/iommu/intel/pasid.h +++ b/drivers/iommu/intel/pasid.h @@ -48,6 +48,7 @@ */ #define PASID_FLAG_SUPERVISOR_MODE BIT(0) #define PASID_FLAG_NESTED BIT(1) +#define PASID_FLAG_PAGE_SNOOP BIT(2) /* * The PASID_FLAG_FL5LP flag Indicates using 5-level paging for first- diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 574a7e657a9a..ecb6314fdd5c 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -862,7 +862,7 @@ intel_svm_prq_report(struct device *dev, struct page_req_dsc *desc) /* Fill in event data for device specific processing */ memset(&event, 0, sizeof(struct iommu_fault_event)); event.fault.type = IOMMU_FAULT_PAGE_REQ; - event.fault.prm.addr = desc->addr; + event.fault.prm.addr = (u64)desc->addr << VTD_PAGE_SHIFT; event.fault.prm.pasid = desc->pasid; event.fault.prm.grpid = desc->prg_index; event.fault.prm.perm = prq_to_iommu_prot(desc); @@ -920,7 +920,17 @@ static irqreturn_t prq_event_thread(int irq, void *d) ((unsigned long long *)req)[1]); goto no_pasid; } - + /* We shall not receive page request for supervisor SVM */ + if (req->pm_req && (req->rd_req | req->wr_req)) { + pr_err("Unexpected page request in Privilege Mode"); + /* No need to find the matching sdev as for bad_req */ + goto no_pasid; + } + /* DMA read with exec requeset is not supported. */ + if (req->exe_req && req->rd_req) { + pr_err("Execution request not supported\n"); + goto no_pasid; + } if (!svm || svm->pasid != req->pasid) { rcu_read_lock(); svm = ioasid_find(NULL, req->pasid, NULL); @@ -1021,12 +1031,12 @@ static irqreturn_t prq_event_thread(int irq, void *d) QI_PGRP_RESP_TYPE; resp.qw1 = QI_PGRP_IDX(req->prg_index) | QI_PGRP_LPIG(req->lpig); + resp.qw2 = 0; + resp.qw3 = 0; if (req->priv_data_present) memcpy(&resp.qw2, req->priv_data, sizeof(req->priv_data)); - resp.qw2 = 0; - resp.qw3 = 0; qi_submit_sync(iommu, &resp, 1, 0); } prq_advance: diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index d0b0a15dba84..e10cfa99057c 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2878,10 +2878,12 @@ EXPORT_SYMBOL_GPL(iommu_fwspec_add_ids); */ int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat) { - const struct iommu_ops *ops = dev->bus->iommu_ops; + if (dev->iommu && dev->iommu->iommu_dev) { + const struct iommu_ops *ops = dev->iommu->iommu_dev->ops; - if (ops && ops->dev_enable_feat) - return ops->dev_enable_feat(dev, feat); + if (ops->dev_enable_feat) + return ops->dev_enable_feat(dev, feat); + } return -ENODEV; } @@ -2894,10 +2896,12 @@ EXPORT_SYMBOL_GPL(iommu_dev_enable_feature); */ int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat) { - const struct iommu_ops *ops = dev->bus->iommu_ops; + if (dev->iommu && dev->iommu->iommu_dev) { + const struct iommu_ops *ops = dev->iommu->iommu_dev->ops; - if (ops && ops->dev_disable_feat) - return ops->dev_disable_feat(dev, feat); + if (ops->dev_disable_feat) + return ops->dev_disable_feat(dev, feat); + } return -EBUSY; } @@ -2905,10 +2909,12 @@ EXPORT_SYMBOL_GPL(iommu_dev_disable_feature); bool iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features feat) { - const struct iommu_ops *ops = dev->bus->iommu_ops; + if (dev->iommu && dev->iommu->iommu_dev) { + const struct iommu_ops *ops = dev->iommu->iommu_dev->ops; - if (ops && ops->dev_feat_enabled) - return ops->dev_feat_enabled(dev, feat); + if (ops->dev_feat_enabled) + return ops->dev_feat_enabled(dev, feat); + } return false; } diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 6ecc007f07cd..e168a682806a 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -688,13 +688,6 @@ static const struct iommu_ops mtk_iommu_ops = { static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) { u32 regval; - int ret; - - ret = clk_prepare_enable(data->bclk); - if (ret) { - dev_err(data->dev, "Failed to enable iommu bclk(%d)\n", ret); - return ret; - } if (data->plat_data->m4u_plat == M4U_MT8173) { regval = F_MMU_PREFETCH_RT_REPLACE_MOD | @@ -760,7 +753,6 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) if (devm_request_irq(data->dev, data->irq, mtk_iommu_isr, 0, dev_name(data->dev), (void *)data)) { writel_relaxed(0, data->base + REG_MMU_PT_BASE_ADDR); - clk_disable_unprepare(data->bclk); dev_err(data->dev, "Failed @ IRQ-%d Request\n", data->irq); return -ENODEV; } @@ -977,14 +969,19 @@ static int __maybe_unused mtk_iommu_runtime_resume(struct device *dev) void __iomem *base = data->base; int ret; - /* Avoid first resume to affect the default value of registers below. */ - if (!m4u_dom) - return 0; ret = clk_prepare_enable(data->bclk); if (ret) { dev_err(data->dev, "Failed to enable clk(%d) in resume\n", ret); return ret; } + + /* + * Uppon first resume, only enable the clk and return, since the values of the + * registers are not yet set. + */ + if (!m4u_dom) + return 0; + writel_relaxed(reg->wr_len_ctrl, base + REG_MMU_WR_LEN_CTRL); writel_relaxed(reg->misc_ctrl, base + REG_MMU_MISC_CTRL); writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM_DIS); diff --git a/drivers/irqchip/irq-gic-v3-mbi.c b/drivers/irqchip/irq-gic-v3-mbi.c index 563a9b366294..e81e89a81cb5 100644 --- a/drivers/irqchip/irq-gic-v3-mbi.c +++ b/drivers/irqchip/irq-gic-v3-mbi.c @@ -303,7 +303,7 @@ int __init mbi_init(struct fwnode_handle *fwnode, struct irq_domain *parent) reg = of_get_property(np, "mbi-alias", NULL); if (reg) { mbi_phys_base = of_translate_address(np, reg); - if (mbi_phys_base == OF_BAD_ADDR) { + if (mbi_phys_base == (phys_addr_t)OF_BAD_ADDR) { ret = -ENXIO; goto err_free_mbi; } diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index eb0ee356a629..00404024d7cd 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -648,6 +648,10 @@ static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs irqnr = gic_read_iar(); + /* Check for special IDs first */ + if ((irqnr >= 1020 && irqnr <= 1023)) + return; + if (gic_supports_nmi() && unlikely(gic_read_rpr() == GICD_INT_NMI_PRI)) { gic_handle_nmi(irqnr, regs); @@ -659,10 +663,6 @@ static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs gic_arch_enable_irqs(); } - /* Check for special IDs first */ - if ((irqnr >= 1020 && irqnr <= 1023)) - return; - if (static_branch_likely(&supports_deactivate_key)) gic_write_eoir(irqnr); else diff --git a/drivers/leds/blink/Kconfig b/drivers/leds/blink/Kconfig index 265b53476a80..6dedc58c47b3 100644 --- a/drivers/leds/blink/Kconfig +++ b/drivers/leds/blink/Kconfig @@ -9,6 +9,7 @@ if LEDS_BLINK config LEDS_BLINK_LGM tristate "LED support for Intel LGM SoC series" + depends on GPIOLIB depends on LEDS_CLASS depends on MFD_SYSCON depends on OF diff --git a/drivers/mailbox/sprd-mailbox.c b/drivers/mailbox/sprd-mailbox.c index 4c325301a2fe..94d9067dc8d0 100644 --- a/drivers/mailbox/sprd-mailbox.c +++ b/drivers/mailbox/sprd-mailbox.c @@ -60,6 +60,8 @@ struct sprd_mbox_priv { struct clk *clk; u32 outbox_fifo_depth; + struct mutex lock; + u32 refcnt; struct mbox_chan chan[SPRD_MBOX_CHAN_MAX]; }; @@ -115,7 +117,11 @@ static irqreturn_t sprd_mbox_outbox_isr(int irq, void *data) id = readl(priv->outbox_base + SPRD_MBOX_ID); chan = &priv->chan[id]; - mbox_chan_received_data(chan, (void *)msg); + if (chan->cl) + mbox_chan_received_data(chan, (void *)msg); + else + dev_warn_ratelimited(priv->dev, + "message's been dropped at ch[%d]\n", id); /* Trigger to update outbox FIFO pointer */ writel(0x1, priv->outbox_base + SPRD_MBOX_TRIGGER); @@ -215,18 +221,22 @@ static int sprd_mbox_startup(struct mbox_chan *chan) struct sprd_mbox_priv *priv = to_sprd_mbox_priv(chan->mbox); u32 val; - /* Select outbox FIFO mode and reset the outbox FIFO status */ - writel(0x0, priv->outbox_base + SPRD_MBOX_FIFO_RST); + mutex_lock(&priv->lock); + if (priv->refcnt++ == 0) { + /* Select outbox FIFO mode and reset the outbox FIFO status */ + writel(0x0, priv->outbox_base + SPRD_MBOX_FIFO_RST); - /* Enable inbox FIFO overflow and delivery interrupt */ - val = readl(priv->inbox_base + SPRD_MBOX_IRQ_MSK); - val &= ~(SPRD_INBOX_FIFO_OVERFLOW_IRQ | SPRD_INBOX_FIFO_DELIVER_IRQ); - writel(val, priv->inbox_base + SPRD_MBOX_IRQ_MSK); + /* Enable inbox FIFO overflow and delivery interrupt */ + val = readl(priv->inbox_base + SPRD_MBOX_IRQ_MSK); + val &= ~(SPRD_INBOX_FIFO_OVERFLOW_IRQ | SPRD_INBOX_FIFO_DELIVER_IRQ); + writel(val, priv->inbox_base + SPRD_MBOX_IRQ_MSK); - /* Enable outbox FIFO not empty interrupt */ - val = readl(priv->outbox_base + SPRD_MBOX_IRQ_MSK); - val &= ~SPRD_OUTBOX_FIFO_NOT_EMPTY_IRQ; - writel(val, priv->outbox_base + SPRD_MBOX_IRQ_MSK); + /* Enable outbox FIFO not empty interrupt */ + val = readl(priv->outbox_base + SPRD_MBOX_IRQ_MSK); + val &= ~SPRD_OUTBOX_FIFO_NOT_EMPTY_IRQ; + writel(val, priv->outbox_base + SPRD_MBOX_IRQ_MSK); + } + mutex_unlock(&priv->lock); return 0; } @@ -235,9 +245,13 @@ static void sprd_mbox_shutdown(struct mbox_chan *chan) { struct sprd_mbox_priv *priv = to_sprd_mbox_priv(chan->mbox); - /* Disable inbox & outbox interrupt */ - writel(SPRD_INBOX_FIFO_IRQ_MASK, priv->inbox_base + SPRD_MBOX_IRQ_MSK); - writel(SPRD_OUTBOX_FIFO_IRQ_MASK, priv->outbox_base + SPRD_MBOX_IRQ_MSK); + mutex_lock(&priv->lock); + if (--priv->refcnt == 0) { + /* Disable inbox & outbox interrupt */ + writel(SPRD_INBOX_FIFO_IRQ_MASK, priv->inbox_base + SPRD_MBOX_IRQ_MSK); + writel(SPRD_OUTBOX_FIFO_IRQ_MASK, priv->outbox_base + SPRD_MBOX_IRQ_MSK); + } + mutex_unlock(&priv->lock); } static const struct mbox_chan_ops sprd_mbox_ops = { @@ -266,6 +280,7 @@ static int sprd_mbox_probe(struct platform_device *pdev) return -ENOMEM; priv->dev = dev; + mutex_init(&priv->lock); /* * The Spreadtrum mailbox uses an inbox to send messages to the target diff --git a/drivers/md/bcache/writeback.c b/drivers/md/bcache/writeback.c index 82d4e0880a99..4fb635c0baa0 100644 --- a/drivers/md/bcache/writeback.c +++ b/drivers/md/bcache/writeback.c @@ -110,13 +110,13 @@ static void __update_writeback_rate(struct cached_dev *dc) int64_t fps; if (c->gc_stats.in_use <= BCH_WRITEBACK_FRAGMENT_THRESHOLD_MID) { - fp_term = dc->writeback_rate_fp_term_low * + fp_term = (int64_t)dc->writeback_rate_fp_term_low * (c->gc_stats.in_use - BCH_WRITEBACK_FRAGMENT_THRESHOLD_LOW); } else if (c->gc_stats.in_use <= BCH_WRITEBACK_FRAGMENT_THRESHOLD_HIGH) { - fp_term = dc->writeback_rate_fp_term_mid * + fp_term = (int64_t)dc->writeback_rate_fp_term_mid * (c->gc_stats.in_use - BCH_WRITEBACK_FRAGMENT_THRESHOLD_MID); } else { - fp_term = dc->writeback_rate_fp_term_high * + fp_term = (int64_t)dc->writeback_rate_fp_term_high * (c->gc_stats.in_use - BCH_WRITEBACK_FRAGMENT_THRESHOLD_HIGH); } fps = div_s64(dirty, dirty_buckets) * fp_term; diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index 46b5d542b8fe..362c887d33b3 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -4039,6 +4039,7 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) if (val >= (uint64_t)UINT_MAX * 1000 / HZ) { r = -EINVAL; ti->error = "Invalid bitmap_flush_interval argument"; + goto bad; } ic->bitmap_flush_interval = msecs_to_jiffies(val); } else if (!strncmp(opt_string, "internal_hash:", strlen("internal_hash:"))) { diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c index cab12b2251ba..91461b6904c1 100644 --- a/drivers/md/dm-raid.c +++ b/drivers/md/dm-raid.c @@ -1868,6 +1868,14 @@ static bool rs_takeover_requested(struct raid_set *rs) return rs->md.new_level != rs->md.level; } +/* True if layout is set to reshape. */ +static bool rs_is_layout_change(struct raid_set *rs, bool use_mddev) +{ + return (use_mddev ? rs->md.delta_disks : rs->delta_disks) || + rs->md.new_layout != rs->md.layout || + rs->md.new_chunk_sectors != rs->md.chunk_sectors; +} + /* True if @rs is requested to reshape by ctr */ static bool rs_reshape_requested(struct raid_set *rs) { @@ -1880,9 +1888,7 @@ static bool rs_reshape_requested(struct raid_set *rs) if (rs_is_raid0(rs)) return false; - change = mddev->new_layout != mddev->layout || - mddev->new_chunk_sectors != mddev->chunk_sectors || - rs->delta_disks; + change = rs_is_layout_change(rs, false); /* Historical case to support raid1 reshape without delta disks */ if (rs_is_raid1(rs)) { @@ -2817,7 +2823,7 @@ static sector_t _get_reshape_sectors(struct raid_set *rs) } /* - * + * Reshape: * - change raid layout * - change chunk size * - add disks @@ -2926,6 +2932,20 @@ static int rs_setup_reshape(struct raid_set *rs) return r; } +/* + * If the md resync thread has updated superblock with max reshape position + * at the end of a reshape but not (yet) reset the layout configuration + * changes -> reset the latter. + */ +static void rs_reset_inconclusive_reshape(struct raid_set *rs) +{ + if (!rs_is_reshaping(rs) && rs_is_layout_change(rs, true)) { + rs_set_cur(rs); + rs->md.delta_disks = 0; + rs->md.reshape_backwards = 0; + } +} + /* * Enable/disable discard support on RAID set depending on * RAID level and discard properties of underlying RAID members. @@ -3212,11 +3232,14 @@ static int raid_ctr(struct dm_target *ti, unsigned int argc, char **argv) if (r) goto bad; + /* Catch any inconclusive reshape superblock content. */ + rs_reset_inconclusive_reshape(rs); + /* Start raid set read-only and assumed clean to change in raid_resume() */ rs->md.ro = 1; rs->md.in_sync = 1; - /* Keep array frozen */ + /* Keep array frozen until resume. */ set_bit(MD_RECOVERY_FROZEN, &rs->md.recovery); /* Has to be held on running the array */ @@ -3230,7 +3253,6 @@ static int raid_ctr(struct dm_target *ti, unsigned int argc, char **argv) } r = md_start(&rs->md); - if (r) { ti->error = "Failed to start raid array"; mddev_unlock(&rs->md); diff --git a/drivers/md/dm-rq.c b/drivers/md/dm-rq.c index 13b4385f4d5a..9c3bc3711b33 100644 --- a/drivers/md/dm-rq.c +++ b/drivers/md/dm-rq.c @@ -569,6 +569,7 @@ int dm_mq_init_request_queue(struct mapped_device *md, struct dm_table *t) blk_mq_free_tag_set(md->tag_set); out_kfree_tag_set: kfree(md->tag_set); + md->tag_set = NULL; return err; } @@ -578,6 +579,7 @@ void dm_mq_cleanup_mapped_device(struct mapped_device *md) if (md->tag_set) { blk_mq_free_tag_set(md->tag_set); kfree(md->tag_set); + md->tag_set = NULL; } } diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c index 200c5d0f08bf..ea3130e11680 100644 --- a/drivers/md/md-bitmap.c +++ b/drivers/md/md-bitmap.c @@ -1722,6 +1722,8 @@ void md_bitmap_flush(struct mddev *mddev) md_bitmap_daemon_work(mddev); bitmap->daemon_lastrun -= sleep; md_bitmap_daemon_work(mddev); + if (mddev->bitmap_info.external) + md_super_wait(mddev); md_bitmap_update_sb(bitmap); } diff --git a/drivers/md/md.c b/drivers/md/md.c index 21da0c48f6c2..2a9553efc2d1 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -734,7 +734,34 @@ void mddev_init(struct mddev *mddev) } EXPORT_SYMBOL_GPL(mddev_init); +static struct mddev *mddev_find_locked(dev_t unit) +{ + struct mddev *mddev; + + list_for_each_entry(mddev, &all_mddevs, all_mddevs) + if (mddev->unit == unit) + return mddev; + + return NULL; +} + static struct mddev *mddev_find(dev_t unit) +{ + struct mddev *mddev; + + if (MAJOR(unit) != MD_MAJOR) + unit &= ~((1 << MdpMinorShift) - 1); + + spin_lock(&all_mddevs_lock); + mddev = mddev_find_locked(unit); + if (mddev) + mddev_get(mddev); + spin_unlock(&all_mddevs_lock); + + return mddev; +} + +static struct mddev *mddev_find_or_alloc(dev_t unit) { struct mddev *mddev, *new = NULL; @@ -745,13 +772,13 @@ static struct mddev *mddev_find(dev_t unit) spin_lock(&all_mddevs_lock); if (unit) { - list_for_each_entry(mddev, &all_mddevs, all_mddevs) - if (mddev->unit == unit) { - mddev_get(mddev); - spin_unlock(&all_mddevs_lock); - kfree(new); - return mddev; - } + mddev = mddev_find_locked(unit); + if (mddev) { + mddev_get(mddev); + spin_unlock(&all_mddevs_lock); + kfree(new); + return mddev; + } if (new) { list_add(&new->all_mddevs, &all_mddevs); @@ -777,12 +804,7 @@ static struct mddev *mddev_find(dev_t unit) return NULL; } - is_free = 1; - list_for_each_entry(mddev, &all_mddevs, all_mddevs) - if (mddev->unit == dev) { - is_free = 0; - break; - } + is_free = !mddev_find_locked(dev); } new->unit = dev; new->md_minor = MINOR(dev); @@ -5644,7 +5666,7 @@ static int md_alloc(dev_t dev, char *name) * writing to /sys/module/md_mod/parameters/new_array. */ static DEFINE_MUTEX(disks_mutex); - struct mddev *mddev = mddev_find(dev); + struct mddev *mddev = mddev_find_or_alloc(dev); struct gendisk *disk; int partitioned; int shift; @@ -6524,11 +6546,9 @@ static void autorun_devices(int part) md_probe(dev); mddev = mddev_find(dev); - if (!mddev || !mddev->gendisk) { - if (mddev) - mddev_put(mddev); + if (!mddev) break; - } + if (mddev_lock(mddev)) pr_warn("md: %s locked, cannot run\n", mdname(mddev)); else if (mddev->raid_disks || mddev->major_version @@ -7821,8 +7841,7 @@ static int md_open(struct block_device *bdev, fmode_t mode) /* Wait until bdev->bd_disk is definitely gone */ if (work_pending(&mddev->del_work)) flush_workqueue(md_misc_wq); - /* Then retry the open from the top */ - return -ERESTARTSYS; + return -EBUSY; } BUG_ON(mddev != bdev->bd_disk->private_data); @@ -8153,7 +8172,11 @@ static void *md_seq_start(struct seq_file *seq, loff_t *pos) loff_t l = *pos; struct mddev *mddev; - if (l >= 0x10000) + if (l == 0x10000) { + ++*pos; + return (void *)2; + } + if (l > 0x10000) return NULL; if (!l--) /* header */ @@ -9251,11 +9274,11 @@ void md_check_recovery(struct mddev *mddev) } if (mddev_is_clustered(mddev)) { - struct md_rdev *rdev; + struct md_rdev *rdev, *tmp; /* kick the device if another node issued a * remove disk. */ - rdev_for_each(rdev, mddev) { + rdev_for_each_safe(rdev, tmp, mddev) { if (test_and_clear_bit(ClusterRemove, &rdev->flags) && rdev->raid_disk < 0) md_kick_rdev_from_array(rdev); @@ -9569,7 +9592,7 @@ static int __init md_init(void) static void check_sb_changes(struct mddev *mddev, struct md_rdev *rdev) { struct mdp_superblock_1 *sb = page_address(rdev->sb_page); - struct md_rdev *rdev2; + struct md_rdev *rdev2, *tmp; int role, ret; char b[BDEVNAME_SIZE]; @@ -9586,7 +9609,7 @@ static void check_sb_changes(struct mddev *mddev, struct md_rdev *rdev) } /* Check for change of roles in the active devices */ - rdev_for_each(rdev2, mddev) { + rdev_for_each_safe(rdev2, tmp, mddev) { if (test_bit(Faulty, &rdev2->flags)) continue; diff --git a/drivers/md/persistent-data/dm-btree-internal.h b/drivers/md/persistent-data/dm-btree-internal.h index fe073d92f01e..70cfdea27efd 100644 --- a/drivers/md/persistent-data/dm-btree-internal.h +++ b/drivers/md/persistent-data/dm-btree-internal.h @@ -34,12 +34,12 @@ struct node_header { __le32 max_entries; __le32 value_size; __le32 padding; -} __packed; +} __attribute__((packed, aligned(8))); struct btree_node { struct node_header header; __le64 keys[]; -} __packed; +} __attribute__((packed, aligned(8))); /* diff --git a/drivers/md/persistent-data/dm-space-map-common.c b/drivers/md/persistent-data/dm-space-map-common.c index d8b4125e338c..a213bf11738f 100644 --- a/drivers/md/persistent-data/dm-space-map-common.c +++ b/drivers/md/persistent-data/dm-space-map-common.c @@ -339,6 +339,8 @@ int sm_ll_find_free_block(struct ll_disk *ll, dm_block_t begin, */ begin = do_div(index_begin, ll->entries_per_block); end = do_div(end, ll->entries_per_block); + if (end == 0) + end = ll->entries_per_block; for (i = index_begin; i < index_end; i++, begin = 0) { struct dm_block *blk; diff --git a/drivers/md/persistent-data/dm-space-map-common.h b/drivers/md/persistent-data/dm-space-map-common.h index 8de63ce39bdd..87e17909ef52 100644 --- a/drivers/md/persistent-data/dm-space-map-common.h +++ b/drivers/md/persistent-data/dm-space-map-common.h @@ -33,7 +33,7 @@ struct disk_index_entry { __le64 blocknr; __le32 nr_free; __le32 none_free_before; -} __packed; +} __attribute__ ((packed, aligned(8))); #define MAX_METADATA_BITMAPS 255 @@ -43,7 +43,7 @@ struct disk_metadata_index { __le64 blocknr; struct disk_index_entry index[MAX_METADATA_BITMAPS]; -} __packed; +} __attribute__ ((packed, aligned(8))); struct ll_disk; @@ -86,7 +86,7 @@ struct disk_sm_root { __le64 nr_allocated; __le64 bitmap_root; __le64 ref_count_root; -} __packed; +} __attribute__ ((packed, aligned(8))); #define ENTRIES_PER_BYTE 4 @@ -94,7 +94,7 @@ struct disk_bitmap_header { __le32 csum; __le32 not_used; __le64 blocknr; -} __packed; +} __attribute__ ((packed, aligned(8))); enum allocation_event { SM_NONE, diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index d2378765dc15..ced076ba560e 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -478,6 +478,8 @@ static void raid1_end_write_request(struct bio *bio) if (!test_bit(Faulty, &rdev->flags)) set_bit(R1BIO_WriteError, &r1_bio->state); else { + /* Fail the request */ + set_bit(R1BIO_Degraded, &r1_bio->state); /* Finished with this branch */ r1_bio->bios[mirror] = NULL; to_put = bio; diff --git a/drivers/media/common/saa7146/saa7146_core.c b/drivers/media/common/saa7146/saa7146_core.c index f2d13b71416c..e50fa0ff7c5d 100644 --- a/drivers/media/common/saa7146/saa7146_core.c +++ b/drivers/media/common/saa7146/saa7146_core.c @@ -253,7 +253,7 @@ int saa7146_pgtable_build_single(struct pci_dev *pci, struct saa7146_pgtable *pt i, sg_dma_address(list), sg_dma_len(list), list->offset); */ - for (p = 0; p * 4096 < list->length; p++, ptr++) { + for (p = 0; p * 4096 < sg_dma_len(list); p++, ptr++) { *ptr = cpu_to_le32(sg_dma_address(list) + p * 4096); nr_pages++; } diff --git a/drivers/media/common/saa7146/saa7146_video.c b/drivers/media/common/saa7146/saa7146_video.c index 7b8795eca589..66215d9106a4 100644 --- a/drivers/media/common/saa7146/saa7146_video.c +++ b/drivers/media/common/saa7146/saa7146_video.c @@ -247,9 +247,8 @@ static int saa7146_pgtable_build(struct saa7146_dev *dev, struct saa7146_buf *bu /* walk all pages, copy all page addresses to ptr1 */ for (i = 0; i < length; i++, list++) { - for (p = 0; p * 4096 < list->length; p++, ptr1++) { + for (p = 0; p * 4096 < sg_dma_len(list); p++, ptr1++) *ptr1 = cpu_to_le32(sg_dma_address(list) - list->offset); - } } /* ptr1 = pt1->cpu; diff --git a/drivers/media/dvb-core/dvbdev.c b/drivers/media/dvb-core/dvbdev.c index 5ff7bedee247..3862ddc86ec4 100644 --- a/drivers/media/dvb-core/dvbdev.c +++ b/drivers/media/dvb-core/dvbdev.c @@ -241,6 +241,7 @@ static void dvb_media_device_free(struct dvb_device *dvbdev) if (dvbdev->adapter->conn) { media_device_unregister_entity(dvbdev->adapter->conn); + kfree(dvbdev->adapter->conn); dvbdev->adapter->conn = NULL; kfree(dvbdev->adapter->conn_pads); dvbdev->adapter->conn_pads = NULL; diff --git a/drivers/media/dvb-frontends/m88ds3103.c b/drivers/media/dvb-frontends/m88ds3103.c index cfa4cdde99d8..02e8aa11e36e 100644 --- a/drivers/media/dvb-frontends/m88ds3103.c +++ b/drivers/media/dvb-frontends/m88ds3103.c @@ -1904,8 +1904,8 @@ static int m88ds3103_probe(struct i2c_client *client, dev->dt_client = i2c_new_dummy_device(client->adapter, dev->dt_addr); - if (!dev->dt_client) { - ret = -ENODEV; + if (IS_ERR(dev->dt_client)) { + ret = PTR_ERR(dev->dt_client); goto err_kfree; } } diff --git a/drivers/media/i2c/adv7511-v4l2.c b/drivers/media/i2c/adv7511-v4l2.c index a3161d709015..ab7883cff8b2 100644 --- a/drivers/media/i2c/adv7511-v4l2.c +++ b/drivers/media/i2c/adv7511-v4l2.c @@ -1964,7 +1964,7 @@ static int adv7511_remove(struct i2c_client *client) adv7511_set_isr(sd, false); adv7511_init_setup(sd); - cancel_delayed_work(&state->edid_handler); + cancel_delayed_work_sync(&state->edid_handler); i2c_unregister_device(state->i2c_edid); i2c_unregister_device(state->i2c_cec); i2c_unregister_device(state->i2c_pktmem); diff --git a/drivers/media/i2c/adv7604.c b/drivers/media/i2c/adv7604.c index 09004d928d11..d1f58795794f 100644 --- a/drivers/media/i2c/adv7604.c +++ b/drivers/media/i2c/adv7604.c @@ -3616,7 +3616,7 @@ static int adv76xx_remove(struct i2c_client *client) io_write(sd, 0x6e, 0); io_write(sd, 0x73, 0); - cancel_delayed_work(&state->delayed_work_enable_hotplug); + cancel_delayed_work_sync(&state->delayed_work_enable_hotplug); v4l2_async_unregister_subdev(sd); media_entity_cleanup(&sd->entity); adv76xx_unregister_clients(to_state(sd)); diff --git a/drivers/media/i2c/adv7842.c b/drivers/media/i2c/adv7842.c index 0855f648416d..f7d2b6cd3008 100644 --- a/drivers/media/i2c/adv7842.c +++ b/drivers/media/i2c/adv7842.c @@ -3586,7 +3586,7 @@ static int adv7842_remove(struct i2c_client *client) struct adv7842_state *state = to_state(sd); adv7842_irq_enable(sd, false); - cancel_delayed_work(&state->delayed_work_enable_hotplug); + cancel_delayed_work_sync(&state->delayed_work_enable_hotplug); v4l2_device_unregister_subdev(sd); media_entity_cleanup(&sd->entity); adv7842_unregister_clients(sd); diff --git a/drivers/media/i2c/ccs/ccs-core.c b/drivers/media/i2c/ccs/ccs-core.c index 15afbb4f5b31..4505594996bd 100644 --- a/drivers/media/i2c/ccs/ccs-core.c +++ b/drivers/media/i2c/ccs/ccs-core.c @@ -3522,11 +3522,11 @@ static int ccs_probe(struct i2c_client *client) sensor->pll.scale_n = CCS_LIM(sensor, SCALER_N_MIN); ccs_create_subdev(sensor, sensor->scaler, " scaler", 2, - MEDIA_ENT_F_CAM_SENSOR); + MEDIA_ENT_F_PROC_VIDEO_SCALER); ccs_create_subdev(sensor, sensor->binner, " binner", 2, MEDIA_ENT_F_PROC_VIDEO_SCALER); ccs_create_subdev(sensor, sensor->pixel_array, " pixel_array", 1, - MEDIA_ENT_F_PROC_VIDEO_SCALER); + MEDIA_ENT_F_CAM_SENSOR); rval = ccs_init_controls(sensor); if (rval < 0) diff --git a/drivers/media/i2c/imx219.c b/drivers/media/i2c/imx219.c index 6e3382b85a90..49ba39418360 100644 --- a/drivers/media/i2c/imx219.c +++ b/drivers/media/i2c/imx219.c @@ -1035,29 +1035,47 @@ static int imx219_start_streaming(struct imx219 *imx219) const struct imx219_reg_list *reg_list; int ret; + ret = pm_runtime_get_sync(&client->dev); + if (ret < 0) { + pm_runtime_put_noidle(&client->dev); + return ret; + } + /* Apply default values of current mode */ reg_list = &imx219->mode->reg_list; ret = imx219_write_regs(imx219, reg_list->regs, reg_list->num_of_regs); if (ret) { dev_err(&client->dev, "%s failed to set mode\n", __func__); - return ret; + goto err_rpm_put; } ret = imx219_set_framefmt(imx219); if (ret) { dev_err(&client->dev, "%s failed to set frame format: %d\n", __func__, ret); - return ret; + goto err_rpm_put; } /* Apply customized values from user */ ret = __v4l2_ctrl_handler_setup(imx219->sd.ctrl_handler); if (ret) - return ret; + goto err_rpm_put; /* set stream on register */ - return imx219_write_reg(imx219, IMX219_REG_MODE_SELECT, - IMX219_REG_VALUE_08BIT, IMX219_MODE_STREAMING); + ret = imx219_write_reg(imx219, IMX219_REG_MODE_SELECT, + IMX219_REG_VALUE_08BIT, IMX219_MODE_STREAMING); + if (ret) + goto err_rpm_put; + + /* vflip and hflip cannot change during streaming */ + __v4l2_ctrl_grab(imx219->vflip, true); + __v4l2_ctrl_grab(imx219->hflip, true); + + return 0; + +err_rpm_put: + pm_runtime_put(&client->dev); + return ret; } static void imx219_stop_streaming(struct imx219 *imx219) @@ -1070,12 +1088,16 @@ static void imx219_stop_streaming(struct imx219 *imx219) IMX219_REG_VALUE_08BIT, IMX219_MODE_STANDBY); if (ret) dev_err(&client->dev, "%s failed to set stream\n", __func__); + + __v4l2_ctrl_grab(imx219->vflip, false); + __v4l2_ctrl_grab(imx219->hflip, false); + + pm_runtime_put(&client->dev); } static int imx219_set_stream(struct v4l2_subdev *sd, int enable) { struct imx219 *imx219 = to_imx219(sd); - struct i2c_client *client = v4l2_get_subdevdata(sd); int ret = 0; mutex_lock(&imx219->mutex); @@ -1085,36 +1107,23 @@ static int imx219_set_stream(struct v4l2_subdev *sd, int enable) } if (enable) { - ret = pm_runtime_get_sync(&client->dev); - if (ret < 0) { - pm_runtime_put_noidle(&client->dev); - goto err_unlock; - } - /* * Apply default & customized values * and then start streaming. */ ret = imx219_start_streaming(imx219); if (ret) - goto err_rpm_put; + goto err_unlock; } else { imx219_stop_streaming(imx219); - pm_runtime_put(&client->dev); } imx219->streaming = enable; - /* vflip and hflip cannot change during streaming */ - __v4l2_ctrl_grab(imx219->vflip, enable); - __v4l2_ctrl_grab(imx219->hflip, enable); - mutex_unlock(&imx219->mutex); return ret; -err_rpm_put: - pm_runtime_put(&client->dev); err_unlock: mutex_unlock(&imx219->mutex); diff --git a/drivers/media/i2c/rdacm21.c b/drivers/media/i2c/rdacm21.c index dcc21515e5a4..179d107f494c 100644 --- a/drivers/media/i2c/rdacm21.c +++ b/drivers/media/i2c/rdacm21.c @@ -345,7 +345,7 @@ static int ov10640_initialize(struct rdacm21_device *dev) /* Read OV10640 ID to test communications. */ ov490_write_reg(dev, OV490_SCCB_SLAVE0_DIR, OV490_SCCB_SLAVE_READ); ov490_write_reg(dev, OV490_SCCB_SLAVE0_ADDR_HIGH, OV10640_CHIP_ID >> 8); - ov490_write_reg(dev, OV490_SCCB_SLAVE0_ADDR_LOW, (u8)OV10640_CHIP_ID); + ov490_write_reg(dev, OV490_SCCB_SLAVE0_ADDR_LOW, OV10640_CHIP_ID & 0xff); /* Trigger SCCB slave transaction and give it some time to complete. */ ov490_write_reg(dev, OV490_HOST_CMD, OV490_HOST_CMD_TRIGGER); diff --git a/drivers/media/i2c/tc358743.c b/drivers/media/i2c/tc358743.c index 831b5b54fd78..1b309bb743c7 100644 --- a/drivers/media/i2c/tc358743.c +++ b/drivers/media/i2c/tc358743.c @@ -2193,7 +2193,7 @@ static int tc358743_remove(struct i2c_client *client) del_timer_sync(&state->timer); flush_work(&state->work_i2c_poll); } - cancel_delayed_work(&state->delayed_work_enable_hotplug); + cancel_delayed_work_sync(&state->delayed_work_enable_hotplug); cec_unregister_adapter(state->cec_adap); v4l2_async_unregister_subdev(sd); v4l2_device_unregister_subdev(sd); diff --git a/drivers/media/i2c/tda1997x.c b/drivers/media/i2c/tda1997x.c index a09bf0a39d05..89bb7e6dc7a4 100644 --- a/drivers/media/i2c/tda1997x.c +++ b/drivers/media/i2c/tda1997x.c @@ -2804,7 +2804,7 @@ static int tda1997x_remove(struct i2c_client *client) media_entity_cleanup(&sd->entity); v4l2_ctrl_handler_free(&state->hdl); regulator_bulk_disable(TDA1997X_NUM_SUPPLIES, state->supplies); - cancel_delayed_work(&state->delayed_work_enable_hpd); + cancel_delayed_work_sync(&state->delayed_work_enable_hpd); mutex_destroy(&state->page_lock); mutex_destroy(&state->lock); diff --git a/drivers/media/pci/cx23885/cx23885-core.c b/drivers/media/pci/cx23885/cx23885-core.c index 22f55a7840a6..d0ca260ecf70 100644 --- a/drivers/media/pci/cx23885/cx23885-core.c +++ b/drivers/media/pci/cx23885/cx23885-core.c @@ -2077,6 +2077,15 @@ static struct { * 0x1423 is the PCI ID for the IOMMU found on Kaveri */ { PCI_VENDOR_ID_AMD, 0x1423 }, + /* 0x1481 is the PCI ID for the IOMMU found on Starship/Matisse + */ + { PCI_VENDOR_ID_AMD, 0x1481 }, + /* 0x1419 is the PCI ID for the IOMMU found on 15h (Models 10h-1fh) family + */ + { PCI_VENDOR_ID_AMD, 0x1419 }, + /* 0x5a23 is the PCI ID for the IOMMU found on RD890S/RD990 + */ + { PCI_VENDOR_ID_ATI, 0x5a23 }, }; static bool cx23885_does_need_dma_reset(void) diff --git a/drivers/media/pci/intel/ipu3/ipu3-cio2-main.c b/drivers/media/pci/intel/ipu3/ipu3-cio2-main.c index 6e8c0c230e11..fecef85bd62e 100644 --- a/drivers/media/pci/intel/ipu3/ipu3-cio2-main.c +++ b/drivers/media/pci/intel/ipu3/ipu3-cio2-main.c @@ -302,7 +302,7 @@ static int cio2_csi2_calc_timing(struct cio2_device *cio2, struct cio2_queue *q, if (!q->sensor) return -ENODEV; - freq = v4l2_get_link_freq(q->sensor->ctrl_handler, bpp, lanes); + freq = v4l2_get_link_freq(q->sensor->ctrl_handler, bpp, lanes * 2); if (freq < 0) { dev_err(dev, "error %lld, invalid link_freq\n", freq); return freq; diff --git a/drivers/media/pci/saa7134/saa7134-core.c b/drivers/media/pci/saa7134/saa7134-core.c index 391572a6ec76..efb757d5168a 100644 --- a/drivers/media/pci/saa7134/saa7134-core.c +++ b/drivers/media/pci/saa7134/saa7134-core.c @@ -243,7 +243,7 @@ int saa7134_pgtable_build(struct pci_dev *pci, struct saa7134_pgtable *pt, ptr = pt->cpu + startpage; for (i = 0; i < length; i++, list = sg_next(list)) { - for (p = 0; p * 4096 < list->length; p++, ptr++) + for (p = 0; p * 4096 < sg_dma_len(list); p++, ptr++) *ptr = cpu_to_le32(sg_dma_address(list) + list->offset + p * 4096); } diff --git a/drivers/media/pci/saa7164/saa7164-encoder.c b/drivers/media/pci/saa7164/saa7164-encoder.c index 11e1eb6a6809..1d1d32e043f1 100644 --- a/drivers/media/pci/saa7164/saa7164-encoder.c +++ b/drivers/media/pci/saa7164/saa7164-encoder.c @@ -1008,7 +1008,7 @@ int saa7164_encoder_register(struct saa7164_port *port) printk(KERN_ERR "%s() failed (errno = %d), NO PCI configuration\n", __func__, result); result = -ENOMEM; - goto failed; + goto fail_pci; } /* Establish encoder defaults here */ @@ -1062,7 +1062,7 @@ int saa7164_encoder_register(struct saa7164_port *port) 100000, ENCODER_DEF_BITRATE); if (hdl->error) { result = hdl->error; - goto failed; + goto fail_hdl; } port->std = V4L2_STD_NTSC_M; @@ -1080,7 +1080,7 @@ int saa7164_encoder_register(struct saa7164_port *port) printk(KERN_INFO "%s: can't allocate mpeg device\n", dev->name); result = -ENOMEM; - goto failed; + goto fail_hdl; } port->v4l_device->ctrl_handler = hdl; @@ -1091,10 +1091,7 @@ int saa7164_encoder_register(struct saa7164_port *port) if (result < 0) { printk(KERN_INFO "%s: can't register mpeg device\n", dev->name); - /* TODO: We're going to leak here if we don't dealloc - The buffers above. The unreg function can't deal wit it. - */ - goto failed; + goto fail_reg; } printk(KERN_INFO "%s: registered device video%d [mpeg]\n", @@ -1116,9 +1113,14 @@ int saa7164_encoder_register(struct saa7164_port *port) saa7164_api_set_encoder(port); saa7164_api_get_encoder(port); + return 0; - result = 0; -failed: +fail_reg: + video_device_release(port->v4l_device); + port->v4l_device = NULL; +fail_hdl: + v4l2_ctrl_handler_free(hdl); +fail_pci: return result; } diff --git a/drivers/media/pci/sta2x11/Kconfig b/drivers/media/pci/sta2x11/Kconfig index 4dd98f94a91e..27bb78513631 100644 --- a/drivers/media/pci/sta2x11/Kconfig +++ b/drivers/media/pci/sta2x11/Kconfig @@ -3,6 +3,7 @@ config STA2X11_VIP tristate "STA2X11 VIP Video For Linux" depends on PCI && VIDEO_V4L2 && VIRT_TO_BUS && I2C depends on STA2X11 || COMPILE_TEST + select GPIOLIB if MEDIA_SUBDRV_AUTOSELECT select VIDEO_ADV7180 if MEDIA_SUBDRV_AUTOSELECT select VIDEOBUF2_DMA_CONTIG select MEDIA_CONTROLLER diff --git a/drivers/media/platform/Kconfig b/drivers/media/platform/Kconfig index fd1831e97b22..1ddb5d6354cf 100644 --- a/drivers/media/platform/Kconfig +++ b/drivers/media/platform/Kconfig @@ -244,6 +244,7 @@ config VIDEO_MEDIATEK_JPEG depends on MTK_IOMMU_V1 || MTK_IOMMU || COMPILE_TEST depends on VIDEO_DEV && VIDEO_V4L2 depends on ARCH_MEDIATEK || COMPILE_TEST + depends on MTK_SMI || (COMPILE_TEST && MTK_SMI=n) select VIDEOBUF2_DMA_CONTIG select V4L2_MEM2MEM_DEV help @@ -271,6 +272,7 @@ config VIDEO_MEDIATEK_MDP depends on MTK_IOMMU || COMPILE_TEST depends on VIDEO_DEV && VIDEO_V4L2 depends on ARCH_MEDIATEK || COMPILE_TEST + depends on MTK_SMI || (COMPILE_TEST && MTK_SMI=n) select VIDEOBUF2_DMA_CONTIG select V4L2_MEM2MEM_DEV select VIDEO_MEDIATEK_VPU @@ -291,6 +293,7 @@ config VIDEO_MEDIATEK_VCODEC # our dependencies, to avoid missing symbols during link. depends on VIDEO_MEDIATEK_VPU || !VIDEO_MEDIATEK_VPU depends on MTK_SCP || !MTK_SCP + depends on MTK_SMI || (COMPILE_TEST && MTK_SMI=n) select VIDEOBUF2_DMA_CONTIG select V4L2_MEM2MEM_DEV select VIDEO_MEDIATEK_VCODEC_VPU if VIDEO_MEDIATEK_VPU diff --git a/drivers/media/platform/aspeed-video.c b/drivers/media/platform/aspeed-video.c index f2c4dadd6a0e..7bb6babdcade 100644 --- a/drivers/media/platform/aspeed-video.c +++ b/drivers/media/platform/aspeed-video.c @@ -514,8 +514,8 @@ static void aspeed_video_off(struct aspeed_video *video) aspeed_video_write(video, VE_INTERRUPT_STATUS, 0xffffffff); /* Turn off the relevant clocks */ - clk_disable(video->vclk); clk_disable(video->eclk); + clk_disable(video->vclk); clear_bit(VIDEO_CLOCKS_ON, &video->flags); } @@ -526,8 +526,8 @@ static void aspeed_video_on(struct aspeed_video *video) return; /* Turn on the relevant clocks */ - clk_enable(video->eclk); clk_enable(video->vclk); + clk_enable(video->eclk); set_bit(VIDEO_CLOCKS_ON, &video->flags); } @@ -1719,8 +1719,11 @@ static int aspeed_video_probe(struct platform_device *pdev) return rc; rc = aspeed_video_setup_video(video); - if (rc) + if (rc) { + clk_unprepare(video->vclk); + clk_unprepare(video->eclk); return rc; + } return 0; } diff --git a/drivers/media/platform/coda/coda-common.c b/drivers/media/platform/coda/coda-common.c index 995e95272e51..e600764dce96 100644 --- a/drivers/media/platform/coda/coda-common.c +++ b/drivers/media/platform/coda/coda-common.c @@ -2062,7 +2062,9 @@ static int coda_start_streaming(struct vb2_queue *q, unsigned int count) if (q_data_dst->fourcc == V4L2_PIX_FMT_JPEG) ctx->params.gop_size = 1; ctx->gopcounter = ctx->params.gop_size - 1; - v4l2_ctrl_s_ctrl(ctx->mb_err_cnt_ctrl, 0); + /* Only decoders have this control */ + if (ctx->mb_err_cnt_ctrl) + v4l2_ctrl_s_ctrl(ctx->mb_err_cnt_ctrl, 0); ret = ctx->ops->start_streaming(ctx); if (ctx->inst_type == CODA_INST_DECODER) { diff --git a/drivers/media/platform/meson/ge2d/ge2d.c b/drivers/media/platform/meson/ge2d/ge2d.c index 153612ca96fc..a1393fefa8ae 100644 --- a/drivers/media/platform/meson/ge2d/ge2d.c +++ b/drivers/media/platform/meson/ge2d/ge2d.c @@ -757,7 +757,7 @@ static int ge2d_s_ctrl(struct v4l2_ctrl *ctrl) if (ctrl->val == 90) { ctx->hflip = 0; - ctx->vflip = 0; + ctx->vflip = 1; ctx->xy_swap = 1; } else if (ctrl->val == 180) { ctx->hflip = 1; @@ -765,7 +765,7 @@ static int ge2d_s_ctrl(struct v4l2_ctrl *ctrl) ctx->xy_swap = 0; } else if (ctrl->val == 270) { ctx->hflip = 1; - ctx->vflip = 1; + ctx->vflip = 0; ctx->xy_swap = 1; } else { ctx->hflip = 0; diff --git a/drivers/media/platform/qcom/venus/core.c b/drivers/media/platform/qcom/venus/core.c index f9896c121fd8..ae374bb2a48f 100644 --- a/drivers/media/platform/qcom/venus/core.c +++ b/drivers/media/platform/qcom/venus/core.c @@ -218,18 +218,17 @@ static int venus_probe(struct platform_device *pdev) return -ENOMEM; core->dev = dev; - platform_set_drvdata(pdev, core); r = platform_get_resource(pdev, IORESOURCE_MEM, 0); core->base = devm_ioremap_resource(dev, r); if (IS_ERR(core->base)) return PTR_ERR(core->base); - core->video_path = of_icc_get(dev, "video-mem"); + core->video_path = devm_of_icc_get(dev, "video-mem"); if (IS_ERR(core->video_path)) return PTR_ERR(core->video_path); - core->cpucfg_path = of_icc_get(dev, "cpu-cfg"); + core->cpucfg_path = devm_of_icc_get(dev, "cpu-cfg"); if (IS_ERR(core->cpucfg_path)) return PTR_ERR(core->cpucfg_path); @@ -248,7 +247,7 @@ static int venus_probe(struct platform_device *pdev) return -ENODEV; if (core->pm_ops->core_get) { - ret = core->pm_ops->core_get(dev); + ret = core->pm_ops->core_get(core); if (ret) return ret; } @@ -273,6 +272,12 @@ static int venus_probe(struct platform_device *pdev) if (ret) goto err_core_put; + ret = v4l2_device_register(dev, &core->v4l2_dev); + if (ret) + goto err_core_deinit; + + platform_set_drvdata(pdev, core); + pm_runtime_enable(dev); ret = pm_runtime_get_sync(dev); @@ -307,10 +312,6 @@ static int venus_probe(struct platform_device *pdev) if (ret) goto err_venus_shutdown; - ret = v4l2_device_register(dev, &core->v4l2_dev); - if (ret) - goto err_core_deinit; - ret = pm_runtime_put_sync(dev); if (ret) { pm_runtime_get_noresume(dev); @@ -323,8 +324,6 @@ static int venus_probe(struct platform_device *pdev) err_dev_unregister: v4l2_device_unregister(&core->v4l2_dev); -err_core_deinit: - hfi_core_deinit(core, false); err_venus_shutdown: venus_shutdown(core); err_runtime_disable: @@ -332,9 +331,11 @@ static int venus_probe(struct platform_device *pdev) pm_runtime_set_suspended(dev); pm_runtime_disable(dev); hfi_destroy(core); +err_core_deinit: + hfi_core_deinit(core, false); err_core_put: if (core->pm_ops->core_put) - core->pm_ops->core_put(dev); + core->pm_ops->core_put(core); return ret; } @@ -360,14 +361,14 @@ static int venus_remove(struct platform_device *pdev) pm_runtime_disable(dev); if (pm_ops->core_put) - pm_ops->core_put(dev); + pm_ops->core_put(core); - hfi_destroy(core); + v4l2_device_unregister(&core->v4l2_dev); - icc_put(core->video_path); - icc_put(core->cpucfg_path); + hfi_destroy(core); v4l2_device_unregister(&core->v4l2_dev); + mutex_destroy(&core->pm_lock); mutex_destroy(&core->lock); venus_dbgfs_deinit(core); @@ -396,7 +397,7 @@ static __maybe_unused int venus_runtime_suspend(struct device *dev) return ret; if (pm_ops->core_power) { - ret = pm_ops->core_power(dev, POWER_OFF); + ret = pm_ops->core_power(core, POWER_OFF); if (ret) return ret; } @@ -414,7 +415,7 @@ static __maybe_unused int venus_runtime_suspend(struct device *dev) err_video_path: icc_set_bw(core->cpucfg_path, kbps_to_icc(1000), 0); err_cpucfg_path: - pm_ops->core_power(dev, POWER_ON); + pm_ops->core_power(core, POWER_ON); return ret; } @@ -434,7 +435,7 @@ static __maybe_unused int venus_runtime_resume(struct device *dev) return ret; if (pm_ops->core_power) { - ret = pm_ops->core_power(dev, POWER_ON); + ret = pm_ops->core_power(core, POWER_ON); if (ret) return ret; } diff --git a/drivers/media/platform/qcom/venus/hfi_cmds.c b/drivers/media/platform/qcom/venus/hfi_cmds.c index 4f7565834469..558510a8dfc8 100644 --- a/drivers/media/platform/qcom/venus/hfi_cmds.c +++ b/drivers/media/platform/qcom/venus/hfi_cmds.c @@ -1039,6 +1039,18 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt, pkt->shdr.hdr.size += sizeof(u32) + sizeof(*hierp); break; } + case HFI_PROPERTY_PARAM_UNCOMPRESSED_PLANE_ACTUAL_INFO: { + struct hfi_uncompressed_plane_actual_info *in = pdata; + struct hfi_uncompressed_plane_actual_info *info = prop_data; + + info->buffer_type = in->buffer_type; + info->num_planes = in->num_planes; + info->plane_format[0] = in->plane_format[0]; + if (in->num_planes > 1) + info->plane_format[1] = in->plane_format[1]; + pkt->shdr.hdr.size += sizeof(u32) + sizeof(*info); + break; + } /* FOLLOWING PROPERTIES ARE NOT IMPLEMENTED IN CORE YET */ case HFI_PROPERTY_CONFIG_BUFFER_REQUIREMENTS: @@ -1205,18 +1217,6 @@ pkt_session_set_property_4xx(struct hfi_session_set_property_pkt *pkt, pkt->shdr.hdr.size += sizeof(u32) + sizeof(*cu); break; } - case HFI_PROPERTY_PARAM_UNCOMPRESSED_PLANE_ACTUAL_INFO: { - struct hfi_uncompressed_plane_actual_info *in = pdata; - struct hfi_uncompressed_plane_actual_info *info = prop_data; - - info->buffer_type = in->buffer_type; - info->num_planes = in->num_planes; - info->plane_format[0] = in->plane_format[0]; - if (in->num_planes > 1) - info->plane_format[1] = in->plane_format[1]; - pkt->shdr.hdr.size += sizeof(u32) + sizeof(*info); - break; - } case HFI_PROPERTY_CONFIG_VENC_MAX_BITRATE: case HFI_PROPERTY_CONFIG_VDEC_POST_LOOP_DEBLOCKER: case HFI_PROPERTY_PARAM_BUFFER_ALLOC_MODE: diff --git a/drivers/media/platform/qcom/venus/hfi_parser.c b/drivers/media/platform/qcom/venus/hfi_parser.c index 7263c0c32695..5b8389b98299 100644 --- a/drivers/media/platform/qcom/venus/hfi_parser.c +++ b/drivers/media/platform/qcom/venus/hfi_parser.c @@ -235,13 +235,13 @@ static int hfi_platform_parser(struct venus_core *core, struct venus_inst *inst) u32 enc_codecs, dec_codecs, count = 0; unsigned int entries; - if (inst) - return 0; - plat = hfi_platform_get(core->res->hfi_version); if (!plat) return -EINVAL; + if (inst) + return 0; + if (plat->codecs) plat->codecs(&enc_codecs, &dec_codecs, &count); @@ -277,8 +277,10 @@ u32 hfi_parser(struct venus_core *core, struct venus_inst *inst, void *buf, parser_init(inst, &codecs, &domain); - core->codecs_count = 0; - memset(core->caps, 0, sizeof(core->caps)); + if (core->res->hfi_version > HFI_VERSION_1XX) { + core->codecs_count = 0; + memset(core->caps, 0, sizeof(core->caps)); + } while (words_count) { data = word + 1; diff --git a/drivers/media/platform/qcom/venus/pm_helpers.c b/drivers/media/platform/qcom/venus/pm_helpers.c index 43c4e3d9e281..95b4d40ff6a5 100644 --- a/drivers/media/platform/qcom/venus/pm_helpers.c +++ b/drivers/media/platform/qcom/venus/pm_helpers.c @@ -277,16 +277,28 @@ static int load_scale_v1(struct venus_inst *inst) return 0; } -static int core_get_v1(struct device *dev) +static int core_get_v1(struct venus_core *core) { - struct venus_core *core = dev_get_drvdata(dev); + int ret; + + ret = core_clks_get(core); + if (ret) + return ret; - return core_clks_get(core); + core->opp_table = dev_pm_opp_set_clkname(core->dev, "core"); + if (IS_ERR(core->opp_table)) + return PTR_ERR(core->opp_table); + + return 0; } -static int core_power_v1(struct device *dev, int on) +static void core_put_v1(struct venus_core *core) +{ + dev_pm_opp_put_clkname(core->opp_table); +} + +static int core_power_v1(struct venus_core *core, int on) { - struct venus_core *core = dev_get_drvdata(dev); int ret = 0; if (on == POWER_ON) @@ -299,6 +311,7 @@ static int core_power_v1(struct device *dev, int on) static const struct venus_pm_ops pm_ops_v1 = { .core_get = core_get_v1, + .core_put = core_put_v1, .core_power = core_power_v1, .load_scale = load_scale_v1, }; @@ -371,6 +384,7 @@ static int venc_power_v3(struct device *dev, int on) static const struct venus_pm_ops pm_ops_v3 = { .core_get = core_get_v1, + .core_put = core_put_v1, .core_power = core_power_v1, .vdec_get = vdec_get_v3, .vdec_power = vdec_power_v3, @@ -753,12 +767,12 @@ static int venc_power_v4(struct device *dev, int on) return ret; } -static int vcodec_domains_get(struct device *dev) +static int vcodec_domains_get(struct venus_core *core) { int ret; struct opp_table *opp_table; struct device **opp_virt_dev; - struct venus_core *core = dev_get_drvdata(dev); + struct device *dev = core->dev; const struct venus_resources *res = core->res; struct device *pd; unsigned int i; @@ -809,9 +823,8 @@ static int vcodec_domains_get(struct device *dev) return ret; } -static void vcodec_domains_put(struct device *dev) +static void vcodec_domains_put(struct venus_core *core) { - struct venus_core *core = dev_get_drvdata(dev); const struct venus_resources *res = core->res; unsigned int i; @@ -834,9 +847,9 @@ static void vcodec_domains_put(struct device *dev) dev_pm_opp_detach_genpd(core->opp_table); } -static int core_get_v4(struct device *dev) +static int core_get_v4(struct venus_core *core) { - struct venus_core *core = dev_get_drvdata(dev); + struct device *dev = core->dev; const struct venus_resources *res = core->res; int ret; @@ -875,7 +888,7 @@ static int core_get_v4(struct device *dev) } } - ret = vcodec_domains_get(dev); + ret = vcodec_domains_get(core); if (ret) { if (core->has_opp_table) dev_pm_opp_of_remove_table(dev); @@ -886,14 +899,14 @@ static int core_get_v4(struct device *dev) return 0; } -static void core_put_v4(struct device *dev) +static void core_put_v4(struct venus_core *core) { - struct venus_core *core = dev_get_drvdata(dev); + struct device *dev = core->dev; if (legacy_binding) return; - vcodec_domains_put(dev); + vcodec_domains_put(core); if (core->has_opp_table) dev_pm_opp_of_remove_table(dev); @@ -901,9 +914,9 @@ static void core_put_v4(struct device *dev) } -static int core_power_v4(struct device *dev, int on) +static int core_power_v4(struct venus_core *core, int on) { - struct venus_core *core = dev_get_drvdata(dev); + struct device *dev = core->dev; struct device *pmctrl = core->pmdomains[0]; int ret = 0; diff --git a/drivers/media/platform/qcom/venus/pm_helpers.h b/drivers/media/platform/qcom/venus/pm_helpers.h index aa2f6afa2354..a492c50c5543 100644 --- a/drivers/media/platform/qcom/venus/pm_helpers.h +++ b/drivers/media/platform/qcom/venus/pm_helpers.h @@ -4,14 +4,15 @@ #define __VENUS_PM_HELPERS_H__ struct device; +struct venus_core; #define POWER_ON 1 #define POWER_OFF 0 struct venus_pm_ops { - int (*core_get)(struct device *dev); - void (*core_put)(struct device *dev); - int (*core_power)(struct device *dev, int on); + int (*core_get)(struct venus_core *core); + void (*core_put)(struct venus_core *core); + int (*core_power)(struct venus_core *core, int on); int (*vdec_get)(struct device *dev); void (*vdec_put)(struct device *dev); diff --git a/drivers/media/platform/qcom/venus/venc_ctrls.c b/drivers/media/platform/qcom/venus/venc_ctrls.c index a52b80055173..abef0037bf55 100644 --- a/drivers/media/platform/qcom/venus/venc_ctrls.c +++ b/drivers/media/platform/qcom/venus/venc_ctrls.c @@ -359,7 +359,7 @@ int venc_ctrl_init(struct venus_inst *inst) V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME, ~((1 << V4L2_MPEG_VIDEO_HEADER_MODE_SEPARATE) | (1 << V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME)), - V4L2_MPEG_VIDEO_HEADER_MODE_SEPARATE); + V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME); v4l2_ctrl_new_std_menu(&inst->ctrl_handler, &venc_ctrl_ops, V4L2_CID_MPEG_VIDEO_MULTI_SLICE_MODE, diff --git a/drivers/media/platform/rockchip/rkisp1/rkisp1-resizer.c b/drivers/media/platform/rockchip/rkisp1/rkisp1-resizer.c index 813670ed9577..79deed8adcea 100644 --- a/drivers/media/platform/rockchip/rkisp1/rkisp1-resizer.c +++ b/drivers/media/platform/rockchip/rkisp1/rkisp1-resizer.c @@ -520,14 +520,15 @@ static void rkisp1_rsz_set_src_fmt(struct rkisp1_resizer *rsz, struct v4l2_mbus_framefmt *format, unsigned int which) { - const struct rkisp1_isp_mbus_info *mbus_info; - struct v4l2_mbus_framefmt *src_fmt; + const struct rkisp1_isp_mbus_info *sink_mbus_info; + struct v4l2_mbus_framefmt *src_fmt, *sink_fmt; + sink_fmt = rkisp1_rsz_get_pad_fmt(rsz, cfg, RKISP1_RSZ_PAD_SINK, which); src_fmt = rkisp1_rsz_get_pad_fmt(rsz, cfg, RKISP1_RSZ_PAD_SRC, which); - mbus_info = rkisp1_isp_mbus_info_get(src_fmt->code); + sink_mbus_info = rkisp1_isp_mbus_info_get(sink_fmt->code); /* for YUV formats, userspace can change the mbus code on the src pad if it is supported */ - if (mbus_info->pixel_enc == V4L2_PIXEL_ENC_YUV && + if (sink_mbus_info->pixel_enc == V4L2_PIXEL_ENC_YUV && rkisp1_rsz_get_yuv_mbus_info(format->code)) src_fmt->code = format->code; diff --git a/drivers/media/platform/sti/bdisp/bdisp-debug.c b/drivers/media/platform/sti/bdisp/bdisp-debug.c index 2b270093009c..a27f638df11c 100644 --- a/drivers/media/platform/sti/bdisp/bdisp-debug.c +++ b/drivers/media/platform/sti/bdisp/bdisp-debug.c @@ -480,7 +480,7 @@ static int regs_show(struct seq_file *s, void *data) int ret; unsigned int i; - ret = pm_runtime_get_sync(bdisp->dev); + ret = pm_runtime_resume_and_get(bdisp->dev); if (ret < 0) { seq_puts(s, "Cannot wake up IP\n"); return 0; diff --git a/drivers/media/platform/sunxi/sun6i-csi/sun6i_video.c b/drivers/media/platform/sunxi/sun6i-csi/sun6i_video.c index b55de9ab64d8..3181d0781b61 100644 --- a/drivers/media/platform/sunxi/sun6i-csi/sun6i_video.c +++ b/drivers/media/platform/sunxi/sun6i-csi/sun6i_video.c @@ -151,8 +151,10 @@ static int sun6i_video_start_streaming(struct vb2_queue *vq, unsigned int count) } subdev = sun6i_video_remote_subdev(video, NULL); - if (!subdev) + if (!subdev) { + ret = -EINVAL; goto stop_media_pipeline; + } config.pixelformat = video->fmt.fmt.pix.pixelformat; config.code = video->mbus_code; diff --git a/drivers/media/platform/sunxi/sun8i-di/sun8i-di.c b/drivers/media/platform/sunxi/sun8i-di/sun8i-di.c index ed863bf5ea80..671e4a928993 100644 --- a/drivers/media/platform/sunxi/sun8i-di/sun8i-di.c +++ b/drivers/media/platform/sunxi/sun8i-di/sun8i-di.c @@ -589,7 +589,7 @@ static int deinterlace_start_streaming(struct vb2_queue *vq, unsigned int count) int ret; if (V4L2_TYPE_IS_OUTPUT(vq->type)) { - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0) { dev_err(dev, "Failed to enable module\n"); diff --git a/drivers/media/rc/ite-cir.c b/drivers/media/rc/ite-cir.c index 0c6229592e13..e5c4a6941d26 100644 --- a/drivers/media/rc/ite-cir.c +++ b/drivers/media/rc/ite-cir.c @@ -276,8 +276,14 @@ static irqreturn_t ite_cir_isr(int irq, void *data) /* read the interrupt flags */ iflags = dev->params.get_irq_causes(dev); + /* Check for RX overflow */ + if (iflags & ITE_IRQ_RX_FIFO_OVERRUN) { + dev_warn(&dev->rdev->dev, "receive overflow\n"); + ir_raw_event_reset(dev->rdev); + } + /* check for the receive interrupt */ - if (iflags & (ITE_IRQ_RX_FIFO | ITE_IRQ_RX_FIFO_OVERRUN)) { + if (iflags & ITE_IRQ_RX_FIFO) { /* read the FIFO bytes */ rx_bytes = dev->params.get_rx_bytes(dev, rx_buf, diff --git a/drivers/media/test-drivers/vivid/vivid-core.c b/drivers/media/test-drivers/vivid/vivid-core.c index 0dc65ef3aa14..ca0ebf6ad9cc 100644 --- a/drivers/media/test-drivers/vivid/vivid-core.c +++ b/drivers/media/test-drivers/vivid/vivid-core.c @@ -205,13 +205,13 @@ static const u8 vivid_hdmi_edid[256] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x7b, - 0x02, 0x03, 0x3f, 0xf0, 0x51, 0x61, 0x60, 0x5f, + 0x02, 0x03, 0x3f, 0xf1, 0x51, 0x61, 0x60, 0x5f, 0x5e, 0x5d, 0x10, 0x1f, 0x04, 0x13, 0x22, 0x21, 0x20, 0x05, 0x14, 0x02, 0x11, 0x01, 0x23, 0x09, 0x07, 0x07, 0x83, 0x01, 0x00, 0x00, 0x6d, 0x03, 0x0c, 0x00, 0x10, 0x00, 0x00, 0x3c, 0x21, 0x00, 0x60, 0x01, 0x02, 0x03, 0x67, 0xd8, 0x5d, 0xc4, - 0x01, 0x78, 0x00, 0x00, 0xe2, 0x00, 0xea, 0xe3, + 0x01, 0x78, 0x00, 0x00, 0xe2, 0x00, 0xca, 0xe3, 0x05, 0x00, 0x00, 0xe3, 0x06, 0x01, 0x00, 0x4d, 0xd0, 0x00, 0xa0, 0xf0, 0x70, 0x3e, 0x80, 0x30, 0x20, 0x35, 0x00, 0xc0, 0x1c, 0x32, 0x00, 0x00, @@ -220,7 +220,7 @@ static const u8 vivid_hdmi_edid[256] = { 0x00, 0x00, 0x1a, 0x1a, 0x1d, 0x00, 0x80, 0x51, 0xd0, 0x1c, 0x20, 0x40, 0x80, 0x35, 0x00, 0xc0, 0x1c, 0x32, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x82, }; static int vidioc_querycap(struct file *file, void *priv, diff --git a/drivers/media/test-drivers/vivid/vivid-vid-out.c b/drivers/media/test-drivers/vivid/vivid-vid-out.c index ac1e981e8342..9f731f085179 100644 --- a/drivers/media/test-drivers/vivid/vivid-vid-out.c +++ b/drivers/media/test-drivers/vivid/vivid-vid-out.c @@ -1021,7 +1021,7 @@ int vivid_vid_out_s_fbuf(struct file *file, void *fh, return -EINVAL; } dev->fbuf_out_flags &= ~(chroma_flags | alpha_flags); - dev->fbuf_out_flags = a->flags & (chroma_flags | alpha_flags); + dev->fbuf_out_flags |= a->flags & (chroma_flags | alpha_flags); return 0; } diff --git a/drivers/media/tuners/m88rs6000t.c b/drivers/media/tuners/m88rs6000t.c index b3505f402476..8647c50b66e5 100644 --- a/drivers/media/tuners/m88rs6000t.c +++ b/drivers/media/tuners/m88rs6000t.c @@ -525,7 +525,7 @@ static int m88rs6000t_get_rf_strength(struct dvb_frontend *fe, u16 *strength) PGA2_cri = PGA2_GC >> 2; PGA2_crf = PGA2_GC & 0x03; - for (i = 0; i <= RF_GC; i++) + for (i = 0; i <= RF_GC && i < ARRAY_SIZE(RFGS); i++) RFG += RFGS[i]; if (RF_GC == 0) @@ -537,12 +537,12 @@ static int m88rs6000t_get_rf_strength(struct dvb_frontend *fe, u16 *strength) if (RF_GC == 3) RFG += 100; - for (i = 0; i <= IF_GC; i++) + for (i = 0; i <= IF_GC && i < ARRAY_SIZE(IFGS); i++) IFG += IFGS[i]; TIAG = TIA_GC * TIA_GS; - for (i = 0; i <= BB_GC; i++) + for (i = 0; i <= BB_GC && i < ARRAY_SIZE(BBGS); i++) BBG += BBGS[i]; PGA2G = PGA2_cri * PGA2_cri_GS + PGA2_crf * PGA2_crf_GS; diff --git a/drivers/media/usb/dvb-usb/dvb-usb-init.c b/drivers/media/usb/dvb-usb/dvb-usb-init.c index c1a7634e27b4..28e1fd64dd3c 100644 --- a/drivers/media/usb/dvb-usb/dvb-usb-init.c +++ b/drivers/media/usb/dvb-usb/dvb-usb-init.c @@ -79,11 +79,17 @@ static int dvb_usb_adapter_init(struct dvb_usb_device *d, short *adapter_nrs) } } - if ((ret = dvb_usb_adapter_stream_init(adap)) || - (ret = dvb_usb_adapter_dvb_init(adap, adapter_nrs)) || - (ret = dvb_usb_adapter_frontend_init(adap))) { + ret = dvb_usb_adapter_stream_init(adap); + if (ret) return ret; - } + + ret = dvb_usb_adapter_dvb_init(adap, adapter_nrs); + if (ret) + goto dvb_init_err; + + ret = dvb_usb_adapter_frontend_init(adap); + if (ret) + goto frontend_init_err; /* use exclusive FE lock if there is multiple shared FEs */ if (adap->fe_adap[1].fe) @@ -103,6 +109,12 @@ static int dvb_usb_adapter_init(struct dvb_usb_device *d, short *adapter_nrs) } return 0; + +frontend_init_err: + dvb_usb_adapter_dvb_exit(adap); +dvb_init_err: + dvb_usb_adapter_stream_exit(adap); + return ret; } static int dvb_usb_adapter_exit(struct dvb_usb_device *d) @@ -158,22 +170,20 @@ static int dvb_usb_init(struct dvb_usb_device *d, short *adapter_nums) if (d->props.priv_init != NULL) { ret = d->props.priv_init(d); - if (ret != 0) { - kfree(d->priv); - d->priv = NULL; - return ret; - } + if (ret != 0) + goto err_priv_init; } } /* check the capabilities and set appropriate variables */ dvb_usb_device_power_ctrl(d, 1); - if ((ret = dvb_usb_i2c_init(d)) || - (ret = dvb_usb_adapter_init(d, adapter_nums))) { - dvb_usb_exit(d); - return ret; - } + ret = dvb_usb_i2c_init(d); + if (ret) + goto err_i2c_init; + ret = dvb_usb_adapter_init(d, adapter_nums); + if (ret) + goto err_adapter_init; if ((ret = dvb_usb_remote_init(d))) err("could not initialize remote control."); @@ -181,6 +191,17 @@ static int dvb_usb_init(struct dvb_usb_device *d, short *adapter_nums) dvb_usb_device_power_ctrl(d, 0); return 0; + +err_adapter_init: + dvb_usb_adapter_exit(d); +err_i2c_init: + dvb_usb_i2c_exit(d); + if (d->priv && d->props.priv_destroy) + d->props.priv_destroy(d); +err_priv_init: + kfree(d->priv); + d->priv = NULL; + return ret; } /* determine the name and the state of the just found USB device */ @@ -255,41 +276,50 @@ int dvb_usb_device_init(struct usb_interface *intf, if (du != NULL) *du = NULL; - if ((desc = dvb_usb_find_device(udev, props, &cold)) == NULL) { + d = kzalloc(sizeof(*d), GFP_KERNEL); + if (!d) { + err("no memory for 'struct dvb_usb_device'"); + return -ENOMEM; + } + + memcpy(&d->props, props, sizeof(struct dvb_usb_device_properties)); + + desc = dvb_usb_find_device(udev, &d->props, &cold); + if (!desc) { deb_err("something went very wrong, device was not found in current device list - let's see what comes next.\n"); - return -ENODEV; + ret = -ENODEV; + goto error; } if (cold) { info("found a '%s' in cold state, will try to load a firmware", desc->name); ret = dvb_usb_download_firmware(udev, props); if (!props->no_reconnect || ret != 0) - return ret; + goto error; } info("found a '%s' in warm state.", desc->name); - d = kzalloc(sizeof(struct dvb_usb_device), GFP_KERNEL); - if (d == NULL) { - err("no memory for 'struct dvb_usb_device'"); - return -ENOMEM; - } - d->udev = udev; - memcpy(&d->props, props, sizeof(struct dvb_usb_device_properties)); d->desc = desc; d->owner = owner; usb_set_intfdata(intf, d); - if (du != NULL) + ret = dvb_usb_init(d, adapter_nums); + if (ret) { + info("%s error while loading driver (%d)", desc->name, ret); + goto error; + } + + if (du) *du = d; - ret = dvb_usb_init(d, adapter_nums); + info("%s successfully initialized and connected.", desc->name); + return 0; - if (ret == 0) - info("%s successfully initialized and connected.", desc->name); - else - info("%s error while loading driver (%d)", desc->name, ret); + error: + usb_set_intfdata(intf, NULL); + kfree(d); return ret; } EXPORT_SYMBOL(dvb_usb_device_init); diff --git a/drivers/media/usb/dvb-usb/dvb-usb.h b/drivers/media/usb/dvb-usb/dvb-usb.h index 741be0e69447..2b8ad2bde8a4 100644 --- a/drivers/media/usb/dvb-usb/dvb-usb.h +++ b/drivers/media/usb/dvb-usb/dvb-usb.h @@ -487,7 +487,7 @@ extern int __must_check dvb_usb_generic_write(struct dvb_usb_device *, u8 *, u16); /* commonly used remote control parsing */ -extern int dvb_usb_nec_rc_key_to_event(struct dvb_usb_device *, u8[], u32 *, int *); +extern int dvb_usb_nec_rc_key_to_event(struct dvb_usb_device *, u8[5], u32 *, int *); /* commonly used firmware download types and function */ struct hexline { diff --git a/drivers/media/usb/em28xx/em28xx-dvb.c b/drivers/media/usb/em28xx/em28xx-dvb.c index 526424279637..471bd74667e3 100644 --- a/drivers/media/usb/em28xx/em28xx-dvb.c +++ b/drivers/media/usb/em28xx/em28xx-dvb.c @@ -2010,6 +2010,7 @@ static int em28xx_dvb_init(struct em28xx *dev) return result; out_free: + em28xx_uninit_usb_xfer(dev, EM28XX_DIGITAL_MODE); kfree(dvb); dev->dvb = NULL; goto ret; diff --git a/drivers/media/usb/gspca/gspca.c b/drivers/media/usb/gspca/gspca.c index 158c8e28ed2c..47d8f28bfdfc 100644 --- a/drivers/media/usb/gspca/gspca.c +++ b/drivers/media/usb/gspca/gspca.c @@ -1576,6 +1576,8 @@ int gspca_dev_probe2(struct usb_interface *intf, #endif v4l2_ctrl_handler_free(gspca_dev->vdev.ctrl_handler); v4l2_device_unregister(&gspca_dev->v4l2_dev); + if (sd_desc->probe_error) + sd_desc->probe_error(gspca_dev); kfree(gspca_dev->usb_buf); kfree(gspca_dev); return ret; diff --git a/drivers/media/usb/gspca/gspca.h b/drivers/media/usb/gspca/gspca.h index b0ced2e14006..a6554d5e9e1a 100644 --- a/drivers/media/usb/gspca/gspca.h +++ b/drivers/media/usb/gspca/gspca.h @@ -105,6 +105,7 @@ struct sd_desc { cam_cf_op config; /* called on probe */ cam_op init; /* called on probe and resume */ cam_op init_controls; /* called on probe */ + cam_v_op probe_error; /* called if probe failed, do cleanup here */ cam_op start; /* called on stream on after URBs creation */ cam_pkt_op pkt_scan; /* optional operations */ diff --git a/drivers/media/usb/gspca/sq905.c b/drivers/media/usb/gspca/sq905.c index 97799cfb832e..949111070971 100644 --- a/drivers/media/usb/gspca/sq905.c +++ b/drivers/media/usb/gspca/sq905.c @@ -158,7 +158,7 @@ static int sq905_read_data(struct gspca_dev *gspca_dev, u8 *data, int size, int need_lock) { int ret; - int act_len; + int act_len = 0; gspca_dev->usb_buf[0] = '\0'; if (need_lock) diff --git a/drivers/media/usb/gspca/stv06xx/stv06xx.c b/drivers/media/usb/gspca/stv06xx/stv06xx.c index 95673fc0a99c..d9bc2aacc885 100644 --- a/drivers/media/usb/gspca/stv06xx/stv06xx.c +++ b/drivers/media/usb/gspca/stv06xx/stv06xx.c @@ -529,12 +529,21 @@ static int sd_int_pkt_scan(struct gspca_dev *gspca_dev, static int stv06xx_config(struct gspca_dev *gspca_dev, const struct usb_device_id *id); +static void stv06xx_probe_error(struct gspca_dev *gspca_dev) +{ + struct sd *sd = (struct sd *)gspca_dev; + + kfree(sd->sensor_priv); + sd->sensor_priv = NULL; +} + /* sub-driver description */ static const struct sd_desc sd_desc = { .name = MODULE_NAME, .config = stv06xx_config, .init = stv06xx_init, .init_controls = stv06xx_init_controls, + .probe_error = stv06xx_probe_error, .start = stv06xx_start, .stopN = stv06xx_stopN, .pkt_scan = stv06xx_pkt_scan, diff --git a/drivers/media/usb/uvc/uvc_driver.c b/drivers/media/usb/uvc/uvc_driver.c index 30ef2a3110f7..9a791d8ef200 100644 --- a/drivers/media/usb/uvc/uvc_driver.c +++ b/drivers/media/usb/uvc/uvc_driver.c @@ -1712,10 +1712,35 @@ static int uvc_scan_chain_forward(struct uvc_video_chain *chain, if (forward->bNrInPins != 1) { uvc_dbg(chain->dev, DESCR, "Extension unit %d has more than 1 input pin\n", - entity->id); + forward->id); return -EINVAL; } + /* + * Some devices reference an output terminal as the + * source of extension units. This is incorrect, as + * output terminals only have an input pin, and thus + * can't be connected to any entity in the forward + * direction. The resulting topology would cause issues + * when registering the media controller graph. To + * avoid this problem, connect the extension unit to + * the source of the output terminal instead. + */ + if (UVC_ENTITY_IS_OTERM(entity)) { + struct uvc_entity *source; + + source = uvc_entity_by_id(chain->dev, + entity->baSourceID[0]); + if (!source) { + uvc_dbg(chain->dev, DESCR, + "Can't connect extension unit %u in chain\n", + forward->id); + break; + } + + forward->baSourceID[0] = source->id; + } + list_add_tail(&forward->chain, &chain->entities); if (!found) uvc_dbg_cont(PROBE, " (->"); @@ -1735,6 +1760,13 @@ static int uvc_scan_chain_forward(struct uvc_video_chain *chain, return -EINVAL; } + if (UVC_ENTITY_IS_OTERM(entity)) { + uvc_dbg(chain->dev, DESCR, + "Unsupported connection between output terminals %u and %u\n", + entity->id, forward->id); + break; + } + list_add_tail(&forward->chain, &chain->entities); if (!found) uvc_dbg_cont(PROBE, " (->"); diff --git a/drivers/media/usb/zr364xx/zr364xx.c b/drivers/media/usb/zr364xx/zr364xx.c index d29b861367ea..1ef611e08323 100644 --- a/drivers/media/usb/zr364xx/zr364xx.c +++ b/drivers/media/usb/zr364xx/zr364xx.c @@ -1430,7 +1430,7 @@ static int zr364xx_probe(struct usb_interface *intf, if (hdl->error) { err = hdl->error; dev_err(&udev->dev, "couldn't register control\n"); - goto unregister; + goto free_hdlr_and_unreg_dev; } /* save the init method used by this camera */ cam->method = id->driver_info; @@ -1503,7 +1503,7 @@ static int zr364xx_probe(struct usb_interface *intf, if (!cam->read_endpoint) { err = -ENOMEM; dev_err(&intf->dev, "Could not find bulk-in endpoint\n"); - goto unregister; + goto free_hdlr_and_unreg_dev; } /* v4l */ @@ -1515,7 +1515,7 @@ static int zr364xx_probe(struct usb_interface *intf, /* load zr364xx board specific */ err = zr364xx_board_init(cam); if (err) - goto unregister; + goto free_hdlr_and_unreg_dev; err = v4l2_ctrl_handler_setup(hdl); if (err) goto board_uninit; @@ -1533,7 +1533,7 @@ static int zr364xx_probe(struct usb_interface *intf, err = video_register_device(&cam->vdev, VFL_TYPE_VIDEO, -1); if (err) { dev_err(&udev->dev, "video_register_device failed\n"); - goto free_handler; + goto board_uninit; } cam->v4l2_dev.release = zr364xx_release; @@ -1541,11 +1541,10 @@ static int zr364xx_probe(struct usb_interface *intf, video_device_node_name(&cam->vdev)); return 0; -free_handler: - v4l2_ctrl_handler_free(hdl); board_uninit: zr364xx_board_uninit(cam); -unregister: +free_hdlr_and_unreg_dev: + v4l2_ctrl_handler_free(hdl); v4l2_device_unregister(&cam->v4l2_dev); free_cam: kfree(cam); diff --git a/drivers/media/v4l2-core/Kconfig b/drivers/media/v4l2-core/Kconfig index bf49f83cb86f..56664074a4d4 100644 --- a/drivers/media/v4l2-core/Kconfig +++ b/drivers/media/v4l2-core/Kconfig @@ -48,6 +48,11 @@ config VIDEO_TUNER config V4L2_JPEG_HELPER tristate +config V4L2_LOOPBACK + tristate "V4L2 loopback device" + help + V4L2 loopback device + # Used by drivers that need v4l2-h264.ko config V4L2_H264 tristate diff --git a/drivers/media/v4l2-core/Makefile b/drivers/media/v4l2-core/Makefile index e4cd589b99a5..8db53a37f68a 100644 --- a/drivers/media/v4l2-core/Makefile +++ b/drivers/media/v4l2-core/Makefile @@ -27,6 +27,8 @@ obj-$(CONFIG_V4L2_FLASH_LED_CLASS) += v4l2-flash-led-class.o obj-$(CONFIG_V4L2_JPEG_HELPER) += v4l2-jpeg.o +obj-$(CONFIG_V4L2_LOOPBACK) += v4l2loopback.o + obj-$(CONFIG_VIDEOBUF_GEN) += videobuf-core.o obj-$(CONFIG_VIDEOBUF_DMA_SG) += videobuf-dma-sg.o obj-$(CONFIG_VIDEOBUF_DMA_CONTIG) += videobuf-dma-contig.o diff --git a/drivers/media/v4l2-core/v4l2-ctrls.c b/drivers/media/v4l2-core/v4l2-ctrls.c index 016cf6204cbb..6219c8185782 100644 --- a/drivers/media/v4l2-core/v4l2-ctrls.c +++ b/drivers/media/v4l2-core/v4l2-ctrls.c @@ -1675,6 +1675,8 @@ static void std_init_compound(const struct v4l2_ctrl *ctrl, u32 idx, p_fwht_params->version = V4L2_FWHT_VERSION; p_fwht_params->width = 1280; p_fwht_params->height = 720; + p_fwht_params->flags = V4L2_FWHT_FL_PIXENC_YUV | + (2 << V4L2_FWHT_FL_COMPONENTS_NUM_OFFSET); break; } } @@ -2395,7 +2397,16 @@ static void new_to_req(struct v4l2_ctrl_ref *ref) if (!ref) return; ptr_to_ptr(ref->ctrl, ref->ctrl->p_new, ref->p_req); - ref->req = ref; + ref->valid_p_req = true; +} + +/* Copy the current value to the request value */ +static void cur_to_req(struct v4l2_ctrl_ref *ref) +{ + if (!ref) + return; + ptr_to_ptr(ref->ctrl, ref->ctrl->p_cur, ref->p_req); + ref->valid_p_req = true; } /* Copy the request value to the new value */ @@ -2403,8 +2414,8 @@ static void req_to_new(struct v4l2_ctrl_ref *ref) { if (!ref) return; - if (ref->req) - ptr_to_ptr(ref->ctrl, ref->req->p_req, ref->ctrl->p_new); + if (ref->valid_p_req) + ptr_to_ptr(ref->ctrl, ref->p_req, ref->ctrl->p_new); else ptr_to_ptr(ref->ctrl, ref->ctrl->p_cur, ref->ctrl->p_new); } @@ -2541,7 +2552,15 @@ void v4l2_ctrl_handler_free(struct v4l2_ctrl_handler *hdl) if (hdl == NULL || hdl->buckets == NULL) return; - if (!hdl->req_obj.req && !list_empty(&hdl->requests)) { + /* + * If the main handler is freed and it is used by handler objects in + * outstanding requests, then unbind and put those objects before + * freeing the main handler. + * + * The main handler can be identified by having a NULL ops pointer in + * the request object. + */ + if (!hdl->req_obj.ops && !list_empty(&hdl->requests)) { struct v4l2_ctrl_handler *req, *next_req; list_for_each_entry_safe(req, next_req, &hdl->requests, requests) { @@ -3571,39 +3590,8 @@ static void v4l2_ctrl_request_queue(struct media_request_object *obj) struct v4l2_ctrl_handler *hdl = container_of(obj, struct v4l2_ctrl_handler, req_obj); struct v4l2_ctrl_handler *main_hdl = obj->priv; - struct v4l2_ctrl_handler *prev_hdl = NULL; - struct v4l2_ctrl_ref *ref_ctrl, *ref_ctrl_prev = NULL; mutex_lock(main_hdl->lock); - if (list_empty(&main_hdl->requests_queued)) - goto queue; - - prev_hdl = list_last_entry(&main_hdl->requests_queued, - struct v4l2_ctrl_handler, requests_queued); - /* - * Note: prev_hdl and hdl must contain the same list of control - * references, so if any differences are detected then that is a - * driver bug and the WARN_ON is triggered. - */ - mutex_lock(prev_hdl->lock); - ref_ctrl_prev = list_first_entry(&prev_hdl->ctrl_refs, - struct v4l2_ctrl_ref, node); - list_for_each_entry(ref_ctrl, &hdl->ctrl_refs, node) { - if (ref_ctrl->req) - continue; - while (ref_ctrl_prev->ctrl->id < ref_ctrl->ctrl->id) { - /* Should never happen, but just in case... */ - if (list_is_last(&ref_ctrl_prev->node, - &prev_hdl->ctrl_refs)) - break; - ref_ctrl_prev = list_next_entry(ref_ctrl_prev, node); - } - if (WARN_ON(ref_ctrl_prev->ctrl->id != ref_ctrl->ctrl->id)) - break; - ref_ctrl->req = ref_ctrl_prev->req; - } - mutex_unlock(prev_hdl->lock); -queue: list_add_tail(&hdl->requests_queued, &main_hdl->requests_queued); hdl->request_is_queued = true; mutex_unlock(main_hdl->lock); @@ -3615,8 +3603,8 @@ static void v4l2_ctrl_request_unbind(struct media_request_object *obj) container_of(obj, struct v4l2_ctrl_handler, req_obj); struct v4l2_ctrl_handler *main_hdl = obj->priv; - list_del_init(&hdl->requests); mutex_lock(main_hdl->lock); + list_del_init(&hdl->requests); if (hdl->request_is_queued) { list_del_init(&hdl->requests_queued); hdl->request_is_queued = false; @@ -3660,7 +3648,7 @@ v4l2_ctrl_request_hdl_ctrl_find(struct v4l2_ctrl_handler *hdl, u32 id) { struct v4l2_ctrl_ref *ref = find_ref_lock(hdl, id); - return (ref && ref->req == ref) ? ref->ctrl : NULL; + return (ref && ref->valid_p_req) ? ref->ctrl : NULL; } EXPORT_SYMBOL_GPL(v4l2_ctrl_request_hdl_ctrl_find); @@ -3675,8 +3663,11 @@ static int v4l2_ctrl_request_bind(struct media_request *req, if (!ret) { ret = media_request_object_bind(req, &req_ops, from, false, &hdl->req_obj); - if (!ret) + if (!ret) { + mutex_lock(from->lock); list_add_tail(&hdl->requests, &from->requests); + mutex_unlock(from->lock); + } } return ret; } @@ -3846,7 +3837,13 @@ static int class_check(struct v4l2_ctrl_handler *hdl, u32 which) return find_ref_lock(hdl, which | 1) ? 0 : -EINVAL; } -/* Get extended controls. Allocates the helpers array if needed. */ +/* + * Get extended controls. Allocates the helpers array if needed. + * + * Note that v4l2_g_ext_ctrls_common() with 'which' set to + * V4L2_CTRL_WHICH_REQUEST_VAL is only called if the request was + * completed, and in that case valid_p_req is true for all controls. + */ static int v4l2_g_ext_ctrls_common(struct v4l2_ctrl_handler *hdl, struct v4l2_ext_controls *cs, struct video_device *vdev) @@ -3855,9 +3852,10 @@ static int v4l2_g_ext_ctrls_common(struct v4l2_ctrl_handler *hdl, struct v4l2_ctrl_helper *helpers = helper; int ret; int i, j; - bool def_value; + bool is_default, is_request; - def_value = (cs->which == V4L2_CTRL_WHICH_DEF_VAL); + is_default = (cs->which == V4L2_CTRL_WHICH_DEF_VAL); + is_request = (cs->which == V4L2_CTRL_WHICH_REQUEST_VAL); cs->error_idx = cs->count; cs->which = V4L2_CTRL_ID2WHICH(cs->which); @@ -3883,11 +3881,9 @@ static int v4l2_g_ext_ctrls_common(struct v4l2_ctrl_handler *hdl, ret = -EACCES; for (i = 0; !ret && i < cs->count; i++) { - int (*ctrl_to_user)(struct v4l2_ext_control *c, - struct v4l2_ctrl *ctrl); struct v4l2_ctrl *master; - - ctrl_to_user = def_value ? def_to_user : cur_to_user; + bool is_volatile = false; + u32 idx = i; if (helpers[i].mref == NULL) continue; @@ -3897,31 +3893,48 @@ static int v4l2_g_ext_ctrls_common(struct v4l2_ctrl_handler *hdl, v4l2_ctrl_lock(master); - /* g_volatile_ctrl will update the new control values */ - if (!def_value && + /* + * g_volatile_ctrl will update the new control values. + * This makes no sense for V4L2_CTRL_WHICH_DEF_VAL and + * V4L2_CTRL_WHICH_REQUEST_VAL. In the case of requests + * it is v4l2_ctrl_request_complete() that copies the + * volatile controls at the time of request completion + * to the request, so you don't want to do that again. + */ + if (!is_default && !is_request && ((master->flags & V4L2_CTRL_FLAG_VOLATILE) || (master->has_volatiles && !is_cur_manual(master)))) { for (j = 0; j < master->ncontrols; j++) cur_to_new(master->cluster[j]); ret = call_op(master, g_volatile_ctrl); - ctrl_to_user = new_to_user; + is_volatile = true; } - /* If OK, then copy the current (for non-volatile controls) - or the new (for volatile controls) control values to the - caller */ - if (!ret) { - u32 idx = i; - do { - if (helpers[idx].ref->req) - ret = req_to_user(cs->controls + idx, - helpers[idx].ref->req); - else - ret = ctrl_to_user(cs->controls + idx, - helpers[idx].ref->ctrl); - idx = helpers[idx].next; - } while (!ret && idx); + if (ret) { + v4l2_ctrl_unlock(master); + break; } + + /* + * Copy the default value (if is_default is true), the + * request value (if is_request is true and p_req is valid), + * the new volatile value (if is_volatile is true) or the + * current value. + */ + do { + struct v4l2_ctrl_ref *ref = helpers[idx].ref; + + if (is_default) + ret = def_to_user(cs->controls + idx, ref->ctrl); + else if (is_request && ref->valid_p_req) + ret = req_to_user(cs->controls + idx, ref); + else if (is_volatile) + ret = new_to_user(cs->controls + idx, ref->ctrl); + else + ret = cur_to_user(cs->controls + idx, ref->ctrl); + idx = helpers[idx].next; + } while (!ret && idx); + v4l2_ctrl_unlock(master); } @@ -4564,8 +4577,6 @@ void v4l2_ctrl_request_complete(struct media_request *req, unsigned int i; if (ctrl->flags & V4L2_CTRL_FLAG_VOLATILE) { - ref->req = ref; - v4l2_ctrl_lock(master); /* g_volatile_ctrl will update the current control values */ for (i = 0; i < master->ncontrols; i++) @@ -4575,21 +4586,12 @@ void v4l2_ctrl_request_complete(struct media_request *req, v4l2_ctrl_unlock(master); continue; } - if (ref->req == ref) + if (ref->valid_p_req) continue; + /* Copy the current control value into the request */ v4l2_ctrl_lock(ctrl); - if (ref->req) { - ptr_to_ptr(ctrl, ref->req->p_req, ref->p_req); - } else { - ptr_to_ptr(ctrl, ctrl->p_cur, ref->p_req); - /* - * Set ref->req to ensure that when userspace wants to - * obtain the controls of this request it will take - * this value and not the current value of the control. - */ - ref->req = ref; - } + cur_to_req(ref); v4l2_ctrl_unlock(ctrl); } @@ -4653,7 +4655,7 @@ int v4l2_ctrl_request_setup(struct media_request *req, struct v4l2_ctrl_ref *r = find_ref(hdl, master->cluster[i]->id); - if (r->req && r == r->req) { + if (r->valid_p_req) { have_new_data = true; break; } diff --git a/drivers/media/v4l2-core/v4l2loopback.c b/drivers/media/v4l2-core/v4l2loopback.c new file mode 100644 index 000000000000..74af58d558c5 --- /dev/null +++ b/drivers/media/v4l2-core/v4l2loopback.c @@ -0,0 +1,2904 @@ +/* -*- c-file-style: "linux" -*- */ +/* + * v4l2loopback.c -- video4linux2 loopback driver + * + * Copyright (C) 2005-2009 Vasily Levin (vasaka@gmail.com) + * Copyright (C) 2010-2019 IOhannes m zmoelnig (zmoelnig@iem.at) + * Copyright (C) 2011 Stefan Diewald (stefan.diewald@mytum.de) + * Copyright (C) 2012 Anton Novikov (random.plant@gmail.com) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29) +#define HAVE__V4L2_DEVICE +#include +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 36) +#define HAVE__V4L2_CTRLS +#include +#endif +#include + +#include +#include "v4l2loopback.h" + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 6, 1) +#define kstrtoul strict_strtoul +#endif + +#if defined(timer_setup) && defined(from_timer) +#define HAVE_TIMER_SETUP +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 7, 0) +#define VFL_TYPE_VIDEO VFL_TYPE_GRABBER +#endif + +#define V4L2LOOPBACK_VERSION_CODE \ + KERNEL_VERSION(V4L2LOOPBACK_VERSION_MAJOR, V4L2LOOPBACK_VERSION_MINOR, \ + V4L2LOOPBACK_VERSION_BUGFIX) + +MODULE_DESCRIPTION("V4L2 loopback video device"); +MODULE_AUTHOR("Vasily Levin, " + "IOhannes m zmoelnig ," + "Stefan Diewald," + "Anton Novikov" + "et al."); +MODULE_LICENSE("GPL"); + +/* + * helpers + */ +#define STRINGIFY(s) #s +#define STRINGIFY2(s) STRINGIFY(s) + +#define dprintk(fmt, args...) \ + do { \ + if (debug > 0) { \ + printk(KERN_INFO "v4l2-loopback[" STRINGIFY2( \ + __LINE__) "]: " fmt, \ + ##args); \ + } \ + } while (0) + +#define MARK() \ + do { \ + if (debug > 1) { \ + printk(KERN_INFO "%s:%d[%s]\n", __FILE__, __LINE__, \ + __func__); \ + } \ + } while (0) + +#define dprintkrw(fmt, args...) \ + do { \ + if (debug > 2) { \ + printk(KERN_INFO "v4l2-loopback[" STRINGIFY2( \ + __LINE__) "]: " fmt, \ + ##args); \ + } \ + } while (0) + +/* + * compatibility hacks + */ + +#ifndef HAVE__V4L2_CTRLS +struct v4l2_ctrl_handler { + int error; +}; +struct v4l2_ctrl_config { + void *ops; + u32 id; + const char *name; + int type; + s32 min; + s32 max; + u32 step; + s32 def; +}; +int v4l2_ctrl_handler_init(struct v4l2_ctrl_handler *hdl, + unsigned nr_of_controls_hint) +{ + hdl->error = 0; + return 0; +} +void v4l2_ctrl_handler_free(struct v4l2_ctrl_handler *hdl) +{ +} +void *v4l2_ctrl_new_custom(struct v4l2_ctrl_handler *hdl, + const struct v4l2_ctrl_config *conf, void *priv) +{ + return NULL; +} +#endif /* HAVE__V4L2_CTRLS */ + +#ifndef HAVE__V4L2_DEVICE +/* dummy v4l2_device struct/functions */ +#define V4L2_DEVICE_NAME_SIZE (20 + 16) +struct v4l2_device { + char name[V4L2_DEVICE_NAME_SIZE]; + struct v4l2_ctrl_handler *ctrl_handler; +}; +static inline int v4l2_device_register(void *dev, void *v4l2_dev) +{ + return 0; +} +static inline void v4l2_device_unregister(struct v4l2_device *v4l2_dev) +{ + return; +} +#endif /* HAVE__V4L2_DEVICE */ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 29) +#define v4l2_file_operations file_operations +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 37) +void *v4l2l_vzalloc(unsigned long size) +{ + void *data = vmalloc(size); + + memset(data, 0, size); + return data; +} +#else +#define v4l2l_vzalloc vzalloc +#endif + +static inline void v4l2l_get_timestamp(struct v4l2_buffer *b) +{ + /* ktime_get_ts is considered deprecated, so use ktime_get_ts64 if possible */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) + struct timespec ts; + ktime_get_ts(&ts); +#else + struct timespec64 ts; + ktime_get_ts64(&ts); +#endif + + b->timestamp.tv_sec = ts.tv_sec; + b->timestamp.tv_usec = (ts.tv_nsec / NSEC_PER_USEC); +} + +#if !defined(__poll_t) +typedef unsigned __poll_t; +#endif + +/* module constants + * can be overridden during he build process using something like + * make KCPPFLAGS="-DMAX_DEVICES=100" + */ + +/* maximum number of v4l2loopback devices that can be created */ +#ifndef MAX_DEVICES +#define MAX_DEVICES 8 +#endif + +/* whether the default is to announce capabilities exclusively or not */ +#ifndef V4L2LOOPBACK_DEFAULT_EXCLUSIVECAPS +#define V4L2LOOPBACK_DEFAULT_EXCLUSIVECAPS 0 +#endif + +/* when a producer is considered to have gone stale */ +#ifndef MAX_TIMEOUT +#define MAX_TIMEOUT (100 * 1000) /* in msecs */ +#endif + +/* max buffers that can be mapped, actually they + * are all mapped to max_buffers buffers */ +#ifndef MAX_BUFFERS +#define MAX_BUFFERS 32 +#endif + +/* module parameters */ +static int debug = 0; +module_param(debug, int, S_IRUGO | S_IWUSR); +MODULE_PARM_DESC(debug, "debugging level (higher values == more verbose)"); + +#define V4L2LOOPBACK_DEFAULT_MAX_BUFFERS 2 +static int max_buffers = V4L2LOOPBACK_DEFAULT_MAX_BUFFERS; +module_param(max_buffers, int, S_IRUGO); +MODULE_PARM_DESC(max_buffers, + "how many buffers should be allocated [DEFAULT: " STRINGIFY2( + V4L2LOOPBACK_DEFAULT_MAX_BUFFERS) "]"); + +/* how many times a device can be opened + * the per-module default value can be overridden on a per-device basis using + * the /sys/devices interface + * + * note that max_openers should be at least 2 in order to get a working system: + * one opener for the producer and one opener for the consumer + * however, we leave that to the user + */ +#define V4L2LOOPBACK_DEFAULT_MAX_OPENERS 10 +static int max_openers = V4L2LOOPBACK_DEFAULT_MAX_OPENERS; +module_param(max_openers, int, S_IRUGO | S_IWUSR); +MODULE_PARM_DESC( + max_openers, + "how many users can open the loopback device [DEFAULT: " STRINGIFY2( + V4L2LOOPBACK_DEFAULT_MAX_OPENERS) "]"); + +static int devices = -1; +module_param(devices, int, 0); +MODULE_PARM_DESC(devices, "how many devices should be created"); + +static int video_nr[MAX_DEVICES] = { [0 ...(MAX_DEVICES - 1)] = -1 }; +module_param_array(video_nr, int, NULL, 0444); +MODULE_PARM_DESC(video_nr, + "video device numbers (-1=auto, 0=/dev/video0, etc.)"); + +static char *card_label[MAX_DEVICES]; +module_param_array(card_label, charp, NULL, 0000); +MODULE_PARM_DESC(card_label, "card labels for each device"); + +static bool exclusive_caps[MAX_DEVICES] = { + [0 ...(MAX_DEVICES - 1)] = V4L2LOOPBACK_DEFAULT_EXCLUSIVECAPS +}; +module_param_array(exclusive_caps, bool, NULL, 0444); +/* FIXXME: wording */ +MODULE_PARM_DESC( + exclusive_caps, + "whether to announce OUTPUT/CAPTURE capabilities exclusively or not [DEFAULT: " STRINGIFY2( + V4L2LOOPBACK_DEFAULT_EXCLUSIVECAPS) "]"); + +/* format specifications */ +#define V4L2LOOPBACK_SIZE_MIN_WIDTH 48 +#define V4L2LOOPBACK_SIZE_MIN_HEIGHT 32 +#define V4L2LOOPBACK_SIZE_DEFAULT_MAX_WIDTH 8192 +#define V4L2LOOPBACK_SIZE_DEFAULT_MAX_HEIGHT 8192 + +#define V4L2LOOPBACK_SIZE_DEFAULT_WIDTH 640 +#define V4L2LOOPBACK_SIZE_DEFAULT_HEIGHT 480 + +static int max_width = V4L2LOOPBACK_SIZE_DEFAULT_MAX_WIDTH; +module_param(max_width, int, S_IRUGO); +MODULE_PARM_DESC(max_width, "maximum allowed frame width [DEFAULT: " STRINGIFY2( + V4L2LOOPBACK_SIZE_DEFAULT_MAX_WIDTH) "]"); +static int max_height = V4L2LOOPBACK_SIZE_DEFAULT_MAX_HEIGHT; +module_param(max_height, int, S_IRUGO); +MODULE_PARM_DESC(max_height, + "maximum allowed frame height [DEFAULT: " STRINGIFY2( + V4L2LOOPBACK_SIZE_DEFAULT_MAX_HEIGHT) "]"); + +static DEFINE_IDR(v4l2loopback_index_idr); +static DEFINE_MUTEX(v4l2loopback_ctl_mutex); + +/* control IDs */ +#ifndef HAVE__V4L2_CTRLS +#define V4L2LOOPBACK_CID_BASE (V4L2_CID_PRIVATE_BASE) +#else +#define V4L2LOOPBACK_CID_BASE (V4L2_CID_USER_BASE | 0xf000) +#endif +#define CID_KEEP_FORMAT (V4L2LOOPBACK_CID_BASE + 0) +#define CID_SUSTAIN_FRAMERATE (V4L2LOOPBACK_CID_BASE + 1) +#define CID_TIMEOUT (V4L2LOOPBACK_CID_BASE + 2) +#define CID_TIMEOUT_IMAGE_IO (V4L2LOOPBACK_CID_BASE + 3) + +static int v4l2loopback_s_ctrl(struct v4l2_ctrl *ctrl); +static const struct v4l2_ctrl_ops v4l2loopback_ctrl_ops = { + .s_ctrl = v4l2loopback_s_ctrl, +}; +static const struct v4l2_ctrl_config v4l2loopback_ctrl_keepformat = { + // clang-format off + .ops = &v4l2loopback_ctrl_ops, + .id = CID_KEEP_FORMAT, + .name = "keep_format", + .type = V4L2_CTRL_TYPE_BOOLEAN, + .min = 0, + .max = 1, + .step = 1, + .def = 0, + // clang-format on +}; +static const struct v4l2_ctrl_config v4l2loopback_ctrl_sustainframerate = { + // clang-format off + .ops = &v4l2loopback_ctrl_ops, + .id = CID_SUSTAIN_FRAMERATE, + .name = "sustain_framerate", + .type = V4L2_CTRL_TYPE_BOOLEAN, + .min = 0, + .max = 1, + .step = 1, + .def = 0, + // clang-format on +}; +static const struct v4l2_ctrl_config v4l2loopback_ctrl_timeout = { + // clang-format off + .ops = &v4l2loopback_ctrl_ops, + .id = CID_TIMEOUT, + .name = "timeout", + .type = V4L2_CTRL_TYPE_INTEGER, + .min = 0, + .max = MAX_TIMEOUT, + .step = 1, + .def = 0, + // clang-format on +}; +static const struct v4l2_ctrl_config v4l2loopback_ctrl_timeoutimageio = { + // clang-format off + .ops = &v4l2loopback_ctrl_ops, + .id = CID_TIMEOUT_IMAGE_IO, + .name = "timeout_image_io", + .type = V4L2_CTRL_TYPE_BOOLEAN, + .min = 0, + .max = 1, + .step = 1, + .def = 0, + // clang-format on +}; + +/* module structures */ +struct v4l2loopback_private { + int device_nr; +}; + +/* TODO(vasaka) use typenames which are common to kernel, but first find out if + * it is needed */ +/* struct keeping state and settings of loopback device */ + +struct v4l2l_buffer { + struct v4l2_buffer buffer; + struct list_head list_head; + int use_count; +}; + +struct v4l2_loopback_device { + struct v4l2_device v4l2_dev; + struct v4l2_ctrl_handler ctrl_handler; + struct video_device *vdev; + /* pixel and stream format */ + struct v4l2_pix_format pix_format; + struct v4l2_captureparm capture_param; + unsigned long frame_jiffies; + + /* ctrls */ + int keep_format; /* CID_KEEP_FORMAT; stay ready_for_capture even when all + openers close() the device */ + int sustain_framerate; /* CID_SUSTAIN_FRAMERATE; duplicate frames to maintain + (close to) nominal framerate */ + + /* buffers stuff */ + u8 *image; /* pointer to actual buffers data */ + unsigned long int imagesize; /* size of buffers data */ + int buffers_number; /* should not be big, 4 is a good choice */ + struct v4l2l_buffer buffers[MAX_BUFFERS]; /* inner driver buffers */ + int used_buffers; /* number of the actually used buffers */ + int max_openers; /* how many times can this device be opened */ + + int write_position; /* number of last written frame + 1 */ + struct list_head outbufs_list; /* buffers in output DQBUF order */ + int bufpos2index + [MAX_BUFFERS]; /* mapping of (read/write_position % used_buffers) + * to inner buffer index */ + long buffer_size; + + /* sustain_framerate stuff */ + struct timer_list sustain_timer; + unsigned int reread_count; + + /* timeout stuff */ + unsigned long timeout_jiffies; /* CID_TIMEOUT; 0 means disabled */ + int timeout_image_io; /* CID_TIMEOUT_IMAGE_IO; next opener will + * read/write to timeout_image */ + u8 *timeout_image; /* copy of it will be captured when timeout passes */ + struct v4l2l_buffer timeout_image_buffer; + struct timer_list timeout_timer; + int timeout_happened; + + /* sync stuff */ + atomic_t open_count; + + int ready_for_capture; /* set to the number of writers that opened the + * device and negotiated format. */ + int ready_for_output; /* set to true when no writer is currently attached + * this differs slightly from !ready_for_capture, + * e.g. when using fallback images */ + int announce_all_caps; /* set to false, if device caps (OUTPUT/CAPTURE) + * should only be announced if the resp. "ready" + * flag is set; default=TRUE */ + + int max_width; + int max_height; + + char card_label[32]; + + wait_queue_head_t read_event; + spinlock_t lock; +}; + +/* types of opener shows what opener wants to do with loopback */ +enum opener_type { + // clang-format off + UNNEGOTIATED = 0, + READER = 1, + WRITER = 2, + // clang-format on +}; + +/* struct keeping state and type of opener */ +struct v4l2_loopback_opener { + enum opener_type type; + int vidioc_enum_frameintervals_calls; + int read_position; /* number of last processed frame + 1 or + * write_position - 1 if reader went out of sync */ + unsigned int reread_count; + struct v4l2_buffer *buffers; + int buffers_number; /* should not be big, 4 is a good choice */ + int timeout_image_io; + + struct v4l2_fh fh; +}; + +#define fh_to_opener(ptr) container_of((ptr), struct v4l2_loopback_opener, fh) + +/* this is heavily inspired by the bttv driver found in the linux kernel */ +struct v4l2l_format { + char *name; + int fourcc; /* video4linux 2 */ + int depth; /* bit/pixel */ + int flags; +}; +/* set the v4l2l_format.flags to PLANAR for non-packed formats */ +#define FORMAT_FLAGS_PLANAR 0x01 +#define FORMAT_FLAGS_COMPRESSED 0x02 + +static const struct v4l2l_format formats[] = { +#include "v4l2loopback_formats.h" +}; + +static const unsigned int FORMATS = ARRAY_SIZE(formats); + +static char *fourcc2str(unsigned int fourcc, char buf[4]) +{ + buf[0] = (fourcc >> 0) & 0xFF; + buf[1] = (fourcc >> 8) & 0xFF; + buf[2] = (fourcc >> 16) & 0xFF; + buf[3] = (fourcc >> 24) & 0xFF; + + return buf; +} + +static const struct v4l2l_format *format_by_fourcc(int fourcc) +{ + unsigned int i; + + for (i = 0; i < FORMATS; i++) { + if (formats[i].fourcc == fourcc) + return formats + i; + } + + dprintk("unsupported format '%c%c%c%c'\n", (fourcc >> 0) & 0xFF, + (fourcc >> 8) & 0xFF, (fourcc >> 16) & 0xFF, + (fourcc >> 24) & 0xFF); + return NULL; +} + +static void pix_format_set_size(struct v4l2_pix_format *f, + const struct v4l2l_format *fmt, + unsigned int width, unsigned int height) +{ + f->width = width; + f->height = height; + + if (fmt->flags & FORMAT_FLAGS_PLANAR) { + f->bytesperline = width; /* Y plane */ + f->sizeimage = (width * height * fmt->depth) >> 3; + } else if (fmt->flags & FORMAT_FLAGS_COMPRESSED) { + /* doesn't make sense for compressed formats */ + f->bytesperline = 0; + f->sizeimage = (width * height * fmt->depth) >> 3; + } else { + f->bytesperline = (width * fmt->depth) >> 3; + f->sizeimage = height * f->bytesperline; + } +} + +static int set_timeperframe(struct v4l2_loopback_device *dev, + struct v4l2_fract *tpf) +{ + if ((tpf->denominator < 1) || (tpf->numerator < 1)) { + return -EINVAL; + } + dev->capture_param.timeperframe = *tpf; + dev->frame_jiffies = max(1UL, msecs_to_jiffies(1000) * tpf->numerator / + tpf->denominator); + return 0; +} + +static struct v4l2_loopback_device *v4l2loopback_cd2dev(struct device *cd); + +/* device attributes */ +/* available via sysfs: /sys/devices/virtual/video4linux/video* */ + +static ssize_t attr_show_format(struct device *cd, + struct device_attribute *attr, char *buf) +{ + /* gets the current format as "FOURCC:WxH@f/s", e.g. "YUYV:320x240@1000/30" */ + struct v4l2_loopback_device *dev = v4l2loopback_cd2dev(cd); + const struct v4l2_fract *tpf; + char buf4cc[5], buf_fps[32]; + + if (!dev || !dev->ready_for_capture) + return 0; + tpf = &dev->capture_param.timeperframe; + + fourcc2str(dev->pix_format.pixelformat, buf4cc); + buf4cc[4] = 0; + if (tpf->numerator == 1) + snprintf(buf_fps, sizeof(buf_fps), "%d", tpf->denominator); + else + snprintf(buf_fps, sizeof(buf_fps), "%d/%d", tpf->denominator, + tpf->numerator); + return sprintf(buf, "%4s:%dx%d@%s\n", buf4cc, dev->pix_format.width, + dev->pix_format.height, buf_fps); +} + +static ssize_t attr_store_format(struct device *cd, + struct device_attribute *attr, const char *buf, + size_t len) +{ + struct v4l2_loopback_device *dev = v4l2loopback_cd2dev(cd); + int fps_num = 0, fps_den = 1; + + /* only fps changing is supported */ + if (sscanf(buf, "@%d/%d", &fps_num, &fps_den) > 0) { + struct v4l2_fract f = { .numerator = fps_den, + .denominator = fps_num }; + int err = 0; + if ((err = set_timeperframe(dev, &f)) < 0) + return err; + return len; + } + return -EINVAL; +} + +static DEVICE_ATTR(format, S_IRUGO | S_IWUSR, attr_show_format, + attr_store_format); + +static ssize_t attr_show_buffers(struct device *cd, + struct device_attribute *attr, char *buf) +{ + struct v4l2_loopback_device *dev = v4l2loopback_cd2dev(cd); + + return sprintf(buf, "%d\n", dev->used_buffers); +} + +static DEVICE_ATTR(buffers, S_IRUGO, attr_show_buffers, NULL); + +static ssize_t attr_show_maxopeners(struct device *cd, + struct device_attribute *attr, char *buf) +{ + struct v4l2_loopback_device *dev = v4l2loopback_cd2dev(cd); + + return sprintf(buf, "%d\n", dev->max_openers); +} + +static ssize_t attr_store_maxopeners(struct device *cd, + struct device_attribute *attr, + const char *buf, size_t len) +{ + struct v4l2_loopback_device *dev = NULL; + unsigned long curr = 0; + + if (kstrtoul(buf, 0, &curr)) + return -EINVAL; + + dev = v4l2loopback_cd2dev(cd); + + if (dev->max_openers == curr) + return len; + + if (dev->open_count.counter > curr) { + /* request to limit to less openers as are currently attached to us */ + return -EINVAL; + } + + dev->max_openers = (int)curr; + + return len; +} + +static DEVICE_ATTR(max_openers, S_IRUGO | S_IWUSR, attr_show_maxopeners, + attr_store_maxopeners); + +static void v4l2loopback_remove_sysfs(struct video_device *vdev) +{ +#define V4L2_SYSFS_DESTROY(x) device_remove_file(&vdev->dev, &dev_attr_##x) + + if (vdev) { + V4L2_SYSFS_DESTROY(format); + V4L2_SYSFS_DESTROY(buffers); + V4L2_SYSFS_DESTROY(max_openers); + /* ... */ + } +} + +static void v4l2loopback_create_sysfs(struct video_device *vdev) +{ + int res = 0; + +#define V4L2_SYSFS_CREATE(x) \ + res = device_create_file(&vdev->dev, &dev_attr_##x); \ + if (res < 0) \ + break + if (!vdev) + return; + do { + V4L2_SYSFS_CREATE(format); + V4L2_SYSFS_CREATE(buffers); + V4L2_SYSFS_CREATE(max_openers); + /* ... */ + } while (0); + + if (res >= 0) + return; + dev_err(&vdev->dev, "%s error: %d\n", __func__, res); +} + +/* global module data */ +/* find a device based on it's device-number (e.g. '3' for /dev/video3) */ +struct v4l2loopback_lookup_cb_data { + int device_nr; + struct v4l2_loopback_device *device; +}; +static int v4l2loopback_lookup_cb(int id, void *ptr, void *data) +{ + struct v4l2_loopback_device *device = ptr; + struct v4l2loopback_lookup_cb_data *cbdata = data; + if (cbdata && device && device->vdev) { + if (device->vdev->num == cbdata->device_nr) { + cbdata->device = device; + cbdata->device_nr = id; + return 1; + } + } + return 0; +} +static int v4l2loopback_lookup(int device_nr, + struct v4l2_loopback_device **device) +{ + struct v4l2loopback_lookup_cb_data data = { + .device_nr = device_nr, + .device = NULL, + }; + int err = idr_for_each(&v4l2loopback_index_idr, &v4l2loopback_lookup_cb, + &data); + if (1 == err) { + if (device) + *device = data.device; + return data.device_nr; + } + return -ENODEV; +} +static struct v4l2_loopback_device *v4l2loopback_cd2dev(struct device *cd) +{ + struct video_device *loopdev = to_video_device(cd); + struct v4l2loopback_private *ptr = + (struct v4l2loopback_private *)video_get_drvdata(loopdev); + int nr = ptr->device_nr; + + return idr_find(&v4l2loopback_index_idr, nr); +} + +static struct v4l2_loopback_device *v4l2loopback_getdevice(struct file *f) +{ + struct video_device *loopdev = video_devdata(f); + struct v4l2loopback_private *ptr = + (struct v4l2loopback_private *)video_get_drvdata(loopdev); + int nr = ptr->device_nr; + + return idr_find(&v4l2loopback_index_idr, nr); +} + +/* forward declarations */ +static void init_buffers(struct v4l2_loopback_device *dev); +static int allocate_buffers(struct v4l2_loopback_device *dev); +static int free_buffers(struct v4l2_loopback_device *dev); +static void try_free_buffers(struct v4l2_loopback_device *dev); +static int allocate_timeout_image(struct v4l2_loopback_device *dev); +static void check_timers(struct v4l2_loopback_device *dev); +static const struct v4l2_file_operations v4l2_loopback_fops; +static const struct v4l2_ioctl_ops v4l2_loopback_ioctl_ops; + +/* Queue helpers */ +/* next functions sets buffer flags and adjusts counters accordingly */ +static inline void set_done(struct v4l2l_buffer *buffer) +{ + buffer->buffer.flags &= ~V4L2_BUF_FLAG_QUEUED; + buffer->buffer.flags |= V4L2_BUF_FLAG_DONE; +} + +static inline void set_queued(struct v4l2l_buffer *buffer) +{ + buffer->buffer.flags &= ~V4L2_BUF_FLAG_DONE; + buffer->buffer.flags |= V4L2_BUF_FLAG_QUEUED; +} + +static inline void unset_flags(struct v4l2l_buffer *buffer) +{ + buffer->buffer.flags &= ~V4L2_BUF_FLAG_QUEUED; + buffer->buffer.flags &= ~V4L2_BUF_FLAG_DONE; +} + +/* V4L2 ioctl caps and params calls */ +/* returns device capabilities + * called on VIDIOC_QUERYCAP + */ +static int vidioc_querycap(struct file *file, void *priv, + struct v4l2_capability *cap) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + int labellen = (sizeof(cap->card) < sizeof(dev->card_label)) ? + sizeof(cap->card) : + sizeof(dev->card_label); + int device_nr = + ((struct v4l2loopback_private *)video_get_drvdata(dev->vdev)) + ->device_nr; + __u32 capabilities = V4L2_CAP_STREAMING | V4L2_CAP_READWRITE; + + strlcpy(cap->driver, "v4l2 loopback", sizeof(cap->driver)); + snprintf(cap->card, labellen, dev->card_label); + snprintf(cap->bus_info, sizeof(cap->bus_info), + "platform:v4l2loopback-%03d", device_nr); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0) + /* since 3.1.0, the v4l2-core system is supposed to set the version */ + cap->version = V4L2LOOPBACK_VERSION_CODE; +#endif + +#ifdef V4L2_CAP_VIDEO_M2M + capabilities |= V4L2_CAP_VIDEO_M2M; +#endif /* V4L2_CAP_VIDEO_M2M */ + + if (dev->announce_all_caps) { + capabilities |= V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_VIDEO_OUTPUT; + } else { + if (dev->ready_for_capture) { + capabilities |= V4L2_CAP_VIDEO_CAPTURE; + } + if (dev->ready_for_output) { + capabilities |= V4L2_CAP_VIDEO_OUTPUT; + } + } + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 7, 0) + dev->vdev->device_caps = +#endif /* >=linux-4.7.0 */ + cap->device_caps = cap->capabilities = capabilities; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 3, 0) + cap->capabilities |= V4L2_CAP_DEVICE_CAPS; +#endif + + memset(cap->reserved, 0, sizeof(cap->reserved)); + return 0; +} + +static int vidioc_enum_framesizes(struct file *file, void *fh, + struct v4l2_frmsizeenum *argp) +{ + struct v4l2_loopback_device *dev; + + /* LATER: what does the index really mean? + * if it's about enumerating formats, we can safely ignore it + * (CHECK) + */ + + /* there can be only one... */ + if (argp->index) + return -EINVAL; + + dev = v4l2loopback_getdevice(file); + if (dev->ready_for_capture) { + /* format has already been negotiated + * cannot change during runtime + */ + argp->type = V4L2_FRMSIZE_TYPE_DISCRETE; + + argp->discrete.width = dev->pix_format.width; + argp->discrete.height = dev->pix_format.height; + } else { + /* if the format has not been negotiated yet, we accept anything + */ + argp->type = V4L2_FRMSIZE_TYPE_CONTINUOUS; + + argp->stepwise.min_width = V4L2LOOPBACK_SIZE_MIN_WIDTH; + argp->stepwise.min_height = V4L2LOOPBACK_SIZE_MIN_HEIGHT; + + argp->stepwise.max_width = dev->max_width; + argp->stepwise.max_height = dev->max_height; + + argp->stepwise.step_width = 1; + argp->stepwise.step_height = 1; + } + return 0; +} + +/* returns frameinterval (fps) for the set resolution + * called on VIDIOC_ENUM_FRAMEINTERVALS + */ +static int vidioc_enum_frameintervals(struct file *file, void *fh, + struct v4l2_frmivalenum *argp) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + + if (dev->ready_for_capture) { + if (opener->vidioc_enum_frameintervals_calls > 0) + return -EINVAL; + if (argp->width == dev->pix_format.width && + argp->height == dev->pix_format.height) { + argp->type = V4L2_FRMIVAL_TYPE_DISCRETE; + argp->discrete = dev->capture_param.timeperframe; + opener->vidioc_enum_frameintervals_calls++; + return 0; + } + return -EINVAL; + } + return 0; +} + +/* ------------------ CAPTURE ----------------------- */ + +/* returns device formats + * called on VIDIOC_ENUM_FMT, with v4l2_buf_type set to V4L2_BUF_TYPE_VIDEO_CAPTURE + */ +static int vidioc_enum_fmt_cap(struct file *file, void *fh, + struct v4l2_fmtdesc *f) +{ + struct v4l2_loopback_device *dev; + MARK(); + + dev = v4l2loopback_getdevice(file); + + if (f->index) + return -EINVAL; + if (dev->ready_for_capture) { + const __u32 format = dev->pix_format.pixelformat; + + snprintf(f->description, sizeof(f->description), "[%c%c%c%c]", + (format >> 0) & 0xFF, (format >> 8) & 0xFF, + (format >> 16) & 0xFF, (format >> 24) & 0xFF); + + f->pixelformat = dev->pix_format.pixelformat; + } else { + return -EINVAL; + } + f->flags = 0; + MARK(); + return 0; +} + +/* returns current video format format fmt + * called on VIDIOC_G_FMT, with v4l2_buf_type set to V4L2_BUF_TYPE_VIDEO_CAPTURE + */ +static int vidioc_g_fmt_cap(struct file *file, void *priv, + struct v4l2_format *fmt) +{ + struct v4l2_loopback_device *dev; + MARK(); + + dev = v4l2loopback_getdevice(file); + + if (!dev->ready_for_capture) + return -EINVAL; + + fmt->fmt.pix = dev->pix_format; + MARK(); + return 0; +} + +/* checks if it is OK to change to format fmt; + * actual check is done by inner_try_fmt_cap + * just checking that pixelformat is OK and set other parameters, app should + * obey this decision + * called on VIDIOC_TRY_FMT, with v4l2_buf_type set to V4L2_BUF_TYPE_VIDEO_CAPTURE + */ +static int vidioc_try_fmt_cap(struct file *file, void *priv, + struct v4l2_format *fmt) +{ + struct v4l2_loopback_device *dev; + char buf[5]; + + dev = v4l2loopback_getdevice(file); + + if (0 == dev->ready_for_capture) { + dprintk("setting fmt_cap not possible yet\n"); + return -EBUSY; + } + + if (fmt->fmt.pix.pixelformat != dev->pix_format.pixelformat) + return -EINVAL; + + fmt->fmt.pix = dev->pix_format; + + buf[4] = 0; + dprintk("capFOURCC=%s\n", fourcc2str(dev->pix_format.pixelformat, buf)); + return 0; +} + +/* sets new output format, if possible + * actually format is set by input and we even do not check it, just return + * current one, but it is possible to set subregions of input TODO(vasaka) + * called on VIDIOC_S_FMT, with v4l2_buf_type set to V4L2_BUF_TYPE_VIDEO_CAPTURE + */ +static int vidioc_s_fmt_cap(struct file *file, void *priv, + struct v4l2_format *fmt) +{ + return vidioc_try_fmt_cap(file, priv, fmt); +} + +/* ------------------ OUTPUT ----------------------- */ + +/* returns device formats; + * LATER: allow all formats + * called on VIDIOC_ENUM_FMT, with v4l2_buf_type set to V4L2_BUF_TYPE_VIDEO_OUTPUT + */ +static int vidioc_enum_fmt_out(struct file *file, void *fh, + struct v4l2_fmtdesc *f) +{ + struct v4l2_loopback_device *dev; + const struct v4l2l_format *fmt; + + dev = v4l2loopback_getdevice(file); + + if (dev->ready_for_capture) { + const __u32 format = dev->pix_format.pixelformat; + + /* format has been fixed by the writer, so only one single format is supported */ + if (f->index) + return -EINVAL; + + fmt = format_by_fourcc(format); + if (NULL == fmt) + return -EINVAL; + + f->type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + /* f->flags = ??; */ + snprintf(f->description, sizeof(f->description), "%s", + fmt->name); + + f->pixelformat = dev->pix_format.pixelformat; + } else { + /* fill in a dummy format */ + /* coverity[unsigned_compare] */ + if (f->index < 0 || f->index >= FORMATS) + return -EINVAL; + + fmt = &formats[f->index]; + + f->pixelformat = fmt->fourcc; + snprintf(f->description, sizeof(f->description), "%s", + fmt->name); + } + f->flags = 0; + + return 0; +} + +/* returns current video format format fmt */ +/* NOTE: this is called from the producer + * so if format has not been negotiated yet, + * it should return ALL of available formats, + * called on VIDIOC_G_FMT, with v4l2_buf_type set to V4L2_BUF_TYPE_VIDEO_OUTPUT + */ +static int vidioc_g_fmt_out(struct file *file, void *priv, + struct v4l2_format *fmt) +{ + struct v4l2_loopback_device *dev; + MARK(); + + dev = v4l2loopback_getdevice(file); + + /* + * LATER: this should return the currently valid format + * gstreamer doesn't like it, if this returns -EINVAL, as it + * then concludes that there is _no_ valid format + * CHECK whether this assumption is wrong, + * or whether we have to always provide a valid format + */ + + fmt->fmt.pix = dev->pix_format; + return 0; +} + +/* checks if it is OK to change to format fmt; + * if format is negotiated do not change it + * called on VIDIOC_TRY_FMT with v4l2_buf_type set to V4L2_BUF_TYPE_VIDEO_OUTPUT + */ +static int vidioc_try_fmt_out(struct file *file, void *priv, + struct v4l2_format *fmt) +{ + struct v4l2_loopback_device *dev; + MARK(); + + dev = v4l2loopback_getdevice(file); + + /* TODO(vasaka) loopback does not care about formats writer want to set, + * maybe it is a good idea to restrict format somehow */ + if (dev->ready_for_capture) { + fmt->fmt.pix = dev->pix_format; + } else { + __u32 w = fmt->fmt.pix.width; + __u32 h = fmt->fmt.pix.height; + __u32 pixfmt = fmt->fmt.pix.pixelformat; + const struct v4l2l_format *format = format_by_fourcc(pixfmt); + + if (w > dev->max_width) + w = dev->max_width; + if (h > dev->max_height) + h = dev->max_height; + + dprintk("trying image %dx%d\n", w, h); + + if (w < 1) + w = V4L2LOOPBACK_SIZE_DEFAULT_WIDTH; + + if (h < 1) + h = V4L2LOOPBACK_SIZE_DEFAULT_HEIGHT; + + if (NULL == format) + format = &formats[0]; + + pix_format_set_size(&fmt->fmt.pix, format, w, h); + + fmt->fmt.pix.pixelformat = format->fourcc; + fmt->fmt.pix.colorspace = V4L2_COLORSPACE_SRGB; + + if (V4L2_FIELD_ANY == fmt->fmt.pix.field) + fmt->fmt.pix.field = V4L2_FIELD_NONE; + + /* FIXXME: try_fmt should never modify the device-state */ + dev->pix_format = fmt->fmt.pix; + } + return 0; +} + +/* sets new output format, if possible; + * allocate data here because we do not know if it will be streaming or + * read/write IO + * called on VIDIOC_S_FMT with v4l2_buf_type set to V4L2_BUF_TYPE_VIDEO_OUTPUT + */ +static int vidioc_s_fmt_out(struct file *file, void *priv, + struct v4l2_format *fmt) +{ + struct v4l2_loopback_device *dev; + char buf[5]; + int ret; + MARK(); + + dev = v4l2loopback_getdevice(file); + ret = vidioc_try_fmt_out(file, priv, fmt); + + dprintk("s_fmt_out(%d) %d...%d\n", ret, dev->ready_for_capture, + dev->pix_format.sizeimage); + + buf[4] = 0; + dprintk("outFOURCC=%s\n", fourcc2str(dev->pix_format.pixelformat, buf)); + + if (ret < 0) + return ret; + + if (!dev->ready_for_capture) { + dev->buffer_size = PAGE_ALIGN(dev->pix_format.sizeimage); + fmt->fmt.pix.sizeimage = dev->buffer_size; + allocate_buffers(dev); + } + return ret; +} + +// #define V4L2L_OVERLAY +#ifdef V4L2L_OVERLAY +/* ------------------ OVERLAY ----------------------- */ +/* currently unsupported */ +/* GSTreamer's v4l2sink is buggy, as it requires the overlay to work + * while it should only require it, if overlay is requested + * once the gstreamer element is fixed, remove the overlay dummies + */ +#warning OVERLAY dummies +static int vidioc_g_fmt_overlay(struct file *file, void *priv, + struct v4l2_format *fmt) +{ + return 0; +} + +static int vidioc_s_fmt_overlay(struct file *file, void *priv, + struct v4l2_format *fmt) +{ + return 0; +} +#endif /* V4L2L_OVERLAY */ + +/* ------------------ PARAMs ----------------------- */ + +/* get some data flow parameters, only capability, fps and readbuffers has + * effect on this driver + * called on VIDIOC_G_PARM + */ +static int vidioc_g_parm(struct file *file, void *priv, + struct v4l2_streamparm *parm) +{ + /* do not care about type of opener, hope these enums would always be + * compatible */ + struct v4l2_loopback_device *dev; + MARK(); + + dev = v4l2loopback_getdevice(file); + parm->parm.capture = dev->capture_param; + return 0; +} + +/* get some data flow parameters, only capability, fps and readbuffers has + * effect on this driver + * called on VIDIOC_S_PARM + */ +static int vidioc_s_parm(struct file *file, void *priv, + struct v4l2_streamparm *parm) +{ + struct v4l2_loopback_device *dev; + int err = 0; + MARK(); + + dev = v4l2loopback_getdevice(file); + dprintk("vidioc_s_parm called frate=%d/%d\n", + parm->parm.capture.timeperframe.numerator, + parm->parm.capture.timeperframe.denominator); + + switch (parm->type) { + case V4L2_BUF_TYPE_VIDEO_CAPTURE: + if ((err = set_timeperframe( + dev, &parm->parm.capture.timeperframe)) < 0) + return err; + break; + case V4L2_BUF_TYPE_VIDEO_OUTPUT: + if ((err = set_timeperframe( + dev, &parm->parm.capture.timeperframe)) < 0) + return err; + break; + default: + return -1; + } + + parm->parm.capture = dev->capture_param; + return 0; +} + +#ifdef V4L2LOOPBACK_WITH_STD +/* sets a tv standard, actually we do not need to handle this any special way + * added to support effecttv + * called on VIDIOC_S_STD + */ +static int vidioc_s_std(struct file *file, void *fh, v4l2_std_id *_std) +{ + v4l2_std_id req_std = 0, supported_std = 0; + const v4l2_std_id all_std = V4L2_STD_ALL, no_std = 0; + + if (_std) { + req_std = *_std; + *_std = all_std; + } + + /* we support everything in V4L2_STD_ALL, but not more... */ + supported_std = (all_std & req_std); + if (no_std == supported_std) + return -EINVAL; + + return 0; +} + +/* gets a fake video standard + * called on VIDIOC_G_STD + */ +static int vidioc_g_std(struct file *file, void *fh, v4l2_std_id *norm) +{ + if (norm) + *norm = V4L2_STD_ALL; + return 0; +} +/* gets a fake video standard + * called on VIDIOC_QUERYSTD + */ +static int vidioc_querystd(struct file *file, void *fh, v4l2_std_id *norm) +{ + if (norm) + *norm = V4L2_STD_ALL; + return 0; +} +#endif /* V4L2LOOPBACK_WITH_STD */ + +/* get ctrls info + * called on VIDIOC_QUERYCTRL + */ +static int vidioc_queryctrl(struct file *file, void *fh, + struct v4l2_queryctrl *q) +{ + const struct v4l2_ctrl_config *cnf = 0; + switch (q->id) { + case CID_KEEP_FORMAT: + cnf = &v4l2loopback_ctrl_keepformat; + break; + case CID_SUSTAIN_FRAMERATE: + cnf = &v4l2loopback_ctrl_sustainframerate; + break; + case CID_TIMEOUT: + cnf = &v4l2loopback_ctrl_timeout; + break; + case CID_TIMEOUT_IMAGE_IO: + cnf = &v4l2loopback_ctrl_timeoutimageio; + break; + default: + return -EINVAL; + } + if (!cnf) + BUG(); + + strcpy(q->name, cnf->name); + q->default_value = cnf->def; + q->type = cnf->type; + q->minimum = cnf->min; + q->maximum = cnf->max; + q->step = cnf->step; + + memset(q->reserved, 0, sizeof(q->reserved)); + return 0; +} + +static int vidioc_g_ctrl(struct file *file, void *fh, struct v4l2_control *c) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + + switch (c->id) { + case CID_KEEP_FORMAT: + c->value = dev->keep_format; + break; + case CID_SUSTAIN_FRAMERATE: + c->value = dev->sustain_framerate; + break; + case CID_TIMEOUT: + c->value = jiffies_to_msecs(dev->timeout_jiffies); + break; + case CID_TIMEOUT_IMAGE_IO: + c->value = dev->timeout_image_io; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int v4l2loopback_set_ctrl(struct v4l2_loopback_device *dev, u32 id, + s64 val) +{ + switch (id) { + case CID_KEEP_FORMAT: + if (val < 0 || val > 1) + return -EINVAL; + dev->keep_format = val; + try_free_buffers(dev); /* will only free buffers if !keep_format */ + break; + case CID_SUSTAIN_FRAMERATE: + if (val < 0 || val > 1) + return -EINVAL; + spin_lock_bh(&dev->lock); + dev->sustain_framerate = val; + check_timers(dev); + spin_unlock_bh(&dev->lock); + break; + case CID_TIMEOUT: + if (val < 0 || val > MAX_TIMEOUT) + return -EINVAL; + spin_lock_bh(&dev->lock); + dev->timeout_jiffies = msecs_to_jiffies(val); + check_timers(dev); + spin_unlock_bh(&dev->lock); + allocate_timeout_image(dev); + break; + case CID_TIMEOUT_IMAGE_IO: + if (val < 0 || val > 1) + return -EINVAL; + dev->timeout_image_io = val; + break; + default: + return -EINVAL; + } + return 0; +} + +static int v4l2loopback_s_ctrl(struct v4l2_ctrl *ctrl) +{ + struct v4l2_loopback_device *dev = container_of( + ctrl->handler, struct v4l2_loopback_device, ctrl_handler); + return v4l2loopback_set_ctrl(dev, ctrl->id, ctrl->val); +} +static int vidioc_s_ctrl(struct file *file, void *fh, struct v4l2_control *c) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + return v4l2loopback_set_ctrl(dev, c->id, c->value); +} + +/* returns set of device outputs, in our case there is only one + * called on VIDIOC_ENUMOUTPUT + */ +static int vidioc_enum_output(struct file *file, void *fh, + struct v4l2_output *outp) +{ + __u32 index = outp->index; + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + MARK(); + + if (!dev->announce_all_caps && !dev->ready_for_output) + return -ENOTTY; + + if (0 != index) + return -EINVAL; + + /* clear all data (including the reserved fields) */ + memset(outp, 0, sizeof(*outp)); + + outp->index = index; + strlcpy(outp->name, "loopback in", sizeof(outp->name)); + outp->type = V4L2_OUTPUT_TYPE_ANALOG; + outp->audioset = 0; + outp->modulator = 0; +#ifdef V4L2LOOPBACK_WITH_STD + outp->std = V4L2_STD_ALL; +#ifdef V4L2_OUT_CAP_STD + outp->capabilities |= V4L2_OUT_CAP_STD; +#endif /* V4L2_OUT_CAP_STD */ +#endif /* V4L2LOOPBACK_WITH_STD */ + + return 0; +} + +/* which output is currently active, + * called on VIDIOC_G_OUTPUT + */ +static int vidioc_g_output(struct file *file, void *fh, unsigned int *i) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + if (!dev->announce_all_caps && !dev->ready_for_output) + return -ENOTTY; + if (i) + *i = 0; + return 0; +} + +/* set output, can make sense if we have more than one video src, + * called on VIDIOC_S_OUTPUT + */ +static int vidioc_s_output(struct file *file, void *fh, unsigned int i) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + if (!dev->announce_all_caps && !dev->ready_for_output) + return -ENOTTY; + + if (i) + return -EINVAL; + + return 0; +} + +/* returns set of device inputs, in our case there is only one, + * but later I may add more + * called on VIDIOC_ENUMINPUT + */ +static int vidioc_enum_input(struct file *file, void *fh, + struct v4l2_input *inp) +{ + __u32 index = inp->index; + MARK(); + + if (0 != index) + return -EINVAL; + + /* clear all data (including the reserved fields) */ + memset(inp, 0, sizeof(*inp)); + + inp->index = index; + strlcpy(inp->name, "loopback", sizeof(inp->name)); + inp->type = V4L2_INPUT_TYPE_CAMERA; + inp->audioset = 0; + inp->tuner = 0; + inp->status = 0; + +#ifdef V4L2LOOPBACK_WITH_STD + inp->std = V4L2_STD_ALL; +#ifdef V4L2_IN_CAP_STD + inp->capabilities |= V4L2_IN_CAP_STD; +#endif +#endif /* V4L2LOOPBACK_WITH_STD */ + + return 0; +} + +/* which input is currently active, + * called on VIDIOC_G_INPUT + */ +static int vidioc_g_input(struct file *file, void *fh, unsigned int *i) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + if (!dev->announce_all_caps && !dev->ready_for_capture) + return -ENOTTY; + if (i) + *i = 0; + return 0; +} + +/* set input, can make sense if we have more than one video src, + * called on VIDIOC_S_INPUT + */ +static int vidioc_s_input(struct file *file, void *fh, unsigned int i) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + if (!dev->announce_all_caps && !dev->ready_for_capture) + return -ENOTTY; + if (i == 0) + return 0; + return -EINVAL; +} + +/* --------------- V4L2 ioctl buffer related calls ----------------- */ + +/* negotiate buffer type + * only mmap streaming supported + * called on VIDIOC_REQBUFS + */ +static int vidioc_reqbufs(struct file *file, void *fh, + struct v4l2_requestbuffers *b) +{ + struct v4l2_loopback_device *dev; + struct v4l2_loopback_opener *opener; + int i; + MARK(); + + dev = v4l2loopback_getdevice(file); + opener = fh_to_opener(fh); + + dprintk("reqbufs: %d\t%d=%d\n", b->memory, b->count, + dev->buffers_number); + if (opener->timeout_image_io) { + if (b->memory != V4L2_MEMORY_MMAP) + return -EINVAL; + b->count = 1; + return 0; + } + + init_buffers(dev); + switch (b->memory) { + case V4L2_MEMORY_MMAP: + /* do nothing here, buffers are always allocated */ + if (b->count < 1 || dev->buffers_number < 1) + return 0; + + if (b->count > dev->buffers_number) + b->count = dev->buffers_number; + + /* make sure that outbufs_list contains buffers from 0 to used_buffers-1 + * actually, it will have been already populated via v4l2_loopback_init() + * at this point */ + if (list_empty(&dev->outbufs_list)) { + for (i = 0; i < dev->used_buffers; ++i) + list_add_tail(&dev->buffers[i].list_head, + &dev->outbufs_list); + } + + /* also, if dev->used_buffers is going to be decreased, we should remove + * out-of-range buffers from outbufs_list, and fix bufpos2index mapping */ + if (b->count < dev->used_buffers) { + struct v4l2l_buffer *pos, *n; + + list_for_each_entry_safe (pos, n, &dev->outbufs_list, + list_head) { + if (pos->buffer.index >= b->count) + list_del(&pos->list_head); + } + + /* after we update dev->used_buffers, buffers in outbufs_list will + * correspond to dev->write_position + [0;b->count-1] range */ + i = dev->write_position; + list_for_each_entry (pos, &dev->outbufs_list, + list_head) { + dev->bufpos2index[i % b->count] = + pos->buffer.index; + ++i; + } + } + + opener->buffers_number = b->count; + if (opener->buffers_number < dev->used_buffers) + dev->used_buffers = opener->buffers_number; + return 0; + default: + return -EINVAL; + } +} + +/* returns buffer asked for; + * give app as many buffers as it wants, if it less than MAX, + * but map them in our inner buffers + * called on VIDIOC_QUERYBUF + */ +static int vidioc_querybuf(struct file *file, void *fh, struct v4l2_buffer *b) +{ + enum v4l2_buf_type type; + int index; + struct v4l2_loopback_device *dev; + struct v4l2_loopback_opener *opener; + + MARK(); + + type = b->type; + index = b->index; + dev = v4l2loopback_getdevice(file); + opener = fh_to_opener(fh); + + if ((b->type != V4L2_BUF_TYPE_VIDEO_CAPTURE) && + (b->type != V4L2_BUF_TYPE_VIDEO_OUTPUT)) { + return -EINVAL; + } + if (b->index > max_buffers) + return -EINVAL; + + if (opener->timeout_image_io) + *b = dev->timeout_image_buffer.buffer; + else + *b = dev->buffers[b->index % dev->used_buffers].buffer; + + b->type = type; + b->index = index; + dprintkrw("buffer type: %d (of %d with size=%ld)\n", b->memory, + dev->buffers_number, dev->buffer_size); + + /* Hopefully fix 'DQBUF return bad index if queue bigger then 2 for capture' + https://github.com/umlaeute/v4l2loopback/issues/60 */ + b->flags &= ~V4L2_BUF_FLAG_DONE; + b->flags |= V4L2_BUF_FLAG_QUEUED; + + return 0; +} + +static void buffer_written(struct v4l2_loopback_device *dev, + struct v4l2l_buffer *buf) +{ + del_timer_sync(&dev->sustain_timer); + del_timer_sync(&dev->timeout_timer); + spin_lock_bh(&dev->lock); + + dev->bufpos2index[dev->write_position % dev->used_buffers] = + buf->buffer.index; + list_move_tail(&buf->list_head, &dev->outbufs_list); + ++dev->write_position; + dev->reread_count = 0; + + check_timers(dev); + spin_unlock_bh(&dev->lock); +} + +/* put buffer to queue + * called on VIDIOC_QBUF + */ +static int vidioc_qbuf(struct file *file, void *fh, struct v4l2_buffer *buf) +{ + struct v4l2_loopback_device *dev; + struct v4l2_loopback_opener *opener; + struct v4l2l_buffer *b; + int index; + + dev = v4l2loopback_getdevice(file); + opener = fh_to_opener(fh); + + if (buf->index > max_buffers) + return -EINVAL; + if (opener->timeout_image_io) + return 0; + + index = buf->index % dev->used_buffers; + b = &dev->buffers[index]; + + switch (buf->type) { + case V4L2_BUF_TYPE_VIDEO_CAPTURE: + dprintkrw("capture QBUF index: %d\n", index); + set_queued(b); + return 0; + case V4L2_BUF_TYPE_VIDEO_OUTPUT: + dprintkrw("output QBUF pos: %d index: %d\n", + dev->write_position, index); + if (buf->timestamp.tv_sec == 0 && buf->timestamp.tv_usec == 0) + v4l2l_get_timestamp(&b->buffer); + else + b->buffer.timestamp = buf->timestamp; + b->buffer.bytesused = buf->bytesused; + set_done(b); + buffer_written(dev, b); + + /* Hopefully fix 'DQBUF return bad index if queue bigger then 2 for capture' + https://github.com/umlaeute/v4l2loopback/issues/60 */ + buf->flags &= ~V4L2_BUF_FLAG_DONE; + buf->flags |= V4L2_BUF_FLAG_QUEUED; + + wake_up_all(&dev->read_event); + return 0; + default: + return -EINVAL; + } +} + +static int can_read(struct v4l2_loopback_device *dev, + struct v4l2_loopback_opener *opener) +{ + int ret; + + spin_lock_bh(&dev->lock); + check_timers(dev); + ret = dev->write_position > opener->read_position || + dev->reread_count > opener->reread_count || dev->timeout_happened; + spin_unlock_bh(&dev->lock); + return ret; +} + +static int get_capture_buffer(struct file *file) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(file->private_data); + int pos, ret; + int timeout_happened; + + if ((file->f_flags & O_NONBLOCK) && + (dev->write_position <= opener->read_position && + dev->reread_count <= opener->reread_count && + !dev->timeout_happened)) + return -EAGAIN; + wait_event_interruptible(dev->read_event, can_read(dev, opener)); + + spin_lock_bh(&dev->lock); + if (dev->write_position == opener->read_position) { + if (dev->reread_count > opener->reread_count + 2) + opener->reread_count = dev->reread_count - 1; + ++opener->reread_count; + pos = (opener->read_position + dev->used_buffers - 1) % + dev->used_buffers; + } else { + opener->reread_count = 0; + if (dev->write_position > + opener->read_position + dev->used_buffers) + opener->read_position = dev->write_position - 1; + pos = opener->read_position % dev->used_buffers; + ++opener->read_position; + } + timeout_happened = dev->timeout_happened; + dev->timeout_happened = 0; + spin_unlock_bh(&dev->lock); + + ret = dev->bufpos2index[pos]; + if (timeout_happened) { + /* although allocated on-demand, timeout_image is freed only + * in free_buffers(), so we don't need to worry about it being + * deallocated suddenly */ + memcpy(dev->image + dev->buffers[ret].buffer.m.offset, + dev->timeout_image, dev->buffer_size); + } + return ret; +} + +/* put buffer to dequeue + * called on VIDIOC_DQBUF + */ +static int vidioc_dqbuf(struct file *file, void *fh, struct v4l2_buffer *buf) +{ + struct v4l2_loopback_device *dev; + struct v4l2_loopback_opener *opener; + int index; + struct v4l2l_buffer *b; + + dev = v4l2loopback_getdevice(file); + opener = fh_to_opener(fh); + if (opener->timeout_image_io) { + *buf = dev->timeout_image_buffer.buffer; + return 0; + } + + switch (buf->type) { + case V4L2_BUF_TYPE_VIDEO_CAPTURE: + index = get_capture_buffer(file); + if (index < 0) + return index; + dprintkrw("capture DQBUF pos: %d index: %d\n", + opener->read_position - 1, index); + if (!(dev->buffers[index].buffer.flags & + V4L2_BUF_FLAG_MAPPED)) { + dprintk("trying to return not mapped buf[%d]\n", index); + return -EINVAL; + } + unset_flags(&dev->buffers[index]); + *buf = dev->buffers[index].buffer; + return 0; + case V4L2_BUF_TYPE_VIDEO_OUTPUT: + b = list_entry(dev->outbufs_list.prev, struct v4l2l_buffer, + list_head); + list_move_tail(&b->list_head, &dev->outbufs_list); + dprintkrw("output DQBUF index: %d\n", b->buffer.index); + unset_flags(b); + *buf = b->buffer; + buf->type = V4L2_BUF_TYPE_VIDEO_OUTPUT; + return 0; + default: + return -EINVAL; + } +} + +/* ------------- STREAMING ------------------- */ + +/* start streaming + * called on VIDIOC_STREAMON + */ +static int vidioc_streamon(struct file *file, void *fh, enum v4l2_buf_type type) +{ + struct v4l2_loopback_device *dev; + struct v4l2_loopback_opener *opener; + MARK(); + + dev = v4l2loopback_getdevice(file); + opener = fh_to_opener(fh); + + switch (type) { + case V4L2_BUF_TYPE_VIDEO_OUTPUT: + opener->type = WRITER; + dev->ready_for_output = 0; + if (!dev->ready_for_capture) { + int ret = allocate_buffers(dev); + if (ret < 0) + return ret; + } + dev->ready_for_capture++; + return 0; + case V4L2_BUF_TYPE_VIDEO_CAPTURE: + opener->type = READER; + if (!dev->ready_for_capture) + return -EIO; + return 0; + default: + return -EINVAL; + } + return -EINVAL; +} + +/* stop streaming + * called on VIDIOC_STREAMOFF + */ +static int vidioc_streamoff(struct file *file, void *fh, + enum v4l2_buf_type type) +{ + struct v4l2_loopback_device *dev; + MARK(); + dprintk("%d\n", type); + + dev = v4l2loopback_getdevice(file); + + switch (type) { + case V4L2_BUF_TYPE_VIDEO_OUTPUT: + if (dev->ready_for_capture > 0) + dev->ready_for_capture--; + return 0; + case V4L2_BUF_TYPE_VIDEO_CAPTURE: + return 0; + default: + return -EINVAL; + } + return -EINVAL; +} + +#ifdef CONFIG_VIDEO_V4L1_COMPAT +static int vidiocgmbuf(struct file *file, void *fh, struct video_mbuf *p) +{ + struct v4l2_loopback_device *dev; + MARK(); + + dev = v4l2loopback_getdevice(file); + p->frames = dev->buffers_number; + p->offsets[0] = 0; + p->offsets[1] = 0; + p->size = dev->buffer_size; + return 0; +} +#endif + +static int vidioc_subscribe_event(struct v4l2_fh *fh, + const struct v4l2_event_subscription *sub) +{ + switch (sub->type) { + case V4L2_EVENT_CTRL: + return v4l2_ctrl_subscribe_event(fh, sub); + } + + return -EINVAL; +} + +/* file operations */ +static void vm_open(struct vm_area_struct *vma) +{ + struct v4l2l_buffer *buf; + MARK(); + + buf = vma->vm_private_data; + buf->use_count++; +} + +static void vm_close(struct vm_area_struct *vma) +{ + struct v4l2l_buffer *buf; + MARK(); + + buf = vma->vm_private_data; + buf->use_count--; +} + +static struct vm_operations_struct vm_ops = { + .open = vm_open, + .close = vm_close, +}; + +static int v4l2_loopback_mmap(struct file *file, struct vm_area_struct *vma) +{ + unsigned long addr; + unsigned long start; + unsigned long size; + struct v4l2_loopback_device *dev; + struct v4l2_loopback_opener *opener; + struct v4l2l_buffer *buffer = NULL; + MARK(); + + start = (unsigned long)vma->vm_start; + size = (unsigned long)(vma->vm_end - vma->vm_start); + + dev = v4l2loopback_getdevice(file); + opener = fh_to_opener(file->private_data); + + if (size > dev->buffer_size) { + dprintk("userspace tries to mmap too much, fail\n"); + return -EINVAL; + } + if (opener->timeout_image_io) { + /* we are going to map the timeout_image_buffer */ + if ((vma->vm_pgoff << PAGE_SHIFT) != + dev->buffer_size * MAX_BUFFERS) { + dprintk("invalid mmap offset for timeout_image_io mode\n"); + return -EINVAL; + } + } else if ((vma->vm_pgoff << PAGE_SHIFT) > + dev->buffer_size * (dev->buffers_number - 1)) { + dprintk("userspace tries to mmap too far, fail\n"); + return -EINVAL; + } + + /* FIXXXXXME: allocation should not happen here! */ + if (NULL == dev->image) + if (allocate_buffers(dev) < 0) + return -EINVAL; + + if (opener->timeout_image_io) { + buffer = &dev->timeout_image_buffer; + addr = (unsigned long)dev->timeout_image; + } else { + int i; + for (i = 0; i < dev->buffers_number; ++i) { + buffer = &dev->buffers[i]; + if ((buffer->buffer.m.offset >> PAGE_SHIFT) == + vma->vm_pgoff) + break; + } + + if (NULL == buffer) + return -EINVAL; + + addr = (unsigned long)dev->image + + (vma->vm_pgoff << PAGE_SHIFT); + } + + while (size > 0) { + struct page *page; + + page = (void *)vmalloc_to_page((void *)addr); + + if (vm_insert_page(vma, start, page) < 0) + return -EAGAIN; + + start += PAGE_SIZE; + addr += PAGE_SIZE; + size -= PAGE_SIZE; + } + + vma->vm_ops = &vm_ops; + vma->vm_private_data = buffer; + buffer->buffer.flags |= V4L2_BUF_FLAG_MAPPED; + + vm_open(vma); + + MARK(); + return 0; +} + +static unsigned int v4l2_loopback_poll(struct file *file, + struct poll_table_struct *pts) +{ + struct v4l2_loopback_opener *opener; + struct v4l2_loopback_device *dev; + __poll_t req_events = poll_requested_events(pts); + int ret_mask = 0; + MARK(); + + opener = fh_to_opener(file->private_data); + dev = v4l2loopback_getdevice(file); + + if (req_events & POLLPRI) { + if (!v4l2_event_pending(&opener->fh)) + poll_wait(file, &opener->fh.wait, pts); + if (v4l2_event_pending(&opener->fh)) { + ret_mask |= POLLPRI; + if (!(req_events & DEFAULT_POLLMASK)) + return ret_mask; + } + } + + switch (opener->type) { + case WRITER: + ret_mask |= POLLOUT | POLLWRNORM; + break; + case READER: + if (!can_read(dev, opener)) { + if (ret_mask) + return ret_mask; + poll_wait(file, &dev->read_event, pts); + } + if (can_read(dev, opener)) + ret_mask |= POLLIN | POLLRDNORM; + if (v4l2_event_pending(&opener->fh)) + ret_mask |= POLLPRI; + break; + default: + break; + } + + MARK(); + return ret_mask; +} + +/* do not want to limit device opens, it can be as many readers as user want, + * writers are limited by means of setting writer field */ +static int v4l2_loopback_open(struct file *file) +{ + struct v4l2_loopback_device *dev; + struct v4l2_loopback_opener *opener; + MARK(); + dev = v4l2loopback_getdevice(file); + if (dev->open_count.counter >= dev->max_openers) + return -EBUSY; + /* kfree on close */ + opener = kzalloc(sizeof(*opener), GFP_KERNEL); + if (opener == NULL) + return -ENOMEM; + + v4l2_fh_init(&opener->fh, video_devdata(file)); + file->private_data = &opener->fh; + atomic_inc(&dev->open_count); + + opener->timeout_image_io = dev->timeout_image_io; + dev->timeout_image_io = 0; + + if (opener->timeout_image_io) { + int r = allocate_timeout_image(dev); + + if (r < 0) { + dprintk("timeout image allocation failed\n"); + return r; + } + } + + v4l2_fh_add(&opener->fh); + dprintk("opened dev:%p with image:%p\n", dev, dev ? dev->image : NULL); + MARK(); + return 0; +} + +static int v4l2_loopback_close(struct file *file) +{ + struct v4l2_loopback_opener *opener; + struct v4l2_loopback_device *dev; + int iswriter = 0; + MARK(); + + opener = fh_to_opener(file->private_data); + dev = v4l2loopback_getdevice(file); + + if (WRITER == opener->type) + iswriter = 1; + + atomic_dec(&dev->open_count); + if (dev->open_count.counter == 0) { + del_timer_sync(&dev->sustain_timer); + del_timer_sync(&dev->timeout_timer); + } + try_free_buffers(dev); + + v4l2_fh_del(&opener->fh); + v4l2_fh_exit(&opener->fh); + + kfree(opener); + if (iswriter) { + dev->ready_for_output = 1; + } + MARK(); + return 0; +} + +static ssize_t v4l2_loopback_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + int read_index; + struct v4l2_loopback_device *dev; + struct v4l2_buffer *b; + MARK(); + + dev = v4l2loopback_getdevice(file); + + read_index = get_capture_buffer(file); + if (read_index < 0) + return read_index; + if (count > dev->buffer_size) + count = dev->buffer_size; + b = &dev->buffers[read_index].buffer; + if (count > b->bytesused) + count = b->bytesused; + if (copy_to_user((void *)buf, (void *)(dev->image + b->m.offset), + count)) { + printk(KERN_ERR + "v4l2-loopback: failed copy_to_user() in read buf\n"); + return -EFAULT; + } + dprintkrw("leave v4l2_loopback_read()\n"); + return count; +} + +static ssize_t v4l2_loopback_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct v4l2_loopback_device *dev; + int write_index; + struct v4l2_buffer *b; + MARK(); + + dev = v4l2loopback_getdevice(file); + + /* there's at least one writer, so don't stop announcing output capabilities */ + dev->ready_for_output = 0; + + if (!dev->ready_for_capture) { + int ret = allocate_buffers(dev); + if (ret < 0) + return ret; + dev->ready_for_capture = 1; + } + dprintkrw("v4l2_loopback_write() trying to write %zu bytes\n", count); + if (count > dev->buffer_size) + count = dev->buffer_size; + + write_index = dev->write_position % dev->used_buffers; + b = &dev->buffers[write_index].buffer; + + if (copy_from_user((void *)(dev->image + b->m.offset), (void *)buf, + count)) { + printk(KERN_ERR + "v4l2-loopback: failed copy_from_user() in write buf, could not write %zu\n", + count); + return -EFAULT; + } + v4l2l_get_timestamp(b); + b->bytesused = count; + b->sequence = dev->write_position; + buffer_written(dev, &dev->buffers[write_index]); + wake_up_all(&dev->read_event); + dprintkrw("leave v4l2_loopback_write()\n"); + return count; +} + +/* init functions */ +/* frees buffers, if already allocated */ +static int free_buffers(struct v4l2_loopback_device *dev) +{ + MARK(); + dprintk("freeing image@%p for dev:%p\n", dev ? dev->image : NULL, dev); + if (dev->image) { + vfree(dev->image); + dev->image = NULL; + } + if (dev->timeout_image) { + vfree(dev->timeout_image); + dev->timeout_image = NULL; + } + dev->imagesize = 0; + + return 0; +} +/* frees buffers, if they are no longer needed */ +static void try_free_buffers(struct v4l2_loopback_device *dev) +{ + MARK(); + if (0 == dev->open_count.counter && !dev->keep_format) { + free_buffers(dev); + dev->ready_for_capture = 0; + dev->buffer_size = 0; + dev->write_position = 0; + } +} +/* allocates buffers, if buffer_size is set */ +static int allocate_buffers(struct v4l2_loopback_device *dev) +{ + MARK(); + /* vfree on close file operation in case no open handles left */ + if (0 == dev->buffer_size) + return -EINVAL; + + if (dev->image) { + dprintk("allocating buffers again: %ld %ld\n", + dev->buffer_size * dev->buffers_number, dev->imagesize); + /* FIXME: prevent double allocation more intelligently! */ + if (dev->buffer_size * dev->buffers_number == dev->imagesize) + return 0; + + /* if there is only one writer, no problem should occur */ + if (dev->open_count.counter == 1) + free_buffers(dev); + else + return -EINVAL; + } + + dev->imagesize = dev->buffer_size * dev->buffers_number; + + dprintk("allocating %ld = %ldx%d\n", dev->imagesize, dev->buffer_size, + dev->buffers_number); + + dev->image = vmalloc(dev->imagesize); + if (dev->timeout_jiffies > 0) + allocate_timeout_image(dev); + + if (dev->image == NULL) + return -ENOMEM; + dprintk("vmallocated %ld bytes\n", dev->imagesize); + MARK(); + init_buffers(dev); + return 0; +} + +/* init inner buffers, they are capture mode and flags are set as + * for capture mod buffers */ +static void init_buffers(struct v4l2_loopback_device *dev) +{ + int i; + int buffer_size; + int bytesused; + MARK(); + + buffer_size = dev->buffer_size; + bytesused = dev->pix_format.sizeimage; + + for (i = 0; i < dev->buffers_number; ++i) { + struct v4l2_buffer *b = &dev->buffers[i].buffer; + b->index = i; + b->bytesused = bytesused; + b->length = buffer_size; + b->field = V4L2_FIELD_NONE; + b->flags = 0; +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 6, 1) + b->input = 0; +#endif + b->m.offset = i * buffer_size; + b->memory = V4L2_MEMORY_MMAP; + b->sequence = 0; + b->timestamp.tv_sec = 0; + b->timestamp.tv_usec = 0; + b->type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + + v4l2l_get_timestamp(b); + } + dev->timeout_image_buffer = dev->buffers[0]; + dev->timeout_image_buffer.buffer.m.offset = MAX_BUFFERS * buffer_size; + MARK(); +} + +static int allocate_timeout_image(struct v4l2_loopback_device *dev) +{ + MARK(); + if (dev->buffer_size <= 0) + return -EINVAL; + + if (dev->timeout_image == NULL) { + dev->timeout_image = v4l2l_vzalloc(dev->buffer_size); + if (dev->timeout_image == NULL) + return -ENOMEM; + } + return 0; +} + +/* fills and register video device */ +static void init_vdev(struct video_device *vdev, int nr) +{ + MARK(); + +#ifdef V4L2LOOPBACK_WITH_STD + vdev->tvnorms = V4L2_STD_ALL; +#endif /* V4L2LOOPBACK_WITH_STD */ + + vdev->vfl_type = VFL_TYPE_VIDEO; + vdev->fops = &v4l2_loopback_fops; + vdev->ioctl_ops = &v4l2_loopback_ioctl_ops; + vdev->release = &video_device_release; + vdev->minor = -1; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 7, 0) + vdev->device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_VIDEO_OUTPUT | + V4L2_CAP_READWRITE | V4L2_CAP_STREAMING; +#ifdef V4L2_CAP_VIDEO_M2M + vdev->device_caps |= V4L2_CAP_VIDEO_M2M; +#endif +#endif /* >=linux-4.7.0 */ + + if (debug > 1) +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 20, 0) + vdev->debug = V4L2_DEBUG_IOCTL | V4L2_DEBUG_IOCTL_ARG; +#else + vdev->dev_debug = + V4L2_DEV_DEBUG_IOCTL | V4L2_DEV_DEBUG_IOCTL_ARG; +#endif + + /* since kernel-3.7, there is a new field 'vfl_dir' that has to be + * set to VFL_DIR_M2M for bidirectional devices */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) + vdev->vfl_dir = VFL_DIR_M2M; +#endif + + MARK(); +} + +/* init default capture parameters, only fps may be changed in future */ +static void init_capture_param(struct v4l2_captureparm *capture_param) +{ + MARK(); + capture_param->capability = 0; + capture_param->capturemode = 0; + capture_param->extendedmode = 0; + capture_param->readbuffers = max_buffers; + capture_param->timeperframe.numerator = 1; + capture_param->timeperframe.denominator = 30; +} + +static void check_timers(struct v4l2_loopback_device *dev) +{ + if (!dev->ready_for_capture) + return; + + if (dev->timeout_jiffies > 0 && !timer_pending(&dev->timeout_timer)) + mod_timer(&dev->timeout_timer, jiffies + dev->timeout_jiffies); + if (dev->sustain_framerate && !timer_pending(&dev->sustain_timer)) + mod_timer(&dev->sustain_timer, + jiffies + dev->frame_jiffies * 3 / 2); +} +#ifdef HAVE_TIMER_SETUP +static void sustain_timer_clb(struct timer_list *t) +{ + struct v4l2_loopback_device *dev = from_timer(dev, t, sustain_timer); +#else +static void sustain_timer_clb(unsigned long nr) +{ + struct v4l2_loopback_device *dev = + idr_find(&v4l2loopback_index_idr, nr); +#endif + spin_lock(&dev->lock); + if (dev->sustain_framerate) { + dev->reread_count++; + dprintkrw("reread: %d %d\n", dev->write_position, + dev->reread_count); + if (dev->reread_count == 1) + mod_timer(&dev->sustain_timer, + jiffies + max(1UL, dev->frame_jiffies / 2)); + else + mod_timer(&dev->sustain_timer, + jiffies + dev->frame_jiffies); + wake_up_all(&dev->read_event); + } + spin_unlock(&dev->lock); +} +#ifdef HAVE_TIMER_SETUP +static void timeout_timer_clb(struct timer_list *t) +{ + struct v4l2_loopback_device *dev = from_timer(dev, t, timeout_timer); +#else +static void timeout_timer_clb(unsigned long nr) +{ + struct v4l2_loopback_device *dev = + idr_find(&v4l2loopback_index_idr, nr); +#endif + spin_lock(&dev->lock); + if (dev->timeout_jiffies > 0) { + dev->timeout_happened = 1; + mod_timer(&dev->timeout_timer, jiffies + dev->timeout_jiffies); + wake_up_all(&dev->read_event); + } + spin_unlock(&dev->lock); +} + +/* init loopback main structure */ +#define DEFAULT_FROM_CONF(confmember, default_condition, default_value) \ + ((conf) ? \ + ((conf->confmember default_condition) ? (default_value) : \ + (conf->confmember)) : \ + default_value) + +static int v4l2_loopback_add(struct v4l2_loopback_config *conf, int *ret_nr) +{ + struct v4l2_loopback_device *dev; + struct v4l2_ctrl_handler *hdl; + + int err = -ENOMEM; + + int _max_width = DEFAULT_FROM_CONF( + max_width, <= V4L2LOOPBACK_SIZE_MIN_WIDTH, max_width); + int _max_height = DEFAULT_FROM_CONF( + max_height, <= V4L2LOOPBACK_SIZE_MIN_HEIGHT, max_height); + bool _announce_all_caps = (conf && conf->announce_all_caps >= 0) ? + (conf->announce_all_caps) : + V4L2LOOPBACK_DEFAULT_EXCLUSIVECAPS; + + int _max_buffers = DEFAULT_FROM_CONF(max_buffers, <= 0, max_buffers); + int _max_openers = DEFAULT_FROM_CONF(max_openers, <= 0, max_openers); + + int nr = -1; + if (conf) { + if (conf->capture_nr >= 0 && + conf->output_nr == conf->capture_nr) { + nr = conf->capture_nr; + } else if (conf->capture_nr < 0 && conf->output_nr < 0) { + nr = -1; + } else if (conf->capture_nr < 0) { + nr = conf->output_nr; + } else if (conf->output_nr < 0) { + nr = conf->capture_nr; + } else { + printk(KERN_ERR + "split OUTPUT and CAPTURE devices not yet supported."); + printk(KERN_INFO + "both devices must have the same number (%d != %d).", + conf->output_nr, conf->capture_nr); + return -EINVAL; + } + } + + if (idr_find(&v4l2loopback_index_idr, nr)) + return -EEXIST; + + dprintk("creating v4l2loopback-device #%d\n", nr); + dev = kzalloc(sizeof(*dev), GFP_KERNEL); + if (!dev) + return -ENOMEM; + + /* allocate id, if @id >= 0, we're requesting that specific id */ + if (nr >= 0) { + err = idr_alloc(&v4l2loopback_index_idr, dev, nr, nr + 1, + GFP_KERNEL); + if (err == -ENOSPC) + err = -EEXIST; + } else { + err = idr_alloc(&v4l2loopback_index_idr, dev, 0, 0, GFP_KERNEL); + } + if (err < 0) + goto out_free_dev; + nr = err; + err = -ENOMEM; + + if (conf && conf->card_label && *(conf->card_label)) { + snprintf(dev->card_label, sizeof(dev->card_label), "%s", + conf->card_label); + } else { + snprintf(dev->card_label, sizeof(dev->card_label), + "Dummy video device (0x%04X)", nr); + } + snprintf(dev->v4l2_dev.name, sizeof(dev->v4l2_dev.name), + "v4l2loopback-%03d", nr); + + err = v4l2_device_register(NULL, &dev->v4l2_dev); + if (err) + goto out_free_idr; + MARK(); + + dev->vdev = video_device_alloc(); + if (dev->vdev == NULL) { + err = -ENOMEM; + goto out_unregister; + } + video_set_drvdata(dev->vdev, + kzalloc(sizeof(struct v4l2loopback_private), + GFP_KERNEL)); + if (video_get_drvdata(dev->vdev) == NULL) { + err = -ENOMEM; + goto out_unregister; + } + + MARK(); + snprintf(dev->vdev->name, sizeof(dev->vdev->name), dev->card_label); + + ((struct v4l2loopback_private *)video_get_drvdata(dev->vdev)) + ->device_nr = nr; + + init_vdev(dev->vdev, nr); + dev->vdev->v4l2_dev = &dev->v4l2_dev; + init_capture_param(&dev->capture_param); + set_timeperframe(dev, &dev->capture_param.timeperframe); + dev->keep_format = 0; + dev->sustain_framerate = 0; + + dev->announce_all_caps = _announce_all_caps; + dev->max_width = _max_width; + dev->max_height = _max_height; + dev->max_openers = _max_openers; + dev->buffers_number = dev->used_buffers = _max_buffers; + + dev->write_position = 0; + + MARK(); + spin_lock_init(&dev->lock); + INIT_LIST_HEAD(&dev->outbufs_list); + if (list_empty(&dev->outbufs_list)) { + int i; + + for (i = 0; i < dev->used_buffers; ++i) + list_add_tail(&dev->buffers[i].list_head, + &dev->outbufs_list); + } + memset(dev->bufpos2index, 0, sizeof(dev->bufpos2index)); + atomic_set(&dev->open_count, 0); + dev->ready_for_capture = 0; + dev->ready_for_output = 1; + + dev->buffer_size = 0; + dev->image = NULL; + dev->imagesize = 0; +#ifdef HAVE_TIMER_SETUP + timer_setup(&dev->sustain_timer, sustain_timer_clb, 0); + timer_setup(&dev->timeout_timer, timeout_timer_clb, 0); +#else + setup_timer(&dev->sustain_timer, sustain_timer_clb, nr); + setup_timer(&dev->timeout_timer, timeout_timer_clb, nr); +#endif + dev->reread_count = 0; + dev->timeout_jiffies = 0; + dev->timeout_image = NULL; + dev->timeout_happened = 0; + + hdl = &dev->ctrl_handler; + err = v4l2_ctrl_handler_init(hdl, 4); + if (err) + goto out_unregister; + v4l2_ctrl_new_custom(hdl, &v4l2loopback_ctrl_keepformat, NULL); + v4l2_ctrl_new_custom(hdl, &v4l2loopback_ctrl_sustainframerate, NULL); + v4l2_ctrl_new_custom(hdl, &v4l2loopback_ctrl_timeout, NULL); + v4l2_ctrl_new_custom(hdl, &v4l2loopback_ctrl_timeoutimageio, NULL); + if (hdl->error) { + err = hdl->error; + goto out_free_handler; + } + dev->v4l2_dev.ctrl_handler = hdl; + + err = v4l2_ctrl_handler_setup(hdl); + + /* FIXME set buffers to 0 */ + + /* Set initial format */ + dev->pix_format.width = 0; /* V4L2LOOPBACK_SIZE_DEFAULT_WIDTH; */ + dev->pix_format.height = 0; /* V4L2LOOPBACK_SIZE_DEFAULT_HEIGHT; */ + dev->pix_format.pixelformat = formats[0].fourcc; + dev->pix_format.colorspace = + V4L2_COLORSPACE_SRGB; /* do we need to set this ? */ + dev->pix_format.field = V4L2_FIELD_NONE; + + dev->buffer_size = PAGE_ALIGN(dev->pix_format.sizeimage); + dprintk("buffer_size = %ld (=%d)\n", dev->buffer_size, + dev->pix_format.sizeimage); + allocate_buffers(dev); + + init_waitqueue_head(&dev->read_event); + + /* register the device -> it creates /dev/video* */ + if (video_register_device(dev->vdev, VFL_TYPE_VIDEO, nr) < 0) { + printk(KERN_ERR + "v4l2loopback: failed video_register_device()\n"); + err = -EFAULT; + goto out_free_device; + } + v4l2loopback_create_sysfs(dev->vdev); + + MARK(); + if (ret_nr) + *ret_nr = dev->vdev->num; + return 0; + +out_free_device: + video_device_release(dev->vdev); +out_free_handler: + v4l2_ctrl_handler_free(&dev->ctrl_handler); +out_unregister: + v4l2_device_unregister(&dev->v4l2_dev); +out_free_idr: + idr_remove(&v4l2loopback_index_idr, nr); +out_free_dev: + kfree(dev); + return err; +} + +static void v4l2_loopback_remove(struct v4l2_loopback_device *dev) +{ + free_buffers(dev); + v4l2loopback_remove_sysfs(dev->vdev); + kfree(video_get_drvdata(dev->vdev)); + video_unregister_device(dev->vdev); + v4l2_device_unregister(&dev->v4l2_dev); + v4l2_ctrl_handler_free(&dev->ctrl_handler); + kfree(dev); +} + +static long v4l2loopback_control_ioctl(struct file *file, unsigned int cmd, + unsigned long parm) +{ + struct v4l2_loopback_device *dev; + struct v4l2_loopback_config conf; + struct v4l2_loopback_config *confptr = &conf; + int device_nr; + int ret; + + ret = mutex_lock_killable(&v4l2loopback_ctl_mutex); + if (ret) + return ret; + + ret = -EINVAL; + switch (cmd) { + default: + ret = -ENOSYS; + break; + /* add a v4l2loopback device (pair), based on the user-provided specs */ + case V4L2LOOPBACK_CTL_ADD: + if (parm) { + if ((ret = copy_from_user(&conf, (void *)parm, + sizeof(conf))) < 0) + break; + } else + confptr = NULL; + ret = v4l2_loopback_add(confptr, &device_nr); + if (ret >= 0) + ret = device_nr; + break; + /* remove a v4l2loopback device (both capture and output) */ + case V4L2LOOPBACK_CTL_REMOVE: + ret = v4l2loopback_lookup((int)parm, &dev); + if (ret >= 0 && dev) { + int nr = ret; + ret = -EBUSY; + if (dev->open_count.counter > 0) + break; + idr_remove(&v4l2loopback_index_idr, nr); + v4l2_loopback_remove(dev); + ret = 0; + }; + break; + /* get information for a loopback device. + * this is mostly about limits (which cannot be queried directly with VIDIOC_G_FMT and friends + */ + case V4L2LOOPBACK_CTL_QUERY: + if (!parm) + break; + if ((ret = copy_from_user(&conf, (void *)parm, sizeof(conf))) < + 0) + break; + device_nr = + (conf.output_nr < 0) ? conf.capture_nr : conf.output_nr; + MARK(); + /* get the device from either capture_nr or output_nr (whatever is valid) */ + if ((ret = v4l2loopback_lookup(device_nr, &dev)) < 0) + break; + MARK(); + /* if we got the device from output_nr and there is a valid capture_nr, + * make sure that both refer to the same device (or bail out) + */ + if ((device_nr != conf.capture_nr) && (conf.capture_nr >= 0) && + (ret != v4l2loopback_lookup(conf.capture_nr, 0))) + break; + MARK(); + /* if otoh, we got the device from capture_nr and there is a valid output_nr, + * make sure that both refer to the same device (or bail out) + */ + if ((device_nr != conf.output_nr) && (conf.output_nr >= 0) && + (ret != v4l2loopback_lookup(conf.output_nr, 0))) + break; + MARK(); + + /* v4l2_loopback_config identified a single device, so fetch the data */ + snprintf(conf.card_label, sizeof(conf.card_label), "%s", + dev->card_label); + MARK(); + conf.output_nr = conf.capture_nr = dev->vdev->num; + conf.max_width = dev->max_width; + conf.max_height = dev->max_height; + conf.announce_all_caps = dev->announce_all_caps; + conf.max_buffers = dev->buffers_number; + conf.max_openers = dev->max_openers; + conf.debug = debug; + MARK(); + if (copy_to_user((void *)parm, &conf, sizeof(conf))) { + ret = -EFAULT; + break; + } + MARK(); + ret = 0; + ; + break; + } + + MARK(); + mutex_unlock(&v4l2loopback_ctl_mutex); + MARK(); + return ret; +} + +/* LINUX KERNEL */ + +static const struct file_operations v4l2loopback_ctl_fops = { + // clang-format off + .open = nonseekable_open, + .unlocked_ioctl = v4l2loopback_control_ioctl, + .compat_ioctl = v4l2loopback_control_ioctl, + .owner = THIS_MODULE, + .llseek = noop_llseek, + // clang-format on +}; + +static struct miscdevice v4l2loopback_misc = { + // clang-format off + .minor = MISC_DYNAMIC_MINOR, + .name = "v4l2loopback", + .fops = &v4l2loopback_ctl_fops, + // clang-format on +}; + +static const struct v4l2_file_operations v4l2_loopback_fops = { + // clang-format off + .owner = THIS_MODULE, + .open = v4l2_loopback_open, + .release = v4l2_loopback_close, + .read = v4l2_loopback_read, + .write = v4l2_loopback_write, + .poll = v4l2_loopback_poll, + .mmap = v4l2_loopback_mmap, + .unlocked_ioctl = video_ioctl2, + // clang-format on +}; + +static const struct v4l2_ioctl_ops v4l2_loopback_ioctl_ops = { + // clang-format off + .vidioc_querycap = &vidioc_querycap, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29) + .vidioc_enum_framesizes = &vidioc_enum_framesizes, + .vidioc_enum_frameintervals = &vidioc_enum_frameintervals, +#endif + +#ifndef HAVE__V4L2_CTRLS + .vidioc_queryctrl = &vidioc_queryctrl, + .vidioc_g_ctrl = &vidioc_g_ctrl, + .vidioc_s_ctrl = &vidioc_s_ctrl, +#endif /* HAVE__V4L2_CTRLS */ + + .vidioc_enum_output = &vidioc_enum_output, + .vidioc_g_output = &vidioc_g_output, + .vidioc_s_output = &vidioc_s_output, + + .vidioc_enum_input = &vidioc_enum_input, + .vidioc_g_input = &vidioc_g_input, + .vidioc_s_input = &vidioc_s_input, + + .vidioc_enum_fmt_vid_cap = &vidioc_enum_fmt_cap, + .vidioc_g_fmt_vid_cap = &vidioc_g_fmt_cap, + .vidioc_s_fmt_vid_cap = &vidioc_s_fmt_cap, + .vidioc_try_fmt_vid_cap = &vidioc_try_fmt_cap, + + .vidioc_enum_fmt_vid_out = &vidioc_enum_fmt_out, + .vidioc_s_fmt_vid_out = &vidioc_s_fmt_out, + .vidioc_g_fmt_vid_out = &vidioc_g_fmt_out, + .vidioc_try_fmt_vid_out = &vidioc_try_fmt_out, + +#ifdef V4L2L_OVERLAY + .vidioc_s_fmt_vid_overlay = &vidioc_s_fmt_overlay, + .vidioc_g_fmt_vid_overlay = &vidioc_g_fmt_overlay, +#endif + +#ifdef V4L2LOOPBACK_WITH_STD + .vidioc_s_std = &vidioc_s_std, + .vidioc_g_std = &vidioc_g_std, + .vidioc_querystd = &vidioc_querystd, +#endif /* V4L2LOOPBACK_WITH_STD */ + + .vidioc_g_parm = &vidioc_g_parm, + .vidioc_s_parm = &vidioc_s_parm, + + .vidioc_reqbufs = &vidioc_reqbufs, + .vidioc_querybuf = &vidioc_querybuf, + .vidioc_qbuf = &vidioc_qbuf, + .vidioc_dqbuf = &vidioc_dqbuf, + + .vidioc_streamon = &vidioc_streamon, + .vidioc_streamoff = &vidioc_streamoff, + +#ifdef CONFIG_VIDEO_V4L1_COMPAT + .vidiocgmbuf = &vidiocgmbuf, +#endif + + .vidioc_subscribe_event = &vidioc_subscribe_event, + .vidioc_unsubscribe_event = &v4l2_event_unsubscribe, + // clang-format on +}; + +static int free_device_cb(int id, void *ptr, void *data) +{ + struct v4l2_loopback_device *dev = ptr; + v4l2_loopback_remove(dev); + return 0; +} +static void free_devices(void) +{ + idr_for_each(&v4l2loopback_index_idr, &free_device_cb, NULL); + idr_destroy(&v4l2loopback_index_idr); +} + +static int __init v4l2loopback_init_module(void) +{ + int err; + int i; + MARK(); + + err = misc_register(&v4l2loopback_misc); + if (err < 0) + return err; + + if (devices < 0) { + devices = 1; + + /* try guessing the devices from the "video_nr" parameter */ + for (i = MAX_DEVICES - 1; i >= 0; i--) { + if (video_nr[i] >= 0) { + devices = i + 1; + break; + } + } + } + + if (devices > MAX_DEVICES) { + devices = MAX_DEVICES; + printk(KERN_INFO + "v4l2loopback: number of initial devices is limited to: %d\n", + MAX_DEVICES); + } + + if (max_buffers > MAX_BUFFERS) { + max_buffers = MAX_BUFFERS; + printk(KERN_INFO + "v4l2loopback: number of buffers is limited to: %d\n", + MAX_BUFFERS); + } + + if (max_openers < 0) { + printk(KERN_INFO + "v4l2loopback: allowing %d openers rather than %d\n", + 2, max_openers); + max_openers = 2; + } + + if (max_width < 1) { + max_width = V4L2LOOPBACK_SIZE_DEFAULT_MAX_WIDTH; + printk(KERN_INFO "v4l2loopback: using max_width %d\n", + max_width); + } + if (max_height < 1) { + max_height = V4L2LOOPBACK_SIZE_DEFAULT_MAX_HEIGHT; + printk(KERN_INFO "v4l2loopback: using max_height %d\n", + max_height); + } + + /* kfree on module release */ + for (i = 0; i < devices; i++) { + struct v4l2_loopback_config cfg = { + // clang-format off + .output_nr = video_nr[i], + .capture_nr = video_nr[i], + .max_width = max_width, + .max_height = max_height, + .announce_all_caps = (!exclusive_caps[i]), + .max_buffers = max_buffers, + .max_openers = max_openers, + .debug = debug, + // clang-format on + }; + cfg.card_label[0] = 0; + if (card_label[i]) + snprintf(cfg.card_label, sizeof(cfg.card_label), "%s", + card_label[i]); + err = v4l2_loopback_add(&cfg, 0); + if (err) { + free_devices(); + goto error; + } + } + + dprintk("module installed\n"); + + printk(KERN_INFO "v4l2loopback driver version %d.%d.%d loaded\n", + // clang-format off + (V4L2LOOPBACK_VERSION_CODE >> 16) & 0xff, + (V4L2LOOPBACK_VERSION_CODE >> 8) & 0xff, + (V4L2LOOPBACK_VERSION_CODE ) & 0xff); + // clang-format on + + return 0; +error: + misc_deregister(&v4l2loopback_misc); + return err; +} + +#ifdef MODULE +static void v4l2loopback_cleanup_module(void) +{ + MARK(); + /* unregister the device -> it deletes /dev/video* */ + free_devices(); + /* and get rid of /dev/v4l2loopback */ + misc_deregister(&v4l2loopback_misc); + dprintk("module removed\n"); +} +#endif + +MODULE_ALIAS_MISCDEV(MISC_DYNAMIC_MINOR); +MODULE_ALIAS("devname:v4l2loopback"); + +#ifdef MODULE +int __init init_module(void) +{ + return v4l2loopback_init_module(); +} +void __exit cleanup_module(void) +{ + return v4l2loopback_cleanup_module(); +} +#else +late_initcall(v4l2loopback_init_module); +#endif + +/* + * fake usage of unused functions + */ +#ifdef HAVE__V4L2_CTRLS +static int vidioc_queryctrl(struct file *file, void *fh, + struct v4l2_queryctrl *q) __attribute__((unused)); +static int vidioc_g_ctrl(struct file *file, void *fh, struct v4l2_control *c) + __attribute__((unused)); +static int vidioc_s_ctrl(struct file *file, void *fh, struct v4l2_control *c) + __attribute__((unused)); +#endif /* HAVE__V4L2_CTRLS */ diff --git a/drivers/media/v4l2-core/v4l2loopback.h b/drivers/media/v4l2-core/v4l2loopback.h new file mode 100644 index 000000000000..77c671971ef2 --- /dev/null +++ b/drivers/media/v4l2-core/v4l2loopback.h @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ +/* + * v4l2loopback.h + * + * Written by IOhannes m zmölnig, 7/1/20. + * + * Copyright 2020 by IOhannes m zmölnig. Redistribution of this file is + * permitted under the GNU General Public License. + */ +#ifndef _V4L2LOOPBACK_H +#define _V4L2LOOPBACK_H + +#define V4L2LOOPBACK_VERSION_MAJOR 0 +#define V4L2LOOPBACK_VERSION_MINOR 12 +#define V4L2LOOPBACK_VERSION_BUGFIX 5 + +/* /dev/v4l2loopback interface */ + +struct v4l2_loopback_config { + /** + * the device-number (/dev/video) + * V4L2LOOPBACK_CTL_ADD: + * setting this to a value<0, will allocate an available one + * if nr>=0 and the device already exists, the ioctl will EEXIST + * if output_nr and capture_nr are the same, only a single device will be created + * + * V4L2LOOPBACK_CTL_QUERY: + * either both output_nr and capture_nr must refer to the same loopback, + * or one (and only one) of them must be -1 + * + */ + int output_nr; + int capture_nr; + + /** + * a nice name for your device + * if (*card_label)==0, an automatic name is assigned + */ + char card_label[32]; + + /** + * maximum allowed frame size + * if too low, default values are used + */ + int max_width; + int max_height; + + /** + * whether to announce OUTPUT/CAPTURE capabilities exclusively + * for this device or not + * (!exclusive_caps) + * FIXXME: this ought to be removed (if superseded by output_nr vs capture_nr) + */ + int announce_all_caps; + + /** + * number of buffers to allocate for the queue + * if set to <=0, default values are used + */ + int max_buffers; + + /** + * how many consumers are allowed to open this device concurrently + * if set to <=0, default values are used + */ + int max_openers; + + /** + * set the debugging level for this device + */ + int debug; +}; + +/* a pointer to a (struct v4l2_loopback_config) that has all values you wish to impose on the + * to-be-created device set. + * if the ptr is NULL, a new device is created with default values at the driver's discretion. + * + * returns the device_nr of the OUTPUT device (which can be used with V4L2LOOPBACK_CTL_QUERY, + * to get more information on the device) + */ +#define V4L2LOOPBACK_CTL_ADD 0x4C80 + +/* a pointer to a (struct v4l2_loopback_config) that has output_nr and/or capture_nr set + * (the two values must either refer to video-devices associated with the same loopback device + * or exactly one of them must be <0 + */ +#define V4L2LOOPBACK_CTL_QUERY 0x4C82 + +/* the device-number (either CAPTURE or OUTPUT) associated with the loopback-device */ +#define V4L2LOOPBACK_CTL_REMOVE 0x4C81 + +#endif /* _V4L2LOOPBACK_H */ diff --git a/drivers/media/v4l2-core/v4l2loopback_formats.h b/drivers/media/v4l2-core/v4l2loopback_formats.h new file mode 100644 index 000000000000..6cb6f842cd1a --- /dev/null +++ b/drivers/media/v4l2-core/v4l2loopback_formats.h @@ -0,0 +1,427 @@ +#ifndef V4L2_PIX_FMT_VP9 +#define V4L2_PIX_FMT_VP9 v4l2_fourcc('V', 'P', '9', '0') +#endif +#ifndef V4L2_PIX_FMT_HEVC +#define V4L2_PIX_FMT_HEVC v4l2_fourcc('H', 'E', 'V', 'C') +#endif + +/* here come the packed formats */ +{ + .name = "32 bpp RGB, le", + .fourcc = V4L2_PIX_FMT_BGR32, + .depth = 32, + .flags = 0, +}, + { + .name = "32 bpp RGB, be", + .fourcc = V4L2_PIX_FMT_RGB32, + .depth = 32, + .flags = 0, + }, + { + .name = "24 bpp RGB, le", + .fourcc = V4L2_PIX_FMT_BGR24, + .depth = 24, + .flags = 0, + }, + { + .name = "24 bpp RGB, be", + .fourcc = V4L2_PIX_FMT_RGB24, + .depth = 24, + .flags = 0, + }, +#ifdef V4L2_PIX_FMT_RGB332 + { + .name = "8 bpp RGB-3-3-2", + .fourcc = V4L2_PIX_FMT_RGB332, + .depth = 8, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_RGB332 */ +#ifdef V4L2_PIX_FMT_RGB444 + { + .name = "16 bpp RGB (xxxxrrrr ggggbbbb)", + .fourcc = V4L2_PIX_FMT_RGB444, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_RGB444 */ +#ifdef V4L2_PIX_FMT_RGB555 + { + .name = "16 bpp RGB-5-5-5", + .fourcc = V4L2_PIX_FMT_RGB555, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_RGB555 */ +#ifdef V4L2_PIX_FMT_RGB565 + { + .name = "16 bpp RGB-5-6-5", + .fourcc = V4L2_PIX_FMT_RGB565, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_RGB565 */ +#ifdef V4L2_PIX_FMT_RGB555X + { + .name = "16 bpp RGB-5-5-5 BE", + .fourcc = V4L2_PIX_FMT_RGB555X, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_RGB555X */ +#ifdef V4L2_PIX_FMT_RGB565X + { + .name = "16 bpp RGB-5-6-5 BE", + .fourcc = V4L2_PIX_FMT_RGB565X, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_RGB565X */ +#ifdef V4L2_PIX_FMT_BGR666 + { + .name = "18 bpp BGR-6-6-6", + .fourcc = V4L2_PIX_FMT_BGR666, + .depth = 18, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_BGR666 */ + { + .name = "4:2:2, packed, YUYV", + .fourcc = V4L2_PIX_FMT_YUYV, + .depth = 16, + .flags = 0, + }, + { + .name = "4:2:2, packed, UYVY", + .fourcc = V4L2_PIX_FMT_UYVY, + .depth = 16, + .flags = 0, + }, +#ifdef V4L2_PIX_FMT_YVYU + { + .name = "4:2:2, packed YVYU", + .fourcc = V4L2_PIX_FMT_YVYU, + .depth = 16, + .flags = 0, + }, +#endif +#ifdef V4L2_PIX_FMT_VYUY + { + .name = "4:2:2, packed VYUY", + .fourcc = V4L2_PIX_FMT_VYUY, + .depth = 16, + .flags = 0, + }, +#endif + { + .name = "4:2:2, packed YYUV", + .fourcc = V4L2_PIX_FMT_YYUV, + .depth = 16, + .flags = 0, + }, + { + .name = "YUV-8-8-8-8", + .fourcc = V4L2_PIX_FMT_YUV32, + .depth = 32, + .flags = 0, + }, + { + .name = "8 bpp, Greyscale", + .fourcc = V4L2_PIX_FMT_GREY, + .depth = 8, + .flags = 0, + }, +#ifdef V4L2_PIX_FMT_Y4 + { + .name = "4 bpp Greyscale", + .fourcc = V4L2_PIX_FMT_Y4, + .depth = 4, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_Y4 */ +#ifdef V4L2_PIX_FMT_Y6 + { + .name = "6 bpp Greyscale", + .fourcc = V4L2_PIX_FMT_Y6, + .depth = 6, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_Y6 */ +#ifdef V4L2_PIX_FMT_Y10 + { + .name = "10 bpp Greyscale", + .fourcc = V4L2_PIX_FMT_Y10, + .depth = 10, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_Y10 */ +#ifdef V4L2_PIX_FMT_Y12 + { + .name = "12 bpp Greyscale", + .fourcc = V4L2_PIX_FMT_Y12, + .depth = 12, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_Y12 */ + { + .name = "16 bpp, Greyscale", + .fourcc = V4L2_PIX_FMT_Y16, + .depth = 16, + .flags = 0, + }, +#ifdef V4L2_PIX_FMT_YUV444 + { + .name = "16 bpp xxxxyyyy uuuuvvvv", + .fourcc = V4L2_PIX_FMT_YUV444, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_YUV444 */ +#ifdef V4L2_PIX_FMT_YUV555 + { + .name = "16 bpp YUV-5-5-5", + .fourcc = V4L2_PIX_FMT_YUV555, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_YUV555 */ +#ifdef V4L2_PIX_FMT_YUV565 + { + .name = "16 bpp YUV-5-6-5", + .fourcc = V4L2_PIX_FMT_YUV565, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_YUV565 */ + +/* bayer formats */ +#ifdef V4L2_PIX_FMT_SRGGB8 + { + .name = "Bayer RGGB 8bit", + .fourcc = V4L2_PIX_FMT_SRGGB8, + .depth = 8, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_SRGGB8 */ +#ifdef V4L2_PIX_FMT_SGRBG8 + { + .name = "Bayer GRBG 8bit", + .fourcc = V4L2_PIX_FMT_SGRBG8, + .depth = 8, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_SGRBG8 */ +#ifdef V4L2_PIX_FMT_SGBRG8 + { + .name = "Bayer GBRG 8bit", + .fourcc = V4L2_PIX_FMT_SGBRG8, + .depth = 8, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_SGBRG8 */ +#ifdef V4L2_PIX_FMT_SBGGR8 + { + .name = "Bayer BA81 8bit", + .fourcc = V4L2_PIX_FMT_SBGGR8, + .depth = 8, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_SBGGR8 */ + + /* here come the planar formats */ + { + .name = "4:1:0, planar, Y-Cr-Cb", + .fourcc = V4L2_PIX_FMT_YVU410, + .depth = 9, + .flags = FORMAT_FLAGS_PLANAR, + }, + { + .name = "4:2:0, planar, Y-Cr-Cb", + .fourcc = V4L2_PIX_FMT_YVU420, + .depth = 12, + .flags = FORMAT_FLAGS_PLANAR, + }, + { + .name = "4:1:0, planar, Y-Cb-Cr", + .fourcc = V4L2_PIX_FMT_YUV410, + .depth = 9, + .flags = FORMAT_FLAGS_PLANAR, + }, + { + .name = "4:2:0, planar, Y-Cb-Cr", + .fourcc = V4L2_PIX_FMT_YUV420, + .depth = 12, + .flags = FORMAT_FLAGS_PLANAR, + }, +#ifdef V4L2_PIX_FMT_YUV422P + { + .name = "16 bpp YVU422 planar", + .fourcc = V4L2_PIX_FMT_YUV422P, + .depth = 16, + .flags = FORMAT_FLAGS_PLANAR, + }, +#endif /* V4L2_PIX_FMT_YUV422P */ +#ifdef V4L2_PIX_FMT_YUV411P + { + .name = "16 bpp YVU411 planar", + .fourcc = V4L2_PIX_FMT_YUV411P, + .depth = 16, + .flags = FORMAT_FLAGS_PLANAR, + }, +#endif /* V4L2_PIX_FMT_YUV411P */ +#ifdef V4L2_PIX_FMT_Y41P + { + .name = "12 bpp YUV 4:1:1", + .fourcc = V4L2_PIX_FMT_Y41P, + .depth = 12, + .flags = FORMAT_FLAGS_PLANAR, + }, +#endif /* V4L2_PIX_FMT_Y41P */ +#ifdef V4L2_PIX_FMT_NV12 + { + .name = "12 bpp Y/CbCr 4:2:0 ", + .fourcc = V4L2_PIX_FMT_NV12, + .depth = 12, + .flags = FORMAT_FLAGS_PLANAR, + }, +#endif /* V4L2_PIX_FMT_NV12 */ + +/* here come the compressed formats */ + +#ifdef V4L2_PIX_FMT_MJPEG + { + .name = "Motion-JPEG", + .fourcc = V4L2_PIX_FMT_MJPEG, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_MJPEG */ +#ifdef V4L2_PIX_FMT_JPEG + { + .name = "JFIF JPEG", + .fourcc = V4L2_PIX_FMT_JPEG, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_JPEG */ +#ifdef V4L2_PIX_FMT_DV + { + .name = "DV1394", + .fourcc = V4L2_PIX_FMT_DV, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_DV */ +#ifdef V4L2_PIX_FMT_MPEG + { + .name = "MPEG-1/2/4 Multiplexed", + .fourcc = V4L2_PIX_FMT_MPEG, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_MPEG */ +#ifdef V4L2_PIX_FMT_H264 + { + .name = "H264 with start codes", + .fourcc = V4L2_PIX_FMT_H264, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_H264 */ +#ifdef V4L2_PIX_FMT_H264_NO_SC + { + .name = "H264 without start codes", + .fourcc = V4L2_PIX_FMT_H264_NO_SC, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_H264_NO_SC */ +#ifdef V4L2_PIX_FMT_H264_MVC + { + .name = "H264 MVC", + .fourcc = V4L2_PIX_FMT_H264_MVC, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_H264_MVC */ +#ifdef V4L2_PIX_FMT_H263 + { + .name = "H263", + .fourcc = V4L2_PIX_FMT_H263, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_H263 */ +#ifdef V4L2_PIX_FMT_MPEG1 + { + .name = "MPEG-1 ES", + .fourcc = V4L2_PIX_FMT_MPEG1, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_MPEG1 */ +#ifdef V4L2_PIX_FMT_MPEG2 + { + .name = "MPEG-2 ES", + .fourcc = V4L2_PIX_FMT_MPEG2, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_MPEG2 */ +#ifdef V4L2_PIX_FMT_MPEG4 + { + .name = "MPEG-4 part 2 ES", + .fourcc = V4L2_PIX_FMT_MPEG4, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_MPEG4 */ +#ifdef V4L2_PIX_FMT_XVID + { + .name = "Xvid", + .fourcc = V4L2_PIX_FMT_XVID, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_XVID */ +#ifdef V4L2_PIX_FMT_VC1_ANNEX_G + { + .name = "SMPTE 421M Annex G compliant stream", + .fourcc = V4L2_PIX_FMT_VC1_ANNEX_G, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_VC1_ANNEX_G */ +#ifdef V4L2_PIX_FMT_VC1_ANNEX_L + { + .name = "SMPTE 421M Annex L compliant stream", + .fourcc = V4L2_PIX_FMT_VC1_ANNEX_L, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_VC1_ANNEX_L */ +#ifdef V4L2_PIX_FMT_VP8 + { + .name = "VP8", + .fourcc = V4L2_PIX_FMT_VP8, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_VP8 */ +#ifdef V4L2_PIX_FMT_VP9 + { + .name = "VP9", + .fourcc = V4L2_PIX_FMT_VP9, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_VP9 */ +#ifdef V4L2_PIX_FMT_HEVC + { + .name = "HEVC", + .fourcc = V4L2_PIX_FMT_HEVC, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_HEVC */ diff --git a/drivers/memory/omap-gpmc.c b/drivers/memory/omap-gpmc.c index cfa730cfd145..f80c2ea39ca4 100644 --- a/drivers/memory/omap-gpmc.c +++ b/drivers/memory/omap-gpmc.c @@ -1009,8 +1009,8 @@ EXPORT_SYMBOL(gpmc_cs_request); void gpmc_cs_free(int cs) { - struct gpmc_cs_data *gpmc = &gpmc_cs[cs]; - struct resource *res = &gpmc->mem; + struct gpmc_cs_data *gpmc; + struct resource *res; spin_lock(&gpmc_mem_lock); if (cs >= gpmc_cs_num || cs < 0 || !gpmc_cs_reserved(cs)) { @@ -1018,6 +1018,9 @@ void gpmc_cs_free(int cs) spin_unlock(&gpmc_mem_lock); return; } + gpmc = &gpmc_cs[cs]; + res = &gpmc->mem; + gpmc_cs_disable_mem(cs); if (res->flags) release_resource(res); diff --git a/drivers/memory/pl353-smc.c b/drivers/memory/pl353-smc.c index 3b5b1045edd9..9c0a28416777 100644 --- a/drivers/memory/pl353-smc.c +++ b/drivers/memory/pl353-smc.c @@ -63,7 +63,7 @@ /* ECC memory config register specific constants */ #define PL353_SMC_ECC_MEMCFG_MODE_MASK 0xC #define PL353_SMC_ECC_MEMCFG_MODE_SHIFT 2 -#define PL353_SMC_ECC_MEMCFG_PGSIZE_MASK 0xC +#define PL353_SMC_ECC_MEMCFG_PGSIZE_MASK 0x3 #define PL353_SMC_DC_UPT_NAND_REGS ((4 << 23) | /* CS: NAND chip */ \ (2 << 21)) /* UpdateRegs operation */ diff --git a/drivers/memory/renesas-rpc-if.c b/drivers/memory/renesas-rpc-if.c index 8d36e221def1..45eed659b0c6 100644 --- a/drivers/memory/renesas-rpc-if.c +++ b/drivers/memory/renesas-rpc-if.c @@ -192,10 +192,10 @@ int rpcif_sw_init(struct rpcif *rpc, struct device *dev) } res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dirmap"); - rpc->size = resource_size(res); rpc->dirmap = devm_ioremap_resource(&pdev->dev, res); if (IS_ERR(rpc->dirmap)) rpc->dirmap = NULL; + rpc->size = resource_size(res); rpc->rstc = devm_reset_control_get_exclusive(&pdev->dev, NULL); diff --git a/drivers/memory/samsung/exynos5422-dmc.c b/drivers/memory/samsung/exynos5422-dmc.c index 1dabb509dec3..dee503640e12 100644 --- a/drivers/memory/samsung/exynos5422-dmc.c +++ b/drivers/memory/samsung/exynos5422-dmc.c @@ -1298,7 +1298,9 @@ static int exynos5_dmc_init_clks(struct exynos5_dmc *dmc) dmc->curr_volt = target_volt; - clk_set_parent(dmc->mout_mx_mspll_ccore, dmc->mout_spll); + ret = clk_set_parent(dmc->mout_mx_mspll_ccore, dmc->mout_spll); + if (ret) + return ret; clk_prepare_enable(dmc->fout_bpll); clk_prepare_enable(dmc->mout_bpll); diff --git a/drivers/mfd/arizona-irq.c b/drivers/mfd/arizona-irq.c index 077d9ab112b7..d919ae9691e2 100644 --- a/drivers/mfd/arizona-irq.c +++ b/drivers/mfd/arizona-irq.c @@ -100,7 +100,7 @@ static irqreturn_t arizona_irq_thread(int irq, void *data) unsigned int val; int ret; - ret = pm_runtime_get_sync(arizona->dev); + ret = pm_runtime_resume_and_get(arizona->dev); if (ret < 0) { dev_err(arizona->dev, "Failed to resume device: %d\n", ret); return IRQ_NONE; diff --git a/drivers/mfd/da9063-i2c.c b/drivers/mfd/da9063-i2c.c index 3781d0bb7786..783a14af18e2 100644 --- a/drivers/mfd/da9063-i2c.c +++ b/drivers/mfd/da9063-i2c.c @@ -442,6 +442,16 @@ static int da9063_i2c_probe(struct i2c_client *i2c, return ret; } + /* If SMBus is not available and only I2C is possible, enter I2C mode */ + if (i2c_check_functionality(i2c->adapter, I2C_FUNC_I2C)) { + ret = regmap_clear_bits(da9063->regmap, DA9063_REG_CONFIG_J, + DA9063_TWOWIRE_TO); + if (ret < 0) { + dev_err(da9063->dev, "Failed to set Two-Wire Bus Mode.\n"); + return -EIO; + } + } + return da9063_device_init(da9063, i2c->irq); } diff --git a/drivers/mfd/intel_pmt.c b/drivers/mfd/intel_pmt.c index 744b230cdcca..65da2b17a204 100644 --- a/drivers/mfd/intel_pmt.c +++ b/drivers/mfd/intel_pmt.c @@ -79,19 +79,18 @@ static int pmt_add_dev(struct pci_dev *pdev, struct intel_dvsec_header *header, case DVSEC_INTEL_ID_WATCHER: if (quirks & PMT_QUIRK_NO_WATCHER) { dev_info(dev, "Watcher not supported\n"); - return 0; + return -EINVAL; } name = "pmt_watcher"; break; case DVSEC_INTEL_ID_CRASHLOG: if (quirks & PMT_QUIRK_NO_CRASHLOG) { dev_info(dev, "Crashlog not supported\n"); - return 0; + return -EINVAL; } name = "pmt_crashlog"; break; default: - dev_err(dev, "Unrecognized PMT capability: %d\n", id); return -EINVAL; } @@ -174,12 +173,8 @@ static int pmt_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) header.offset = INTEL_DVSEC_TABLE_OFFSET(table); ret = pmt_add_dev(pdev, &header, quirks); - if (ret) { - dev_warn(&pdev->dev, - "Failed to add device for DVSEC id %d\n", - header.id); + if (ret) continue; - } found_devices = true; } while (true); diff --git a/drivers/mfd/stm32-timers.c b/drivers/mfd/stm32-timers.c index add603359124..44ed2fce0319 100644 --- a/drivers/mfd/stm32-timers.c +++ b/drivers/mfd/stm32-timers.c @@ -158,13 +158,18 @@ static const struct regmap_config stm32_timers_regmap_cfg = { static void stm32_timers_get_arr_size(struct stm32_timers *ddata) { + u32 arr; + + /* Backup ARR to restore it after getting the maximum value */ + regmap_read(ddata->regmap, TIM_ARR, &arr); + /* * Only the available bits will be written so when readback * we get the maximum value of auto reload register */ regmap_write(ddata->regmap, TIM_ARR, ~0L); regmap_read(ddata->regmap, TIM_ARR, &ddata->max_arr); - regmap_write(ddata->regmap, TIM_ARR, 0x0); + regmap_write(ddata->regmap, TIM_ARR, arr); } static int stm32_timers_dma_probe(struct device *dev, diff --git a/drivers/mfd/stmpe.c b/drivers/mfd/stmpe.c index 90f3292230c9..1dd39483e7c1 100644 --- a/drivers/mfd/stmpe.c +++ b/drivers/mfd/stmpe.c @@ -312,7 +312,7 @@ EXPORT_SYMBOL_GPL(stmpe_set_altfunc); * GPIO (all variants) */ -static const struct resource stmpe_gpio_resources[] = { +static struct resource stmpe_gpio_resources[] = { /* Start and end filled dynamically */ { .flags = IORESOURCE_IRQ, @@ -336,7 +336,8 @@ static const struct mfd_cell stmpe_gpio_cell_noirq = { * Keypad (1601, 2401, 2403) */ -static const struct resource stmpe_keypad_resources[] = { +static struct resource stmpe_keypad_resources[] = { + /* Start and end filled dynamically */ { .name = "KEYPAD", .flags = IORESOURCE_IRQ, @@ -357,7 +358,8 @@ static const struct mfd_cell stmpe_keypad_cell = { /* * PWM (1601, 2401, 2403) */ -static const struct resource stmpe_pwm_resources[] = { +static struct resource stmpe_pwm_resources[] = { + /* Start and end filled dynamically */ { .name = "PWM0", .flags = IORESOURCE_IRQ, @@ -445,7 +447,8 @@ static struct stmpe_variant_info stmpe801_noirq = { * Touchscreen (STMPE811 or STMPE610) */ -static const struct resource stmpe_ts_resources[] = { +static struct resource stmpe_ts_resources[] = { + /* Start and end filled dynamically */ { .name = "TOUCH_DET", .flags = IORESOURCE_IRQ, @@ -467,7 +470,8 @@ static const struct mfd_cell stmpe_ts_cell = { * ADC (STMPE811) */ -static const struct resource stmpe_adc_resources[] = { +static struct resource stmpe_adc_resources[] = { + /* Start and end filled dynamically */ { .name = "STMPE_TEMP_SENS", .flags = IORESOURCE_IRQ, diff --git a/drivers/misc/kgdbts.c b/drivers/misc/kgdbts.c index 945701bce553..2e081a58da6c 100644 --- a/drivers/misc/kgdbts.c +++ b/drivers/misc/kgdbts.c @@ -95,19 +95,19 @@ #include -#define v1printk(a...) do { \ - if (verbose) \ - printk(KERN_INFO a); \ - } while (0) -#define v2printk(a...) do { \ - if (verbose > 1) \ - printk(KERN_INFO a); \ - touch_nmi_watchdog(); \ - } while (0) -#define eprintk(a...) do { \ - printk(KERN_ERR a); \ - WARN_ON(1); \ - } while (0) +#define v1printk(a...) do { \ + if (verbose) \ + printk(KERN_INFO a); \ +} while (0) +#define v2printk(a...) do { \ + if (verbose > 1) \ + printk(KERN_INFO a); \ + touch_nmi_watchdog(); \ +} while (0) +#define eprintk(a...) do { \ + printk(KERN_ERR a); \ + WARN_ON(1); \ +} while (0) #define MAX_CONFIG_LEN 40 static struct kgdb_io kgdbts_io_ops; diff --git a/drivers/misc/lis3lv02d/lis3lv02d.c b/drivers/misc/lis3lv02d/lis3lv02d.c index dd65cedf3b12..9d14bf444481 100644 --- a/drivers/misc/lis3lv02d/lis3lv02d.c +++ b/drivers/misc/lis3lv02d/lis3lv02d.c @@ -208,7 +208,7 @@ static int lis3_3dc_rates[16] = {0, 1, 10, 25, 50, 100, 200, 400, 1600, 5000}; static int lis3_3dlh_rates[4] = {50, 100, 400, 1000}; /* ODR is Output Data Rate */ -static int lis3lv02d_get_odr(struct lis3lv02d *lis3) +static int lis3lv02d_get_odr_index(struct lis3lv02d *lis3) { u8 ctrl; int shift; @@ -216,15 +216,23 @@ static int lis3lv02d_get_odr(struct lis3lv02d *lis3) lis3->read(lis3, CTRL_REG1, &ctrl); ctrl &= lis3->odr_mask; shift = ffs(lis3->odr_mask) - 1; - return lis3->odrs[(ctrl >> shift)]; + return (ctrl >> shift); } static int lis3lv02d_get_pwron_wait(struct lis3lv02d *lis3) { - int div = lis3lv02d_get_odr(lis3); + int odr_idx = lis3lv02d_get_odr_index(lis3); + int div = lis3->odrs[odr_idx]; - if (WARN_ONCE(div == 0, "device returned spurious data")) + if (div == 0) { + if (odr_idx == 0) { + /* Power-down mode, not sampling no need to sleep */ + return 0; + } + + dev_err(&lis3->pdev->dev, "Error unknown odrs-index: %d\n", odr_idx); return -ENXIO; + } /* LIS3 power on delay is quite long */ msleep(lis3->pwron_delay / div); @@ -816,9 +824,12 @@ static ssize_t lis3lv02d_rate_show(struct device *dev, struct device_attribute *attr, char *buf) { struct lis3lv02d *lis3 = dev_get_drvdata(dev); + int odr_idx; lis3lv02d_sysfs_poweron(lis3); - return sprintf(buf, "%d\n", lis3lv02d_get_odr(lis3)); + + odr_idx = lis3lv02d_get_odr_index(lis3); + return sprintf(buf, "%d\n", lis3->odrs[odr_idx]); } static ssize_t lis3lv02d_rate_set(struct device *dev, diff --git a/drivers/misc/mei/hw-me-regs.h b/drivers/misc/mei/hw-me-regs.h index 14be76d4c2e6..cb34925e10f1 100644 --- a/drivers/misc/mei/hw-me-regs.h +++ b/drivers/misc/mei/hw-me-regs.h @@ -105,6 +105,7 @@ #define MEI_DEV_ID_ADP_S 0x7AE8 /* Alder Lake Point S */ #define MEI_DEV_ID_ADP_LP 0x7A60 /* Alder Lake Point LP */ +#define MEI_DEV_ID_ADP_P 0x51E0 /* Alder Lake Point P */ /* * MEI HW Section diff --git a/drivers/misc/mei/pci-me.c b/drivers/misc/mei/pci-me.c index a7e179626b63..c3393b383e59 100644 --- a/drivers/misc/mei/pci-me.c +++ b/drivers/misc/mei/pci-me.c @@ -111,6 +111,7 @@ static const struct pci_device_id mei_me_pci_tbl[] = { {MEI_PCI_DEVICE(MEI_DEV_ID_ADP_S, MEI_ME_PCH15_CFG)}, {MEI_PCI_DEVICE(MEI_DEV_ID_ADP_LP, MEI_ME_PCH15_CFG)}, + {MEI_PCI_DEVICE(MEI_DEV_ID_ADP_P, MEI_ME_PCH15_CFG)}, /* required last entry */ {0, } diff --git a/drivers/misc/vmw_vmci/vmci_doorbell.c b/drivers/misc/vmw_vmci/vmci_doorbell.c index 345addd9306d..fa8a7fce4481 100644 --- a/drivers/misc/vmw_vmci/vmci_doorbell.c +++ b/drivers/misc/vmw_vmci/vmci_doorbell.c @@ -326,7 +326,7 @@ int vmci_dbell_host_context_notify(u32 src_cid, struct vmci_handle handle) bool vmci_dbell_register_notification_bitmap(u64 bitmap_ppn) { int result; - struct vmci_notify_bm_set_msg bitmap_set_msg; + struct vmci_notify_bm_set_msg bitmap_set_msg = { }; bitmap_set_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID, VMCI_SET_NOTIFY_BITMAP); diff --git a/drivers/misc/vmw_vmci/vmci_guest.c b/drivers/misc/vmw_vmci/vmci_guest.c index cc8eeb361fcd..1018dc77269d 100644 --- a/drivers/misc/vmw_vmci/vmci_guest.c +++ b/drivers/misc/vmw_vmci/vmci_guest.c @@ -168,7 +168,7 @@ static int vmci_check_host_caps(struct pci_dev *pdev) VMCI_UTIL_NUM_RESOURCES * sizeof(u32); struct vmci_datagram *check_msg; - check_msg = kmalloc(msg_size, GFP_KERNEL); + check_msg = kzalloc(msg_size, GFP_KERNEL); if (!check_msg) { dev_err(&pdev->dev, "%s: Insufficient memory\n", __func__); return -ENOMEM; diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c index d666e24fbe0e..a4c06ef67394 100644 --- a/drivers/mmc/core/block.c +++ b/drivers/mmc/core/block.c @@ -572,6 +572,18 @@ static int __mmc_blk_ioctl_cmd(struct mmc_card *card, struct mmc_blk_data *md, main_md->part_curr = value & EXT_CSD_PART_CONFIG_ACC_MASK; } + /* + * Make sure to update CACHE_CTRL in case it was changed. The cache + * will get turned back on if the card is re-initialized, e.g. + * suspend/resume or hw reset in recovery. + */ + if ((MMC_EXTRACT_INDEX_FROM_ARG(cmd.arg) == EXT_CSD_CACHE_CTRL) && + (cmd.opcode == MMC_SWITCH)) { + u8 value = MMC_EXTRACT_VALUE_FROM_ARG(cmd.arg) & 1; + + card->ext_csd.cache_ctrl = value; + } + /* * According to the SD specs, some commands require a delay after * issuing the command. @@ -2224,6 +2236,10 @@ enum mmc_issued mmc_blk_mq_issue_rq(struct mmc_queue *mq, struct request *req) case MMC_ISSUE_ASYNC: switch (req_op(req)) { case REQ_OP_FLUSH: + if (!mmc_cache_enabled(host)) { + blk_mq_end_request(req, BLK_STS_OK); + return MMC_REQ_FINISHED; + } ret = mmc_blk_cqe_issue_flush(mq, req); break; case REQ_OP_READ: diff --git a/drivers/mmc/core/core.c b/drivers/mmc/core/core.c index 1136b859ddd8..e30c4e88e404 100644 --- a/drivers/mmc/core/core.c +++ b/drivers/mmc/core/core.c @@ -1207,7 +1207,7 @@ int mmc_set_uhs_voltage(struct mmc_host *host, u32 ocr) err = mmc_wait_for_cmd(host, &cmd, 0); if (err) - return err; + goto power_cycle; if (!mmc_host_is_spi(host) && (cmd.resp[0] & R1_ERROR)) return -EIO; @@ -2369,80 +2369,6 @@ void mmc_stop_host(struct mmc_host *host) mmc_release_host(host); } -#ifdef CONFIG_PM_SLEEP -/* Do the card removal on suspend if card is assumed removeable - * Do that in pm notifier while userspace isn't yet frozen, so we will be able - to sync the card. -*/ -static int mmc_pm_notify(struct notifier_block *notify_block, - unsigned long mode, void *unused) -{ - struct mmc_host *host = container_of( - notify_block, struct mmc_host, pm_notify); - unsigned long flags; - int err = 0; - - switch (mode) { - case PM_HIBERNATION_PREPARE: - case PM_SUSPEND_PREPARE: - case PM_RESTORE_PREPARE: - spin_lock_irqsave(&host->lock, flags); - host->rescan_disable = 1; - spin_unlock_irqrestore(&host->lock, flags); - cancel_delayed_work_sync(&host->detect); - - if (!host->bus_ops) - break; - - /* Validate prerequisites for suspend */ - if (host->bus_ops->pre_suspend) - err = host->bus_ops->pre_suspend(host); - if (!err) - break; - - if (!mmc_card_is_removable(host)) { - dev_warn(mmc_dev(host), - "pre_suspend failed for non-removable host: " - "%d\n", err); - /* Avoid removing non-removable hosts */ - break; - } - - /* Calling bus_ops->remove() with a claimed host can deadlock */ - host->bus_ops->remove(host); - mmc_claim_host(host); - mmc_detach_bus(host); - mmc_power_off(host); - mmc_release_host(host); - host->pm_flags = 0; - break; - - case PM_POST_SUSPEND: - case PM_POST_HIBERNATION: - case PM_POST_RESTORE: - - spin_lock_irqsave(&host->lock, flags); - host->rescan_disable = 0; - spin_unlock_irqrestore(&host->lock, flags); - _mmc_detect_change(host, 0, false); - - } - - return 0; -} - -void mmc_register_pm_notifier(struct mmc_host *host) -{ - host->pm_notify.notifier_call = mmc_pm_notify; - register_pm_notifier(&host->pm_notify); -} - -void mmc_unregister_pm_notifier(struct mmc_host *host) -{ - unregister_pm_notifier(&host->pm_notify); -} -#endif - static int __init mmc_init(void) { int ret; diff --git a/drivers/mmc/core/core.h b/drivers/mmc/core/core.h index 575ac0257af2..db3c9c68875d 100644 --- a/drivers/mmc/core/core.h +++ b/drivers/mmc/core/core.h @@ -29,6 +29,7 @@ struct mmc_bus_ops { int (*shutdown)(struct mmc_host *); int (*hw_reset)(struct mmc_host *); int (*sw_reset)(struct mmc_host *); + bool (*cache_enabled)(struct mmc_host *); }; void mmc_attach_bus(struct mmc_host *host, const struct mmc_bus_ops *ops); @@ -93,14 +94,6 @@ int mmc_execute_tuning(struct mmc_card *card); int mmc_hs200_to_hs400(struct mmc_card *card); int mmc_hs400_to_hs200(struct mmc_card *card); -#ifdef CONFIG_PM_SLEEP -void mmc_register_pm_notifier(struct mmc_host *host); -void mmc_unregister_pm_notifier(struct mmc_host *host); -#else -static inline void mmc_register_pm_notifier(struct mmc_host *host) { } -static inline void mmc_unregister_pm_notifier(struct mmc_host *host) { } -#endif - void mmc_wait_for_req_done(struct mmc_host *host, struct mmc_request *mrq); bool mmc_is_req_done(struct mmc_host *host, struct mmc_request *mrq); @@ -171,4 +164,12 @@ static inline void mmc_post_req(struct mmc_host *host, struct mmc_request *mrq, host->ops->post_req(host, mrq, err); } +static inline bool mmc_cache_enabled(struct mmc_host *host) +{ + if (host->bus_ops->cache_enabled) + return host->bus_ops->cache_enabled(host); + + return false; +} + #endif diff --git a/drivers/mmc/core/host.c b/drivers/mmc/core/host.c index 9b89a91b6b47..fe05b3645fe9 100644 --- a/drivers/mmc/core/host.c +++ b/drivers/mmc/core/host.c @@ -35,6 +35,42 @@ static DEFINE_IDA(mmc_host_ida); +#ifdef CONFIG_PM_SLEEP +static int mmc_host_class_prepare(struct device *dev) +{ + struct mmc_host *host = cls_dev_to_mmc_host(dev); + + /* + * It's safe to access the bus_ops pointer, as both userspace and the + * workqueue for detecting cards are frozen at this point. + */ + if (!host->bus_ops) + return 0; + + /* Validate conditions for system suspend. */ + if (host->bus_ops->pre_suspend) + return host->bus_ops->pre_suspend(host); + + return 0; +} + +static void mmc_host_class_complete(struct device *dev) +{ + struct mmc_host *host = cls_dev_to_mmc_host(dev); + + _mmc_detect_change(host, 0, false); +} + +static const struct dev_pm_ops mmc_host_class_dev_pm_ops = { + .prepare = mmc_host_class_prepare, + .complete = mmc_host_class_complete, +}; + +#define MMC_HOST_CLASS_DEV_PM_OPS (&mmc_host_class_dev_pm_ops) +#else +#define MMC_HOST_CLASS_DEV_PM_OPS NULL +#endif + static void mmc_host_classdev_release(struct device *dev) { struct mmc_host *host = cls_dev_to_mmc_host(dev); @@ -46,6 +82,7 @@ static void mmc_host_classdev_release(struct device *dev) static struct class mmc_host_class = { .name = "mmc_host", .dev_release = mmc_host_classdev_release, + .pm = MMC_HOST_CLASS_DEV_PM_OPS, }; int mmc_register_host_class(void) @@ -538,8 +575,6 @@ int mmc_add_host(struct mmc_host *host) #endif mmc_start_host(host); - mmc_register_pm_notifier(host); - return 0; } @@ -555,7 +590,6 @@ EXPORT_SYMBOL(mmc_add_host); */ void mmc_remove_host(struct mmc_host *host) { - mmc_unregister_pm_notifier(host); mmc_stop_host(host); #ifdef CONFIG_DEBUG_FS diff --git a/drivers/mmc/core/mmc.c b/drivers/mmc/core/mmc.c index 8741271d3971..4d2b4b0da93c 100644 --- a/drivers/mmc/core/mmc.c +++ b/drivers/mmc/core/mmc.c @@ -2029,6 +2029,12 @@ static void mmc_detect(struct mmc_host *host) } } +static bool _mmc_cache_enabled(struct mmc_host *host) +{ + return host->card->ext_csd.cache_size > 0 && + host->card->ext_csd.cache_ctrl & 1; +} + static int _mmc_suspend(struct mmc_host *host, bool is_suspend) { int err = 0; @@ -2208,6 +2214,7 @@ static const struct mmc_bus_ops mmc_ops = { .alive = mmc_alive, .shutdown = mmc_shutdown, .hw_reset = _mmc_hw_reset, + .cache_enabled = _mmc_cache_enabled, }; /* diff --git a/drivers/mmc/core/mmc_ops.c b/drivers/mmc/core/mmc_ops.c index 265d95ec82ce..c458f6b626a2 100644 --- a/drivers/mmc/core/mmc_ops.c +++ b/drivers/mmc/core/mmc_ops.c @@ -988,9 +988,7 @@ int mmc_flush_cache(struct mmc_card *card) { int err = 0; - if (mmc_card_mmc(card) && - (card->ext_csd.cache_size > 0) && - (card->ext_csd.cache_ctrl & 1)) { + if (mmc_cache_enabled(card->host)) { err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL, EXT_CSD_FLUSH_CACHE, 1, MMC_CACHE_FLUSH_TIMEOUT_MS); diff --git a/drivers/mmc/core/sd.c b/drivers/mmc/core/sd.c index 6fa51a6ed058..2c48d6504101 100644 --- a/drivers/mmc/core/sd.c +++ b/drivers/mmc/core/sd.c @@ -135,6 +135,9 @@ static int mmc_decode_csd(struct mmc_card *card) csd->erase_size = UNSTUFF_BITS(resp, 39, 7) + 1; csd->erase_size <<= csd->write_blkbits - 9; } + + if (UNSTUFF_BITS(resp, 13, 1)) + mmc_card_set_readonly(card); break; case 1: /* @@ -169,6 +172,9 @@ static int mmc_decode_csd(struct mmc_card *card) csd->write_blkbits = 9; csd->write_partial = 0; csd->erase_size = 1; + + if (UNSTUFF_BITS(resp, 13, 1)) + mmc_card_set_readonly(card); break; default: pr_err("%s: unrecognised CSD structure version %d\n", diff --git a/drivers/mmc/core/sdio.c b/drivers/mmc/core/sdio.c index 0fda7784cab2..3eb94ac2712e 100644 --- a/drivers/mmc/core/sdio.c +++ b/drivers/mmc/core/sdio.c @@ -985,21 +985,37 @@ static void mmc_sdio_detect(struct mmc_host *host) */ static int mmc_sdio_pre_suspend(struct mmc_host *host) { - int i, err = 0; + int i; for (i = 0; i < host->card->sdio_funcs; i++) { struct sdio_func *func = host->card->sdio_func[i]; if (func && sdio_func_present(func) && func->dev.driver) { const struct dev_pm_ops *pmops = func->dev.driver->pm; - if (!pmops || !pmops->suspend || !pmops->resume) { + if (!pmops || !pmops->suspend || !pmops->resume) /* force removal of entire card in that case */ - err = -ENOSYS; - break; - } + goto remove; } } - return err; + return 0; + +remove: + if (!mmc_card_is_removable(host)) { + dev_warn(mmc_dev(host), + "missing suspend/resume ops for non-removable SDIO card\n"); + /* Don't remove a non-removable card - we can't re-detect it. */ + return 0; + } + + /* Remove the SDIO card and let it be re-detected later on. */ + mmc_sdio_remove(host); + mmc_claim_host(host); + mmc_detach_bus(host); + mmc_power_off(host); + mmc_release_host(host); + host->pm_flags = 0; + + return 0; } /* diff --git a/drivers/mmc/host/sdhci-brcmstb.c b/drivers/mmc/host/sdhci-brcmstb.c index f9780c65ebe9..f24623aac2db 100644 --- a/drivers/mmc/host/sdhci-brcmstb.c +++ b/drivers/mmc/host/sdhci-brcmstb.c @@ -199,7 +199,6 @@ static int sdhci_brcmstb_add_host(struct sdhci_host *host, if (dma64) { dev_dbg(mmc_dev(host->mmc), "Using 64 bit DMA\n"); cq_host->caps |= CQHCI_TASK_DESC_SZ_128; - cq_host->quirks |= CQHCI_QUIRK_SHORT_TXFR_DESC_SZ; } ret = cqhci_init(cq_host, host->mmc, dma64); diff --git a/drivers/mmc/host/sdhci-esdhc-imx.c b/drivers/mmc/host/sdhci-esdhc-imx.c index a20459744d21..94327988da91 100644 --- a/drivers/mmc/host/sdhci-esdhc-imx.c +++ b/drivers/mmc/host/sdhci-esdhc-imx.c @@ -1488,7 +1488,7 @@ sdhci_esdhc_imx_probe_dt(struct platform_device *pdev, mmc_of_parse_voltage(np, &host->ocr_mask); - if (esdhc_is_usdhc(imx_data)) { + if (esdhc_is_usdhc(imx_data) && !IS_ERR(imx_data->pinctrl)) { imx_data->pins_100mhz = pinctrl_lookup_state(imx_data->pinctrl, ESDHC_PINCTRL_STATE_100MHZ); imx_data->pins_200mhz = pinctrl_lookup_state(imx_data->pinctrl, diff --git a/drivers/mmc/host/sdhci-pci-core.c b/drivers/mmc/host/sdhci-pci-core.c index 9552708846ca..bf04a08eeba1 100644 --- a/drivers/mmc/host/sdhci-pci-core.c +++ b/drivers/mmc/host/sdhci-pci-core.c @@ -516,6 +516,7 @@ struct intel_host { int drv_strength; bool d3_retune; bool rpm_retune_ok; + bool needs_pwr_off; u32 glk_rx_ctrl1; u32 glk_tun_val; u32 active_ltr; @@ -643,9 +644,25 @@ static int bxt_get_cd(struct mmc_host *mmc) static void sdhci_intel_set_power(struct sdhci_host *host, unsigned char mode, unsigned short vdd) { + struct sdhci_pci_slot *slot = sdhci_priv(host); + struct intel_host *intel_host = sdhci_pci_priv(slot); int cntr; u8 reg; + /* + * Bus power may control card power, but a full reset still may not + * reset the power, whereas a direct write to SDHCI_POWER_CONTROL can. + * That might be needed to initialize correctly, if the card was left + * powered on previously. + */ + if (intel_host->needs_pwr_off) { + intel_host->needs_pwr_off = false; + if (mode != MMC_POWER_OFF) { + sdhci_writeb(host, 0, SDHCI_POWER_CONTROL); + usleep_range(10000, 12500); + } + } + sdhci_set_power(host, mode, vdd); if (mode == MMC_POWER_OFF) @@ -1135,6 +1152,14 @@ static int byt_sdio_probe_slot(struct sdhci_pci_slot *slot) return 0; } +static void byt_needs_pwr_off(struct sdhci_pci_slot *slot) +{ + struct intel_host *intel_host = sdhci_pci_priv(slot); + u8 reg = sdhci_readb(slot->host, SDHCI_POWER_CONTROL); + + intel_host->needs_pwr_off = reg & SDHCI_POWER_ON; +} + static int byt_sd_probe_slot(struct sdhci_pci_slot *slot) { byt_probe_slot(slot); @@ -1152,6 +1177,8 @@ static int byt_sd_probe_slot(struct sdhci_pci_slot *slot) slot->chip->pdev->subsystem_device == PCI_SUBDEVICE_ID_NI_78E3) slot->host->mmc->caps2 |= MMC_CAP2_AVOID_3_3V; + byt_needs_pwr_off(slot); + return 0; } @@ -1903,6 +1930,8 @@ static const struct pci_device_id pci_ids[] = { SDHCI_PCI_DEVICE(INTEL, CMLH_SD, intel_byt_sd), SDHCI_PCI_DEVICE(INTEL, JSL_EMMC, intel_glk_emmc), SDHCI_PCI_DEVICE(INTEL, JSL_SD, intel_byt_sd), + SDHCI_PCI_DEVICE(INTEL, LKF_EMMC, intel_glk_emmc), + SDHCI_PCI_DEVICE(INTEL, LKF_SD, intel_byt_sd), SDHCI_PCI_DEVICE(O2, 8120, o2), SDHCI_PCI_DEVICE(O2, 8220, o2), SDHCI_PCI_DEVICE(O2, 8221, o2), diff --git a/drivers/mmc/host/sdhci-pci.h b/drivers/mmc/host/sdhci-pci.h index d0ed232af0eb..8f90c4163bb5 100644 --- a/drivers/mmc/host/sdhci-pci.h +++ b/drivers/mmc/host/sdhci-pci.h @@ -57,6 +57,8 @@ #define PCI_DEVICE_ID_INTEL_CMLH_SD 0x06f5 #define PCI_DEVICE_ID_INTEL_JSL_EMMC 0x4dc4 #define PCI_DEVICE_ID_INTEL_JSL_SD 0x4df8 +#define PCI_DEVICE_ID_INTEL_LKF_EMMC 0x98c4 +#define PCI_DEVICE_ID_INTEL_LKF_SD 0x98f8 #define PCI_DEVICE_ID_SYSKONNECT_8000 0x8000 #define PCI_DEVICE_ID_VIA_95D0 0x95d0 diff --git a/drivers/mmc/host/sdhci-tegra.c b/drivers/mmc/host/sdhci-tegra.c index 41d193fa77bb..8ea9132ebca4 100644 --- a/drivers/mmc/host/sdhci-tegra.c +++ b/drivers/mmc/host/sdhci-tegra.c @@ -119,6 +119,10 @@ /* SDMMC CQE Base Address for Tegra Host Ver 4.1 and Higher */ #define SDHCI_TEGRA_CQE_BASE_ADDR 0xF000 +#define SDHCI_TEGRA_CQE_TRNS_MODE (SDHCI_TRNS_MULTI | \ + SDHCI_TRNS_BLK_CNT_EN | \ + SDHCI_TRNS_DMA) + struct sdhci_tegra_soc_data { const struct sdhci_pltfm_data *pdata; u64 dma_mask; @@ -1156,6 +1160,7 @@ static void tegra_sdhci_voltage_switch(struct sdhci_host *host) static void tegra_cqhci_writel(struct cqhci_host *cq_host, u32 val, int reg) { struct mmc_host *mmc = cq_host->mmc; + struct sdhci_host *host = mmc_priv(mmc); u8 ctrl; ktime_t timeout; bool timed_out; @@ -1170,6 +1175,7 @@ static void tegra_cqhci_writel(struct cqhci_host *cq_host, u32 val, int reg) */ if (reg == CQHCI_CTL && !(val & CQHCI_HALT) && cqhci_readl(cq_host, CQHCI_CTL) & CQHCI_HALT) { + sdhci_writew(host, SDHCI_TEGRA_CQE_TRNS_MODE, SDHCI_TRANSFER_MODE); sdhci_cqe_enable(mmc); writel(val, cq_host->mmio + reg); timeout = ktime_add_us(ktime_get(), 50); @@ -1205,6 +1211,7 @@ static void sdhci_tegra_update_dcmd_desc(struct mmc_host *mmc, static void sdhci_tegra_cqe_enable(struct mmc_host *mmc) { struct cqhci_host *cq_host = mmc->cqe_private; + struct sdhci_host *host = mmc_priv(mmc); u32 val; /* @@ -1218,6 +1225,7 @@ static void sdhci_tegra_cqe_enable(struct mmc_host *mmc) if (val & CQHCI_ENABLE) cqhci_writel(cq_host, (val & ~CQHCI_ENABLE), CQHCI_CFG); + sdhci_writew(host, SDHCI_TEGRA_CQE_TRNS_MODE, SDHCI_TRANSFER_MODE); sdhci_cqe_enable(mmc); if (val & CQHCI_ENABLE) cqhci_writel(cq_host, val, CQHCI_CFG); @@ -1281,12 +1289,36 @@ static void tegra_sdhci_set_timeout(struct sdhci_host *host, __sdhci_set_timeout(host, cmd); } +static void sdhci_tegra_cqe_pre_enable(struct mmc_host *mmc) +{ + struct cqhci_host *cq_host = mmc->cqe_private; + u32 reg; + + reg = cqhci_readl(cq_host, CQHCI_CFG); + reg |= CQHCI_ENABLE; + cqhci_writel(cq_host, reg, CQHCI_CFG); +} + +static void sdhci_tegra_cqe_post_disable(struct mmc_host *mmc) +{ + struct cqhci_host *cq_host = mmc->cqe_private; + struct sdhci_host *host = mmc_priv(mmc); + u32 reg; + + reg = cqhci_readl(cq_host, CQHCI_CFG); + reg &= ~CQHCI_ENABLE; + cqhci_writel(cq_host, reg, CQHCI_CFG); + sdhci_writew(host, 0x0, SDHCI_TRANSFER_MODE); +} + static const struct cqhci_host_ops sdhci_tegra_cqhci_ops = { .write_l = tegra_cqhci_writel, .enable = sdhci_tegra_cqe_enable, .disable = sdhci_cqe_disable, .dumpregs = sdhci_tegra_dumpregs, .update_dcmd_desc = sdhci_tegra_update_dcmd_desc, + .pre_enable = sdhci_tegra_cqe_pre_enable, + .post_disable = sdhci_tegra_cqe_post_disable, }; static int tegra_sdhci_set_dma_mask(struct sdhci_host *host) diff --git a/drivers/mmc/host/sdhci.c b/drivers/mmc/host/sdhci.c index 2d73407ee52e..a9e20818ff3a 100644 --- a/drivers/mmc/host/sdhci.c +++ b/drivers/mmc/host/sdhci.c @@ -2996,6 +2996,37 @@ static bool sdhci_request_done(struct sdhci_host *host) return true; } + /* + * The controller needs a reset of internal state machines + * upon error conditions. + */ + if (sdhci_needs_reset(host, mrq)) { + /* + * Do not finish until command and data lines are available for + * reset. Note there can only be one other mrq, so it cannot + * also be in mrqs_done, otherwise host->cmd and host->data_cmd + * would both be null. + */ + if (host->cmd || host->data_cmd) { + spin_unlock_irqrestore(&host->lock, flags); + return true; + } + + /* Some controllers need this kick or reset won't work here */ + if (host->quirks & SDHCI_QUIRK_CLOCK_BEFORE_RESET) + /* This is to force an update */ + host->ops->set_clock(host, host->clock); + + /* + * Spec says we should do both at the same time, but Ricoh + * controllers do not like that. + */ + sdhci_do_reset(host, SDHCI_RESET_CMD); + sdhci_do_reset(host, SDHCI_RESET_DATA); + + host->pending_reset = false; + } + /* * Always unmap the data buffers if they were mapped by * sdhci_prepare_data() whenever we finish with a request. @@ -3059,35 +3090,6 @@ static bool sdhci_request_done(struct sdhci_host *host) } } - /* - * The controller needs a reset of internal state machines - * upon error conditions. - */ - if (sdhci_needs_reset(host, mrq)) { - /* - * Do not finish until command and data lines are available for - * reset. Note there can only be one other mrq, so it cannot - * also be in mrqs_done, otherwise host->cmd and host->data_cmd - * would both be null. - */ - if (host->cmd || host->data_cmd) { - spin_unlock_irqrestore(&host->lock, flags); - return true; - } - - /* Some controllers need this kick or reset won't work here */ - if (host->quirks & SDHCI_QUIRK_CLOCK_BEFORE_RESET) - /* This is to force an update */ - host->ops->set_clock(host, host->clock); - - /* Spec says we should do both at the same time, but Ricoh - controllers do not like that. */ - sdhci_do_reset(host, SDHCI_RESET_CMD); - sdhci_do_reset(host, SDHCI_RESET_DATA); - - host->pending_reset = false; - } - host->mrqs_done[i] = NULL; spin_unlock_irqrestore(&host->lock, flags); diff --git a/drivers/mmc/host/uniphier-sd.c b/drivers/mmc/host/uniphier-sd.c index 2413b6750cec..ccbf9885a52b 100644 --- a/drivers/mmc/host/uniphier-sd.c +++ b/drivers/mmc/host/uniphier-sd.c @@ -635,7 +635,7 @@ static int uniphier_sd_probe(struct platform_device *pdev) ret = tmio_mmc_host_probe(host); if (ret) - goto free_host; + goto disable_clk; ret = devm_request_irq(dev, irq, tmio_mmc_irq, IRQF_SHARED, dev_name(dev), host); @@ -646,6 +646,8 @@ static int uniphier_sd_probe(struct platform_device *pdev) remove_host: tmio_mmc_host_remove(host); +disable_clk: + uniphier_sd_clk_disable(host); free_host: tmio_mmc_host_free(host); @@ -658,6 +660,7 @@ static int uniphier_sd_remove(struct platform_device *pdev) tmio_mmc_host_remove(host); uniphier_sd_clk_disable(host); + tmio_mmc_host_free(host); return 0; } diff --git a/drivers/mtd/maps/physmap-bt1-rom.c b/drivers/mtd/maps/physmap-bt1-rom.c index a35450002284..58782cfaf71c 100644 --- a/drivers/mtd/maps/physmap-bt1-rom.c +++ b/drivers/mtd/maps/physmap-bt1-rom.c @@ -79,7 +79,7 @@ static void __xipram bt1_rom_map_copy_from(struct map_info *map, if (shift) { chunk = min_t(ssize_t, 4 - shift, len); data = readl_relaxed(src - shift); - memcpy(to, &data + shift, chunk); + memcpy(to, (char *)&data + shift, chunk); src += chunk; to += chunk; len -= chunk; diff --git a/drivers/mtd/maps/physmap-core.c b/drivers/mtd/maps/physmap-core.c index 001ed5deb622..4f63b8430c71 100644 --- a/drivers/mtd/maps/physmap-core.c +++ b/drivers/mtd/maps/physmap-core.c @@ -69,8 +69,10 @@ static int physmap_flash_remove(struct platform_device *dev) int i, err = 0; info = platform_get_drvdata(dev); - if (!info) + if (!info) { + err = -EINVAL; goto out; + } if (info->cmtd) { err = mtd_device_unregister(info->cmtd); diff --git a/drivers/mtd/mtdchar.c b/drivers/mtd/mtdchar.c index 323035d4f2d0..688de663cabf 100644 --- a/drivers/mtd/mtdchar.c +++ b/drivers/mtd/mtdchar.c @@ -651,16 +651,12 @@ static int mtdchar_ioctl(struct file *file, u_int cmd, u_long arg) case MEMGETINFO: case MEMREADOOB: case MEMREADOOB64: - case MEMLOCK: - case MEMUNLOCK: case MEMISLOCKED: case MEMGETOOBSEL: case MEMGETBADBLOCK: - case MEMSETBADBLOCK: case OTPSELECT: case OTPGETREGIONCOUNT: case OTPGETREGIONINFO: - case OTPLOCK: case ECCGETLAYOUT: case ECCGETSTATS: case MTDFILEMODE: @@ -671,9 +667,13 @@ static int mtdchar_ioctl(struct file *file, u_int cmd, u_long arg) /* "dangerous" commands */ case MEMERASE: case MEMERASE64: + case MEMLOCK: + case MEMUNLOCK: + case MEMSETBADBLOCK: case MEMWRITEOOB: case MEMWRITEOOB64: case MEMWRITE: + case OTPLOCK: if (!(file->f_mode & FMODE_WRITE)) return -EPERM; break; diff --git a/drivers/mtd/mtdcore.c b/drivers/mtd/mtdcore.c index 2d6423d89a17..d97ddc65b5d4 100644 --- a/drivers/mtd/mtdcore.c +++ b/drivers/mtd/mtdcore.c @@ -820,6 +820,9 @@ int mtd_device_parse_register(struct mtd_info *mtd, const char * const *types, /* Prefer parsed partitions over driver-provided fallback */ ret = parse_mtd_partitions(mtd, types, parser_data); + if (ret == -EPROBE_DEFER) + goto out; + if (ret > 0) ret = 0; else if (nr_parts) diff --git a/drivers/mtd/mtdpart.c b/drivers/mtd/mtdpart.c index 12ca4f19cb14..665fd9020b76 100644 --- a/drivers/mtd/mtdpart.c +++ b/drivers/mtd/mtdpart.c @@ -331,7 +331,7 @@ static int __del_mtd_partitions(struct mtd_info *mtd) list_for_each_entry_safe(child, next, &mtd->partitions, part.node) { if (mtd_has_partitions(child)) - del_mtd_partitions(child); + __del_mtd_partitions(child); pr_info("Deleting %s MTD partition\n", child->name); ret = del_mtd_device(child); diff --git a/drivers/mtd/nand/raw/atmel/nand-controller.c b/drivers/mtd/nand/raw/atmel/nand-controller.c index e6ceec8f50dc..8aab1017b460 100644 --- a/drivers/mtd/nand/raw/atmel/nand-controller.c +++ b/drivers/mtd/nand/raw/atmel/nand-controller.c @@ -883,10 +883,12 @@ static int atmel_nand_pmecc_correct_data(struct nand_chip *chip, void *buf, NULL, 0, chip->ecc.strength); - if (ret >= 0) + if (ret >= 0) { + mtd->ecc_stats.corrected += ret; max_bitflips = max(ret, max_bitflips); - else + } else { mtd->ecc_stats.failed++; + } databuf += chip->ecc.size; eccbuf += chip->ecc.bytes; diff --git a/drivers/mtd/nand/raw/brcmnand/brcmnand.c b/drivers/mtd/nand/raw/brcmnand/brcmnand.c index 659eaa6f0980..5ff4291380c5 100644 --- a/drivers/mtd/nand/raw/brcmnand/brcmnand.c +++ b/drivers/mtd/nand/raw/brcmnand/brcmnand.c @@ -2688,6 +2688,12 @@ static int brcmnand_attach_chip(struct nand_chip *chip) ret = brcmstb_choose_ecc_layout(host); + /* If OOB is written with ECC enabled it will cause ECC errors */ + if (is_hamming_ecc(host->ctrl, &host->hwcfg)) { + chip->ecc.write_oob = brcmnand_write_oob_raw; + chip->ecc.read_oob = brcmnand_read_oob_raw; + } + return ret; } diff --git a/drivers/mtd/nand/raw/fsmc_nand.c b/drivers/mtd/nand/raw/fsmc_nand.c index 0101c0fab50a..a24e2f57fa68 100644 --- a/drivers/mtd/nand/raw/fsmc_nand.c +++ b/drivers/mtd/nand/raw/fsmc_nand.c @@ -1077,11 +1077,13 @@ static int __init fsmc_nand_probe(struct platform_device *pdev) host->read_dma_chan = dma_request_channel(mask, filter, NULL); if (!host->read_dma_chan) { dev_err(&pdev->dev, "Unable to get read dma channel\n"); + ret = -ENODEV; goto disable_clk; } host->write_dma_chan = dma_request_channel(mask, filter, NULL); if (!host->write_dma_chan) { dev_err(&pdev->dev, "Unable to get write dma channel\n"); + ret = -ENODEV; goto release_dma_read_chan; } } diff --git a/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c b/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c index 3fa8c22d3f36..4d08e4ab5c1b 100644 --- a/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c +++ b/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c @@ -2449,7 +2449,7 @@ static int gpmi_nand_init(struct gpmi_nand_data *this) this->bch_geometry.auxiliary_size = 128; ret = gpmi_alloc_dma_buffer(this); if (ret) - goto err_out; + return ret; nand_controller_init(&this->base); this->base.ops = &gpmi_nand_controller_ops; diff --git a/drivers/mtd/nand/raw/qcom_nandc.c b/drivers/mtd/nand/raw/qcom_nandc.c index fd4c318b520f..87c23bb320bf 100644 --- a/drivers/mtd/nand/raw/qcom_nandc.c +++ b/drivers/mtd/nand/raw/qcom_nandc.c @@ -2898,7 +2898,7 @@ static int qcom_probe_nand_devices(struct qcom_nand_controller *nandc) struct device *dev = nandc->dev; struct device_node *dn = dev->of_node, *child; struct qcom_nand_host *host; - int ret; + int ret = -ENODEV; for_each_available_child_of_node(dn, child) { host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL); @@ -2916,10 +2916,7 @@ static int qcom_probe_nand_devices(struct qcom_nand_controller *nandc) list_add_tail(&host->node, &nandc->host_list); } - if (list_empty(&nandc->host_list)) - return -ENODEV; - - return 0; + return ret; } /* parse custom DT properties here */ diff --git a/drivers/mtd/nand/spi/core.c b/drivers/mtd/nand/spi/core.c index 61d932c1b718..17f63f95f4a2 100644 --- a/drivers/mtd/nand/spi/core.c +++ b/drivers/mtd/nand/spi/core.c @@ -1263,12 +1263,14 @@ static const struct spi_device_id spinand_ids[] = { { .name = "spi-nand" }, { /* sentinel */ }, }; +MODULE_DEVICE_TABLE(spi, spinand_ids); #ifdef CONFIG_OF static const struct of_device_id spinand_of_ids[] = { { .compatible = "spi-nand" }, { /* sentinel */ }, }; +MODULE_DEVICE_TABLE(of, spinand_of_ids); #endif static struct spi_mem_driver spinand_drv = { diff --git a/drivers/mtd/parsers/qcomsmempart.c b/drivers/mtd/parsers/qcomsmempart.c index 808cb33d71f8..d9083308f6ba 100644 --- a/drivers/mtd/parsers/qcomsmempart.c +++ b/drivers/mtd/parsers/qcomsmempart.c @@ -65,6 +65,13 @@ static int parse_qcomsmem_part(struct mtd_info *mtd, int ret, i, numparts; char *name, *c; + if (IS_ENABLED(CONFIG_MTD_SPI_NOR_USE_4K_SECTORS) + && mtd->type == MTD_NORFLASH) { + pr_err("%s: SMEM partition parser is incompatible with 4K sectors\n", + mtd->name); + return -EINVAL; + } + pr_debug("Parsing partition table info from SMEM\n"); ptable = qcom_smem_get(SMEM_APPS, SMEM_AARM_PARTITION_TABLE, &len); if (IS_ERR(ptable)) { @@ -104,7 +111,7 @@ static int parse_qcomsmem_part(struct mtd_info *mtd, * complete partition table */ ptable = qcom_smem_get(SMEM_APPS, SMEM_AARM_PARTITION_TABLE, &len); - if (IS_ERR_OR_NULL(ptable)) { + if (IS_ERR(ptable)) { pr_err("Error reading partition table\n"); return PTR_ERR(ptable); } diff --git a/drivers/mtd/spi-nor/core.c b/drivers/mtd/spi-nor/core.c index 0522304f52fa..72bc1342c3ff 100644 --- a/drivers/mtd/spi-nor/core.c +++ b/drivers/mtd/spi-nor/core.c @@ -3301,6 +3301,37 @@ static void spi_nor_resume(struct mtd_info *mtd) dev_err(dev, "resume() failed\n"); } +static int spi_nor_get_device(struct mtd_info *mtd) +{ + struct mtd_info *master = mtd_get_master(mtd); + struct spi_nor *nor = mtd_to_spi_nor(master); + struct device *dev; + + if (nor->spimem) + dev = nor->spimem->spi->controller->dev.parent; + else + dev = nor->dev; + + if (!try_module_get(dev->driver->owner)) + return -ENODEV; + + return 0; +} + +static void spi_nor_put_device(struct mtd_info *mtd) +{ + struct mtd_info *master = mtd_get_master(mtd); + struct spi_nor *nor = mtd_to_spi_nor(master); + struct device *dev; + + if (nor->spimem) + dev = nor->spimem->spi->controller->dev.parent; + else + dev = nor->dev; + + module_put(dev->driver->owner); +} + void spi_nor_restore(struct spi_nor *nor) { /* restore the addressing mode */ @@ -3495,6 +3526,8 @@ int spi_nor_scan(struct spi_nor *nor, const char *name, mtd->_read = spi_nor_read; mtd->_suspend = spi_nor_suspend; mtd->_resume = spi_nor_resume; + mtd->_get_device = spi_nor_get_device; + mtd->_put_device = spi_nor_put_device; if (nor->params->locking_ops) { mtd->_lock = spi_nor_lock; diff --git a/drivers/mtd/spi-nor/macronix.c b/drivers/mtd/spi-nor/macronix.c index 9203abaac229..662b212787d4 100644 --- a/drivers/mtd/spi-nor/macronix.c +++ b/drivers/mtd/spi-nor/macronix.c @@ -73,9 +73,6 @@ static const struct flash_info macronix_parts[] = { SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, { "mx25l25655e", INFO(0xc22619, 0, 64 * 1024, 512, 0) }, - { "mx25l51245g", INFO(0xc2201a, 0, 64 * 1024, 1024, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | - SPI_NOR_4B_OPCODES) }, { "mx66l51235l", INFO(0xc2201a, 0, 64 * 1024, 1024, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | SPI_NOR_4B_OPCODES) }, diff --git a/drivers/net/can/dev/skb.c b/drivers/net/can/dev/skb.c index 6a64fe410987..c3508109263e 100644 --- a/drivers/net/can/dev/skb.c +++ b/drivers/net/can/dev/skb.c @@ -151,7 +151,11 @@ void can_free_echo_skb(struct net_device *dev, unsigned int idx) { struct can_priv *priv = netdev_priv(dev); - BUG_ON(idx >= priv->echo_skb_max); + if (idx >= priv->echo_skb_max) { + netdev_err(dev, "%s: BUG! Trying to access can_priv::echo_skb out of bounds (%u/max %u)\n", + __func__, idx, priv->echo_skb_max); + return; + } if (priv->echo_skb[idx]) { dev_kfree_skb_any(priv->echo_skb[idx]); diff --git a/drivers/net/can/m_can/m_can.c b/drivers/net/can/m_can/m_can.c index 0c8d36bc668c..f71127229caf 100644 --- a/drivers/net/can/m_can/m_can.c +++ b/drivers/net/can/m_can/m_can.c @@ -1455,6 +1455,8 @@ static netdev_tx_t m_can_tx_handler(struct m_can_classdev *cdev) int i; int putidx; + cdev->tx_skb = NULL; + /* Generate ID field for TX buffer Element */ /* Common to all supported M_CAN versions */ if (cf->can_id & CAN_EFF_FLAG) { @@ -1571,7 +1573,6 @@ static void m_can_tx_work_queue(struct work_struct *ws) tx_work); m_can_tx_handler(cdev); - cdev->tx_skb = NULL; } static netdev_tx_t m_can_start_xmit(struct sk_buff *skb, diff --git a/drivers/net/can/spi/mcp251x.c b/drivers/net/can/spi/mcp251x.c index a57da43680d8..bd7d0251be10 100644 --- a/drivers/net/can/spi/mcp251x.c +++ b/drivers/net/can/spi/mcp251x.c @@ -956,8 +956,6 @@ static int mcp251x_stop(struct net_device *net) priv->force_quit = 1; free_irq(spi->irq, priv); - destroy_workqueue(priv->wq); - priv->wq = NULL; mutex_lock(&priv->mcp_lock); @@ -1224,24 +1222,15 @@ static int mcp251x_open(struct net_device *net) goto out_close; } - priv->wq = alloc_workqueue("mcp251x_wq", WQ_FREEZABLE | WQ_MEM_RECLAIM, - 0); - if (!priv->wq) { - ret = -ENOMEM; - goto out_clean; - } - INIT_WORK(&priv->tx_work, mcp251x_tx_work_handler); - INIT_WORK(&priv->restart_work, mcp251x_restart_work_handler); - ret = mcp251x_hw_wake(spi); if (ret) - goto out_free_wq; + goto out_free_irq; ret = mcp251x_setup(net, spi); if (ret) - goto out_free_wq; + goto out_free_irq; ret = mcp251x_set_normal_mode(spi); if (ret) - goto out_free_wq; + goto out_free_irq; can_led_event(net, CAN_LED_EVENT_OPEN); @@ -1250,9 +1239,7 @@ static int mcp251x_open(struct net_device *net) return 0; -out_free_wq: - destroy_workqueue(priv->wq); -out_clean: +out_free_irq: free_irq(spi->irq, priv); mcp251x_hw_sleep(spi); out_close: @@ -1373,6 +1360,15 @@ static int mcp251x_can_probe(struct spi_device *spi) if (ret) goto out_clk; + priv->wq = alloc_workqueue("mcp251x_wq", WQ_FREEZABLE | WQ_MEM_RECLAIM, + 0); + if (!priv->wq) { + ret = -ENOMEM; + goto out_clk; + } + INIT_WORK(&priv->tx_work, mcp251x_tx_work_handler); + INIT_WORK(&priv->restart_work, mcp251x_restart_work_handler); + priv->spi = spi; mutex_init(&priv->mcp_lock); @@ -1417,6 +1413,8 @@ static int mcp251x_can_probe(struct spi_device *spi) return 0; error_probe: + destroy_workqueue(priv->wq); + priv->wq = NULL; mcp251x_power_enable(priv->power, 0); out_clk: @@ -1438,6 +1436,9 @@ static int mcp251x_can_remove(struct spi_device *spi) mcp251x_power_enable(priv->power, 0); + destroy_workqueue(priv->wq); + priv->wq = NULL; + clk_disable_unprepare(priv->clk); free_candev(net); diff --git a/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c b/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c index 799e9d5d3481..4a742aa5c417 100644 --- a/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c +++ b/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c @@ -2856,8 +2856,8 @@ static int mcp251xfd_probe(struct spi_device *spi) clk = devm_clk_get(&spi->dev, NULL); if (IS_ERR(clk)) - dev_err_probe(&spi->dev, PTR_ERR(clk), - "Failed to get Oscillator (clock)!\n"); + return dev_err_probe(&spi->dev, PTR_ERR(clk), + "Failed to get Oscillator (clock)!\n"); freq = clk_get_rate(clk); /* Sanity check */ @@ -2957,10 +2957,12 @@ static int mcp251xfd_probe(struct spi_device *spi) err = mcp251xfd_register(priv); if (err) - goto out_free_candev; + goto out_can_rx_offload_del; return 0; + out_can_rx_offload_del: + can_rx_offload_del(&priv->offload); out_free_candev: spi->max_speed_hz = priv->spi_max_speed_hz_orig; diff --git a/drivers/net/dsa/bcm_sf2.c b/drivers/net/dsa/bcm_sf2.c index ba5d546d06aa..9c86cacc4a72 100644 --- a/drivers/net/dsa/bcm_sf2.c +++ b/drivers/net/dsa/bcm_sf2.c @@ -32,6 +32,36 @@ #include "b53/b53_priv.h" #include "b53/b53_regs.h" +static u16 bcm_sf2_reg_rgmii_cntrl(struct bcm_sf2_priv *priv, int port) +{ + switch (priv->type) { + case BCM4908_DEVICE_ID: + switch (port) { + case 7: + return REG_RGMII_11_CNTRL; + default: + break; + } + break; + default: + switch (port) { + case 0: + return REG_RGMII_0_CNTRL; + case 1: + return REG_RGMII_1_CNTRL; + case 2: + return REG_RGMII_2_CNTRL; + default: + break; + } + } + + WARN_ONCE(1, "Unsupported port %d\n", port); + + /* RO fallback reg */ + return REG_SWITCH_STATUS; +} + /* Return the number of active ports, not counting the IMP (CPU) port */ static unsigned int bcm_sf2_num_active_ports(struct dsa_switch *ds) { @@ -647,6 +677,7 @@ static void bcm_sf2_sw_mac_config(struct dsa_switch *ds, int port, { struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds); u32 id_mode_dis = 0, port_mode; + u32 reg_rgmii_ctrl; u32 reg; if (port == core_readl(priv, CORE_IMP0_PRT_ID)) @@ -670,10 +701,12 @@ static void bcm_sf2_sw_mac_config(struct dsa_switch *ds, int port, return; } + reg_rgmii_ctrl = bcm_sf2_reg_rgmii_cntrl(priv, port); + /* Clear id_mode_dis bit, and the existing port mode, let * RGMII_MODE_EN bet set by mac_link_{up,down} */ - reg = reg_readl(priv, REG_RGMII_CNTRL_P(port)); + reg = reg_readl(priv, reg_rgmii_ctrl); reg &= ~ID_MODE_DIS; reg &= ~(PORT_MODE_MASK << PORT_MODE_SHIFT); @@ -681,13 +714,14 @@ static void bcm_sf2_sw_mac_config(struct dsa_switch *ds, int port, if (id_mode_dis) reg |= ID_MODE_DIS; - reg_writel(priv, reg, REG_RGMII_CNTRL_P(port)); + reg_writel(priv, reg, reg_rgmii_ctrl); } static void bcm_sf2_sw_mac_link_set(struct dsa_switch *ds, int port, phy_interface_t interface, bool link) { struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds); + u32 reg_rgmii_ctrl; u32 reg; if (!phy_interface_mode_is_rgmii(interface) && @@ -695,13 +729,15 @@ static void bcm_sf2_sw_mac_link_set(struct dsa_switch *ds, int port, interface != PHY_INTERFACE_MODE_REVMII) return; + reg_rgmii_ctrl = bcm_sf2_reg_rgmii_cntrl(priv, port); + /* If the link is down, just disable the interface to conserve power */ - reg = reg_readl(priv, REG_RGMII_CNTRL_P(port)); + reg = reg_readl(priv, reg_rgmii_ctrl); if (link) reg |= RGMII_MODE_EN; else reg &= ~RGMII_MODE_EN; - reg_writel(priv, reg, REG_RGMII_CNTRL_P(port)); + reg_writel(priv, reg, reg_rgmii_ctrl); } static void bcm_sf2_sw_mac_link_down(struct dsa_switch *ds, int port, @@ -735,11 +771,15 @@ static void bcm_sf2_sw_mac_link_up(struct dsa_switch *ds, int port, { struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds); struct ethtool_eee *p = &priv->dev->ports[port].eee; - u32 reg, offset; bcm_sf2_sw_mac_link_set(ds, port, interface, true); if (port != core_readl(priv, CORE_IMP0_PRT_ID)) { + u32 reg_rgmii_ctrl; + u32 reg, offset; + + reg_rgmii_ctrl = bcm_sf2_reg_rgmii_cntrl(priv, port); + if (priv->type == BCM4908_DEVICE_ID || priv->type == BCM7445_DEVICE_ID) offset = CORE_STS_OVERRIDE_GMIIP_PORT(port); @@ -750,7 +790,7 @@ static void bcm_sf2_sw_mac_link_up(struct dsa_switch *ds, int port, interface == PHY_INTERFACE_MODE_RGMII_TXID || interface == PHY_INTERFACE_MODE_MII || interface == PHY_INTERFACE_MODE_REVMII) { - reg = reg_readl(priv, REG_RGMII_CNTRL_P(port)); + reg = reg_readl(priv, reg_rgmii_ctrl); reg &= ~(RX_PAUSE_EN | TX_PAUSE_EN); if (tx_pause) @@ -758,7 +798,7 @@ static void bcm_sf2_sw_mac_link_up(struct dsa_switch *ds, int port, if (rx_pause) reg |= RX_PAUSE_EN; - reg_writel(priv, reg, REG_RGMII_CNTRL_P(port)); + reg_writel(priv, reg, reg_rgmii_ctrl); } reg = SW_OVERRIDE | LINK_STS; @@ -1144,9 +1184,7 @@ static const u16 bcm_sf2_4908_reg_offsets[] = { [REG_PHY_REVISION] = 0x14, [REG_SPHY_CNTRL] = 0x24, [REG_CROSSBAR] = 0xc8, - [REG_RGMII_0_CNTRL] = 0xe0, - [REG_RGMII_1_CNTRL] = 0xec, - [REG_RGMII_2_CNTRL] = 0xf8, + [REG_RGMII_11_CNTRL] = 0x014c, [REG_LED_0_CNTRL] = 0x40, [REG_LED_1_CNTRL] = 0x4c, [REG_LED_2_CNTRL] = 0x58, diff --git a/drivers/net/dsa/bcm_sf2_regs.h b/drivers/net/dsa/bcm_sf2_regs.h index 1d2d55c9f8aa..9e141d1a0b07 100644 --- a/drivers/net/dsa/bcm_sf2_regs.h +++ b/drivers/net/dsa/bcm_sf2_regs.h @@ -21,6 +21,7 @@ enum bcm_sf2_reg_offs { REG_RGMII_0_CNTRL, REG_RGMII_1_CNTRL, REG_RGMII_2_CNTRL, + REG_RGMII_11_CNTRL, REG_LED_0_CNTRL, REG_LED_1_CNTRL, REG_LED_2_CNTRL, @@ -48,8 +49,6 @@ enum bcm_sf2_reg_offs { #define PHY_PHYAD_SHIFT 8 #define PHY_PHYAD_MASK 0x1F -#define REG_RGMII_CNTRL_P(x) (REG_RGMII_0_CNTRL + (x)) - /* Relative to REG_RGMII_CNTRL */ #define RGMII_MODE_EN (1 << 0) #define ID_MODE_DIS (1 << 1) diff --git a/drivers/net/dsa/mv88e6xxx/devlink.c b/drivers/net/dsa/mv88e6xxx/devlink.c index 21953d6d484c..ada7a38d4d31 100644 --- a/drivers/net/dsa/mv88e6xxx/devlink.c +++ b/drivers/net/dsa/mv88e6xxx/devlink.c @@ -678,7 +678,7 @@ static int mv88e6xxx_setup_devlink_regions_global(struct dsa_switch *ds, sizeof(struct mv88e6xxx_devlink_atu_entry); break; case MV88E6XXX_REGION_VTU: - size = mv88e6xxx_max_vid(chip) * + size = (mv88e6xxx_max_vid(chip) + 1) * sizeof(struct mv88e6xxx_devlink_vtu_entry); break; } diff --git a/drivers/net/dsa/mv88e6xxx/serdes.c b/drivers/net/dsa/mv88e6xxx/serdes.c index 3195936dc5be..2ce04fef698d 100644 --- a/drivers/net/dsa/mv88e6xxx/serdes.c +++ b/drivers/net/dsa/mv88e6xxx/serdes.c @@ -443,15 +443,15 @@ int mv88e6185_serdes_power(struct mv88e6xxx_chip *chip, int port, u8 lane, u8 mv88e6185_serdes_get_lane(struct mv88e6xxx_chip *chip, int port) { /* There are no configurable serdes lanes on this switch chip but we - * need to return non-zero so that callers of + * need to return a non-negative lane number so that callers of * mv88e6xxx_serdes_get_lane() know this is a serdes port. */ switch (chip->ports[port].cmode) { case MV88E6185_PORT_STS_CMODE_SERDES: case MV88E6185_PORT_STS_CMODE_1000BASE_X: - return 0xff; - default: return 0; + default: + return -ENODEV; } } diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index b53a0d87371a..cf4249d59383 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -122,7 +122,10 @@ enum board_idx { NETXTREME_E_VF, NETXTREME_C_VF, NETXTREME_S_VF, + NETXTREME_C_VF_HV, + NETXTREME_E_VF_HV, NETXTREME_E_P5_VF, + NETXTREME_E_P5_VF_HV, }; /* indexed by enum above */ @@ -170,7 +173,10 @@ static const struct { [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" }, [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" }, [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" }, + [NETXTREME_C_VF_HV] = { "Broadcom NetXtreme-C Virtual Function for Hyper-V" }, + [NETXTREME_E_VF_HV] = { "Broadcom NetXtreme-E Virtual Function for Hyper-V" }, [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" }, + [NETXTREME_E_P5_VF_HV] = { "Broadcom BCM5750X NetXtreme-E Virtual Function for Hyper-V" }, }; static const struct pci_device_id bnxt_pci_tbl[] = { @@ -222,15 +228,25 @@ static const struct pci_device_id bnxt_pci_tbl[] = { { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 }, #ifdef CONFIG_BNXT_SRIOV { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF }, + { PCI_VDEVICE(BROADCOM, 0x1607), .driver_data = NETXTREME_E_VF_HV }, + { PCI_VDEVICE(BROADCOM, 0x1608), .driver_data = NETXTREME_E_VF_HV }, { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF }, + { PCI_VDEVICE(BROADCOM, 0x16bd), .driver_data = NETXTREME_E_VF_HV }, { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF }, + { PCI_VDEVICE(BROADCOM, 0x16c2), .driver_data = NETXTREME_C_VF_HV }, + { PCI_VDEVICE(BROADCOM, 0x16c3), .driver_data = NETXTREME_C_VF_HV }, + { PCI_VDEVICE(BROADCOM, 0x16c4), .driver_data = NETXTREME_E_VF_HV }, + { PCI_VDEVICE(BROADCOM, 0x16c5), .driver_data = NETXTREME_E_VF_HV }, { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF }, { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF }, { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF }, { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF }, { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF }, + { PCI_VDEVICE(BROADCOM, 0x16e6), .driver_data = NETXTREME_C_VF_HV }, { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF }, { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF }, + { PCI_VDEVICE(BROADCOM, 0x1808), .driver_data = NETXTREME_E_P5_VF_HV }, + { PCI_VDEVICE(BROADCOM, 0x1809), .driver_data = NETXTREME_E_P5_VF_HV }, { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF }, #endif { 0 } @@ -265,7 +281,8 @@ static struct workqueue_struct *bnxt_pf_wq; static bool bnxt_vf_pciid(enum board_idx idx) { return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF || - idx == NETXTREME_S_VF || idx == NETXTREME_E_P5_VF); + idx == NETXTREME_S_VF || idx == NETXTREME_C_VF_HV || + idx == NETXTREME_E_VF_HV || idx == NETXTREME_E_P5_VF); } #define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID) @@ -1732,14 +1749,16 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, cons = rxcmp->rx_cmp_opaque; if (unlikely(cons != rxr->rx_next_cons)) { - int rc1 = bnxt_discard_rx(bp, cpr, raw_cons, rxcmp); + int rc1 = bnxt_discard_rx(bp, cpr, &tmp_raw_cons, rxcmp); /* 0xffff is forced error, don't print it */ if (rxr->rx_next_cons != 0xffff) netdev_warn(bp->dev, "RX cons %x != expected cons %x\n", cons, rxr->rx_next_cons); bnxt_sched_reset(bp, rxr); - return rc1; + if (rc1) + return rc1; + goto next_rx_no_prod_no_len; } rx_buf = &rxr->rx_buf_ring[cons]; data = rx_buf->data; @@ -9736,7 +9755,9 @@ static ssize_t bnxt_show_temp(struct device *dev, if (!rc) len = sprintf(buf, "%u\n", resp->temp * 1000); /* display millidegree */ mutex_unlock(&bp->hwrm_cmd_lock); - return rc ?: len; + if (rc) + return rc; + return len; } static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0); diff --git a/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_regs.h b/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_regs.h index e6d4ad99cc38..3f1c189646f4 100644 --- a/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_regs.h +++ b/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_regs.h @@ -521,7 +521,7 @@ #define CN23XX_BAR1_INDEX_OFFSET 3 #define CN23XX_PEM_BAR1_INDEX_REG(port, idx) \ - (CN23XX_PEM_BAR1_INDEX_START + ((port) << CN23XX_PEM_OFFSET) + \ + (CN23XX_PEM_BAR1_INDEX_START + (((u64)port) << CN23XX_PEM_OFFSET) + \ ((idx) << CN23XX_BAR1_INDEX_OFFSET)) /*############################ DPI #########################*/ diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_queues.c b/drivers/net/ethernet/cavium/thunder/nicvf_queues.c index f782e6af45e9..50bbe79fb93d 100644 --- a/drivers/net/ethernet/cavium/thunder/nicvf_queues.c +++ b/drivers/net/ethernet/cavium/thunder/nicvf_queues.c @@ -776,7 +776,7 @@ static void nicvf_rcv_queue_config(struct nicvf *nic, struct queue_set *qs, mbx.rq.msg = NIC_MBOX_MSG_RQ_CFG; mbx.rq.qs_num = qs->vnic_id; mbx.rq.rq_num = qidx; - mbx.rq.cfg = (rq->caching << 26) | (rq->cq_qs << 19) | + mbx.rq.cfg = ((u64)rq->caching << 26) | (rq->cq_qs << 19) | (rq->cq_idx << 16) | (rq->cont_rbdr_qs << 9) | (rq->cont_qs_rbdr_idx << 8) | (rq->start_rbdr_qs << 1) | (rq->start_qs_rbdr_idx); diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.c index 83b46440408b..bde8494215c4 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.c @@ -174,31 +174,31 @@ static void set_nat_params(struct adapter *adap, struct filter_entry *f, WORD_MASK, f->fs.nat_lip[15] | f->fs.nat_lip[14] << 8 | f->fs.nat_lip[13] << 16 | - f->fs.nat_lip[12] << 24, 1); + (u64)f->fs.nat_lip[12] << 24, 1); set_tcb_field(adap, f, tid, TCB_SND_UNA_RAW_W + 1, WORD_MASK, f->fs.nat_lip[11] | f->fs.nat_lip[10] << 8 | f->fs.nat_lip[9] << 16 | - f->fs.nat_lip[8] << 24, 1); + (u64)f->fs.nat_lip[8] << 24, 1); set_tcb_field(adap, f, tid, TCB_SND_UNA_RAW_W + 2, WORD_MASK, f->fs.nat_lip[7] | f->fs.nat_lip[6] << 8 | f->fs.nat_lip[5] << 16 | - f->fs.nat_lip[4] << 24, 1); + (u64)f->fs.nat_lip[4] << 24, 1); set_tcb_field(adap, f, tid, TCB_SND_UNA_RAW_W + 3, WORD_MASK, f->fs.nat_lip[3] | f->fs.nat_lip[2] << 8 | f->fs.nat_lip[1] << 16 | - f->fs.nat_lip[0] << 24, 1); + (u64)f->fs.nat_lip[0] << 24, 1); } else { set_tcb_field(adap, f, tid, TCB_RX_FRAG3_LEN_RAW_W, WORD_MASK, f->fs.nat_lip[3] | f->fs.nat_lip[2] << 8 | f->fs.nat_lip[1] << 16 | - f->fs.nat_lip[0] << 24, 1); + (u64)f->fs.nat_lip[0] << 25, 1); } } @@ -208,25 +208,25 @@ static void set_nat_params(struct adapter *adap, struct filter_entry *f, WORD_MASK, f->fs.nat_fip[15] | f->fs.nat_fip[14] << 8 | f->fs.nat_fip[13] << 16 | - f->fs.nat_fip[12] << 24, 1); + (u64)f->fs.nat_fip[12] << 24, 1); set_tcb_field(adap, f, tid, TCB_RX_FRAG2_PTR_RAW_W + 1, WORD_MASK, f->fs.nat_fip[11] | f->fs.nat_fip[10] << 8 | f->fs.nat_fip[9] << 16 | - f->fs.nat_fip[8] << 24, 1); + (u64)f->fs.nat_fip[8] << 24, 1); set_tcb_field(adap, f, tid, TCB_RX_FRAG2_PTR_RAW_W + 2, WORD_MASK, f->fs.nat_fip[7] | f->fs.nat_fip[6] << 8 | f->fs.nat_fip[5] << 16 | - f->fs.nat_fip[4] << 24, 1); + (u64)f->fs.nat_fip[4] << 24, 1); set_tcb_field(adap, f, tid, TCB_RX_FRAG2_PTR_RAW_W + 3, WORD_MASK, f->fs.nat_fip[3] | f->fs.nat_fip[2] << 8 | f->fs.nat_fip[1] << 16 | - f->fs.nat_fip[0] << 24, 1); + (u64)f->fs.nat_fip[0] << 24, 1); } else { set_tcb_field(adap, f, tid, @@ -234,13 +234,13 @@ static void set_nat_params(struct adapter *adap, struct filter_entry *f, WORD_MASK, f->fs.nat_fip[3] | f->fs.nat_fip[2] << 8 | f->fs.nat_fip[1] << 16 | - f->fs.nat_fip[0] << 24, 1); + (u64)f->fs.nat_fip[0] << 24, 1); } } set_tcb_field(adap, f, tid, TCB_PDU_HDR_LEN_W, WORD_MASK, (dp ? (nat_lp[1] | nat_lp[0] << 8) : 0) | - (sp ? (nat_fp[1] << 16 | nat_fp[0] << 24) : 0), + (sp ? (nat_fp[1] << 16 | (u64)nat_fp[0] << 24) : 0), 1); } diff --git a/drivers/net/ethernet/chelsio/cxgb4/sge.c b/drivers/net/ethernet/chelsio/cxgb4/sge.c index 256fae15e032..1e5f2edb70cf 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/sge.c +++ b/drivers/net/ethernet/chelsio/cxgb4/sge.c @@ -2563,12 +2563,12 @@ int cxgb4_ethofld_send_flowc(struct net_device *dev, u32 eotid, u32 tc) spin_lock_bh(&eosw_txq->lock); if (tc != FW_SCHED_CLS_NONE) { if (eosw_txq->state != CXGB4_EO_STATE_CLOSED) - goto out_unlock; + goto out_free_skb; next_state = CXGB4_EO_STATE_FLOWC_OPEN_SEND; } else { if (eosw_txq->state != CXGB4_EO_STATE_ACTIVE) - goto out_unlock; + goto out_free_skb; next_state = CXGB4_EO_STATE_FLOWC_CLOSE_SEND; } @@ -2604,17 +2604,19 @@ int cxgb4_ethofld_send_flowc(struct net_device *dev, u32 eotid, u32 tc) eosw_txq_flush_pending_skbs(eosw_txq); ret = eosw_txq_enqueue(eosw_txq, skb); - if (ret) { - dev_consume_skb_any(skb); - goto out_unlock; - } + if (ret) + goto out_free_skb; eosw_txq->state = next_state; eosw_txq->flowc_idx = eosw_txq->pidx; eosw_txq_advance(eosw_txq, 1); ethofld_xmit(dev, eosw_txq); -out_unlock: + spin_unlock_bh(&eosw_txq->lock); + return 0; + +out_free_skb: + dev_consume_skb_any(skb); spin_unlock_bh(&eosw_txq->lock); return ret; } diff --git a/drivers/net/ethernet/cisco/enic/enic_main.c b/drivers/net/ethernet/cisco/enic/enic_main.c index f04ec53544ae..b1443ff439de 100644 --- a/drivers/net/ethernet/cisco/enic/enic_main.c +++ b/drivers/net/ethernet/cisco/enic/enic_main.c @@ -768,7 +768,7 @@ static inline int enic_queue_wq_skb_encap(struct enic *enic, struct vnic_wq *wq, return err; } -static inline void enic_queue_wq_skb(struct enic *enic, +static inline int enic_queue_wq_skb(struct enic *enic, struct vnic_wq *wq, struct sk_buff *skb) { unsigned int mss = skb_shinfo(skb)->gso_size; @@ -814,6 +814,7 @@ static inline void enic_queue_wq_skb(struct enic *enic, wq->to_use = buf->next; dev_kfree_skb(skb); } + return err; } /* netif_tx_lock held, process context with BHs disabled, or BH */ @@ -857,7 +858,8 @@ static netdev_tx_t enic_hard_start_xmit(struct sk_buff *skb, return NETDEV_TX_BUSY; } - enic_queue_wq_skb(enic, wq, skb); + if (enic_queue_wq_skb(enic, wq, skb)) + goto error; if (vnic_wq_desc_avail(wq) < MAX_SKB_FRAGS + ENIC_DESC_MAX_SPLITS) netif_tx_stop_queue(txq); @@ -865,6 +867,7 @@ static netdev_tx_t enic_hard_start_xmit(struct sk_buff *skb, if (!netdev_xmit_more() || netif_xmit_stopped(txq)) vnic_wq_doorbell(wq); +error: spin_unlock(&enic->wq_lock[txq_map]); return NETDEV_TX_OK; diff --git a/drivers/net/ethernet/freescale/Makefile b/drivers/net/ethernet/freescale/Makefile index 67c436400352..de7b31842233 100644 --- a/drivers/net/ethernet/freescale/Makefile +++ b/drivers/net/ethernet/freescale/Makefile @@ -24,6 +24,4 @@ obj-$(CONFIG_FSL_DPAA_ETH) += dpaa/ obj-$(CONFIG_FSL_DPAA2_ETH) += dpaa2/ -obj-$(CONFIG_FSL_ENETC) += enetc/ -obj-$(CONFIG_FSL_ENETC_MDIO) += enetc/ -obj-$(CONFIG_FSL_ENETC_VF) += enetc/ +obj-y += enetc/ diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c index 3db882322b2b..70aea9c274fe 100644 --- a/drivers/net/ethernet/freescale/fec_main.c +++ b/drivers/net/ethernet/freescale/fec_main.c @@ -2048,6 +2048,8 @@ static int fec_enet_mii_probe(struct net_device *ndev) fep->link = 0; fep->full_duplex = 0; + phy_dev->mac_managed_pm = 1; + phy_attached_info(phy_dev); return 0; @@ -3864,6 +3866,7 @@ static int __maybe_unused fec_resume(struct device *dev) netif_device_attach(ndev); netif_tx_unlock_bh(ndev); napi_enable(&fep->napi); + phy_init_hw(ndev->phydev); phy_start(ndev->phydev); } rtnl_unlock(); diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c index bf4302a5cf95..0f70158c2551 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c @@ -576,8 +576,8 @@ static int hns3_nic_net_stop(struct net_device *netdev) if (h->ae_algo->ops->set_timer_task) h->ae_algo->ops->set_timer_task(priv->ae_handle, false); - netif_tx_stop_all_queues(netdev); netif_carrier_off(netdev); + netif_tx_disable(netdev); hns3_nic_net_down(netdev); @@ -823,7 +823,7 @@ static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto, * and it is udp packet, which has a dest port as the IANA assigned. * the hardware is expected to do the checksum offload, but the * hardware will not do the checksum offload when udp dest port is - * 4789 or 6081. + * 4789, 4790 or 6081. */ static bool hns3_tunnel_csum_bug(struct sk_buff *skb) { @@ -841,7 +841,8 @@ static bool hns3_tunnel_csum_bug(struct sk_buff *skb) if (!(!skb->encapsulation && (l4.udp->dest == htons(IANA_VXLAN_UDP_PORT) || - l4.udp->dest == htons(GENEVE_UDP_PORT)))) + l4.udp->dest == htons(GENEVE_UDP_PORT) || + l4.udp->dest == htons(4790)))) return false; skb_checksum_help(skb); @@ -1277,23 +1278,21 @@ static unsigned int hns3_skb_bd_num(struct sk_buff *skb, unsigned int *bd_size, } static unsigned int hns3_tx_bd_num(struct sk_buff *skb, unsigned int *bd_size, - u8 max_non_tso_bd_num) + u8 max_non_tso_bd_num, unsigned int bd_num, + unsigned int recursion_level) { +#define HNS3_MAX_RECURSION_LEVEL 24 + struct sk_buff *frag_skb; - unsigned int bd_num = 0; /* If the total len is within the max bd limit */ - if (likely(skb->len <= HNS3_MAX_BD_SIZE && !skb_has_frag_list(skb) && + if (likely(skb->len <= HNS3_MAX_BD_SIZE && !recursion_level && + !skb_has_frag_list(skb) && skb_shinfo(skb)->nr_frags < max_non_tso_bd_num)) return skb_shinfo(skb)->nr_frags + 1U; - /* The below case will always be linearized, return - * HNS3_MAX_BD_NUM_TSO + 1U to make sure it is linearized. - */ - if (unlikely(skb->len > HNS3_MAX_TSO_SIZE || - (!skb_is_gso(skb) && skb->len > - HNS3_MAX_NON_TSO_SIZE(max_non_tso_bd_num)))) - return HNS3_MAX_TSO_BD_NUM + 1U; + if (unlikely(recursion_level >= HNS3_MAX_RECURSION_LEVEL)) + return UINT_MAX; bd_num = hns3_skb_bd_num(skb, bd_size, bd_num); @@ -1301,7 +1300,8 @@ static unsigned int hns3_tx_bd_num(struct sk_buff *skb, unsigned int *bd_size, return bd_num; skb_walk_frags(skb, frag_skb) { - bd_num = hns3_skb_bd_num(frag_skb, bd_size, bd_num); + bd_num = hns3_tx_bd_num(frag_skb, bd_size, max_non_tso_bd_num, + bd_num, recursion_level + 1); if (bd_num > HNS3_MAX_TSO_BD_NUM) return bd_num; } @@ -1361,6 +1361,43 @@ void hns3_shinfo_pack(struct skb_shared_info *shinfo, __u32 *size) size[i] = skb_frag_size(&shinfo->frags[i]); } +static int hns3_skb_linearize(struct hns3_enet_ring *ring, + struct sk_buff *skb, + u8 max_non_tso_bd_num, + unsigned int bd_num) +{ + /* 'bd_num == UINT_MAX' means the skb' fraglist has a + * recursion level of over HNS3_MAX_RECURSION_LEVEL. + */ + if (bd_num == UINT_MAX) { + u64_stats_update_begin(&ring->syncp); + ring->stats.over_max_recursion++; + u64_stats_update_end(&ring->syncp); + return -ENOMEM; + } + + /* The skb->len has exceeded the hw limitation, linearization + * will not help. + */ + if (skb->len > HNS3_MAX_TSO_SIZE || + (!skb_is_gso(skb) && skb->len > + HNS3_MAX_NON_TSO_SIZE(max_non_tso_bd_num))) { + u64_stats_update_begin(&ring->syncp); + ring->stats.hw_limitation++; + u64_stats_update_end(&ring->syncp); + return -ENOMEM; + } + + if (__skb_linearize(skb)) { + u64_stats_update_begin(&ring->syncp); + ring->stats.sw_err_cnt++; + u64_stats_update_end(&ring->syncp); + return -ENOMEM; + } + + return 0; +} + static int hns3_nic_maybe_stop_tx(struct hns3_enet_ring *ring, struct net_device *netdev, struct sk_buff *skb) @@ -1370,7 +1407,7 @@ static int hns3_nic_maybe_stop_tx(struct hns3_enet_ring *ring, unsigned int bd_size[HNS3_MAX_TSO_BD_NUM + 1U]; unsigned int bd_num; - bd_num = hns3_tx_bd_num(skb, bd_size, max_non_tso_bd_num); + bd_num = hns3_tx_bd_num(skb, bd_size, max_non_tso_bd_num, 0, 0); if (unlikely(bd_num > max_non_tso_bd_num)) { if (bd_num <= HNS3_MAX_TSO_BD_NUM && skb_is_gso(skb) && !hns3_skb_need_linearized(skb, bd_size, bd_num, @@ -1379,16 +1416,11 @@ static int hns3_nic_maybe_stop_tx(struct hns3_enet_ring *ring, goto out; } - if (__skb_linearize(skb)) + if (hns3_skb_linearize(ring, skb, max_non_tso_bd_num, + bd_num)) return -ENOMEM; bd_num = hns3_tx_bd_count(skb->len); - if ((skb_is_gso(skb) && bd_num > HNS3_MAX_TSO_BD_NUM) || - (!skb_is_gso(skb) && - bd_num > max_non_tso_bd_num)) { - trace_hns3_over_max_bd(skb); - return -ENOMEM; - } u64_stats_update_begin(&ring->syncp); ring->stats.tx_copy++; @@ -1412,6 +1444,10 @@ static int hns3_nic_maybe_stop_tx(struct hns3_enet_ring *ring, return bd_num; } + u64_stats_update_begin(&ring->syncp); + ring->stats.tx_busy++; + u64_stats_update_end(&ring->syncp); + return -EBUSY; } @@ -1459,6 +1495,7 @@ static int hns3_fill_skb_to_desc(struct hns3_enet_ring *ring, struct sk_buff *skb, enum hns_desc_type type) { unsigned int size = skb_headlen(skb); + struct sk_buff *frag_skb; int i, ret, bd_num = 0; if (size) { @@ -1483,6 +1520,15 @@ static int hns3_fill_skb_to_desc(struct hns3_enet_ring *ring, bd_num += ret; } + skb_walk_frags(skb, frag_skb) { + ret = hns3_fill_skb_to_desc(ring, frag_skb, + DESC_TYPE_FRAGLIST_SKB); + if (unlikely(ret < 0)) + return ret; + + bd_num += ret; + } + return bd_num; } @@ -1513,8 +1559,6 @@ netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev) struct hns3_enet_ring *ring = &priv->ring[skb->queue_mapping]; struct netdev_queue *dev_queue; int pre_ntu, next_to_use_head; - struct sk_buff *frag_skb; - int bd_num = 0; bool doorbell; int ret; @@ -1530,15 +1574,8 @@ netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev) ret = hns3_nic_maybe_stop_tx(ring, netdev, skb); if (unlikely(ret <= 0)) { if (ret == -EBUSY) { - u64_stats_update_begin(&ring->syncp); - ring->stats.tx_busy++; - u64_stats_update_end(&ring->syncp); hns3_tx_doorbell(ring, 0, true); return NETDEV_TX_BUSY; - } else if (ret == -ENOMEM) { - u64_stats_update_begin(&ring->syncp); - ring->stats.sw_err_cnt++; - u64_stats_update_end(&ring->syncp); } hns3_rl_err(netdev, "xmit error: %d!\n", ret); @@ -1551,21 +1588,14 @@ netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev) if (unlikely(ret < 0)) goto fill_err; + /* 'ret < 0' means filling error, 'ret == 0' means skb->len is + * zero, which is unlikely, and 'ret > 0' means how many tx desc + * need to be notified to the hw. + */ ret = hns3_fill_skb_to_desc(ring, skb, DESC_TYPE_SKB); - if (unlikely(ret < 0)) + if (unlikely(ret <= 0)) goto fill_err; - bd_num += ret; - - skb_walk_frags(skb, frag_skb) { - ret = hns3_fill_skb_to_desc(ring, frag_skb, - DESC_TYPE_FRAGLIST_SKB); - if (unlikely(ret < 0)) - goto fill_err; - - bd_num += ret; - } - pre_ntu = ring->next_to_use ? (ring->next_to_use - 1) : (ring->desc_num - 1); ring->desc[pre_ntu].tx.bdtp_fe_sc_vld_ra_ri |= @@ -1576,7 +1606,7 @@ netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev) dev_queue = netdev_get_tx_queue(netdev, ring->queue_index); doorbell = __netdev_tx_sent_queue(dev_queue, skb->len, netdev_xmit_more()); - hns3_tx_doorbell(ring, bd_num, doorbell); + hns3_tx_doorbell(ring, ret, doorbell); return NETDEV_TX_OK; @@ -1748,11 +1778,15 @@ static void hns3_nic_get_stats64(struct net_device *netdev, tx_drop += ring->stats.tx_l4_proto_err; tx_drop += ring->stats.tx_l2l3l4_err; tx_drop += ring->stats.tx_tso_err; + tx_drop += ring->stats.over_max_recursion; + tx_drop += ring->stats.hw_limitation; tx_errors += ring->stats.sw_err_cnt; tx_errors += ring->stats.tx_vlan_err; tx_errors += ring->stats.tx_l4_proto_err; tx_errors += ring->stats.tx_l2l3l4_err; tx_errors += ring->stats.tx_tso_err; + tx_errors += ring->stats.over_max_recursion; + tx_errors += ring->stats.hw_limitation; } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); /* fetch the rx stats */ @@ -3704,7 +3738,6 @@ static void hns3_nic_set_cpumask(struct hns3_nic_priv *priv) static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv) { - struct hnae3_ring_chain_node vector_ring_chain; struct hnae3_handle *h = priv->ae_handle; struct hns3_enet_tqp_vector *tqp_vector; int ret; @@ -3736,6 +3769,8 @@ static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv) } for (i = 0; i < priv->vector_num; i++) { + struct hnae3_ring_chain_node vector_ring_chain; + tqp_vector = &priv->tqp_vector[i]; tqp_vector->rx_group.total_bytes = 0; @@ -4554,6 +4589,11 @@ static int hns3_reset_notify_up_enet(struct hnae3_handle *handle) struct hns3_nic_priv *priv = netdev_priv(kinfo->netdev); int ret = 0; + if (!test_bit(HNS3_NIC_STATE_INITED, &priv->state)) { + netdev_err(kinfo->netdev, "device is not initialized yet\n"); + return -EFAULT; + } + clear_bit(HNS3_NIC_STATE_RESETTING, &priv->state); if (netif_running(kinfo->netdev)) { diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h index d069b04ee587..e44224e23315 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h @@ -376,6 +376,8 @@ struct ring_stats { u64 tx_l4_proto_err; u64 tx_l2l3l4_err; u64 tx_tso_err; + u64 over_max_recursion; + u64 hw_limitation; }; struct { u64 rx_pkts; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c index adcec4ea7cb9..d20f2e246017 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c @@ -44,6 +44,8 @@ static const struct hns3_stats hns3_txq_stats[] = { HNS3_TQP_STAT("l4_proto_err", tx_l4_proto_err), HNS3_TQP_STAT("l2l3l4_err", tx_l2l3l4_err), HNS3_TQP_STAT("tso_err", tx_tso_err), + HNS3_TQP_STAT("over_max_recursion", over_max_recursion), + HNS3_TQP_STAT("hw_limitation", hw_limitation), }; #define HNS3_TXQ_STATS_COUNT ARRAY_SIZE(hns3_txq_stats) diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_err.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_err.c index 0ca7f1b984bf..78d3eb142df8 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_err.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_err.c @@ -753,8 +753,9 @@ static int hclge_config_igu_egu_hw_err_int(struct hclge_dev *hdev, bool en) /* configure IGU,EGU error interrupts */ hclge_cmd_setup_basic_desc(&desc, HCLGE_IGU_COMMON_INT_EN, false); + desc.data[0] = cpu_to_le32(HCLGE_IGU_ERR_INT_TYPE); if (en) - desc.data[0] = cpu_to_le32(HCLGE_IGU_ERR_INT_EN); + desc.data[0] |= cpu_to_le32(HCLGE_IGU_ERR_INT_EN); desc.data[1] = cpu_to_le32(HCLGE_IGU_ERR_INT_EN_MASK); diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_err.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_err.h index 608fe26fc3fe..d647f3c84134 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_err.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_err.h @@ -32,7 +32,8 @@ #define HCLGE_TQP_ECC_ERR_INT_EN_MASK 0x0FFF #define HCLGE_MSIX_SRAM_ECC_ERR_INT_EN_MASK 0x0F000000 #define HCLGE_MSIX_SRAM_ECC_ERR_INT_EN 0x0F000000 -#define HCLGE_IGU_ERR_INT_EN 0x0000066F +#define HCLGE_IGU_ERR_INT_EN 0x0000000F +#define HCLGE_IGU_ERR_INT_TYPE 0x00000660 #define HCLGE_IGU_ERR_INT_EN_MASK 0x000F #define HCLGE_IGU_TNL_ERR_INT_EN 0x0002AABF #define HCLGE_IGU_TNL_ERR_INT_EN_MASK 0x003F diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c index b0dbe6dcaa7b..7a560d0e19b9 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c @@ -11379,7 +11379,6 @@ static int hclge_get_64_bit_regs(struct hclge_dev *hdev, u32 regs_num, #define REG_LEN_PER_LINE (REG_NUM_PER_LINE * sizeof(u32)) #define REG_SEPARATOR_LINE 1 #define REG_NUM_REMAIN_MASK 3 -#define BD_LIST_MAX_NUM 30 int hclge_query_bd_num_cmd_send(struct hclge_dev *hdev, struct hclge_desc *desc) { @@ -11473,15 +11472,19 @@ static int hclge_get_dfx_reg_len(struct hclge_dev *hdev, int *len) { u32 dfx_reg_type_num = ARRAY_SIZE(hclge_dfx_bd_offset_list); int data_len_per_desc, bd_num, i; - int bd_num_list[BD_LIST_MAX_NUM]; + int *bd_num_list; u32 data_len; int ret; + bd_num_list = kcalloc(dfx_reg_type_num, sizeof(int), GFP_KERNEL); + if (!bd_num_list) + return -ENOMEM; + ret = hclge_get_dfx_reg_bd_num(hdev, bd_num_list, dfx_reg_type_num); if (ret) { dev_err(&hdev->pdev->dev, "Get dfx reg bd num fail, status is %d.\n", ret); - return ret; + goto out; } data_len_per_desc = sizeof_field(struct hclge_desc, data); @@ -11492,6 +11495,8 @@ static int hclge_get_dfx_reg_len(struct hclge_dev *hdev, int *len) *len += (data_len / REG_LEN_PER_LINE + 1) * REG_LEN_PER_LINE; } +out: + kfree(bd_num_list); return ret; } @@ -11499,16 +11504,20 @@ static int hclge_get_dfx_reg(struct hclge_dev *hdev, void *data) { u32 dfx_reg_type_num = ARRAY_SIZE(hclge_dfx_bd_offset_list); int bd_num, bd_num_max, buf_len, i; - int bd_num_list[BD_LIST_MAX_NUM]; struct hclge_desc *desc_src; + int *bd_num_list; u32 *reg = data; int ret; + bd_num_list = kcalloc(dfx_reg_type_num, sizeof(int), GFP_KERNEL); + if (!bd_num_list) + return -ENOMEM; + ret = hclge_get_dfx_reg_bd_num(hdev, bd_num_list, dfx_reg_type_num); if (ret) { dev_err(&hdev->pdev->dev, "Get dfx reg bd num fail, status is %d.\n", ret); - return ret; + goto out; } bd_num_max = bd_num_list[0]; @@ -11517,8 +11526,10 @@ static int hclge_get_dfx_reg(struct hclge_dev *hdev, void *data) buf_len = sizeof(*desc_src) * bd_num_max; desc_src = kzalloc(buf_len, GFP_KERNEL); - if (!desc_src) - return -ENOMEM; + if (!desc_src) { + ret = -ENOMEM; + goto out; + } for (i = 0; i < dfx_reg_type_num; i++) { bd_num = bd_num_list[i]; @@ -11534,6 +11545,8 @@ static int hclge_get_dfx_reg(struct hclge_dev *hdev, void *data) } kfree(desc_src); +out: + kfree(bd_num_list); return ret; } diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c index 51a36e74f088..c3bb16b1f060 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c @@ -535,7 +535,7 @@ static void hclge_get_link_mode(struct hclge_vport *vport, unsigned long advertising; unsigned long supported; unsigned long send_data; - u8 msg_data[10]; + u8 msg_data[10] = {}; u8 dest_vfid; advertising = hdev->hw.mac.advertising[0]; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c index e89820702540..c194bba187d6 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c @@ -255,6 +255,8 @@ void hclge_mac_start_phy(struct hclge_dev *hdev) if (!phydev) return; + phy_loopback(phydev, false); + phy_start(phydev); } diff --git a/drivers/net/ethernet/intel/i40e/i40e.h b/drivers/net/ethernet/intel/i40e/i40e.h index 15f93b355099..5069f690cf0b 100644 --- a/drivers/net/ethernet/intel/i40e/i40e.h +++ b/drivers/net/ethernet/intel/i40e/i40e.h @@ -1142,7 +1142,6 @@ static inline bool i40e_is_sw_dcb(struct i40e_pf *pf) return !!(pf->flags & I40E_FLAG_DISABLE_FW_LLDP); } -void i40e_set_lldp_forwarding(struct i40e_pf *pf, bool enable); #ifdef CONFIG_I40E_DCB void i40e_dcbnl_flush_apps(struct i40e_pf *pf, struct i40e_dcbx_config *old_cfg, diff --git a/drivers/net/ethernet/intel/i40e/i40e_adminq_cmd.h b/drivers/net/ethernet/intel/i40e/i40e_adminq_cmd.h index ce626eace692..140b677f114d 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_adminq_cmd.h +++ b/drivers/net/ethernet/intel/i40e/i40e_adminq_cmd.h @@ -1566,8 +1566,10 @@ enum i40e_aq_phy_type { I40E_PHY_TYPE_25GBASE_LR = 0x22, I40E_PHY_TYPE_25GBASE_AOC = 0x23, I40E_PHY_TYPE_25GBASE_ACC = 0x24, - I40E_PHY_TYPE_2_5GBASE_T = 0x30, - I40E_PHY_TYPE_5GBASE_T = 0x31, + I40E_PHY_TYPE_2_5GBASE_T = 0x26, + I40E_PHY_TYPE_5GBASE_T = 0x27, + I40E_PHY_TYPE_2_5GBASE_T_LINK_STATUS = 0x30, + I40E_PHY_TYPE_5GBASE_T_LINK_STATUS = 0x31, I40E_PHY_TYPE_MAX, I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP = 0xFD, I40E_PHY_TYPE_EMPTY = 0xFE, diff --git a/drivers/net/ethernet/intel/i40e/i40e_client.c b/drivers/net/ethernet/intel/i40e/i40e_client.c index a2dba32383f6..32f3facbed1a 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_client.c +++ b/drivers/net/ethernet/intel/i40e/i40e_client.c @@ -375,6 +375,7 @@ void i40e_client_subtask(struct i40e_pf *pf) clear_bit(__I40E_CLIENT_INSTANCE_OPENED, &cdev->state); i40e_client_del_instance(pf); + return; } } } diff --git a/drivers/net/ethernet/intel/i40e/i40e_common.c b/drivers/net/ethernet/intel/i40e/i40e_common.c index ec19e18305ec..ce35e064cf60 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_common.c +++ b/drivers/net/ethernet/intel/i40e/i40e_common.c @@ -1154,8 +1154,8 @@ static enum i40e_media_type i40e_get_media_type(struct i40e_hw *hw) break; case I40E_PHY_TYPE_100BASE_TX: case I40E_PHY_TYPE_1000BASE_T: - case I40E_PHY_TYPE_2_5GBASE_T: - case I40E_PHY_TYPE_5GBASE_T: + case I40E_PHY_TYPE_2_5GBASE_T_LINK_STATUS: + case I40E_PHY_TYPE_5GBASE_T_LINK_STATUS: case I40E_PHY_TYPE_10GBASE_T: media = I40E_MEDIA_TYPE_BASET; break; diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c index 0e92668012e3..93dd58fda272 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c @@ -841,8 +841,8 @@ static void i40e_get_settings_link_up(struct i40e_hw *hw, 10000baseT_Full); break; case I40E_PHY_TYPE_10GBASE_T: - case I40E_PHY_TYPE_5GBASE_T: - case I40E_PHY_TYPE_2_5GBASE_T: + case I40E_PHY_TYPE_5GBASE_T_LINK_STATUS: + case I40E_PHY_TYPE_2_5GBASE_T_LINK_STATUS: case I40E_PHY_TYPE_1000BASE_T: case I40E_PHY_TYPE_100BASE_TX: ethtool_link_ksettings_add_link_mode(ks, supported, Autoneg); @@ -1409,7 +1409,8 @@ static int i40e_set_fec_cfg(struct net_device *netdev, u8 fec_cfg) memset(&config, 0, sizeof(config)); config.phy_type = abilities.phy_type; - config.abilities = abilities.abilities; + config.abilities = abilities.abilities | + I40E_AQ_PHY_ENABLE_ATOMIC_LINK; config.phy_type_ext = abilities.phy_type_ext; config.link_speed = abilities.link_speed; config.eee_capability = abilities.eee_capability; @@ -5287,7 +5288,6 @@ static int i40e_set_priv_flags(struct net_device *dev, u32 flags) i40e_aq_cfg_lldp_mib_change_event(&pf->hw, false, NULL); i40e_aq_stop_lldp(&pf->hw, true, false, NULL); } else { - i40e_set_lldp_forwarding(pf, false); status = i40e_aq_start_lldp(&pf->hw, false, NULL); if (status) { adq_err = pf->hw.aq.asq_last_status; diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c index 527023ee4c07..ac4b44fc19f1 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_main.c +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c @@ -6878,40 +6878,6 @@ static int i40e_init_pf_dcb(struct i40e_pf *pf) } #endif /* CONFIG_I40E_DCB */ -/** - * i40e_set_lldp_forwarding - set forwarding of lldp frames - * @pf: PF being configured - * @enable: if forwarding to OS shall be enabled - * - * Toggle forwarding of lldp frames behavior, - * When passing DCB control from firmware to software - * lldp frames must be forwarded to the software based - * lldp agent. - */ -void i40e_set_lldp_forwarding(struct i40e_pf *pf, bool enable) -{ - if (pf->lan_vsi == I40E_NO_VSI) - return; - - if (!pf->vsi[pf->lan_vsi]) - return; - - /* No need to check the outcome, commands may fail - * if desired value is already set - */ - i40e_aq_add_rem_control_packet_filter(&pf->hw, NULL, ETH_P_LLDP, - I40E_AQC_ADD_CONTROL_PACKET_FLAGS_TX | - I40E_AQC_ADD_CONTROL_PACKET_FLAGS_IGNORE_MAC, - pf->vsi[pf->lan_vsi]->seid, 0, - enable, NULL, NULL); - - i40e_aq_add_rem_control_packet_filter(&pf->hw, NULL, ETH_P_LLDP, - I40E_AQC_ADD_CONTROL_PACKET_FLAGS_RX | - I40E_AQC_ADD_CONTROL_PACKET_FLAGS_IGNORE_MAC, - pf->vsi[pf->lan_vsi]->seid, 0, - enable, NULL, NULL); -} - /** * i40e_print_link_message - print link up or down * @vsi: the VSI for which link needs a message @@ -10735,10 +10701,6 @@ static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired) */ i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, pf->main_vsi_seid); -#ifdef CONFIG_I40E_DCB - if (pf->flags & I40E_FLAG_DISABLE_FW_LLDP) - i40e_set_lldp_forwarding(pf, true); -#endif /* CONFIG_I40E_DCB */ /* restart the VSIs that were rebuilt and running before the reset */ i40e_pf_unquiesce_all_vsi(pf); @@ -15753,10 +15715,6 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) */ i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, pf->main_vsi_seid); -#ifdef CONFIG_I40E_DCB - if (pf->flags & I40E_FLAG_DISABLE_FW_LLDP) - i40e_set_lldp_forwarding(pf, true); -#endif /* CONFIG_I40E_DCB */ if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) || (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4)) diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c index 06b4271219b1..70b515049540 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c +++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c @@ -1961,10 +1961,6 @@ static bool i40e_cleanup_headers(struct i40e_ring *rx_ring, struct sk_buff *skb, union i40e_rx_desc *rx_desc) { - /* XDP packets use error pointer so abort at this point */ - if (IS_ERR(skb)) - return true; - /* ERR_MASK will only have valid bits if EOP set, and * what we are doing here is actually checking * I40E_RX_DESC_ERROR_RXE_SHIFT, since it is the zeroth bit in @@ -2534,7 +2530,7 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget) } /* exit if we failed to retrieve a buffer */ - if (!skb) { + if (!xdp_res && !skb) { rx_ring->rx_stats.alloc_buff_failed++; rx_buffer->pagecnt_bias++; break; @@ -2547,7 +2543,7 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget) if (i40e_is_non_eop(rx_ring, rx_desc)) continue; - if (i40e_cleanup_headers(rx_ring, skb, rx_desc)) { + if (xdp_res || i40e_cleanup_headers(rx_ring, skb, rx_desc)) { skb = NULL; continue; } diff --git a/drivers/net/ethernet/intel/i40e/i40e_type.h b/drivers/net/ethernet/intel/i40e/i40e_type.h index 5c10faaca790..c81109a63e90 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_type.h +++ b/drivers/net/ethernet/intel/i40e/i40e_type.h @@ -239,11 +239,8 @@ struct i40e_phy_info { #define I40E_CAP_PHY_TYPE_25GBASE_ACC BIT_ULL(I40E_PHY_TYPE_25GBASE_ACC + \ I40E_PHY_TYPE_OFFSET) /* Offset for 2.5G/5G PHY Types value to bit number conversion */ -#define I40E_PHY_TYPE_OFFSET2 (-10) -#define I40E_CAP_PHY_TYPE_2_5GBASE_T BIT_ULL(I40E_PHY_TYPE_2_5GBASE_T + \ - I40E_PHY_TYPE_OFFSET2) -#define I40E_CAP_PHY_TYPE_5GBASE_T BIT_ULL(I40E_PHY_TYPE_5GBASE_T + \ - I40E_PHY_TYPE_OFFSET2) +#define I40E_CAP_PHY_TYPE_2_5GBASE_T BIT_ULL(I40E_PHY_TYPE_2_5GBASE_T) +#define I40E_CAP_PHY_TYPE_5GBASE_T BIT_ULL(I40E_PHY_TYPE_5GBASE_T) #define I40E_HW_CAP_MAX_GPIO 30 /* Capabilities of a PF or a VF or the whole device */ struct i40e_hw_capabilities { diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c b/drivers/net/ethernet/intel/iavf/iavf_main.c index dc5b3c06d1e0..ebd08543791b 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_main.c +++ b/drivers/net/ethernet/intel/iavf/iavf_main.c @@ -3899,8 +3899,6 @@ static void iavf_remove(struct pci_dev *pdev) iounmap(hw->hw_addr); pci_release_regions(pdev); - iavf_free_all_tx_resources(adapter); - iavf_free_all_rx_resources(adapter); iavf_free_queues(adapter); kfree(adapter->vf_res); spin_lock_bh(&adapter->mac_vlan_list_lock); diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c index d13c7fc8fb0a..195d122c9cb2 100644 --- a/drivers/net/ethernet/intel/ice/ice_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_lib.c @@ -2818,38 +2818,46 @@ int ice_vsi_release(struct ice_vsi *vsi) } /** - * ice_vsi_rebuild_update_coalesce - set coalesce for a q_vector + * ice_vsi_rebuild_update_coalesce_intrl - set interrupt rate limit for a q_vector * @q_vector: pointer to q_vector which is being updated - * @coalesce: pointer to array of struct with stored coalesce + * @stored_intrl_setting: original INTRL setting * * Set coalesce param in q_vector and update these parameters in HW. */ static void -ice_vsi_rebuild_update_coalesce(struct ice_q_vector *q_vector, - struct ice_coalesce_stored *coalesce) +ice_vsi_rebuild_update_coalesce_intrl(struct ice_q_vector *q_vector, + u16 stored_intrl_setting) { - struct ice_ring_container *rx_rc = &q_vector->rx; - struct ice_ring_container *tx_rc = &q_vector->tx; struct ice_hw *hw = &q_vector->vsi->back->hw; - tx_rc->itr_setting = coalesce->itr_tx; - rx_rc->itr_setting = coalesce->itr_rx; - - /* dynamic ITR values will be updated during Tx/Rx */ - if (!ITR_IS_DYNAMIC(tx_rc->itr_setting)) - wr32(hw, GLINT_ITR(tx_rc->itr_idx, q_vector->reg_idx), - ITR_REG_ALIGN(tx_rc->itr_setting) >> - ICE_ITR_GRAN_S); - if (!ITR_IS_DYNAMIC(rx_rc->itr_setting)) - wr32(hw, GLINT_ITR(rx_rc->itr_idx, q_vector->reg_idx), - ITR_REG_ALIGN(rx_rc->itr_setting) >> - ICE_ITR_GRAN_S); - - q_vector->intrl = coalesce->intrl; + q_vector->intrl = stored_intrl_setting; wr32(hw, GLINT_RATE(q_vector->reg_idx), ice_intrl_usec_to_reg(q_vector->intrl, hw->intrl_gran)); } +/** + * ice_vsi_rebuild_update_coalesce_itr - set coalesce for a q_vector + * @q_vector: pointer to q_vector which is being updated + * @rc: pointer to ring container + * @stored_itr_setting: original ITR setting + * + * Set coalesce param in q_vector and update these parameters in HW. + */ +static void +ice_vsi_rebuild_update_coalesce_itr(struct ice_q_vector *q_vector, + struct ice_ring_container *rc, + u16 stored_itr_setting) +{ + struct ice_hw *hw = &q_vector->vsi->back->hw; + + rc->itr_setting = stored_itr_setting; + + /* dynamic ITR values will be updated during Tx/Rx */ + if (!ITR_IS_DYNAMIC(rc->itr_setting)) + wr32(hw, GLINT_ITR(rc->itr_idx, q_vector->reg_idx), + ITR_REG_ALIGN(rc->itr_setting) >> ICE_ITR_GRAN_S); +} + /** * ice_vsi_rebuild_get_coalesce - get coalesce from all q_vectors * @vsi: VSI connected with q_vectors @@ -2869,6 +2877,11 @@ ice_vsi_rebuild_get_coalesce(struct ice_vsi *vsi, coalesce[i].itr_tx = q_vector->tx.itr_setting; coalesce[i].itr_rx = q_vector->rx.itr_setting; coalesce[i].intrl = q_vector->intrl; + + if (i < vsi->num_txq) + coalesce[i].tx_valid = true; + if (i < vsi->num_rxq) + coalesce[i].rx_valid = true; } return vsi->num_q_vectors; @@ -2893,17 +2906,59 @@ ice_vsi_rebuild_set_coalesce(struct ice_vsi *vsi, if ((size && !coalesce) || !vsi) return; - for (i = 0; i < size && i < vsi->num_q_vectors; i++) - ice_vsi_rebuild_update_coalesce(vsi->q_vectors[i], - &coalesce[i]); - - /* number of q_vectors increased, so assume coalesce settings were - * changed globally (i.e. ethtool -C eth0 instead of per-queue) and use - * the previous settings from q_vector 0 for all of the new q_vectors + /* There are a couple of cases that have to be handled here: + * 1. The case where the number of queue vectors stays the same, but + * the number of Tx or Rx rings changes (the first for loop) + * 2. The case where the number of queue vectors increased (the + * second for loop) */ - for (; i < vsi->num_q_vectors; i++) - ice_vsi_rebuild_update_coalesce(vsi->q_vectors[i], - &coalesce[0]); + for (i = 0; i < size && i < vsi->num_q_vectors; i++) { + /* There are 2 cases to handle here and they are the same for + * both Tx and Rx: + * if the entry was valid previously (coalesce[i].[tr]x_valid + * and the loop variable is less than the number of rings + * allocated, then write the previous values + * + * if the entry was not valid previously, but the number of + * rings is less than are allocated (this means the number of + * rings increased from previously), then write out the + * values in the first element + */ + if (i < vsi->alloc_rxq && coalesce[i].rx_valid) + ice_vsi_rebuild_update_coalesce_itr(vsi->q_vectors[i], + &vsi->q_vectors[i]->rx, + coalesce[i].itr_rx); + else if (i < vsi->alloc_rxq) + ice_vsi_rebuild_update_coalesce_itr(vsi->q_vectors[i], + &vsi->q_vectors[i]->rx, + coalesce[0].itr_rx); + + if (i < vsi->alloc_txq && coalesce[i].tx_valid) + ice_vsi_rebuild_update_coalesce_itr(vsi->q_vectors[i], + &vsi->q_vectors[i]->tx, + coalesce[i].itr_tx); + else if (i < vsi->alloc_txq) + ice_vsi_rebuild_update_coalesce_itr(vsi->q_vectors[i], + &vsi->q_vectors[i]->tx, + coalesce[0].itr_tx); + + ice_vsi_rebuild_update_coalesce_intrl(vsi->q_vectors[i], + coalesce[i].intrl); + } + + /* the number of queue vectors increased so write whatever is in + * the first element + */ + for (; i < vsi->num_q_vectors; i++) { + ice_vsi_rebuild_update_coalesce_itr(vsi->q_vectors[i], + &vsi->q_vectors[i]->tx, + coalesce[0].itr_tx); + ice_vsi_rebuild_update_coalesce_itr(vsi->q_vectors[i], + &vsi->q_vectors[i]->rx, + coalesce[0].itr_rx); + ice_vsi_rebuild_update_coalesce_intrl(vsi->q_vectors[i], + coalesce[0].intrl); + } } /** @@ -2932,9 +2987,11 @@ int ice_vsi_rebuild(struct ice_vsi *vsi, bool init_vsi) coalesce = kcalloc(vsi->num_q_vectors, sizeof(struct ice_coalesce_stored), GFP_KERNEL); - if (coalesce) - prev_num_q_vectors = ice_vsi_rebuild_get_coalesce(vsi, - coalesce); + if (!coalesce) + return -ENOMEM; + + prev_num_q_vectors = ice_vsi_rebuild_get_coalesce(vsi, coalesce); + ice_rm_vsi_lan_cfg(vsi->port_info, vsi->idx); ice_vsi_free_q_vectors(vsi); diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.h b/drivers/net/ethernet/intel/ice/ice_txrx.h index 5dab77504fa5..672a7ff0ee36 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.h +++ b/drivers/net/ethernet/intel/ice/ice_txrx.h @@ -351,6 +351,8 @@ struct ice_coalesce_stored { u16 itr_tx; u16 itr_rx; u8 intrl; + u8 tx_valid; + u8 rx_valid; }; /* iterator for handling rings in ring container */ diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c index a45cd2b416c8..b1085ada7723 100644 --- a/drivers/net/ethernet/intel/igb/igb_main.c +++ b/drivers/net/ethernet/intel/igb/igb_main.c @@ -7977,12 +7977,16 @@ static void igb_ring_irq_enable(struct igb_q_vector *q_vector) **/ static int igb_poll(struct napi_struct *napi, int budget) { - struct igb_q_vector *q_vector = container_of(napi, - struct igb_q_vector, - napi); - bool clean_complete = true; + struct igb_q_vector *q_vector; + bool clean_complete; int work_done = 0; + /* if budget is zero, we have a special case for netconsole, so + * make sure to set clean_complete to false in that case. + */ + clean_complete = !!budget; + + q_vector = container_of(napi, struct igb_q_vector, napi); #ifdef CONFIG_IGB_DCA if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED) igb_update_dca(q_vector); diff --git a/drivers/net/ethernet/marvell/prestera/prestera_main.c b/drivers/net/ethernet/marvell/prestera/prestera_main.c index 25dd903a3e92..d849b0f65de2 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_main.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_main.c @@ -431,7 +431,8 @@ static void prestera_port_handle_event(struct prestera_switch *sw, netif_carrier_on(port->dev); if (!delayed_work_pending(caching_dw)) queue_delayed_work(prestera_wq, caching_dw, 0); - } else { + } else if (netif_running(port->dev) && + netif_carrier_ok(port->dev)) { netif_carrier_off(port->dev); if (delayed_work_pending(caching_dw)) cancel_delayed_work(caching_dw); diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c index 01d3ee4b5829..bcd5e7ae8482 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c @@ -1319,7 +1319,7 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget, skb->protocol = eth_type_trans(skb, netdev); if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX && - RX_DMA_VID(trxd.rxd3)) + (trxd.rxd2 & RX_DMA_VTAG)) __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), RX_DMA_VID(trxd.rxd3)); skb_record_rx_queue(skb, 0); diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.h b/drivers/net/ethernet/mediatek/mtk_eth_soc.h index fd3cec8f06ba..c47272100615 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h @@ -296,6 +296,7 @@ #define RX_DMA_LSO BIT(30) #define RX_DMA_PLEN0(_x) (((_x) & 0x3fff) << 16) #define RX_DMA_GET_PLEN0(_x) (((_x) >> 16) & 0x3fff) +#define RX_DMA_VTAG BIT(15) /* QDMA descriptor rxd3 */ #define RX_DMA_VID(_x) ((_x) & 0xfff) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c index bdbffe484fce..d2efe2455955 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c @@ -576,7 +576,7 @@ static void mlx5e_tx_mpwqe_session_start(struct mlx5e_txqsq *sq, pi = mlx5e_txqsq_get_next_pi(sq, MLX5E_TX_MPW_MAX_WQEBBS); wqe = MLX5E_TX_FETCH_WQE(sq, pi); - prefetchw(wqe->data); + net_prefetchw(wqe->data); *session = (struct mlx5e_tx_mpwqe) { .wqe = wqe, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.c index 22bee4990232..bb61f52d782d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.c @@ -850,7 +850,7 @@ mlx5_fpga_ipsec_release_sa_ctx(struct mlx5_fpga_ipsec_sa_ctx *sa_ctx) return; } - if (sa_ctx->fpga_xfrm->accel_xfrm.attrs.action & + if (sa_ctx->fpga_xfrm->accel_xfrm.attrs.action == MLX5_ACCEL_ESP_ACTION_DECRYPT) ida_simple_remove(&fipsec->halloc, sa_ctx->sa_handle); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.c index 9143ec326ebf..f146c618a78e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.c @@ -1532,6 +1532,7 @@ static void dr_ste_v1_build_src_gvmi_qpn_bit_mask(struct mlx5dr_match_param *val DR_STE_SET_ONES(src_gvmi_qp_v1, bit_mask, source_gvmi, misc_mask, source_port); DR_STE_SET_ONES(src_gvmi_qp_v1, bit_mask, source_qp, misc_mask, source_sqn); + misc_mask->source_eswitch_owner_vhca_id = 0; } static int dr_ste_v1_build_src_gvmi_qpn_tag(struct mlx5dr_match_param *value, diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_mr.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_mr.c index 7846a21555ef..1f6bc0c7e91d 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_mr.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_mr.c @@ -535,6 +535,16 @@ mlxsw_sp_mr_route_evif_resolve(struct mlxsw_sp_mr_table *mr_table, u16 erif_index = 0; int err; + /* Add the eRIF */ + if (mlxsw_sp_mr_vif_valid(rve->mr_vif)) { + erif_index = mlxsw_sp_rif_index(rve->mr_vif->rif); + err = mr->mr_ops->route_erif_add(mlxsw_sp, + rve->mr_route->route_priv, + erif_index); + if (err) + return err; + } + /* Update the route action, as the new eVIF can be a tunnel or a pimreg * device which will require updating the action. */ @@ -544,17 +554,7 @@ mlxsw_sp_mr_route_evif_resolve(struct mlxsw_sp_mr_table *mr_table, rve->mr_route->route_priv, route_action); if (err) - return err; - } - - /* Add the eRIF */ - if (mlxsw_sp_mr_vif_valid(rve->mr_vif)) { - erif_index = mlxsw_sp_rif_index(rve->mr_vif->rif); - err = mr->mr_ops->route_erif_add(mlxsw_sp, - rve->mr_route->route_priv, - erif_index); - if (err) - goto err_route_erif_add; + goto err_route_action_update; } /* Update the minimum MTU */ @@ -572,14 +572,14 @@ mlxsw_sp_mr_route_evif_resolve(struct mlxsw_sp_mr_table *mr_table, return 0; err_route_min_mtu_update: - if (mlxsw_sp_mr_vif_valid(rve->mr_vif)) - mr->mr_ops->route_erif_del(mlxsw_sp, rve->mr_route->route_priv, - erif_index); -err_route_erif_add: if (route_action != rve->mr_route->route_action) mr->mr_ops->route_action_update(mlxsw_sp, rve->mr_route->route_priv, rve->mr_route->route_action); +err_route_action_update: + if (mlxsw_sp_mr_vif_valid(rve->mr_vif)) + mr->mr_ops->route_erif_del(mlxsw_sp, rve->mr_route->route_priv, + erif_index); return err; } diff --git a/drivers/net/ethernet/netronome/nfp/nfp_devlink.c b/drivers/net/ethernet/netronome/nfp/nfp_devlink.c index 713ee3041d49..bea978df7713 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_devlink.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_devlink.c @@ -364,6 +364,7 @@ int nfp_devlink_port_register(struct nfp_app *app, struct nfp_port *port) attrs.split = eth_port.is_split; attrs.splittable = !attrs.split; + attrs.lanes = eth_port.port_lanes; attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL; attrs.phys.port_number = eth_port.label_port; attrs.phys.split_subport_number = eth_port.label_subport; diff --git a/drivers/net/ethernet/qualcomm/emac/emac-mac.c b/drivers/net/ethernet/qualcomm/emac/emac-mac.c index 117188e3c7de..87b8c032195d 100644 --- a/drivers/net/ethernet/qualcomm/emac/emac-mac.c +++ b/drivers/net/ethernet/qualcomm/emac/emac-mac.c @@ -1437,6 +1437,7 @@ netdev_tx_t emac_mac_tx_buf_send(struct emac_adapter *adpt, { struct emac_tpd tpd; u32 prod_idx; + int len; memset(&tpd, 0, sizeof(tpd)); @@ -1456,9 +1457,10 @@ netdev_tx_t emac_mac_tx_buf_send(struct emac_adapter *adpt, if (skb_network_offset(skb) != ETH_HLEN) TPD_TYP_SET(&tpd, 1); + len = skb->len; emac_tx_fill_tpd(adpt, tx_q, skb, &tpd); - netdev_sent_queue(adpt->netdev, skb->len); + netdev_sent_queue(adpt->netdev, len); /* Make sure the are enough free descriptors to hold one * maximum-sized SKB. We need one desc for each fragment, diff --git a/drivers/net/ethernet/renesas/ravb_main.c b/drivers/net/ethernet/renesas/ravb_main.c index eb0c03bdb12d..cad57d58d764 100644 --- a/drivers/net/ethernet/renesas/ravb_main.c +++ b/drivers/net/ethernet/renesas/ravb_main.c @@ -911,31 +911,20 @@ static int ravb_poll(struct napi_struct *napi, int budget) int q = napi - priv->napi; int mask = BIT(q); int quota = budget; - u32 ris0, tis; - for (;;) { - tis = ravb_read(ndev, TIS); - ris0 = ravb_read(ndev, RIS0); - if (!((ris0 & mask) || (tis & mask))) - break; + /* Processing RX Descriptor Ring */ + /* Clear RX interrupt */ + ravb_write(ndev, ~(mask | RIS0_RESERVED), RIS0); + if (ravb_rx(ndev, "a, q)) + goto out; - /* Processing RX Descriptor Ring */ - if (ris0 & mask) { - /* Clear RX interrupt */ - ravb_write(ndev, ~(mask | RIS0_RESERVED), RIS0); - if (ravb_rx(ndev, "a, q)) - goto out; - } - /* Processing TX Descriptor Ring */ - if (tis & mask) { - spin_lock_irqsave(&priv->lock, flags); - /* Clear TX interrupt */ - ravb_write(ndev, ~(mask | TIS_RESERVED), TIS); - ravb_tx_free(ndev, q, true); - netif_wake_subqueue(ndev, q); - spin_unlock_irqrestore(&priv->lock, flags); - } - } + /* Processing RX Descriptor Ring */ + spin_lock_irqsave(&priv->lock, flags); + /* Clear TX interrupt */ + ravb_write(ndev, ~(mask | TIS_RESERVED), TIS); + ravb_tx_free(ndev, q, true); + netif_wake_subqueue(ndev, q); + spin_unlock_irqrestore(&priv->lock, flags); napi_complete(napi); diff --git a/drivers/net/ethernet/sfc/ef10.c b/drivers/net/ethernet/sfc/ef10.c index da6886dcac37..4fa72b573c17 100644 --- a/drivers/net/ethernet/sfc/ef10.c +++ b/drivers/net/ethernet/sfc/ef10.c @@ -2928,8 +2928,7 @@ efx_ef10_handle_tx_event(struct efx_channel *channel, efx_qword_t *event) /* Get the transmit queue */ tx_ev_q_label = EFX_QWORD_FIELD(*event, ESF_DZ_TX_QLABEL); - tx_queue = efx_channel_get_tx_queue(channel, - tx_ev_q_label % EFX_MAX_TXQ_PER_CHANNEL); + tx_queue = channel->tx_queue + (tx_ev_q_label % EFX_MAX_TXQ_PER_CHANNEL); if (!tx_queue->timestamping) { /* Transmit completion */ diff --git a/drivers/net/ethernet/sfc/efx_channels.c b/drivers/net/ethernet/sfc/efx_channels.c index 1bfeee283ea9..a3ca406a3561 100644 --- a/drivers/net/ethernet/sfc/efx_channels.c +++ b/drivers/net/ethernet/sfc/efx_channels.c @@ -914,6 +914,8 @@ int efx_set_channels(struct efx_nic *efx) } } } + if (xdp_queue_number) + efx->xdp_tx_queue_count = xdp_queue_number; rc = netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels); if (rc) diff --git a/drivers/net/ethernet/sfc/farch.c b/drivers/net/ethernet/sfc/farch.c index d75cf5ff5686..49df02ecee91 100644 --- a/drivers/net/ethernet/sfc/farch.c +++ b/drivers/net/ethernet/sfc/farch.c @@ -835,14 +835,14 @@ efx_farch_handle_tx_event(struct efx_channel *channel, efx_qword_t *event) /* Transmit completion */ tx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_DESC_PTR); tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL); - tx_queue = efx_channel_get_tx_queue( - channel, tx_ev_q_label % EFX_MAX_TXQ_PER_CHANNEL); + tx_queue = channel->tx_queue + + (tx_ev_q_label % EFX_MAX_TXQ_PER_CHANNEL); efx_xmit_done(tx_queue, tx_ev_desc_ptr); } else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_WQ_FF_FULL)) { /* Rewrite the FIFO write pointer */ tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL); - tx_queue = efx_channel_get_tx_queue( - channel, tx_ev_q_label % EFX_MAX_TXQ_PER_CHANNEL); + tx_queue = channel->tx_queue + + (tx_ev_q_label % EFX_MAX_TXQ_PER_CHANNEL); netif_tx_lock(efx->net_dev); efx_farch_notify_tx_desc(tx_queue); @@ -1081,16 +1081,16 @@ static void efx_farch_handle_tx_flush_done(struct efx_nic *efx, efx_qword_t *event) { struct efx_tx_queue *tx_queue; + struct efx_channel *channel; int qid; qid = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBDATA); if (qid < EFX_MAX_TXQ_PER_CHANNEL * (efx->n_tx_channels + efx->n_extra_tx_channels)) { - tx_queue = efx_get_tx_queue(efx, qid / EFX_MAX_TXQ_PER_CHANNEL, - qid % EFX_MAX_TXQ_PER_CHANNEL); - if (atomic_cmpxchg(&tx_queue->flush_outstanding, 1, 0)) { + channel = efx_get_tx_channel(efx, qid / EFX_MAX_TXQ_PER_CHANNEL); + tx_queue = channel->tx_queue + (qid % EFX_MAX_TXQ_PER_CHANNEL); + if (atomic_cmpxchg(&tx_queue->flush_outstanding, 1, 0)) efx_farch_magic_event(tx_queue->channel, EFX_CHANNEL_MAGIC_TX_DRAIN(tx_queue)); - } } } diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-ipq806x.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-ipq806x.c index bf3250e0e59c..749585fe6fc9 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-ipq806x.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-ipq806x.c @@ -352,6 +352,8 @@ static int ipq806x_gmac_probe(struct platform_device *pdev) plat_dat->bsp_priv = gmac; plat_dat->fix_mac_speed = ipq806x_gmac_fix_mac_speed; plat_dat->multicast_filter_bins = 0; + plat_dat->tx_fifo_size = 8192; + plat_dat->rx_fifo_size = 8192; err = stmmac_dvr_probe(&pdev->dev, plat_dat, &stmmac_res); if (err) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c index 29f765a246a0..aaf37598cbd3 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c @@ -638,6 +638,7 @@ static void dwmac4_set_filter(struct mac_device_info *hw, value &= ~GMAC_PACKET_FILTER_PCF; value &= ~GMAC_PACKET_FILTER_PM; value &= ~GMAC_PACKET_FILTER_PR; + value &= ~GMAC_PACKET_FILTER_RA; if (dev->flags & IFF_PROMISC) { /* VLAN Tag Filter Fail Packets Queuing */ if (hw->vlan_fail_q_en) { diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c index 62aa0e95beb7..a7249e4071f1 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c @@ -222,7 +222,7 @@ static void dwmac4_dma_rx_chan_op_mode(void __iomem *ioaddr, int mode, u32 channel, int fifosz, u8 qmode) { unsigned int rqs = fifosz / 256 - 1; - u32 mtl_rx_op, mtl_rx_int; + u32 mtl_rx_op; mtl_rx_op = readl(ioaddr + MTL_CHAN_RX_OP_MODE(channel)); @@ -283,11 +283,6 @@ static void dwmac4_dma_rx_chan_op_mode(void __iomem *ioaddr, int mode, } writel(mtl_rx_op, ioaddr + MTL_CHAN_RX_OP_MODE(channel)); - - /* Enable MTL RX overflow */ - mtl_rx_int = readl(ioaddr + MTL_CHAN_INT_CTRL(channel)); - writel(mtl_rx_int | MTL_RX_OVERFLOW_INT_EN, - ioaddr + MTL_CHAN_INT_CTRL(channel)); } static void dwmac4_dma_tx_chan_op_mode(void __iomem *ioaddr, int mode, diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 4749bd0af160..369d7cde3993 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -2757,8 +2757,15 @@ static int stmmac_hw_setup(struct net_device *dev, bool init_ptp) /* Enable TSO */ if (priv->tso) { - for (chan = 0; chan < tx_cnt; chan++) + for (chan = 0; chan < tx_cnt; chan++) { + struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; + + /* TSO and TBS cannot co-exist */ + if (tx_q->tbs & STMMAC_TBS_AVAIL) + continue; + stmmac_enable_tso(priv, priv->ioaddr, 1, chan); + } } /* Enable Split Header */ @@ -2850,9 +2857,8 @@ static int stmmac_open(struct net_device *dev) struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en; + /* Setup per-TXQ tbs flag before TX descriptor alloc */ tx_q->tbs |= tbs_en ? STMMAC_TBS_AVAIL : 0; - if (stmmac_enable_tbs(priv, priv->ioaddr, tbs_en, chan)) - tx_q->tbs &= ~STMMAC_TBS_AVAIL; } ret = alloc_dma_desc_resources(priv); @@ -4162,7 +4168,6 @@ static irqreturn_t stmmac_interrupt(int irq, void *dev_id) /* To handle GMAC own interrupts */ if ((priv->plat->has_gmac) || xmac) { int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats); - int mtl_status; if (unlikely(status)) { /* For LPI we need to save the tx status */ @@ -4173,17 +4178,8 @@ static irqreturn_t stmmac_interrupt(int irq, void *dev_id) } for (queue = 0; queue < queues_count; queue++) { - struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; - - mtl_status = stmmac_host_mtl_irq_status(priv, priv->hw, - queue); - if (mtl_status != -EINVAL) - status |= mtl_status; - - if (status & CORE_IRQ_MTL_RX_OVERFLOW) - stmmac_set_rx_tail_ptr(priv, priv->ioaddr, - rx_q->rx_tail_addr, - queue); + status = stmmac_host_mtl_irq_status(priv, priv->hw, + queue); } /* PCS link status */ diff --git a/drivers/net/ethernet/ti/davinci_emac.c b/drivers/net/ethernet/ti/davinci_emac.c index c7031e1960d4..03055c96f076 100644 --- a/drivers/net/ethernet/ti/davinci_emac.c +++ b/drivers/net/ethernet/ti/davinci_emac.c @@ -169,11 +169,11 @@ static const char emac_version_string[] = "TI DaVinci EMAC Linux v6.1"; /* EMAC mac_status register */ #define EMAC_MACSTATUS_TXERRCODE_MASK (0xF00000) #define EMAC_MACSTATUS_TXERRCODE_SHIFT (20) -#define EMAC_MACSTATUS_TXERRCH_MASK (0x7) +#define EMAC_MACSTATUS_TXERRCH_MASK (0x70000) #define EMAC_MACSTATUS_TXERRCH_SHIFT (16) #define EMAC_MACSTATUS_RXERRCODE_MASK (0xF000) #define EMAC_MACSTATUS_RXERRCODE_SHIFT (12) -#define EMAC_MACSTATUS_RXERRCH_MASK (0x7) +#define EMAC_MACSTATUS_RXERRCH_MASK (0x700) #define EMAC_MACSTATUS_RXERRCH_SHIFT (8) /* EMAC RX register masks */ diff --git a/drivers/net/ethernet/xilinx/Kconfig b/drivers/net/ethernet/xilinx/Kconfig index c6eb7f2368aa..911b5ef9e680 100644 --- a/drivers/net/ethernet/xilinx/Kconfig +++ b/drivers/net/ethernet/xilinx/Kconfig @@ -18,12 +18,14 @@ if NET_VENDOR_XILINX config XILINX_EMACLITE tristate "Xilinx 10/100 Ethernet Lite support" + depends on HAS_IOMEM select PHYLIB help This driver supports the 10/100 Ethernet Lite from Xilinx. config XILINX_AXI_EMAC tristate "Xilinx 10/100/1000 AXI Ethernet support" + depends on HAS_IOMEM select PHYLINK help This driver supports the 10/100/1000 Ethernet from Xilinx for the @@ -31,6 +33,7 @@ config XILINX_AXI_EMAC config XILINX_LL_TEMAC tristate "Xilinx LL TEMAC (LocalLink Tri-mode Ethernet MAC) driver" + depends on HAS_IOMEM select PHYLIB help This driver supports the Xilinx 10/100/1000 LocalLink TEMAC diff --git a/drivers/net/ethernet/xscale/ixp4xx_eth.c b/drivers/net/ethernet/xscale/ixp4xx_eth.c index 0152f1e70783..9defaa21a1a9 100644 --- a/drivers/net/ethernet/xscale/ixp4xx_eth.c +++ b/drivers/net/ethernet/xscale/ixp4xx_eth.c @@ -1085,7 +1085,7 @@ static int init_queues(struct port *port) int i; if (!ports_open) { - dma_pool = dma_pool_create(DRV_NAME, port->netdev->dev.parent, + dma_pool = dma_pool_create(DRV_NAME, &port->netdev->dev, POOL_ALLOC_SIZE, 32, 0); if (!dma_pool) return -ENOMEM; @@ -1435,6 +1435,9 @@ static int ixp4xx_eth_probe(struct platform_device *pdev) ndev->netdev_ops = &ixp4xx_netdev_ops; ndev->ethtool_ops = &ixp4xx_ethtool_ops; ndev->tx_queue_len = 100; + /* Inherit the DMA masks from the platform device */ + ndev->dev.dma_mask = dev->dma_mask; + ndev->dev.coherent_dma_mask = dev->coherent_dma_mask; netif_napi_add(ndev, &port->napi, eth_poll, NAPI_WEIGHT); diff --git a/drivers/net/fddi/Kconfig b/drivers/net/fddi/Kconfig index f722079dfb6a..f99c1048c97e 100644 --- a/drivers/net/fddi/Kconfig +++ b/drivers/net/fddi/Kconfig @@ -40,17 +40,20 @@ config DEFXX config DEFXX_MMIO bool - prompt "Use MMIO instead of PIO" if PCI || EISA + prompt "Use MMIO instead of IOP" if PCI || EISA depends on DEFXX - default n if PCI || EISA + default n if EISA default y help This instructs the driver to use EISA or PCI memory-mapped I/O - (MMIO) as appropriate instead of programmed I/O ports (PIO). + (MMIO) as appropriate instead of programmed I/O ports (IOP). Enabling this gives an improvement in processing time in parts - of the driver, but it may cause problems with EISA (DEFEA) - adapters. TURBOchannel does not have the concept of I/O ports, - so MMIO is always used for these (DEFTA) adapters. + of the driver, but it requires a memory window to be configured + for EISA (DEFEA) adapters that may not always be available. + Conversely some PCIe host bridges do not support IOP, so MMIO + may be required to access PCI (DEFPA) adapters on downstream PCI + buses with some systems. TURBOchannel does not have the concept + of I/O ports, so MMIO is always used for these (DEFTA) adapters. If unsure, say N. diff --git a/drivers/net/fddi/defxx.c b/drivers/net/fddi/defxx.c index 077c68498f04..c7ce6d5491af 100644 --- a/drivers/net/fddi/defxx.c +++ b/drivers/net/fddi/defxx.c @@ -495,6 +495,25 @@ static const struct net_device_ops dfx_netdev_ops = { .ndo_set_mac_address = dfx_ctl_set_mac_address, }; +static void dfx_register_res_alloc_err(const char *print_name, bool mmio, + bool eisa) +{ + pr_err("%s: Cannot use %s, no address set, aborting\n", + print_name, mmio ? "MMIO" : "I/O"); + pr_err("%s: Recompile driver with \"CONFIG_DEFXX_MMIO=%c\"\n", + print_name, mmio ? 'n' : 'y'); + if (eisa && mmio) + pr_err("%s: Or run ECU and set adapter's MMIO location\n", + print_name); +} + +static void dfx_register_res_err(const char *print_name, bool mmio, + unsigned long start, unsigned long len) +{ + pr_err("%s: Cannot reserve %s resource 0x%lx @ 0x%lx, aborting\n", + print_name, mmio ? "MMIO" : "I/O", len, start); +} + /* * ================ * = dfx_register = @@ -568,15 +587,12 @@ static int dfx_register(struct device *bdev) dev_set_drvdata(bdev, dev); dfx_get_bars(bdev, bar_start, bar_len); - if (dfx_bus_eisa && dfx_use_mmio && bar_start[0] == 0) { - pr_err("%s: Cannot use MMIO, no address set, aborting\n", - print_name); - pr_err("%s: Run ECU and set adapter's MMIO location\n", - print_name); - pr_err("%s: Or recompile driver with \"CONFIG_DEFXX_MMIO=n\"" - "\n", print_name); + if (bar_len[0] == 0 || + (dfx_bus_eisa && dfx_use_mmio && bar_start[0] == 0)) { + dfx_register_res_alloc_err(print_name, dfx_use_mmio, + dfx_bus_eisa); err = -ENXIO; - goto err_out; + goto err_out_disable; } if (dfx_use_mmio) @@ -585,18 +601,16 @@ static int dfx_register(struct device *bdev) else region = request_region(bar_start[0], bar_len[0], print_name); if (!region) { - pr_err("%s: Cannot reserve %s resource 0x%lx @ 0x%lx, " - "aborting\n", dfx_use_mmio ? "MMIO" : "I/O", print_name, - (long)bar_len[0], (long)bar_start[0]); + dfx_register_res_err(print_name, dfx_use_mmio, + bar_start[0], bar_len[0]); err = -EBUSY; goto err_out_disable; } if (bar_start[1] != 0) { region = request_region(bar_start[1], bar_len[1], print_name); if (!region) { - pr_err("%s: Cannot reserve I/O resource " - "0x%lx @ 0x%lx, aborting\n", print_name, - (long)bar_len[1], (long)bar_start[1]); + dfx_register_res_err(print_name, 0, + bar_start[1], bar_len[1]); err = -EBUSY; goto err_out_csr_region; } @@ -604,9 +618,8 @@ static int dfx_register(struct device *bdev) if (bar_start[2] != 0) { region = request_region(bar_start[2], bar_len[2], print_name); if (!region) { - pr_err("%s: Cannot reserve I/O resource " - "0x%lx @ 0x%lx, aborting\n", print_name, - (long)bar_len[2], (long)bar_start[2]); + dfx_register_res_err(print_name, 0, + bar_start[2], bar_len[2]); err = -EBUSY; goto err_out_bh_region; } diff --git a/drivers/net/geneve.c b/drivers/net/geneve.c index 42f31c681846..61cd3dd4deab 100644 --- a/drivers/net/geneve.c +++ b/drivers/net/geneve.c @@ -891,7 +891,7 @@ static int geneve_xmit_skb(struct sk_buff *skb, struct net_device *dev, __be16 sport; int err; - if (!pskb_network_may_pull(skb, sizeof(struct iphdr))) + if (!pskb_inet_may_pull(skb)) return -EINVAL; sport = udp_flow_src_port(geneve->net, skb, 1, USHRT_MAX, true); @@ -988,7 +988,7 @@ static int geneve6_xmit_skb(struct sk_buff *skb, struct net_device *dev, __be16 sport; int err; - if (!pskb_network_may_pull(skb, sizeof(struct ipv6hdr))) + if (!pskb_inet_may_pull(skb)) return -EINVAL; sport = udp_flow_src_port(geneve->net, skb, 1, USHRT_MAX, true); diff --git a/drivers/net/ipa/gsi.c b/drivers/net/ipa/gsi.c index 390d3403386a..144892060718 100644 --- a/drivers/net/ipa/gsi.c +++ b/drivers/net/ipa/gsi.c @@ -211,8 +211,8 @@ static void gsi_irq_setup(struct gsi *gsi) iowrite32(0, gsi->virt + GSI_CNTXT_SRC_IEOB_IRQ_MSK_OFFSET); /* The inter-EE registers are in the non-adjusted address range */ - iowrite32(0, gsi->virt_raw + GSI_INTER_EE_SRC_CH_IRQ_OFFSET); - iowrite32(0, gsi->virt_raw + GSI_INTER_EE_SRC_EV_CH_IRQ_OFFSET); + iowrite32(0, gsi->virt_raw + GSI_INTER_EE_SRC_CH_IRQ_MSK_OFFSET); + iowrite32(0, gsi->virt_raw + GSI_INTER_EE_SRC_EV_CH_IRQ_MSK_OFFSET); iowrite32(0, gsi->virt + GSI_CNTXT_GSI_IRQ_EN_OFFSET); } diff --git a/drivers/net/ipa/gsi_reg.h b/drivers/net/ipa/gsi_reg.h index 1622d8cf8dea..48ef04afab79 100644 --- a/drivers/net/ipa/gsi_reg.h +++ b/drivers/net/ipa/gsi_reg.h @@ -53,15 +53,15 @@ #define GSI_EE_REG_ADJUST 0x0000d000 /* IPA v4.5+ */ /* The two inter-EE IRQ register offsets are relative to gsi->virt_raw */ -#define GSI_INTER_EE_SRC_CH_IRQ_OFFSET \ - GSI_INTER_EE_N_SRC_CH_IRQ_OFFSET(GSI_EE_AP) -#define GSI_INTER_EE_N_SRC_CH_IRQ_OFFSET(ee) \ - (0x0000c018 + 0x1000 * (ee)) - -#define GSI_INTER_EE_SRC_EV_CH_IRQ_OFFSET \ - GSI_INTER_EE_N_SRC_EV_CH_IRQ_OFFSET(GSI_EE_AP) -#define GSI_INTER_EE_N_SRC_EV_CH_IRQ_OFFSET(ee) \ - (0x0000c01c + 0x1000 * (ee)) +#define GSI_INTER_EE_SRC_CH_IRQ_MSK_OFFSET \ + GSI_INTER_EE_N_SRC_CH_IRQ_MSK_OFFSET(GSI_EE_AP) +#define GSI_INTER_EE_N_SRC_CH_IRQ_MSK_OFFSET(ee) \ + (0x0000c020 + 0x1000 * (ee)) + +#define GSI_INTER_EE_SRC_EV_CH_IRQ_MSK_OFFSET \ + GSI_INTER_EE_N_SRC_EV_CH_IRQ_MSK_OFFSET(GSI_EE_AP) +#define GSI_INTER_EE_N_SRC_EV_CH_IRQ_MSK_OFFSET(ee) \ + (0x0000c024 + 0x1000 * (ee)) /* All other register offsets are relative to gsi->virt */ #define GSI_CH_C_CNTXT_0_OFFSET(ch) \ diff --git a/drivers/net/phy/intel-xway.c b/drivers/net/phy/intel-xway.c index 6eac50d4b42f..d453ec016168 100644 --- a/drivers/net/phy/intel-xway.c +++ b/drivers/net/phy/intel-xway.c @@ -11,6 +11,18 @@ #define XWAY_MDIO_IMASK 0x19 /* interrupt mask */ #define XWAY_MDIO_ISTAT 0x1A /* interrupt status */ +#define XWAY_MDIO_LED 0x1B /* led control */ + +/* bit 15:12 are reserved */ +#define XWAY_MDIO_LED_LED3_EN BIT(11) /* Enable the integrated function of LED3 */ +#define XWAY_MDIO_LED_LED2_EN BIT(10) /* Enable the integrated function of LED2 */ +#define XWAY_MDIO_LED_LED1_EN BIT(9) /* Enable the integrated function of LED1 */ +#define XWAY_MDIO_LED_LED0_EN BIT(8) /* Enable the integrated function of LED0 */ +/* bit 7:4 are reserved */ +#define XWAY_MDIO_LED_LED3_DA BIT(3) /* Direct Access to LED3 */ +#define XWAY_MDIO_LED_LED2_DA BIT(2) /* Direct Access to LED2 */ +#define XWAY_MDIO_LED_LED1_DA BIT(1) /* Direct Access to LED1 */ +#define XWAY_MDIO_LED_LED0_DA BIT(0) /* Direct Access to LED0 */ #define XWAY_MDIO_INIT_WOL BIT(15) /* Wake-On-LAN */ #define XWAY_MDIO_INIT_MSRE BIT(14) @@ -159,6 +171,15 @@ static int xway_gphy_config_init(struct phy_device *phydev) /* Clear all pending interrupts */ phy_read(phydev, XWAY_MDIO_ISTAT); + /* Ensure that integrated led function is enabled for all leds */ + err = phy_write(phydev, XWAY_MDIO_LED, + XWAY_MDIO_LED_LED0_EN | + XWAY_MDIO_LED_LED1_EN | + XWAY_MDIO_LED_LED2_EN | + XWAY_MDIO_LED_LED3_EN); + if (err) + return err; + phy_write_mmd(phydev, MDIO_MMD_VEND2, XWAY_MMD_LEDCH, XWAY_MMD_LEDCH_NACS_NONE | XWAY_MMD_LEDCH_SBF_F02HZ | diff --git a/drivers/net/phy/marvell.c b/drivers/net/phy/marvell.c index 8018ddf7f316..f86c9ddc609e 100644 --- a/drivers/net/phy/marvell.c +++ b/drivers/net/phy/marvell.c @@ -967,22 +967,28 @@ static int m88e1111_get_downshift(struct phy_device *phydev, u8 *data) static int m88e1111_set_downshift(struct phy_device *phydev, u8 cnt) { - int val; + int val, err; if (cnt > MII_M1111_PHY_EXT_CR_DOWNSHIFT_MAX) return -E2BIG; - if (!cnt) - return phy_clear_bits(phydev, MII_M1111_PHY_EXT_CR, - MII_M1111_PHY_EXT_CR_DOWNSHIFT_EN); + if (!cnt) { + err = phy_clear_bits(phydev, MII_M1111_PHY_EXT_CR, + MII_M1111_PHY_EXT_CR_DOWNSHIFT_EN); + } else { + val = MII_M1111_PHY_EXT_CR_DOWNSHIFT_EN; + val |= FIELD_PREP(MII_M1111_PHY_EXT_CR_DOWNSHIFT_MASK, cnt - 1); - val = MII_M1111_PHY_EXT_CR_DOWNSHIFT_EN; - val |= FIELD_PREP(MII_M1111_PHY_EXT_CR_DOWNSHIFT_MASK, cnt - 1); + err = phy_modify(phydev, MII_M1111_PHY_EXT_CR, + MII_M1111_PHY_EXT_CR_DOWNSHIFT_EN | + MII_M1111_PHY_EXT_CR_DOWNSHIFT_MASK, + val); + } - return phy_modify(phydev, MII_M1111_PHY_EXT_CR, - MII_M1111_PHY_EXT_CR_DOWNSHIFT_EN | - MII_M1111_PHY_EXT_CR_DOWNSHIFT_MASK, - val); + if (err < 0) + return err; + + return genphy_soft_reset(phydev); } static int m88e1111_get_tunable(struct phy_device *phydev, @@ -1025,22 +1031,28 @@ static int m88e1011_get_downshift(struct phy_device *phydev, u8 *data) static int m88e1011_set_downshift(struct phy_device *phydev, u8 cnt) { - int val; + int val, err; if (cnt > MII_M1011_PHY_SCR_DOWNSHIFT_MAX) return -E2BIG; - if (!cnt) - return phy_clear_bits(phydev, MII_M1011_PHY_SCR, - MII_M1011_PHY_SCR_DOWNSHIFT_EN); + if (!cnt) { + err = phy_clear_bits(phydev, MII_M1011_PHY_SCR, + MII_M1011_PHY_SCR_DOWNSHIFT_EN); + } else { + val = MII_M1011_PHY_SCR_DOWNSHIFT_EN; + val |= FIELD_PREP(MII_M1011_PHY_SCR_DOWNSHIFT_MASK, cnt - 1); - val = MII_M1011_PHY_SCR_DOWNSHIFT_EN; - val |= FIELD_PREP(MII_M1011_PHY_SCR_DOWNSHIFT_MASK, cnt - 1); + err = phy_modify(phydev, MII_M1011_PHY_SCR, + MII_M1011_PHY_SCR_DOWNSHIFT_EN | + MII_M1011_PHY_SCR_DOWNSHIFT_MASK, + val); + } - return phy_modify(phydev, MII_M1011_PHY_SCR, - MII_M1011_PHY_SCR_DOWNSHIFT_EN | - MII_M1011_PHY_SCR_DOWNSHIFT_MASK, - val); + if (err < 0) + return err; + + return genphy_soft_reset(phydev); } static int m88e1011_get_tunable(struct phy_device *phydev, diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index cc38e326405a..af2e1759b523 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -273,6 +273,9 @@ static __maybe_unused int mdio_bus_phy_suspend(struct device *dev) { struct phy_device *phydev = to_phy_device(dev); + if (phydev->mac_managed_pm) + return 0; + /* We must stop the state machine manually, otherwise it stops out of * control, possibly with the phydev->lock held. Upon resume, netdev * may call phy routines that try to grab the same lock, and that may @@ -294,6 +297,9 @@ static __maybe_unused int mdio_bus_phy_resume(struct device *dev) struct phy_device *phydev = to_phy_device(dev); int ret; + if (phydev->mac_managed_pm) + return 0; + if (!phydev->suspended_by_mdio_bus) goto no_resume; diff --git a/drivers/net/phy/smsc.c b/drivers/net/phy/smsc.c index ddb78fb4d6dc..d8cac02a79b9 100644 --- a/drivers/net/phy/smsc.c +++ b/drivers/net/phy/smsc.c @@ -185,10 +185,13 @@ static int lan87xx_config_aneg(struct phy_device *phydev) return genphy_config_aneg(phydev); } -static int lan87xx_config_aneg_ext(struct phy_device *phydev) +static int lan95xx_config_aneg_ext(struct phy_device *phydev) { int rc; + if (phydev->phy_id != 0x0007c0f0) /* not (LAN9500A or LAN9505A) */ + return lan87xx_config_aneg(phydev); + /* Extend Manual AutoMDIX timer */ rc = phy_read(phydev, PHY_EDPD_CONFIG); if (rc < 0) @@ -441,7 +444,7 @@ static struct phy_driver smsc_phy_driver[] = { .read_status = lan87xx_read_status, .config_init = smsc_phy_config_init, .soft_reset = smsc_phy_reset, - .config_aneg = lan87xx_config_aneg_ext, + .config_aneg = lan95xx_config_aneg_ext, /* IRQ related */ .config_intr = smsc_phy_config_intr, diff --git a/drivers/net/usb/ax88179_178a.c b/drivers/net/usb/ax88179_178a.c index d650b39b6e5d..c1316718304d 100644 --- a/drivers/net/usb/ax88179_178a.c +++ b/drivers/net/usb/ax88179_178a.c @@ -296,12 +296,12 @@ static int ax88179_read_cmd(struct usbnet *dev, u8 cmd, u16 value, u16 index, int ret; if (2 == size) { - u16 buf; + u16 buf = 0; ret = __ax88179_read_cmd(dev, cmd, value, index, size, &buf, 0); le16_to_cpus(&buf); *((u16 *)data) = buf; } else if (4 == size) { - u32 buf; + u32 buf = 0; ret = __ax88179_read_cmd(dev, cmd, value, index, size, &buf, 0); le32_to_cpus(&buf); *((u32 *)data) = buf; @@ -1296,6 +1296,8 @@ static void ax88179_get_mac_addr(struct usbnet *dev) { u8 mac[ETH_ALEN]; + memset(mac, 0, sizeof(mac)); + /* Maybe the boot loader passed the MAC address via device tree */ if (!eth_platform_get_mac_address(&dev->udev->dev, mac)) { netif_dbg(dev, ifup, dev->net, diff --git a/drivers/net/usb/hso.c b/drivers/net/usb/hso.c index 9bc58e64b5b7..3ef4b2841402 100644 --- a/drivers/net/usb/hso.c +++ b/drivers/net/usb/hso.c @@ -3104,7 +3104,7 @@ static void hso_free_interface(struct usb_interface *interface) cancel_work_sync(&serial_table[i]->async_put_intf); cancel_work_sync(&serial_table[i]->async_get_intf); hso_serial_tty_unregister(serial); - kref_put(&serial_table[i]->ref, hso_serial_ref_free); + kref_put(&serial->parent->ref, hso_serial_ref_free); } } diff --git a/drivers/net/wan/hdlc_fr.c b/drivers/net/wan/hdlc_fr.c index 4d9dc7d15908..0720f5f92caa 100644 --- a/drivers/net/wan/hdlc_fr.c +++ b/drivers/net/wan/hdlc_fr.c @@ -415,7 +415,7 @@ static netdev_tx_t pvc_xmit(struct sk_buff *skb, struct net_device *dev) if (pad > 0) { /* Pad the frame with zeros */ if (__skb_pad(skb, pad, false)) - goto out; + goto drop; skb_put(skb, pad); } } @@ -448,9 +448,8 @@ static netdev_tx_t pvc_xmit(struct sk_buff *skb, struct net_device *dev) return NETDEV_TX_OK; drop: - kfree_skb(skb); -out: dev->stats.tx_dropped++; + kfree_skb(skb); return NETDEV_TX_OK; } diff --git a/drivers/net/wan/lapbether.c b/drivers/net/wan/lapbether.c index c3372498f4f1..8fda0446ff71 100644 --- a/drivers/net/wan/lapbether.c +++ b/drivers/net/wan/lapbether.c @@ -51,6 +51,8 @@ struct lapbethdev { struct list_head node; struct net_device *ethdev; /* link to ethernet device */ struct net_device *axdev; /* lapbeth device (lapb#) */ + bool up; + spinlock_t up_lock; /* Protects "up" */ }; static LIST_HEAD(lapbeth_devices); @@ -101,8 +103,9 @@ static int lapbeth_rcv(struct sk_buff *skb, struct net_device *dev, struct packe rcu_read_lock(); lapbeth = lapbeth_get_x25_dev(dev); if (!lapbeth) - goto drop_unlock; - if (!netif_running(lapbeth->axdev)) + goto drop_unlock_rcu; + spin_lock_bh(&lapbeth->up_lock); + if (!lapbeth->up) goto drop_unlock; len = skb->data[0] + skb->data[1] * 256; @@ -117,11 +120,14 @@ static int lapbeth_rcv(struct sk_buff *skb, struct net_device *dev, struct packe goto drop_unlock; } out: + spin_unlock_bh(&lapbeth->up_lock); rcu_read_unlock(); return 0; drop_unlock: kfree_skb(skb); goto out; +drop_unlock_rcu: + rcu_read_unlock(); drop: kfree_skb(skb); return 0; @@ -151,13 +157,11 @@ static int lapbeth_data_indication(struct net_device *dev, struct sk_buff *skb) static netdev_tx_t lapbeth_xmit(struct sk_buff *skb, struct net_device *dev) { + struct lapbethdev *lapbeth = netdev_priv(dev); int err; - /* - * Just to be *really* sure not to send anything if the interface - * is down, the ethernet device may have gone. - */ - if (!netif_running(dev)) + spin_lock_bh(&lapbeth->up_lock); + if (!lapbeth->up) goto drop; /* There should be a pseudo header of 1 byte added by upper layers. @@ -194,6 +198,7 @@ static netdev_tx_t lapbeth_xmit(struct sk_buff *skb, goto drop; } out: + spin_unlock_bh(&lapbeth->up_lock); return NETDEV_TX_OK; drop: kfree_skb(skb); @@ -285,6 +290,7 @@ static const struct lapb_register_struct lapbeth_callbacks = { */ static int lapbeth_open(struct net_device *dev) { + struct lapbethdev *lapbeth = netdev_priv(dev); int err; if ((err = lapb_register(dev, &lapbeth_callbacks)) != LAPB_OK) { @@ -292,13 +298,22 @@ static int lapbeth_open(struct net_device *dev) return -ENODEV; } + spin_lock_bh(&lapbeth->up_lock); + lapbeth->up = true; + spin_unlock_bh(&lapbeth->up_lock); + return 0; } static int lapbeth_close(struct net_device *dev) { + struct lapbethdev *lapbeth = netdev_priv(dev); int err; + spin_lock_bh(&lapbeth->up_lock); + lapbeth->up = false; + spin_unlock_bh(&lapbeth->up_lock); + if ((err = lapb_unregister(dev)) != LAPB_OK) pr_err("lapb_unregister error: %d\n", err); @@ -356,6 +371,9 @@ static int lapbeth_new_device(struct net_device *dev) dev_hold(dev); lapbeth->ethdev = dev; + lapbeth->up = false; + spin_lock_init(&lapbeth->up_lock); + rc = -EIO; if (register_netdevice(ndev)) goto fail; diff --git a/drivers/net/wireless/ath/ath10k/htc.c b/drivers/net/wireless/ath/ath10k/htc.c index 0a37be6a7d33..fab398046a3f 100644 --- a/drivers/net/wireless/ath/ath10k/htc.c +++ b/drivers/net/wireless/ath/ath10k/htc.c @@ -669,7 +669,7 @@ static int ath10k_htc_send_bundle(struct ath10k_htc_ep *ep, ath10k_dbg(ar, ATH10K_DBG_HTC, "bundle tx status %d eid %d req count %d count %d len %d\n", - ret, ep->eid, skb_queue_len(&ep->tx_req_head), cn, bundle_skb->len); + ret, ep->eid, skb_queue_len(&ep->tx_req_head), cn, skb_len); return ret; } diff --git a/drivers/net/wireless/ath/ath10k/wmi-tlv.c b/drivers/net/wireless/ath/ath10k/wmi-tlv.c index d97b33f789e4..7efbe03fbca8 100644 --- a/drivers/net/wireless/ath/ath10k/wmi-tlv.c +++ b/drivers/net/wireless/ath/ath10k/wmi-tlv.c @@ -592,6 +592,9 @@ static void ath10k_wmi_event_tdls_peer(struct ath10k *ar, struct sk_buff *skb) GFP_ATOMIC ); break; + default: + kfree(tb); + return; } exit: diff --git a/drivers/net/wireless/ath/ath11k/wmi.c b/drivers/net/wireless/ath/ath11k/wmi.c index cccfd3bd4d27..ca5cda890d58 100644 --- a/drivers/net/wireless/ath/ath11k/wmi.c +++ b/drivers/net/wireless/ath/ath11k/wmi.c @@ -5417,31 +5417,6 @@ int ath11k_wmi_pull_fw_stats(struct ath11k_base *ab, struct sk_buff *skb, return 0; } -static int -ath11k_pull_pdev_temp_ev(struct ath11k_base *ab, u8 *evt_buf, - u32 len, const struct wmi_pdev_temperature_event *ev) -{ - const void **tb; - int ret; - - tb = ath11k_wmi_tlv_parse_alloc(ab, evt_buf, len, GFP_ATOMIC); - if (IS_ERR(tb)) { - ret = PTR_ERR(tb); - ath11k_warn(ab, "failed to parse tlv: %d\n", ret); - return ret; - } - - ev = tb[WMI_TAG_PDEV_TEMPERATURE_EVENT]; - if (!ev) { - ath11k_warn(ab, "failed to fetch pdev temp ev"); - kfree(tb); - return -EPROTO; - } - - kfree(tb); - return 0; -} - size_t ath11k_wmi_fw_stats_num_vdevs(struct list_head *head) { struct ath11k_fw_stats_vdev *i; @@ -6849,23 +6824,37 @@ ath11k_wmi_pdev_temperature_event(struct ath11k_base *ab, struct sk_buff *skb) { struct ath11k *ar; - struct wmi_pdev_temperature_event ev = {0}; + const void **tb; + const struct wmi_pdev_temperature_event *ev; + int ret; + + tb = ath11k_wmi_tlv_parse_alloc(ab, skb->data, skb->len, GFP_ATOMIC); + if (IS_ERR(tb)) { + ret = PTR_ERR(tb); + ath11k_warn(ab, "failed to parse tlv: %d\n", ret); + return; + } - if (ath11k_pull_pdev_temp_ev(ab, skb->data, skb->len, &ev) != 0) { - ath11k_warn(ab, "failed to extract pdev temperature event"); + ev = tb[WMI_TAG_PDEV_TEMPERATURE_EVENT]; + if (!ev) { + ath11k_warn(ab, "failed to fetch pdev temp ev"); + kfree(tb); return; } ath11k_dbg(ab, ATH11K_DBG_WMI, - "pdev temperature ev temp %d pdev_id %d\n", ev.temp, ev.pdev_id); + "pdev temperature ev temp %d pdev_id %d\n", ev->temp, ev->pdev_id); - ar = ath11k_mac_get_ar_by_pdev_id(ab, ev.pdev_id); + ar = ath11k_mac_get_ar_by_pdev_id(ab, ev->pdev_id); if (!ar) { - ath11k_warn(ab, "invalid pdev id in pdev temperature ev %d", ev.pdev_id); + ath11k_warn(ab, "invalid pdev id in pdev temperature ev %d", ev->pdev_id); + kfree(tb); return; } - ath11k_thermal_event_temperature(ar, ev.temp); + ath11k_thermal_event_temperature(ar, ev->temp); + + kfree(tb); } static void ath11k_fils_discovery_event(struct ath11k_base *ab, diff --git a/drivers/net/wireless/ath/ath9k/htc_drv_init.c b/drivers/net/wireless/ath/ath9k/htc_drv_init.c index db0c6fa9c9dc..ff61ae34ecdf 100644 --- a/drivers/net/wireless/ath/ath9k/htc_drv_init.c +++ b/drivers/net/wireless/ath/ath9k/htc_drv_init.c @@ -246,7 +246,7 @@ static unsigned int ath9k_regread(void *hw_priv, u32 reg_offset) if (unlikely(r)) { ath_dbg(common, WMI, "REGISTER READ FAILED: (0x%04x, %d)\n", reg_offset, r); - return -EIO; + return -1; } return be32_to_cpu(val); diff --git a/drivers/net/wireless/ath/ath9k/hw.c b/drivers/net/wireless/ath/ath9k/hw.c index 5abc2a5526ec..2ca3b86714a9 100644 --- a/drivers/net/wireless/ath/ath9k/hw.c +++ b/drivers/net/wireless/ath/ath9k/hw.c @@ -286,7 +286,7 @@ static bool ath9k_hw_read_revisions(struct ath_hw *ah) srev = REG_READ(ah, AR_SREV); - if (srev == -EIO) { + if (srev == -1) { ath_err(ath9k_hw_common(ah), "Failed to read SREV register"); return false; diff --git a/drivers/net/wireless/cisco/airo.c b/drivers/net/wireless/cisco/airo.c index 60db38c38960..fd37d4d2983b 100644 --- a/drivers/net/wireless/cisco/airo.c +++ b/drivers/net/wireless/cisco/airo.c @@ -3817,6 +3817,68 @@ static inline void set_auth_type(struct airo_info *local, int auth_type) local->last_auth = auth_type; } +static int noinline_for_stack airo_readconfig(struct airo_info *ai, u8 *mac, int lock) +{ + int i, status; + /* large variables, so don't inline this function, + * maybe change to kmalloc + */ + tdsRssiRid rssi_rid; + CapabilityRid cap_rid; + + kfree(ai->SSID); + ai->SSID = NULL; + // general configuration (read/modify/write) + status = readConfigRid(ai, lock); + if (status != SUCCESS) return ERROR; + + status = readCapabilityRid(ai, &cap_rid, lock); + if (status != SUCCESS) return ERROR; + + status = PC4500_readrid(ai, RID_RSSI, &rssi_rid, sizeof(rssi_rid), lock); + if (status == SUCCESS) { + if (ai->rssi || (ai->rssi = kmalloc(512, GFP_KERNEL)) != NULL) + memcpy(ai->rssi, (u8*)&rssi_rid + 2, 512); /* Skip RID length member */ + } + else { + kfree(ai->rssi); + ai->rssi = NULL; + if (cap_rid.softCap & cpu_to_le16(8)) + ai->config.rmode |= RXMODE_NORMALIZED_RSSI; + else + airo_print_warn(ai->dev->name, "unknown received signal " + "level scale"); + } + ai->config.opmode = adhoc ? MODE_STA_IBSS : MODE_STA_ESS; + set_auth_type(ai, AUTH_OPEN); + ai->config.modulation = MOD_CCK; + + if (le16_to_cpu(cap_rid.len) >= sizeof(cap_rid) && + (cap_rid.extSoftCap & cpu_to_le16(1)) && + micsetup(ai) == SUCCESS) { + ai->config.opmode |= MODE_MIC; + set_bit(FLAG_MIC_CAPABLE, &ai->flags); + } + + /* Save off the MAC */ + for (i = 0; i < ETH_ALEN; i++) { + mac[i] = ai->config.macAddr[i]; + } + + /* Check to see if there are any insmod configured + rates to add */ + if (rates[0]) { + memset(ai->config.rates, 0, sizeof(ai->config.rates)); + for (i = 0; i < 8 && rates[i]; i++) { + ai->config.rates[i] = rates[i]; + } + } + set_bit (FLAG_COMMIT, &ai->flags); + + return SUCCESS; +} + + static u16 setup_card(struct airo_info *ai, u8 *mac, int lock) { Cmd cmd; @@ -3863,58 +3925,9 @@ static u16 setup_card(struct airo_info *ai, u8 *mac, int lock) if (lock) up(&ai->sem); if (ai->config.len == 0) { - int i; - tdsRssiRid rssi_rid; - CapabilityRid cap_rid; - - kfree(ai->SSID); - ai->SSID = NULL; - // general configuration (read/modify/write) - status = readConfigRid(ai, lock); - if (status != SUCCESS) return ERROR; - - status = readCapabilityRid(ai, &cap_rid, lock); - if (status != SUCCESS) return ERROR; - - status = PC4500_readrid(ai, RID_RSSI,&rssi_rid, sizeof(rssi_rid), lock); - if (status == SUCCESS) { - if (ai->rssi || (ai->rssi = kmalloc(512, GFP_KERNEL)) != NULL) - memcpy(ai->rssi, (u8*)&rssi_rid + 2, 512); /* Skip RID length member */ - } - else { - kfree(ai->rssi); - ai->rssi = NULL; - if (cap_rid.softCap & cpu_to_le16(8)) - ai->config.rmode |= RXMODE_NORMALIZED_RSSI; - else - airo_print_warn(ai->dev->name, "unknown received signal " - "level scale"); - } - ai->config.opmode = adhoc ? MODE_STA_IBSS : MODE_STA_ESS; - set_auth_type(ai, AUTH_OPEN); - ai->config.modulation = MOD_CCK; - - if (le16_to_cpu(cap_rid.len) >= sizeof(cap_rid) && - (cap_rid.extSoftCap & cpu_to_le16(1)) && - micsetup(ai) == SUCCESS) { - ai->config.opmode |= MODE_MIC; - set_bit(FLAG_MIC_CAPABLE, &ai->flags); - } - - /* Save off the MAC */ - for (i = 0; i < ETH_ALEN; i++) { - mac[i] = ai->config.macAddr[i]; - } - - /* Check to see if there are any insmod configured - rates to add */ - if (rates[0]) { - memset(ai->config.rates, 0, sizeof(ai->config.rates)); - for (i = 0; i < 8 && rates[i]; i++) { - ai->config.rates[i] = rates[i]; - } - } - set_bit (FLAG_COMMIT, &ai->flags); + status = airo_readconfig(ai, mac, lock); + if (status != SUCCESS) + return ERROR; } /* Setup the SSIDs if present */ diff --git a/drivers/net/wireless/intel/ipw2x00/libipw_wx.c b/drivers/net/wireless/intel/ipw2x00/libipw_wx.c index a0cf78c418ac..903de34028ef 100644 --- a/drivers/net/wireless/intel/ipw2x00/libipw_wx.c +++ b/drivers/net/wireless/intel/ipw2x00/libipw_wx.c @@ -633,8 +633,10 @@ int libipw_wx_set_encodeext(struct libipw_device *ieee, } if (ext->alg != IW_ENCODE_ALG_NONE) { - memcpy(sec.keys[idx], ext->key, ext->key_len); - sec.key_sizes[idx] = ext->key_len; + int key_len = clamp_val(ext->key_len, 0, SCM_KEY_LEN); + + memcpy(sec.keys[idx], ext->key, key_len); + sec.key_sizes[idx] = key_len; sec.flags |= (1 << idx); if (ext->alg == IW_ENCODE_ALG_WEP) { sec.encode_alg[idx] = SEC_ALG_WEP; diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c b/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c index 579bc81cc0ae..4cd8c39cc3e9 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c +++ b/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* - * Copyright (C) 2018-2020 Intel Corporation + * Copyright (C) 2018-2021 Intel Corporation */ #include #include "iwl-drv.h" @@ -426,7 +426,8 @@ void iwl_dbg_tlv_load_bin(struct device *dev, struct iwl_trans *trans) const struct firmware *fw; int res; - if (!iwlwifi_mod_params.enable_ini) + if (!iwlwifi_mod_params.enable_ini || + trans->trans_cfg->device_family <= IWL_DEVICE_FAMILY_9000) return; res = firmware_request_nowarn(&fw, "iwl-debug-yoyo.bin", dev); diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-trans.c b/drivers/net/wireless/intel/iwlwifi/iwl-trans.c index 60e0db4a5e20..9236f9106826 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-trans.c +++ b/drivers/net/wireless/intel/iwlwifi/iwl-trans.c @@ -2,7 +2,7 @@ /* * Copyright (C) 2015 Intel Mobile Communications GmbH * Copyright (C) 2016-2017 Intel Deutschland GmbH - * Copyright (C) 2019-2020 Intel Corporation + * Copyright (C) 2019-2021 Intel Corporation */ #include #include @@ -21,7 +21,6 @@ struct iwl_trans *iwl_trans_alloc(unsigned int priv_size, const struct iwl_cfg_trans_params *cfg_trans) { struct iwl_trans *trans; - int txcmd_size, txcmd_align; #ifdef CONFIG_LOCKDEP static struct lock_class_key __key; #endif @@ -31,10 +30,40 @@ struct iwl_trans *iwl_trans_alloc(unsigned int priv_size, return NULL; trans->trans_cfg = cfg_trans; - if (!cfg_trans->gen2) { + +#ifdef CONFIG_LOCKDEP + lockdep_init_map(&trans->sync_cmd_lockdep_map, "sync_cmd_lockdep_map", + &__key, 0); +#endif + + trans->dev = dev; + trans->ops = ops; + trans->num_rx_queues = 1; + + WARN_ON(!ops->wait_txq_empty && !ops->wait_tx_queues_empty); + + if (trans->trans_cfg->use_tfh) { + trans->txqs.tfd.addr_size = 64; + trans->txqs.tfd.max_tbs = IWL_TFH_NUM_TBS; + trans->txqs.tfd.size = sizeof(struct iwl_tfh_tfd); + } else { + trans->txqs.tfd.addr_size = 36; + trans->txqs.tfd.max_tbs = IWL_NUM_OF_TBS; + trans->txqs.tfd.size = sizeof(struct iwl_tfd); + } + trans->max_skb_frags = IWL_TRANS_MAX_FRAGS(trans); + + return trans; +} + +int iwl_trans_init(struct iwl_trans *trans) +{ + int txcmd_size, txcmd_align; + + if (!trans->trans_cfg->gen2) { txcmd_size = sizeof(struct iwl_tx_cmd); txcmd_align = sizeof(void *); - } else if (cfg_trans->device_family < IWL_DEVICE_FAMILY_AX210) { + } else if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_AX210) { txcmd_size = sizeof(struct iwl_tx_cmd_gen2); txcmd_align = 64; } else { @@ -46,17 +75,8 @@ struct iwl_trans *iwl_trans_alloc(unsigned int priv_size, txcmd_size += 36; /* biggest possible 802.11 header */ /* Ensure device TX cmd cannot reach/cross a page boundary in gen2 */ - if (WARN_ON(cfg_trans->gen2 && txcmd_size >= txcmd_align)) - return ERR_PTR(-EINVAL); - -#ifdef CONFIG_LOCKDEP - lockdep_init_map(&trans->sync_cmd_lockdep_map, "sync_cmd_lockdep_map", - &__key, 0); -#endif - - trans->dev = dev; - trans->ops = ops; - trans->num_rx_queues = 1; + if (WARN_ON(trans->trans_cfg->gen2 && txcmd_size >= txcmd_align)) + return -EINVAL; if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) trans->txqs.bc_tbl_size = sizeof(struct iwl_gen3_bc_tbl); @@ -68,23 +88,16 @@ struct iwl_trans *iwl_trans_alloc(unsigned int priv_size, * allocate here. */ if (trans->trans_cfg->gen2) { - trans->txqs.bc_pool = dmam_pool_create("iwlwifi:bc", dev, + trans->txqs.bc_pool = dmam_pool_create("iwlwifi:bc", trans->dev, trans->txqs.bc_tbl_size, 256, 0); if (!trans->txqs.bc_pool) - return NULL; + return -ENOMEM; } - if (trans->trans_cfg->use_tfh) { - trans->txqs.tfd.addr_size = 64; - trans->txqs.tfd.max_tbs = IWL_TFH_NUM_TBS; - trans->txqs.tfd.size = sizeof(struct iwl_tfh_tfd); - } else { - trans->txqs.tfd.addr_size = 36; - trans->txqs.tfd.max_tbs = IWL_NUM_OF_TBS; - trans->txqs.tfd.size = sizeof(struct iwl_tfd); - } - trans->max_skb_frags = IWL_TRANS_MAX_FRAGS(trans); + /* Some things must not change even if the config does */ + WARN_ON(trans->txqs.tfd.addr_size != + (trans->trans_cfg->use_tfh ? 64 : 36)); snprintf(trans->dev_cmd_pool_name, sizeof(trans->dev_cmd_pool_name), "iwl_cmd_pool:%s", dev_name(trans->dev)); @@ -93,35 +106,35 @@ struct iwl_trans *iwl_trans_alloc(unsigned int priv_size, txcmd_size, txcmd_align, SLAB_HWCACHE_ALIGN, NULL); if (!trans->dev_cmd_pool) - return NULL; - - WARN_ON(!ops->wait_txq_empty && !ops->wait_tx_queues_empty); + return -ENOMEM; trans->txqs.tso_hdr_page = alloc_percpu(struct iwl_tso_hdr_page); if (!trans->txqs.tso_hdr_page) { kmem_cache_destroy(trans->dev_cmd_pool); - return NULL; + return -ENOMEM; } /* Initialize the wait queue for commands */ init_waitqueue_head(&trans->wait_command_queue); - return trans; + return 0; } void iwl_trans_free(struct iwl_trans *trans) { int i; - for_each_possible_cpu(i) { - struct iwl_tso_hdr_page *p = - per_cpu_ptr(trans->txqs.tso_hdr_page, i); + if (trans->txqs.tso_hdr_page) { + for_each_possible_cpu(i) { + struct iwl_tso_hdr_page *p = + per_cpu_ptr(trans->txqs.tso_hdr_page, i); - if (p->page) - __free_page(p->page); - } + if (p && p->page) + __free_page(p->page); + } - free_percpu(trans->txqs.tso_hdr_page); + free_percpu(trans->txqs.tso_hdr_page); + } kmem_cache_destroy(trans->dev_cmd_pool); } diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h index 4a5822c1be13..3e0df6fbb642 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h +++ b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h @@ -1438,6 +1438,7 @@ struct iwl_trans *iwl_trans_alloc(unsigned int priv_size, struct device *dev, const struct iwl_trans_ops *ops, const struct iwl_cfg_trans_params *cfg_trans); +int iwl_trans_init(struct iwl_trans *trans); void iwl_trans_free(struct iwl_trans *trans); /***************************************************** diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c b/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c index 8772b65c9dab..2d58cb969918 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* * Copyright (C) 2017 Intel Deutschland GmbH - * Copyright (C) 2018-2020 Intel Corporation + * Copyright (C) 2018-2021 Intel Corporation */ #include "rs.h" #include "fw-api.h" @@ -72,19 +72,15 @@ static u16 rs_fw_get_config_flags(struct iwl_mvm *mvm, bool vht_ena = vht_cap->vht_supported; u16 flags = 0; + /* get STBC flags */ if (mvm->cfg->ht_params->stbc && (num_of_ant(iwl_mvm_get_valid_tx_ant(mvm)) > 1)) { - if (he_cap->has_he) { - if (he_cap->he_cap_elem.phy_cap_info[2] & - IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ) - flags |= IWL_TLC_MNG_CFG_FLAGS_STBC_MSK; - - if (he_cap->he_cap_elem.phy_cap_info[7] & - IEEE80211_HE_PHY_CAP7_STBC_RX_ABOVE_80MHZ) - flags |= IWL_TLC_MNG_CFG_FLAGS_HE_STBC_160MHZ_MSK; - } else if ((ht_cap->cap & IEEE80211_HT_CAP_RX_STBC) || - (vht_ena && - (vht_cap->cap & IEEE80211_VHT_CAP_RXSTBC_MASK))) + if (he_cap->has_he && he_cap->he_cap_elem.phy_cap_info[2] & + IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ) + flags |= IWL_TLC_MNG_CFG_FLAGS_STBC_MSK; + else if (vht_cap->cap & IEEE80211_VHT_CAP_RXSTBC_MASK) + flags |= IWL_TLC_MNG_CFG_FLAGS_STBC_MSK; + else if (ht_cap->cap & IEEE80211_HT_CAP_RX_STBC) flags |= IWL_TLC_MNG_CFG_FLAGS_STBC_MSK; } diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c index 558a0b2ef0fc..66faf7914bd8 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c +++ b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c @@ -17,10 +17,20 @@ #include "iwl-prph.h" #include "internal.h" +#define TRANS_CFG_MARKER BIT(0) +#define _IS_A(cfg, _struct) __builtin_types_compatible_p(typeof(cfg), \ + struct _struct) +extern int _invalid_type; +#define _TRANS_CFG_MARKER(cfg) \ + (__builtin_choose_expr(_IS_A(cfg, iwl_cfg_trans_params), \ + TRANS_CFG_MARKER, \ + __builtin_choose_expr(_IS_A(cfg, iwl_cfg), 0, _invalid_type))) +#define _ASSIGN_CFG(cfg) (_TRANS_CFG_MARKER(cfg) + (kernel_ulong_t)&(cfg)) + #define IWL_PCI_DEVICE(dev, subdev, cfg) \ .vendor = PCI_VENDOR_ID_INTEL, .device = (dev), \ .subvendor = PCI_ANY_ID, .subdevice = (subdev), \ - .driver_data = (kernel_ulong_t)&(cfg) + .driver_data = _ASSIGN_CFG(cfg) /* Hardware specific file defines the PCI IDs table for that hardware module */ static const struct pci_device_id iwl_hw_card_ids[] = { @@ -1075,19 +1085,22 @@ static const struct iwl_dev_info iwl_dev_info_table[] = { static int iwl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { - const struct iwl_cfg_trans_params *trans = - (struct iwl_cfg_trans_params *)(ent->driver_data); + const struct iwl_cfg_trans_params *trans; const struct iwl_cfg *cfg_7265d __maybe_unused = NULL; struct iwl_trans *iwl_trans; struct iwl_trans_pcie *trans_pcie; int i, ret; + const struct iwl_cfg *cfg; + + trans = (void *)(ent->driver_data & ~TRANS_CFG_MARKER); + /* * This is needed for backwards compatibility with the old * tables, so we don't need to change all the config structs * at the same time. The cfg is used to compare with the old * full cfg structs. */ - const struct iwl_cfg *cfg = (struct iwl_cfg *)(ent->driver_data); + cfg = (void *)(ent->driver_data & ~TRANS_CFG_MARKER); /* make sure trans is the first element in iwl_cfg */ BUILD_BUG_ON(offsetof(struct iwl_cfg, trans)); @@ -1202,11 +1215,19 @@ static int iwl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) #endif /* - * If we didn't set the cfg yet, assume the trans is actually - * a full cfg from the old tables. + * If we didn't set the cfg yet, the PCI ID table entry should have + * been a full config - if yes, use it, otherwise fail. */ - if (!iwl_trans->cfg) + if (!iwl_trans->cfg) { + if (ent->driver_data & TRANS_CFG_MARKER) { + pr_err("No config found for PCI dev %04x/%04x, rev=0x%x, rfid=0x%x\n", + pdev->device, pdev->subsystem_device, + iwl_trans->hw_rev, iwl_trans->hw_rf_id); + ret = -EINVAL; + goto out_free_trans; + } iwl_trans->cfg = cfg; + } /* if we don't have a name yet, copy name from the old cfg */ if (!iwl_trans->name) @@ -1222,6 +1243,10 @@ static int iwl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) trans_pcie->num_rx_bufs = RX_QUEUE_SIZE; } + ret = iwl_trans_init(iwl_trans); + if (ret) + goto out_free_trans; + pci_set_drvdata(pdev, iwl_trans); iwl_trans->drv = iwl_drv_start(iwl_trans); diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c b/drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c index 94ffc1ae484d..af9412bd697e 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c +++ b/drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* * Copyright (C) 2017 Intel Deutschland GmbH - * Copyright (C) 2018-2020 Intel Corporation + * Copyright (C) 2018-2021 Intel Corporation */ #include "iwl-trans.h" #include "iwl-prph.h" @@ -143,7 +143,7 @@ void _iwl_trans_pcie_gen2_stop_device(struct iwl_trans *trans) if (test_and_clear_bit(STATUS_DEVICE_ENABLED, &trans->status)) { IWL_DEBUG_INFO(trans, "DEVICE_ENABLED bit was set and is now cleared\n"); - iwl_txq_gen2_tx_stop(trans); + iwl_txq_gen2_tx_free(trans); iwl_pcie_rx_stop(trans); } diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/tx-gen2.c b/drivers/net/wireless/intel/iwlwifi/pcie/tx-gen2.c index 4456abb9a074..34bde8c87324 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/tx-gen2.c +++ b/drivers/net/wireless/intel/iwlwifi/pcie/tx-gen2.c @@ -40,6 +40,7 @@ int iwl_pcie_gen2_enqueue_hcmd(struct iwl_trans *trans, const u8 *cmddata[IWL_MAX_CMD_TBS_PER_TFD]; u16 cmdlen[IWL_MAX_CMD_TBS_PER_TFD]; struct iwl_tfh_tfd *tfd; + unsigned long flags; copy_size = sizeof(struct iwl_cmd_header_wide); cmd_size = sizeof(struct iwl_cmd_header_wide); @@ -108,14 +109,14 @@ int iwl_pcie_gen2_enqueue_hcmd(struct iwl_trans *trans, goto free_dup_buf; } - spin_lock_bh(&txq->lock); + spin_lock_irqsave(&txq->lock, flags); idx = iwl_txq_get_cmd_index(txq, txq->write_ptr); tfd = iwl_txq_get_tfd(trans, txq, txq->write_ptr); memset(tfd, 0, sizeof(*tfd)); if (iwl_txq_space(trans, txq) < ((cmd->flags & CMD_ASYNC) ? 2 : 1)) { - spin_unlock_bh(&txq->lock); + spin_unlock_irqrestore(&txq->lock, flags); IWL_ERR(trans, "No space in command queue\n"); iwl_op_mode_cmd_queue_full(trans->op_mode); @@ -250,7 +251,7 @@ int iwl_pcie_gen2_enqueue_hcmd(struct iwl_trans *trans, spin_unlock(&trans_pcie->reg_lock); out: - spin_unlock_bh(&txq->lock); + spin_unlock_irqrestore(&txq->lock, flags); free_dup_buf: if (idx < 0) kfree(dup_buf); diff --git a/drivers/net/wireless/intel/iwlwifi/queue/tx.c b/drivers/net/wireless/intel/iwlwifi/queue/tx.c index 833f43d1ca7a..810dcb3df242 100644 --- a/drivers/net/wireless/intel/iwlwifi/queue/tx.c +++ b/drivers/net/wireless/intel/iwlwifi/queue/tx.c @@ -13,30 +13,6 @@ #include "iwl-scd.h" #include -/* - * iwl_txq_gen2_tx_stop - Stop all Tx DMA channels - */ -void iwl_txq_gen2_tx_stop(struct iwl_trans *trans) -{ - int txq_id; - - /* - * This function can be called before the op_mode disabled the - * queues. This happens when we have an rfkill interrupt. - * Since we stop Tx altogether - mark the queues as stopped. - */ - memset(trans->txqs.queue_stopped, 0, - sizeof(trans->txqs.queue_stopped)); - memset(trans->txqs.queue_used, 0, sizeof(trans->txqs.queue_used)); - - /* Unmap DMA from host system and free skb's */ - for (txq_id = 0; txq_id < ARRAY_SIZE(trans->txqs.txq); txq_id++) { - if (!trans->txqs.txq[txq_id]) - continue; - iwl_txq_gen2_unmap(trans, txq_id); - } -} - /* * iwl_txq_update_byte_tbl - Set up entry in Tx byte-count array */ @@ -1189,6 +1165,12 @@ static int iwl_txq_alloc_response(struct iwl_trans *trans, struct iwl_txq *txq, goto error_free_resp; } + if (WARN_ONCE(trans->txqs.txq[qid], + "queue %d already allocated\n", qid)) { + ret = -EIO; + goto error_free_resp; + } + txq->id = qid; trans->txqs.txq[qid] = txq; wr_ptr &= (trans->trans_cfg->base_params->max_tfd_queue_size - 1); diff --git a/drivers/net/wireless/intel/iwlwifi/queue/tx.h b/drivers/net/wireless/intel/iwlwifi/queue/tx.h index af1dbdf5617a..20efc62acf13 100644 --- a/drivers/net/wireless/intel/iwlwifi/queue/tx.h +++ b/drivers/net/wireless/intel/iwlwifi/queue/tx.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ /* - * Copyright (C) 2020 Intel Corporation + * Copyright (C) 2020-2021 Intel Corporation */ #ifndef __iwl_trans_queue_tx_h__ #define __iwl_trans_queue_tx_h__ @@ -123,7 +123,6 @@ int iwl_txq_gen2_tx(struct iwl_trans *trans, struct sk_buff *skb, void iwl_txq_dyn_free(struct iwl_trans *trans, int queue); void iwl_txq_gen2_free_tfd(struct iwl_trans *trans, struct iwl_txq *txq); void iwl_txq_inc_wr_ptr(struct iwl_trans *trans, struct iwl_txq *txq); -void iwl_txq_gen2_tx_stop(struct iwl_trans *trans); void iwl_txq_gen2_tx_free(struct iwl_trans *trans); int iwl_txq_init(struct iwl_trans *trans, struct iwl_txq *txq, int slots_num, bool cmd_queue); diff --git a/drivers/net/wireless/marvell/mwl8k.c b/drivers/net/wireless/marvell/mwl8k.c index c9f8c056aa51..84b32a5f01ee 100644 --- a/drivers/net/wireless/marvell/mwl8k.c +++ b/drivers/net/wireless/marvell/mwl8k.c @@ -1473,6 +1473,7 @@ static int mwl8k_txq_init(struct ieee80211_hw *hw, int index) if (txq->skb == NULL) { dma_free_coherent(&priv->pdev->dev, size, txq->txd, txq->txd_dma); + txq->txd = NULL; return -ENOMEM; } diff --git a/drivers/net/wireless/mediatek/mt76/dma.c b/drivers/net/wireless/mediatek/mt76/dma.c index 2f27c43ad76d..7196fa9047e6 100644 --- a/drivers/net/wireless/mediatek/mt76/dma.c +++ b/drivers/net/wireless/mediatek/mt76/dma.c @@ -309,7 +309,7 @@ static int mt76_dma_tx_queue_skb_raw(struct mt76_dev *dev, struct mt76_queue *q, struct sk_buff *skb, u32 tx_info) { - struct mt76_queue_buf buf; + struct mt76_queue_buf buf = {}; dma_addr_t addr; if (q->queued + 1 >= q->ndesc - 1) diff --git a/drivers/net/wireless/mediatek/mt76/mt76.h b/drivers/net/wireless/mediatek/mt76/mt76.h index 8bf45497cfca..36a430f09f64 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76.h +++ b/drivers/net/wireless/mediatek/mt76/mt76.h @@ -222,6 +222,7 @@ struct mt76_wcid { u16 idx; u8 hw_key_idx; + u8 hw_key_idx2; u8 sta:1; u8 ext_phy:1; diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/eeprom.c b/drivers/net/wireless/mediatek/mt76/mt7615/eeprom.c index 2eab23898c77..6dbaaf95ee38 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/eeprom.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/eeprom.c @@ -86,6 +86,7 @@ static int mt7615_check_eeprom(struct mt76_dev *dev) switch (val) { case 0x7615: case 0x7622: + case 0x7663: return 0; default: return -EINVAL; diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/mac.c b/drivers/net/wireless/mediatek/mt76/mt7615/mac.c index 59fdd0fc2ad4..8dccb589b756 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/mac.c @@ -690,7 +690,7 @@ mt7615_txp_skb_unmap_fw(struct mt76_dev *dev, struct mt7615_fw_txp *txp) { int i; - for (i = 1; i < txp->nbuf; i++) + for (i = 0; i < txp->nbuf; i++) dma_unmap_single(dev->dev, le32_to_cpu(txp->buf[i]), le16_to_cpu(txp->len[i]), DMA_TO_DEVICE); } @@ -966,6 +966,7 @@ void mt7615_mac_set_rates(struct mt7615_phy *phy, struct mt7615_sta *sta, struct mt7615_dev *dev = phy->dev; struct mt7615_rate_desc rd; u32 w5, w27, addr; + u16 idx = sta->vif->mt76.omac_idx; if (!mt76_is_mmio(&dev->mt76)) { mt7615_mac_queue_rate_update(phy, sta, probe_rate, rates); @@ -1017,7 +1018,10 @@ void mt7615_mac_set_rates(struct mt7615_phy *phy, struct mt7615_sta *sta, mt76_wr(dev, addr + 27 * 4, w27); - mt76_set(dev, MT_LPON_T0CR, MT_LPON_T0CR_MODE); /* TSF read */ + idx = idx > HW_BSSID_MAX ? HW_BSSID_0 : idx; + addr = idx > 1 ? MT_LPON_TCR2(idx): MT_LPON_TCR0(idx); + + mt76_set(dev, addr, MT_LPON_TCR_MODE); /* TSF read */ sta->rate_set_tsf = mt76_rr(dev, MT_LPON_UTTR0) & ~BIT(0); sta->rate_set_tsf |= rd.rateset; @@ -1033,7 +1037,7 @@ EXPORT_SYMBOL_GPL(mt7615_mac_set_rates); static int mt7615_mac_wtbl_update_key(struct mt7615_dev *dev, struct mt76_wcid *wcid, struct ieee80211_key_conf *key, - enum mt7615_cipher_type cipher, + enum mt7615_cipher_type cipher, u16 cipher_mask, enum set_key_cmd cmd) { u32 addr = mt7615_mac_wtbl_addr(dev, wcid->idx) + 30 * 4; @@ -1050,22 +1054,22 @@ mt7615_mac_wtbl_update_key(struct mt7615_dev *dev, struct mt76_wcid *wcid, memcpy(data + 16, key->key + 24, 8); memcpy(data + 24, key->key + 16, 8); } else { - if (cipher != MT_CIPHER_BIP_CMAC_128 && wcid->cipher) - memmove(data + 16, data, 16); - if (cipher != MT_CIPHER_BIP_CMAC_128 || !wcid->cipher) + if (cipher_mask == BIT(cipher)) memcpy(data, key->key, key->keylen); - else if (cipher == MT_CIPHER_BIP_CMAC_128) + else if (cipher != MT_CIPHER_BIP_CMAC_128) + memcpy(data, key->key, 16); + if (cipher == MT_CIPHER_BIP_CMAC_128) memcpy(data + 16, key->key, 16); } } else { - if (wcid->cipher & ~BIT(cipher)) { - if (cipher != MT_CIPHER_BIP_CMAC_128) - memmove(data, data + 16, 16); + if (cipher == MT_CIPHER_BIP_CMAC_128) memset(data + 16, 0, 16); - } else { + else if (cipher_mask) + memset(data, 0, 16); + if (!cipher_mask) memset(data, 0, sizeof(data)); - } } + mt76_wr_copy(dev, addr, data, sizeof(data)); return 0; @@ -1073,7 +1077,7 @@ mt7615_mac_wtbl_update_key(struct mt7615_dev *dev, struct mt76_wcid *wcid, static int mt7615_mac_wtbl_update_pk(struct mt7615_dev *dev, struct mt76_wcid *wcid, - enum mt7615_cipher_type cipher, + enum mt7615_cipher_type cipher, u16 cipher_mask, int keyidx, enum set_key_cmd cmd) { u32 addr = mt7615_mac_wtbl_addr(dev, wcid->idx), w0, w1; @@ -1083,20 +1087,23 @@ mt7615_mac_wtbl_update_pk(struct mt7615_dev *dev, struct mt76_wcid *wcid, w0 = mt76_rr(dev, addr); w1 = mt76_rr(dev, addr + 4); - if (cmd == SET_KEY) { - w0 |= MT_WTBL_W0_RX_KEY_VALID | - FIELD_PREP(MT_WTBL_W0_RX_IK_VALID, - cipher == MT_CIPHER_BIP_CMAC_128); - if (cipher != MT_CIPHER_BIP_CMAC_128 || - !wcid->cipher) - w0 |= FIELD_PREP(MT_WTBL_W0_KEY_IDX, keyidx); - } else { - if (!(wcid->cipher & ~BIT(cipher))) - w0 &= ~(MT_WTBL_W0_RX_KEY_VALID | - MT_WTBL_W0_KEY_IDX); - if (cipher == MT_CIPHER_BIP_CMAC_128) - w0 &= ~MT_WTBL_W0_RX_IK_VALID; + + if (cipher_mask) + w0 |= MT_WTBL_W0_RX_KEY_VALID; + else + w0 &= ~(MT_WTBL_W0_RX_KEY_VALID | MT_WTBL_W0_KEY_IDX); + if (cipher_mask & BIT(MT_CIPHER_BIP_CMAC_128)) + w0 |= MT_WTBL_W0_RX_IK_VALID; + else + w0 &= ~MT_WTBL_W0_RX_IK_VALID; + + if (cmd == SET_KEY && + (cipher != MT_CIPHER_BIP_CMAC_128 || + cipher_mask == BIT(cipher))) { + w0 &= ~MT_WTBL_W0_KEY_IDX; + w0 |= FIELD_PREP(MT_WTBL_W0_KEY_IDX, keyidx); } + mt76_wr(dev, MT_WTBL_RICR0, w0); mt76_wr(dev, MT_WTBL_RICR1, w1); @@ -1109,24 +1116,25 @@ mt7615_mac_wtbl_update_pk(struct mt7615_dev *dev, struct mt76_wcid *wcid, static void mt7615_mac_wtbl_update_cipher(struct mt7615_dev *dev, struct mt76_wcid *wcid, - enum mt7615_cipher_type cipher, + enum mt7615_cipher_type cipher, u16 cipher_mask, enum set_key_cmd cmd) { u32 addr = mt7615_mac_wtbl_addr(dev, wcid->idx); - if (cmd == SET_KEY) { - if (cipher != MT_CIPHER_BIP_CMAC_128 || !wcid->cipher) - mt76_rmw(dev, addr + 2 * 4, MT_WTBL_W2_KEY_TYPE, - FIELD_PREP(MT_WTBL_W2_KEY_TYPE, cipher)); - } else { - if (cipher != MT_CIPHER_BIP_CMAC_128 && - wcid->cipher & BIT(MT_CIPHER_BIP_CMAC_128)) - mt76_rmw(dev, addr + 2 * 4, MT_WTBL_W2_KEY_TYPE, - FIELD_PREP(MT_WTBL_W2_KEY_TYPE, - MT_CIPHER_BIP_CMAC_128)); - else if (!(wcid->cipher & ~BIT(cipher))) - mt76_clear(dev, addr + 2 * 4, MT_WTBL_W2_KEY_TYPE); + if (!cipher_mask) { + mt76_clear(dev, addr + 2 * 4, MT_WTBL_W2_KEY_TYPE); + return; } + + if (cmd != SET_KEY) + return; + + if (cipher == MT_CIPHER_BIP_CMAC_128 && + cipher_mask & ~BIT(MT_CIPHER_BIP_CMAC_128)) + return; + + mt76_rmw(dev, addr + 2 * 4, MT_WTBL_W2_KEY_TYPE, + FIELD_PREP(MT_WTBL_W2_KEY_TYPE, cipher)); } int __mt7615_mac_wtbl_set_key(struct mt7615_dev *dev, @@ -1135,25 +1143,30 @@ int __mt7615_mac_wtbl_set_key(struct mt7615_dev *dev, enum set_key_cmd cmd) { enum mt7615_cipher_type cipher; + u16 cipher_mask = wcid->cipher; int err; cipher = mt7615_mac_get_cipher(key->cipher); if (cipher == MT_CIPHER_NONE) return -EOPNOTSUPP; - mt7615_mac_wtbl_update_cipher(dev, wcid, cipher, cmd); - err = mt7615_mac_wtbl_update_key(dev, wcid, key, cipher, cmd); + if (cmd == SET_KEY) + cipher_mask |= BIT(cipher); + else + cipher_mask &= ~BIT(cipher); + + mt7615_mac_wtbl_update_cipher(dev, wcid, cipher, cipher_mask, cmd); + err = mt7615_mac_wtbl_update_key(dev, wcid, key, cipher, cipher_mask, + cmd); if (err < 0) return err; - err = mt7615_mac_wtbl_update_pk(dev, wcid, cipher, key->keyidx, cmd); + err = mt7615_mac_wtbl_update_pk(dev, wcid, cipher, cipher_mask, + key->keyidx, cmd); if (err < 0) return err; - if (cmd == SET_KEY) - wcid->cipher |= BIT(cipher); - else - wcid->cipher &= ~BIT(cipher); + wcid->cipher = cipher_mask; return 0; } @@ -1821,10 +1834,8 @@ mt7615_mac_update_mib_stats(struct mt7615_phy *phy) int i, aggr; u32 val, val2; - memset(mib, 0, sizeof(*mib)); - - mib->fcs_err_cnt = mt76_get_field(dev, MT_MIB_SDR3(ext_phy), - MT_MIB_SDR3_FCS_ERR_MASK); + mib->fcs_err_cnt += mt76_get_field(dev, MT_MIB_SDR3(ext_phy), + MT_MIB_SDR3_FCS_ERR_MASK); val = mt76_get_field(dev, MT_MIB_SDR14(ext_phy), MT_MIB_AMPDU_MPDU_COUNT); @@ -1837,24 +1848,16 @@ mt7615_mac_update_mib_stats(struct mt7615_phy *phy) aggr = ext_phy ? ARRAY_SIZE(dev->mt76.aggr_stats) / 2 : 0; for (i = 0; i < 4; i++) { val = mt76_rr(dev, MT_MIB_MB_SDR1(ext_phy, i)); - - val2 = FIELD_GET(MT_MIB_ACK_FAIL_COUNT_MASK, val); - if (val2 > mib->ack_fail_cnt) - mib->ack_fail_cnt = val2; - - val2 = FIELD_GET(MT_MIB_BA_MISS_COUNT_MASK, val); - if (val2 > mib->ba_miss_cnt) - mib->ba_miss_cnt = val2; + mib->ba_miss_cnt += FIELD_GET(MT_MIB_BA_MISS_COUNT_MASK, val); + mib->ack_fail_cnt += FIELD_GET(MT_MIB_ACK_FAIL_COUNT_MASK, + val); val = mt76_rr(dev, MT_MIB_MB_SDR0(ext_phy, i)); - val2 = FIELD_GET(MT_MIB_RTS_RETRIES_COUNT_MASK, val); - if (val2 > mib->rts_retries_cnt) { - mib->rts_cnt = FIELD_GET(MT_MIB_RTS_COUNT_MASK, val); - mib->rts_retries_cnt = val2; - } + mib->rts_cnt += FIELD_GET(MT_MIB_RTS_COUNT_MASK, val); + mib->rts_retries_cnt += FIELD_GET(MT_MIB_RTS_RETRIES_COUNT_MASK, + val); val = mt76_rr(dev, MT_TX_AGG_CNT(ext_phy, i)); - dev->mt76.aggr_stats[aggr++] += val & 0xffff; dev->mt76.aggr_stats[aggr++] += val >> 16; } @@ -1976,15 +1979,17 @@ void mt7615_dma_reset(struct mt7615_dev *dev) mt76_clear(dev, MT_WPDMA_GLO_CFG, MT_WPDMA_GLO_CFG_RX_DMA_EN | MT_WPDMA_GLO_CFG_TX_DMA_EN | MT_WPDMA_GLO_CFG_TX_WRITEBACK_DONE); + usleep_range(1000, 2000); - mt76_queue_tx_cleanup(dev, dev->mt76.q_mcu[MT_MCUQ_WM], true); for (i = 0; i < __MT_TXQ_MAX; i++) mt76_queue_tx_cleanup(dev, dev->mphy.q_tx[i], true); - mt76_for_each_q_rx(&dev->mt76, i) { + for (i = 0; i < __MT_MCUQ_MAX; i++) + mt76_queue_tx_cleanup(dev, dev->mt76.q_mcu[i], true); + + mt76_for_each_q_rx(&dev->mt76, i) mt76_queue_rx_reset(dev, i); - } mt76_set(dev, MT_WPDMA_GLO_CFG, MT_WPDMA_GLO_CFG_RX_DMA_EN | MT_WPDMA_GLO_CFG_TX_DMA_EN | @@ -2000,8 +2005,12 @@ void mt7615_tx_token_put(struct mt7615_dev *dev) spin_lock_bh(&dev->token_lock); idr_for_each_entry(&dev->token, txwi, id) { mt7615_txp_skb_unmap(&dev->mt76, txwi); - if (txwi->skb) - dev_kfree_skb_any(txwi->skb); + if (txwi->skb) { + struct ieee80211_hw *hw; + + hw = mt76_tx_status_get_hw(&dev->mt76, txwi->skb); + ieee80211_free_txskb(hw, txwi->skb); + } mt76_put_txwi(&dev->mt76, txwi); } spin_unlock_bh(&dev->token_lock); @@ -2304,8 +2313,10 @@ void mt7615_coredump_work(struct work_struct *work) break; skb_pull(skb, sizeof(struct mt7615_mcu_rxd)); - if (data + skb->len - dump > MT76_CONNAC_COREDUMP_SZ) - break; + if (data + skb->len - dump > MT76_CONNAC_COREDUMP_SZ) { + dev_kfree_skb(skb); + continue; + } memcpy(data, skb->data, skb->len); data += skb->len; diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/main.c b/drivers/net/wireless/mediatek/mt76/mt7615/main.c index 25faf486d279..d334491667a4 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/main.c @@ -217,8 +217,6 @@ static int mt7615_add_interface(struct ieee80211_hw *hw, ret = mt7615_mcu_add_dev_info(phy, vif, true); if (ret) goto out; - - mt7615_mac_set_beacon_filter(phy, vif, true); out: mt7615_mutex_release(dev); @@ -244,7 +242,6 @@ static void mt7615_remove_interface(struct ieee80211_hw *hw, mt76_connac_free_pending_tx_skbs(&dev->pm, &msta->wcid); - mt7615_mac_set_beacon_filter(phy, vif, false); mt7615_mcu_add_dev_info(phy, vif, false); rcu_assign_pointer(dev->mt76.wcid[idx], NULL); @@ -337,7 +334,8 @@ static int mt7615_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, struct mt7615_sta *msta = sta ? (struct mt7615_sta *)sta->drv_priv : &mvif->sta; struct mt76_wcid *wcid = &msta->wcid; - int idx = key->keyidx, err; + int idx = key->keyidx, err = 0; + u8 *wcid_keyidx = &wcid->hw_key_idx; /* The hardware does not support per-STA RX GTK, fallback * to software mode for these. @@ -352,6 +350,7 @@ static int mt7615_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, /* fall back to sw encryption for unsupported ciphers */ switch (key->cipher) { case WLAN_CIPHER_SUITE_AES_CMAC: + wcid_keyidx = &wcid->hw_key_idx2; key->flags |= IEEE80211_KEY_FLAG_GENERATE_MMIE; break; case WLAN_CIPHER_SUITE_TKIP: @@ -369,12 +368,13 @@ static int mt7615_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, mt7615_mutex_acquire(dev); - if (cmd == SET_KEY) { - key->hw_key_idx = wcid->idx; - wcid->hw_key_idx = idx; - } else if (idx == wcid->hw_key_idx) { - wcid->hw_key_idx = -1; - } + if (cmd == SET_KEY) + *wcid_keyidx = idx; + else if (idx == *wcid_keyidx) + *wcid_keyidx = -1; + else + goto out; + mt76_wcid_key_setup(&dev->mt76, wcid, cmd == SET_KEY ? key : NULL); @@ -383,6 +383,7 @@ static int mt7615_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, else err = __mt7615_mac_wtbl_set_key(dev, wcid, key, cmd); +out: mt7615_mutex_release(dev); return err; @@ -544,6 +545,9 @@ static void mt7615_bss_info_changed(struct ieee80211_hw *hw, if (changed & BSS_CHANGED_ARP_FILTER) mt7615_mcu_update_arp_filter(hw, vif, info); + if (changed & BSS_CHANGED_ASSOC) + mt7615_mac_set_beacon_filter(phy, vif, info->assoc); + mt7615_mutex_release(dev); } @@ -803,26 +807,38 @@ mt7615_get_stats(struct ieee80211_hw *hw, struct mt7615_phy *phy = mt7615_hw_phy(hw); struct mib_stats *mib = &phy->mib; + mt7615_mutex_acquire(phy->dev); + stats->dot11RTSSuccessCount = mib->rts_cnt; stats->dot11RTSFailureCount = mib->rts_retries_cnt; stats->dot11FCSErrorCount = mib->fcs_err_cnt; stats->dot11ACKFailureCount = mib->ack_fail_cnt; + memset(mib, 0, sizeof(*mib)); + + mt7615_mutex_release(phy->dev); + return 0; } static u64 mt7615_get_tsf(struct ieee80211_hw *hw, struct ieee80211_vif *vif) { + struct mt7615_vif *mvif = (struct mt7615_vif *)vif->drv_priv; struct mt7615_dev *dev = mt7615_hw_dev(hw); union { u64 t64; u32 t32[2]; } tsf; + u16 idx = mvif->mt76.omac_idx; + u32 reg; + + idx = idx > HW_BSSID_MAX ? HW_BSSID_0 : idx; + reg = idx > 1 ? MT_LPON_TCR2(idx): MT_LPON_TCR0(idx); mt7615_mutex_acquire(dev); - mt76_set(dev, MT_LPON_T0CR, MT_LPON_T0CR_MODE); /* TSF read */ + mt76_set(dev, reg, MT_LPON_TCR_MODE); /* TSF read */ tsf.t32[0] = mt76_rr(dev, MT_LPON_UTTR0); tsf.t32[1] = mt76_rr(dev, MT_LPON_UTTR1); @@ -835,18 +851,24 @@ static void mt7615_set_tsf(struct ieee80211_hw *hw, struct ieee80211_vif *vif, u64 timestamp) { + struct mt7615_vif *mvif = (struct mt7615_vif *)vif->drv_priv; struct mt7615_dev *dev = mt7615_hw_dev(hw); union { u64 t64; u32 t32[2]; } tsf = { .t64 = timestamp, }; + u16 idx = mvif->mt76.omac_idx; + u32 reg; + + idx = idx > HW_BSSID_MAX ? HW_BSSID_0 : idx; + reg = idx > 1 ? MT_LPON_TCR2(idx): MT_LPON_TCR0(idx); mt7615_mutex_acquire(dev); mt76_wr(dev, MT_LPON_UTTR0, tsf.t32[0]); mt76_wr(dev, MT_LPON_UTTR1, tsf.t32[1]); /* TSF software overwrite */ - mt76_set(dev, MT_LPON_T0CR, MT_LPON_T0CR_WRITE); + mt76_set(dev, reg, MT_LPON_TCR_WRITE); mt7615_mutex_release(dev); } diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c index 631596fc2f36..198e9025b681 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c @@ -291,12 +291,20 @@ static int mt7615_mcu_drv_pmctrl(struct mt7615_dev *dev) u32 addr; int err; - addr = is_mt7663(mdev) ? MT_PCIE_DOORBELL_PUSH : MT_CFG_LPCR_HOST; + if (is_mt7663(mdev)) { + /* Clear firmware own via N9 eint */ + mt76_wr(dev, MT_PCIE_DOORBELL_PUSH, MT_CFG_LPCR_HOST_DRV_OWN); + mt76_poll(dev, MT_CONN_ON_MISC, MT_CFG_LPCR_HOST_FW_OWN, 0, 3000); + + addr = MT_CONN_HIF_ON_LPCTL; + } else { + addr = MT_CFG_LPCR_HOST; + } + mt76_wr(dev, addr, MT_CFG_LPCR_HOST_DRV_OWN); mt7622_trigger_hif_int(dev, true); - addr = is_mt7663(mdev) ? MT_CONN_HIF_ON_LPCTL : MT_CFG_LPCR_HOST; err = !mt76_poll_msec(dev, addr, MT_CFG_LPCR_HOST_FW_OWN, 0, 3000); mt7622_trigger_hif_int(dev, false); @@ -1040,6 +1048,9 @@ mt7615_mcu_sta_ba(struct mt7615_dev *dev, wtbl_hdr = mt76_connac_mcu_alloc_wtbl_req(&dev->mt76, &msta->wcid, WTBL_SET, sta_wtbl, &skb); + if (IS_ERR(wtbl_hdr)) + return PTR_ERR(wtbl_hdr); + mt76_connac_mcu_wtbl_ba_tlv(&dev->mt76, skb, params, enable, tx, sta_wtbl, wtbl_hdr); diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/mt7615.h b/drivers/net/wireless/mediatek/mt76/mt7615/mt7615.h index 491841bc6291..4bc0c379c579 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/mt7615.h +++ b/drivers/net/wireless/mediatek/mt76/mt7615/mt7615.h @@ -133,11 +133,11 @@ struct mt7615_vif { }; struct mib_stats { - u16 ack_fail_cnt; - u16 fcs_err_cnt; - u16 rts_cnt; - u16 rts_retries_cnt; - u16 ba_miss_cnt; + u32 ack_fail_cnt; + u32 fcs_err_cnt; + u32 rts_cnt; + u32 rts_retries_cnt; + u32 ba_miss_cnt; unsigned long aggr_per; }; diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/pci_init.c b/drivers/net/wireless/mediatek/mt76/mt7615/pci_init.c index 72395925ddee..15b417d6d889 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/pci_init.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/pci_init.c @@ -163,10 +163,9 @@ void mt7615_unregister_device(struct mt7615_dev *dev) mt76_unregister_device(&dev->mt76); if (mcu_running) mt7615_mcu_exit(dev); - mt7615_dma_cleanup(dev); mt7615_tx_token_put(dev); - + mt7615_dma_cleanup(dev); tasklet_disable(&dev->irq_tasklet); mt76_free_device(&dev->mt76); diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/regs.h b/drivers/net/wireless/mediatek/mt76/mt7615/regs.h index 6e5db015b32c..6e4710d3ddd3 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/regs.h +++ b/drivers/net/wireless/mediatek/mt76/mt7615/regs.h @@ -447,9 +447,10 @@ enum mt7615_reg_base { #define MT_LPON(_n) ((dev)->reg_map[MT_LPON_BASE] + (_n)) -#define MT_LPON_T0CR MT_LPON(0x010) -#define MT_LPON_T0CR_MODE GENMASK(1, 0) -#define MT_LPON_T0CR_WRITE BIT(0) +#define MT_LPON_TCR0(_n) MT_LPON(0x010 + ((_n) * 4)) +#define MT_LPON_TCR2(_n) MT_LPON(0x0f8 + ((_n) - 2) * 4) +#define MT_LPON_TCR_MODE GENMASK(1, 0) +#define MT_LPON_TCR_WRITE BIT(0) #define MT_LPON_UTTR0 MT_LPON(0x018) #define MT_LPON_UTTR1 MT_LPON(0x01c) diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/sdio_txrx.c b/drivers/net/wireless/mediatek/mt76/mt7615/sdio_txrx.c index 9fb506f2ace6..4393dd21ebbb 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/sdio_txrx.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/sdio_txrx.c @@ -218,12 +218,15 @@ static int mt7663s_tx_run_queue(struct mt76_dev *dev, struct mt76_queue *q) int qid, err, nframes = 0, len = 0, pse_sz = 0, ple_sz = 0; bool mcu = q == dev->q_mcu[MT_MCUQ_WM]; struct mt76_sdio *sdio = &dev->sdio; + u8 pad; qid = mcu ? ARRAY_SIZE(sdio->xmit_buf) - 1 : q->qid; while (q->first != q->head) { struct mt76_queue_entry *e = &q->entry[q->first]; struct sk_buff *iter; + smp_rmb(); + if (!test_bit(MT76_STATE_MCU_RUNNING, &dev->phy.state)) { __skb_put_zero(e->skb, 4); err = __mt7663s_xmit_queue(dev, e->skb->data, @@ -234,7 +237,8 @@ static int mt7663s_tx_run_queue(struct mt76_dev *dev, struct mt76_queue *q) goto next; } - if (len + e->skb->len + 4 > MT76S_XMIT_BUF_SZ) + pad = roundup(e->skb->len, 4) - e->skb->len; + if (len + e->skb->len + pad + 4 > MT76S_XMIT_BUF_SZ) break; if (mt7663s_tx_pick_quota(sdio, mcu, e->buf_sz, &pse_sz, @@ -252,6 +256,11 @@ static int mt7663s_tx_run_queue(struct mt76_dev *dev, struct mt76_queue *q) len += iter->len; nframes++; } + + if (unlikely(pad)) { + memset(sdio->xmit_buf[qid] + len, 0, pad); + len += pad; + } next: q->first = (q->first + 1) % q->ndesc; e->done = true; diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/usb_sdio.c b/drivers/net/wireless/mediatek/mt76/mt7615/usb_sdio.c index 203256862dfd..f8d3673c2cae 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/usb_sdio.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/usb_sdio.c @@ -67,6 +67,7 @@ static int mt7663_usb_sdio_set_rates(struct mt7615_dev *dev, struct mt7615_rate_desc *rate = &wrd->rate; struct mt7615_sta *sta = wrd->sta; u32 w5, w27, addr, val; + u16 idx; lockdep_assert_held(&dev->mt76.mutex); @@ -118,7 +119,11 @@ static int mt7663_usb_sdio_set_rates(struct mt7615_dev *dev, sta->rate_probe = sta->rateset[rate->rateset].probe_rate.idx != -1; - mt76_set(dev, MT_LPON_T0CR, MT_LPON_T0CR_MODE); /* TSF read */ + idx = sta->vif->mt76.omac_idx; + idx = idx > HW_BSSID_MAX ? HW_BSSID_0 : idx; + addr = idx > 1 ? MT_LPON_TCR2(idx): MT_LPON_TCR0(idx); + + mt76_set(dev, addr, MT_LPON_TCR_MODE); /* TSF read */ val = mt76_rr(dev, MT_LPON_UTTR0); sta->rate_set_tsf = (val & ~BIT(0)) | rate->rateset; diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c index 6cbccfb05f8b..cefd33b74a87 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c @@ -833,6 +833,9 @@ int mt76_connac_mcu_add_sta_cmd(struct mt76_phy *phy, wtbl_hdr = mt76_connac_mcu_alloc_wtbl_req(dev, wcid, WTBL_RESET_AND_SET, sta_wtbl, &skb); + if (IS_ERR(wtbl_hdr)) + return PTR_ERR(wtbl_hdr); + if (enable) { mt76_connac_mcu_wtbl_generic_tlv(dev, skb, vif, sta, sta_wtbl, wtbl_hdr); @@ -946,6 +949,7 @@ int mt76_connac_mcu_uni_add_dev(struct mt76_phy *phy, switch (vif->type) { case NL80211_IFTYPE_MESH_POINT: + case NL80211_IFTYPE_MONITOR: case NL80211_IFTYPE_AP: basic_req.basic.conn_type = cpu_to_le32(CONNECTION_INFRA_AP); break; @@ -1195,6 +1199,7 @@ int mt76_connac_mcu_uni_add_bss(struct mt76_phy *phy, .center_chan = ieee80211_frequency_to_channel(freq1), .center_chan2 = ieee80211_frequency_to_channel(freq2), .tx_streams = hweight8(phy->antenna_mask), + .ht_op_info = 4, /* set HT 40M allowed */ .rx_streams = phy->chainmask, .short_st = true, }, @@ -1287,6 +1292,7 @@ int mt76_connac_mcu_uni_add_bss(struct mt76_phy *phy, case NL80211_CHAN_WIDTH_20: default: rlm_req.rlm.bw = CMD_CBW_20MHZ; + rlm_req.rlm.ht_op_info = 0; break; } @@ -1306,7 +1312,7 @@ int mt76_connac_mcu_hw_scan(struct mt76_phy *phy, struct ieee80211_vif *vif, { struct mt76_vif *mvif = (struct mt76_vif *)vif->drv_priv; struct cfg80211_scan_request *sreq = &scan_req->req; - int n_ssids = 0, err, i, duration = MT76_CONNAC_SCAN_CHANNEL_TIME; + int n_ssids = 0, err, i, duration; int ext_channels_num = max_t(int, sreq->n_channels - 32, 0); struct ieee80211_channel **scan_list = sreq->channels; struct mt76_dev *mdev = phy->dev; @@ -1343,6 +1349,7 @@ int mt76_connac_mcu_hw_scan(struct mt76_phy *phy, struct ieee80211_vif *vif, req->ssid_type_ext = n_ssids ? BIT(0) : 0; req->ssids_num = n_ssids; + duration = is_mt7921(phy->dev) ? 0 : MT76_CONNAC_SCAN_CHANNEL_TIME; /* increase channel time for passive scan */ if (!sreq->n_ssids) duration *= 2; diff --git a/drivers/net/wireless/mediatek/mt76/mt76x02_util.c b/drivers/net/wireless/mediatek/mt76/mt76x02_util.c index ab671e21f882..02db5d66735d 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x02_util.c +++ b/drivers/net/wireless/mediatek/mt76/mt76x02_util.c @@ -447,6 +447,10 @@ int mt76x02_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, !(key->flags & IEEE80211_KEY_FLAG_PAIRWISE)) return -EOPNOTSUPP; + /* MT76x0 GTK offloading does not work with more than one VIF */ + if (is_mt76x0(dev) && !(key->flags & IEEE80211_KEY_FLAG_PAIRWISE)) + return -EOPNOTSUPP; + msta = sta ? (struct mt76x02_sta *)sta->drv_priv : NULL; wcid = msta ? &msta->wcid : &mvif->group_wcid; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/debugfs.c b/drivers/net/wireless/mediatek/mt76/mt7915/debugfs.c index 77dcd71e49a5..2f706620686e 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/debugfs.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/debugfs.c @@ -124,7 +124,7 @@ mt7915_ampdu_stat_read_phy(struct mt7915_phy *phy, range[i] = mt76_rr(dev, MT_MIB_ARNG(ext_phy, i)); for (i = 0; i < ARRAY_SIZE(bound); i++) - bound[i] = MT_MIB_ARNCR_RANGE(range[i / 4], i) + 1; + bound[i] = MT_MIB_ARNCR_RANGE(range[i / 4], i % 4) + 1; seq_printf(file, "\nPhy %d\n", ext_phy); diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.c b/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.c index 660398ac53c2..738ecf8f4fa2 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.c @@ -124,7 +124,7 @@ int mt7915_eeprom_get_target_power(struct mt7915_dev *dev, struct ieee80211_channel *chan, u8 chain_idx) { - int index; + int index, target_power; bool tssi_on; if (chain_idx > 3) @@ -133,15 +133,22 @@ int mt7915_eeprom_get_target_power(struct mt7915_dev *dev, tssi_on = mt7915_tssi_enabled(dev, chan->band); if (chan->band == NL80211_BAND_2GHZ) { - index = MT_EE_TX0_POWER_2G + chain_idx * 3 + !tssi_on; + index = MT_EE_TX0_POWER_2G + chain_idx * 3; + target_power = mt7915_eeprom_read(dev, index); + + if (!tssi_on) + target_power += mt7915_eeprom_read(dev, index + 1); } else { - int group = tssi_on ? - mt7915_get_channel_group(chan->hw_value) : 8; + int group = mt7915_get_channel_group(chan->hw_value); + + index = MT_EE_TX0_POWER_5G + chain_idx * 12; + target_power = mt7915_eeprom_read(dev, index + group); - index = MT_EE_TX0_POWER_5G + chain_idx * 12 + group; + if (!tssi_on) + target_power += mt7915_eeprom_read(dev, index + 8); } - return mt7915_eeprom_read(dev, index); + return target_power; } static const u8 sku_cck_delta_map[] = { diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/init.c b/drivers/net/wireless/mediatek/mt76/mt7915/init.c index ad4e5b95158b..c7d4268d860a 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/init.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/init.c @@ -4,6 +4,7 @@ #include #include "mt7915.h" #include "mac.h" +#include "mcu.h" #include "eeprom.h" #define CCK_RATE(_idx, _rate) { \ @@ -283,9 +284,50 @@ static void mt7915_init_work(struct work_struct *work) mt7915_register_ext_phy(dev); } +static void mt7915_wfsys_reset(struct mt7915_dev *dev) +{ + u32 val = MT_TOP_PWR_KEY | MT_TOP_PWR_SW_PWR_ON | MT_TOP_PWR_PWR_ON; + u32 reg = mt7915_reg_map_l1(dev, MT_TOP_MISC); + +#define MT_MCU_DUMMY_RANDOM GENMASK(15, 0) +#define MT_MCU_DUMMY_DEFAULT GENMASK(31, 16) + + mt76_wr(dev, MT_MCU_WFDMA0_DUMMY_CR, MT_MCU_DUMMY_RANDOM); + + /* change to software control */ + val |= MT_TOP_PWR_SW_RST; + mt76_wr(dev, MT_TOP_PWR_CTRL, val); + + /* reset wfsys */ + val &= ~MT_TOP_PWR_SW_RST; + mt76_wr(dev, MT_TOP_PWR_CTRL, val); + + /* release wfsys then mcu re-excutes romcode */ + val |= MT_TOP_PWR_SW_RST; + mt76_wr(dev, MT_TOP_PWR_CTRL, val); + + /* switch to hw control */ + val &= ~MT_TOP_PWR_SW_RST; + val |= MT_TOP_PWR_HW_CTRL; + mt76_wr(dev, MT_TOP_PWR_CTRL, val); + + /* check whether mcu resets to default */ + if (!mt76_poll_msec(dev, MT_MCU_WFDMA0_DUMMY_CR, MT_MCU_DUMMY_DEFAULT, + MT_MCU_DUMMY_DEFAULT, 1000)) { + dev_err(dev->mt76.dev, "wifi subsystem reset failure\n"); + return; + } + + /* wfsys reset won't clear host registers */ + mt76_clear(dev, reg, MT_TOP_MISC_FW_STATE); + + msleep(100); +} + static int mt7915_init_hardware(struct mt7915_dev *dev) { int ret, idx; + u32 val; mt76_wr(dev, MT_INT_SOURCE_CSR, ~0); @@ -295,6 +337,12 @@ static int mt7915_init_hardware(struct mt7915_dev *dev) dev->dbdc_support = !!(mt7915_l1_rr(dev, MT_HW_BOUND) & BIT(5)); + val = mt76_rr(dev, mt7915_reg_map_l1(dev, MT_TOP_MISC)); + + /* If MCU was already running, it is likely in a bad state */ + if (FIELD_GET(MT_TOP_MISC_FW_STATE, val) > FW_STATE_FW_DOWNLOAD) + mt7915_wfsys_reset(dev); + ret = mt7915_dma_init(dev); if (ret) return ret; @@ -308,8 +356,14 @@ static int mt7915_init_hardware(struct mt7915_dev *dev) mt76_wr(dev, MT_SWDEF_MODE, MT_SWDEF_NORMAL_MODE); ret = mt7915_mcu_init(dev); - if (ret) - return ret; + if (ret) { + /* Reset and try again */ + mt7915_wfsys_reset(dev); + + ret = mt7915_mcu_init(dev); + if (ret) + return ret; + } ret = mt7915_eeprom_init(dev); if (ret < 0) @@ -675,9 +729,8 @@ void mt7915_unregister_device(struct mt7915_dev *dev) mt7915_unregister_ext_phy(dev); mt76_unregister_device(&dev->mt76); mt7915_mcu_exit(dev); - mt7915_dma_cleanup(dev); - mt7915_tx_token_put(dev); + mt7915_dma_cleanup(dev); mt76_free_device(&dev->mt76); } diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mac.c b/drivers/net/wireless/mediatek/mt76/mt7915/mac.c index e5a258958ac9..819670767521 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mac.c @@ -1091,7 +1091,7 @@ void mt7915_txp_skb_unmap(struct mt76_dev *dev, int i; txp = mt7915_txwi_to_txp(dev, t); - for (i = 1; i < txp->nbuf; i++) + for (i = 0; i < txp->nbuf; i++) dma_unmap_single(dev->dev, le32_to_cpu(txp->buf[i]), le16_to_cpu(txp->len[i]), DMA_TO_DEVICE); } @@ -1470,9 +1470,8 @@ mt7915_update_beacons(struct mt7915_dev *dev) } static void -mt7915_dma_reset(struct mt7915_phy *phy) +mt7915_dma_reset(struct mt7915_dev *dev) { - struct mt7915_dev *dev = phy->dev; struct mt76_phy *mphy_ext = dev->mt76.phy2; u32 hif1_ofs = MT_WFDMA1_PCIE1_BASE - MT_WFDMA1_BASE; int i; @@ -1489,18 +1488,20 @@ mt7915_dma_reset(struct mt7915_phy *phy) (MT_WFDMA1_GLO_CFG_TX_DMA_EN | MT_WFDMA1_GLO_CFG_RX_DMA_EN)); } + usleep_range(1000, 2000); - mt76_queue_tx_cleanup(dev, dev->mt76.q_mcu[MT_MCUQ_WA], true); for (i = 0; i < __MT_TXQ_MAX; i++) { - mt76_queue_tx_cleanup(dev, phy->mt76->q_tx[i], true); + mt76_queue_tx_cleanup(dev, dev->mphy.q_tx[i], true); if (mphy_ext) mt76_queue_tx_cleanup(dev, mphy_ext->q_tx[i], true); } - mt76_for_each_q_rx(&dev->mt76, i) { + for (i = 0; i < __MT_MCUQ_MAX; i++) + mt76_queue_tx_cleanup(dev, dev->mt76.q_mcu[i], true); + + mt76_for_each_q_rx(&dev->mt76, i) mt76_queue_rx_reset(dev, i); - } /* re-init prefetch settings after reset */ mt7915_dma_prefetch(dev); @@ -1584,7 +1585,7 @@ void mt7915_mac_reset_work(struct work_struct *work) idr_init(&dev->token); if (mt7915_wait_reset_state(dev, MT_MCU_CMD_RESET_DONE)) { - mt7915_dma_reset(&dev->phy); + mt7915_dma_reset(dev); mt76_wr(dev, MT_MCU_INT_EVENT, MT_MCU_INT_EVENT_DMA_INIT); mt7915_wait_reset_state(dev, MT_MCU_CMD_RECOVERY_DONE); @@ -1633,39 +1634,30 @@ mt7915_mac_update_mib_stats(struct mt7915_phy *phy) bool ext_phy = phy != &dev->phy; int i, aggr0, aggr1; - memset(mib, 0, sizeof(*mib)); - - mib->fcs_err_cnt = mt76_get_field(dev, MT_MIB_SDR3(ext_phy), - MT_MIB_SDR3_FCS_ERR_MASK); + mib->fcs_err_cnt += mt76_get_field(dev, MT_MIB_SDR3(ext_phy), + MT_MIB_SDR3_FCS_ERR_MASK); aggr0 = ext_phy ? ARRAY_SIZE(dev->mt76.aggr_stats) / 2 : 0; for (i = 0, aggr1 = aggr0 + 4; i < 4; i++) { - u32 val, val2; + u32 val; val = mt76_rr(dev, MT_MIB_MB_SDR1(ext_phy, i)); - - val2 = FIELD_GET(MT_MIB_ACK_FAIL_COUNT_MASK, val); - if (val2 > mib->ack_fail_cnt) - mib->ack_fail_cnt = val2; - - val2 = FIELD_GET(MT_MIB_BA_MISS_COUNT_MASK, val); - if (val2 > mib->ba_miss_cnt) - mib->ba_miss_cnt = val2; + mib->ba_miss_cnt += FIELD_GET(MT_MIB_BA_MISS_COUNT_MASK, val); + mib->ack_fail_cnt += + FIELD_GET(MT_MIB_ACK_FAIL_COUNT_MASK, val); val = mt76_rr(dev, MT_MIB_MB_SDR0(ext_phy, i)); - val2 = FIELD_GET(MT_MIB_RTS_RETRIES_COUNT_MASK, val); - if (val2 > mib->rts_retries_cnt) { - mib->rts_cnt = FIELD_GET(MT_MIB_RTS_COUNT_MASK, val); - mib->rts_retries_cnt = val2; - } + mib->rts_cnt += FIELD_GET(MT_MIB_RTS_COUNT_MASK, val); + mib->rts_retries_cnt += + FIELD_GET(MT_MIB_RTS_RETRIES_COUNT_MASK, val); val = mt76_rr(dev, MT_TX_AGG_CNT(ext_phy, i)); - val2 = mt76_rr(dev, MT_TX_AGG_CNT2(ext_phy, i)); - dev->mt76.aggr_stats[aggr0++] += val & 0xffff; dev->mt76.aggr_stats[aggr0++] += val >> 16; - dev->mt76.aggr_stats[aggr1++] += val2 & 0xffff; - dev->mt76.aggr_stats[aggr1++] += val2 >> 16; + + val = mt76_rr(dev, MT_TX_AGG_CNT2(ext_phy, i)); + dev->mt76.aggr_stats[aggr1++] += val & 0xffff; + dev->mt76.aggr_stats[aggr1++] += val >> 16; } } diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/main.c b/drivers/net/wireless/mediatek/mt76/mt7915/main.c index d4969b2e1ffb..bf032d943f74 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/main.c @@ -317,7 +317,9 @@ static int mt7915_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, struct mt7915_sta *msta = sta ? (struct mt7915_sta *)sta->drv_priv : &mvif->sta; struct mt76_wcid *wcid = &msta->wcid; + u8 *wcid_keyidx = &wcid->hw_key_idx; int idx = key->keyidx; + int err = 0; /* The hardware does not support per-STA RX GTK, fallback * to software mode for these. @@ -332,6 +334,7 @@ static int mt7915_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, /* fall back to sw encryption for unsupported ciphers */ switch (key->cipher) { case WLAN_CIPHER_SUITE_AES_CMAC: + wcid_keyidx = &wcid->hw_key_idx2; key->flags |= IEEE80211_KEY_FLAG_GENERATE_MMIE; break; case WLAN_CIPHER_SUITE_TKIP: @@ -347,16 +350,24 @@ static int mt7915_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, return -EOPNOTSUPP; } - if (cmd == SET_KEY) { - key->hw_key_idx = wcid->idx; - wcid->hw_key_idx = idx; - } else if (idx == wcid->hw_key_idx) { - wcid->hw_key_idx = -1; - } + mutex_lock(&dev->mt76.mutex); + + if (cmd == SET_KEY) + *wcid_keyidx = idx; + else if (idx == *wcid_keyidx) + *wcid_keyidx = -1; + else + goto out; + mt76_wcid_key_setup(&dev->mt76, wcid, cmd == SET_KEY ? key : NULL); - return mt7915_mcu_add_key(dev, vif, msta, key, cmd); + err = mt7915_mcu_add_key(dev, vif, msta, key, cmd); + +out: + mutex_unlock(&dev->mt76.mutex); + + return err; } static int mt7915_config(struct ieee80211_hw *hw, u32 changed) @@ -717,13 +728,19 @@ mt7915_get_stats(struct ieee80211_hw *hw, struct ieee80211_low_level_stats *stats) { struct mt7915_phy *phy = mt7915_hw_phy(hw); + struct mt7915_dev *dev = mt7915_hw_dev(hw); struct mib_stats *mib = &phy->mib; + mutex_lock(&dev->mt76.mutex); stats->dot11RTSSuccessCount = mib->rts_cnt; stats->dot11RTSFailureCount = mib->rts_retries_cnt; stats->dot11FCSErrorCount = mib->fcs_err_cnt; stats->dot11ACKFailureCount = mib->ack_fail_cnt; + memset(mib, 0, sizeof(*mib)); + + mutex_unlock(&dev->mt76.mutex); + return 0; } @@ -833,9 +850,12 @@ static void mt7915_sta_statistics(struct ieee80211_hw *hw, struct mt7915_phy *phy = mt7915_hw_phy(hw); struct mt7915_sta *msta = (struct mt7915_sta *)sta->drv_priv; struct mt7915_sta_stats *stats = &msta->stats; + struct rate_info rxrate = {}; - if (mt7915_mcu_get_rx_rate(phy, vif, sta, &sinfo->rxrate) == 0) + if (!mt7915_mcu_get_rx_rate(phy, vif, sta, &rxrate)) { + sinfo->rxrate = rxrate; sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_BITRATE); + } if (!stats->tx_rate.legacy && !stats->tx_rate.flags) return; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c index 195929242b72..f069a5a03e14 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c @@ -351,54 +351,62 @@ mt7915_mcu_rx_radar_detected(struct mt7915_dev *dev, struct sk_buff *skb) dev->hw_pattern++; } -static void +static int mt7915_mcu_tx_rate_parse(struct mt76_phy *mphy, struct mt7915_mcu_ra_info *ra, struct rate_info *rate, u16 r) { struct ieee80211_supported_band *sband; u16 ru_idx = le16_to_cpu(ra->ru_idx); - u16 flags = 0; + bool cck = false; rate->mcs = FIELD_GET(MT_RA_RATE_MCS, r); rate->nss = FIELD_GET(MT_RA_RATE_NSS, r) + 1; switch (FIELD_GET(MT_RA_RATE_TX_MODE, r)) { case MT_PHY_TYPE_CCK: + cck = true; + fallthrough; case MT_PHY_TYPE_OFDM: if (mphy->chandef.chan->band == NL80211_BAND_5GHZ) sband = &mphy->sband_5g.sband; else sband = &mphy->sband_2g.sband; + rate->mcs = mt76_get_rate(mphy->dev, sband, rate->mcs, cck); rate->legacy = sband->bitrates[rate->mcs].bitrate; break; case MT_PHY_TYPE_HT: case MT_PHY_TYPE_HT_GF: rate->mcs += (rate->nss - 1) * 8; - flags |= RATE_INFO_FLAGS_MCS; + if (rate->mcs > 31) + return -EINVAL; + rate->flags = RATE_INFO_FLAGS_MCS; if (ra->gi) - flags |= RATE_INFO_FLAGS_SHORT_GI; + rate->flags |= RATE_INFO_FLAGS_SHORT_GI; break; case MT_PHY_TYPE_VHT: - flags |= RATE_INFO_FLAGS_VHT_MCS; + if (rate->mcs > 9) + return -EINVAL; + rate->flags = RATE_INFO_FLAGS_VHT_MCS; if (ra->gi) - flags |= RATE_INFO_FLAGS_SHORT_GI; + rate->flags |= RATE_INFO_FLAGS_SHORT_GI; break; case MT_PHY_TYPE_HE_SU: case MT_PHY_TYPE_HE_EXT_SU: case MT_PHY_TYPE_HE_TB: case MT_PHY_TYPE_HE_MU: + if (ra->gi > NL80211_RATE_INFO_HE_GI_3_2 || rate->mcs > 11) + return -EINVAL; + rate->he_gi = ra->gi; rate->he_dcm = FIELD_GET(MT_RA_RATE_DCM_EN, r); - - flags |= RATE_INFO_FLAGS_HE_MCS; + rate->flags = RATE_INFO_FLAGS_HE_MCS; break; default: - break; + return -EINVAL; } - rate->flags = flags; if (ru_idx) { switch (ru_idx) { @@ -435,6 +443,8 @@ mt7915_mcu_tx_rate_parse(struct mt76_phy *mphy, struct mt7915_mcu_ra_info *ra, break; } } + + return 0; } static void @@ -465,12 +475,12 @@ mt7915_mcu_tx_rate_report(struct mt7915_dev *dev, struct sk_buff *skb) mphy = dev->mt76.phy2; /* current rate */ - mt7915_mcu_tx_rate_parse(mphy, ra, &rate, curr); - stats->tx_rate = rate; + if (!mt7915_mcu_tx_rate_parse(mphy, ra, &rate, curr)) + stats->tx_rate = rate; /* probing rate */ - mt7915_mcu_tx_rate_parse(mphy, ra, &prob_rate, probe); - stats->prob_rate = prob_rate; + if (!mt7915_mcu_tx_rate_parse(mphy, ra, &prob_rate, probe)) + stats->prob_rate = prob_rate; if (attempts) { u16 success = le16_to_cpu(ra->success); @@ -1188,6 +1198,9 @@ mt7915_mcu_sta_ba(struct mt7915_dev *dev, wtbl_hdr = mt7915_mcu_alloc_wtbl_req(dev, msta, WTBL_SET, sta_wtbl, &skb); + if (IS_ERR(wtbl_hdr)) + return PTR_ERR(wtbl_hdr); + mt7915_mcu_wtbl_ba_tlv(skb, params, enable, tx, sta_wtbl, wtbl_hdr); ret = mt76_mcu_skb_send_msg(&dev->mt76, skb, @@ -1704,6 +1717,9 @@ int mt7915_mcu_sta_update_hdr_trans(struct mt7915_dev *dev, return -ENOMEM; wtbl_hdr = mt7915_mcu_alloc_wtbl_req(dev, msta, WTBL_SET, NULL, &skb); + if (IS_ERR(wtbl_hdr)) + return PTR_ERR(wtbl_hdr); + mt7915_mcu_wtbl_hdr_trans_tlv(skb, vif, sta, NULL, wtbl_hdr); return mt76_mcu_skb_send_msg(&dev->mt76, skb, MCU_EXT_CMD(WTBL_UPDATE), @@ -1728,6 +1744,9 @@ int mt7915_mcu_add_smps(struct mt7915_dev *dev, struct ieee80211_vif *vif, wtbl_hdr = mt7915_mcu_alloc_wtbl_req(dev, msta, WTBL_SET, sta_wtbl, &skb); + if (IS_ERR(wtbl_hdr)) + return PTR_ERR(wtbl_hdr); + mt7915_mcu_wtbl_smps_tlv(skb, sta, sta_wtbl, wtbl_hdr); return mt76_mcu_skb_send_msg(&dev->mt76, skb, @@ -2253,6 +2272,9 @@ int mt7915_mcu_add_sta(struct mt7915_dev *dev, struct ieee80211_vif *vif, wtbl_hdr = mt7915_mcu_alloc_wtbl_req(dev, msta, WTBL_RESET_AND_SET, sta_wtbl, &skb); + if (IS_ERR(wtbl_hdr)) + return PTR_ERR(wtbl_hdr); + if (enable) { mt7915_mcu_wtbl_generic_tlv(skb, vif, sta, sta_wtbl, wtbl_hdr); mt7915_mcu_wtbl_hdr_trans_tlv(skb, vif, sta, sta_wtbl, wtbl_hdr); @@ -2742,21 +2764,8 @@ static int mt7915_load_ram(struct mt7915_dev *dev) static int mt7915_load_firmware(struct mt7915_dev *dev) { + u32 reg = mt7915_reg_map_l1(dev, MT_TOP_MISC); int ret; - u32 val, reg = mt7915_reg_map_l1(dev, MT_TOP_MISC); - - val = FIELD_PREP(MT_TOP_MISC_FW_STATE, FW_STATE_FW_DOWNLOAD); - - if (!mt76_poll_msec(dev, reg, MT_TOP_MISC_FW_STATE, val, 1000)) { - /* restart firmware once */ - __mt76_mcu_restart(&dev->mt76); - if (!mt76_poll_msec(dev, reg, MT_TOP_MISC_FW_STATE, - val, 1000)) { - dev_err(dev->mt76.dev, - "Firmware is not ready for download\n"); - return -EIO; - } - } ret = mt7915_load_patch(dev); if (ret) @@ -3501,9 +3510,8 @@ int mt7915_mcu_get_rx_rate(struct mt7915_phy *phy, struct ieee80211_vif *vif, struct ieee80211_supported_band *sband; struct mt7915_mcu_phy_rx_info *res; struct sk_buff *skb; - u16 flags = 0; int ret; - int i; + bool cck = false; ret = mt76_mcu_send_and_get_msg(&dev->mt76, MCU_EXT_CMD(PHY_STAT_INFO), &req, sizeof(req), true, &skb); @@ -3517,48 +3525,53 @@ int mt7915_mcu_get_rx_rate(struct mt7915_phy *phy, struct ieee80211_vif *vif, switch (res->mode) { case MT_PHY_TYPE_CCK: + cck = true; + fallthrough; case MT_PHY_TYPE_OFDM: if (mphy->chandef.chan->band == NL80211_BAND_5GHZ) sband = &mphy->sband_5g.sband; else sband = &mphy->sband_2g.sband; - for (i = 0; i < sband->n_bitrates; i++) { - if (rate->mcs != (sband->bitrates[i].hw_value & 0xf)) - continue; - - rate->legacy = sband->bitrates[i].bitrate; - break; - } + rate->mcs = mt76_get_rate(&dev->mt76, sband, rate->mcs, cck); + rate->legacy = sband->bitrates[rate->mcs].bitrate; break; case MT_PHY_TYPE_HT: case MT_PHY_TYPE_HT_GF: - if (rate->mcs > 31) - return -EINVAL; - - flags |= RATE_INFO_FLAGS_MCS; + if (rate->mcs > 31) { + ret = -EINVAL; + goto out; + } + rate->flags = RATE_INFO_FLAGS_MCS; if (res->gi) - flags |= RATE_INFO_FLAGS_SHORT_GI; + rate->flags |= RATE_INFO_FLAGS_SHORT_GI; break; case MT_PHY_TYPE_VHT: - flags |= RATE_INFO_FLAGS_VHT_MCS; + if (rate->mcs > 9) { + ret = -EINVAL; + goto out; + } + rate->flags = RATE_INFO_FLAGS_VHT_MCS; if (res->gi) - flags |= RATE_INFO_FLAGS_SHORT_GI; + rate->flags |= RATE_INFO_FLAGS_SHORT_GI; break; case MT_PHY_TYPE_HE_SU: case MT_PHY_TYPE_HE_EXT_SU: case MT_PHY_TYPE_HE_TB: case MT_PHY_TYPE_HE_MU: + if (res->gi > NL80211_RATE_INFO_HE_GI_3_2 || rate->mcs > 11) { + ret = -EINVAL; + goto out; + } rate->he_gi = res->gi; - - flags |= RATE_INFO_FLAGS_HE_MCS; + rate->flags = RATE_INFO_FLAGS_HE_MCS; break; default: - break; + ret = -EINVAL; + goto out; } - rate->flags = flags; switch (res->bw) { case IEEE80211_STA_RX_BW_160: @@ -3575,7 +3588,8 @@ int mt7915_mcu_get_rx_rate(struct mt7915_phy *phy, struct ieee80211_vif *vif, break; } +out: dev_kfree_skb(skb); - return 0; + return ret; } diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h b/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h index 5c7eefdf2013..1160d1bf8a7c 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h @@ -108,11 +108,11 @@ struct mt7915_vif { }; struct mib_stats { - u16 ack_fail_cnt; - u16 fcs_err_cnt; - u16 rts_cnt; - u16 rts_retries_cnt; - u16 ba_miss_cnt; + u32 ack_fail_cnt; + u32 fcs_err_cnt; + u32 rts_cnt; + u32 rts_retries_cnt; + u32 ba_miss_cnt; }; struct mt7915_hif { diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/regs.h b/drivers/net/wireless/mediatek/mt76/mt7915/regs.h index ed0c9a24bb53..dfb8880657bf 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/regs.h +++ b/drivers/net/wireless/mediatek/mt76/mt7915/regs.h @@ -4,6 +4,11 @@ #ifndef __MT7915_REGS_H #define __MT7915_REGS_H +/* MCU WFDMA0 */ +#define MT_MCU_WFDMA0_BASE 0x2000 +#define MT_MCU_WFDMA0(ofs) (MT_MCU_WFDMA0_BASE + (ofs)) +#define MT_MCU_WFDMA0_DUMMY_CR MT_MCU_WFDMA0(0x120) + /* MCU WFDMA1 */ #define MT_MCU_WFDMA1_BASE 0x3000 #define MT_MCU_WFDMA1(ofs) (MT_MCU_WFDMA1_BASE + (ofs)) @@ -396,6 +401,14 @@ #define MT_WFDMA1_PCIE1_BUSY_ENA_TX_FIFO1 BIT(1) #define MT_WFDMA1_PCIE1_BUSY_ENA_RX_FIFO BIT(2) +#define MT_TOP_RGU_BASE 0xf0000 +#define MT_TOP_PWR_CTRL (MT_TOP_RGU_BASE + (0x0)) +#define MT_TOP_PWR_KEY (0x5746 << 16) +#define MT_TOP_PWR_SW_RST BIT(0) +#define MT_TOP_PWR_SW_PWR_ON GENMASK(3, 2) +#define MT_TOP_PWR_HW_CTRL BIT(4) +#define MT_TOP_PWR_PWR_ON BIT(7) + #define MT_INFRA_CFG_BASE 0xf1000 #define MT_INFRA(ofs) (MT_INFRA_CFG_BASE + (ofs)) diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/debugfs.c b/drivers/net/wireless/mediatek/mt76/mt7921/debugfs.c index 0dc8e25e18e4..87a7ea12f3b3 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/debugfs.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/debugfs.c @@ -9,10 +9,13 @@ mt7921_fw_debug_set(void *data, u64 val) { struct mt7921_dev *dev = data; - dev->fw_debug = (u8)val; + mt7921_mutex_acquire(dev); + dev->fw_debug = (u8)val; mt7921_mcu_fw_log_2_host(dev, dev->fw_debug); + mt7921_mutex_release(dev); + return 0; } @@ -44,14 +47,13 @@ mt7921_ampdu_stat_read_phy(struct mt7921_phy *phy, range[i] = mt76_rr(dev, MT_MIB_ARNG(0, i)); for (i = 0; i < ARRAY_SIZE(bound); i++) - bound[i] = MT_MIB_ARNCR_RANGE(range[i / 4], i) + 1; + bound[i] = MT_MIB_ARNCR_RANGE(range[i / 4], i % 4) + 1; seq_printf(file, "\nPhy0\n"); seq_printf(file, "Length: %8d | ", bound[0]); for (i = 0; i < ARRAY_SIZE(bound) - 1; i++) - seq_printf(file, "%3d -%3d | ", - bound[i] + 1, bound[i + 1]); + seq_printf(file, "%3d %3d | ", bound[i] + 1, bound[i + 1]); seq_puts(file, "\nCount: "); for (i = 0; i < ARRAY_SIZE(bound); i++) diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mac.c b/drivers/net/wireless/mediatek/mt76/mt7921/mac.c index 3f9097481a5e..a6d2a25b3495 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/mac.c @@ -400,7 +400,9 @@ int mt7921_mac_fill_rx(struct mt7921_dev *dev, struct sk_buff *skb) /* RXD Group 3 - P-RXV */ if (rxd1 & MT_RXD1_NORMAL_GROUP_3) { - u32 v0, v1, v2; + u8 stbc, gi; + u32 v0, v1; + bool cck; rxv = rxd; rxd += 2; @@ -409,7 +411,6 @@ int mt7921_mac_fill_rx(struct mt7921_dev *dev, struct sk_buff *skb) v0 = le32_to_cpu(rxv[0]); v1 = le32_to_cpu(rxv[1]); - v2 = le32_to_cpu(rxv[2]); if (v0 & MT_PRXV_HT_AD_CODE) status->enc_flags |= RX_ENC_FLAG_LDPC; @@ -429,87 +430,87 @@ int mt7921_mac_fill_rx(struct mt7921_dev *dev, struct sk_buff *skb) status->chain_signal[i]); } - /* RXD Group 5 - C-RXV */ - if (rxd1 & MT_RXD1_NORMAL_GROUP_5) { - u8 stbc = FIELD_GET(MT_CRXV_HT_STBC, v2); - u8 gi = FIELD_GET(MT_CRXV_HT_SHORT_GI, v2); - bool cck = false; + stbc = FIELD_GET(MT_PRXV_STBC, v0); + gi = FIELD_GET(MT_PRXV_SGI, v0); + cck = false; - rxd += 18; - if ((u8 *)rxd - skb->data >= skb->len) - return -EINVAL; + idx = i = FIELD_GET(MT_PRXV_TX_RATE, v0); + mode = FIELD_GET(MT_PRXV_TX_MODE, v0); - idx = i = FIELD_GET(MT_PRXV_TX_RATE, v0); - mode = FIELD_GET(MT_CRXV_TX_MODE, v2); - - switch (mode) { - case MT_PHY_TYPE_CCK: - cck = true; - fallthrough; - case MT_PHY_TYPE_OFDM: - i = mt76_get_rate(&dev->mt76, sband, i, cck); - break; - case MT_PHY_TYPE_HT_GF: - case MT_PHY_TYPE_HT: - status->encoding = RX_ENC_HT; - if (i > 31) - return -EINVAL; - break; - case MT_PHY_TYPE_VHT: - status->nss = - FIELD_GET(MT_PRXV_NSTS, v0) + 1; - status->encoding = RX_ENC_VHT; - if (i > 9) - return -EINVAL; - break; - case MT_PHY_TYPE_HE_MU: - status->flag |= RX_FLAG_RADIOTAP_HE_MU; - fallthrough; - case MT_PHY_TYPE_HE_SU: - case MT_PHY_TYPE_HE_EXT_SU: - case MT_PHY_TYPE_HE_TB: - status->nss = - FIELD_GET(MT_PRXV_NSTS, v0) + 1; - status->encoding = RX_ENC_HE; - status->flag |= RX_FLAG_RADIOTAP_HE; - i &= GENMASK(3, 0); - - if (gi <= NL80211_RATE_INFO_HE_GI_3_2) - status->he_gi = gi; - - status->he_dcm = !!(idx & MT_PRXV_TX_DCM); - break; - default: + switch (mode) { + case MT_PHY_TYPE_CCK: + cck = true; + fallthrough; + case MT_PHY_TYPE_OFDM: + i = mt76_get_rate(&dev->mt76, sband, i, cck); + break; + case MT_PHY_TYPE_HT_GF: + case MT_PHY_TYPE_HT: + status->encoding = RX_ENC_HT; + if (i > 31) return -EINVAL; - } - status->rate_idx = i; - - switch (FIELD_GET(MT_CRXV_FRAME_MODE, v2)) { - case IEEE80211_STA_RX_BW_20: - break; - case IEEE80211_STA_RX_BW_40: - if (mode & MT_PHY_TYPE_HE_EXT_SU && - (idx & MT_PRXV_TX_ER_SU_106T)) { - status->bw = RATE_INFO_BW_HE_RU; - status->he_ru = - NL80211_RATE_INFO_HE_RU_ALLOC_106; - } else { - status->bw = RATE_INFO_BW_40; - } - break; - case IEEE80211_STA_RX_BW_80: - status->bw = RATE_INFO_BW_80; - break; - case IEEE80211_STA_RX_BW_160: - status->bw = RATE_INFO_BW_160; - break; - default: + break; + case MT_PHY_TYPE_VHT: + status->nss = + FIELD_GET(MT_PRXV_NSTS, v0) + 1; + status->encoding = RX_ENC_VHT; + if (i > 9) return -EINVAL; + break; + case MT_PHY_TYPE_HE_MU: + status->flag |= RX_FLAG_RADIOTAP_HE_MU; + fallthrough; + case MT_PHY_TYPE_HE_SU: + case MT_PHY_TYPE_HE_EXT_SU: + case MT_PHY_TYPE_HE_TB: + status->nss = + FIELD_GET(MT_PRXV_NSTS, v0) + 1; + status->encoding = RX_ENC_HE; + status->flag |= RX_FLAG_RADIOTAP_HE; + i &= GENMASK(3, 0); + + if (gi <= NL80211_RATE_INFO_HE_GI_3_2) + status->he_gi = gi; + + status->he_dcm = !!(idx & MT_PRXV_TX_DCM); + break; + default: + return -EINVAL; + } + + status->rate_idx = i; + + switch (FIELD_GET(MT_PRXV_FRAME_MODE, v0)) { + case IEEE80211_STA_RX_BW_20: + break; + case IEEE80211_STA_RX_BW_40: + if (mode & MT_PHY_TYPE_HE_EXT_SU && + (idx & MT_PRXV_TX_ER_SU_106T)) { + status->bw = RATE_INFO_BW_HE_RU; + status->he_ru = + NL80211_RATE_INFO_HE_RU_ALLOC_106; + } else { + status->bw = RATE_INFO_BW_40; } + break; + case IEEE80211_STA_RX_BW_80: + status->bw = RATE_INFO_BW_80; + break; + case IEEE80211_STA_RX_BW_160: + status->bw = RATE_INFO_BW_160; + break; + default: + return -EINVAL; + } - status->enc_flags |= RX_ENC_FLAG_STBC_MASK * stbc; - if (mode < MT_PHY_TYPE_HE_SU && gi) - status->enc_flags |= RX_ENC_FLAG_SHORT_GI; + status->enc_flags |= RX_ENC_FLAG_STBC_MASK * stbc; + if (mode < MT_PHY_TYPE_HE_SU && gi) + status->enc_flags |= RX_ENC_FLAG_SHORT_GI; + + if (rxd1 & MT_RXD1_NORMAL_GROUP_5) { + rxd += 18; + if ((u8 *)rxd - skb->data >= skb->len) + return -EINVAL; } } @@ -1317,31 +1318,20 @@ mt7921_mac_update_mib_stats(struct mt7921_phy *phy) struct mib_stats *mib = &phy->mib; int i, aggr0 = 0, aggr1; - memset(mib, 0, sizeof(*mib)); - - mib->fcs_err_cnt = mt76_get_field(dev, MT_MIB_SDR3(0), - MT_MIB_SDR3_FCS_ERR_MASK); + mib->fcs_err_cnt += mt76_get_field(dev, MT_MIB_SDR3(0), + MT_MIB_SDR3_FCS_ERR_MASK); + mib->ack_fail_cnt += mt76_get_field(dev, MT_MIB_MB_BSDR3(0), + MT_MIB_ACK_FAIL_COUNT_MASK); + mib->ba_miss_cnt += mt76_get_field(dev, MT_MIB_MB_BSDR2(0), + MT_MIB_BA_FAIL_COUNT_MASK); + mib->rts_cnt += mt76_get_field(dev, MT_MIB_MB_BSDR0(0), + MT_MIB_RTS_COUNT_MASK); + mib->rts_retries_cnt += mt76_get_field(dev, MT_MIB_MB_BSDR1(0), + MT_MIB_RTS_FAIL_COUNT_MASK); for (i = 0, aggr1 = aggr0 + 4; i < 4; i++) { u32 val, val2; - val = mt76_rr(dev, MT_MIB_MB_SDR1(0, i)); - - val2 = FIELD_GET(MT_MIB_ACK_FAIL_COUNT_MASK, val); - if (val2 > mib->ack_fail_cnt) - mib->ack_fail_cnt = val2; - - val2 = FIELD_GET(MT_MIB_BA_MISS_COUNT_MASK, val); - if (val2 > mib->ba_miss_cnt) - mib->ba_miss_cnt = val2; - - val = mt76_rr(dev, MT_MIB_MB_SDR0(0, i)); - val2 = FIELD_GET(MT_MIB_RTS_RETRIES_COUNT_MASK, val); - if (val2 > mib->rts_retries_cnt) { - mib->rts_cnt = FIELD_GET(MT_MIB_RTS_COUNT_MASK, val); - mib->rts_retries_cnt = val2; - } - val = mt76_rr(dev, MT_TX_AGG_CNT(0, i)); val2 = mt76_rr(dev, MT_TX_AGG_CNT2(0, i)); @@ -1503,8 +1493,10 @@ void mt7921_coredump_work(struct work_struct *work) break; skb_pull(skb, sizeof(struct mt7921_mcu_rxd)); - if (data + skb->len - dump > MT76_CONNAC_COREDUMP_SZ) - break; + if (data + skb->len - dump > MT76_CONNAC_COREDUMP_SZ) { + dev_kfree_skb(skb); + continue; + } memcpy(data, skb->data, skb->len); data += skb->len; diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mac.h b/drivers/net/wireless/mediatek/mt76/mt7921/mac.h index a0c1fa0f20e4..109c8849d106 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/mac.h +++ b/drivers/net/wireless/mediatek/mt76/mt7921/mac.h @@ -97,18 +97,24 @@ enum rx_pkt_type { #define MT_RXD3_NORMAL_PF_MODE BIT(29) #define MT_RXD3_NORMAL_PF_STS GENMASK(31, 30) -/* P-RXV */ +/* P-RXV DW0 */ #define MT_PRXV_TX_RATE GENMASK(6, 0) #define MT_PRXV_TX_DCM BIT(4) #define MT_PRXV_TX_ER_SU_106T BIT(5) #define MT_PRXV_NSTS GENMASK(9, 7) #define MT_PRXV_HT_AD_CODE BIT(11) +#define MT_PRXV_FRAME_MODE GENMASK(14, 12) +#define MT_PRXV_SGI GENMASK(16, 15) +#define MT_PRXV_STBC GENMASK(23, 22) +#define MT_PRXV_TX_MODE GENMASK(27, 24) #define MT_PRXV_HE_RU_ALLOC_L GENMASK(31, 28) -#define MT_PRXV_HE_RU_ALLOC_H GENMASK(3, 0) + +/* P-RXV DW1 */ #define MT_PRXV_RCPI3 GENMASK(31, 24) #define MT_PRXV_RCPI2 GENMASK(23, 16) #define MT_PRXV_RCPI1 GENMASK(15, 8) #define MT_PRXV_RCPI0 GENMASK(7, 0) +#define MT_PRXV_HE_RU_ALLOC_H GENMASK(3, 0) /* C-RXV */ #define MT_CRXV_HT_STBC GENMASK(1, 0) diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/main.c b/drivers/net/wireless/mediatek/mt76/mt7921/main.c index 729f6c42cdde..ada943c7a950 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/main.c @@ -348,6 +348,7 @@ static void mt7921_remove_interface(struct ieee80211_hw *hw, if (vif == phy->monitor_vif) phy->monitor_vif = NULL; + mt7921_mutex_acquire(dev); mt76_connac_free_pending_tx_skbs(&dev->pm, &msta->wcid); if (dev->pm.enable) { @@ -360,7 +361,6 @@ static void mt7921_remove_interface(struct ieee80211_hw *hw, rcu_assign_pointer(dev->mt76.wcid[idx], NULL); - mt7921_mutex_acquire(dev); dev->mt76.vif_mask &= ~BIT(mvif->mt76.idx); phy->omac_mask &= ~BIT_ULL(mvif->mt76.omac_idx); mt7921_mutex_release(dev); @@ -413,7 +413,8 @@ static int mt7921_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, struct mt7921_sta *msta = sta ? (struct mt7921_sta *)sta->drv_priv : &mvif->sta; struct mt76_wcid *wcid = &msta->wcid; - int idx = key->keyidx; + u8 *wcid_keyidx = &wcid->hw_key_idx; + int idx = key->keyidx, err = 0; /* The hardware does not support per-STA RX GTK, fallback * to software mode for these. @@ -429,6 +430,7 @@ static int mt7921_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, switch (key->cipher) { case WLAN_CIPHER_SUITE_AES_CMAC: key->flags |= IEEE80211_KEY_FLAG_GENERATE_MMIE; + wcid_keyidx = &wcid->hw_key_idx2; break; case WLAN_CIPHER_SUITE_TKIP: case WLAN_CIPHER_SUITE_CCMP: @@ -443,16 +445,23 @@ static int mt7921_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, return -EOPNOTSUPP; } - if (cmd == SET_KEY) { - key->hw_key_idx = wcid->idx; - wcid->hw_key_idx = idx; - } else if (idx == wcid->hw_key_idx) { - wcid->hw_key_idx = -1; - } + mt7921_mutex_acquire(dev); + + if (cmd == SET_KEY) + *wcid_keyidx = idx; + else if (idx == *wcid_keyidx) + *wcid_keyidx = -1; + else + goto out; + mt76_wcid_key_setup(&dev->mt76, wcid, cmd == SET_KEY ? key : NULL); - return mt7921_mcu_add_key(dev, vif, msta, key, cmd); + err = mt7921_mcu_add_key(dev, vif, msta, key, cmd); +out: + mt7921_mutex_release(dev); + + return err; } static int mt7921_config(struct ieee80211_hw *hw, u32 changed) @@ -587,6 +596,9 @@ static void mt7921_bss_info_changed(struct ieee80211_hw *hw, if (changed & BSS_CHANGED_PS) mt7921_mcu_uni_bss_ps(dev, vif); + if (changed & BSS_CHANGED_ARP_FILTER) + mt7921_mcu_update_arp_filter(hw, vif, info); + mt7921_mutex_release(dev); } @@ -814,11 +826,17 @@ mt7921_get_stats(struct ieee80211_hw *hw, struct mt7921_phy *phy = mt7921_hw_phy(hw); struct mib_stats *mib = &phy->mib; + mt7921_mutex_acquire(phy->dev); + stats->dot11RTSSuccessCount = mib->rts_cnt; stats->dot11RTSFailureCount = mib->rts_retries_cnt; stats->dot11FCSErrorCount = mib->fcs_err_cnt; stats->dot11ACKFailureCount = mib->ack_fail_cnt; + memset(mib, 0, sizeof(*mib)); + + mt7921_mutex_release(phy->dev); + return 0; } diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c index b5cc72e7e81c..62afbad77596 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c @@ -1304,3 +1304,47 @@ mt7921_pm_interface_iter(void *priv, u8 *mac, struct ieee80211_vif *vif) mt76_clear(dev, MT_WF_RFCR(0), MT_WF_RFCR_DROP_OTHER_BEACON); } } + +int mt7921_mcu_update_arp_filter(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_bss_conf *info) +{ + struct mt7921_vif *mvif = (struct mt7921_vif *)vif->drv_priv; + struct mt7921_dev *dev = mt7921_hw_dev(hw); + struct sk_buff *skb; + int i, len = min_t(int, info->arp_addr_cnt, + IEEE80211_BSS_ARP_ADDR_LIST_LEN); + struct { + struct { + u8 bss_idx; + u8 pad[3]; + } __packed hdr; + struct mt76_connac_arpns_tlv arp; + } req_hdr = { + .hdr = { + .bss_idx = mvif->mt76.idx, + }, + .arp = { + .tag = cpu_to_le16(UNI_OFFLOAD_OFFLOAD_ARP), + .len = cpu_to_le16(sizeof(struct mt76_connac_arpns_tlv)), + .ips_num = len, + .mode = 2, /* update */ + .option = 1, + }, + }; + + skb = mt76_mcu_msg_alloc(&dev->mt76, NULL, + sizeof(req_hdr) + len * sizeof(__be32)); + if (!skb) + return -ENOMEM; + + skb_put_data(skb, &req_hdr, sizeof(req_hdr)); + for (i = 0; i < len; i++) { + u8 *addr = (u8 *)skb_put(skb, sizeof(__be32)); + + memcpy(addr, &info->arp_addr_list[i], sizeof(__be32)); + } + + return mt76_mcu_skb_send_msg(&dev->mt76, skb, MCU_UNI_CMD_OFFLOAD, + true); +} diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h b/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h index 46e6aeec35ae..25a1a6acb6ba 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h +++ b/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h @@ -102,11 +102,11 @@ struct mt7921_vif { }; struct mib_stats { - u16 ack_fail_cnt; - u16 fcs_err_cnt; - u16 rts_cnt; - u16 rts_retries_cnt; - u16 ba_miss_cnt; + u32 ack_fail_cnt; + u32 fcs_err_cnt; + u32 rts_cnt; + u32 rts_retries_cnt; + u32 ba_miss_cnt; }; struct mt7921_phy { @@ -339,4 +339,7 @@ int mt7921_mac_set_beacon_filter(struct mt7921_phy *phy, bool enable); void mt7921_pm_interface_iter(void *priv, u8 *mac, struct ieee80211_vif *vif); void mt7921_coredump_work(struct work_struct *work); +int mt7921_mcu_update_arp_filter(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_bss_conf *info); #endif diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/pci.c b/drivers/net/wireless/mediatek/mt76/mt7921/pci.c index 5570b4a50531..80f6f29892a4 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/pci.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/pci.c @@ -137,7 +137,7 @@ static int mt7921_pci_probe(struct pci_dev *pdev, mt76_wr(dev, MT_WFDMA0_HOST_INT_ENA, 0); - mt7921_l1_wr(dev, MT_PCIE_MAC_INT_ENABLE, 0xff); + mt76_wr(dev, MT_PCIE_MAC_INT_ENABLE, 0xff); ret = devm_request_irq(mdev->dev, pdev->irq, mt7921_irq_handler, IRQF_SHARED, KBUILD_MODNAME, dev); @@ -146,10 +146,12 @@ static int mt7921_pci_probe(struct pci_dev *pdev, ret = mt7921_register_device(dev); if (ret) - goto err_free_dev; + goto err_free_irq; return 0; +err_free_irq: + devm_free_irq(&pdev->dev, pdev->irq, dev); err_free_dev: mt76_free_device(&dev->mt76); err_free_pci_vec: @@ -193,7 +195,6 @@ static int mt7921_pci_suspend(struct pci_dev *pdev, pm_message_t state) mt76_for_each_q_rx(mdev, i) { napi_disable(&mdev->napi[i]); } - tasklet_kill(&dev->irq_tasklet); pci_enable_wake(pdev, pci_choose_state(pdev, state), true); @@ -208,13 +209,16 @@ static int mt7921_pci_suspend(struct pci_dev *pdev, pm_message_t state) /* disable interrupt */ mt76_wr(dev, MT_WFDMA0_HOST_INT_ENA, 0); + mt76_wr(dev, MT_PCIE_MAC_INT_ENABLE, 0x0); + synchronize_irq(pdev->irq); + tasklet_kill(&dev->irq_tasklet); - pci_save_state(pdev); - err = pci_set_power_state(pdev, pci_choose_state(pdev, state)); + err = mt7921_mcu_fw_pmctrl(dev); if (err) goto restore; - err = mt7921_mcu_drv_pmctrl(dev); + pci_save_state(pdev); + err = pci_set_power_state(pdev, pci_choose_state(pdev, state)); if (err) goto restore; @@ -237,18 +241,18 @@ static int mt7921_pci_resume(struct pci_dev *pdev) struct mt7921_dev *dev = container_of(mdev, struct mt7921_dev, mt76); int i, err; - err = mt7921_mcu_fw_pmctrl(dev); - if (err < 0) - return err; - err = pci_set_power_state(pdev, PCI_D0); if (err) return err; pci_restore_state(pdev); + err = mt7921_mcu_drv_pmctrl(dev); + if (err < 0) + return err; + /* enable interrupt */ - mt7921_l1_wr(dev, MT_PCIE_MAC_INT_ENABLE, 0xff); + mt76_wr(dev, MT_PCIE_MAC_INT_ENABLE, 0xff); mt7921_irq_enable(dev, MT_INT_RX_DONE_ALL | MT_INT_TX_DONE_ALL | MT_INT_MCU_CMD); diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/regs.h b/drivers/net/wireless/mediatek/mt76/mt7921/regs.h index 6dad7f6ab09d..73878d3e2495 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/regs.h +++ b/drivers/net/wireless/mediatek/mt76/mt7921/regs.h @@ -96,8 +96,8 @@ #define MT_WF_MIB_BASE(_band) ((_band) ? 0xa4800 : 0x24800) #define MT_WF_MIB(_band, ofs) (MT_WF_MIB_BASE(_band) + (ofs)) -#define MT_MIB_SDR3(_band) MT_WF_MIB(_band, 0x014) -#define MT_MIB_SDR3_FCS_ERR_MASK GENMASK(15, 0) +#define MT_MIB_SDR3(_band) MT_WF_MIB(_band, 0x698) +#define MT_MIB_SDR3_FCS_ERR_MASK GENMASK(31, 16) #define MT_MIB_SDR9(_band) MT_WF_MIB(_band, 0x02c) #define MT_MIB_SDR9_BUSY_MASK GENMASK(23, 0) @@ -121,16 +121,21 @@ #define MT_MIB_RTS_RETRIES_COUNT_MASK GENMASK(31, 16) #define MT_MIB_RTS_COUNT_MASK GENMASK(15, 0) -#define MT_MIB_MB_SDR1(_band, n) MT_WF_MIB(_band, 0x104 + ((n) << 4)) -#define MT_MIB_BA_MISS_COUNT_MASK GENMASK(15, 0) -#define MT_MIB_ACK_FAIL_COUNT_MASK GENMASK(31, 16) +#define MT_MIB_MB_BSDR0(_band) MT_WF_MIB(_band, 0x688) +#define MT_MIB_RTS_COUNT_MASK GENMASK(15, 0) +#define MT_MIB_MB_BSDR1(_band) MT_WF_MIB(_band, 0x690) +#define MT_MIB_RTS_FAIL_COUNT_MASK GENMASK(15, 0) +#define MT_MIB_MB_BSDR2(_band) MT_WF_MIB(_band, 0x518) +#define MT_MIB_BA_FAIL_COUNT_MASK GENMASK(15, 0) +#define MT_MIB_MB_BSDR3(_band) MT_WF_MIB(_band, 0x520) +#define MT_MIB_ACK_FAIL_COUNT_MASK GENMASK(15, 0) #define MT_MIB_MB_SDR2(_band, n) MT_WF_MIB(_band, 0x108 + ((n) << 4)) #define MT_MIB_FRAME_RETRIES_COUNT_MASK GENMASK(15, 0) -#define MT_TX_AGG_CNT(_band, n) MT_WF_MIB(_band, 0x0a8 + ((n) << 2)) -#define MT_TX_AGG_CNT2(_band, n) MT_WF_MIB(_band, 0x164 + ((n) << 2)) -#define MT_MIB_ARNG(_band, n) MT_WF_MIB(_band, 0x4b8 + ((n) << 2)) +#define MT_TX_AGG_CNT(_band, n) MT_WF_MIB(_band, 0x7dc + ((n) << 2)) +#define MT_TX_AGG_CNT2(_band, n) MT_WF_MIB(_band, 0x7ec + ((n) << 2)) +#define MT_MIB_ARNG(_band, n) MT_WF_MIB(_band, 0x0b0 + ((n) << 2)) #define MT_MIB_ARNCR_RANGE(val, n) (((val) >> ((n) << 3)) & GENMASK(7, 0)) #define MT_WTBLON_TOP_BASE 0x34000 @@ -357,11 +362,11 @@ #define MT_INFRA_CFG_BASE 0xfe000 #define MT_INFRA(ofs) (MT_INFRA_CFG_BASE + (ofs)) -#define MT_HIF_REMAP_L1 MT_INFRA(0x260) +#define MT_HIF_REMAP_L1 MT_INFRA(0x24c) #define MT_HIF_REMAP_L1_MASK GENMASK(15, 0) #define MT_HIF_REMAP_L1_OFFSET GENMASK(15, 0) #define MT_HIF_REMAP_L1_BASE GENMASK(31, 16) -#define MT_HIF_REMAP_BASE_L1 0xe0000 +#define MT_HIF_REMAP_BASE_L1 0x40000 #define MT_SWDEF_BASE 0x41f200 #define MT_SWDEF(ofs) (MT_SWDEF_BASE + (ofs)) @@ -384,7 +389,7 @@ #define MT_HW_CHIPID 0x70010200 #define MT_HW_REV 0x70010204 -#define MT_PCIE_MAC_BASE 0x74030000 +#define MT_PCIE_MAC_BASE 0x10000 #define MT_PCIE_MAC(ofs) (MT_PCIE_MAC_BASE + (ofs)) #define MT_PCIE_MAC_INT_ENABLE MT_PCIE_MAC(0x188) diff --git a/drivers/net/wireless/mediatek/mt76/sdio.c b/drivers/net/wireless/mediatek/mt76/sdio.c index 0b6facb17ff7..a18d2896ee1f 100644 --- a/drivers/net/wireless/mediatek/mt76/sdio.c +++ b/drivers/net/wireless/mediatek/mt76/sdio.c @@ -256,6 +256,9 @@ mt76s_tx_queue_skb(struct mt76_dev *dev, struct mt76_queue *q, q->entry[q->head].skb = tx_info.skb; q->entry[q->head].buf_sz = len; + + smp_wmb(); + q->head = (q->head + 1) % q->ndesc; q->queued++; diff --git a/drivers/net/wireless/mediatek/mt76/tx.c b/drivers/net/wireless/mediatek/mt76/tx.c index b8fe8adc43a3..451ed60c6296 100644 --- a/drivers/net/wireless/mediatek/mt76/tx.c +++ b/drivers/net/wireless/mediatek/mt76/tx.c @@ -461,11 +461,11 @@ mt76_txq_schedule_list(struct mt76_phy *phy, enum mt76_txq_id qid) int ret = 0; while (1) { + int n_frames = 0; + if (test_bit(MT76_STATE_PM, &phy->state) || - test_bit(MT76_RESET, &phy->state)) { - ret = -EBUSY; - break; - } + test_bit(MT76_RESET, &phy->state)) + return -EBUSY; if (dev->queue_ops->tx_cleanup && q->queued + 2 * MT_TXQ_FREE_THR >= q->ndesc) { @@ -497,11 +497,16 @@ mt76_txq_schedule_list(struct mt76_phy *phy, enum mt76_txq_id qid) } if (!mt76_txq_stopped(q)) - ret += mt76_txq_send_burst(phy, q, mtxq); + n_frames = mt76_txq_send_burst(phy, q, mtxq); spin_unlock_bh(&q->lock); ieee80211_return_txq(phy->hw, txq, false); + + if (unlikely(n_frames < 0)) + return n_frames; + + ret += n_frames; } return ret; diff --git a/drivers/net/wireless/mediatek/mt7601u/eeprom.c b/drivers/net/wireless/mediatek/mt7601u/eeprom.c index c868582c5d22..aa3b64902cf9 100644 --- a/drivers/net/wireless/mediatek/mt7601u/eeprom.c +++ b/drivers/net/wireless/mediatek/mt7601u/eeprom.c @@ -99,7 +99,7 @@ mt7601u_has_tssi(struct mt7601u_dev *dev, u8 *eeprom) { u16 nic_conf1 = get_unaligned_le16(eeprom + MT_EE_NIC_CONF_1); - return ~nic_conf1 && (nic_conf1 & MT_EE_NIC_CONF_1_TX_ALC_EN); + return (u16)~nic_conf1 && (nic_conf1 & MT_EE_NIC_CONF_1_TX_ALC_EN); } static void diff --git a/drivers/net/wireless/microchip/wilc1000/netdev.c b/drivers/net/wireless/microchip/wilc1000/netdev.c index 1b205e7d97a8..37f40039e4ca 100644 --- a/drivers/net/wireless/microchip/wilc1000/netdev.c +++ b/drivers/net/wireless/microchip/wilc1000/netdev.c @@ -575,7 +575,6 @@ static int wilc_mac_open(struct net_device *ndev) { struct wilc_vif *vif = netdev_priv(ndev); struct wilc *wl = vif->wilc; - unsigned char mac_add[ETH_ALEN] = {0}; int ret = 0; struct mgmt_frame_regs mgmt_regs = {}; @@ -598,9 +597,12 @@ static int wilc_mac_open(struct net_device *ndev) wilc_set_operation_mode(vif, wilc_get_vif_idx(vif), vif->iftype, vif->idx); - wilc_get_mac_address(vif, mac_add); - netdev_dbg(ndev, "Mac address: %pM\n", mac_add); - ether_addr_copy(ndev->dev_addr, mac_add); + + if (is_valid_ether_addr(ndev->dev_addr)) + wilc_set_mac_address(vif, ndev->dev_addr); + else + wilc_get_mac_address(vif, ndev->dev_addr); + netdev_dbg(ndev, "Mac address: %pM\n", ndev->dev_addr); if (!is_valid_ether_addr(ndev->dev_addr)) { netdev_err(ndev, "Wrong MAC address\n"); @@ -639,7 +641,14 @@ static int wilc_set_mac_addr(struct net_device *dev, void *p) int srcu_idx; if (!is_valid_ether_addr(addr->sa_data)) - return -EINVAL; + return -EADDRNOTAVAIL; + + if (!vif->mac_opened) { + eth_commit_mac_addr_change(dev, p); + return 0; + } + + /* Verify MAC Address is not already in use: */ srcu_idx = srcu_read_lock(&wilc->srcu); list_for_each_entry_rcu(tmp_vif, &wilc->vif_list, list) { @@ -647,7 +656,7 @@ static int wilc_set_mac_addr(struct net_device *dev, void *p) if (ether_addr_equal(addr->sa_data, mac_addr)) { if (vif != tmp_vif) { srcu_read_unlock(&wilc->srcu, srcu_idx); - return -EINVAL; + return -EADDRNOTAVAIL; } srcu_read_unlock(&wilc->srcu, srcu_idx); return 0; @@ -659,9 +668,7 @@ static int wilc_set_mac_addr(struct net_device *dev, void *p) if (result) return result; - ether_addr_copy(vif->bssid, addr->sa_data); - ether_addr_copy(vif->ndev->dev_addr, addr->sa_data); - + eth_commit_mac_addr_change(dev, p); return result; } diff --git a/drivers/net/wireless/microchip/wilc1000/sdio.c b/drivers/net/wireless/microchip/wilc1000/sdio.c index 351ff909ab1c..e14b9fc2c67a 100644 --- a/drivers/net/wireless/microchip/wilc1000/sdio.c +++ b/drivers/net/wireless/microchip/wilc1000/sdio.c @@ -947,7 +947,7 @@ static int wilc_sdio_sync_ext(struct wilc *wilc, int nint) for (i = 0; (i < 3) && (nint > 0); i++, nint--) reg |= BIT(i); - ret = wilc_sdio_read_reg(wilc, WILC_INTR2_ENABLE, ®); + ret = wilc_sdio_write_reg(wilc, WILC_INTR2_ENABLE, reg); if (ret) { dev_err(&func->dev, "Failed write reg (%08x)...\n", diff --git a/drivers/net/wireless/quantenna/qtnfmac/event.c b/drivers/net/wireless/quantenna/qtnfmac/event.c index c775c177933b..8dc80574d08d 100644 --- a/drivers/net/wireless/quantenna/qtnfmac/event.c +++ b/drivers/net/wireless/quantenna/qtnfmac/event.c @@ -570,8 +570,10 @@ qtnf_event_handle_external_auth(struct qtnf_vif *vif, return 0; if (ev->ssid_len) { - memcpy(auth.ssid.ssid, ev->ssid, ev->ssid_len); - auth.ssid.ssid_len = ev->ssid_len; + int len = clamp_val(ev->ssid_len, 0, IEEE80211_MAX_SSID_LEN); + + memcpy(auth.ssid.ssid, ev->ssid, len); + auth.ssid.ssid_len = len; } auth.key_mgmt_suite = le32_to_cpu(ev->akm_suite); diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/table.c b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/table.c index 27c8a5d96520..fcaaf664cbec 100644 --- a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/table.c +++ b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/table.c @@ -249,7 +249,7 @@ u32 RTL8821AE_PHY_REG_ARRAY[] = { 0x824, 0x00030FE0, 0x828, 0x00000000, 0x82C, 0x002081DD, - 0x830, 0x2AAA8E24, + 0x830, 0x2AAAEEC8, 0x834, 0x0037A706, 0x838, 0x06489B44, 0x83C, 0x0000095B, @@ -324,10 +324,10 @@ u32 RTL8821AE_PHY_REG_ARRAY[] = { 0x9D8, 0x00000000, 0x9DC, 0x00000000, 0x9E0, 0x00005D00, - 0x9E4, 0x00000002, + 0x9E4, 0x00000003, 0x9E8, 0x00000001, 0xA00, 0x00D047C8, - 0xA04, 0x01FF000C, + 0xA04, 0x01FF800C, 0xA08, 0x8C8A8300, 0xA0C, 0x2E68000F, 0xA10, 0x9500BB78, @@ -1320,7 +1320,11 @@ u32 RTL8821AE_RADIOA_ARRAY[] = { 0x083, 0x00021800, 0x084, 0x00028000, 0x085, 0x00048000, + 0x80000111, 0x00000000, 0x40000000, 0x00000000, + 0x086, 0x0009483A, + 0xA0000000, 0x00000000, 0x086, 0x00094838, + 0xB0000000, 0x00000000, 0x087, 0x00044980, 0x088, 0x00048000, 0x089, 0x0000D480, @@ -1409,36 +1413,32 @@ u32 RTL8821AE_RADIOA_ARRAY[] = { 0x03C, 0x000CA000, 0x0EF, 0x00000000, 0x0EF, 0x00001100, - 0xFF0F0104, 0xABCD, + 0x80000111, 0x00000000, 0x40000000, 0x00000000, 0x034, 0x0004ADF3, 0x034, 0x00049DF0, - 0xFF0F0204, 0xCDEF, + 0x90000110, 0x00000000, 0x40000000, 0x00000000, 0x034, 0x0004ADF3, 0x034, 0x00049DF0, - 0xFF0F0404, 0xCDEF, - 0x034, 0x0004ADF3, - 0x034, 0x00049DF0, - 0xFF0F0200, 0xCDEF, + 0x90000210, 0x00000000, 0x40000000, 0x00000000, 0x034, 0x0004ADF5, 0x034, 0x00049DF2, - 0xFF0F02C0, 0xCDEF, + 0x9000020c, 0x00000000, 0x40000000, 0x00000000, + 0x034, 0x0004A0F3, + 0x034, 0x000490B1, + 0x9000040c, 0x00000000, 0x40000000, 0x00000000, 0x034, 0x0004A0F3, 0x034, 0x000490B1, - 0xCDCDCDCD, 0xCDCD, + 0x90000200, 0x00000000, 0x40000000, 0x00000000, + 0x034, 0x0004ADF5, + 0x034, 0x00049DF2, + 0x90000410, 0x00000000, 0x40000000, 0x00000000, + 0x034, 0x0004ADF3, + 0x034, 0x00049DF0, + 0xA0000000, 0x00000000, 0x034, 0x0004ADF7, 0x034, 0x00049DF3, - 0xFF0F0104, 0xDEAD, - 0xFF0F0104, 0xABCD, - 0x034, 0x00048DED, - 0x034, 0x00047DEA, - 0x034, 0x00046DE7, - 0x034, 0x00045CE9, - 0x034, 0x00044CE6, - 0x034, 0x000438C6, - 0x034, 0x00042886, - 0x034, 0x00041486, - 0x034, 0x00040447, - 0xFF0F0204, 0xCDEF, + 0xB0000000, 0x00000000, + 0x80000111, 0x00000000, 0x40000000, 0x00000000, 0x034, 0x00048DED, 0x034, 0x00047DEA, 0x034, 0x00046DE7, @@ -1448,7 +1448,7 @@ u32 RTL8821AE_RADIOA_ARRAY[] = { 0x034, 0x00042886, 0x034, 0x00041486, 0x034, 0x00040447, - 0xFF0F0404, 0xCDEF, + 0x90000110, 0x00000000, 0x40000000, 0x00000000, 0x034, 0x00048DED, 0x034, 0x00047DEA, 0x034, 0x00046DE7, @@ -1458,7 +1458,17 @@ u32 RTL8821AE_RADIOA_ARRAY[] = { 0x034, 0x00042886, 0x034, 0x00041486, 0x034, 0x00040447, - 0xFF0F02C0, 0xCDEF, + 0x9000020c, 0x00000000, 0x40000000, 0x00000000, + 0x034, 0x000480AE, + 0x034, 0x000470AB, + 0x034, 0x0004608B, + 0x034, 0x00045069, + 0x034, 0x00044048, + 0x034, 0x00043045, + 0x034, 0x00042026, + 0x034, 0x00041023, + 0x034, 0x00040002, + 0x9000040c, 0x00000000, 0x40000000, 0x00000000, 0x034, 0x000480AE, 0x034, 0x000470AB, 0x034, 0x0004608B, @@ -1468,7 +1478,17 @@ u32 RTL8821AE_RADIOA_ARRAY[] = { 0x034, 0x00042026, 0x034, 0x00041023, 0x034, 0x00040002, - 0xCDCDCDCD, 0xCDCD, + 0x90000410, 0x00000000, 0x40000000, 0x00000000, + 0x034, 0x00048DED, + 0x034, 0x00047DEA, + 0x034, 0x00046DE7, + 0x034, 0x00045CE9, + 0x034, 0x00044CE6, + 0x034, 0x000438C6, + 0x034, 0x00042886, + 0x034, 0x00041486, + 0x034, 0x00040447, + 0xA0000000, 0x00000000, 0x034, 0x00048DEF, 0x034, 0x00047DEC, 0x034, 0x00046DE9, @@ -1478,38 +1498,36 @@ u32 RTL8821AE_RADIOA_ARRAY[] = { 0x034, 0x0004248A, 0x034, 0x0004108D, 0x034, 0x0004008A, - 0xFF0F0104, 0xDEAD, - 0xFF0F0200, 0xABCD, + 0xB0000000, 0x00000000, + 0x80000210, 0x00000000, 0x40000000, 0x00000000, 0x034, 0x0002ADF4, - 0xFF0F02C0, 0xCDEF, + 0x9000020c, 0x00000000, 0x40000000, 0x00000000, + 0x034, 0x0002A0F3, + 0x9000040c, 0x00000000, 0x40000000, 0x00000000, 0x034, 0x0002A0F3, - 0xCDCDCDCD, 0xCDCD, + 0x90000200, 0x00000000, 0x40000000, 0x00000000, + 0x034, 0x0002ADF4, + 0xA0000000, 0x00000000, 0x034, 0x0002ADF7, - 0xFF0F0200, 0xDEAD, - 0xFF0F0104, 0xABCD, - 0x034, 0x00029DF4, - 0xFF0F0204, 0xCDEF, + 0xB0000000, 0x00000000, + 0x80000111, 0x00000000, 0x40000000, 0x00000000, 0x034, 0x00029DF4, - 0xFF0F0404, 0xCDEF, + 0x90000110, 0x00000000, 0x40000000, 0x00000000, 0x034, 0x00029DF4, - 0xFF0F0200, 0xCDEF, + 0x90000210, 0x00000000, 0x40000000, 0x00000000, 0x034, 0x00029DF1, - 0xFF0F02C0, 0xCDEF, + 0x9000020c, 0x00000000, 0x40000000, 0x00000000, + 0x034, 0x000290F0, + 0x9000040c, 0x00000000, 0x40000000, 0x00000000, 0x034, 0x000290F0, - 0xCDCDCDCD, 0xCDCD, + 0x90000200, 0x00000000, 0x40000000, 0x00000000, + 0x034, 0x00029DF1, + 0x90000410, 0x00000000, 0x40000000, 0x00000000, + 0x034, 0x00029DF4, + 0xA0000000, 0x00000000, 0x034, 0x00029DF2, - 0xFF0F0104, 0xDEAD, - 0xFF0F0104, 0xABCD, - 0x034, 0x00028DF1, - 0x034, 0x00027DEE, - 0x034, 0x00026DEB, - 0x034, 0x00025CEC, - 0x034, 0x00024CE9, - 0x034, 0x000238CA, - 0x034, 0x00022889, - 0x034, 0x00021489, - 0x034, 0x0002044A, - 0xFF0F0204, 0xCDEF, + 0xB0000000, 0x00000000, + 0x80000111, 0x00000000, 0x40000000, 0x00000000, 0x034, 0x00028DF1, 0x034, 0x00027DEE, 0x034, 0x00026DEB, @@ -1519,7 +1537,7 @@ u32 RTL8821AE_RADIOA_ARRAY[] = { 0x034, 0x00022889, 0x034, 0x00021489, 0x034, 0x0002044A, - 0xFF0F0404, 0xCDEF, + 0x90000110, 0x00000000, 0x40000000, 0x00000000, 0x034, 0x00028DF1, 0x034, 0x00027DEE, 0x034, 0x00026DEB, @@ -1529,7 +1547,7 @@ u32 RTL8821AE_RADIOA_ARRAY[] = { 0x034, 0x00022889, 0x034, 0x00021489, 0x034, 0x0002044A, - 0xFF0F02C0, 0xCDEF, + 0x9000020c, 0x00000000, 0x40000000, 0x00000000, 0x034, 0x000280AF, 0x034, 0x000270AC, 0x034, 0x0002608B, @@ -1539,7 +1557,27 @@ u32 RTL8821AE_RADIOA_ARRAY[] = { 0x034, 0x00022026, 0x034, 0x00021023, 0x034, 0x00020002, - 0xCDCDCDCD, 0xCDCD, + 0x9000040c, 0x00000000, 0x40000000, 0x00000000, + 0x034, 0x000280AF, + 0x034, 0x000270AC, + 0x034, 0x0002608B, + 0x034, 0x00025069, + 0x034, 0x00024048, + 0x034, 0x00023045, + 0x034, 0x00022026, + 0x034, 0x00021023, + 0x034, 0x00020002, + 0x90000410, 0x00000000, 0x40000000, 0x00000000, + 0x034, 0x00028DF1, + 0x034, 0x00027DEE, + 0x034, 0x00026DEB, + 0x034, 0x00025CEC, + 0x034, 0x00024CE9, + 0x034, 0x000238CA, + 0x034, 0x00022889, + 0x034, 0x00021489, + 0x034, 0x0002044A, + 0xA0000000, 0x00000000, 0x034, 0x00028DEE, 0x034, 0x00027DEB, 0x034, 0x00026CCD, @@ -1549,27 +1587,24 @@ u32 RTL8821AE_RADIOA_ARRAY[] = { 0x034, 0x00022849, 0x034, 0x00021449, 0x034, 0x0002004D, - 0xFF0F0104, 0xDEAD, - 0xFF0F02C0, 0xABCD, + 0xB0000000, 0x00000000, + 0x8000020c, 0x00000000, 0x40000000, 0x00000000, + 0x034, 0x0000A0D7, + 0x034, 0x000090D3, + 0x034, 0x000080B1, + 0x034, 0x000070AE, + 0x9000040c, 0x00000000, 0x40000000, 0x00000000, 0x034, 0x0000A0D7, 0x034, 0x000090D3, 0x034, 0x000080B1, 0x034, 0x000070AE, - 0xCDCDCDCD, 0xCDCD, + 0xA0000000, 0x00000000, 0x034, 0x0000ADF7, 0x034, 0x00009DF4, 0x034, 0x00008DF1, 0x034, 0x00007DEE, - 0xFF0F02C0, 0xDEAD, - 0xFF0F0104, 0xABCD, - 0x034, 0x00006DEB, - 0x034, 0x00005CEC, - 0x034, 0x00004CE9, - 0x034, 0x000038CA, - 0x034, 0x00002889, - 0x034, 0x00001489, - 0x034, 0x0000044A, - 0xFF0F0204, 0xCDEF, + 0xB0000000, 0x00000000, + 0x80000111, 0x00000000, 0x40000000, 0x00000000, 0x034, 0x00006DEB, 0x034, 0x00005CEC, 0x034, 0x00004CE9, @@ -1577,7 +1612,7 @@ u32 RTL8821AE_RADIOA_ARRAY[] = { 0x034, 0x00002889, 0x034, 0x00001489, 0x034, 0x0000044A, - 0xFF0F0404, 0xCDEF, + 0x90000110, 0x00000000, 0x40000000, 0x00000000, 0x034, 0x00006DEB, 0x034, 0x00005CEC, 0x034, 0x00004CE9, @@ -1585,7 +1620,7 @@ u32 RTL8821AE_RADIOA_ARRAY[] = { 0x034, 0x00002889, 0x034, 0x00001489, 0x034, 0x0000044A, - 0xFF0F02C0, 0xCDEF, + 0x9000020c, 0x00000000, 0x40000000, 0x00000000, 0x034, 0x0000608D, 0x034, 0x0000506B, 0x034, 0x0000404A, @@ -1593,7 +1628,23 @@ u32 RTL8821AE_RADIOA_ARRAY[] = { 0x034, 0x00002044, 0x034, 0x00001025, 0x034, 0x00000004, - 0xCDCDCDCD, 0xCDCD, + 0x9000040c, 0x00000000, 0x40000000, 0x00000000, + 0x034, 0x0000608D, + 0x034, 0x0000506B, + 0x034, 0x0000404A, + 0x034, 0x00003047, + 0x034, 0x00002044, + 0x034, 0x00001025, + 0x034, 0x00000004, + 0x90000410, 0x00000000, 0x40000000, 0x00000000, + 0x034, 0x00006DEB, + 0x034, 0x00005CEC, + 0x034, 0x00004CE9, + 0x034, 0x000038CA, + 0x034, 0x00002889, + 0x034, 0x00001489, + 0x034, 0x0000044A, + 0xA0000000, 0x00000000, 0x034, 0x00006DCD, 0x034, 0x00005CCD, 0x034, 0x00004CCA, @@ -1601,11 +1652,11 @@ u32 RTL8821AE_RADIOA_ARRAY[] = { 0x034, 0x00002888, 0x034, 0x00001488, 0x034, 0x00000486, - 0xFF0F0104, 0xDEAD, + 0xB0000000, 0x00000000, 0x0EF, 0x00000000, 0x018, 0x0001712A, 0x0EF, 0x00000040, - 0xFF0F0104, 0xABCD, + 0x80000111, 0x00000000, 0x40000000, 0x00000000, 0x035, 0x00000187, 0x035, 0x00008187, 0x035, 0x00010187, @@ -1615,7 +1666,7 @@ u32 RTL8821AE_RADIOA_ARRAY[] = { 0x035, 0x00040188, 0x035, 0x00048188, 0x035, 0x00050188, - 0xFF0F0204, 0xCDEF, + 0x90000110, 0x00000000, 0x40000000, 0x00000000, 0x035, 0x00000187, 0x035, 0x00008187, 0x035, 0x00010187, @@ -1625,7 +1676,37 @@ u32 RTL8821AE_RADIOA_ARRAY[] = { 0x035, 0x00040188, 0x035, 0x00048188, 0x035, 0x00050188, - 0xFF0F0404, 0xCDEF, + 0x90000210, 0x00000000, 0x40000000, 0x00000000, + 0x035, 0x00000128, + 0x035, 0x00008128, + 0x035, 0x00010128, + 0x035, 0x000201C8, + 0x035, 0x000281C8, + 0x035, 0x000301C8, + 0x035, 0x000401C8, + 0x035, 0x000481C8, + 0x035, 0x000501C8, + 0x9000040c, 0x00000000, 0x40000000, 0x00000000, + 0x035, 0x00000145, + 0x035, 0x00008145, + 0x035, 0x00010145, + 0x035, 0x00020196, + 0x035, 0x00028196, + 0x035, 0x00030196, + 0x035, 0x000401C7, + 0x035, 0x000481C7, + 0x035, 0x000501C7, + 0x90000200, 0x00000000, 0x40000000, 0x00000000, + 0x035, 0x00000128, + 0x035, 0x00008128, + 0x035, 0x00010128, + 0x035, 0x000201C8, + 0x035, 0x000281C8, + 0x035, 0x000301C8, + 0x035, 0x000401C8, + 0x035, 0x000481C8, + 0x035, 0x000501C8, + 0x90000410, 0x00000000, 0x40000000, 0x00000000, 0x035, 0x00000187, 0x035, 0x00008187, 0x035, 0x00010187, @@ -1635,7 +1716,7 @@ u32 RTL8821AE_RADIOA_ARRAY[] = { 0x035, 0x00040188, 0x035, 0x00048188, 0x035, 0x00050188, - 0xCDCDCDCD, 0xCDCD, + 0xA0000000, 0x00000000, 0x035, 0x00000145, 0x035, 0x00008145, 0x035, 0x00010145, @@ -1645,11 +1726,11 @@ u32 RTL8821AE_RADIOA_ARRAY[] = { 0x035, 0x000401C7, 0x035, 0x000481C7, 0x035, 0x000501C7, - 0xFF0F0104, 0xDEAD, + 0xB0000000, 0x00000000, 0x0EF, 0x00000000, 0x018, 0x0001712A, 0x0EF, 0x00000010, - 0xFF0F0104, 0xABCD, + 0x80000111, 0x00000000, 0x40000000, 0x00000000, 0x036, 0x00085733, 0x036, 0x0008D733, 0x036, 0x00095733, @@ -1662,7 +1743,7 @@ u32 RTL8821AE_RADIOA_ARRAY[] = { 0x036, 0x000CE4B4, 0x036, 0x000D64B4, 0x036, 0x000DE4B4, - 0xFF0F0204, 0xCDEF, + 0x90000110, 0x00000000, 0x40000000, 0x00000000, 0x036, 0x00085733, 0x036, 0x0008D733, 0x036, 0x00095733, @@ -1675,7 +1756,46 @@ u32 RTL8821AE_RADIOA_ARRAY[] = { 0x036, 0x000CE4B4, 0x036, 0x000D64B4, 0x036, 0x000DE4B4, - 0xFF0F0404, 0xCDEF, + 0x90000210, 0x00000000, 0x40000000, 0x00000000, + 0x036, 0x000063B5, + 0x036, 0x0000E3B5, + 0x036, 0x000163B5, + 0x036, 0x0001E3B5, + 0x036, 0x000263B5, + 0x036, 0x0002E3B5, + 0x036, 0x000363B5, + 0x036, 0x0003E3B5, + 0x036, 0x000463B5, + 0x036, 0x0004E3B5, + 0x036, 0x000563B5, + 0x036, 0x0005E3B5, + 0x9000040c, 0x00000000, 0x40000000, 0x00000000, + 0x036, 0x000056B3, + 0x036, 0x0000D6B3, + 0x036, 0x000156B3, + 0x036, 0x0001D6B3, + 0x036, 0x00026634, + 0x036, 0x0002E634, + 0x036, 0x00036634, + 0x036, 0x0003E634, + 0x036, 0x000467B4, + 0x036, 0x0004E7B4, + 0x036, 0x000567B4, + 0x036, 0x0005E7B4, + 0x90000200, 0x00000000, 0x40000000, 0x00000000, + 0x036, 0x000063B5, + 0x036, 0x0000E3B5, + 0x036, 0x000163B5, + 0x036, 0x0001E3B5, + 0x036, 0x000263B5, + 0x036, 0x0002E3B5, + 0x036, 0x000363B5, + 0x036, 0x0003E3B5, + 0x036, 0x000463B5, + 0x036, 0x0004E3B5, + 0x036, 0x000563B5, + 0x036, 0x0005E3B5, + 0x90000410, 0x00000000, 0x40000000, 0x00000000, 0x036, 0x00085733, 0x036, 0x0008D733, 0x036, 0x00095733, @@ -1688,7 +1808,7 @@ u32 RTL8821AE_RADIOA_ARRAY[] = { 0x036, 0x000CE4B4, 0x036, 0x000D64B4, 0x036, 0x000DE4B4, - 0xCDCDCDCD, 0xCDCD, + 0xA0000000, 0x00000000, 0x036, 0x000056B3, 0x036, 0x0000D6B3, 0x036, 0x000156B3, @@ -1701,103 +1821,162 @@ u32 RTL8821AE_RADIOA_ARRAY[] = { 0x036, 0x0004E7B4, 0x036, 0x000567B4, 0x036, 0x0005E7B4, - 0xFF0F0104, 0xDEAD, + 0xB0000000, 0x00000000, 0x0EF, 0x00000000, 0x0EF, 0x00000008, - 0xFF0F0104, 0xABCD, + 0x80000111, 0x00000000, 0x40000000, 0x00000000, 0x03C, 0x000001C8, 0x03C, 0x00000492, - 0xFF0F0204, 0xCDEF, + 0x90000110, 0x00000000, 0x40000000, 0x00000000, 0x03C, 0x000001C8, 0x03C, 0x00000492, - 0xFF0F0404, 0xCDEF, + 0x90000210, 0x00000000, 0x40000000, 0x00000000, + 0x03C, 0x000001B6, + 0x03C, 0x00000492, + 0x9000040c, 0x00000000, 0x40000000, 0x00000000, + 0x03C, 0x0000022A, + 0x03C, 0x00000594, + 0x90000200, 0x00000000, 0x40000000, 0x00000000, + 0x03C, 0x000001B6, + 0x03C, 0x00000492, + 0x90000410, 0x00000000, 0x40000000, 0x00000000, 0x03C, 0x000001C8, 0x03C, 0x00000492, - 0xCDCDCDCD, 0xCDCD, + 0xA0000000, 0x00000000, 0x03C, 0x0000022A, 0x03C, 0x00000594, - 0xFF0F0104, 0xDEAD, - 0xFF0F0104, 0xABCD, + 0xB0000000, 0x00000000, + 0x80000111, 0x00000000, 0x40000000, 0x00000000, 0x03C, 0x00000800, - 0xFF0F0204, 0xCDEF, + 0x90000110, 0x00000000, 0x40000000, 0x00000000, 0x03C, 0x00000800, - 0xFF0F0404, 0xCDEF, + 0x90000210, 0x00000000, 0x40000000, 0x00000000, 0x03C, 0x00000800, - 0xFF0F02C0, 0xCDEF, + 0x9000020c, 0x00000000, 0x40000000, 0x00000000, 0x03C, 0x00000820, - 0xCDCDCDCD, 0xCDCD, + 0x9000040c, 0x00000000, 0x40000000, 0x00000000, + 0x03C, 0x00000820, + 0x90000200, 0x00000000, 0x40000000, 0x00000000, + 0x03C, 0x00000800, + 0x90000410, 0x00000000, 0x40000000, 0x00000000, + 0x03C, 0x00000800, + 0xA0000000, 0x00000000, 0x03C, 0x00000900, - 0xFF0F0104, 0xDEAD, + 0xB0000000, 0x00000000, 0x0EF, 0x00000000, 0x018, 0x0001712A, 0x0EF, 0x00000002, - 0xFF0F0104, 0xABCD, + 0x80000111, 0x00000000, 0x40000000, 0x00000000, 0x008, 0x0004E400, - 0xFF0F0204, 0xCDEF, + 0x90000110, 0x00000000, 0x40000000, 0x00000000, 0x008, 0x0004E400, - 0xFF0F0404, 0xCDEF, + 0x90000210, 0x00000000, 0x40000000, 0x00000000, + 0x008, 0x00002000, + 0x9000020c, 0x00000000, 0x40000000, 0x00000000, + 0x008, 0x00002000, + 0x9000040c, 0x00000000, 0x40000000, 0x00000000, + 0x008, 0x00002000, + 0x90000200, 0x00000000, 0x40000000, 0x00000000, + 0x008, 0x00002000, + 0x90000410, 0x00000000, 0x40000000, 0x00000000, 0x008, 0x0004E400, - 0xCDCDCDCD, 0xCDCD, + 0xA0000000, 0x00000000, 0x008, 0x00002000, - 0xFF0F0104, 0xDEAD, + 0xB0000000, 0x00000000, 0x0EF, 0x00000000, 0x0DF, 0x000000C0, - 0x01F, 0x00040064, - 0xFF0F0104, 0xABCD, + 0x01F, 0x00000064, + 0x80000111, 0x00000000, 0x40000000, 0x00000000, 0x058, 0x000A7284, 0x059, 0x000600EC, - 0xFF0F0204, 0xCDEF, + 0x90000110, 0x00000000, 0x40000000, 0x00000000, 0x058, 0x000A7284, 0x059, 0x000600EC, - 0xFF0F0404, 0xCDEF, + 0x9000020c, 0x00000000, 0x40000000, 0x00000000, + 0x058, 0x00081184, + 0x059, 0x0006016C, + 0x9000040c, 0x00000000, 0x40000000, 0x00000000, + 0x058, 0x00081184, + 0x059, 0x0006016C, + 0x90000200, 0x00000000, 0x40000000, 0x00000000, + 0x058, 0x00081184, + 0x059, 0x0006016C, + 0x90000410, 0x00000000, 0x40000000, 0x00000000, 0x058, 0x000A7284, 0x059, 0x000600EC, - 0xCDCDCDCD, 0xCDCD, + 0xA0000000, 0x00000000, 0x058, 0x00081184, 0x059, 0x0006016C, - 0xFF0F0104, 0xDEAD, - 0xFF0F0104, 0xABCD, + 0xB0000000, 0x00000000, + 0x80000111, 0x00000000, 0x40000000, 0x00000000, 0x061, 0x000E8D73, 0x062, 0x00093FC5, - 0xFF0F0204, 0xCDEF, + 0x90000110, 0x00000000, 0x40000000, 0x00000000, 0x061, 0x000E8D73, 0x062, 0x00093FC5, - 0xFF0F0404, 0xCDEF, + 0x90000210, 0x00000000, 0x40000000, 0x00000000, + 0x061, 0x000EFD83, + 0x062, 0x00093FCC, + 0x9000040c, 0x00000000, 0x40000000, 0x00000000, + 0x061, 0x000EAD53, + 0x062, 0x00093BC4, + 0x90000200, 0x00000000, 0x40000000, 0x00000000, + 0x061, 0x000EFD83, + 0x062, 0x00093FCC, + 0x90000410, 0x00000000, 0x40000000, 0x00000000, 0x061, 0x000E8D73, 0x062, 0x00093FC5, - 0xCDCDCDCD, 0xCDCD, + 0xA0000000, 0x00000000, 0x061, 0x000EAD53, 0x062, 0x00093BC4, - 0xFF0F0104, 0xDEAD, - 0xFF0F0104, 0xABCD, + 0xB0000000, 0x00000000, + 0x80000111, 0x00000000, 0x40000000, 0x00000000, 0x063, 0x000110E9, - 0xFF0F0204, 0xCDEF, + 0x90000110, 0x00000000, 0x40000000, 0x00000000, 0x063, 0x000110E9, - 0xFF0F0404, 0xCDEF, + 0x90000210, 0x00000000, 0x40000000, 0x00000000, + 0x063, 0x000110EB, + 0x9000020c, 0x00000000, 0x40000000, 0x00000000, 0x063, 0x000110E9, - 0xFF0F0200, 0xCDEF, - 0x063, 0x000710E9, - 0xFF0F02C0, 0xCDEF, + 0x9000040c, 0x00000000, 0x40000000, 0x00000000, 0x063, 0x000110E9, - 0xCDCDCDCD, 0xCDCD, + 0x90000200, 0x00000000, 0x40000000, 0x00000000, + 0x063, 0x000110EB, + 0x90000410, 0x00000000, 0x40000000, 0x00000000, + 0x063, 0x000110E9, + 0xA0000000, 0x00000000, 0x063, 0x000714E9, - 0xFF0F0104, 0xDEAD, - 0xFF0F0104, 0xABCD, + 0xB0000000, 0x00000000, + 0x80000111, 0x00000000, 0x40000000, 0x00000000, + 0x064, 0x0001C27C, + 0x90000110, 0x00000000, 0x40000000, 0x00000000, + 0x064, 0x0001C27C, + 0x90000210, 0x00000000, 0x40000000, 0x00000000, 0x064, 0x0001C27C, - 0xFF0F0204, 0xCDEF, + 0x9000040c, 0x00000000, 0x40000000, 0x00000000, + 0x064, 0x0001C67C, + 0x90000200, 0x00000000, 0x40000000, 0x00000000, 0x064, 0x0001C27C, - 0xFF0F0404, 0xCDEF, + 0x90000410, 0x00000000, 0x40000000, 0x00000000, 0x064, 0x0001C27C, - 0xCDCDCDCD, 0xCDCD, + 0xA0000000, 0x00000000, 0x064, 0x0001C67C, - 0xFF0F0104, 0xDEAD, - 0xFF0F0200, 0xABCD, + 0xB0000000, 0x00000000, + 0x80000111, 0x00000000, 0x40000000, 0x00000000, + 0x065, 0x00091016, + 0x90000110, 0x00000000, 0x40000000, 0x00000000, + 0x065, 0x00091016, + 0x90000210, 0x00000000, 0x40000000, 0x00000000, 0x065, 0x00093016, - 0xFF0F02C0, 0xCDEF, + 0x9000020c, 0x00000000, 0x40000000, 0x00000000, 0x065, 0x00093015, - 0xCDCDCDCD, 0xCDCD, + 0x9000040c, 0x00000000, 0x40000000, 0x00000000, + 0x065, 0x00093015, + 0x90000200, 0x00000000, 0x40000000, 0x00000000, + 0x065, 0x00093016, + 0xA0000000, 0x00000000, 0x065, 0x00091016, - 0xFF0F0200, 0xDEAD, + 0xB0000000, 0x00000000, 0x018, 0x00000006, 0x0EF, 0x00002000, 0x03B, 0x0003824B, @@ -1895,9 +2074,10 @@ u32 RTL8821AE_RADIOA_ARRAY[] = { 0x0B4, 0x0001214C, 0x0B7, 0x0003000C, 0x01C, 0x000539D2, + 0x0C4, 0x000AFE00, 0x018, 0x0001F12A, - 0x0FE, 0x00000000, - 0x0FE, 0x00000000, + 0xFFE, 0x00000000, + 0xFFE, 0x00000000, 0x018, 0x0001712A, }; @@ -2017,6 +2197,7 @@ u32 RTL8812AE_MAC_REG_ARRAY[] = { u32 RTL8812AE_MAC_1T_ARRAYLEN = ARRAY_SIZE(RTL8812AE_MAC_REG_ARRAY); u32 RTL8821AE_MAC_REG_ARRAY[] = { + 0x421, 0x0000000F, 0x428, 0x0000000A, 0x429, 0x00000010, 0x430, 0x00000000, @@ -2485,7 +2666,7 @@ u32 RTL8821AE_AGC_TAB_ARRAY[] = { 0x81C, 0xA6360001, 0x81C, 0xA5380001, 0x81C, 0xA43A0001, - 0x81C, 0xA33C0001, + 0x81C, 0x683C0001, 0x81C, 0x673E0001, 0x81C, 0x66400001, 0x81C, 0x65420001, @@ -2519,7 +2700,66 @@ u32 RTL8821AE_AGC_TAB_ARRAY[] = { 0x81C, 0x017A0001, 0x81C, 0x017C0001, 0x81C, 0x017E0001, - 0xFF0F02C0, 0xABCD, + 0x8000020c, 0x00000000, 0x40000000, 0x00000000, + 0x81C, 0xFB000101, + 0x81C, 0xFA020101, + 0x81C, 0xF9040101, + 0x81C, 0xF8060101, + 0x81C, 0xF7080101, + 0x81C, 0xF60A0101, + 0x81C, 0xF50C0101, + 0x81C, 0xF40E0101, + 0x81C, 0xF3100101, + 0x81C, 0xF2120101, + 0x81C, 0xF1140101, + 0x81C, 0xF0160101, + 0x81C, 0xEF180101, + 0x81C, 0xEE1A0101, + 0x81C, 0xED1C0101, + 0x81C, 0xEC1E0101, + 0x81C, 0xEB200101, + 0x81C, 0xEA220101, + 0x81C, 0xE9240101, + 0x81C, 0xE8260101, + 0x81C, 0xE7280101, + 0x81C, 0xE62A0101, + 0x81C, 0xE52C0101, + 0x81C, 0xE42E0101, + 0x81C, 0xE3300101, + 0x81C, 0xA5320101, + 0x81C, 0xA4340101, + 0x81C, 0xA3360101, + 0x81C, 0x87380101, + 0x81C, 0x863A0101, + 0x81C, 0x853C0101, + 0x81C, 0x843E0101, + 0x81C, 0x69400101, + 0x81C, 0x68420101, + 0x81C, 0x67440101, + 0x81C, 0x66460101, + 0x81C, 0x49480101, + 0x81C, 0x484A0101, + 0x81C, 0x474C0101, + 0x81C, 0x2A4E0101, + 0x81C, 0x29500101, + 0x81C, 0x28520101, + 0x81C, 0x27540101, + 0x81C, 0x26560101, + 0x81C, 0x25580101, + 0x81C, 0x245A0101, + 0x81C, 0x235C0101, + 0x81C, 0x055E0101, + 0x81C, 0x04600101, + 0x81C, 0x03620101, + 0x81C, 0x02640101, + 0x81C, 0x01660101, + 0x81C, 0x01680101, + 0x81C, 0x016A0101, + 0x81C, 0x016C0101, + 0x81C, 0x016E0101, + 0x81C, 0x01700101, + 0x81C, 0x01720101, + 0x9000040c, 0x00000000, 0x40000000, 0x00000000, 0x81C, 0xFB000101, 0x81C, 0xFA020101, 0x81C, 0xF9040101, @@ -2578,7 +2818,7 @@ u32 RTL8821AE_AGC_TAB_ARRAY[] = { 0x81C, 0x016E0101, 0x81C, 0x01700101, 0x81C, 0x01720101, - 0xCDCDCDCD, 0xCDCD, + 0xA0000000, 0x00000000, 0x81C, 0xFF000101, 0x81C, 0xFF020101, 0x81C, 0xFE040101, @@ -2637,7 +2877,7 @@ u32 RTL8821AE_AGC_TAB_ARRAY[] = { 0x81C, 0x046E0101, 0x81C, 0x03700101, 0x81C, 0x02720101, - 0xFF0F02C0, 0xDEAD, + 0xB0000000, 0x00000000, 0x81C, 0x01740101, 0x81C, 0x01760101, 0x81C, 0x01780101, diff --git a/drivers/net/wireless/realtek/rtw88/debug.c b/drivers/net/wireless/realtek/rtw88/debug.c index 948cb79050ea..e7d51ac9b689 100644 --- a/drivers/net/wireless/realtek/rtw88/debug.c +++ b/drivers/net/wireless/realtek/rtw88/debug.c @@ -270,7 +270,7 @@ static ssize_t rtw_debugfs_set_rsvd_page(struct file *filp, if (num != 2) { rtw_warn(rtwdev, "invalid arguments\n"); - return num; + return -EINVAL; } debugfs_priv->rsvd_page.page_offset = offset; diff --git a/drivers/net/wireless/realtek/rtw88/main.h b/drivers/net/wireless/realtek/rtw88/main.h index 35afea91fd29..92b9cf1f9525 100644 --- a/drivers/net/wireless/realtek/rtw88/main.h +++ b/drivers/net/wireless/realtek/rtw88/main.h @@ -1166,6 +1166,7 @@ struct rtw_chip_info { bool en_dis_dpd; u16 dpd_ratemask; u8 iqk_threshold; + u8 lck_threshold; const struct rtw_pwr_track_tbl *pwr_track_tbl; u8 bfer_su_max_num; @@ -1534,6 +1535,7 @@ struct rtw_dm_info { u32 rrsr_mask_min; u8 thermal_avg[RTW_RF_PATH_MAX]; u8 thermal_meter_k; + u8 thermal_meter_lck; s8 delta_power_index[RTW_RF_PATH_MAX]; s8 delta_power_index_last[RTW_RF_PATH_MAX]; u8 default_ofdm_index; diff --git a/drivers/net/wireless/realtek/rtw88/pci.c b/drivers/net/wireless/realtek/rtw88/pci.c index 786a48649946..6b5c885798a4 100644 --- a/drivers/net/wireless/realtek/rtw88/pci.c +++ b/drivers/net/wireless/realtek/rtw88/pci.c @@ -581,23 +581,30 @@ static int rtw_pci_start(struct rtw_dev *rtwdev) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; + rtw_pci_napi_start(rtwdev); + spin_lock_bh(&rtwpci->irq_lock); + rtwpci->running = true; rtw_pci_enable_interrupt(rtwdev, rtwpci, false); spin_unlock_bh(&rtwpci->irq_lock); - rtw_pci_napi_start(rtwdev); - return 0; } static void rtw_pci_stop(struct rtw_dev *rtwdev) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; + struct pci_dev *pdev = rtwpci->pdev; + spin_lock_bh(&rtwpci->irq_lock); + rtwpci->running = false; + rtw_pci_disable_interrupt(rtwdev, rtwpci); + spin_unlock_bh(&rtwpci->irq_lock); + + synchronize_irq(pdev->irq); rtw_pci_napi_stop(rtwdev); spin_lock_bh(&rtwpci->irq_lock); - rtw_pci_disable_interrupt(rtwdev, rtwpci); rtw_pci_dma_release(rtwdev, rtwpci); spin_unlock_bh(&rtwpci->irq_lock); } @@ -1138,7 +1145,8 @@ static irqreturn_t rtw_pci_interrupt_threadfn(int irq, void *dev) rtw_fw_c2h_cmd_isr(rtwdev); /* all of the jobs for this interrupt have been done */ - rtw_pci_enable_interrupt(rtwdev, rtwpci, rx); + if (rtwpci->running) + rtw_pci_enable_interrupt(rtwdev, rtwpci, rx); spin_unlock_bh(&rtwpci->irq_lock); return IRQ_HANDLED; @@ -1558,7 +1566,8 @@ static int rtw_pci_napi_poll(struct napi_struct *napi, int budget) if (work_done < budget) { napi_complete_done(napi, work_done); spin_lock_bh(&rtwpci->irq_lock); - rtw_pci_enable_interrupt(rtwdev, rtwpci, false); + if (rtwpci->running) + rtw_pci_enable_interrupt(rtwdev, rtwpci, false); spin_unlock_bh(&rtwpci->irq_lock); /* When ISR happens during polling and before napi_complete * while no further data is received. Data on the dma_ring will diff --git a/drivers/net/wireless/realtek/rtw88/pci.h b/drivers/net/wireless/realtek/rtw88/pci.h index e76fc549a788..0ffae887527a 100644 --- a/drivers/net/wireless/realtek/rtw88/pci.h +++ b/drivers/net/wireless/realtek/rtw88/pci.h @@ -211,6 +211,7 @@ struct rtw_pci { spinlock_t irq_lock; u32 irq_mask[4]; bool irq_enabled; + bool running; /* napi structure */ struct net_device netdev; diff --git a/drivers/net/wireless/realtek/rtw88/phy.c b/drivers/net/wireless/realtek/rtw88/phy.c index e114ddecac09..21e77fcfa4d5 100644 --- a/drivers/net/wireless/realtek/rtw88/phy.c +++ b/drivers/net/wireless/realtek/rtw88/phy.c @@ -1584,7 +1584,7 @@ void rtw_phy_load_tables(struct rtw_dev *rtwdev) } EXPORT_SYMBOL(rtw_phy_load_tables); -static u8 rtw_get_channel_group(u8 channel) +static u8 rtw_get_channel_group(u8 channel, u8 rate) { switch (channel) { default: @@ -1628,6 +1628,7 @@ static u8 rtw_get_channel_group(u8 channel) case 106: return 4; case 14: + return rate <= DESC_RATE11M ? 5 : 4; case 108: case 110: case 112: @@ -1879,7 +1880,7 @@ void rtw_get_tx_power_params(struct rtw_dev *rtwdev, u8 path, u8 rate, u8 bw, s8 *remnant = &pwr_param->pwr_remnant; pwr_idx = &rtwdev->efuse.txpwr_idx_table[path]; - group = rtw_get_channel_group(ch); + group = rtw_get_channel_group(ch, rate); /* base power index for 2.4G/5G */ if (IS_CH_2G_BAND(ch)) { @@ -2219,6 +2220,20 @@ s8 rtw_phy_pwrtrack_get_pwridx(struct rtw_dev *rtwdev, } EXPORT_SYMBOL(rtw_phy_pwrtrack_get_pwridx); +bool rtw_phy_pwrtrack_need_lck(struct rtw_dev *rtwdev) +{ + struct rtw_dm_info *dm_info = &rtwdev->dm_info; + u8 delta_lck; + + delta_lck = abs(dm_info->thermal_avg[0] - dm_info->thermal_meter_lck); + if (delta_lck >= rtwdev->chip->lck_threshold) { + dm_info->thermal_meter_lck = dm_info->thermal_avg[0]; + return true; + } + return false; +} +EXPORT_SYMBOL(rtw_phy_pwrtrack_need_lck); + bool rtw_phy_pwrtrack_need_iqk(struct rtw_dev *rtwdev) { struct rtw_dm_info *dm_info = &rtwdev->dm_info; diff --git a/drivers/net/wireless/realtek/rtw88/phy.h b/drivers/net/wireless/realtek/rtw88/phy.h index a4fcfb878550..a0742a69446d 100644 --- a/drivers/net/wireless/realtek/rtw88/phy.h +++ b/drivers/net/wireless/realtek/rtw88/phy.h @@ -55,6 +55,7 @@ u8 rtw_phy_pwrtrack_get_delta(struct rtw_dev *rtwdev, u8 path); s8 rtw_phy_pwrtrack_get_pwridx(struct rtw_dev *rtwdev, struct rtw_swing_table *swing_table, u8 tbl_path, u8 therm_path, u8 delta); +bool rtw_phy_pwrtrack_need_lck(struct rtw_dev *rtwdev); bool rtw_phy_pwrtrack_need_iqk(struct rtw_dev *rtwdev); void rtw_phy_config_swing_table(struct rtw_dev *rtwdev, struct rtw_swing_table *swing_table); diff --git a/drivers/net/wireless/realtek/rtw88/reg.h b/drivers/net/wireless/realtek/rtw88/reg.h index ea518aa78552..819af34dac34 100644 --- a/drivers/net/wireless/realtek/rtw88/reg.h +++ b/drivers/net/wireless/realtek/rtw88/reg.h @@ -652,8 +652,13 @@ #define RF_TXATANK 0x64 #define RF_TRXIQ 0x66 #define RF_RXIQGEN 0x8d +#define RF_SYN_PFD 0xb0 #define RF_XTALX2 0xb8 +#define RF_SYN_CTRL 0xbb #define RF_MALSEL 0xbe +#define RF_SYN_AAC 0xc9 +#define RF_AAC_CTRL 0xca +#define RF_FAST_LCK 0xcc #define RF_RCKD 0xde #define RF_TXADBG 0xde #define RF_LUTDBG 0xdf diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822c.c b/drivers/net/wireless/realtek/rtw88/rtw8822c.c index dd560c28abb2..448922cb2e63 100644 --- a/drivers/net/wireless/realtek/rtw88/rtw8822c.c +++ b/drivers/net/wireless/realtek/rtw88/rtw8822c.c @@ -1126,6 +1126,7 @@ static void rtw8822c_pwrtrack_init(struct rtw_dev *rtwdev) dm_info->pwr_trk_triggered = false; dm_info->thermal_meter_k = rtwdev->efuse.thermal_meter_k; + dm_info->thermal_meter_lck = rtwdev->efuse.thermal_meter_k; } static void rtw8822c_phy_set_param(struct rtw_dev *rtwdev) @@ -2108,6 +2109,26 @@ static void rtw8822c_false_alarm_statistics(struct rtw_dev *rtwdev) rtw_write32_set(rtwdev, REG_RX_BREAK, BIT_COM_RX_GCK_EN); } +static void rtw8822c_do_lck(struct rtw_dev *rtwdev) +{ + u32 val; + + rtw_write_rf(rtwdev, RF_PATH_A, RF_SYN_CTRL, RFREG_MASK, 0x80010); + rtw_write_rf(rtwdev, RF_PATH_A, RF_SYN_PFD, RFREG_MASK, 0x1F0FA); + fsleep(1); + rtw_write_rf(rtwdev, RF_PATH_A, RF_AAC_CTRL, RFREG_MASK, 0x80000); + rtw_write_rf(rtwdev, RF_PATH_A, RF_SYN_AAC, RFREG_MASK, 0x80001); + read_poll_timeout(rtw_read_rf, val, val != 0x1, 1000, 100000, + true, rtwdev, RF_PATH_A, RF_AAC_CTRL, 0x1000); + rtw_write_rf(rtwdev, RF_PATH_A, RF_SYN_PFD, RFREG_MASK, 0x1F0F8); + rtw_write_rf(rtwdev, RF_PATH_B, RF_SYN_CTRL, RFREG_MASK, 0x80010); + + rtw_write_rf(rtwdev, RF_PATH_A, RF_FAST_LCK, RFREG_MASK, 0x0f000); + rtw_write_rf(rtwdev, RF_PATH_A, RF_FAST_LCK, RFREG_MASK, 0x4f000); + fsleep(1); + rtw_write_rf(rtwdev, RF_PATH_A, RF_FAST_LCK, RFREG_MASK, 0x0f000); +} + static void rtw8822c_do_iqk(struct rtw_dev *rtwdev) { struct rtw_iqk_para para = {0}; @@ -3538,11 +3559,12 @@ static void __rtw8822c_pwr_track(struct rtw_dev *rtwdev) rtw_phy_config_swing_table(rtwdev, &swing_table); + if (rtw_phy_pwrtrack_need_lck(rtwdev)) + rtw8822c_do_lck(rtwdev); + for (i = 0; i < rtwdev->hal.rf_path_num; i++) rtw8822c_pwr_track_path(rtwdev, &swing_table, i); - if (rtw_phy_pwrtrack_need_iqk(rtwdev)) - rtw8822c_do_iqk(rtwdev); } static void rtw8822c_pwr_track(struct rtw_dev *rtwdev) @@ -4351,6 +4373,7 @@ struct rtw_chip_info rtw8822c_hw_spec = { .dpd_ratemask = DIS_DPD_RATEALL, .pwr_track_tbl = &rtw8822c_rtw_pwr_track_tbl, .iqk_threshold = 8, + .lck_threshold = 8, .bfer_su_max_num = 2, .bfer_mu_max_num = 1, .rx_ldpc = true, diff --git a/drivers/net/wireless/rsi/rsi_91x_sdio.c b/drivers/net/wireless/rsi/rsi_91x_sdio.c index fe0287b22a25..e0c502bc4270 100644 --- a/drivers/net/wireless/rsi/rsi_91x_sdio.c +++ b/drivers/net/wireless/rsi/rsi_91x_sdio.c @@ -1513,7 +1513,7 @@ static int rsi_restore(struct device *dev) } static const struct dev_pm_ops rsi_pm_ops = { .suspend = rsi_suspend, - .resume = rsi_resume, + .resume_noirq = rsi_resume, .freeze = rsi_freeze, .thaw = rsi_thaw, .restore = rsi_restore, diff --git a/drivers/net/wireless/ti/wlcore/boot.c b/drivers/net/wireless/ti/wlcore/boot.c index e14d88e558f0..85abd0a2d1c9 100644 --- a/drivers/net/wireless/ti/wlcore/boot.c +++ b/drivers/net/wireless/ti/wlcore/boot.c @@ -72,6 +72,7 @@ static int wlcore_validate_fw_ver(struct wl1271 *wl) unsigned int *min_ver = (wl->fw_type == WL12XX_FW_TYPE_MULTI) ? wl->min_mr_fw_ver : wl->min_sr_fw_ver; char min_fw_str[32] = ""; + int off = 0; int i; /* the chip must be exactly equal */ @@ -105,13 +106,15 @@ static int wlcore_validate_fw_ver(struct wl1271 *wl) return 0; fail: - for (i = 0; i < NUM_FW_VER; i++) + for (i = 0; i < NUM_FW_VER && off < sizeof(min_fw_str); i++) if (min_ver[i] == WLCORE_FW_VER_IGNORE) - snprintf(min_fw_str, sizeof(min_fw_str), - "%s*.", min_fw_str); + off += snprintf(min_fw_str + off, + sizeof(min_fw_str) - off, + "*."); else - snprintf(min_fw_str, sizeof(min_fw_str), - "%s%u.", min_fw_str, min_ver[i]); + off += snprintf(min_fw_str + off, + sizeof(min_fw_str) - off, + "%u.", min_ver[i]); wl1271_error("Your WiFi FW version (%u.%u.%u.%u.%u) is invalid.\n" "Please use at least FW %s\n" diff --git a/drivers/net/wireless/ti/wlcore/debugfs.h b/drivers/net/wireless/ti/wlcore/debugfs.h index b143293e694f..a9e13e6d65c5 100644 --- a/drivers/net/wireless/ti/wlcore/debugfs.h +++ b/drivers/net/wireless/ti/wlcore/debugfs.h @@ -78,13 +78,14 @@ static ssize_t sub## _ ##name## _read(struct file *file, \ struct wl1271 *wl = file->private_data; \ struct struct_type *stats = wl->stats.fw_stats; \ char buf[DEBUGFS_FORMAT_BUFFER_SIZE] = ""; \ + int pos = 0; \ int i; \ \ wl1271_debugfs_update_stats(wl); \ \ - for (i = 0; i < len; i++) \ - snprintf(buf, sizeof(buf), "%s[%d] = %d\n", \ - buf, i, stats->sub.name[i]); \ + for (i = 0; i < len && pos < sizeof(buf); i++) \ + pos += snprintf(buf + pos, sizeof(buf) - pos, \ + "[%d] = %d\n", i, stats->sub.name[i]); \ \ return wl1271_format_buffer(userbuf, count, ppos, "%s", buf); \ } \ diff --git a/drivers/net/wireless/wl3501.h b/drivers/net/wireless/wl3501.h index e98e04ee9a2c..59b7b93c5963 100644 --- a/drivers/net/wireless/wl3501.h +++ b/drivers/net/wireless/wl3501.h @@ -379,16 +379,7 @@ struct wl3501_get_confirm { u8 mib_value[100]; }; -struct wl3501_join_req { - u16 next_blk; - u8 sig_id; - u8 reserved; - struct iw_mgmt_data_rset operational_rset; - u16 reserved2; - u16 timeout; - u16 probe_delay; - u8 timestamp[8]; - u8 local_time[8]; +struct wl3501_req { u16 beacon_period; u16 dtim_period; u16 cap_info; @@ -401,6 +392,19 @@ struct wl3501_join_req { struct iw_mgmt_data_rset bss_basic_rset; }; +struct wl3501_join_req { + u16 next_blk; + u8 sig_id; + u8 reserved; + struct iw_mgmt_data_rset operational_rset; + u16 reserved2; + u16 timeout; + u16 probe_delay; + u8 timestamp[8]; + u8 local_time[8]; + struct wl3501_req req; +}; + struct wl3501_join_confirm { u16 next_blk; u8 sig_id; @@ -443,16 +447,7 @@ struct wl3501_scan_confirm { u16 status; char timestamp[8]; char localtime[8]; - u16 beacon_period; - u16 dtim_period; - u16 cap_info; - u8 bss_type; - u8 bssid[ETH_ALEN]; - struct iw_mgmt_essid_pset ssid; - struct iw_mgmt_ds_pset ds_pset; - struct iw_mgmt_cf_pset cf_pset; - struct iw_mgmt_ibss_pset ibss_pset; - struct iw_mgmt_data_rset bss_basic_rset; + struct wl3501_req req; u8 rssi; }; @@ -471,8 +466,10 @@ struct wl3501_md_req { u16 size; u8 pri; u8 service_class; - u8 daddr[ETH_ALEN]; - u8 saddr[ETH_ALEN]; + struct { + u8 daddr[ETH_ALEN]; + u8 saddr[ETH_ALEN]; + } addr; }; struct wl3501_md_ind { @@ -484,8 +481,10 @@ struct wl3501_md_ind { u8 reception; u8 pri; u8 service_class; - u8 daddr[ETH_ALEN]; - u8 saddr[ETH_ALEN]; + struct { + u8 daddr[ETH_ALEN]; + u8 saddr[ETH_ALEN]; + } addr; }; struct wl3501_md_confirm { diff --git a/drivers/net/wireless/wl3501_cs.c b/drivers/net/wireless/wl3501_cs.c index 8ca5789c7b37..672f5d5f3f2c 100644 --- a/drivers/net/wireless/wl3501_cs.c +++ b/drivers/net/wireless/wl3501_cs.c @@ -469,6 +469,7 @@ static int wl3501_send_pkt(struct wl3501_card *this, u8 *data, u16 len) struct wl3501_md_req sig = { .sig_id = WL3501_SIG_MD_REQ, }; + size_t sig_addr_len = sizeof(sig.addr); u8 *pdata = (char *)data; int rc = -EIO; @@ -484,9 +485,9 @@ static int wl3501_send_pkt(struct wl3501_card *this, u8 *data, u16 len) goto out; } rc = 0; - memcpy(&sig.daddr[0], pdata, 12); - pktlen = len - 12; - pdata += 12; + memcpy(&sig.addr, pdata, sig_addr_len); + pktlen = len - sig_addr_len; + pdata += sig_addr_len; sig.data = bf; if (((*pdata) * 256 + (*(pdata + 1))) > 1500) { u8 addr4[ETH_ALEN] = { @@ -589,7 +590,7 @@ static int wl3501_mgmt_join(struct wl3501_card *this, u16 stas) struct wl3501_join_req sig = { .sig_id = WL3501_SIG_JOIN_REQ, .timeout = 10, - .ds_pset = { + .req.ds_pset = { .el = { .id = IW_MGMT_INFO_ELEMENT_DS_PARAMETER_SET, .len = 1, @@ -598,7 +599,7 @@ static int wl3501_mgmt_join(struct wl3501_card *this, u16 stas) }, }; - memcpy(&sig.beacon_period, &this->bss_set[stas].beacon_period, 72); + memcpy(&sig.req, &this->bss_set[stas].req, sizeof(sig.req)); return wl3501_esbq_exec(this, &sig, sizeof(sig)); } @@ -666,35 +667,37 @@ static void wl3501_mgmt_scan_confirm(struct wl3501_card *this, u16 addr) if (sig.status == WL3501_STATUS_SUCCESS) { pr_debug("success"); if ((this->net_type == IW_MODE_INFRA && - (sig.cap_info & WL3501_MGMT_CAPABILITY_ESS)) || + (sig.req.cap_info & WL3501_MGMT_CAPABILITY_ESS)) || (this->net_type == IW_MODE_ADHOC && - (sig.cap_info & WL3501_MGMT_CAPABILITY_IBSS)) || + (sig.req.cap_info & WL3501_MGMT_CAPABILITY_IBSS)) || this->net_type == IW_MODE_AUTO) { if (!this->essid.el.len) matchflag = 1; else if (this->essid.el.len == 3 && !memcmp(this->essid.essid, "ANY", 3)) matchflag = 1; - else if (this->essid.el.len != sig.ssid.el.len) + else if (this->essid.el.len != sig.req.ssid.el.len) matchflag = 0; - else if (memcmp(this->essid.essid, sig.ssid.essid, + else if (memcmp(this->essid.essid, sig.req.ssid.essid, this->essid.el.len)) matchflag = 0; else matchflag = 1; if (matchflag) { for (i = 0; i < this->bss_cnt; i++) { - if (ether_addr_equal_unaligned(this->bss_set[i].bssid, sig.bssid)) { + if (ether_addr_equal_unaligned(this->bss_set[i].req.bssid, + sig.req.bssid)) { matchflag = 0; break; } } } if (matchflag && (i < 20)) { - memcpy(&this->bss_set[i].beacon_period, - &sig.beacon_period, 73); + memcpy(&this->bss_set[i].req, + &sig.req, sizeof(sig.req)); this->bss_cnt++; this->rssi = sig.rssi; + this->bss_set[i].rssi = sig.rssi; } } } else if (sig.status == WL3501_STATUS_TIMEOUT) { @@ -886,19 +889,19 @@ static void wl3501_mgmt_join_confirm(struct net_device *dev, u16 addr) if (this->join_sta_bss < this->bss_cnt) { const int i = this->join_sta_bss; memcpy(this->bssid, - this->bss_set[i].bssid, ETH_ALEN); - this->chan = this->bss_set[i].ds_pset.chan; + this->bss_set[i].req.bssid, ETH_ALEN); + this->chan = this->bss_set[i].req.ds_pset.chan; iw_copy_mgmt_info_element(&this->keep_essid.el, - &this->bss_set[i].ssid.el); + &this->bss_set[i].req.ssid.el); wl3501_mgmt_auth(this); } } else { const int i = this->join_sta_bss; - memcpy(&this->bssid, &this->bss_set[i].bssid, ETH_ALEN); - this->chan = this->bss_set[i].ds_pset.chan; + memcpy(&this->bssid, &this->bss_set[i].req.bssid, ETH_ALEN); + this->chan = this->bss_set[i].req.ds_pset.chan; iw_copy_mgmt_info_element(&this->keep_essid.el, - &this->bss_set[i].ssid.el); + &this->bss_set[i].req.ssid.el); wl3501_online(dev); } } else { @@ -980,7 +983,8 @@ static inline void wl3501_md_ind_interrupt(struct net_device *dev, } else { skb->dev = dev; skb_reserve(skb, 2); /* IP headers on 16 bytes boundaries */ - skb_copy_to_linear_data(skb, (unsigned char *)&sig.daddr, 12); + skb_copy_to_linear_data(skb, (unsigned char *)&sig.addr, + sizeof(sig.addr)); wl3501_receive(this, skb->data, pkt_len); skb_put(skb, pkt_len); skb->protocol = eth_type_trans(skb, dev); @@ -1571,30 +1575,30 @@ static int wl3501_get_scan(struct net_device *dev, struct iw_request_info *info, for (i = 0; i < this->bss_cnt; ++i) { iwe.cmd = SIOCGIWAP; iwe.u.ap_addr.sa_family = ARPHRD_ETHER; - memcpy(iwe.u.ap_addr.sa_data, this->bss_set[i].bssid, ETH_ALEN); + memcpy(iwe.u.ap_addr.sa_data, this->bss_set[i].req.bssid, ETH_ALEN); current_ev = iwe_stream_add_event(info, current_ev, extra + IW_SCAN_MAX_DATA, &iwe, IW_EV_ADDR_LEN); iwe.cmd = SIOCGIWESSID; iwe.u.data.flags = 1; - iwe.u.data.length = this->bss_set[i].ssid.el.len; + iwe.u.data.length = this->bss_set[i].req.ssid.el.len; current_ev = iwe_stream_add_point(info, current_ev, extra + IW_SCAN_MAX_DATA, &iwe, - this->bss_set[i].ssid.essid); + this->bss_set[i].req.ssid.essid); iwe.cmd = SIOCGIWMODE; - iwe.u.mode = this->bss_set[i].bss_type; + iwe.u.mode = this->bss_set[i].req.bss_type; current_ev = iwe_stream_add_event(info, current_ev, extra + IW_SCAN_MAX_DATA, &iwe, IW_EV_UINT_LEN); iwe.cmd = SIOCGIWFREQ; - iwe.u.freq.m = this->bss_set[i].ds_pset.chan; + iwe.u.freq.m = this->bss_set[i].req.ds_pset.chan; iwe.u.freq.e = 0; current_ev = iwe_stream_add_event(info, current_ev, extra + IW_SCAN_MAX_DATA, &iwe, IW_EV_FREQ_LEN); iwe.cmd = SIOCGIWENCODE; - if (this->bss_set[i].cap_info & WL3501_MGMT_CAPABILITY_PRIVACY) + if (this->bss_set[i].req.cap_info & WL3501_MGMT_CAPABILITY_PRIVACY) iwe.u.data.flags = IW_ENCODE_ENABLED | IW_ENCODE_NOKEY; else iwe.u.data.flags = IW_ENCODE_DISABLED; diff --git a/drivers/nfc/pn533/pn533.c b/drivers/nfc/pn533/pn533.c index f1469ac8ff42..3fe5b81eda2d 100644 --- a/drivers/nfc/pn533/pn533.c +++ b/drivers/nfc/pn533/pn533.c @@ -706,6 +706,9 @@ static bool pn533_target_type_a_is_valid(struct pn533_target_type_a *type_a, if (PN533_TYPE_A_SEL_CASCADE(type_a->sel_res) != 0) return false; + if (type_a->nfcid_len > NFC_NFCID1_MAXSIZE) + return false; + return true; } diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 0896e21642be..d5d7e0cdd78d 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -2681,7 +2681,8 @@ static void nvme_set_latency_tolerance(struct device *dev, s32 val) if (ctrl->ps_max_latency_us != latency) { ctrl->ps_max_latency_us = latency; - nvme_configure_apst(ctrl); + if (ctrl->state == NVME_CTRL_LIVE) + nvme_configure_apst(ctrl); } } diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index a1d476e1ac02..ec1e454848e5 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -668,6 +668,10 @@ void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id) if (desc.state) { /* found the group desc: update */ nvme_update_ns_ana_state(&desc, ns); + } else { + /* group desc not found: trigger a re-read */ + set_bit(NVME_NS_ANA_PENDING, &ns->flags); + queue_work(nvme_wq, &ns->ctrl->ana_work); } } else { ns->ana_state = NVME_ANA_OPTIMIZED; diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 7249ae74f71f..c92a15c3fbc5 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -852,7 +852,7 @@ static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req, return nvme_setup_prp_simple(dev, req, &cmnd->rw, &bv); - if (iod->nvmeq->qid && + if (iod->nvmeq->qid && sgl_threshold && dev->ctrl.sgls & ((1 << 0) | (1 << 1))) return nvme_setup_sgl_simple(dev, req, &cmnd->rw, &bv); diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index a0f00cb8f9f3..d7d7c81d0701 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -874,7 +874,7 @@ static void nvme_tcp_state_change(struct sock *sk) { struct nvme_tcp_queue *queue; - read_lock(&sk->sk_callback_lock); + read_lock_bh(&sk->sk_callback_lock); queue = sk->sk_user_data; if (!queue) goto done; @@ -895,7 +895,7 @@ static void nvme_tcp_state_change(struct sock *sk) queue->state_change(sk); done: - read_unlock(&sk->sk_callback_lock); + read_unlock_bh(&sk->sk_callback_lock); } static inline bool nvme_tcp_queue_more(struct nvme_tcp_queue *queue) diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c index fe6b8aa90b53..5a1ab49908c3 100644 --- a/drivers/nvme/target/admin-cmd.c +++ b/drivers/nvme/target/admin-cmd.c @@ -307,7 +307,7 @@ static void nvmet_execute_get_log_page(struct nvmet_req *req) case NVME_LOG_ANA: return nvmet_execute_get_log_page_ana(req); } - pr_err("unhandled lid %d on qid %d\n", + pr_debug("unhandled lid %d on qid %d\n", req->cmd->get_log_page.lid, req->sq->qid); req->error_loc = offsetof(struct nvme_get_log_page_command, lid); nvmet_req_complete(req, NVME_SC_INVALID_FIELD | NVME_SC_DNR); @@ -659,7 +659,7 @@ static void nvmet_execute_identify(struct nvmet_req *req) return nvmet_execute_identify_desclist(req); } - pr_err("unhandled identify cns %d on qid %d\n", + pr_debug("unhandled identify cns %d on qid %d\n", req->cmd->identify.cns, req->sq->qid); req->error_loc = offsetof(struct nvme_identify, cns); nvmet_req_complete(req, NVME_SC_INVALID_FIELD | NVME_SC_DNR); @@ -919,15 +919,21 @@ void nvmet_execute_async_event(struct nvmet_req *req) void nvmet_execute_keep_alive(struct nvmet_req *req) { struct nvmet_ctrl *ctrl = req->sq->ctrl; + u16 status = 0; if (!nvmet_check_transfer_len(req, 0)) return; + if (!ctrl->kato) { + status = NVME_SC_KA_TIMEOUT_INVALID; + goto out; + } + pr_debug("ctrl %d update keep-alive timer for %d secs\n", ctrl->cntlid, ctrl->kato); - mod_delayed_work(system_wq, &ctrl->ka_work, ctrl->kato * HZ); - nvmet_req_complete(req, 0); +out: + nvmet_req_complete(req, status); } u16 nvmet_parse_admin_cmd(struct nvmet_req *req) @@ -971,7 +977,7 @@ u16 nvmet_parse_admin_cmd(struct nvmet_req *req) return 0; } - pr_err("unhandled cmd %d on qid %d\n", cmd->common.opcode, + pr_debug("unhandled cmd %d on qid %d\n", cmd->common.opcode, req->sq->qid); req->error_loc = offsetof(struct nvme_common_command, opcode); return NVME_SC_INVALID_OPCODE | NVME_SC_DNR; diff --git a/drivers/nvme/target/discovery.c b/drivers/nvme/target/discovery.c index 682854e0e079..4845d12e374a 100644 --- a/drivers/nvme/target/discovery.c +++ b/drivers/nvme/target/discovery.c @@ -178,12 +178,14 @@ static void nvmet_execute_disc_get_log_page(struct nvmet_req *req) if (req->cmd->get_log_page.lid != NVME_LOG_DISC) { req->error_loc = offsetof(struct nvme_get_log_page_command, lid); - status = NVME_SC_INVALID_OPCODE | NVME_SC_DNR; + status = NVME_SC_INVALID_FIELD | NVME_SC_DNR; goto out; } /* Spec requires dword aligned offsets */ if (offset & 0x3) { + req->error_loc = + offsetof(struct nvme_get_log_page_command, lpo); status = NVME_SC_INVALID_FIELD | NVME_SC_DNR; goto out; } @@ -250,7 +252,7 @@ static void nvmet_execute_disc_identify(struct nvmet_req *req) if (req->cmd->identify.cns != NVME_ID_CNS_CTRL) { req->error_loc = offsetof(struct nvme_identify, cns); - status = NVME_SC_INVALID_OPCODE | NVME_SC_DNR; + status = NVME_SC_INVALID_FIELD | NVME_SC_DNR; goto out; } diff --git a/drivers/nvme/target/io-cmd-bdev.c b/drivers/nvme/target/io-cmd-bdev.c index 9a8b3726a37c..429263ca9b97 100644 --- a/drivers/nvme/target/io-cmd-bdev.c +++ b/drivers/nvme/target/io-cmd-bdev.c @@ -258,7 +258,7 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req) sector = nvmet_lba_to_sect(req->ns, req->cmd->rw.slba); - if (req->transfer_len <= NVMET_MAX_INLINE_DATA_LEN) { + if (nvmet_use_inline_bvec(req)) { bio = &req->b.inline_bio; bio_init(bio, req->inline_bvec, ARRAY_SIZE(req->inline_bvec)); } else { diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index 4b84edb49f22..5aad34b106dc 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -614,4 +614,10 @@ static inline sector_t nvmet_lba_to_sect(struct nvmet_ns *ns, __le64 lba) return le64_to_cpu(lba) << (ns->blksize_shift - SECTOR_SHIFT); } +static inline bool nvmet_use_inline_bvec(struct nvmet_req *req) +{ + return req->transfer_len <= NVMET_MAX_INLINE_DATA_LEN && + req->sg_cnt <= NVMET_MAX_INLINE_BIOVEC; +} + #endif /* _NVMET_H */ diff --git a/drivers/nvme/target/passthru.c b/drivers/nvme/target/passthru.c index 2798944899b7..39b1473f7204 100644 --- a/drivers/nvme/target/passthru.c +++ b/drivers/nvme/target/passthru.c @@ -194,7 +194,7 @@ static int nvmet_passthru_map_sg(struct nvmet_req *req, struct request *rq) if (req->sg_cnt > BIO_MAX_VECS) return -EINVAL; - if (req->transfer_len <= NVMET_MAX_INLINE_DATA_LEN) { + if (nvmet_use_inline_bvec(req)) { bio = &req->p.inline_bio; bio_init(bio, req->inline_bvec, ARRAY_SIZE(req->inline_bvec)); } else { diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c index 6c1f3ab7649c..7d607f435e36 100644 --- a/drivers/nvme/target/rdma.c +++ b/drivers/nvme/target/rdma.c @@ -700,7 +700,7 @@ static void nvmet_rdma_send_done(struct ib_cq *cq, struct ib_wc *wc) { struct nvmet_rdma_rsp *rsp = container_of(wc->wr_cqe, struct nvmet_rdma_rsp, send_cqe); - struct nvmet_rdma_queue *queue = cq->cq_context; + struct nvmet_rdma_queue *queue = wc->qp->qp_context; nvmet_rdma_release_rsp(rsp); @@ -786,7 +786,7 @@ static void nvmet_rdma_write_data_done(struct ib_cq *cq, struct ib_wc *wc) { struct nvmet_rdma_rsp *rsp = container_of(wc->wr_cqe, struct nvmet_rdma_rsp, write_cqe); - struct nvmet_rdma_queue *queue = cq->cq_context; + struct nvmet_rdma_queue *queue = wc->qp->qp_context; struct rdma_cm_id *cm_id = rsp->queue->cm_id; u16 status; diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index d658c6e8263a..d958b5da9b88 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -525,11 +525,36 @@ static void nvmet_tcp_queue_response(struct nvmet_req *req) struct nvmet_tcp_cmd *cmd = container_of(req, struct nvmet_tcp_cmd, req); struct nvmet_tcp_queue *queue = cmd->queue; + struct nvme_sgl_desc *sgl; + u32 len; + + if (unlikely(cmd == queue->cmd)) { + sgl = &cmd->req.cmd->common.dptr.sgl; + len = le32_to_cpu(sgl->length); + + /* + * Wait for inline data before processing the response. + * Avoid using helpers, this might happen before + * nvmet_req_init is completed. + */ + if (queue->rcv_state == NVMET_TCP_RECV_PDU && + len && len < cmd->req.port->inline_data_size && + nvme_is_write(cmd->req.cmd)) + return; + } llist_add(&cmd->lentry, &queue->resp_list); queue_work_on(queue_cpu(queue), nvmet_tcp_wq, &cmd->queue->io_work); } +static void nvmet_tcp_execute_request(struct nvmet_tcp_cmd *cmd) +{ + if (unlikely(cmd->flags & NVMET_TCP_F_INIT_FAILED)) + nvmet_tcp_queue_response(&cmd->req); + else + cmd->req.execute(&cmd->req); +} + static int nvmet_try_send_data_pdu(struct nvmet_tcp_cmd *cmd) { u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue); @@ -961,7 +986,7 @@ static int nvmet_tcp_done_recv_pdu(struct nvmet_tcp_queue *queue) le32_to_cpu(req->cmd->common.dptr.sgl.length)); nvmet_tcp_handle_req_failure(queue, queue->cmd, req); - return -EAGAIN; + return 0; } ret = nvmet_tcp_map_data(queue->cmd); @@ -1104,10 +1129,8 @@ static int nvmet_tcp_try_recv_data(struct nvmet_tcp_queue *queue) return 0; } - if (!(cmd->flags & NVMET_TCP_F_INIT_FAILED) && - cmd->rbytes_done == cmd->req.transfer_len) { - cmd->req.execute(&cmd->req); - } + if (cmd->rbytes_done == cmd->req.transfer_len) + nvmet_tcp_execute_request(cmd); nvmet_prepare_receive_pdu(queue); return 0; @@ -1144,9 +1167,9 @@ static int nvmet_tcp_try_recv_ddgst(struct nvmet_tcp_queue *queue) goto out; } - if (!(cmd->flags & NVMET_TCP_F_INIT_FAILED) && - cmd->rbytes_done == cmd->req.transfer_len) - cmd->req.execute(&cmd->req); + if (cmd->rbytes_done == cmd->req.transfer_len) + nvmet_tcp_execute_request(cmd); + ret = 0; out: nvmet_prepare_receive_pdu(queue); @@ -1434,7 +1457,7 @@ static void nvmet_tcp_state_change(struct sock *sk) { struct nvmet_tcp_queue *queue; - write_lock_bh(&sk->sk_callback_lock); + read_lock_bh(&sk->sk_callback_lock); queue = sk->sk_user_data; if (!queue) goto done; @@ -1452,7 +1475,7 @@ static void nvmet_tcp_state_change(struct sock *sk) queue->idx, sk->sk_state); } done: - write_unlock_bh(&sk->sk_callback_lock); + read_unlock_bh(&sk->sk_callback_lock); } static int nvmet_tcp_set_queue_sock(struct nvmet_tcp_queue *queue) diff --git a/drivers/nvmem/Kconfig b/drivers/nvmem/Kconfig index 75d2594c16e1..267a0d9e99ba 100644 --- a/drivers/nvmem/Kconfig +++ b/drivers/nvmem/Kconfig @@ -272,6 +272,7 @@ config SPRD_EFUSE config NVMEM_RMEM tristate "Reserved Memory Based Driver Support" + depends on HAS_IOMEM help This driver maps reserved memory into an nvmem device. It might be useful to expose information left by firmware in memory. diff --git a/drivers/nvmem/qfprom.c b/drivers/nvmem/qfprom.c index 6cace24dfbf7..100d69d8f2e1 100644 --- a/drivers/nvmem/qfprom.c +++ b/drivers/nvmem/qfprom.c @@ -127,6 +127,16 @@ static void qfprom_disable_fuse_blowing(const struct qfprom_priv *priv, { int ret; + /* + * This may be a shared rail and may be able to run at a lower rate + * when we're not blowing fuses. At the moment, the regulator framework + * applies voltage constraints even on disabled rails, so remove our + * constraints and allow the rail to be adjusted by other users. + */ + ret = regulator_set_voltage(priv->vcc, 0, INT_MAX); + if (ret) + dev_warn(priv->dev, "Failed to set 0 voltage (ignoring)\n"); + ret = regulator_disable(priv->vcc); if (ret) dev_warn(priv->dev, "Failed to disable regulator (ignoring)\n"); @@ -172,6 +182,17 @@ static int qfprom_enable_fuse_blowing(const struct qfprom_priv *priv, goto err_clk_prepared; } + /* + * Hardware requires 1.8V min for fuse blowing; this may be + * a rail shared do don't specify a max--regulator constraints + * will handle. + */ + ret = regulator_set_voltage(priv->vcc, 1800000, INT_MAX); + if (ret) { + dev_err(priv->dev, "Failed to set 1.8 voltage\n"); + goto err_clk_rate_set; + } + ret = regulator_enable(priv->vcc); if (ret) { dev_err(priv->dev, "Failed to enable regulator\n"); diff --git a/drivers/of/overlay.c b/drivers/of/overlay.c index 23effe5e50ec..2d132949572d 100644 --- a/drivers/of/overlay.c +++ b/drivers/of/overlay.c @@ -796,6 +796,7 @@ static int init_overlay_changeset(struct overlay_changeset *ovcs, if (!fragment->target) { of_node_put(fragment->overlay); ret = -EINVAL; + of_node_put(node); goto err_free_fragments; } diff --git a/drivers/pci/controller/dwc/pci-keystone.c b/drivers/pci/controller/dwc/pci-keystone.c index 53aa35cb3a49..a59ecbec601f 100644 --- a/drivers/pci/controller/dwc/pci-keystone.c +++ b/drivers/pci/controller/dwc/pci-keystone.c @@ -798,7 +798,8 @@ static int __init ks_pcie_host_init(struct pcie_port *pp) int ret; pp->bridge->ops = &ks_pcie_ops; - pp->bridge->child_ops = &ks_child_pcie_ops; + if (!ks_pcie->is_am6) + pp->bridge->child_ops = &ks_child_pcie_ops; ret = ks_pcie_config_legacy_irq(ks_pcie); if (ret) diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c index 1c25d8337151..8d028a88b375 100644 --- a/drivers/pci/controller/dwc/pcie-designware-ep.c +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c @@ -705,6 +705,8 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep) } } + dw_pcie_iatu_detect(pci); + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "addr_space"); if (!res) return -EINVAL; diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c index 7e55b2b66182..24192b40e3a2 100644 --- a/drivers/pci/controller/dwc/pcie-designware-host.c +++ b/drivers/pci/controller/dwc/pcie-designware-host.c @@ -398,6 +398,7 @@ int dw_pcie_host_init(struct pcie_port *pp) if (ret) goto err_free_msi; } + dw_pcie_iatu_detect(pci); dw_pcie_setup_rc(pp); dw_pcie_msi_init(pp); diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c index 004cb860e266..a945f0c0e73d 100644 --- a/drivers/pci/controller/dwc/pcie-designware.c +++ b/drivers/pci/controller/dwc/pcie-designware.c @@ -660,11 +660,9 @@ static void dw_pcie_iatu_detect_regions(struct dw_pcie *pci) pci->num_ob_windows = ob; } -void dw_pcie_setup(struct dw_pcie *pci) +void dw_pcie_iatu_detect(struct dw_pcie *pci) { - u32 val; struct device *dev = pci->dev; - struct device_node *np = dev->of_node; struct platform_device *pdev = to_platform_device(dev); if (pci->version >= 0x480A || (!pci->version && @@ -693,6 +691,13 @@ void dw_pcie_setup(struct dw_pcie *pci) dev_info(pci->dev, "Detected iATU regions: %u outbound, %u inbound", pci->num_ob_windows, pci->num_ib_windows); +} + +void dw_pcie_setup(struct dw_pcie *pci) +{ + u32 val; + struct device *dev = pci->dev; + struct device_node *np = dev->of_node; if (pci->link_gen > 0) dw_pcie_link_set_max_speed(pci, pci->link_gen); diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h index 7247c8b01f04..7d6e9b7576be 100644 --- a/drivers/pci/controller/dwc/pcie-designware.h +++ b/drivers/pci/controller/dwc/pcie-designware.h @@ -306,6 +306,7 @@ int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, u8 func_no, int index, void dw_pcie_disable_atu(struct dw_pcie *pci, int index, enum dw_pcie_region_type type); void dw_pcie_setup(struct dw_pcie *pci); +void dw_pcie_iatu_detect(struct dw_pcie *pci); static inline void dw_pcie_writel_dbi(struct dw_pcie *pci, u32 reg, u32 val) { diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c index 6fa216e52d14..0e94190ca4e8 100644 --- a/drivers/pci/controller/dwc/pcie-tegra194.c +++ b/drivers/pci/controller/dwc/pcie-tegra194.c @@ -1645,7 +1645,7 @@ static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie) if (pcie->ep_state == EP_STATE_ENABLED) return; - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0) { dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n", ret); diff --git a/drivers/pci/controller/pci-thunder-ecam.c b/drivers/pci/controller/pci-thunder-ecam.c index f964fd26f7e0..ffd84656544f 100644 --- a/drivers/pci/controller/pci-thunder-ecam.c +++ b/drivers/pci/controller/pci-thunder-ecam.c @@ -116,7 +116,7 @@ static int thunder_ecam_p2_config_read(struct pci_bus *bus, unsigned int devfn, * the config space access window. Since we are working with * the high-order 32 bits, shift everything down by 32 bits. */ - node_bits = (cfg->res.start >> 32) & (1 << 12); + node_bits = upper_32_bits(cfg->res.start) & (1 << 12); v |= node_bits; set_val(v, where, size, val); diff --git a/drivers/pci/controller/pci-thunder-pem.c b/drivers/pci/controller/pci-thunder-pem.c index 1a3f70ac61fc..0660b9da204f 100644 --- a/drivers/pci/controller/pci-thunder-pem.c +++ b/drivers/pci/controller/pci-thunder-pem.c @@ -12,6 +12,7 @@ #include #include #include +#include #include "../pci.h" #if defined(CONFIG_PCI_HOST_THUNDER_PEM) || (defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS)) @@ -324,9 +325,9 @@ static int thunder_pem_init(struct device *dev, struct pci_config_window *cfg, * structure here for the BAR. */ bar4_start = res_pem->start + 0xf00000; - pem_pci->ea_entry[0] = (u32)bar4_start | 2; - pem_pci->ea_entry[1] = (u32)(res_pem->end - bar4_start) & ~3u; - pem_pci->ea_entry[2] = (u32)(bar4_start >> 32); + pem_pci->ea_entry[0] = lower_32_bits(bar4_start) | 2; + pem_pci->ea_entry[1] = lower_32_bits(res_pem->end - bar4_start) & ~3u; + pem_pci->ea_entry[2] = upper_32_bits(bar4_start); cfg->priv = pem_pci; return 0; @@ -334,9 +335,9 @@ static int thunder_pem_init(struct device *dev, struct pci_config_window *cfg, #if defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS) -#define PEM_RES_BASE 0x87e0c0000000UL -#define PEM_NODE_MASK GENMASK(45, 44) -#define PEM_INDX_MASK GENMASK(26, 24) +#define PEM_RES_BASE 0x87e0c0000000ULL +#define PEM_NODE_MASK GENMASK_ULL(45, 44) +#define PEM_INDX_MASK GENMASK_ULL(26, 24) #define PEM_MIN_DOM_IN_NODE 4 #define PEM_MAX_DOM_IN_NODE 10 diff --git a/drivers/pci/controller/pci-xgene.c b/drivers/pci/controller/pci-xgene.c index 2afdc865253e..7f503dd4ff81 100644 --- a/drivers/pci/controller/pci-xgene.c +++ b/drivers/pci/controller/pci-xgene.c @@ -354,7 +354,8 @@ static int xgene_pcie_map_reg(struct xgene_pcie_port *port, if (IS_ERR(port->csr_base)) return PTR_ERR(port->csr_base); - port->cfg_base = devm_platform_ioremap_resource_byname(pdev, "cfg"); + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg"); + port->cfg_base = devm_ioremap_resource(dev, res); if (IS_ERR(port->cfg_base)) return PTR_ERR(port->cfg_base); port->cfg_addr = res->start; diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c index e330e6811f0b..08bc788d9422 100644 --- a/drivers/pci/controller/pcie-brcmstb.c +++ b/drivers/pci/controller/pcie-brcmstb.c @@ -1148,6 +1148,7 @@ static int brcm_pcie_suspend(struct device *dev) brcm_pcie_turn_off(pcie); ret = brcm_phy_stop(pcie); + reset_control_rearm(pcie->rescal); clk_disable_unprepare(pcie->clk); return ret; @@ -1163,9 +1164,13 @@ static int brcm_pcie_resume(struct device *dev) base = pcie->base; clk_prepare_enable(pcie->clk); + ret = reset_control_reset(pcie->rescal); + if (ret) + goto err_disable_clk; + ret = brcm_phy_start(pcie); if (ret) - goto err; + goto err_reset; /* Take bridge out of reset so we can access the SERDES reg */ pcie->bridge_sw_init_set(pcie, 0); @@ -1180,14 +1185,16 @@ static int brcm_pcie_resume(struct device *dev) ret = brcm_pcie_setup(pcie); if (ret) - goto err; + goto err_reset; if (pcie->msi) brcm_msi_set_regs(pcie->msi); return 0; -err: +err_reset: + reset_control_rearm(pcie->rescal); +err_disable_clk: clk_disable_unprepare(pcie->clk); return ret; } @@ -1197,7 +1204,7 @@ static void __brcm_pcie_remove(struct brcm_pcie *pcie) brcm_msi_remove(pcie); brcm_pcie_turn_off(pcie); brcm_phy_stop(pcie); - reset_control_assert(pcie->rescal); + reset_control_rearm(pcie->rescal); clk_disable_unprepare(pcie->clk); } @@ -1278,13 +1285,13 @@ static int brcm_pcie_probe(struct platform_device *pdev) return PTR_ERR(pcie->perst_reset); } - ret = reset_control_deassert(pcie->rescal); + ret = reset_control_reset(pcie->rescal); if (ret) dev_err(&pdev->dev, "failed to deassert 'rescal'\n"); ret = brcm_phy_start(pcie); if (ret) { - reset_control_assert(pcie->rescal); + reset_control_rearm(pcie->rescal); clk_disable_unprepare(pcie->clk); return ret; } @@ -1296,6 +1303,7 @@ static int brcm_pcie_probe(struct platform_device *pdev) pcie->hw_rev = readl(pcie->base + PCIE_MISC_REVISION); if (pcie->type == BCM4908 && pcie->hw_rev >= BRCM_PCIE_HW_REV_3_20) { dev_err(pcie->dev, "hardware revision with unsupported PERST# setup\n"); + ret = -ENODEV; goto fail; } diff --git a/drivers/pci/controller/pcie-iproc-msi.c b/drivers/pci/controller/pcie-iproc-msi.c index 908475d27e0e..eede4e8f3f75 100644 --- a/drivers/pci/controller/pcie-iproc-msi.c +++ b/drivers/pci/controller/pcie-iproc-msi.c @@ -271,7 +271,7 @@ static int iproc_msi_irq_domain_alloc(struct irq_domain *domain, NULL, NULL); } - return hwirq; + return 0; } static void iproc_msi_irq_domain_free(struct irq_domain *domain, diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c index c0ac4e9cbe72..f9760e73d568 100644 --- a/drivers/pci/endpoint/functions/pci-epf-test.c +++ b/drivers/pci/endpoint/functions/pci-epf-test.c @@ -833,15 +833,18 @@ static int pci_epf_test_bind(struct pci_epf *epf) return -EINVAL; epc_features = pci_epc_get_features(epc, epf->func_no); - if (epc_features) { - linkup_notifier = epc_features->linkup_notifier; - core_init_notifier = epc_features->core_init_notifier; - test_reg_bar = pci_epc_get_first_free_bar(epc_features); - if (test_reg_bar < 0) - return -EINVAL; - pci_epf_configure_bar(epf, epc_features); + if (!epc_features) { + dev_err(&epf->dev, "epc_features not implemented\n"); + return -EOPNOTSUPP; } + linkup_notifier = epc_features->linkup_notifier; + core_init_notifier = epc_features->core_init_notifier; + test_reg_bar = pci_epc_get_first_free_bar(epc_features); + if (test_reg_bar < 0) + return -EINVAL; + pci_epf_configure_bar(epf, epc_features); + epf_test->test_reg_bar = test_reg_bar; epf_test->epc_features = epc_features; @@ -922,6 +925,7 @@ static int __init pci_epf_test_init(void) ret = pci_epf_register_driver(&test_driver); if (ret) { + destroy_workqueue(kpcitest_workqueue); pr_err("Failed to register pci epf test driver --> %d\n", ret); return ret; } @@ -932,6 +936,8 @@ module_init(pci_epf_test_init); static void __exit pci_epf_test_exit(void) { + if (kpcitest_workqueue) + destroy_workqueue(kpcitest_workqueue); pci_epf_unregister_driver(&test_driver); } module_exit(pci_epf_test_exit); diff --git a/drivers/pci/hotplug/acpiphp_glue.c b/drivers/pci/hotplug/acpiphp_glue.c index 3365c93abf0e..f031302ad401 100644 --- a/drivers/pci/hotplug/acpiphp_glue.c +++ b/drivers/pci/hotplug/acpiphp_glue.c @@ -533,6 +533,7 @@ static void enable_slot(struct acpiphp_slot *slot, bool bridge) slot->flags &= ~SLOT_ENABLED; continue; } + pci_dev_put(dev); } } diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 16a17215f633..e4d4e399004b 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -1870,20 +1870,10 @@ static int pci_enable_device_flags(struct pci_dev *dev, unsigned long flags) int err; int i, bars = 0; - /* - * Power state could be unknown at this point, either due to a fresh - * boot or a device removal call. So get the current power state - * so that things like MSI message writing will behave as expected - * (e.g. if the device really is in D0 at enable time). - */ - if (dev->pm_cap) { - u16 pmcsr; - pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); - dev->current_state = (pmcsr & PCI_PM_CTRL_STATE_MASK); - } - - if (atomic_inc_return(&dev->enable_cnt) > 1) + if (atomic_inc_return(&dev->enable_cnt) > 1) { + pci_update_current_state(dev, dev->current_state); return 0; /* already enabled */ + } bridge = pci_upstream_bridge(dev); if (bridge) diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index ef7c4661314f..9684b468267f 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -624,6 +624,12 @@ static inline int pci_dev_specific_reset(struct pci_dev *dev, int probe) #if defined(CONFIG_PCI_QUIRKS) && defined(CONFIG_ARM64) int acpi_get_rc_resources(struct device *dev, const char *hid, u16 segment, struct resource *res); +#else +static inline int acpi_get_rc_resources(struct device *dev, const char *hid, + u16 segment, struct resource *res) +{ + return -ENODEV; +} #endif int pci_rebar_get_current_size(struct pci_dev *pdev, int bar); diff --git a/drivers/pci/pcie/rcec.c b/drivers/pci/pcie/rcec.c index 2c5c552994e4..d0bcd141ac9c 100644 --- a/drivers/pci/pcie/rcec.c +++ b/drivers/pci/pcie/rcec.c @@ -32,7 +32,7 @@ static bool rcec_assoc_rciep(struct pci_dev *rcec, struct pci_dev *rciep) /* Same bus, so check bitmap */ for_each_set_bit(devn, &bitmap, 32) - if (devn == rciep->devfn) + if (devn == PCI_SLOT(rciep->devfn)) return true; return false; diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 953f15abc850..be51670572fa 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -2353,6 +2353,7 @@ static struct pci_dev *pci_scan_device(struct pci_bus *bus, int devfn) pci_set_of_node(dev); if (pci_setup_device(dev)) { + pci_release_of_node(dev); pci_bus_put(dev->bus); kfree(dev); return NULL; diff --git a/drivers/pci/vpd.c b/drivers/pci/vpd.c index 7915d10f9aa1..bd549070c011 100644 --- a/drivers/pci/vpd.c +++ b/drivers/pci/vpd.c @@ -570,7 +570,6 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LSI_LOGIC, 0x005d, quirk_blacklist_vpd); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LSI_LOGIC, 0x005f, quirk_blacklist_vpd); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATTANSIC, PCI_ANY_ID, quirk_blacklist_vpd); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_QLOGIC, 0x2261, quirk_blacklist_vpd); /* * The Amazon Annapurna Labs 0x0031 device id is reused for other non Root Port * device types, so the quirk is registered for the PCI_CLASS_BRIDGE_PCI class. diff --git a/drivers/perf/arm_pmu_platform.c b/drivers/perf/arm_pmu_platform.c index 933bd8410fc2..ef9676418c9f 100644 --- a/drivers/perf/arm_pmu_platform.c +++ b/drivers/perf/arm_pmu_platform.c @@ -6,6 +6,7 @@ * Copyright (C) 2010 ARM Ltd., Will Deacon */ #define pr_fmt(fmt) "hw perfevents: " fmt +#define dev_fmt pr_fmt #include #include @@ -100,10 +101,8 @@ static int pmu_parse_irqs(struct arm_pmu *pmu) struct pmu_hw_events __percpu *hw_events = pmu->hw_events; num_irqs = platform_irq_count(pdev); - if (num_irqs < 0) { - pr_err("unable to count PMU IRQs\n"); - return num_irqs; - } + if (num_irqs < 0) + return dev_err_probe(&pdev->dev, num_irqs, "unable to count PMU IRQs\n"); /* * In this case we have no idea which CPUs are covered by the PMU. @@ -236,7 +235,7 @@ int arm_pmu_device_probe(struct platform_device *pdev, ret = armpmu_register(pmu); if (ret) - goto out_free; + goto out_free_irqs; return 0; diff --git a/drivers/phy/cadence/phy-cadence-sierra.c b/drivers/phy/cadence/phy-cadence-sierra.c index 26a0badabe38..19f32ae877b9 100644 --- a/drivers/phy/cadence/phy-cadence-sierra.c +++ b/drivers/phy/cadence/phy-cadence-sierra.c @@ -319,6 +319,12 @@ static int cdns_sierra_phy_on(struct phy *gphy) u32 val; int ret; + ret = reset_control_deassert(sp->phy_rst); + if (ret) { + dev_err(dev, "Failed to take the PHY out of reset\n"); + return ret; + } + /* Take the PHY lane group out of reset */ ret = reset_control_deassert(ins->lnk_rst); if (ret) { @@ -616,7 +622,6 @@ static int cdns_sierra_phy_probe(struct platform_device *pdev) pm_runtime_enable(dev); phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate); - reset_control_deassert(sp->phy_rst); return PTR_ERR_OR_ZERO(phy_provider); put_child: diff --git a/drivers/phy/ingenic/phy-ingenic-usb.c b/drivers/phy/ingenic/phy-ingenic-usb.c index ea127b177f46..28c28d816484 100644 --- a/drivers/phy/ingenic/phy-ingenic-usb.c +++ b/drivers/phy/ingenic/phy-ingenic-usb.c @@ -352,8 +352,8 @@ static int ingenic_usb_phy_probe(struct platform_device *pdev) } priv->phy = devm_phy_create(dev, NULL, &ingenic_usb_phy_ops); - if (IS_ERR(priv)) - return PTR_ERR(priv); + if (IS_ERR(priv->phy)) + return PTR_ERR(priv->phy); phy_set_drvdata(priv->phy, priv); diff --git a/drivers/phy/marvell/Kconfig b/drivers/phy/marvell/Kconfig index 6c96f2bf5266..c8ee23fc3a83 100644 --- a/drivers/phy/marvell/Kconfig +++ b/drivers/phy/marvell/Kconfig @@ -3,8 +3,8 @@ # Phy drivers for Marvell platforms # config ARMADA375_USBCLUSTER_PHY - def_bool y - depends on MACH_ARMADA_375 || COMPILE_TEST + bool "Armada 375 USB cluster PHY support" if COMPILE_TEST + default y if MACH_ARMADA_375 depends on OF && HAS_IOMEM select GENERIC_PHY diff --git a/drivers/phy/ralink/phy-mt7621-pci.c b/drivers/phy/ralink/phy-mt7621-pci.c index 9a610b414b1f..753cb5bab930 100644 --- a/drivers/phy/ralink/phy-mt7621-pci.c +++ b/drivers/phy/ralink/phy-mt7621-pci.c @@ -62,7 +62,7 @@ #define RG_PE1_FRC_MSTCKDIV BIT(5) -#define XTAL_MASK GENMASK(7, 6) +#define XTAL_MASK GENMASK(8, 6) #define MAX_PHYS 2 @@ -319,9 +319,9 @@ static int mt7621_pci_phy_probe(struct platform_device *pdev) return PTR_ERR(phy->regmap); phy->phy = devm_phy_create(dev, dev->of_node, &mt7621_pci_phy_ops); - if (IS_ERR(phy)) { + if (IS_ERR(phy->phy)) { dev_err(dev, "failed to create phy\n"); - return PTR_ERR(phy); + return PTR_ERR(phy->phy); } phy_set_drvdata(phy->phy, phy); diff --git a/drivers/phy/ti/phy-j721e-wiz.c b/drivers/phy/ti/phy-j721e-wiz.c index c9cfafe89cbf..e28e25f98708 100644 --- a/drivers/phy/ti/phy-j721e-wiz.c +++ b/drivers/phy/ti/phy-j721e-wiz.c @@ -615,6 +615,12 @@ static void wiz_clock_cleanup(struct wiz *wiz, struct device_node *node) of_clk_del_provider(clk_node); of_node_put(clk_node); } + + for (i = 0; i < wiz->clk_div_sel_num; i++) { + clk_node = of_get_child_by_name(node, clk_div_sel[i].node_name); + of_clk_del_provider(clk_node); + of_node_put(clk_node); + } } static int wiz_clock_init(struct wiz *wiz, struct device_node *node) @@ -947,27 +953,24 @@ static int wiz_probe(struct platform_device *pdev) goto err_get_sync; } + ret = wiz_init(wiz); + if (ret) { + dev_err(dev, "WIZ initialization failed\n"); + goto err_wiz_init; + } + serdes_pdev = of_platform_device_create(child_node, NULL, dev); if (!serdes_pdev) { dev_WARN(dev, "Unable to create SERDES platform device\n"); ret = -ENOMEM; - goto err_pdev_create; - } - wiz->serdes_pdev = serdes_pdev; - - ret = wiz_init(wiz); - if (ret) { - dev_err(dev, "WIZ initialization failed\n"); goto err_wiz_init; } + wiz->serdes_pdev = serdes_pdev; of_node_put(child_node); return 0; err_wiz_init: - of_platform_device_destroy(&serdes_pdev->dev, NULL); - -err_pdev_create: wiz_clock_cleanup(wiz, node); err_get_sync: diff --git a/drivers/phy/ti/phy-twl4030-usb.c b/drivers/phy/ti/phy-twl4030-usb.c index 9887f908f540..812e5409d359 100644 --- a/drivers/phy/ti/phy-twl4030-usb.c +++ b/drivers/phy/ti/phy-twl4030-usb.c @@ -779,7 +779,7 @@ static int twl4030_usb_remove(struct platform_device *pdev) usb_remove_phy(&twl->phy); pm_runtime_get_sync(twl->dev); - cancel_delayed_work(&twl->id_workaround_work); + cancel_delayed_work_sync(&twl->id_workaround_work); device_remove_file(twl->dev, &dev_attr_vbus); /* set transceiver mode to power on defaults */ diff --git a/drivers/pinctrl/pinctrl-at91-pio4.c b/drivers/pinctrl/pinctrl-at91-pio4.c index e71ebccc479c..03c32b2c5d30 100644 --- a/drivers/pinctrl/pinctrl-at91-pio4.c +++ b/drivers/pinctrl/pinctrl-at91-pio4.c @@ -801,6 +801,10 @@ static int atmel_conf_pin_config_group_set(struct pinctrl_dev *pctldev, conf = atmel_pin_config_read(pctldev, pin_id); + /* Keep slew rate enabled by default. */ + if (atmel_pioctrl->slew_rate_support) + conf |= ATMEL_PIO_SR_MASK; + for (i = 0; i < num_configs; i++) { unsigned int param = pinconf_to_config_param(configs[i]); unsigned int arg = pinconf_to_config_argument(configs[i]); @@ -808,10 +812,6 @@ static int atmel_conf_pin_config_group_set(struct pinctrl_dev *pctldev, dev_dbg(pctldev->dev, "%s: pin=%u, config=0x%lx\n", __func__, pin_id, configs[i]); - /* Keep slew rate enabled by default. */ - if (atmel_pioctrl->slew_rate_support) - conf |= ATMEL_PIO_SR_MASK; - switch (param) { case PIN_CONFIG_BIAS_DISABLE: conf &= (~ATMEL_PIO_PUEN_MASK); diff --git a/drivers/pinctrl/pinctrl-ingenic.c b/drivers/pinctrl/pinctrl-ingenic.c index f2746125b077..3de0f767b7d1 100644 --- a/drivers/pinctrl/pinctrl-ingenic.c +++ b/drivers/pinctrl/pinctrl-ingenic.c @@ -667,7 +667,9 @@ static int jz4770_pwm_pwm7_pins[] = { 0x6b, }; static int jz4770_mac_rmii_pins[] = { 0xa9, 0xab, 0xaa, 0xac, 0xa5, 0xa4, 0xad, 0xae, 0xa6, 0xa8, }; -static int jz4770_mac_mii_pins[] = { 0xa7, 0xaf, }; +static int jz4770_mac_mii_pins[] = { + 0x7b, 0x7a, 0x7d, 0x7c, 0xa7, 0x24, 0xaf, +}; static const struct group_desc jz4770_groups[] = { INGENIC_PIN_GROUP("uart0-data", jz4770_uart0_data, 0), @@ -2107,26 +2109,48 @@ static int ingenic_pinconf_get(struct pinctrl_dev *pctldev, enum pin_config_param param = pinconf_to_config_param(*config); unsigned int idx = pin % PINS_PER_GPIO_CHIP; unsigned int offt = pin / PINS_PER_GPIO_CHIP; - bool pull; + unsigned int bias; + bool pull, pullup, pulldown; - if (jzpc->info->version >= ID_JZ4770) - pull = !ingenic_get_pin_config(jzpc, pin, JZ4770_GPIO_PEN); - else - pull = !ingenic_get_pin_config(jzpc, pin, JZ4740_GPIO_PULL_DIS); + if (jzpc->info->version >= ID_X1830) { + unsigned int half = PINS_PER_GPIO_CHIP / 2; + unsigned int idxh = (pin % half) * 2; + + if (idx < half) + regmap_read(jzpc->map, offt * jzpc->info->reg_offset + + X1830_GPIO_PEL, &bias); + else + regmap_read(jzpc->map, offt * jzpc->info->reg_offset + + X1830_GPIO_PEH, &bias); + + bias = (bias >> idxh) & (GPIO_PULL_UP | GPIO_PULL_DOWN); + + pullup = (bias == GPIO_PULL_UP) && (jzpc->info->pull_ups[offt] & BIT(idx)); + pulldown = (bias == GPIO_PULL_DOWN) && (jzpc->info->pull_downs[offt] & BIT(idx)); + + } else { + if (jzpc->info->version >= ID_JZ4770) + pull = !ingenic_get_pin_config(jzpc, pin, JZ4770_GPIO_PEN); + else + pull = !ingenic_get_pin_config(jzpc, pin, JZ4740_GPIO_PULL_DIS); + + pullup = pull && (jzpc->info->pull_ups[offt] & BIT(idx)); + pulldown = pull && (jzpc->info->pull_downs[offt] & BIT(idx)); + } switch (param) { case PIN_CONFIG_BIAS_DISABLE: - if (pull) + if (pullup || pulldown) return -EINVAL; break; case PIN_CONFIG_BIAS_PULL_UP: - if (!pull || !(jzpc->info->pull_ups[offt] & BIT(idx))) + if (!pullup) return -EINVAL; break; case PIN_CONFIG_BIAS_PULL_DOWN: - if (!pull || !(jzpc->info->pull_downs[offt] & BIT(idx))) + if (!pulldown) return -EINVAL; break; @@ -2144,7 +2168,7 @@ static void ingenic_set_bias(struct ingenic_pinctrl *jzpc, if (jzpc->info->version >= ID_X1830) { unsigned int idx = pin % PINS_PER_GPIO_CHIP; unsigned int half = PINS_PER_GPIO_CHIP / 2; - unsigned int idxh = pin % half * 2; + unsigned int idxh = (pin % half) * 2; unsigned int offt = pin / PINS_PER_GPIO_CHIP; if (idx < half) { diff --git a/drivers/pinctrl/pinctrl-single.c b/drivers/pinctrl/pinctrl-single.c index 7771316dfffa..10890fde9a75 100644 --- a/drivers/pinctrl/pinctrl-single.c +++ b/drivers/pinctrl/pinctrl-single.c @@ -270,20 +270,44 @@ static void __maybe_unused pcs_writel(unsigned val, void __iomem *reg) writel(val, reg); } +static unsigned int pcs_pin_reg_offset_get(struct pcs_device *pcs, + unsigned int pin) +{ + unsigned int mux_bytes = pcs->width / BITS_PER_BYTE; + + if (pcs->bits_per_mux) { + unsigned int pin_offset_bytes; + + pin_offset_bytes = (pcs->bits_per_pin * pin) / BITS_PER_BYTE; + return (pin_offset_bytes / mux_bytes) * mux_bytes; + } + + return pin * mux_bytes; +} + +static unsigned int pcs_pin_shift_reg_get(struct pcs_device *pcs, + unsigned int pin) +{ + return (pin % (pcs->width / pcs->bits_per_pin)) * pcs->bits_per_pin; +} + static void pcs_pin_dbg_show(struct pinctrl_dev *pctldev, struct seq_file *s, unsigned pin) { struct pcs_device *pcs; - unsigned val, mux_bytes; + unsigned int val; unsigned long offset; size_t pa; pcs = pinctrl_dev_get_drvdata(pctldev); - mux_bytes = pcs->width / BITS_PER_BYTE; - offset = pin * mux_bytes; + offset = pcs_pin_reg_offset_get(pcs, pin); val = pcs->read(pcs->base + offset); + + if (pcs->bits_per_mux) + val &= pcs->fmask << pcs_pin_shift_reg_get(pcs, pin); + pa = pcs->res->start + offset; seq_printf(s, "%zx %08x %s ", pa, val, DRIVER_NAME); @@ -384,7 +408,6 @@ static int pcs_request_gpio(struct pinctrl_dev *pctldev, struct pcs_device *pcs = pinctrl_dev_get_drvdata(pctldev); struct pcs_gpiofunc_range *frange = NULL; struct list_head *pos, *tmp; - int mux_bytes = 0; unsigned data; /* If function mask is null, return directly. */ @@ -392,29 +415,27 @@ static int pcs_request_gpio(struct pinctrl_dev *pctldev, return -ENOTSUPP; list_for_each_safe(pos, tmp, &pcs->gpiofuncs) { + u32 offset; + frange = list_entry(pos, struct pcs_gpiofunc_range, node); if (pin >= frange->offset + frange->npins || pin < frange->offset) continue; - mux_bytes = pcs->width / BITS_PER_BYTE; - if (pcs->bits_per_mux) { - int byte_num, offset, pin_shift; + offset = pcs_pin_reg_offset_get(pcs, pin); - byte_num = (pcs->bits_per_pin * pin) / BITS_PER_BYTE; - offset = (byte_num / mux_bytes) * mux_bytes; - pin_shift = pin % (pcs->width / pcs->bits_per_pin) * - pcs->bits_per_pin; + if (pcs->bits_per_mux) { + int pin_shift = pcs_pin_shift_reg_get(pcs, pin); data = pcs->read(pcs->base + offset); data &= ~(pcs->fmask << pin_shift); data |= frange->gpiofunc << pin_shift; pcs->write(data, pcs->base + offset); } else { - data = pcs->read(pcs->base + pin * mux_bytes); + data = pcs->read(pcs->base + offset); data &= ~pcs->fmask; data |= frange->gpiofunc; - pcs->write(data, pcs->base + pin * mux_bytes); + pcs->write(data, pcs->base + offset); } break; } @@ -656,10 +677,8 @@ static const struct pinconf_ops pcs_pinconf_ops = { * pcs_add_pin() - add a pin to the static per controller pin array * @pcs: pcs driver instance * @offset: register offset from base - * @pin_pos: unused */ -static int pcs_add_pin(struct pcs_device *pcs, unsigned offset, - unsigned pin_pos) +static int pcs_add_pin(struct pcs_device *pcs, unsigned int offset) { struct pcs_soc_data *pcs_soc = &pcs->socdata; struct pinctrl_pin_desc *pin; @@ -728,17 +747,9 @@ static int pcs_allocate_pin_table(struct pcs_device *pcs) for (i = 0; i < pcs->desc.npins; i++) { unsigned offset; int res; - int byte_num; - int pin_pos = 0; - if (pcs->bits_per_mux) { - byte_num = (pcs->bits_per_pin * i) / BITS_PER_BYTE; - offset = (byte_num / mux_bytes) * mux_bytes; - pin_pos = i % num_pins_in_register; - } else { - offset = i * mux_bytes; - } - res = pcs_add_pin(pcs, offset, pin_pos); + offset = pcs_pin_reg_offset_get(pcs, i); + res = pcs_add_pin(pcs, offset); if (res < 0) { dev_err(pcs->dev, "error adding pins: %i\n", res); return res; diff --git a/drivers/pinctrl/samsung/pinctrl-exynos.c b/drivers/pinctrl/samsung/pinctrl-exynos.c index 0cd7f33cdf25..2b99f4130e1e 100644 --- a/drivers/pinctrl/samsung/pinctrl-exynos.c +++ b/drivers/pinctrl/samsung/pinctrl-exynos.c @@ -55,7 +55,7 @@ static void exynos_irq_mask(struct irq_data *irqd) struct exynos_irq_chip *our_chip = to_exynos_irq_chip(chip); struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(irqd); unsigned long reg_mask = our_chip->eint_mask + bank->eint_offset; - unsigned long mask; + unsigned int mask; unsigned long flags; raw_spin_lock_irqsave(&bank->slock, flags); @@ -83,7 +83,7 @@ static void exynos_irq_unmask(struct irq_data *irqd) struct exynos_irq_chip *our_chip = to_exynos_irq_chip(chip); struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(irqd); unsigned long reg_mask = our_chip->eint_mask + bank->eint_offset; - unsigned long mask; + unsigned int mask; unsigned long flags; /* @@ -483,7 +483,7 @@ static void exynos_irq_eint0_15(struct irq_desc *desc) chained_irq_exit(chip, desc); } -static inline void exynos_irq_demux_eint(unsigned long pend, +static inline void exynos_irq_demux_eint(unsigned int pend, struct irq_domain *domain) { unsigned int irq; @@ -500,8 +500,8 @@ static void exynos_irq_demux_eint16_31(struct irq_desc *desc) { struct irq_chip *chip = irq_desc_get_chip(desc); struct exynos_muxed_weint_data *eintd = irq_desc_get_handler_data(desc); - unsigned long pend; - unsigned long mask; + unsigned int pend; + unsigned int mask; int i; chained_irq_enter(chip, desc); diff --git a/drivers/platform/chrome/cros_ec_typec.c b/drivers/platform/chrome/cros_ec_typec.c index 0811562deecc..24be8f550ae0 100644 --- a/drivers/platform/chrome/cros_ec_typec.c +++ b/drivers/platform/chrome/cros_ec_typec.c @@ -483,6 +483,11 @@ static int cros_typec_enable_dp(struct cros_typec_data *typec, return -ENOTSUPP; } + if (!pd_ctrl->dp_mode) { + dev_err(typec->dev, "No valid DP mode provided.\n"); + return -EINVAL; + } + /* Status VDO. */ dp_data.status = DP_STATUS_ENABLED; if (port->mux_flags & USB_PD_MUX_HPD_IRQ) diff --git a/drivers/platform/surface/aggregator/controller.c b/drivers/platform/surface/aggregator/controller.c index 5bcb59ed579d..89761d3e1a47 100644 --- a/drivers/platform/surface/aggregator/controller.c +++ b/drivers/platform/surface/aggregator/controller.c @@ -1040,7 +1040,7 @@ static int ssam_dsm_load_u32(acpi_handle handle, u64 funcs, u64 func, u32 *ret) union acpi_object *obj; u64 val; - if (!(funcs & BIT(func))) + if (!(funcs & BIT_ULL(func))) return 0; /* Not supported, leave *ret at its default value */ obj = acpi_evaluate_dsm_typed(handle, &SSAM_SSH_DSM_GUID, diff --git a/drivers/platform/x86/dell/dell-wmi-sysman/sysman.c b/drivers/platform/x86/dell/dell-wmi-sysman/sysman.c index 7410ccae650c..a90ae6ba4a73 100644 --- a/drivers/platform/x86/dell/dell-wmi-sysman/sysman.c +++ b/drivers/platform/x86/dell/dell-wmi-sysman/sysman.c @@ -399,6 +399,7 @@ static int init_bios_attributes(int attr_type, const char *guid) union acpi_object *obj = NULL; union acpi_object *elements; struct kset *tmp_set; + int min_elements; /* instance_id needs to be reset for each type GUID * also, instance IDs are unique within GUID but not across @@ -409,14 +410,38 @@ static int init_bios_attributes(int attr_type, const char *guid) retval = alloc_attributes_data(attr_type); if (retval) return retval; + + switch (attr_type) { + case ENUM: min_elements = 8; break; + case INT: min_elements = 9; break; + case STR: min_elements = 8; break; + case PO: min_elements = 4; break; + default: + pr_err("Error: Unknown attr_type: %d\n", attr_type); + return -EINVAL; + } + /* need to use specific instance_id and guid combination to get right data */ obj = get_wmiobj_pointer(instance_id, guid); - if (!obj || obj->type != ACPI_TYPE_PACKAGE) + if (!obj) return -ENODEV; - elements = obj->package.elements; mutex_lock(&wmi_priv.mutex); - while (elements) { + while (obj) { + if (obj->type != ACPI_TYPE_PACKAGE) { + pr_err("Error: Expected ACPI-package type, got: %d\n", obj->type); + retval = -EIO; + goto err_attr_init; + } + + if (obj->package.count < min_elements) { + pr_err("Error: ACPI-package does not have enough elements: %d < %d\n", + obj->package.count, min_elements); + goto nextobj; + } + + elements = obj->package.elements; + /* sanity checking */ if (elements[ATTR_NAME].type != ACPI_TYPE_STRING) { pr_debug("incorrect element type\n"); @@ -481,7 +506,6 @@ static int init_bios_attributes(int attr_type, const char *guid) kfree(obj); instance_id++; obj = get_wmiobj_pointer(instance_id, guid); - elements = obj ? obj->package.elements : NULL; } mutex_unlock(&wmi_priv.mutex); diff --git a/drivers/platform/x86/intel_pmc_core.c b/drivers/platform/x86/intel_pmc_core.c index b5888aeb4bcf..260d49dca1ad 100644 --- a/drivers/platform/x86/intel_pmc_core.c +++ b/drivers/platform/x86/intel_pmc_core.c @@ -1186,9 +1186,15 @@ static const struct pci_device_id pmc_pci_ids[] = { * the platform BIOS enforces 24Mhz crystal to shutdown * before PMC can assert SLP_S0#. */ +static bool xtal_ignore; static int quirk_xtal_ignore(const struct dmi_system_id *id) { - struct pmc_dev *pmcdev = &pmc; + xtal_ignore = true; + return 0; +} + +static void pmc_core_xtal_ignore(struct pmc_dev *pmcdev) +{ u32 value; value = pmc_core_reg_read(pmcdev, pmcdev->map->pm_vric1_offset); @@ -1197,7 +1203,6 @@ static int quirk_xtal_ignore(const struct dmi_system_id *id) /* Low Voltage Mode Enable */ value &= ~SPT_PMC_VRIC1_SLPS0LVEN; pmc_core_reg_write(pmcdev, pmcdev->map->pm_vric1_offset, value); - return 0; } static const struct dmi_system_id pmc_core_dmi_table[] = { @@ -1212,6 +1217,14 @@ static const struct dmi_system_id pmc_core_dmi_table[] = { {} }; +static void pmc_core_do_dmi_quirks(struct pmc_dev *pmcdev) +{ + dmi_check_system(pmc_core_dmi_table); + + if (xtal_ignore) + pmc_core_xtal_ignore(pmcdev); +} + static int pmc_core_probe(struct platform_device *pdev) { static bool device_initialized; @@ -1253,7 +1266,7 @@ static int pmc_core_probe(struct platform_device *pdev) mutex_init(&pmcdev->lock); platform_set_drvdata(pdev, pmcdev); pmcdev->pmc_xram_read_bit = pmc_core_check_read_lock_bit(); - dmi_check_system(pmc_core_dmi_table); + pmc_core_do_dmi_quirks(pmcdev); /* * On TGL, due to a hardware limitation, the GBE LTR blocks PC10 when diff --git a/drivers/platform/x86/intel_speed_select_if/isst_if_mbox_pci.c b/drivers/platform/x86/intel_speed_select_if/isst_if_mbox_pci.c index a2a2d923e60c..df1fc6c719f3 100644 --- a/drivers/platform/x86/intel_speed_select_if/isst_if_mbox_pci.c +++ b/drivers/platform/x86/intel_speed_select_if/isst_if_mbox_pci.c @@ -21,12 +21,16 @@ #define PUNIT_MAILBOX_BUSY_BIT 31 /* - * The average time to complete some commands is about 40us. The current - * count is enough to satisfy 40us. But when the firmware is very busy, this - * causes timeout occasionally. So increase to deal with some worst case - * scenarios. Most of the command still complete in few us. + * The average time to complete mailbox commands is less than 40us. Most of + * the commands complete in few micro seconds. But the same firmware handles + * requests from all power management features. + * We can create a scenario where we flood the firmware with requests then + * the mailbox response can be delayed for 100s of micro seconds. So define + * two timeouts. One for average case and one for long. + * If the firmware is taking more than average, just call cond_resched(). */ -#define OS_MAILBOX_RETRY_COUNT 100 +#define OS_MAILBOX_TIMEOUT_AVG_US 40 +#define OS_MAILBOX_TIMEOUT_MAX_US 1000 struct isst_if_device { struct mutex mutex; @@ -35,11 +39,13 @@ struct isst_if_device { static int isst_if_mbox_cmd(struct pci_dev *pdev, struct isst_if_mbox_cmd *mbox_cmd) { - u32 retries, data; + s64 tm_delta = 0; + ktime_t tm; + u32 data; int ret; /* Poll for rb bit == 0 */ - retries = OS_MAILBOX_RETRY_COUNT; + tm = ktime_get(); do { ret = pci_read_config_dword(pdev, PUNIT_MAILBOX_INTERFACE, &data); @@ -48,11 +54,14 @@ static int isst_if_mbox_cmd(struct pci_dev *pdev, if (data & BIT_ULL(PUNIT_MAILBOX_BUSY_BIT)) { ret = -EBUSY; + tm_delta = ktime_us_delta(ktime_get(), tm); + if (tm_delta > OS_MAILBOX_TIMEOUT_AVG_US) + cond_resched(); continue; } ret = 0; break; - } while (--retries); + } while (tm_delta < OS_MAILBOX_TIMEOUT_MAX_US); if (ret) return ret; @@ -74,7 +83,8 @@ static int isst_if_mbox_cmd(struct pci_dev *pdev, return ret; /* Poll for rb bit == 0 */ - retries = OS_MAILBOX_RETRY_COUNT; + tm_delta = 0; + tm = ktime_get(); do { ret = pci_read_config_dword(pdev, PUNIT_MAILBOX_INTERFACE, &data); @@ -83,6 +93,9 @@ static int isst_if_mbox_cmd(struct pci_dev *pdev, if (data & BIT_ULL(PUNIT_MAILBOX_BUSY_BIT)) { ret = -EBUSY; + tm_delta = ktime_us_delta(ktime_get(), tm); + if (tm_delta > OS_MAILBOX_TIMEOUT_AVG_US) + cond_resched(); continue; } @@ -96,7 +109,7 @@ static int isst_if_mbox_cmd(struct pci_dev *pdev, mbox_cmd->resp_data = data; ret = 0; break; - } while (--retries); + } while (tm_delta < OS_MAILBOX_TIMEOUT_MAX_US); return ret; } diff --git a/drivers/platform/x86/pmc_atom.c b/drivers/platform/x86/pmc_atom.c index ca684ed760d1..a9d2a4b98e57 100644 --- a/drivers/platform/x86/pmc_atom.c +++ b/drivers/platform/x86/pmc_atom.c @@ -393,34 +393,10 @@ static const struct dmi_system_id critclk_systems[] = { }, { /* pmc_plt_clk* - are used for ethernet controllers */ - .ident = "Beckhoff CB3163", + .ident = "Beckhoff Baytrail", .matches = { DMI_MATCH(DMI_SYS_VENDOR, "Beckhoff Automation"), - DMI_MATCH(DMI_BOARD_NAME, "CB3163"), - }, - }, - { - /* pmc_plt_clk* - are used for ethernet controllers */ - .ident = "Beckhoff CB4063", - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "Beckhoff Automation"), - DMI_MATCH(DMI_BOARD_NAME, "CB4063"), - }, - }, - { - /* pmc_plt_clk* - are used for ethernet controllers */ - .ident = "Beckhoff CB6263", - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "Beckhoff Automation"), - DMI_MATCH(DMI_BOARD_NAME, "CB6263"), - }, - }, - { - /* pmc_plt_clk* - are used for ethernet controllers */ - .ident = "Beckhoff CB6363", - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "Beckhoff Automation"), - DMI_MATCH(DMI_BOARD_NAME, "CB6363"), + DMI_MATCH(DMI_PRODUCT_FAMILY, "CBxx63"), }, }, { diff --git a/drivers/platform/x86/thinkpad_acpi.c b/drivers/platform/x86/thinkpad_acpi.c index 0d9e2ddbf904..61f1c91c62de 100644 --- a/drivers/platform/x86/thinkpad_acpi.c +++ b/drivers/platform/x86/thinkpad_acpi.c @@ -6260,6 +6260,7 @@ enum thermal_access_mode { enum { /* TPACPI_THERMAL_TPEC_* */ TP_EC_THERMAL_TMP0 = 0x78, /* ACPI EC regs TMP 0..7 */ TP_EC_THERMAL_TMP8 = 0xC0, /* ACPI EC regs TMP 8..15 */ + TP_EC_FUNCREV = 0xEF, /* ACPI EC Functional revision */ TP_EC_THERMAL_TMP_NA = -128, /* ACPI EC sensor not available */ TPACPI_THERMAL_SENSOR_NA = -128000, /* Sensor not available */ @@ -6458,7 +6459,7 @@ static const struct attribute_group thermal_temp_input8_group = { static int __init thermal_init(struct ibm_init_struct *iibm) { - u8 t, ta1, ta2; + u8 t, ta1, ta2, ver = 0; int i; int acpi_tmp7; int res; @@ -6473,7 +6474,14 @@ static int __init thermal_init(struct ibm_init_struct *iibm) * 0x78-0x7F, 0xC0-0xC7. Registers return 0x00 for * non-implemented, thermal sensors return 0x80 when * not available + * The above rule is unfortunately flawed. This has been seen with + * 0xC2 (power supply ID) causing thermal control problems. + * The EC version can be determined by offset 0xEF and at least for + * version 3 the Lenovo firmware team confirmed that registers 0xC0-0xC7 + * are not thermal registers. */ + if (!acpi_ec_read(TP_EC_FUNCREV, &ver)) + pr_warn("Thinkpad ACPI EC unable to access EC version\n"); ta1 = ta2 = 0; for (i = 0; i < 8; i++) { @@ -6483,11 +6491,13 @@ static int __init thermal_init(struct ibm_init_struct *iibm) ta1 = 0; break; } - if (acpi_ec_read(TP_EC_THERMAL_TMP8 + i, &t)) { - ta2 |= t; - } else { - ta1 = 0; - break; + if (ver < 3) { + if (acpi_ec_read(TP_EC_THERMAL_TMP8 + i, &t)) { + ta2 |= t; + } else { + ta1 = 0; + break; + } } } if (ta1 == 0) { @@ -6500,9 +6510,12 @@ static int __init thermal_init(struct ibm_init_struct *iibm) thermal_read_mode = TPACPI_THERMAL_NONE; } } else { - thermal_read_mode = - (ta2 != 0) ? - TPACPI_THERMAL_TPEC_16 : TPACPI_THERMAL_TPEC_8; + if (ver >= 3) + thermal_read_mode = TPACPI_THERMAL_TPEC_8; + else + thermal_read_mode = + (ta2 != 0) ? + TPACPI_THERMAL_TPEC_16 : TPACPI_THERMAL_TPEC_8; } } else if (acpi_tmp7) { if (tpacpi_is_ibm() && diff --git a/drivers/power/supply/bq25980_charger.c b/drivers/power/supply/bq25980_charger.c index 530ff4025b31..0008c229fd9c 100644 --- a/drivers/power/supply/bq25980_charger.c +++ b/drivers/power/supply/bq25980_charger.c @@ -606,33 +606,6 @@ static int bq25980_get_state(struct bq25980_device *bq, return 0; } -static int bq25980_set_battery_property(struct power_supply *psy, - enum power_supply_property psp, - const union power_supply_propval *val) -{ - struct bq25980_device *bq = power_supply_get_drvdata(psy); - int ret = 0; - - switch (psp) { - case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT: - ret = bq25980_set_const_charge_curr(bq, val->intval); - if (ret) - return ret; - break; - - case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE: - ret = bq25980_set_const_charge_volt(bq, val->intval); - if (ret) - return ret; - break; - - default: - return -EINVAL; - } - - return ret; -} - static int bq25980_get_battery_property(struct power_supply *psy, enum power_supply_property psp, union power_supply_propval *val) @@ -701,6 +674,18 @@ static int bq25980_set_charger_property(struct power_supply *psy, return ret; break; + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT: + ret = bq25980_set_const_charge_curr(bq, val->intval); + if (ret) + return ret; + break; + + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE: + ret = bq25980_set_const_charge_volt(bq, val->intval); + if (ret) + return ret; + break; + default: return -EINVAL; } @@ -922,7 +907,6 @@ static struct power_supply_desc bq25980_battery_desc = { .name = "bq25980-battery", .type = POWER_SUPPLY_TYPE_BATTERY, .get_property = bq25980_get_battery_property, - .set_property = bq25980_set_battery_property, .properties = bq25980_battery_props, .num_properties = ARRAY_SIZE(bq25980_battery_props), .property_is_writeable = bq25980_property_is_writeable, diff --git a/drivers/power/supply/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c index 4c4a7b1c64c5..20e1dc8a87cf 100644 --- a/drivers/power/supply/bq27xxx_battery.c +++ b/drivers/power/supply/bq27xxx_battery.c @@ -1661,27 +1661,6 @@ static int bq27xxx_battery_read_time(struct bq27xxx_device_info *di, u8 reg) return tval * 60; } -/* - * Read an average power register. - * Return < 0 if something fails. - */ -static int bq27xxx_battery_read_pwr_avg(struct bq27xxx_device_info *di) -{ - int tval; - - tval = bq27xxx_read(di, BQ27XXX_REG_AP, false); - if (tval < 0) { - dev_err(di->dev, "error reading average power register %02x: %d\n", - BQ27XXX_REG_AP, tval); - return tval; - } - - if (di->opts & BQ27XXX_O_ZERO) - return (tval * BQ27XXX_POWER_CONSTANT) / BQ27XXX_RS; - else - return tval; -} - /* * Returns true if a battery over temperature condition is detected */ @@ -1769,8 +1748,6 @@ void bq27xxx_battery_update(struct bq27xxx_device_info *di) } if (di->regs[BQ27XXX_REG_CYCT] != INVALID_REG_ADDR) cache.cycle_count = bq27xxx_battery_read_cyct(di); - if (di->regs[BQ27XXX_REG_AP] != INVALID_REG_ADDR) - cache.power_avg = bq27xxx_battery_read_pwr_avg(di); /* We only have to read charge design full once */ if (di->charge_design_full <= 0) @@ -1827,9 +1804,35 @@ static int bq27xxx_battery_current(struct bq27xxx_device_info *di, val->intval = curr * BQ27XXX_CURRENT_CONSTANT / BQ27XXX_RS; } else { /* Other gauges return signed value */ - val->intval = -(int)((s16)curr) * 1000; + val->intval = (int)((s16)curr) * 1000; + } + + return 0; +} + +/* + * Get the average power in µW + * Return < 0 if something fails. + */ +static int bq27xxx_battery_pwr_avg(struct bq27xxx_device_info *di, + union power_supply_propval *val) +{ + int power; + + power = bq27xxx_read(di, BQ27XXX_REG_AP, false); + if (power < 0) { + dev_err(di->dev, + "error reading average power register %02x: %d\n", + BQ27XXX_REG_AP, power); + return power; } + if (di->opts & BQ27XXX_O_ZERO) + val->intval = (power * BQ27XXX_POWER_CONSTANT) / BQ27XXX_RS; + else + /* Other gauges return a signed value in units of 10mW */ + val->intval = (int)((s16)power) * 10000; + return 0; } @@ -2020,7 +2023,7 @@ static int bq27xxx_battery_get_property(struct power_supply *psy, ret = bq27xxx_simple_value(di->cache.energy, val); break; case POWER_SUPPLY_PROP_POWER_AVG: - ret = bq27xxx_simple_value(di->cache.power_avg, val); + ret = bq27xxx_battery_pwr_avg(di, val); break; case POWER_SUPPLY_PROP_HEALTH: ret = bq27xxx_simple_value(di->cache.health, val); diff --git a/drivers/power/supply/cpcap-battery.c b/drivers/power/supply/cpcap-battery.c index 6d5bcdb9f45d..a3fc0084cda0 100644 --- a/drivers/power/supply/cpcap-battery.c +++ b/drivers/power/supply/cpcap-battery.c @@ -786,7 +786,7 @@ static irqreturn_t cpcap_battery_irq_thread(int irq, void *data) break; } - if (!d) + if (list_entry_is_head(d, &ddata->irq_list, node)) return IRQ_NONE; latest = cpcap_battery_latest(ddata); diff --git a/drivers/power/supply/cpcap-charger.c b/drivers/power/supply/cpcap-charger.c index 641dcad1133f..2a8915c3e73e 100644 --- a/drivers/power/supply/cpcap-charger.c +++ b/drivers/power/supply/cpcap-charger.c @@ -318,7 +318,7 @@ static int cpcap_charger_current_to_regval(int microamp) return CPCAP_REG_CRM_ICHRG(0x0); if (miliamp < 177) return CPCAP_REG_CRM_ICHRG(0x1); - if (miliamp > 1596) + if (miliamp >= 1596) return CPCAP_REG_CRM_ICHRG(0xe); res = microamp / 88666; @@ -668,6 +668,9 @@ static void cpcap_usb_detect(struct work_struct *work) return; } + /* Delay for 80ms to avoid vbus bouncing when usb cable is plugged in */ + usleep_range(80000, 120000); + /* Throttle chrgcurr2 interrupt for charger done and retry */ switch (ddata->status) { case POWER_SUPPLY_STATUS_CHARGING: diff --git a/drivers/power/supply/generic-adc-battery.c b/drivers/power/supply/generic-adc-battery.c index 0032069fbc2b..66039c665dd1 100644 --- a/drivers/power/supply/generic-adc-battery.c +++ b/drivers/power/supply/generic-adc-battery.c @@ -373,7 +373,7 @@ static int gab_remove(struct platform_device *pdev) } kfree(adc_bat->psy_desc.properties); - cancel_delayed_work(&adc_bat->bat_work); + cancel_delayed_work_sync(&adc_bat->bat_work); return 0; } diff --git a/drivers/power/supply/lp8788-charger.c b/drivers/power/supply/lp8788-charger.c index e7931ffb7151..397e5a03b7d9 100644 --- a/drivers/power/supply/lp8788-charger.c +++ b/drivers/power/supply/lp8788-charger.c @@ -501,7 +501,7 @@ static int lp8788_set_irqs(struct platform_device *pdev, ret = request_threaded_irq(virq, NULL, lp8788_charger_irq_thread, - 0, name, pchg); + IRQF_ONESHOT, name, pchg); if (ret) break; } diff --git a/drivers/power/supply/pm2301_charger.c b/drivers/power/supply/pm2301_charger.c index ac06ecf7fc9c..a3bfb9612b17 100644 --- a/drivers/power/supply/pm2301_charger.c +++ b/drivers/power/supply/pm2301_charger.c @@ -1089,7 +1089,7 @@ static int pm2xxx_wall_charger_probe(struct i2c_client *i2c_client, ret = request_threaded_irq(gpio_to_irq(pm2->pdata->gpio_irq_number), NULL, pm2xxx_charger_irq[0].isr, - pm2->pdata->irq_type, + pm2->pdata->irq_type | IRQF_ONESHOT, pm2xxx_charger_irq[0].name, pm2); if (ret != 0) { diff --git a/drivers/power/supply/s3c_adc_battery.c b/drivers/power/supply/s3c_adc_battery.c index a2addc24ee8b..3e3a598f114d 100644 --- a/drivers/power/supply/s3c_adc_battery.c +++ b/drivers/power/supply/s3c_adc_battery.c @@ -395,7 +395,7 @@ static int s3c_adc_bat_remove(struct platform_device *pdev) if (main_bat.charge_finished) free_irq(gpiod_to_irq(main_bat.charge_finished), NULL); - cancel_delayed_work(&bat_work); + cancel_delayed_work_sync(&bat_work); if (pdata->exit) pdata->exit(); diff --git a/drivers/power/supply/tps65090-charger.c b/drivers/power/supply/tps65090-charger.c index 6b0098e5a88b..0990b2fa6cd8 100644 --- a/drivers/power/supply/tps65090-charger.c +++ b/drivers/power/supply/tps65090-charger.c @@ -301,7 +301,7 @@ static int tps65090_charger_probe(struct platform_device *pdev) if (irq != -ENXIO) { ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, - tps65090_charger_isr, 0, "tps65090-charger", cdata); + tps65090_charger_isr, IRQF_ONESHOT, "tps65090-charger", cdata); if (ret) { dev_err(cdata->dev, "Unable to register irq %d err %d\n", irq, diff --git a/drivers/power/supply/tps65217_charger.c b/drivers/power/supply/tps65217_charger.c index 814c2b81fdfe..ba33d1617e0b 100644 --- a/drivers/power/supply/tps65217_charger.c +++ b/drivers/power/supply/tps65217_charger.c @@ -238,7 +238,7 @@ static int tps65217_charger_probe(struct platform_device *pdev) for (i = 0; i < NUM_CHARGER_IRQS; i++) { ret = devm_request_threaded_irq(&pdev->dev, irq[i], NULL, tps65217_charger_irq, - 0, "tps65217-charger", + IRQF_ONESHOT, "tps65217-charger", charger); if (ret) { dev_err(charger->dev, diff --git a/drivers/pwm/pwm-atmel.c b/drivers/pwm/pwm-atmel.c index 5813339b597b..3292158157b6 100644 --- a/drivers/pwm/pwm-atmel.c +++ b/drivers/pwm/pwm-atmel.c @@ -319,7 +319,7 @@ static void atmel_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, cdty = atmel_pwm_ch_readl(atmel_pwm, pwm->hwpwm, atmel_pwm->data->regs.duty); - tmp = (u64)cdty * NSEC_PER_SEC; + tmp = (u64)(cprd - cdty) * NSEC_PER_SEC; tmp <<= pres; state->duty_cycle = DIV64_U64_ROUND_UP(tmp, rate); diff --git a/drivers/regulator/bd9576-regulator.c b/drivers/regulator/bd9576-regulator.c index a8b5832a5a1b..204a2da054f5 100644 --- a/drivers/regulator/bd9576-regulator.c +++ b/drivers/regulator/bd9576-regulator.c @@ -206,7 +206,7 @@ static int bd957x_probe(struct platform_device *pdev) { struct regmap *regmap; struct regulator_config config = { 0 }; - int i, err; + int i; bool vout_mode, ddr_sel; const struct bd957x_regulator_data *reg_data = &bd9576_regulators[0]; unsigned int num_reg_data = ARRAY_SIZE(bd9576_regulators); @@ -279,8 +279,7 @@ static int bd957x_probe(struct platform_device *pdev) break; default: dev_err(&pdev->dev, "Unsupported chip type\n"); - err = -EINVAL; - goto err; + return -EINVAL; } config.dev = pdev->dev.parent; @@ -300,8 +299,7 @@ static int bd957x_probe(struct platform_device *pdev) dev_err(&pdev->dev, "failed to register %s regulator\n", desc->name); - err = PTR_ERR(rdev); - goto err; + return PTR_ERR(rdev); } /* * Clear the VOUT1 GPIO setting - rest of the regulators do not @@ -310,8 +308,7 @@ static int bd957x_probe(struct platform_device *pdev) config.ena_gpiod = NULL; } -err: - return err; + return 0; } static const struct platform_device_id bd957x_pmic_id[] = { diff --git a/drivers/regulator/da9121-regulator.c b/drivers/regulator/da9121-regulator.c index a2ede7d7897e..08cbf688e14d 100644 --- a/drivers/regulator/da9121-regulator.c +++ b/drivers/regulator/da9121-regulator.c @@ -40,6 +40,7 @@ struct da9121 { unsigned int passive_delay; int chip_irq; int variant_id; + int subvariant_id; }; /* Define ranges for different variants, enabling translation to/from @@ -812,7 +813,6 @@ static struct regmap_config da9121_2ch_regmap_config = { static int da9121_check_device_type(struct i2c_client *i2c, struct da9121 *chip) { u32 device_id; - u8 chip_id = chip->variant_id; u32 variant_id; u8 variant_mrc, variant_vrc; char *type; @@ -839,22 +839,34 @@ static int da9121_check_device_type(struct i2c_client *i2c, struct da9121 *chip) variant_vrc = variant_id & DA9121_MASK_OTP_VARIANT_ID_VRC; - switch (variant_vrc) { - case DA9121_VARIANT_VRC: - type = "DA9121/DA9130"; - config_match = (chip_id == DA9121_TYPE_DA9121_DA9130); + switch (chip->subvariant_id) { + case DA9121_SUBTYPE_DA9121: + type = "DA9121"; + config_match = (variant_vrc == DA9121_VARIANT_VRC); break; - case DA9220_VARIANT_VRC: - type = "DA9220/DA9132"; - config_match = (chip_id == DA9121_TYPE_DA9220_DA9132); + case DA9121_SUBTYPE_DA9130: + type = "DA9130"; + config_match = (variant_vrc == DA9130_VARIANT_VRC); break; - case DA9122_VARIANT_VRC: - type = "DA9122/DA9131"; - config_match = (chip_id == DA9121_TYPE_DA9122_DA9131); + case DA9121_SUBTYPE_DA9220: + type = "DA9220"; + config_match = (variant_vrc == DA9220_VARIANT_VRC); break; - case DA9217_VARIANT_VRC: + case DA9121_SUBTYPE_DA9132: + type = "DA9132"; + config_match = (variant_vrc == DA9132_VARIANT_VRC); + break; + case DA9121_SUBTYPE_DA9122: + type = "DA9122"; + config_match = (variant_vrc == DA9122_VARIANT_VRC); + break; + case DA9121_SUBTYPE_DA9131: + type = "DA9131"; + config_match = (variant_vrc == DA9131_VARIANT_VRC); + break; + case DA9121_SUBTYPE_DA9217: type = "DA9217"; - config_match = (chip_id == DA9121_TYPE_DA9217); + config_match = (variant_vrc == DA9217_VARIANT_VRC); break; default: type = "Unknown"; @@ -892,15 +904,27 @@ static int da9121_assign_chip_model(struct i2c_client *i2c, chip->dev = &i2c->dev; - switch (chip->variant_id) { - case DA9121_TYPE_DA9121_DA9130: - fallthrough; - case DA9121_TYPE_DA9217: + /* Use configured subtype to select the regulator descriptor index and + * register map, common to both consumer and automotive grade variants + */ + switch (chip->subvariant_id) { + case DA9121_SUBTYPE_DA9121: + case DA9121_SUBTYPE_DA9130: + chip->variant_id = DA9121_TYPE_DA9121_DA9130; regmap = &da9121_1ch_regmap_config; break; - case DA9121_TYPE_DA9122_DA9131: - fallthrough; - case DA9121_TYPE_DA9220_DA9132: + case DA9121_SUBTYPE_DA9217: + chip->variant_id = DA9121_TYPE_DA9217; + regmap = &da9121_1ch_regmap_config; + break; + case DA9121_SUBTYPE_DA9122: + case DA9121_SUBTYPE_DA9131: + chip->variant_id = DA9121_TYPE_DA9122_DA9131; + regmap = &da9121_2ch_regmap_config; + break; + case DA9121_SUBTYPE_DA9220: + case DA9121_SUBTYPE_DA9132: + chip->variant_id = DA9121_TYPE_DA9220_DA9132; regmap = &da9121_2ch_regmap_config; break; } @@ -975,13 +999,13 @@ static int da9121_config_irq(struct i2c_client *i2c, } static const struct of_device_id da9121_dt_ids[] = { - { .compatible = "dlg,da9121", .data = (void *) DA9121_TYPE_DA9121_DA9130 }, - { .compatible = "dlg,da9130", .data = (void *) DA9121_TYPE_DA9121_DA9130 }, - { .compatible = "dlg,da9217", .data = (void *) DA9121_TYPE_DA9217 }, - { .compatible = "dlg,da9122", .data = (void *) DA9121_TYPE_DA9122_DA9131 }, - { .compatible = "dlg,da9131", .data = (void *) DA9121_TYPE_DA9122_DA9131 }, - { .compatible = "dlg,da9220", .data = (void *) DA9121_TYPE_DA9220_DA9132 }, - { .compatible = "dlg,da9132", .data = (void *) DA9121_TYPE_DA9220_DA9132 }, + { .compatible = "dlg,da9121", .data = (void *) DA9121_SUBTYPE_DA9121 }, + { .compatible = "dlg,da9130", .data = (void *) DA9121_SUBTYPE_DA9130 }, + { .compatible = "dlg,da9217", .data = (void *) DA9121_SUBTYPE_DA9217 }, + { .compatible = "dlg,da9122", .data = (void *) DA9121_SUBTYPE_DA9122 }, + { .compatible = "dlg,da9131", .data = (void *) DA9121_SUBTYPE_DA9131 }, + { .compatible = "dlg,da9220", .data = (void *) DA9121_SUBTYPE_DA9220 }, + { .compatible = "dlg,da9132", .data = (void *) DA9121_SUBTYPE_DA9132 }, { } }; MODULE_DEVICE_TABLE(of, da9121_dt_ids); @@ -1011,7 +1035,7 @@ static int da9121_i2c_probe(struct i2c_client *i2c, } chip->pdata = i2c->dev.platform_data; - chip->variant_id = da9121_of_get_id(&i2c->dev); + chip->subvariant_id = da9121_of_get_id(&i2c->dev); ret = da9121_assign_chip_model(i2c, chip); if (ret < 0) diff --git a/drivers/regulator/da9121-regulator.h b/drivers/regulator/da9121-regulator.h index 3c34cb889ca8..357f416e17c1 100644 --- a/drivers/regulator/da9121-regulator.h +++ b/drivers/regulator/da9121-regulator.h @@ -29,6 +29,16 @@ enum da9121_variant { DA9121_TYPE_DA9217 }; +enum da9121_subvariant { + DA9121_SUBTYPE_DA9121, + DA9121_SUBTYPE_DA9130, + DA9121_SUBTYPE_DA9220, + DA9121_SUBTYPE_DA9132, + DA9121_SUBTYPE_DA9122, + DA9121_SUBTYPE_DA9131, + DA9121_SUBTYPE_DA9217 +}; + /* Minimum, maximum and default polling millisecond periods are provided * here as an example. It is expected that any final implementation will * include a modification of these settings to match the required @@ -279,6 +289,9 @@ enum da9121_variant { #define DA9220_VARIANT_VRC 0x0 #define DA9122_VARIANT_VRC 0x2 #define DA9217_VARIANT_VRC 0x7 +#define DA9130_VARIANT_VRC 0x0 +#define DA9131_VARIANT_VRC 0x1 +#define DA9132_VARIANT_VRC 0x2 /* DA9121_REG_OTP_CUSTOMER_ID */ diff --git a/drivers/remoteproc/pru_rproc.c b/drivers/remoteproc/pru_rproc.c index dcb380e868df..549ed3fed625 100644 --- a/drivers/remoteproc/pru_rproc.c +++ b/drivers/remoteproc/pru_rproc.c @@ -266,12 +266,17 @@ static void pru_rproc_create_debug_entries(struct rproc *rproc) static void pru_dispose_irq_mapping(struct pru_rproc *pru) { - while (pru->evt_count--) { + if (!pru->mapped_irq) + return; + + while (pru->evt_count) { + pru->evt_count--; if (pru->mapped_irq[pru->evt_count] > 0) irq_dispose_mapping(pru->mapped_irq[pru->evt_count]); } kfree(pru->mapped_irq); + pru->mapped_irq = NULL; } /* @@ -284,7 +289,7 @@ static int pru_handle_intrmap(struct rproc *rproc) struct pru_rproc *pru = rproc->priv; struct pru_irq_rsc *rsc = pru->pru_interrupt_map; struct irq_fwspec fwspec; - struct device_node *irq_parent; + struct device_node *parent, *irq_parent; int i, ret = 0; /* not having pru_interrupt_map is not an error */ @@ -307,16 +312,31 @@ static int pru_handle_intrmap(struct rproc *rproc) pru->evt_count = rsc->num_evts; pru->mapped_irq = kcalloc(pru->evt_count, sizeof(unsigned int), GFP_KERNEL); - if (!pru->mapped_irq) + if (!pru->mapped_irq) { + pru->evt_count = 0; return -ENOMEM; + } /* * parse and fill in system event to interrupt channel and - * channel-to-host mapping + * channel-to-host mapping. The interrupt controller to be used + * for these mappings for a given PRU remoteproc is always its + * corresponding sibling PRUSS INTC node. */ - irq_parent = of_irq_find_parent(pru->dev->of_node); + parent = of_get_parent(dev_of_node(pru->dev)); + if (!parent) { + kfree(pru->mapped_irq); + pru->mapped_irq = NULL; + pru->evt_count = 0; + return -ENODEV; + } + + irq_parent = of_get_child_by_name(parent, "interrupt-controller"); + of_node_put(parent); if (!irq_parent) { kfree(pru->mapped_irq); + pru->mapped_irq = NULL; + pru->evt_count = 0; return -ENODEV; } @@ -332,16 +352,20 @@ static int pru_handle_intrmap(struct rproc *rproc) pru->mapped_irq[i] = irq_create_fwspec_mapping(&fwspec); if (!pru->mapped_irq[i]) { - dev_err(dev, "failed to get virq\n"); - ret = pru->mapped_irq[i]; + dev_err(dev, "failed to get virq for fw mapping %d: event %d chnl %d host %d\n", + i, fwspec.param[0], fwspec.param[1], + fwspec.param[2]); + ret = -EINVAL; goto map_fail; } } + of_node_put(irq_parent); return ret; map_fail: pru_dispose_irq_mapping(pru); + of_node_put(irq_parent); return ret; } @@ -387,8 +411,7 @@ static int pru_rproc_stop(struct rproc *rproc) pru_control_write_reg(pru, PRU_CTRL_CTRL, val); /* dispose irq mapping - new firmware can provide new mapping */ - if (pru->mapped_irq) - pru_dispose_irq_mapping(pru); + pru_dispose_irq_mapping(pru); return 0; } diff --git a/drivers/remoteproc/qcom_q6v5_mss.c b/drivers/remoteproc/qcom_q6v5_mss.c index 66106ba25ba3..14e0ce5f18f5 100644 --- a/drivers/remoteproc/qcom_q6v5_mss.c +++ b/drivers/remoteproc/qcom_q6v5_mss.c @@ -1210,6 +1210,14 @@ static int q6v5_mpss_load(struct q6v5 *qproc) goto release_firmware; } + if (phdr->p_filesz > phdr->p_memsz) { + dev_err(qproc->dev, + "refusing to load segment %d with p_filesz > p_memsz\n", + i); + ret = -EINVAL; + goto release_firmware; + } + ptr = memremap(qproc->mpss_phys + offset, phdr->p_memsz, MEMREMAP_WC); if (!ptr) { dev_err(qproc->dev, @@ -1241,6 +1249,16 @@ static int q6v5_mpss_load(struct q6v5 *qproc) goto release_firmware; } + if (seg_fw->size != phdr->p_filesz) { + dev_err(qproc->dev, + "failed to load segment %d from truncated file %s\n", + i, fw_name); + ret = -EINVAL; + release_firmware(seg_fw); + memunmap(ptr); + goto release_firmware; + } + release_firmware(seg_fw); } diff --git a/drivers/rpmsg/qcom_glink_native.c b/drivers/rpmsg/qcom_glink_native.c index 27a05167c18c..4840886532ff 100644 --- a/drivers/rpmsg/qcom_glink_native.c +++ b/drivers/rpmsg/qcom_glink_native.c @@ -857,6 +857,7 @@ static int qcom_glink_rx_data(struct qcom_glink *glink, size_t avail) dev_err(glink->dev, "no intent found for channel %s intent %d", channel->name, liid); + ret = -ENOENT; goto advance_rx; } } diff --git a/drivers/rtc/rtc-ds1307.c b/drivers/rtc/rtc-ds1307.c index cd8e438bc9c4..8752620d8e34 100644 --- a/drivers/rtc/rtc-ds1307.c +++ b/drivers/rtc/rtc-ds1307.c @@ -296,7 +296,11 @@ static int ds1307_get_time(struct device *dev, struct rtc_time *t) t->tm_min = bcd2bin(regs[DS1307_REG_MIN] & 0x7f); tmp = regs[DS1307_REG_HOUR] & 0x3f; t->tm_hour = bcd2bin(tmp); - t->tm_wday = bcd2bin(regs[DS1307_REG_WDAY] & 0x07) - 1; + /* rx8130 is bit position, not BCD */ + if (ds1307->type == rx_8130) + t->tm_wday = fls(regs[DS1307_REG_WDAY] & 0x7f); + else + t->tm_wday = bcd2bin(regs[DS1307_REG_WDAY] & 0x07) - 1; t->tm_mday = bcd2bin(regs[DS1307_REG_MDAY] & 0x3f); tmp = regs[DS1307_REG_MONTH] & 0x1f; t->tm_mon = bcd2bin(tmp) - 1; @@ -343,7 +347,11 @@ static int ds1307_set_time(struct device *dev, struct rtc_time *t) regs[DS1307_REG_SECS] = bin2bcd(t->tm_sec); regs[DS1307_REG_MIN] = bin2bcd(t->tm_min); regs[DS1307_REG_HOUR] = bin2bcd(t->tm_hour); - regs[DS1307_REG_WDAY] = bin2bcd(t->tm_wday + 1); + /* rx8130 is bit position, not BCD */ + if (ds1307->type == rx_8130) + regs[DS1307_REG_WDAY] = 1 << t->tm_wday; + else + regs[DS1307_REG_WDAY] = bin2bcd(t->tm_wday + 1); regs[DS1307_REG_MDAY] = bin2bcd(t->tm_mday); regs[DS1307_REG_MONTH] = bin2bcd(t->tm_mon + 1); diff --git a/drivers/rtc/rtc-fsl-ftm-alarm.c b/drivers/rtc/rtc-fsl-ftm-alarm.c index 57cc09d0a806..c0df49fb978c 100644 --- a/drivers/rtc/rtc-fsl-ftm-alarm.c +++ b/drivers/rtc/rtc-fsl-ftm-alarm.c @@ -310,6 +310,7 @@ static const struct of_device_id ftm_rtc_match[] = { { .compatible = "fsl,lx2160a-ftm-alarm", }, { }, }; +MODULE_DEVICE_TABLE(of, ftm_rtc_match); static const struct acpi_device_id ftm_imx_acpi_ids[] = { {"NXP0014",}, diff --git a/drivers/rtc/rtc-tps65910.c b/drivers/rtc/rtc-tps65910.c index 288abb1abdb8..bc89c62ccb9b 100644 --- a/drivers/rtc/rtc-tps65910.c +++ b/drivers/rtc/rtc-tps65910.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/s390/cio/device.c b/drivers/s390/cio/device.c index 3f026021e95e..84f659cafe76 100644 --- a/drivers/s390/cio/device.c +++ b/drivers/s390/cio/device.c @@ -1532,8 +1532,7 @@ static int io_subchannel_sch_event(struct subchannel *sch, int process) switch (action) { case IO_SCH_ORPH_UNREG: case IO_SCH_UNREG: - if (!cdev) - css_sch_device_unregister(sch); + css_sch_device_unregister(sch); break; case IO_SCH_ORPH_ATTACH: case IO_SCH_UNREG_ATTACH: diff --git a/drivers/s390/cio/qdio.h b/drivers/s390/cio/qdio.h index 34bf2f197c71..0e0044d70844 100644 --- a/drivers/s390/cio/qdio.h +++ b/drivers/s390/cio/qdio.h @@ -181,12 +181,6 @@ struct qdio_input_q { struct qdio_output_q { /* PCIs are enabled for the queue */ int pci_out_enabled; - /* cq: use asynchronous output buffers */ - int use_cq; - /* cq: aobs used for particual SBAL */ - struct qaob **aobs; - /* cq: sbal state related to asynchronous operation */ - struct qdio_outbuf_state *sbal_state; /* timer to check for more outbound work */ struct timer_list timer; /* tasklet to check for completions */ @@ -379,12 +373,8 @@ int qdio_setup_irq(struct qdio_irq *irq_ptr, struct qdio_initialize *init_data); void qdio_shutdown_irq(struct qdio_irq *irq); void qdio_print_subchannel_info(struct qdio_irq *irq_ptr); void qdio_free_queues(struct qdio_irq *irq_ptr); -void qdio_free_async_data(struct qdio_irq *irq_ptr); int qdio_setup_init(void); void qdio_setup_exit(void); -int qdio_enable_async_operation(struct qdio_output_q *q); -void qdio_disable_async_operation(struct qdio_output_q *q); -struct qaob *qdio_allocate_aob(void); int debug_get_buf_state(struct qdio_q *q, unsigned int bufnr, unsigned char *state); diff --git a/drivers/s390/cio/qdio_main.c b/drivers/s390/cio/qdio_main.c index 03a011619908..307ce7ff5ca4 100644 --- a/drivers/s390/cio/qdio_main.c +++ b/drivers/s390/cio/qdio_main.c @@ -517,24 +517,6 @@ static inline int qdio_inbound_q_done(struct qdio_q *q, unsigned int start) return 1; } -static inline unsigned long qdio_aob_for_buffer(struct qdio_output_q *q, - int bufnr) -{ - unsigned long phys_aob = 0; - - if (!q->aobs[bufnr]) { - struct qaob *aob = qdio_allocate_aob(); - q->aobs[bufnr] = aob; - } - if (q->aobs[bufnr]) { - q->aobs[bufnr]->user1 = (u64) q->sbal_state[bufnr].user; - phys_aob = virt_to_phys(q->aobs[bufnr]); - WARN_ON_ONCE(phys_aob & 0xFF); - } - - return phys_aob; -} - static inline int qdio_tasklet_schedule(struct qdio_q *q) { if (likely(q->irq_ptr->state == QDIO_IRQ_STATE_ACTIVE)) { @@ -548,7 +530,6 @@ static int get_outbound_buffer_frontier(struct qdio_q *q, unsigned int start, unsigned int *error) { unsigned char state = 0; - unsigned int i; int count; q->timestamp = get_tod_clock_fast(); @@ -570,10 +551,6 @@ static int get_outbound_buffer_frontier(struct qdio_q *q, unsigned int start, switch (state) { case SLSB_P_OUTPUT_PENDING: - /* detach the utilized QAOBs: */ - for (i = 0; i < count; i++) - q->u.out.aobs[QDIO_BUFNR(start + i)] = NULL; - *error = QDIO_ERROR_SLSB_PENDING; fallthrough; case SLSB_P_OUTPUT_EMPTY: @@ -999,7 +976,6 @@ int qdio_free(struct ccw_device *cdev) cdev->private->qdio_data = NULL; mutex_unlock(&irq_ptr->setup_mutex); - qdio_free_async_data(irq_ptr); qdio_free_queues(irq_ptr); free_page((unsigned long) irq_ptr->qdr); free_page(irq_ptr->chsc_page); @@ -1075,28 +1051,6 @@ int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs, } EXPORT_SYMBOL_GPL(qdio_allocate); -static void qdio_detect_hsicq(struct qdio_irq *irq_ptr) -{ - struct qdio_q *q = irq_ptr->input_qs[0]; - int i, use_cq = 0; - - if (irq_ptr->nr_input_qs > 1 && queue_type(q) == QDIO_IQDIO_QFMT) - use_cq = 1; - - for_each_output_queue(irq_ptr, q, i) { - if (use_cq) { - if (multicast_outbound(q)) - continue; - if (qdio_enable_async_operation(&q->u.out) < 0) { - use_cq = 0; - continue; - } - } else - qdio_disable_async_operation(&q->u.out); - } - DBF_EVENT("use_cq:%d", use_cq); -} - static void qdio_trace_init_data(struct qdio_irq *irq, struct qdio_initialize *data) { @@ -1191,8 +1145,6 @@ int qdio_establish(struct ccw_device *cdev, qdio_setup_ssqd_info(irq_ptr); - qdio_detect_hsicq(irq_ptr); - /* qebsm is now setup if available, initialize buffer states */ qdio_init_buf_states(irq_ptr); @@ -1297,9 +1249,11 @@ static int handle_inbound(struct qdio_q *q, unsigned int callflags, * @callflags: flags * @bufnr: first buffer to process * @count: how many buffers are filled + * @aob: asynchronous operation block */ static int handle_outbound(struct qdio_q *q, unsigned int callflags, - unsigned int bufnr, unsigned int count) + unsigned int bufnr, unsigned int count, + struct qaob *aob) { const unsigned int scan_threshold = q->irq_ptr->scan_threshold; unsigned char state = 0; @@ -1320,11 +1274,9 @@ static int handle_outbound(struct qdio_q *q, unsigned int callflags, q->u.out.pci_out_enabled = 0; if (queue_type(q) == QDIO_IQDIO_QFMT) { - unsigned long phys_aob = 0; - - if (q->u.out.use_cq && count == 1) - phys_aob = qdio_aob_for_buffer(&q->u.out, bufnr); + unsigned long phys_aob = aob ? virt_to_phys(aob) : 0; + WARN_ON_ONCE(!IS_ALIGNED(phys_aob, 256)); rc = qdio_kick_outbound_q(q, count, phys_aob); } else if (need_siga_sync(q)) { rc = qdio_siga_sync_q(q); @@ -1359,9 +1311,10 @@ static int handle_outbound(struct qdio_q *q, unsigned int callflags, * @q_nr: queue number * @bufnr: buffer number * @count: how many buffers to process + * @aob: asynchronous operation block (outbound only) */ int do_QDIO(struct ccw_device *cdev, unsigned int callflags, - int q_nr, unsigned int bufnr, unsigned int count) + int q_nr, unsigned int bufnr, unsigned int count, struct qaob *aob) { struct qdio_irq *irq_ptr = cdev->private->qdio_data; @@ -1383,7 +1336,7 @@ int do_QDIO(struct ccw_device *cdev, unsigned int callflags, callflags, bufnr, count); else if (callflags & QDIO_FLAG_SYNC_OUTPUT) return handle_outbound(irq_ptr->output_qs[q_nr], - callflags, bufnr, count); + callflags, bufnr, count, aob); return -EINVAL; } EXPORT_SYMBOL_GPL(do_QDIO); diff --git a/drivers/s390/cio/qdio_setup.c b/drivers/s390/cio/qdio_setup.c index c8b9620bc688..da67e4979402 100644 --- a/drivers/s390/cio/qdio_setup.c +++ b/drivers/s390/cio/qdio_setup.c @@ -30,6 +30,7 @@ struct qaob *qdio_allocate_aob(void) { return kmem_cache_zalloc(qdio_aob_cache, GFP_ATOMIC); } +EXPORT_SYMBOL_GPL(qdio_allocate_aob); void qdio_release_aob(struct qaob *aob) { @@ -247,8 +248,6 @@ static void setup_queues(struct qdio_irq *irq_ptr, struct qdio_initialize *qdio_init) { struct qdio_q *q; - struct qdio_outbuf_state *output_sbal_state_array = - qdio_init->output_sbal_state_array; int i; for_each_input_queue(irq_ptr, q, i) { @@ -265,9 +264,6 @@ static void setup_queues(struct qdio_irq *irq_ptr, DBF_EVENT("outq:%1d", i); setup_queues_misc(q, irq_ptr, qdio_init->output_handler, i); - q->u.out.sbal_state = output_sbal_state_array; - output_sbal_state_array += QDIO_MAX_BUFFERS_PER_Q; - q->is_input_q = 0; setup_storage_lists(q, irq_ptr, qdio_init->output_sbal_addr_array[i], i); @@ -372,30 +368,6 @@ void qdio_setup_ssqd_info(struct qdio_irq *irq_ptr) DBF_EVENT("3:%4x qib:%4x", irq_ptr->ssqd_desc.qdioac3, irq_ptr->qib.ac); } -void qdio_free_async_data(struct qdio_irq *irq_ptr) -{ - struct qdio_q *q; - int i; - - for (i = 0; i < irq_ptr->max_output_qs; i++) { - q = irq_ptr->output_qs[i]; - if (q->u.out.use_cq) { - unsigned int n; - - for (n = 0; n < QDIO_MAX_BUFFERS_PER_Q; n++) { - struct qaob *aob = q->u.out.aobs[n]; - - if (aob) { - qdio_release_aob(aob); - q->u.out.aobs[n] = NULL; - } - } - - qdio_disable_async_operation(&q->u.out); - } - } -} - static void qdio_fill_qdr_desc(struct qdesfmt0 *desc, struct qdio_q *queue) { desc->sliba = virt_to_phys(queue->slib); @@ -545,25 +517,6 @@ void qdio_print_subchannel_info(struct qdio_irq *irq_ptr) printk(KERN_INFO "%s", s); } -int qdio_enable_async_operation(struct qdio_output_q *outq) -{ - outq->aobs = kcalloc(QDIO_MAX_BUFFERS_PER_Q, sizeof(struct qaob *), - GFP_KERNEL); - if (!outq->aobs) { - outq->use_cq = 0; - return -ENOMEM; - } - outq->use_cq = 1; - return 0; -} - -void qdio_disable_async_operation(struct qdio_output_q *q) -{ - kfree(q->aobs); - q->aobs = NULL; - q->use_cq = 0; -} - int __init qdio_setup_init(void) { int rc; diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 1ffdd411201c..6946a7e26eff 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -294,6 +294,19 @@ static int handle_pqap(struct kvm_vcpu *vcpu) matrix_mdev = container_of(vcpu->kvm->arch.crypto.pqap_hook, struct ap_matrix_mdev, pqap_hook); + /* + * If the KVM pointer is in the process of being set, wait until the + * process has completed. + */ + wait_event_cmd(matrix_mdev->wait_for_kvm, + !matrix_mdev->kvm_busy, + mutex_unlock(&matrix_dev->lock), + mutex_lock(&matrix_dev->lock)); + + /* If the there is no guest using the mdev, there is nothing to do */ + if (!matrix_mdev->kvm) + goto out_unlock; + q = vfio_ap_get_queue(matrix_mdev, apqn); if (!q) goto out_unlock; @@ -337,6 +350,7 @@ static int vfio_ap_mdev_create(struct kobject *kobj, struct mdev_device *mdev) matrix_mdev->mdev = mdev; vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix); + init_waitqueue_head(&matrix_mdev->wait_for_kvm); mdev_set_drvdata(mdev, matrix_mdev); matrix_mdev->pqap_hook.hook = handle_pqap; matrix_mdev->pqap_hook.owner = THIS_MODULE; @@ -351,17 +365,23 @@ static int vfio_ap_mdev_remove(struct mdev_device *mdev) { struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev); - if (matrix_mdev->kvm) + mutex_lock(&matrix_dev->lock); + + /* + * If the KVM pointer is in flux or the guest is running, disallow + * un-assignment of control domain. + */ + if (matrix_mdev->kvm_busy || matrix_mdev->kvm) { + mutex_unlock(&matrix_dev->lock); return -EBUSY; + } - mutex_lock(&matrix_dev->lock); vfio_ap_mdev_reset_queues(mdev); list_del(&matrix_mdev->node); - mutex_unlock(&matrix_dev->lock); - kfree(matrix_mdev); mdev_set_drvdata(mdev, NULL); atomic_inc(&matrix_dev->available_instances); + mutex_unlock(&matrix_dev->lock); return 0; } @@ -606,24 +626,31 @@ static ssize_t assign_adapter_store(struct device *dev, struct mdev_device *mdev = mdev_from_dev(dev); struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev); - /* If the guest is running, disallow assignment of adapter */ - if (matrix_mdev->kvm) - return -EBUSY; + mutex_lock(&matrix_dev->lock); + + /* + * If the KVM pointer is in flux or the guest is running, disallow + * un-assignment of adapter + */ + if (matrix_mdev->kvm_busy || matrix_mdev->kvm) { + ret = -EBUSY; + goto done; + } ret = kstrtoul(buf, 0, &apid); if (ret) - return ret; + goto done; - if (apid > matrix_mdev->matrix.apm_max) - return -ENODEV; + if (apid > matrix_mdev->matrix.apm_max) { + ret = -ENODEV; + goto done; + } /* * Set the bit in the AP mask (APM) corresponding to the AP adapter * number (APID). The bits in the mask, from most significant to least * significant bit, correspond to APIDs 0-255. */ - mutex_lock(&matrix_dev->lock); - ret = vfio_ap_mdev_verify_queues_reserved_for_apid(matrix_mdev, apid); if (ret) goto done; @@ -672,22 +699,31 @@ static ssize_t unassign_adapter_store(struct device *dev, struct mdev_device *mdev = mdev_from_dev(dev); struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev); - /* If the guest is running, disallow un-assignment of adapter */ - if (matrix_mdev->kvm) - return -EBUSY; + mutex_lock(&matrix_dev->lock); + + /* + * If the KVM pointer is in flux or the guest is running, disallow + * un-assignment of adapter + */ + if (matrix_mdev->kvm_busy || matrix_mdev->kvm) { + ret = -EBUSY; + goto done; + } ret = kstrtoul(buf, 0, &apid); if (ret) - return ret; + goto done; - if (apid > matrix_mdev->matrix.apm_max) - return -ENODEV; + if (apid > matrix_mdev->matrix.apm_max) { + ret = -ENODEV; + goto done; + } - mutex_lock(&matrix_dev->lock); clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm); + ret = count; +done: mutex_unlock(&matrix_dev->lock); - - return count; + return ret; } static DEVICE_ATTR_WO(unassign_adapter); @@ -753,17 +789,24 @@ static ssize_t assign_domain_store(struct device *dev, struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev); unsigned long max_apqi = matrix_mdev->matrix.aqm_max; - /* If the guest is running, disallow assignment of domain */ - if (matrix_mdev->kvm) - return -EBUSY; + mutex_lock(&matrix_dev->lock); + + /* + * If the KVM pointer is in flux or the guest is running, disallow + * assignment of domain + */ + if (matrix_mdev->kvm_busy || matrix_mdev->kvm) { + ret = -EBUSY; + goto done; + } ret = kstrtoul(buf, 0, &apqi); if (ret) - return ret; - if (apqi > max_apqi) - return -ENODEV; - - mutex_lock(&matrix_dev->lock); + goto done; + if (apqi > max_apqi) { + ret = -ENODEV; + goto done; + } ret = vfio_ap_mdev_verify_queues_reserved_for_apqi(matrix_mdev, apqi); if (ret) @@ -814,22 +857,32 @@ static ssize_t unassign_domain_store(struct device *dev, struct mdev_device *mdev = mdev_from_dev(dev); struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev); - /* If the guest is running, disallow un-assignment of domain */ - if (matrix_mdev->kvm) - return -EBUSY; + mutex_lock(&matrix_dev->lock); + + /* + * If the KVM pointer is in flux or the guest is running, disallow + * un-assignment of domain + */ + if (matrix_mdev->kvm_busy || matrix_mdev->kvm) { + ret = -EBUSY; + goto done; + } ret = kstrtoul(buf, 0, &apqi); if (ret) - return ret; + goto done; - if (apqi > matrix_mdev->matrix.aqm_max) - return -ENODEV; + if (apqi > matrix_mdev->matrix.aqm_max) { + ret = -ENODEV; + goto done; + } - mutex_lock(&matrix_dev->lock); clear_bit_inv((unsigned long)apqi, matrix_mdev->matrix.aqm); - mutex_unlock(&matrix_dev->lock); + ret = count; - return count; +done: + mutex_unlock(&matrix_dev->lock); + return ret; } static DEVICE_ATTR_WO(unassign_domain); @@ -858,27 +911,36 @@ static ssize_t assign_control_domain_store(struct device *dev, struct mdev_device *mdev = mdev_from_dev(dev); struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev); - /* If the guest is running, disallow assignment of control domain */ - if (matrix_mdev->kvm) - return -EBUSY; + mutex_lock(&matrix_dev->lock); + + /* + * If the KVM pointer is in flux or the guest is running, disallow + * assignment of control domain. + */ + if (matrix_mdev->kvm_busy || matrix_mdev->kvm) { + ret = -EBUSY; + goto done; + } ret = kstrtoul(buf, 0, &id); if (ret) - return ret; + goto done; - if (id > matrix_mdev->matrix.adm_max) - return -ENODEV; + if (id > matrix_mdev->matrix.adm_max) { + ret = -ENODEV; + goto done; + } /* Set the bit in the ADM (bitmask) corresponding to the AP control * domain number (id). The bits in the mask, from most significant to * least significant, correspond to IDs 0 up to the one less than the * number of control domains that can be assigned. */ - mutex_lock(&matrix_dev->lock); set_bit_inv(id, matrix_mdev->matrix.adm); + ret = count; +done: mutex_unlock(&matrix_dev->lock); - - return count; + return ret; } static DEVICE_ATTR_WO(assign_control_domain); @@ -908,21 +970,30 @@ static ssize_t unassign_control_domain_store(struct device *dev, struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev); unsigned long max_domid = matrix_mdev->matrix.adm_max; - /* If the guest is running, disallow un-assignment of control domain */ - if (matrix_mdev->kvm) - return -EBUSY; + mutex_lock(&matrix_dev->lock); + + /* + * If the KVM pointer is in flux or the guest is running, disallow + * un-assignment of control domain. + */ + if (matrix_mdev->kvm_busy || matrix_mdev->kvm) { + ret = -EBUSY; + goto done; + } ret = kstrtoul(buf, 0, &domid); if (ret) - return ret; - if (domid > max_domid) - return -ENODEV; + goto done; + if (domid > max_domid) { + ret = -ENODEV; + goto done; + } - mutex_lock(&matrix_dev->lock); clear_bit_inv(domid, matrix_mdev->matrix.adm); + ret = count; +done: mutex_unlock(&matrix_dev->lock); - - return count; + return ret; } static DEVICE_ATTR_WO(unassign_control_domain); @@ -1027,8 +1098,15 @@ static const struct attribute_group *vfio_ap_mdev_attr_groups[] = { * @matrix_mdev: a mediated matrix device * @kvm: reference to KVM instance * - * Verifies no other mediated matrix device has @kvm and sets a reference to - * it in @matrix_mdev->kvm. + * Sets all data for @matrix_mdev that are needed to manage AP resources + * for the guest whose state is represented by @kvm. + * + * Note: The matrix_dev->lock must be taken prior to calling + * this function; however, the lock will be temporarily released while the + * guest's AP configuration is set to avoid a potential lockdep splat. + * The kvm->lock is taken to set the guest's AP configuration which, under + * certain circumstances, will result in a circular lock dependency if this is + * done under the @matrix_mdev->lock. * * Return 0 if no other mediated matrix device has a reference to @kvm; * otherwise, returns an -EPERM. @@ -1038,14 +1116,25 @@ static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev, { struct ap_matrix_mdev *m; - list_for_each_entry(m, &matrix_dev->mdev_list, node) { - if ((m != matrix_mdev) && (m->kvm == kvm)) - return -EPERM; - } + if (kvm->arch.crypto.crycbd) { + list_for_each_entry(m, &matrix_dev->mdev_list, node) { + if (m != matrix_mdev && m->kvm == kvm) + return -EPERM; + } - matrix_mdev->kvm = kvm; - kvm_get_kvm(kvm); - kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook; + kvm_get_kvm(kvm); + matrix_mdev->kvm_busy = true; + mutex_unlock(&matrix_dev->lock); + kvm_arch_crypto_set_masks(kvm, + matrix_mdev->matrix.apm, + matrix_mdev->matrix.aqm, + matrix_mdev->matrix.adm); + mutex_lock(&matrix_dev->lock); + kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook; + matrix_mdev->kvm = kvm; + matrix_mdev->kvm_busy = false; + wake_up_all(&matrix_mdev->wait_for_kvm); + } return 0; } @@ -1079,51 +1168,65 @@ static int vfio_ap_mdev_iommu_notifier(struct notifier_block *nb, return NOTIFY_DONE; } +/** + * vfio_ap_mdev_unset_kvm + * + * @matrix_mdev: a matrix mediated device + * + * Performs clean-up of resources no longer needed by @matrix_mdev. + * + * Note: The matrix_dev->lock must be taken prior to calling + * this function; however, the lock will be temporarily released while the + * guest's AP configuration is cleared to avoid a potential lockdep splat. + * The kvm->lock is taken to clear the guest's AP configuration which, under + * certain circumstances, will result in a circular lock dependency if this is + * done under the @matrix_mdev->lock. + * + */ static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev) { - kvm_arch_crypto_clear_masks(matrix_mdev->kvm); - matrix_mdev->kvm->arch.crypto.pqap_hook = NULL; - vfio_ap_mdev_reset_queues(matrix_mdev->mdev); - kvm_put_kvm(matrix_mdev->kvm); - matrix_mdev->kvm = NULL; + /* + * If the KVM pointer is in the process of being set, wait until the + * process has completed. + */ + wait_event_cmd(matrix_mdev->wait_for_kvm, + !matrix_mdev->kvm_busy, + mutex_unlock(&matrix_dev->lock), + mutex_lock(&matrix_dev->lock)); + + if (matrix_mdev->kvm) { + matrix_mdev->kvm_busy = true; + mutex_unlock(&matrix_dev->lock); + kvm_arch_crypto_clear_masks(matrix_mdev->kvm); + mutex_lock(&matrix_dev->lock); + vfio_ap_mdev_reset_queues(matrix_mdev->mdev); + matrix_mdev->kvm->arch.crypto.pqap_hook = NULL; + kvm_put_kvm(matrix_mdev->kvm); + matrix_mdev->kvm = NULL; + matrix_mdev->kvm_busy = false; + wake_up_all(&matrix_mdev->wait_for_kvm); + } } static int vfio_ap_mdev_group_notifier(struct notifier_block *nb, unsigned long action, void *data) { - int ret, notify_rc = NOTIFY_OK; + int notify_rc = NOTIFY_OK; struct ap_matrix_mdev *matrix_mdev; if (action != VFIO_GROUP_NOTIFY_SET_KVM) return NOTIFY_OK; - matrix_mdev = container_of(nb, struct ap_matrix_mdev, group_notifier); mutex_lock(&matrix_dev->lock); + matrix_mdev = container_of(nb, struct ap_matrix_mdev, group_notifier); - if (!data) { - if (matrix_mdev->kvm) - vfio_ap_mdev_unset_kvm(matrix_mdev); - goto notify_done; - } - - ret = vfio_ap_mdev_set_kvm(matrix_mdev, data); - if (ret) { - notify_rc = NOTIFY_DONE; - goto notify_done; - } - - /* If there is no CRYCB pointer, then we can't copy the masks */ - if (!matrix_mdev->kvm->arch.crypto.crycbd) { + if (!data) + vfio_ap_mdev_unset_kvm(matrix_mdev); + else if (vfio_ap_mdev_set_kvm(matrix_mdev, data)) notify_rc = NOTIFY_DONE; - goto notify_done; - } - - kvm_arch_crypto_set_masks(matrix_mdev->kvm, matrix_mdev->matrix.apm, - matrix_mdev->matrix.aqm, - matrix_mdev->matrix.adm); -notify_done: mutex_unlock(&matrix_dev->lock); + return notify_rc; } @@ -1258,8 +1361,7 @@ static void vfio_ap_mdev_release(struct mdev_device *mdev) struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev); mutex_lock(&matrix_dev->lock); - if (matrix_mdev->kvm) - vfio_ap_mdev_unset_kvm(matrix_mdev); + vfio_ap_mdev_unset_kvm(matrix_mdev); mutex_unlock(&matrix_dev->lock); vfio_unregister_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY, @@ -1293,6 +1395,7 @@ static ssize_t vfio_ap_mdev_ioctl(struct mdev_device *mdev, unsigned int cmd, unsigned long arg) { int ret; + struct ap_matrix_mdev *matrix_mdev; mutex_lock(&matrix_dev->lock); switch (cmd) { @@ -1300,6 +1403,21 @@ static ssize_t vfio_ap_mdev_ioctl(struct mdev_device *mdev, ret = vfio_ap_mdev_get_device_info(arg); break; case VFIO_DEVICE_RESET: + matrix_mdev = mdev_get_drvdata(mdev); + if (WARN(!matrix_mdev, "Driver data missing from mdev!!")) { + ret = -EINVAL; + break; + } + + /* + * If the KVM pointer is in the process of being set, wait until + * the process has completed. + */ + wait_event_cmd(matrix_mdev->wait_for_kvm, + !matrix_mdev->kvm_busy, + mutex_unlock(&matrix_dev->lock), + mutex_lock(&matrix_dev->lock)); + ret = vfio_ap_mdev_reset_queues(mdev); break; default: diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h index 28e9d9989768..f82a6396acae 100644 --- a/drivers/s390/crypto/vfio_ap_private.h +++ b/drivers/s390/crypto/vfio_ap_private.h @@ -83,6 +83,8 @@ struct ap_matrix_mdev { struct ap_matrix matrix; struct notifier_block group_notifier; struct notifier_block iommu_notifier; + bool kvm_busy; + wait_queue_head_t wait_for_kvm; struct kvm *kvm; struct kvm_s390_module_hook pqap_hook; struct mdev_device *mdev; diff --git a/drivers/s390/crypto/zcrypt_card.c b/drivers/s390/crypto/zcrypt_card.c index 33b23884b133..09fe6bb8880b 100644 --- a/drivers/s390/crypto/zcrypt_card.c +++ b/drivers/s390/crypto/zcrypt_card.c @@ -192,5 +192,6 @@ void zcrypt_card_unregister(struct zcrypt_card *zc) spin_unlock(&zcrypt_list_lock); sysfs_remove_group(&zc->card->ap_dev.device.kobj, &zcrypt_card_attr_group); + zcrypt_card_put(zc); } EXPORT_SYMBOL(zcrypt_card_unregister); diff --git a/drivers/s390/crypto/zcrypt_queue.c b/drivers/s390/crypto/zcrypt_queue.c index 5062eae73d4a..c3ffbd26b73f 100644 --- a/drivers/s390/crypto/zcrypt_queue.c +++ b/drivers/s390/crypto/zcrypt_queue.c @@ -223,5 +223,6 @@ void zcrypt_queue_unregister(struct zcrypt_queue *zq) sysfs_remove_group(&zq->queue->ap_dev.device.kobj, &zcrypt_queue_attr_group); zcrypt_card_put(zc); + zcrypt_queue_put(zq); } EXPORT_SYMBOL(zcrypt_queue_unregister); diff --git a/drivers/s390/net/qeth_core.h b/drivers/s390/net/qeth_core.h index 91acff493612..fd9b869d278e 100644 --- a/drivers/s390/net/qeth_core.h +++ b/drivers/s390/net/qeth_core.h @@ -437,6 +437,7 @@ struct qeth_qdio_out_buffer { struct qeth_qdio_out_q *q; struct list_head list_entry; + struct qaob *aob; }; struct qeth_card; @@ -499,7 +500,6 @@ struct qeth_out_q_stats { struct qeth_qdio_out_q { struct qdio_buffer *qdio_bufs[QDIO_MAX_BUFFERS_PER_Q]; struct qeth_qdio_out_buffer *bufs[QDIO_MAX_BUFFERS_PER_Q]; - struct qdio_outbuf_state *bufstates; /* convenience pointer */ struct list_head pending_bufs; struct qeth_out_q_stats stats; spinlock_t lock; @@ -563,7 +563,6 @@ struct qeth_qdio_info { /* output */ unsigned int no_out_queues; struct qeth_qdio_out_q *out_qs[QETH_MAX_OUT_QUEUES]; - struct qdio_outbuf_state *out_bufstates; /* priority queueing */ int do_prio_queueing; diff --git a/drivers/s390/net/qeth_core_main.c b/drivers/s390/net/qeth_core_main.c index a814698387bc..175b82b98f36 100644 --- a/drivers/s390/net/qeth_core_main.c +++ b/drivers/s390/net/qeth_core_main.c @@ -369,8 +369,7 @@ static int qeth_cq_init(struct qeth_card *card) QDIO_MAX_BUFFERS_PER_Q); card->qdio.c_q->next_buf_to_init = 127; rc = do_QDIO(CARD_DDEV(card), QDIO_FLAG_SYNC_INPUT, - card->qdio.no_in_queues - 1, 0, - 127); + card->qdio.no_in_queues - 1, 0, 127, NULL); if (rc) { QETH_CARD_TEXT_(card, 2, "1err%d", rc); goto out; @@ -383,48 +382,22 @@ static int qeth_cq_init(struct qeth_card *card) static int qeth_alloc_cq(struct qeth_card *card) { - int rc; - if (card->options.cq == QETH_CQ_ENABLED) { - int i; - struct qdio_outbuf_state *outbuf_states; - QETH_CARD_TEXT(card, 2, "cqon"); card->qdio.c_q = qeth_alloc_qdio_queue(); if (!card->qdio.c_q) { - rc = -1; - goto kmsg_out; + dev_err(&card->gdev->dev, "Failed to create completion queue\n"); + return -ENOMEM; } + card->qdio.no_in_queues = 2; - card->qdio.out_bufstates = - kcalloc(card->qdio.no_out_queues * - QDIO_MAX_BUFFERS_PER_Q, - sizeof(struct qdio_outbuf_state), - GFP_KERNEL); - outbuf_states = card->qdio.out_bufstates; - if (outbuf_states == NULL) { - rc = -1; - goto free_cq_out; - } - for (i = 0; i < card->qdio.no_out_queues; ++i) { - card->qdio.out_qs[i]->bufstates = outbuf_states; - outbuf_states += QDIO_MAX_BUFFERS_PER_Q; - } } else { QETH_CARD_TEXT(card, 2, "nocq"); card->qdio.c_q = NULL; card->qdio.no_in_queues = 1; } QETH_CARD_TEXT_(card, 2, "iqc%d", card->qdio.no_in_queues); - rc = 0; -out: - return rc; -free_cq_out: - qeth_free_qdio_queue(card->qdio.c_q); - card->qdio.c_q = NULL; -kmsg_out: - dev_err(&card->gdev->dev, "Failed to create completion queue\n"); - goto out; + return 0; } static void qeth_free_cq(struct qeth_card *card) @@ -434,8 +407,6 @@ static void qeth_free_cq(struct qeth_card *card) qeth_free_qdio_queue(card->qdio.c_q); card->qdio.c_q = NULL; } - kfree(card->qdio.out_bufstates); - card->qdio.out_bufstates = NULL; } static enum iucv_tx_notify qeth_compute_cq_notification(int sbalf15, @@ -487,12 +458,12 @@ static void qeth_qdio_handle_aob(struct qeth_card *card, switch (atomic_xchg(&buffer->state, new_state)) { case QETH_QDIO_BUF_PRIMED: /* Faster than TX completion code, let it handle the async - * completion for us. + * completion for us. It will also recycle the QAOB. */ break; case QETH_QDIO_BUF_PENDING: /* TX completion code is active and will handle the async - * completion for us. + * completion for us. It will also recycle the QAOB. */ break; case QETH_QDIO_BUF_NEED_QAOB: @@ -501,7 +472,7 @@ static void qeth_qdio_handle_aob(struct qeth_card *card, qeth_notify_skbs(buffer->q, buffer, notification); /* Free dangling allocations. The attached skbs are handled by - * qeth_tx_complete_pending_bufs(). + * qeth_tx_complete_pending_bufs(), and so is the QAOB. */ for (i = 0; i < aob->sb_count && i < QETH_MAX_BUFFER_ELEMENTS(card); @@ -520,8 +491,6 @@ static void qeth_qdio_handle_aob(struct qeth_card *card, default: WARN_ON_ONCE(1); } - - qdio_release_aob(aob); } static void qeth_setup_ccw(struct ccw1 *ccw, u8 cmd_code, u8 flags, u32 len, @@ -1451,6 +1420,13 @@ static void qeth_clear_output_buffer(struct qeth_qdio_out_q *queue, atomic_set(&buf->state, QETH_QDIO_BUF_EMPTY); } +static void qeth_free_out_buf(struct qeth_qdio_out_buffer *buf) +{ + if (buf->aob) + qdio_release_aob(buf->aob); + kmem_cache_free(qeth_qdio_outbuf_cache, buf); +} + static void qeth_tx_complete_pending_bufs(struct qeth_card *card, struct qeth_qdio_out_q *queue, bool drain) @@ -1468,7 +1444,7 @@ static void qeth_tx_complete_pending_bufs(struct qeth_card *card, qeth_tx_complete_buf(buf, drain, 0); list_del(&buf->list_entry); - kmem_cache_free(qeth_qdio_outbuf_cache, buf); + qeth_free_out_buf(buf); } } } @@ -1485,7 +1461,7 @@ static void qeth_drain_output_queue(struct qeth_qdio_out_q *q, bool free) qeth_clear_output_buffer(q, q->bufs[j], true, 0); if (free) { - kmem_cache_free(qeth_qdio_outbuf_cache, q->bufs[j]); + qeth_free_out_buf(q->bufs[j]); q->bufs[j] = NULL; } } @@ -2637,7 +2613,7 @@ static struct qeth_qdio_out_q *qeth_alloc_output_queue(void) err_out_bufs: while (i > 0) - kmem_cache_free(qeth_qdio_outbuf_cache, q->bufs[--i]); + qeth_free_out_buf(q->bufs[--i]); qdio_free_buffers(q->qdio_bufs, QDIO_MAX_BUFFERS_PER_Q); err_qdio_bufs: kfree(q); @@ -3024,7 +3000,8 @@ static int qeth_init_qdio_queues(struct qeth_card *card) } card->qdio.in_q->next_buf_to_init = QDIO_BUFNR(rx_bufs); - rc = do_QDIO(CARD_DDEV(card), QDIO_FLAG_SYNC_INPUT, 0, 0, rx_bufs); + rc = do_QDIO(CARD_DDEV(card), QDIO_FLAG_SYNC_INPUT, 0, 0, rx_bufs, + NULL); if (rc) { QETH_CARD_TEXT_(card, 2, "1err%d", rc); return rc; @@ -3516,7 +3493,7 @@ static unsigned int qeth_rx_refill_queue(struct qeth_card *card, } rc = do_QDIO(CARD_DDEV(card), QDIO_FLAG_SYNC_INPUT, 0, - queue->next_buf_to_init, count); + queue->next_buf_to_init, count, NULL); if (rc) { QETH_CARD_TEXT(card, 2, "qinberr"); } @@ -3625,6 +3602,7 @@ static void qeth_flush_buffers(struct qeth_qdio_out_q *queue, int index, struct qeth_qdio_out_buffer *buf = queue->bufs[index]; unsigned int qdio_flags = QDIO_FLAG_SYNC_OUTPUT; struct qeth_card *card = queue->card; + struct qaob *aob = NULL; int rc; int i; @@ -3637,16 +3615,24 @@ static void qeth_flush_buffers(struct qeth_qdio_out_q *queue, int index, SBAL_EFLAGS_LAST_ENTRY; queue->coalesced_frames += buf->frames; - if (queue->bufstates) - queue->bufstates[bidx].user = buf; - if (IS_IQD(card)) { skb_queue_walk(&buf->skb_list, skb) skb_tx_timestamp(skb); } } - if (!IS_IQD(card)) { + if (IS_IQD(card)) { + if (card->options.cq == QETH_CQ_ENABLED && + !qeth_iqd_is_mcast_queue(card, queue) && + count == 1) { + if (!buf->aob) + buf->aob = qdio_allocate_aob(); + if (buf->aob) { + aob = buf->aob; + aob->user1 = (u64) buf; + } + } + } else { if (!queue->do_pack) { if ((atomic_read(&queue->used_buffers) >= (QETH_HIGH_WATERMARK_PACK - @@ -3677,8 +3663,8 @@ static void qeth_flush_buffers(struct qeth_qdio_out_q *queue, int index, } QETH_TXQ_STAT_INC(queue, doorbell); - rc = do_QDIO(CARD_DDEV(queue->card), qdio_flags, - queue->queue_no, index, count); + rc = do_QDIO(CARD_DDEV(card), qdio_flags, queue->queue_no, index, count, + aob); switch (rc) { case 0: @@ -3814,8 +3800,7 @@ static void qeth_qdio_cq_handler(struct qeth_card *card, unsigned int qdio_err, qeth_scrub_qdio_buffer(buffer, QDIO_MAX_ELEMENTS_PER_BUFFER); } rc = do_QDIO(CARD_DDEV(card), QDIO_FLAG_SYNC_INPUT, queue, - card->qdio.c_q->next_buf_to_init, - count); + cq->next_buf_to_init, count, NULL); if (rc) { dev_warn(&card->gdev->dev, "QDIO reported an error, rc=%i\n", rc); @@ -5270,7 +5255,6 @@ static int qeth_qdio_establish(struct qeth_card *card) init_data.int_parm = (unsigned long) card; init_data.input_sbal_addr_array = in_sbal_ptrs; init_data.output_sbal_addr_array = out_sbal_ptrs; - init_data.output_sbal_state_array = card->qdio.out_bufstates; init_data.scan_threshold = IS_IQD(card) ? 0 : 32; if (atomic_cmpxchg(&card->qdio.state, QETH_QDIO_ALLOCATED, @@ -6069,7 +6053,15 @@ static void qeth_iqd_tx_complete(struct qeth_qdio_out_q *queue, bool error = !!qdio_error; if (qdio_error == QDIO_ERROR_SLSB_PENDING) { - WARN_ON_ONCE(card->options.cq != QETH_CQ_ENABLED); + struct qaob *aob = buffer->aob; + + if (!aob) { + netdev_WARN_ONCE(card->dev, + "Pending TX buffer %#x without QAOB on TX queue %u\n", + bidx, queue->queue_no); + qeth_schedule_recovery(card); + return; + } QETH_CARD_TEXT_(card, 5, "pel%u", bidx); @@ -6125,6 +6117,8 @@ static void qeth_iqd_tx_complete(struct qeth_qdio_out_q *queue, default: WARN_ON_ONCE(1); } + + memset(aob, 0, sizeof(*aob)); } else if (card->options.cq == QETH_CQ_ENABLED) { qeth_notify_skbs(queue, buffer, qeth_compute_cq_notification(sflags, 0)); diff --git a/drivers/s390/scsi/zfcp_qdio.c b/drivers/s390/scsi/zfcp_qdio.c index 23ab16d65f2a..049596cbfb5d 100644 --- a/drivers/s390/scsi/zfcp_qdio.c +++ b/drivers/s390/scsi/zfcp_qdio.c @@ -128,7 +128,7 @@ static void zfcp_qdio_int_resp(struct ccw_device *cdev, unsigned int qdio_err, /* * put SBALs back to response queue */ - if (do_QDIO(cdev, QDIO_FLAG_SYNC_INPUT, 0, idx, count)) + if (do_QDIO(cdev, QDIO_FLAG_SYNC_INPUT, 0, idx, count, NULL)) zfcp_erp_adapter_reopen(qdio->adapter, 0, "qdires2"); } @@ -298,7 +298,7 @@ int zfcp_qdio_send(struct zfcp_qdio *qdio, struct zfcp_qdio_req *q_req) atomic_sub(sbal_number, &qdio->req_q_free); retval = do_QDIO(qdio->adapter->ccw_device, QDIO_FLAG_SYNC_OUTPUT, 0, - q_req->sbal_first, sbal_number); + q_req->sbal_first, sbal_number, NULL); if (unlikely(retval)) { /* Failed to submit the IO, roll back our modifications. */ @@ -463,7 +463,8 @@ int zfcp_qdio_open(struct zfcp_qdio *qdio) sbale->addr = 0; } - if (do_QDIO(cdev, QDIO_FLAG_SYNC_INPUT, 0, 0, QDIO_MAX_BUFFERS_PER_Q)) + if (do_QDIO(cdev, QDIO_FLAG_SYNC_INPUT, 0, 0, QDIO_MAX_BUFFERS_PER_Q, + NULL)) goto failed_qdio; /* set index of first available SBALS / number of available SBALS */ diff --git a/drivers/scsi/device_handler/scsi_dh_alua.c b/drivers/scsi/device_handler/scsi_dh_alua.c index ea436a14087f..5eff3368143d 100644 --- a/drivers/scsi/device_handler/scsi_dh_alua.c +++ b/drivers/scsi/device_handler/scsi_dh_alua.c @@ -573,10 +573,11 @@ static int alua_rtpg(struct scsi_device *sdev, struct alua_port_group *pg) * even though it shouldn't according to T10. * The retry without rtpg_ext_hdr_req set * handles this. + * Note: some arrays return a sense key of ILLEGAL_REQUEST + * with ASC 00h if they don't support the extended header. */ if (!(pg->flags & ALUA_RTPG_EXT_HDR_UNSUPP) && - sense_hdr.sense_key == ILLEGAL_REQUEST && - sense_hdr.asc == 0x24 && sense_hdr.ascq == 0) { + sense_hdr.sense_key == ILLEGAL_REQUEST) { pg->flags |= ALUA_RTPG_EXT_HDR_UNSUPP; goto retry; } diff --git a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c index 7451377c4cb6..3e359ac752fd 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c +++ b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c @@ -1646,7 +1646,7 @@ static int interrupt_init_v1_hw(struct hisi_hba *hisi_hba) idx = i * HISI_SAS_PHY_INT_NR; for (j = 0; j < HISI_SAS_PHY_INT_NR; j++, idx++) { irq = platform_get_irq(pdev, idx); - if (!irq) { + if (irq < 0) { dev_err(dev, "irq init: fail map phy interrupt %d\n", idx); return -ENOENT; @@ -1665,7 +1665,7 @@ static int interrupt_init_v1_hw(struct hisi_hba *hisi_hba) idx = hisi_hba->n_phy * HISI_SAS_PHY_INT_NR; for (i = 0; i < hisi_hba->queue_count; i++, idx++) { irq = platform_get_irq(pdev, idx); - if (!irq) { + if (irq < 0) { dev_err(dev, "irq init: could not map cq interrupt %d\n", idx); return -ENOENT; @@ -1683,7 +1683,7 @@ static int interrupt_init_v1_hw(struct hisi_hba *hisi_hba) idx = (hisi_hba->n_phy * HISI_SAS_PHY_INT_NR) + hisi_hba->queue_count; for (i = 0; i < HISI_SAS_FATAL_INT_NR; i++, idx++) { irq = platform_get_irq(pdev, idx); - if (!irq) { + if (irq < 0) { dev_err(dev, "irq init: could not map fatal interrupt %d\n", idx); return -ENOENT; diff --git a/drivers/scsi/ibmvscsi/ibmvfc.c b/drivers/scsi/ibmvscsi/ibmvfc.c index 61831f2fdb30..d6675a25719d 100644 --- a/drivers/scsi/ibmvscsi/ibmvfc.c +++ b/drivers/scsi/ibmvscsi/ibmvfc.c @@ -603,8 +603,17 @@ static void ibmvfc_set_host_action(struct ibmvfc_host *vhost, if (vhost->action == IBMVFC_HOST_ACTION_ALLOC_TGTS) vhost->action = action; break; + case IBMVFC_HOST_ACTION_REENABLE: + case IBMVFC_HOST_ACTION_RESET: + vhost->action = action; + break; case IBMVFC_HOST_ACTION_INIT: case IBMVFC_HOST_ACTION_TGT_DEL: + case IBMVFC_HOST_ACTION_LOGO: + case IBMVFC_HOST_ACTION_QUERY_TGTS: + case IBMVFC_HOST_ACTION_TGT_DEL_FAILED: + case IBMVFC_HOST_ACTION_NONE: + default: switch (vhost->action) { case IBMVFC_HOST_ACTION_RESET: case IBMVFC_HOST_ACTION_REENABLE: @@ -614,15 +623,6 @@ static void ibmvfc_set_host_action(struct ibmvfc_host *vhost, break; } break; - case IBMVFC_HOST_ACTION_LOGO: - case IBMVFC_HOST_ACTION_QUERY_TGTS: - case IBMVFC_HOST_ACTION_TGT_DEL_FAILED: - case IBMVFC_HOST_ACTION_NONE: - case IBMVFC_HOST_ACTION_RESET: - case IBMVFC_HOST_ACTION_REENABLE: - default: - vhost->action = action; - break; } } @@ -5373,30 +5373,49 @@ static void ibmvfc_do_work(struct ibmvfc_host *vhost) case IBMVFC_HOST_ACTION_INIT_WAIT: break; case IBMVFC_HOST_ACTION_RESET: - vhost->action = IBMVFC_HOST_ACTION_TGT_DEL; list_splice_init(&vhost->purge, &purge); spin_unlock_irqrestore(vhost->host->host_lock, flags); ibmvfc_complete_purge(&purge); rc = ibmvfc_reset_crq(vhost); + spin_lock_irqsave(vhost->host->host_lock, flags); - if (rc == H_CLOSED) + if (!rc || rc == H_CLOSED) vio_enable_interrupts(to_vio_dev(vhost->dev)); - if (rc || (rc = ibmvfc_send_crq_init(vhost)) || - (rc = vio_enable_interrupts(to_vio_dev(vhost->dev)))) { - ibmvfc_link_down(vhost, IBMVFC_LINK_DEAD); - dev_err(vhost->dev, "Error after reset (rc=%d)\n", rc); + if (vhost->action == IBMVFC_HOST_ACTION_RESET) { + /* + * The only action we could have changed to would have + * been reenable, in which case, we skip the rest of + * this path and wait until we've done the re-enable + * before sending the crq init. + */ + vhost->action = IBMVFC_HOST_ACTION_TGT_DEL; + + if (rc || (rc = ibmvfc_send_crq_init(vhost)) || + (rc = vio_enable_interrupts(to_vio_dev(vhost->dev)))) { + ibmvfc_link_down(vhost, IBMVFC_LINK_DEAD); + dev_err(vhost->dev, "Error after reset (rc=%d)\n", rc); + } } break; case IBMVFC_HOST_ACTION_REENABLE: - vhost->action = IBMVFC_HOST_ACTION_TGT_DEL; list_splice_init(&vhost->purge, &purge); spin_unlock_irqrestore(vhost->host->host_lock, flags); ibmvfc_complete_purge(&purge); rc = ibmvfc_reenable_crq_queue(vhost); + spin_lock_irqsave(vhost->host->host_lock, flags); - if (rc || (rc = ibmvfc_send_crq_init(vhost))) { - ibmvfc_link_down(vhost, IBMVFC_LINK_DEAD); - dev_err(vhost->dev, "Error after enable (rc=%d)\n", rc); + if (vhost->action == IBMVFC_HOST_ACTION_REENABLE) { + /* + * The only action we could have changed to would have + * been reset, in which case, we skip the rest of this + * path and wait until we've done the reset before + * sending the crq init. + */ + vhost->action = IBMVFC_HOST_ACTION_TGT_DEL; + if (rc || (rc = ibmvfc_send_crq_init(vhost))) { + ibmvfc_link_down(vhost, IBMVFC_LINK_DEAD); + dev_err(vhost->dev, "Error after enable (rc=%d)\n", rc); + } } break; case IBMVFC_HOST_ACTION_LOGO: diff --git a/drivers/scsi/jazz_esp.c b/drivers/scsi/jazz_esp.c index f0ed6863cc70..60a88a95a8e2 100644 --- a/drivers/scsi/jazz_esp.c +++ b/drivers/scsi/jazz_esp.c @@ -143,7 +143,9 @@ static int esp_jazz_probe(struct platform_device *dev) if (!esp->command_block) goto fail_unmap_regs; - host->irq = platform_get_irq(dev, 0); + host->irq = err = platform_get_irq(dev, 0); + if (err < 0) + goto fail_unmap_command_block; err = request_irq(host->irq, scsi_esp_intr, IRQF_SHARED, "ESP", esp); if (err < 0) goto fail_unmap_command_block; diff --git a/drivers/scsi/libfc/fc_lport.c b/drivers/scsi/libfc/fc_lport.c index 22826544da7e..9989669beec3 100644 --- a/drivers/scsi/libfc/fc_lport.c +++ b/drivers/scsi/libfc/fc_lport.c @@ -1731,7 +1731,7 @@ void fc_lport_flogi_resp(struct fc_seq *sp, struct fc_frame *fp, if (mfs < FC_SP_MIN_MAX_PAYLOAD || mfs > FC_SP_MAX_MAX_PAYLOAD) { FC_LPORT_DBG(lport, "FLOGI bad mfs:%hu response, " - "lport->mfs:%hu\n", mfs, lport->mfs); + "lport->mfs:%u\n", mfs, lport->mfs); fc_lport_error(lport, fp); goto out; } diff --git a/drivers/scsi/lpfc/lpfc_attr.c b/drivers/scsi/lpfc/lpfc_attr.c index bdd9a29f4201..0496a60735ef 100644 --- a/drivers/scsi/lpfc/lpfc_attr.c +++ b/drivers/scsi/lpfc/lpfc_attr.c @@ -1687,8 +1687,7 @@ lpfc_set_trunking(struct lpfc_hba *phba, char *buff_out) lpfc_printf_log(phba, KERN_ERR, LOG_MBOX, "0071 Set trunk mode failed with status: %d", rc); - if (rc != MBX_TIMEOUT) - mempool_free(mbox, phba->mbox_mem_pool); + mempool_free(mbox, phba->mbox_mem_pool); return 0; } @@ -6793,15 +6792,19 @@ lpfc_get_stats(struct Scsi_Host *shost) pmboxq->ctx_buf = NULL; pmboxq->vport = vport; - if (vport->fc_flag & FC_OFFLINE_MODE) + if (vport->fc_flag & FC_OFFLINE_MODE) { rc = lpfc_sli_issue_mbox(phba, pmboxq, MBX_POLL); - else - rc = lpfc_sli_issue_mbox_wait(phba, pmboxq, phba->fc_ratov * 2); - - if (rc != MBX_SUCCESS) { - if (rc != MBX_TIMEOUT) + if (rc != MBX_SUCCESS) { mempool_free(pmboxq, phba->mbox_mem_pool); - return NULL; + return NULL; + } + } else { + rc = lpfc_sli_issue_mbox_wait(phba, pmboxq, phba->fc_ratov * 2); + if (rc != MBX_SUCCESS) { + if (rc != MBX_TIMEOUT) + mempool_free(pmboxq, phba->mbox_mem_pool); + return NULL; + } } memset(hs, 0, sizeof (struct fc_host_statistics)); @@ -6825,15 +6828,19 @@ lpfc_get_stats(struct Scsi_Host *shost) pmboxq->ctx_buf = NULL; pmboxq->vport = vport; - if (vport->fc_flag & FC_OFFLINE_MODE) + if (vport->fc_flag & FC_OFFLINE_MODE) { rc = lpfc_sli_issue_mbox(phba, pmboxq, MBX_POLL); - else - rc = lpfc_sli_issue_mbox_wait(phba, pmboxq, phba->fc_ratov * 2); - - if (rc != MBX_SUCCESS) { - if (rc != MBX_TIMEOUT) + if (rc != MBX_SUCCESS) { mempool_free(pmboxq, phba->mbox_mem_pool); - return NULL; + return NULL; + } + } else { + rc = lpfc_sli_issue_mbox_wait(phba, pmboxq, phba->fc_ratov * 2); + if (rc != MBX_SUCCESS) { + if (rc != MBX_TIMEOUT) + mempool_free(pmboxq, phba->mbox_mem_pool); + return NULL; + } } hs->link_failure_count = pmb->un.varRdLnk.linkFailureCnt; @@ -6906,15 +6913,19 @@ lpfc_reset_stats(struct Scsi_Host *shost) pmboxq->vport = vport; if ((vport->fc_flag & FC_OFFLINE_MODE) || - (!(psli->sli_flag & LPFC_SLI_ACTIVE))) + (!(psli->sli_flag & LPFC_SLI_ACTIVE))) { rc = lpfc_sli_issue_mbox(phba, pmboxq, MBX_POLL); - else - rc = lpfc_sli_issue_mbox_wait(phba, pmboxq, phba->fc_ratov * 2); - - if (rc != MBX_SUCCESS) { - if (rc != MBX_TIMEOUT) + if (rc != MBX_SUCCESS) { mempool_free(pmboxq, phba->mbox_mem_pool); - return; + return; + } + } else { + rc = lpfc_sli_issue_mbox_wait(phba, pmboxq, phba->fc_ratov * 2); + if (rc != MBX_SUCCESS) { + if (rc != MBX_TIMEOUT) + mempool_free(pmboxq, phba->mbox_mem_pool); + return; + } } memset(pmboxq, 0, sizeof(LPFC_MBOXQ_t)); @@ -6924,15 +6935,19 @@ lpfc_reset_stats(struct Scsi_Host *shost) pmboxq->vport = vport; if ((vport->fc_flag & FC_OFFLINE_MODE) || - (!(psli->sli_flag & LPFC_SLI_ACTIVE))) + (!(psli->sli_flag & LPFC_SLI_ACTIVE))) { rc = lpfc_sli_issue_mbox(phba, pmboxq, MBX_POLL); - else + if (rc != MBX_SUCCESS) { + mempool_free(pmboxq, phba->mbox_mem_pool); + return; + } + } else { rc = lpfc_sli_issue_mbox_wait(phba, pmboxq, phba->fc_ratov * 2); - - if (rc != MBX_SUCCESS) { - if (rc != MBX_TIMEOUT) - mempool_free( pmboxq, phba->mbox_mem_pool); - return; + if (rc != MBX_SUCCESS) { + if (rc != MBX_TIMEOUT) + mempool_free(pmboxq, phba->mbox_mem_pool); + return; + } } lso->link_failure_count = pmb->un.varRdLnk.linkFailureCnt; diff --git a/drivers/scsi/lpfc/lpfc_crtn.h b/drivers/scsi/lpfc/lpfc_crtn.h index a0aad4896a45..763b1eeb0ca8 100644 --- a/drivers/scsi/lpfc/lpfc_crtn.h +++ b/drivers/scsi/lpfc/lpfc_crtn.h @@ -55,9 +55,6 @@ void lpfc_register_new_vport(struct lpfc_hba *, struct lpfc_vport *, void lpfc_unreg_vpi(struct lpfc_hba *, uint16_t, LPFC_MBOXQ_t *); void lpfc_init_link(struct lpfc_hba *, LPFC_MBOXQ_t *, uint32_t, uint32_t); void lpfc_request_features(struct lpfc_hba *, struct lpfcMboxq *); -void lpfc_supported_pages(struct lpfcMboxq *); -void lpfc_pc_sli4_params(struct lpfcMboxq *); -int lpfc_pc_sli4_params_get(struct lpfc_hba *, LPFC_MBOXQ_t *); int lpfc_sli4_mbox_rsrc_extent(struct lpfc_hba *, struct lpfcMboxq *, uint16_t, uint16_t, bool); int lpfc_get_sli4_parameters(struct lpfc_hba *, LPFC_MBOXQ_t *); @@ -351,8 +348,8 @@ int lpfc_sli_hbq_size(void); int lpfc_sli_issue_abort_iotag(struct lpfc_hba *, struct lpfc_sli_ring *, struct lpfc_iocbq *, void *); int lpfc_sli_sum_iocb(struct lpfc_vport *, uint16_t, uint64_t, lpfc_ctx_cmd); -int lpfc_sli_abort_iocb(struct lpfc_vport *, struct lpfc_sli_ring *, uint16_t, - uint64_t, lpfc_ctx_cmd); +int lpfc_sli_abort_iocb(struct lpfc_vport *vport, u16 tgt_id, u64 lun_id, + lpfc_ctx_cmd abort_cmd); int lpfc_sli_abort_taskmgmt(struct lpfc_vport *, struct lpfc_sli_ring *, uint16_t, uint64_t, lpfc_ctx_cmd); diff --git a/drivers/scsi/lpfc/lpfc_els.c b/drivers/scsi/lpfc/lpfc_els.c index f0a758138ae8..3dd22da3153f 100644 --- a/drivers/scsi/lpfc/lpfc_els.c +++ b/drivers/scsi/lpfc/lpfc_els.c @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2020 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2021 Broadcom. All Rights Reserved. The term * * “Broadcom†refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -1600,7 +1600,7 @@ lpfc_plogi_confirm_nport(struct lpfc_hba *phba, uint32_t *prsp, struct lpfc_nodelist *new_ndlp; struct serv_parm *sp; uint8_t name[sizeof(struct lpfc_name)]; - uint32_t rc, keepDID = 0, keep_nlp_flag = 0; + uint32_t keepDID = 0, keep_nlp_flag = 0; uint32_t keep_new_nlp_flag = 0; uint16_t keep_nlp_state; u32 keep_nlp_fc4_type = 0; @@ -1622,7 +1622,7 @@ lpfc_plogi_confirm_nport(struct lpfc_hba *phba, uint32_t *prsp, new_ndlp = lpfc_findnode_wwpn(vport, &sp->portName); /* return immediately if the WWPN matches ndlp */ - if (new_ndlp == ndlp) + if (!new_ndlp || (new_ndlp == ndlp)) return ndlp; if (phba->sli_rev == LPFC_SLI_REV4) { @@ -1641,30 +1641,11 @@ lpfc_plogi_confirm_nport(struct lpfc_hba *phba, uint32_t *prsp, (new_ndlp ? new_ndlp->nlp_flag : 0), (new_ndlp ? new_ndlp->nlp_fc4_type : 0)); - if (!new_ndlp) { - rc = memcmp(&ndlp->nlp_portname, name, - sizeof(struct lpfc_name)); - if (!rc) { - if (active_rrqs_xri_bitmap) - mempool_free(active_rrqs_xri_bitmap, - phba->active_rrq_pool); - return ndlp; - } - new_ndlp = lpfc_nlp_init(vport, ndlp->nlp_DID); - if (!new_ndlp) { - if (active_rrqs_xri_bitmap) - mempool_free(active_rrqs_xri_bitmap, - phba->active_rrq_pool); - return ndlp; - } - } else { - keepDID = new_ndlp->nlp_DID; - if (phba->sli_rev == LPFC_SLI_REV4 && - active_rrqs_xri_bitmap) - memcpy(active_rrqs_xri_bitmap, - new_ndlp->active_rrqs_xri_bitmap, - phba->cfg_rrq_xri_bitmap_sz); - } + keepDID = new_ndlp->nlp_DID; + + if (phba->sli_rev == LPFC_SLI_REV4 && active_rrqs_xri_bitmap) + memcpy(active_rrqs_xri_bitmap, new_ndlp->active_rrqs_xri_bitmap, + phba->cfg_rrq_xri_bitmap_sz); /* At this point in this routine, we know new_ndlp will be * returned. however, any previous GID_FTs that were done @@ -2063,13 +2044,12 @@ lpfc_cmpl_els_plogi(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, * This routine issues a Port Login (PLOGI) command to a remote N_Port * (with the @did) for a @vport. Before issuing a PLOGI to a remote N_Port, * the ndlp with the remote N_Port DID must exist on the @vport's ndlp list. - * This routine constructs the proper feilds of the PLOGI IOCB and invokes + * This routine constructs the proper fields of the PLOGI IOCB and invokes * the lpfc_sli_issue_iocb() routine to send out PLOGI ELS command. * - * Note that, in lpfc_prep_els_iocb() routine, the reference count of ndlp - * will be incremented by 1 for holding the ndlp and the reference to ndlp - * will be stored into the context1 field of the IOCB for the completion - * callback function to the PLOGI ELS command. + * Note that the ndlp reference count will be incremented by 1 for holding + * the ndlp and the reference to ndlp will be stored into the context1 field + * of the IOCB for the completion callback function to the PLOGI ELS command. * * Return code * 0 - Successfully issued a plogi for @vport @@ -2087,29 +2067,28 @@ lpfc_issue_els_plogi(struct lpfc_vport *vport, uint32_t did, uint8_t retry) int ret; ndlp = lpfc_findnode_did(vport, did); + if (!ndlp) + return 1; - if (ndlp) { - /* Defer the processing of the issue PLOGI until after the - * outstanding UNREG_RPI mbox command completes, unless we - * are going offline. This logic does not apply for Fabric DIDs - */ - if ((ndlp->nlp_flag & NLP_UNREG_INP) && - ((ndlp->nlp_DID & Fabric_DID_MASK) != Fabric_DID_MASK) && - !(vport->fc_flag & FC_OFFLINE_MODE)) { - lpfc_printf_vlog(vport, KERN_INFO, LOG_DISCOVERY, - "4110 Issue PLOGI x%x deferred " - "on NPort x%x rpi x%x Data: x%px\n", - ndlp->nlp_defer_did, ndlp->nlp_DID, - ndlp->nlp_rpi, ndlp); - - /* We can only defer 1st PLOGI */ - if (ndlp->nlp_defer_did == NLP_EVT_NOTHING_PENDING) - ndlp->nlp_defer_did = did; - return 0; - } + /* Defer the processing of the issue PLOGI until after the + * outstanding UNREG_RPI mbox command completes, unless we + * are going offline. This logic does not apply for Fabric DIDs + */ + if ((ndlp->nlp_flag & NLP_UNREG_INP) && + ((ndlp->nlp_DID & Fabric_DID_MASK) != Fabric_DID_MASK) && + !(vport->fc_flag & FC_OFFLINE_MODE)) { + lpfc_printf_vlog(vport, KERN_INFO, LOG_DISCOVERY, + "4110 Issue PLOGI x%x deferred " + "on NPort x%x rpi x%x Data: x%px\n", + ndlp->nlp_defer_did, ndlp->nlp_DID, + ndlp->nlp_rpi, ndlp); + + /* We can only defer 1st PLOGI */ + if (ndlp->nlp_defer_did == NLP_EVT_NOTHING_PENDING) + ndlp->nlp_defer_did = did; + return 0; } - /* If ndlp is not NULL, we will bump the reference count on it */ cmdsize = (sizeof(uint32_t) + sizeof(struct serv_parm)); elsiocb = lpfc_prep_els_iocb(vport, 1, cmdsize, retry, ndlp, did, ELS_CMD_PLOGI); @@ -3829,7 +3808,7 @@ lpfc_els_retry(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, did = irsp->un.elsreq64.remoteID; ndlp = lpfc_findnode_did(vport, did); if (!ndlp && (cmd != ELS_CMD_PLOGI)) - return 1; + return 0; } lpfc_debugfs_disc_trc(vport, LPFC_DISC_TRC_ELS_CMD, @@ -4473,10 +4452,7 @@ lpfc_mbx_cmpl_dflt_rpi(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb) * nlp_flag bitmap in the ndlp data structure, if the mbox command reference * field in the command IOCB is not NULL, the referred mailbox command will * be send out, and then invokes the lpfc_els_free_iocb() routine to release - * the IOCB. Under error conditions, such as when a LS_RJT is returned or a - * link down event occurred during the discovery, the lpfc_nlp_not_used() - * routine shall be invoked trying to release the ndlp if no other threads - * are currently referring it. + * the IOCB. **/ static void lpfc_cmpl_els_rsp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, @@ -4486,10 +4462,8 @@ lpfc_cmpl_els_rsp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, struct lpfc_vport *vport = ndlp ? ndlp->vport : NULL; struct Scsi_Host *shost = vport ? lpfc_shost_from_vport(vport) : NULL; IOCB_t *irsp; - uint8_t *pcmd; LPFC_MBOXQ_t *mbox = NULL; struct lpfc_dmabuf *mp = NULL; - uint32_t ls_rjt = 0; irsp = &rspiocb->iocb; @@ -4501,18 +4475,6 @@ lpfc_cmpl_els_rsp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, if (cmdiocb->context_un.mbox) mbox = cmdiocb->context_un.mbox; - /* First determine if this is a LS_RJT cmpl. Note, this callback - * function can have cmdiocb->contest1 (ndlp) field set to NULL. - */ - pcmd = (uint8_t *) (((struct lpfc_dmabuf *) cmdiocb->context2)->virt); - if (ndlp && (*((uint32_t *) (pcmd)) == ELS_CMD_LS_RJT)) { - /* A LS_RJT associated with Default RPI cleanup has its own - * separate code path. - */ - if (!(ndlp->nlp_flag & NLP_RM_DFLT_RPI)) - ls_rjt = 1; - } - /* Check to see if link went down during discovery */ if (!ndlp || lpfc_els_chk_latt(vport)) { if (mbox) { @@ -4523,15 +4485,6 @@ lpfc_cmpl_els_rsp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, } mempool_free(mbox, phba->mbox_mem_pool); } - if (ndlp && (ndlp->nlp_flag & NLP_RM_DFLT_RPI)) - if (lpfc_nlp_not_used(ndlp)) { - ndlp = NULL; - /* Indicate the node has already released, - * should not reference to it from within - * the routine lpfc_els_free_iocb. - */ - cmdiocb->context1 = NULL; - } goto out; } @@ -4609,29 +4562,6 @@ lpfc_cmpl_els_rsp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, "Data: x%x x%x x%x\n", ndlp->nlp_DID, ndlp->nlp_flag, ndlp->nlp_state, ndlp->nlp_rpi); - - if (lpfc_nlp_not_used(ndlp)) { - ndlp = NULL; - /* Indicate node has already been released, - * should not reference to it from within - * the routine lpfc_els_free_iocb. - */ - cmdiocb->context1 = NULL; - } - } else { - /* Do not drop node for lpfc_els_abort'ed ELS cmds */ - if (!lpfc_error_lost_link(irsp) && - ndlp->nlp_flag & NLP_ACC_REGLOGIN) { - if (lpfc_nlp_not_used(ndlp)) { - ndlp = NULL; - /* Indicate node has already been - * released, should not reference - * to it from within the routine - * lpfc_els_free_iocb. - */ - cmdiocb->context1 = NULL; - } - } } mp = (struct lpfc_dmabuf *)mbox->ctx_buf; if (mp) { @@ -4647,19 +4577,6 @@ lpfc_cmpl_els_rsp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, ndlp->nlp_flag &= ~NLP_ACC_REGLOGIN; ndlp->nlp_flag &= ~NLP_RM_DFLT_RPI; spin_unlock_irq(&ndlp->lock); - - /* If the node is not being used by another discovery thread, - * and we are sending a reject, we are done with it. - * Release driver reference count here and free associated - * resources. - */ - if (ls_rjt) - if (lpfc_nlp_not_used(ndlp)) - /* Indicate node has already been released, - * should not reference to it from within - * the routine lpfc_els_free_iocb. - */ - cmdiocb->context1 = NULL; } /* Release the originating I/O reference. */ diff --git a/drivers/scsi/lpfc/lpfc_hbadisc.c b/drivers/scsi/lpfc/lpfc_hbadisc.c index 48ca4a612f80..c5176f406386 100644 --- a/drivers/scsi/lpfc/lpfc_hbadisc.c +++ b/drivers/scsi/lpfc/lpfc_hbadisc.c @@ -140,11 +140,8 @@ lpfc_terminate_rport_io(struct fc_rport *rport) "rport terminate: sid:x%x did:x%x flg:x%x", ndlp->nlp_sid, ndlp->nlp_DID, ndlp->nlp_flag); - if (ndlp->nlp_sid != NLP_NO_SID) { - lpfc_sli_abort_iocb(vport, - &vport->phba->sli.sli3_ring[LPFC_FCP_RING], - ndlp->nlp_sid, 0, LPFC_CTX_TGT); - } + if (ndlp->nlp_sid != NLP_NO_SID) + lpfc_sli_abort_iocb(vport, ndlp->nlp_sid, 0, LPFC_CTX_TGT); } /* @@ -299,8 +296,7 @@ lpfc_dev_loss_tmo_handler(struct lpfc_nodelist *ndlp) if (ndlp->nlp_sid != NLP_NO_SID) { warn_on = 1; - lpfc_sli_abort_iocb(vport, &phba->sli.sli3_ring[LPFC_FCP_RING], - ndlp->nlp_sid, 0, LPFC_CTX_TGT); + lpfc_sli_abort_iocb(vport, ndlp->nlp_sid, 0, LPFC_CTX_TGT); } if (warn_on) { diff --git a/drivers/scsi/lpfc/lpfc_hw4.h b/drivers/scsi/lpfc/lpfc_hw4.h index 541b9aef6bfe..f5bc2c32a817 100644 --- a/drivers/scsi/lpfc/lpfc_hw4.h +++ b/drivers/scsi/lpfc/lpfc_hw4.h @@ -124,6 +124,7 @@ struct lpfc_sli_intf { /* Define SLI4 Alignment requirements. */ #define LPFC_ALIGN_16_BYTE 16 #define LPFC_ALIGN_64_BYTE 64 +#define SLI4_PAGE_SIZE 4096 /* Define SLI4 specific definitions. */ #define LPFC_MQ_CQE_BYTE_OFFSET 256 @@ -2976,62 +2977,6 @@ struct lpfc_mbx_request_features { #define lpfc_mbx_rq_ftr_rsp_mrqp_WORD word3 }; -struct lpfc_mbx_supp_pages { - uint32_t word1; -#define qs_SHIFT 0 -#define qs_MASK 0x00000001 -#define qs_WORD word1 -#define wr_SHIFT 1 -#define wr_MASK 0x00000001 -#define wr_WORD word1 -#define pf_SHIFT 8 -#define pf_MASK 0x000000ff -#define pf_WORD word1 -#define cpn_SHIFT 16 -#define cpn_MASK 0x000000ff -#define cpn_WORD word1 - uint32_t word2; -#define list_offset_SHIFT 0 -#define list_offset_MASK 0x000000ff -#define list_offset_WORD word2 -#define next_offset_SHIFT 8 -#define next_offset_MASK 0x000000ff -#define next_offset_WORD word2 -#define elem_cnt_SHIFT 16 -#define elem_cnt_MASK 0x000000ff -#define elem_cnt_WORD word2 - uint32_t word3; -#define pn_0_SHIFT 24 -#define pn_0_MASK 0x000000ff -#define pn_0_WORD word3 -#define pn_1_SHIFT 16 -#define pn_1_MASK 0x000000ff -#define pn_1_WORD word3 -#define pn_2_SHIFT 8 -#define pn_2_MASK 0x000000ff -#define pn_2_WORD word3 -#define pn_3_SHIFT 0 -#define pn_3_MASK 0x000000ff -#define pn_3_WORD word3 - uint32_t word4; -#define pn_4_SHIFT 24 -#define pn_4_MASK 0x000000ff -#define pn_4_WORD word4 -#define pn_5_SHIFT 16 -#define pn_5_MASK 0x000000ff -#define pn_5_WORD word4 -#define pn_6_SHIFT 8 -#define pn_6_MASK 0x000000ff -#define pn_6_WORD word4 -#define pn_7_SHIFT 0 -#define pn_7_MASK 0x000000ff -#define pn_7_WORD word4 - uint32_t rsvd[27]; -#define LPFC_SUPP_PAGES 0 -#define LPFC_BLOCK_GUARD_PROFILES 1 -#define LPFC_SLI4_PARAMETERS 2 -}; - struct lpfc_mbx_memory_dump_type3 { uint32_t word1; #define lpfc_mbx_memory_dump_type3_type_SHIFT 0 @@ -3248,121 +3193,6 @@ struct user_eeprom { uint8_t reserved191[57]; }; -struct lpfc_mbx_pc_sli4_params { - uint32_t word1; -#define qs_SHIFT 0 -#define qs_MASK 0x00000001 -#define qs_WORD word1 -#define wr_SHIFT 1 -#define wr_MASK 0x00000001 -#define wr_WORD word1 -#define pf_SHIFT 8 -#define pf_MASK 0x000000ff -#define pf_WORD word1 -#define cpn_SHIFT 16 -#define cpn_MASK 0x000000ff -#define cpn_WORD word1 - uint32_t word2; -#define if_type_SHIFT 0 -#define if_type_MASK 0x00000007 -#define if_type_WORD word2 -#define sli_rev_SHIFT 4 -#define sli_rev_MASK 0x0000000f -#define sli_rev_WORD word2 -#define sli_family_SHIFT 8 -#define sli_family_MASK 0x000000ff -#define sli_family_WORD word2 -#define featurelevel_1_SHIFT 16 -#define featurelevel_1_MASK 0x000000ff -#define featurelevel_1_WORD word2 -#define featurelevel_2_SHIFT 24 -#define featurelevel_2_MASK 0x0000001f -#define featurelevel_2_WORD word2 - uint32_t word3; -#define fcoe_SHIFT 0 -#define fcoe_MASK 0x00000001 -#define fcoe_WORD word3 -#define fc_SHIFT 1 -#define fc_MASK 0x00000001 -#define fc_WORD word3 -#define nic_SHIFT 2 -#define nic_MASK 0x00000001 -#define nic_WORD word3 -#define iscsi_SHIFT 3 -#define iscsi_MASK 0x00000001 -#define iscsi_WORD word3 -#define rdma_SHIFT 4 -#define rdma_MASK 0x00000001 -#define rdma_WORD word3 - uint32_t sge_supp_len; -#define SLI4_PAGE_SIZE 4096 - uint32_t word5; -#define if_page_sz_SHIFT 0 -#define if_page_sz_MASK 0x0000ffff -#define if_page_sz_WORD word5 -#define loopbk_scope_SHIFT 24 -#define loopbk_scope_MASK 0x0000000f -#define loopbk_scope_WORD word5 -#define rq_db_window_SHIFT 28 -#define rq_db_window_MASK 0x0000000f -#define rq_db_window_WORD word5 - uint32_t word6; -#define eq_pages_SHIFT 0 -#define eq_pages_MASK 0x0000000f -#define eq_pages_WORD word6 -#define eqe_size_SHIFT 8 -#define eqe_size_MASK 0x000000ff -#define eqe_size_WORD word6 - uint32_t word7; -#define cq_pages_SHIFT 0 -#define cq_pages_MASK 0x0000000f -#define cq_pages_WORD word7 -#define cqe_size_SHIFT 8 -#define cqe_size_MASK 0x000000ff -#define cqe_size_WORD word7 - uint32_t word8; -#define mq_pages_SHIFT 0 -#define mq_pages_MASK 0x0000000f -#define mq_pages_WORD word8 -#define mqe_size_SHIFT 8 -#define mqe_size_MASK 0x000000ff -#define mqe_size_WORD word8 -#define mq_elem_cnt_SHIFT 16 -#define mq_elem_cnt_MASK 0x000000ff -#define mq_elem_cnt_WORD word8 - uint32_t word9; -#define wq_pages_SHIFT 0 -#define wq_pages_MASK 0x0000ffff -#define wq_pages_WORD word9 -#define wqe_size_SHIFT 8 -#define wqe_size_MASK 0x000000ff -#define wqe_size_WORD word9 - uint32_t word10; -#define rq_pages_SHIFT 0 -#define rq_pages_MASK 0x0000ffff -#define rq_pages_WORD word10 -#define rqe_size_SHIFT 8 -#define rqe_size_MASK 0x000000ff -#define rqe_size_WORD word10 - uint32_t word11; -#define hdr_pages_SHIFT 0 -#define hdr_pages_MASK 0x0000000f -#define hdr_pages_WORD word11 -#define hdr_size_SHIFT 8 -#define hdr_size_MASK 0x0000000f -#define hdr_size_WORD word11 -#define hdr_pp_align_SHIFT 16 -#define hdr_pp_align_MASK 0x0000ffff -#define hdr_pp_align_WORD word11 - uint32_t word12; -#define sgl_pages_SHIFT 0 -#define sgl_pages_MASK 0x0000000f -#define sgl_pages_WORD word12 -#define sgl_pp_align_SHIFT 16 -#define sgl_pp_align_MASK 0x0000ffff -#define sgl_pp_align_WORD word12 - uint32_t rsvd_13_63[51]; -}; #define SLI4_PAGE_ALIGN(addr) (((addr)+((SLI4_PAGE_SIZE)-1)) \ &(~((SLI4_PAGE_SIZE)-1))) @@ -3994,8 +3824,6 @@ struct lpfc_mqe { struct lpfc_mbx_post_hdr_tmpl hdr_tmpl; struct lpfc_mbx_query_fw_config query_fw_cfg; struct lpfc_mbx_set_beacon_config beacon_config; - struct lpfc_mbx_supp_pages supp_pages; - struct lpfc_mbx_pc_sli4_params sli4_params; struct lpfc_mbx_get_sli4_parameters get_sli4_parameters; struct lpfc_mbx_set_link_diag_state link_diag_state; struct lpfc_mbx_set_link_diag_loopback link_diag_loopback; diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c index 71f340dd4fbd..a67051ba3f12 100644 --- a/drivers/scsi/lpfc/lpfc_init.c +++ b/drivers/scsi/lpfc/lpfc_init.c @@ -6573,8 +6573,6 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba) LPFC_MBOXQ_t *mboxq; MAILBOX_t *mb; int rc, i, max_buf_size; - uint8_t pn_page[LPFC_MAX_SUPPORTED_PAGES] = {0}; - struct lpfc_mqe *mqe; int longs; int extra; uint64_t wwn; @@ -6808,32 +6806,6 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba) lpfc_nvme_mod_param_dep(phba); - /* Get the Supported Pages if PORT_CAPABILITIES is supported by port. */ - lpfc_supported_pages(mboxq); - rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL); - if (!rc) { - mqe = &mboxq->u.mqe; - memcpy(&pn_page[0], ((uint8_t *)&mqe->un.supp_pages.word3), - LPFC_MAX_SUPPORTED_PAGES); - for (i = 0; i < LPFC_MAX_SUPPORTED_PAGES; i++) { - switch (pn_page[i]) { - case LPFC_SLI4_PARAMETERS: - phba->sli4_hba.pc_sli4_params.supported = 1; - break; - default: - break; - } - } - /* Read the port's SLI4 Parameters capabilities if supported. */ - if (phba->sli4_hba.pc_sli4_params.supported) - rc = lpfc_pc_sli4_params_get(phba, mboxq); - if (rc) { - mempool_free(mboxq, phba->mbox_mem_pool); - rc = -EIO; - goto out_free_bsmbx; - } - } - /* * Get sli4 parameters that override parameters from Port capabilities. * If this call fails, it isn't critical unless the SLI4 parameters come @@ -9660,8 +9632,7 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) "3250 QUERY_FW_CFG mailbox failed with status " "x%x add_status x%x, mbx status x%x\n", shdr_status, shdr_add_status, rc); - if (rc != MBX_TIMEOUT) - mempool_free(mboxq, phba->mbox_mem_pool); + mempool_free(mboxq, phba->mbox_mem_pool); rc = -ENXIO; goto out_error; } @@ -9677,8 +9648,7 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) "ulp1_mode:x%x\n", phba->sli4_hba.fw_func_mode, phba->sli4_hba.ulp0_mode, phba->sli4_hba.ulp1_mode); - if (rc != MBX_TIMEOUT) - mempool_free(mboxq, phba->mbox_mem_pool); + mempool_free(mboxq, phba->mbox_mem_pool); /* * Set up HBA Event Queues (EQs) @@ -10276,8 +10246,7 @@ lpfc_pci_function_reset(struct lpfc_hba *phba) shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); - if (rc != MBX_TIMEOUT) - mempool_free(mboxq, phba->mbox_mem_pool); + mempool_free(mboxq, phba->mbox_mem_pool); if (shdr_status || shdr_add_status || rc) { lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0495 SLI_FUNCTION_RESET mailbox " @@ -12075,78 +12044,6 @@ lpfc_sli4_hba_unset(struct lpfc_hba *phba) phba->pport->work_port_events = 0; } - /** - * lpfc_pc_sli4_params_get - Get the SLI4_PARAMS port capabilities. - * @phba: Pointer to HBA context object. - * @mboxq: Pointer to the mailboxq memory for the mailbox command response. - * - * This function is called in the SLI4 code path to read the port's - * sli4 capabilities. - * - * This function may be be called from any context that can block-wait - * for the completion. The expectation is that this routine is called - * typically from probe_one or from the online routine. - **/ -int -lpfc_pc_sli4_params_get(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq) -{ - int rc; - struct lpfc_mqe *mqe; - struct lpfc_pc_sli4_params *sli4_params; - uint32_t mbox_tmo; - - rc = 0; - mqe = &mboxq->u.mqe; - - /* Read the port's SLI4 Parameters port capabilities */ - lpfc_pc_sli4_params(mboxq); - if (!phba->sli4_hba.intr_enable) - rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL); - else { - mbox_tmo = lpfc_mbox_tmo_val(phba, mboxq); - rc = lpfc_sli_issue_mbox_wait(phba, mboxq, mbox_tmo); - } - - if (unlikely(rc)) - return 1; - - sli4_params = &phba->sli4_hba.pc_sli4_params; - sli4_params->if_type = bf_get(if_type, &mqe->un.sli4_params); - sli4_params->sli_rev = bf_get(sli_rev, &mqe->un.sli4_params); - sli4_params->sli_family = bf_get(sli_family, &mqe->un.sli4_params); - sli4_params->featurelevel_1 = bf_get(featurelevel_1, - &mqe->un.sli4_params); - sli4_params->featurelevel_2 = bf_get(featurelevel_2, - &mqe->un.sli4_params); - sli4_params->proto_types = mqe->un.sli4_params.word3; - sli4_params->sge_supp_len = mqe->un.sli4_params.sge_supp_len; - sli4_params->if_page_sz = bf_get(if_page_sz, &mqe->un.sli4_params); - sli4_params->rq_db_window = bf_get(rq_db_window, &mqe->un.sli4_params); - sli4_params->loopbk_scope = bf_get(loopbk_scope, &mqe->un.sli4_params); - sli4_params->eq_pages_max = bf_get(eq_pages, &mqe->un.sli4_params); - sli4_params->eqe_size = bf_get(eqe_size, &mqe->un.sli4_params); - sli4_params->cq_pages_max = bf_get(cq_pages, &mqe->un.sli4_params); - sli4_params->cqe_size = bf_get(cqe_size, &mqe->un.sli4_params); - sli4_params->mq_pages_max = bf_get(mq_pages, &mqe->un.sli4_params); - sli4_params->mqe_size = bf_get(mqe_size, &mqe->un.sli4_params); - sli4_params->mq_elem_cnt = bf_get(mq_elem_cnt, &mqe->un.sli4_params); - sli4_params->wq_pages_max = bf_get(wq_pages, &mqe->un.sli4_params); - sli4_params->wqe_size = bf_get(wqe_size, &mqe->un.sli4_params); - sli4_params->rq_pages_max = bf_get(rq_pages, &mqe->un.sli4_params); - sli4_params->rqe_size = bf_get(rqe_size, &mqe->un.sli4_params); - sli4_params->hdr_pages_max = bf_get(hdr_pages, &mqe->un.sli4_params); - sli4_params->hdr_size = bf_get(hdr_size, &mqe->un.sli4_params); - sli4_params->hdr_pp_align = bf_get(hdr_pp_align, &mqe->un.sli4_params); - sli4_params->sgl_pages_max = bf_get(sgl_pages, &mqe->un.sli4_params); - sli4_params->sgl_pp_align = bf_get(sgl_pp_align, &mqe->un.sli4_params); - - /* Make sure that sge_supp_len can be handled by the driver */ - if (sli4_params->sge_supp_len > LPFC_MAX_SGE_SIZE) - sli4_params->sge_supp_len = LPFC_MAX_SGE_SIZE; - - return rc; -} - /** * lpfc_get_sli4_parameters - Get the SLI4 Config PARAMETERS. * @phba: Pointer to HBA context object. @@ -12205,7 +12102,8 @@ lpfc_get_sli4_parameters(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq) else phba->sli3_options &= ~LPFC_SLI4_PHWQ_ENABLED; sli4_params->sge_supp_len = mbx_sli4_parameters->sge_supp_len; - sli4_params->loopbk_scope = bf_get(loopbk_scope, mbx_sli4_parameters); + sli4_params->loopbk_scope = bf_get(cfg_loopbk_scope, + mbx_sli4_parameters); sli4_params->oas_supported = bf_get(cfg_oas, mbx_sli4_parameters); sli4_params->cqv = bf_get(cfg_cqv, mbx_sli4_parameters); sli4_params->mqv = bf_get(cfg_mqv, mbx_sli4_parameters); diff --git a/drivers/scsi/lpfc/lpfc_mbox.c b/drivers/scsi/lpfc/lpfc_mbox.c index c03a7f12dd65..72dd22ad5dcc 100644 --- a/drivers/scsi/lpfc/lpfc_mbox.c +++ b/drivers/scsi/lpfc/lpfc_mbox.c @@ -2624,39 +2624,3 @@ lpfc_resume_rpi(struct lpfcMboxq *mbox, struct lpfc_nodelist *ndlp) resume_rpi->event_tag = ndlp->phba->fc_eventTag; } -/** - * lpfc_supported_pages - Initialize the PORT_CAPABILITIES supported pages - * mailbox command. - * @mbox: pointer to lpfc mbox command to initialize. - * - * The PORT_CAPABILITIES supported pages mailbox command is issued to - * retrieve the particular feature pages supported by the port. - **/ -void -lpfc_supported_pages(struct lpfcMboxq *mbox) -{ - struct lpfc_mbx_supp_pages *supp_pages; - - memset(mbox, 0, sizeof(*mbox)); - supp_pages = &mbox->u.mqe.un.supp_pages; - bf_set(lpfc_mqe_command, &mbox->u.mqe, MBX_PORT_CAPABILITIES); - bf_set(cpn, supp_pages, LPFC_SUPP_PAGES); -} - -/** - * lpfc_pc_sli4_params - Initialize the PORT_CAPABILITIES SLI4 Params mbox cmd. - * @mbox: pointer to lpfc mbox command to initialize. - * - * The PORT_CAPABILITIES SLI4 parameters mailbox command is issued to - * retrieve the particular SLI4 features supported by the port. - **/ -void -lpfc_pc_sli4_params(struct lpfcMboxq *mbox) -{ - struct lpfc_mbx_pc_sli4_params *sli4_params; - - memset(mbox, 0, sizeof(*mbox)); - sli4_params = &mbox->u.mqe.un.sli4_params; - bf_set(lpfc_mqe_command, &mbox->u.mqe, MBX_PORT_CAPABILITIES); - bf_set(cpn, sli4_params, LPFC_SLI4_PARAMETERS); -} diff --git a/drivers/scsi/lpfc/lpfc_nportdisc.c b/drivers/scsi/lpfc/lpfc_nportdisc.c index 135d8e8a42ba..9f05f5e329c6 100644 --- a/drivers/scsi/lpfc/lpfc_nportdisc.c +++ b/drivers/scsi/lpfc/lpfc_nportdisc.c @@ -279,106 +279,43 @@ lpfc_els_abort(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) lpfc_cancel_retry_delay_tmo(phba->pport, ndlp); } -/* lpfc_defer_pt2pt_acc - Complete SLI3 pt2pt processing on link up +/* lpfc_defer_plogi_acc - Issue PLOGI ACC after reg_login completes * @phba: pointer to lpfc hba data structure. - * @link_mbox: pointer to CONFIG_LINK mailbox object + * @login_mbox: pointer to REG_RPI mailbox object * - * This routine is only called if we are SLI3, direct connect pt2pt - * mode and the remote NPort issues the PLOGI after link up. + * The ACC for a rcv'ed PLOGI is deferred until AFTER the REG_RPI completes */ static void -lpfc_defer_pt2pt_acc(struct lpfc_hba *phba, LPFC_MBOXQ_t *link_mbox) +lpfc_defer_plogi_acc(struct lpfc_hba *phba, LPFC_MBOXQ_t *login_mbox) { - LPFC_MBOXQ_t *login_mbox; - MAILBOX_t *mb = &link_mbox->u.mb; struct lpfc_iocbq *save_iocb; struct lpfc_nodelist *ndlp; + MAILBOX_t *mb = &login_mbox->u.mb; + int rc; - ndlp = link_mbox->ctx_ndlp; - login_mbox = link_mbox->context3; + ndlp = login_mbox->ctx_ndlp; save_iocb = login_mbox->context3; - link_mbox->context3 = NULL; - login_mbox->context3 = NULL; - - /* Check for CONFIG_LINK error */ - if (mb->mbxStatus) { - lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, - "4575 CONFIG_LINK fails pt2pt discovery: %x\n", - mb->mbxStatus); - mempool_free(login_mbox, phba->mbox_mem_pool); - mempool_free(link_mbox, phba->mbox_mem_pool); - kfree(save_iocb); - return; - } - /* Now that CONFIG_LINK completed, and our SID is configured, - * we can now proceed with sending the PLOGI ACC. - */ - rc = lpfc_els_rsp_acc(link_mbox->vport, ELS_CMD_PLOGI, - save_iocb, ndlp, login_mbox); - if (rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, - "4576 PLOGI ACC fails pt2pt discovery: %x\n", - rc); - mempool_free(login_mbox, phba->mbox_mem_pool); + if (mb->mbxStatus == MBX_SUCCESS) { + /* Now that REG_RPI completed successfully, + * we can now proceed with sending the PLOGI ACC. + */ + rc = lpfc_els_rsp_acc(login_mbox->vport, ELS_CMD_PLOGI, + save_iocb, ndlp, NULL); + if (rc) { + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, + "4576 PLOGI ACC fails pt2pt discovery: " + "DID %x Data: %x\n", ndlp->nlp_DID, rc); + } } - mempool_free(link_mbox, phba->mbox_mem_pool); + /* Now process the REG_RPI cmpl */ + lpfc_mbx_cmpl_reg_login(phba, login_mbox); + ndlp->nlp_flag &= ~NLP_ACC_REGLOGIN; kfree(save_iocb); } -/** - * lpfc_defer_tgt_acc - Progress SLI4 target rcv PLOGI handler - * @phba: Pointer to HBA context object. - * @pmb: Pointer to mailbox object. - * - * This function provides the unreg rpi mailbox completion handler for a tgt. - * The routine frees the memory resources associated with the completed - * mailbox command and transmits the ELS ACC. - * - * This routine is only called if we are SLI4, acting in target - * mode and the remote NPort issues the PLOGI after link up. - **/ -static void -lpfc_defer_acc_rsp(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb) -{ - struct lpfc_vport *vport = pmb->vport; - struct lpfc_nodelist *ndlp = pmb->ctx_ndlp; - LPFC_MBOXQ_t *mbox = pmb->context3; - struct lpfc_iocbq *piocb = NULL; - int rc; - - if (mbox) { - pmb->context3 = NULL; - piocb = mbox->context3; - mbox->context3 = NULL; - } - - /* - * Complete the unreg rpi mbx request, and update flags. - * This will also restart any deferred events. - */ - lpfc_sli4_unreg_rpi_cmpl_clr(phba, pmb); - - if (!piocb) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, - "4578 PLOGI ACC fail\n"); - if (mbox) - mempool_free(mbox, phba->mbox_mem_pool); - return; - } - - rc = lpfc_els_rsp_acc(vport, ELS_CMD_PLOGI, piocb, ndlp, mbox); - if (rc) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, - "4579 PLOGI ACC fail %x\n", rc); - if (mbox) - mempool_free(mbox, phba->mbox_mem_pool); - } - kfree(piocb); -} - static int lpfc_rcv_plogi(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, struct lpfc_iocbq *cmdiocb) @@ -395,8 +332,7 @@ lpfc_rcv_plogi(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, struct lpfc_iocbq *save_iocb; struct ls_rjt stat; uint32_t vid, flag; - u16 rpi; - int rc, defer_acc; + int rc; memset(&stat, 0, sizeof (struct ls_rjt)); pcmd = (struct lpfc_dmabuf *) cmdiocb->context2; @@ -445,7 +381,6 @@ lpfc_rcv_plogi(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, else ndlp->nlp_fcp_info |= CLASS3; - defer_acc = 0; ndlp->nlp_class_sup = 0; if (sp->cls1.classValid) ndlp->nlp_class_sup |= FC_COS_CLASS1; @@ -539,27 +474,26 @@ lpfc_rcv_plogi(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, memcpy(&phba->fc_fabparam, sp, sizeof(struct serv_parm)); - /* Issue config_link / reg_vfi to account for updated TOV's */ - + /* Issue CONFIG_LINK for SLI3 or REG_VFI for SLI4, + * to account for updated TOV's / parameters + */ if (phba->sli_rev == LPFC_SLI_REV4) lpfc_issue_reg_vfi(vport); else { - defer_acc = 1; link_mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); if (!link_mbox) goto out; lpfc_config_link(phba, link_mbox); - link_mbox->mbox_cmpl = lpfc_defer_pt2pt_acc; + link_mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl; link_mbox->vport = vport; link_mbox->ctx_ndlp = ndlp; - save_iocb = kzalloc(sizeof(*save_iocb), GFP_KERNEL); - if (!save_iocb) + rc = lpfc_sli_issue_mbox(phba, link_mbox, MBX_NOWAIT); + if (rc == MBX_NOT_FINISHED) { + mempool_free(link_mbox, phba->mbox_mem_pool); goto out; - /* Save info from cmd IOCB used in rsp */ - memcpy((uint8_t *)save_iocb, (uint8_t *)cmdiocb, - sizeof(struct lpfc_iocbq)); + } } lpfc_can_disctmo(vport); @@ -578,59 +512,28 @@ lpfc_rcv_plogi(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, if (!login_mbox) goto out; - /* Registering an existing RPI behaves differently for SLI3 vs SLI4 */ - if (phba->nvmet_support && !defer_acc) { - link_mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); - if (!link_mbox) - goto out; - - /* As unique identifiers such as iotag would be overwritten - * with those from the cmdiocb, allocate separate temporary - * storage for the copy. - */ - save_iocb = kzalloc(sizeof(*save_iocb), GFP_KERNEL); - if (!save_iocb) - goto out; - - /* Unreg RPI is required for SLI4. */ - rpi = phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]; - lpfc_unreg_login(phba, vport->vpi, rpi, link_mbox); - link_mbox->vport = vport; - link_mbox->ctx_ndlp = lpfc_nlp_get(ndlp); - if (!link_mbox->ctx_ndlp) - goto out; - - link_mbox->mbox_cmpl = lpfc_defer_acc_rsp; - - if (((ndlp->nlp_DID & Fabric_DID_MASK) != Fabric_DID_MASK) && - (!(vport->fc_flag & FC_OFFLINE_MODE))) - ndlp->nlp_flag |= NLP_UNREG_INP; + save_iocb = kzalloc(sizeof(*save_iocb), GFP_KERNEL); + if (!save_iocb) + goto out; - /* Save info from cmd IOCB used in rsp */ - memcpy(save_iocb, cmdiocb, sizeof(*save_iocb)); + /* Save info from cmd IOCB to be used in rsp after all mbox completes */ + memcpy((uint8_t *)save_iocb, (uint8_t *)cmdiocb, + sizeof(struct lpfc_iocbq)); - /* Delay sending ACC till unreg RPI completes. */ - defer_acc = 1; - } else if (phba->sli_rev == LPFC_SLI_REV4) + /* Registering an existing RPI behaves differently for SLI3 vs SLI4 */ + if (phba->sli_rev == LPFC_SLI_REV4) lpfc_unreg_rpi(vport, ndlp); + /* Issue REG_LOGIN first, before ACCing the PLOGI, thus we will + * always be deferring the ACC. + */ rc = lpfc_reg_rpi(phba, vport->vpi, icmd->un.rcvels.remoteID, (uint8_t *)sp, login_mbox, ndlp->nlp_rpi); if (rc) goto out; - /* ACC PLOGI rsp command needs to execute first, - * queue this login_mbox command to be processed later. - */ login_mbox->mbox_cmpl = lpfc_mbx_cmpl_reg_login; - /* - * login_mbox->ctx_ndlp = lpfc_nlp_get(ndlp) deferred until mailbox - * command issued in lpfc_cmpl_els_acc(). - */ login_mbox->vport = vport; - spin_lock_irq(&ndlp->lock); - ndlp->nlp_flag |= (NLP_ACC_REGLOGIN | NLP_RCV_PLOGI); - spin_unlock_irq(&ndlp->lock); /* * If there is an outstanding PLOGI issued, abort it before @@ -660,7 +563,8 @@ lpfc_rcv_plogi(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, * to register, then unregister the RPI. */ spin_lock_irq(&ndlp->lock); - ndlp->nlp_flag |= NLP_RM_DFLT_RPI; + ndlp->nlp_flag |= (NLP_RM_DFLT_RPI | NLP_ACC_REGLOGIN | + NLP_RCV_PLOGI); spin_unlock_irq(&ndlp->lock); stat.un.b.lsRjtRsnCode = LSRJT_INVALID_CMD; stat.un.b.lsRjtRsnCodeExp = LSEXP_NOTHING_MORE; @@ -670,42 +574,39 @@ lpfc_rcv_plogi(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, mempool_free(login_mbox, phba->mbox_mem_pool); return 1; } - if (defer_acc) { - /* So the order here should be: - * SLI3 pt2pt - * Issue CONFIG_LINK mbox - * CONFIG_LINK cmpl - * SLI4 tgt - * Issue UNREG RPI mbx - * UNREG RPI cmpl - * Issue PLOGI ACC - * PLOGI ACC cmpl - * Issue REG_LOGIN mbox - */ - /* Save the REG_LOGIN mbox for and rcv IOCB copy later */ - link_mbox->context3 = login_mbox; - login_mbox->context3 = save_iocb; + /* So the order here should be: + * SLI3 pt2pt + * Issue CONFIG_LINK mbox + * CONFIG_LINK cmpl + * SLI4 pt2pt + * Issue REG_VFI mbox + * REG_VFI cmpl + * SLI4 + * Issue UNREG RPI mbx + * UNREG RPI cmpl + * Issue REG_RPI mbox + * REG RPI cmpl + * Issue PLOGI ACC + * PLOGI ACC cmpl + */ + login_mbox->mbox_cmpl = lpfc_defer_plogi_acc; + login_mbox->ctx_ndlp = lpfc_nlp_get(ndlp); + login_mbox->context3 = save_iocb; /* For PLOGI ACC */ - /* Start the ball rolling by issuing CONFIG_LINK here */ - rc = lpfc_sli_issue_mbox(phba, link_mbox, MBX_NOWAIT); - if (rc == MBX_NOT_FINISHED) - goto out; - return 1; - } + spin_lock_irq(&ndlp->lock); + ndlp->nlp_flag |= (NLP_ACC_REGLOGIN | NLP_RCV_PLOGI); + spin_unlock_irq(&ndlp->lock); + + /* Start the ball rolling by issuing REG_LOGIN here */ + rc = lpfc_sli_issue_mbox(phba, login_mbox, MBX_NOWAIT); + if (rc == MBX_NOT_FINISHED) + goto out; + lpfc_nlp_set_state(vport, ndlp, NLP_STE_REG_LOGIN_ISSUE); - rc = lpfc_els_rsp_acc(vport, ELS_CMD_PLOGI, cmdiocb, ndlp, login_mbox); - if (rc) - mempool_free(login_mbox, phba->mbox_mem_pool); return 1; out: - if (defer_acc) - lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, - "4577 discovery failure: %p %p %p\n", - save_iocb, link_mbox, login_mbox); kfree(save_iocb); - if (link_mbox) - mempool_free(link_mbox, phba->mbox_mem_pool); if (login_mbox) mempool_free(login_mbox, phba->mbox_mem_pool); @@ -913,9 +814,14 @@ lpfc_rcv_logo(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, } } else if ((!(ndlp->nlp_type & NLP_FABRIC) && ((ndlp->nlp_type & NLP_FCP_TARGET) || - !(ndlp->nlp_type & NLP_FCP_INITIATOR))) || + (ndlp->nlp_type & NLP_NVME_TARGET) || + (vport->fc_flag & FC_PT2PT))) || (ndlp->nlp_state == NLP_STE_ADISC_ISSUE)) { - /* Only try to re-login if this is NOT a Fabric Node */ + /* Only try to re-login if this is NOT a Fabric Node + * AND the remote NPORT is a FCP/NVME Target or we + * are in pt2pt mode. NLP_STE_ADISC_ISSUE is a special + * case for LOGO as a response to ADISC behavior. + */ mod_timer(&ndlp->nlp_delayfunc, jiffies + msecs_to_jiffies(1000 * 1)); spin_lock_irq(&ndlp->lock); @@ -1985,8 +1891,6 @@ lpfc_cmpl_reglogin_reglogin_issue(struct lpfc_vport *vport, ndlp->nlp_last_elscmd = ELS_CMD_PLOGI; lpfc_issue_els_logo(vport, ndlp, 0); - ndlp->nlp_prev_state = NLP_STE_REG_LOGIN_ISSUE; - lpfc_nlp_set_state(vport, ndlp, NLP_STE_NPR_NODE); return ndlp->nlp_state; } @@ -2633,12 +2537,10 @@ static uint32_t lpfc_rcv_prlo_mapped_node(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, void *arg, uint32_t evt) { - struct lpfc_hba *phba = vport->phba; struct lpfc_iocbq *cmdiocb = (struct lpfc_iocbq *) arg; /* flush the target */ - lpfc_sli_abort_iocb(vport, &phba->sli.sli3_ring[LPFC_FCP_RING], - ndlp->nlp_sid, 0, LPFC_CTX_TGT); + lpfc_sli_abort_iocb(vport, ndlp->nlp_sid, 0, LPFC_CTX_TGT); /* Treat like rcv logo */ lpfc_rcv_logo(vport, ndlp, cmdiocb, ELS_CMD_PRLO); diff --git a/drivers/scsi/lpfc/lpfc_nvmet.c b/drivers/scsi/lpfc/lpfc_nvmet.c index bb2a4a0d1295..a3fd959f7431 100644 --- a/drivers/scsi/lpfc/lpfc_nvmet.c +++ b/drivers/scsi/lpfc/lpfc_nvmet.c @@ -3304,7 +3304,6 @@ lpfc_nvmet_unsol_issue_abort(struct lpfc_hba *phba, bf_set(wqe_rcvoxid, &wqe_abts->xmit_sequence.wqe_com, xri); /* Word 10 */ - bf_set(wqe_dbde, &wqe_abts->xmit_sequence.wqe_com, 1); bf_set(wqe_iod, &wqe_abts->xmit_sequence.wqe_com, LPFC_WQE_IOD_WRITE); bf_set(wqe_lenloc, &wqe_abts->xmit_sequence.wqe_com, LPFC_WQE_LENLOC_WORD12); diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c index fa1a714a78f0..920cf329268b 100644 --- a/drivers/scsi/lpfc/lpfc_sli.c +++ b/drivers/scsi/lpfc/lpfc_sli.c @@ -5683,12 +5683,10 @@ lpfc_sli4_get_ctl_attr(struct lpfc_hba *phba) phba->sli4_hba.lnk_info.lnk_no, phba->BIOSVersion); out_free_mboxq: - if (rc != MBX_TIMEOUT) { - if (bf_get(lpfc_mqe_command, &mboxq->u.mqe) == MBX_SLI4_CONFIG) - lpfc_sli4_mbox_cmd_free(phba, mboxq); - else - mempool_free(mboxq, phba->mbox_mem_pool); - } + if (bf_get(lpfc_mqe_command, &mboxq->u.mqe) == MBX_SLI4_CONFIG) + lpfc_sli4_mbox_cmd_free(phba, mboxq); + else + mempool_free(mboxq, phba->mbox_mem_pool); return rc; } @@ -5789,12 +5787,10 @@ lpfc_sli4_retrieve_pport_name(struct lpfc_hba *phba) } out_free_mboxq: - if (rc != MBX_TIMEOUT) { - if (bf_get(lpfc_mqe_command, &mboxq->u.mqe) == MBX_SLI4_CONFIG) - lpfc_sli4_mbox_cmd_free(phba, mboxq); - else - mempool_free(mboxq, phba->mbox_mem_pool); - } + if (bf_get(lpfc_mqe_command, &mboxq->u.mqe) == MBX_SLI4_CONFIG) + lpfc_sli4_mbox_cmd_free(phba, mboxq); + else + mempool_free(mboxq, phba->mbox_mem_pool); return rc; } @@ -11647,7 +11643,7 @@ lpfc_sli_issue_abort_iotag(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, icmd = &cmdiocb->iocb; if (icmd->ulpCommand == CMD_ABORT_XRI_CN || icmd->ulpCommand == CMD_CLOSE_XRI_CN || - (cmdiocb->iocb_flag & LPFC_DRIVER_ABORTED) != 0) + cmdiocb->iocb_flag & LPFC_DRIVER_ABORTED) return IOCB_ABORTING; if (!pring) { @@ -11811,13 +11807,20 @@ lpfc_sli_validate_fcp_iocb(struct lpfc_iocbq *iocbq, struct lpfc_vport *vport, lpfc_ctx_cmd ctx_cmd) { struct lpfc_io_buf *lpfc_cmd; + IOCB_t *icmd = NULL; int rc = 1; if (!iocbq || iocbq->vport != vport) return rc; - if (!(iocbq->iocb_flag & LPFC_IO_FCP) || - !(iocbq->iocb_flag & LPFC_IO_ON_TXCMPLQ)) + if (!(iocbq->iocb_flag & LPFC_IO_FCP) || + !(iocbq->iocb_flag & LPFC_IO_ON_TXCMPLQ) || + iocbq->iocb_flag & LPFC_DRIVER_ABORTED) + return rc; + + icmd = &iocbq->iocb; + if (icmd->ulpCommand == CMD_ABORT_XRI_CN || + icmd->ulpCommand == CMD_CLOSE_XRI_CN) return rc; lpfc_cmd = container_of(iocbq, struct lpfc_io_buf, cur_iocbq); @@ -11945,7 +11948,6 @@ lpfc_sli_abort_fcp_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, /** * lpfc_sli_abort_iocb - issue abort for all commands on a host/target/LUN * @vport: Pointer to virtual port. - * @pring: Pointer to driver SLI ring object. * @tgt_id: SCSI ID of the target. * @lun_id: LUN ID of the scsi device. * @abort_cmd: LPFC_CTX_LUN/LPFC_CTX_TGT/LPFC_CTX_HOST. @@ -11960,18 +11962,22 @@ lpfc_sli_abort_fcp_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, * FCP iocbs associated with SCSI target specified by tgt_id parameter. * When abort_cmd == LPFC_CTX_HOST, the function sends abort to all * FCP iocbs associated with virtual port. + * The pring used for SLI3 is sli3_ring[LPFC_FCP_RING], for SLI4 + * lpfc_sli4_calc_ring is used. * This function returns number of iocbs it failed to abort. * This function is called with no locks held. **/ int -lpfc_sli_abort_iocb(struct lpfc_vport *vport, struct lpfc_sli_ring *pring, - uint16_t tgt_id, uint64_t lun_id, lpfc_ctx_cmd abort_cmd) +lpfc_sli_abort_iocb(struct lpfc_vport *vport, u16 tgt_id, u64 lun_id, + lpfc_ctx_cmd abort_cmd) { struct lpfc_hba *phba = vport->phba; + struct lpfc_sli_ring *pring = NULL; struct lpfc_iocbq *iocbq; int errcnt = 0, ret_val = 0; unsigned long iflags; int i; + void *fcp_cmpl = NULL; /* all I/Os are in process of being flushed */ if (phba->hba_flag & HBA_IOQ_FLUSH) @@ -11985,8 +11991,15 @@ lpfc_sli_abort_iocb(struct lpfc_vport *vport, struct lpfc_sli_ring *pring, continue; spin_lock_irqsave(&phba->hbalock, iflags); + if (phba->sli_rev == LPFC_SLI_REV3) { + pring = &phba->sli.sli3_ring[LPFC_FCP_RING]; + fcp_cmpl = lpfc_sli_abort_fcp_cmpl; + } else if (phba->sli_rev == LPFC_SLI_REV4) { + pring = lpfc_sli4_calc_ring(phba, iocbq); + fcp_cmpl = lpfc_sli4_abort_fcp_cmpl; + } ret_val = lpfc_sli_issue_abort_iotag(phba, pring, iocbq, - lpfc_sli_abort_fcp_cmpl); + fcp_cmpl); spin_unlock_irqrestore(&phba->hbalock, iflags); if (ret_val != IOCB_SUCCESS) errcnt++; @@ -17072,8 +17085,7 @@ lpfc_rq_destroy(struct lpfc_hba *phba, struct lpfc_queue *hrq, "2509 RQ_DESTROY mailbox failed with " "status x%x add_status x%x, mbx status x%x\n", shdr_status, shdr_add_status, rc); - if (rc != MBX_TIMEOUT) - mempool_free(mbox, hrq->phba->mbox_mem_pool); + mempool_free(mbox, hrq->phba->mbox_mem_pool); return -ENXIO; } bf_set(lpfc_mbx_rq_destroy_q_id, &mbox->u.mqe.un.rq_destroy.u.request, @@ -17170,7 +17182,9 @@ lpfc_sli4_post_sgl(struct lpfc_hba *phba, shdr = (union lpfc_sli4_cfg_shdr *) &post_sgl_pages->header.cfg_shdr; shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); - if (rc != MBX_TIMEOUT) + if (!phba->sli4_hba.intr_enable) + mempool_free(mbox, phba->mbox_mem_pool); + else if (rc != MBX_TIMEOUT) mempool_free(mbox, phba->mbox_mem_pool); if (shdr_status || shdr_add_status || rc) { lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, @@ -17367,7 +17381,9 @@ lpfc_sli4_post_sgl_list(struct lpfc_hba *phba, shdr = (union lpfc_sli4_cfg_shdr *) &sgl->cfg_shdr; shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); - if (rc != MBX_TIMEOUT) + if (!phba->sli4_hba.intr_enable) + lpfc_sli4_mbox_cmd_free(phba, mbox); + else if (rc != MBX_TIMEOUT) lpfc_sli4_mbox_cmd_free(phba, mbox); if (shdr_status || shdr_add_status || rc) { lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, @@ -17480,7 +17496,9 @@ lpfc_sli4_post_io_sgl_block(struct lpfc_hba *phba, struct list_head *nblist, shdr = (union lpfc_sli4_cfg_shdr *)&sgl->cfg_shdr; shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); - if (rc != MBX_TIMEOUT) + if (!phba->sli4_hba.intr_enable) + lpfc_sli4_mbox_cmd_free(phba, mbox); + else if (rc != MBX_TIMEOUT) lpfc_sli4_mbox_cmd_free(phba, mbox); if (shdr_status || shdr_add_status || rc) { lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, @@ -18064,7 +18082,6 @@ lpfc_sli4_seq_abort_rsp_cmpl(struct lpfc_hba *phba, if (cmd_iocbq) { ndlp = (struct lpfc_nodelist *)cmd_iocbq->context1; lpfc_nlp_put(ndlp); - lpfc_nlp_not_used(ndlp); lpfc_sli_release_iocbq(phba, cmd_iocbq); } @@ -18831,8 +18848,7 @@ lpfc_sli4_post_rpi_hdr(struct lpfc_hba *phba, struct lpfc_rpi_hdr *rpi_page) shdr = (union lpfc_sli4_cfg_shdr *) &hdr_tmpl->header.cfg_shdr; shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); - if (rc != MBX_TIMEOUT) - mempool_free(mboxq, phba->mbox_mem_pool); + mempool_free(mboxq, phba->mbox_mem_pool); if (shdr_status || shdr_add_status || rc) { lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2514 POST_RPI_HDR mailbox failed with " @@ -20076,7 +20092,9 @@ lpfc_wr_object(struct lpfc_hba *phba, struct list_head *dmabuf_list, break; } } - if (rc != MBX_TIMEOUT) + if (!phba->sli4_hba.intr_enable) + mempool_free(mbox, phba->mbox_mem_pool); + else if (rc != MBX_TIMEOUT) mempool_free(mbox, phba->mbox_mem_pool); if (shdr_status || shdr_add_status || rc) { lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c b/drivers/scsi/mpt3sas/mpt3sas_base.c index ac0eef975f17..b6beacfd0f62 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_base.c +++ b/drivers/scsi/mpt3sas/mpt3sas_base.c @@ -7252,6 +7252,8 @@ _base_diag_reset(struct MPT3SAS_ADAPTER *ioc) ioc_info(ioc, "sending diag reset !!\n"); + pci_cfg_access_lock(ioc->pdev); + drsprintk(ioc, ioc_info(ioc, "clear interrupts\n")); count = 0; @@ -7342,10 +7344,12 @@ _base_diag_reset(struct MPT3SAS_ADAPTER *ioc) goto out; } + pci_cfg_access_unlock(ioc->pdev); ioc_info(ioc, "diag reset: SUCCESS\n"); return 0; out: + pci_cfg_access_unlock(ioc->pdev); ioc_err(ioc, "diag reset: FAILED\n"); return -EFAULT; } diff --git a/drivers/scsi/mpt3sas/mpt3sas_ctl.c b/drivers/scsi/mpt3sas/mpt3sas_ctl.c index 44f9a05db94e..2ec11be62a82 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_ctl.c +++ b/drivers/scsi/mpt3sas/mpt3sas_ctl.c @@ -2507,7 +2507,7 @@ _ctl_addnl_diag_query(struct MPT3SAS_ADAPTER *ioc, void __user *arg) __func__, karg.unique_id); return -EPERM; } - memset(&karg.buffer_rel_condition, 0, sizeof(struct htb_rel_query)); + memset(&karg.rel_query, 0, sizeof(karg.rel_query)); if ((ioc->diag_buffer_status[buffer_type] & MPT3_DIAG_BUFFER_IS_REGISTERED) == 0) { ioc_info(ioc, "%s: buffer_type(0x%02x) is not registered\n", @@ -2520,8 +2520,7 @@ _ctl_addnl_diag_query(struct MPT3SAS_ADAPTER *ioc, void __user *arg) __func__, buffer_type); return -EPERM; } - memcpy(&karg.buffer_rel_condition, &ioc->htb_rel, - sizeof(struct htb_rel_query)); + memcpy(&karg.rel_query, &ioc->htb_rel, sizeof(karg.rel_query)); out: if (copy_to_user(arg, &karg, sizeof(struct mpt3_addnl_diag_query))) { ioc_err(ioc, "%s: unable to write mpt3_addnl_diag_query data @ %p\n", diff --git a/drivers/scsi/mpt3sas/mpt3sas_ctl.h b/drivers/scsi/mpt3sas/mpt3sas_ctl.h index d2ccdafb8df2..8f6ffb40261c 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_ctl.h +++ b/drivers/scsi/mpt3sas/mpt3sas_ctl.h @@ -50,6 +50,8 @@ #include #endif +#include "mpt3sas_base.h" + #ifndef MPT2SAS_MINOR #define MPT2SAS_MINOR (MPT_MINOR + 1) #endif @@ -436,19 +438,13 @@ struct mpt3_diag_read_buffer { * struct mpt3_addnl_diag_query - diagnostic buffer release reason * @hdr - generic header * @unique_id - unique id associated with this buffer. - * @buffer_rel_condition - Release condition ioctl/sysfs/reset - * @reserved1 - * @trigger_type - Master/Event/scsi/MPI - * @trigger_info_dwords - Data Correspondig to trigger type + * @rel_query - release query. * @reserved2 */ struct mpt3_addnl_diag_query { struct mpt3_ioctl_header hdr; uint32_t unique_id; - uint16_t buffer_rel_condition; - uint16_t reserved1; - uint32_t trigger_type; - uint32_t trigger_info_dwords[2]; + struct htb_rel_query rel_query; uint32_t reserved2[2]; }; diff --git a/drivers/scsi/mpt3sas/mpt3sas_scsih.c b/drivers/scsi/mpt3sas/mpt3sas_scsih.c index 6aa6de729187..ae1973878cc7 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_scsih.c +++ b/drivers/scsi/mpt3sas/mpt3sas_scsih.c @@ -6483,6 +6483,9 @@ _scsih_alloc_vphy(struct MPT3SAS_ADAPTER *ioc, u8 port_id, u8 phy_num) if (!vphy) return NULL; + if (!port->vphys_mask) + INIT_LIST_HEAD(&port->vphys_list); + /* * Enable bit corresponding to HBA phy number on its * parent hba_port object's vphys_mask field. @@ -6490,7 +6493,6 @@ _scsih_alloc_vphy(struct MPT3SAS_ADAPTER *ioc, u8 port_id, u8 phy_num) port->vphys_mask |= (1 << phy_num); vphy->phy_mask |= (1 << phy_num); - INIT_LIST_HEAD(&port->vphys_list); list_add_tail(&vphy->list, &port->vphys_list); ioc_info(ioc, diff --git a/drivers/scsi/pm8001/pm8001_hwi.c b/drivers/scsi/pm8001/pm8001_hwi.c index 31e5455d280c..1b1a57f46989 100644 --- a/drivers/scsi/pm8001/pm8001_hwi.c +++ b/drivers/scsi/pm8001/pm8001_hwi.c @@ -643,7 +643,7 @@ static void init_pci_device_addresses(struct pm8001_hba_info *pm8001_ha) */ static int pm8001_chip_init(struct pm8001_hba_info *pm8001_ha) { - u8 i = 0; + u32 i = 0; u16 deviceid; pci_read_config_word(pm8001_ha->pdev, PCI_DEVICE_ID, &deviceid); /* 8081 controllers need BAR shift to access MPI space diff --git a/drivers/scsi/pm8001/pm80xx_hwi.c b/drivers/scsi/pm8001/pm80xx_hwi.c index 84315560e8e1..c6b0834e3806 100644 --- a/drivers/scsi/pm8001/pm80xx_hwi.c +++ b/drivers/scsi/pm8001/pm80xx_hwi.c @@ -1502,9 +1502,9 @@ static int mpi_uninit_check(struct pm8001_hba_info *pm8001_ha) /* wait until Inbound DoorBell Clear Register toggled */ if (IS_SPCV_12G(pm8001_ha->pdev)) { - max_wait_count = 4 * 1000 * 1000;/* 4 sec */ + max_wait_count = 30 * 1000 * 1000; /* 30 sec */ } else { - max_wait_count = 2 * 1000 * 1000;/* 2 sec */ + max_wait_count = 15 * 1000 * 1000; /* 15 sec */ } do { udelay(1); diff --git a/drivers/scsi/qla2xxx/qla_attr.c b/drivers/scsi/qla2xxx/qla_attr.c index 63391c9be05d..3aa9869f6fae 100644 --- a/drivers/scsi/qla2xxx/qla_attr.c +++ b/drivers/scsi/qla2xxx/qla_attr.c @@ -2864,6 +2864,8 @@ qla2x00_reset_host_stats(struct Scsi_Host *shost) vha->qla_stats.jiffies_at_last_reset = get_jiffies_64(); if (IS_FWI2_CAPABLE(ha)) { + int rval; + stats = dma_alloc_coherent(&ha->pdev->dev, sizeof(*stats), &stats_dma, GFP_KERNEL); if (!stats) { @@ -2873,7 +2875,11 @@ qla2x00_reset_host_stats(struct Scsi_Host *shost) } /* reset firmware statistics */ - qla24xx_get_isp_stats(base_vha, stats, stats_dma, BIT_0); + rval = qla24xx_get_isp_stats(base_vha, stats, stats_dma, BIT_0); + if (rval != QLA_SUCCESS) + ql_log(ql_log_warn, vha, 0x70de, + "Resetting ISP statistics failed: rval = %d\n", + rval); dma_free_coherent(&ha->pdev->dev, sizeof(*stats), stats, stats_dma); diff --git a/drivers/scsi/qla2xxx/qla_bsg.c b/drivers/scsi/qla2xxx/qla_bsg.c index bee8cf9f8123..aef2f7cc89d3 100644 --- a/drivers/scsi/qla2xxx/qla_bsg.c +++ b/drivers/scsi/qla2xxx/qla_bsg.c @@ -25,10 +25,11 @@ void qla2x00_bsg_job_done(srb_t *sp, int res) struct bsg_job *bsg_job = sp->u.bsg_job; struct fc_bsg_reply *bsg_reply = bsg_job->reply; + sp->free(sp); + bsg_reply->result = res; bsg_job_done(bsg_job, bsg_reply->result, bsg_reply->reply_payload_rcv_len); - sp->free(sp); } void qla2x00_bsg_sp_free(srb_t *sp) @@ -2583,6 +2584,10 @@ qla2x00_get_host_stats(struct bsg_job *bsg_job) } data = kzalloc(response_len, GFP_KERNEL); + if (!data) { + kfree(req_data); + return -ENOMEM; + } ret = qla2xxx_get_ini_stats(fc_bsg_to_shost(bsg_job), req_data->stat_type, data, response_len); diff --git a/drivers/scsi/qla2xxx/qla_init.c b/drivers/scsi/qla2xxx/qla_init.c index f01f07116bd3..8cb0574cfa91 100644 --- a/drivers/scsi/qla2xxx/qla_init.c +++ b/drivers/scsi/qla2xxx/qla_init.c @@ -1194,6 +1194,9 @@ static int qla24xx_post_prli_work(struct scsi_qla_host *vha, fc_port_t *fcport) { struct qla_work_evt *e; + if (vha->host->active_mode == MODE_TARGET) + return QLA_FUNCTION_FAILED; + e = qla2x00_alloc_work(vha, QLA_EVT_PRLI); if (!e) return QLA_FUNCTION_FAILED; diff --git a/drivers/scsi/qla2xxx/qla_isr.c b/drivers/scsi/qla2xxx/qla_isr.c index 5e188375c871..af4831c9edf9 100644 --- a/drivers/scsi/qla2xxx/qla_isr.c +++ b/drivers/scsi/qla2xxx/qla_isr.c @@ -4005,11 +4005,11 @@ qla24xx_enable_msix(struct qla_hw_data *ha, struct rsp_que *rsp) if (USER_CTRL_IRQ(ha) || !ha->mqiobase) { /* user wants to control IRQ setting for target mode */ ret = pci_alloc_irq_vectors(ha->pdev, min_vecs, - min((u16)ha->msix_count, (u16)num_online_cpus()), + min((u16)ha->msix_count, (u16)(num_online_cpus() + min_vecs)), PCI_IRQ_MSIX); } else ret = pci_alloc_irq_vectors_affinity(ha->pdev, min_vecs, - min((u16)ha->msix_count, (u16)num_online_cpus()), + min((u16)ha->msix_count, (u16)(num_online_cpus() + min_vecs)), PCI_IRQ_MSIX | PCI_IRQ_AFFINITY, &desc); diff --git a/drivers/scsi/qla2xxx/qla_os.c b/drivers/scsi/qla2xxx/qla_os.c index 074392560f3d..0e07b98dfae8 100644 --- a/drivers/scsi/qla2xxx/qla_os.c +++ b/drivers/scsi/qla2xxx/qla_os.c @@ -1013,8 +1013,6 @@ qla2xxx_mqueuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd, if (rval != QLA_SUCCESS) { ql_dbg(ql_dbg_io + ql_dbg_verbose, vha, 0x3078, "Start scsi failed rval=%d for cmd=%p.\n", rval, cmd); - if (rval == QLA_INTERFACE_ERROR) - goto qc24_free_sp_fail_command; goto qc24_host_busy_free_sp; } @@ -1026,11 +1024,6 @@ qla2xxx_mqueuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd, qc24_target_busy: return SCSI_MLQUEUE_TARGET_BUSY; -qc24_free_sp_fail_command: - sp->free(sp); - CMD_SP(cmd) = NULL; - qla2xxx_rel_qpair_sp(sp->qpair, sp); - qc24_fail_command: cmd->scsi_done(cmd); diff --git a/drivers/scsi/smartpqi/smartpqi_init.c b/drivers/scsi/smartpqi/smartpqi_init.c index a1dacb6e993e..c30f6047410f 100644 --- a/drivers/scsi/smartpqi/smartpqi_init.c +++ b/drivers/scsi/smartpqi/smartpqi_init.c @@ -5488,6 +5488,8 @@ static void pqi_fail_io_queued_for_device(struct pqi_ctrl_info *ctrl_info, list_del(&io_request->request_list_entry); set_host_byte(scmd, DID_RESET); + pqi_free_io_request(io_request); + scsi_dma_unmap(scmd); pqi_scsi_done(scmd); } @@ -5524,6 +5526,8 @@ static void pqi_fail_io_queued_for_all_devices(struct pqi_ctrl_info *ctrl_info) list_del(&io_request->request_list_entry); set_host_byte(scmd, DID_RESET); + pqi_free_io_request(io_request); + scsi_dma_unmap(scmd); pqi_scsi_done(scmd); } @@ -6598,6 +6602,7 @@ static int pqi_register_scsi(struct pqi_ctrl_info *ctrl_info) shost->irq = pci_irq_vector(ctrl_info->pci_dev, 0); shost->unique_id = shost->irq; shost->nr_hw_queues = ctrl_info->num_queue_groups; + shost->host_tagset = 1; shost->hostdata[0] = (unsigned long)ctrl_info; rc = scsi_add_host(shost, &ctrl_info->pci_dev->dev); @@ -8216,6 +8221,10 @@ static const struct pci_device_id pqi_pci_id_table[] = { PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, 0x152d, 0x8a37) }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + 0x193d, 0x8460) + }, { PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, 0x193d, 0x1104) @@ -8288,6 +8297,22 @@ static const struct pci_device_id pqi_pci_id_table[] = { PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, 0x1bd4, 0x004f) }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + 0x1bd4, 0x0051) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + 0x1bd4, 0x0052) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + 0x1bd4, 0x0053) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + 0x1bd4, 0x0054) + }, { PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, 0x19e5, 0xd227) @@ -8448,6 +8473,122 @@ static const struct pci_device_id pqi_pci_id_table[] = { PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, PCI_VENDOR_ID_ADAPTEC2, 0x1380) }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x1400) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x1402) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x1410) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x1411) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x1412) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x1420) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x1430) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x1440) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x1441) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x1450) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x1452) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x1460) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x1461) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x1462) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x1470) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x1471) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x1472) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x1480) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x1490) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x1491) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x14a0) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x14a1) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x14b0) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x14b1) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x14c0) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x14c1) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x14d0) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x14e0) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_ADAPTEC2, 0x14f0) + }, { PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, PCI_VENDOR_ID_ADVANTECH, 0x8312) @@ -8512,6 +8653,10 @@ static const struct pci_device_id pqi_pci_id_table[] = { PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, PCI_VENDOR_ID_HP, 0x1001) }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + PCI_VENDOR_ID_HP, 0x1002) + }, { PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, PCI_VENDOR_ID_HP, 0x1100) @@ -8520,6 +8665,22 @@ static const struct pci_device_id pqi_pci_id_table[] = { PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, PCI_VENDOR_ID_HP, 0x1101) }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + 0x1590, 0x0294) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + 0x1590, 0x02db) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + 0x1590, 0x02dc) + }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + 0x1590, 0x032e) + }, { PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, 0x1d8d, 0x0800) diff --git a/drivers/scsi/sni_53c710.c b/drivers/scsi/sni_53c710.c index 9e2e196bc202..97c6f81b1d2a 100644 --- a/drivers/scsi/sni_53c710.c +++ b/drivers/scsi/sni_53c710.c @@ -58,6 +58,7 @@ static int snirm710_probe(struct platform_device *dev) struct NCR_700_Host_Parameters *hostdata; struct Scsi_Host *host; struct resource *res; + int rc; res = platform_get_resource(dev, IORESOURCE_MEM, 0); if (!res) @@ -83,7 +84,9 @@ static int snirm710_probe(struct platform_device *dev) goto out_kfree; host->this_id = 7; host->base = base; - host->irq = platform_get_irq(dev, 0); + host->irq = rc = platform_get_irq(dev, 0); + if (rc < 0) + goto out_put_host; if(request_irq(host->irq, NCR_700_intr, IRQF_SHARED, "snirm710", host)) { printk(KERN_ERR "snirm710: request_irq failed!\n"); goto out_put_host; diff --git a/drivers/scsi/sun3x_esp.c b/drivers/scsi/sun3x_esp.c index 7de82f2c9757..d3489ac7ab28 100644 --- a/drivers/scsi/sun3x_esp.c +++ b/drivers/scsi/sun3x_esp.c @@ -206,7 +206,9 @@ static int esp_sun3x_probe(struct platform_device *dev) if (!esp->command_block) goto fail_unmap_regs_dma; - host->irq = platform_get_irq(dev, 0); + host->irq = err = platform_get_irq(dev, 0); + if (err < 0) + goto fail_unmap_command_block; err = request_irq(host->irq, scsi_esp_intr, IRQF_SHARED, "SUN3X ESP", esp); if (err < 0) diff --git a/drivers/scsi/ufs/ufshcd-pltfrm.c b/drivers/scsi/ufs/ufshcd-pltfrm.c index 1a69949a4ea1..b56d9b4e5f03 100644 --- a/drivers/scsi/ufs/ufshcd-pltfrm.c +++ b/drivers/scsi/ufs/ufshcd-pltfrm.c @@ -377,7 +377,7 @@ int ufshcd_pltfrm_init(struct platform_device *pdev, irq = platform_get_irq(pdev, 0); if (irq < 0) { - err = -ENODEV; + err = irq; goto out; } diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c index d3d05e997c13..0c71a159d08f 100644 --- a/drivers/scsi/ufs/ufshcd.c +++ b/drivers/scsi/ufs/ufshcd.c @@ -8599,7 +8599,7 @@ static void ufshcd_vreg_set_lpm(struct ufs_hba *hba) } else if (!ufshcd_is_ufs_dev_active(hba)) { ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false); vcc_off = true; - if (!ufshcd_is_link_active(hba)) { + if (ufshcd_is_link_hibern8(hba) || ufshcd_is_link_off(hba)) { ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq); ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2); } @@ -8621,7 +8621,7 @@ static int ufshcd_vreg_set_hpm(struct ufs_hba *hba) !hba->dev_info.is_lu_power_on_wp) { ret = ufshcd_setup_vreg(hba, true); } else if (!ufshcd_is_ufs_dev_active(hba)) { - if (!ret && !ufshcd_is_link_active(hba)) { + if (!ufshcd_is_link_active(hba)) { ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq); if (ret) goto vcc_disable; @@ -8978,10 +8978,13 @@ int ufshcd_system_suspend(struct ufs_hba *hba) if (!hba->is_powered) return 0; + cancel_delayed_work_sync(&hba->rpm_dev_flush_recheck_work); + if ((ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl) == hba->curr_dev_pwr_mode) && (ufs_get_pm_lvl_to_link_pwr_state(hba->spm_lvl) == hba->uic_link_state) && + pm_runtime_suspended(hba->dev) && !hba->dev_info.b_rpm_dev_flush_capable) goto out; diff --git a/drivers/soc/aspeed/aspeed-lpc-snoop.c b/drivers/soc/aspeed/aspeed-lpc-snoop.c index 20acac6342ef..5828f94b8a7d 100644 --- a/drivers/soc/aspeed/aspeed-lpc-snoop.c +++ b/drivers/soc/aspeed/aspeed-lpc-snoop.c @@ -95,8 +95,10 @@ static ssize_t snoop_file_read(struct file *file, char __user *buffer, return -EINTR; } ret = kfifo_to_user(&chan->fifo, buffer, count, &copied); + if (ret) + return ret; - return ret ? ret : copied; + return copied; } static __poll_t snoop_file_poll(struct file *file, diff --git a/drivers/soc/mediatek/mt8173-pm-domains.h b/drivers/soc/mediatek/mt8173-pm-domains.h index 3e8ee5dabb43..654c717e5467 100644 --- a/drivers/soc/mediatek/mt8173-pm-domains.h +++ b/drivers/soc/mediatek/mt8173-pm-domains.h @@ -12,24 +12,28 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8173[] = { [MT8173_POWER_DOMAIN_VDEC] = { + .name = "vdec", .sta_mask = PWR_STATUS_VDEC, .ctl_offs = SPM_VDE_PWR_CON, .sram_pdn_bits = GENMASK(11, 8), .sram_pdn_ack_bits = GENMASK(12, 12), }, [MT8173_POWER_DOMAIN_VENC] = { + .name = "venc", .sta_mask = PWR_STATUS_VENC, .ctl_offs = SPM_VEN_PWR_CON, .sram_pdn_bits = GENMASK(11, 8), .sram_pdn_ack_bits = GENMASK(15, 12), }, [MT8173_POWER_DOMAIN_ISP] = { + .name = "isp", .sta_mask = PWR_STATUS_ISP, .ctl_offs = SPM_ISP_PWR_CON, .sram_pdn_bits = GENMASK(11, 8), .sram_pdn_ack_bits = GENMASK(13, 12), }, [MT8173_POWER_DOMAIN_MM] = { + .name = "mm", .sta_mask = PWR_STATUS_DISP, .ctl_offs = SPM_DIS_PWR_CON, .sram_pdn_bits = GENMASK(11, 8), @@ -40,18 +44,21 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8173[] = { }, }, [MT8173_POWER_DOMAIN_VENC_LT] = { + .name = "venc_lt", .sta_mask = PWR_STATUS_VENC_LT, .ctl_offs = SPM_VEN2_PWR_CON, .sram_pdn_bits = GENMASK(11, 8), .sram_pdn_ack_bits = GENMASK(15, 12), }, [MT8173_POWER_DOMAIN_AUDIO] = { + .name = "audio", .sta_mask = PWR_STATUS_AUDIO, .ctl_offs = SPM_AUDIO_PWR_CON, .sram_pdn_bits = GENMASK(11, 8), .sram_pdn_ack_bits = GENMASK(15, 12), }, [MT8173_POWER_DOMAIN_USB] = { + .name = "usb", .sta_mask = PWR_STATUS_USB, .ctl_offs = SPM_USB_PWR_CON, .sram_pdn_bits = GENMASK(11, 8), @@ -59,18 +66,21 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8173[] = { .caps = MTK_SCPD_ACTIVE_WAKEUP, }, [MT8173_POWER_DOMAIN_MFG_ASYNC] = { + .name = "mfg_async", .sta_mask = PWR_STATUS_MFG_ASYNC, .ctl_offs = SPM_MFG_ASYNC_PWR_CON, .sram_pdn_bits = GENMASK(11, 8), .sram_pdn_ack_bits = 0, }, [MT8173_POWER_DOMAIN_MFG_2D] = { + .name = "mfg_2d", .sta_mask = PWR_STATUS_MFG_2D, .ctl_offs = SPM_MFG_2D_PWR_CON, .sram_pdn_bits = GENMASK(11, 8), .sram_pdn_ack_bits = GENMASK(13, 12), }, [MT8173_POWER_DOMAIN_MFG] = { + .name = "mfg", .sta_mask = PWR_STATUS_MFG, .ctl_offs = SPM_MFG_PWR_CON, .sram_pdn_bits = GENMASK(13, 8), diff --git a/drivers/soc/mediatek/mt8183-pm-domains.h b/drivers/soc/mediatek/mt8183-pm-domains.h index aa5230e6c12f..98a9940d05fb 100644 --- a/drivers/soc/mediatek/mt8183-pm-domains.h +++ b/drivers/soc/mediatek/mt8183-pm-domains.h @@ -12,12 +12,14 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8183[] = { [MT8183_POWER_DOMAIN_AUDIO] = { + .name = "audio", .sta_mask = PWR_STATUS_AUDIO, .ctl_offs = 0x0314, .sram_pdn_bits = GENMASK(11, 8), .sram_pdn_ack_bits = GENMASK(15, 12), }, [MT8183_POWER_DOMAIN_CONN] = { + .name = "conn", .sta_mask = PWR_STATUS_CONN, .ctl_offs = 0x032c, .sram_pdn_bits = 0, @@ -28,12 +30,14 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8183[] = { }, }, [MT8183_POWER_DOMAIN_MFG_ASYNC] = { + .name = "mfg_async", .sta_mask = PWR_STATUS_MFG_ASYNC, .ctl_offs = 0x0334, .sram_pdn_bits = 0, .sram_pdn_ack_bits = 0, }, [MT8183_POWER_DOMAIN_MFG] = { + .name = "mfg", .sta_mask = PWR_STATUS_MFG, .ctl_offs = 0x0338, .sram_pdn_bits = GENMASK(8, 8), @@ -41,18 +45,21 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8183[] = { .caps = MTK_SCPD_DOMAIN_SUPPLY, }, [MT8183_POWER_DOMAIN_MFG_CORE0] = { + .name = "mfg_core0", .sta_mask = BIT(7), .ctl_offs = 0x034c, .sram_pdn_bits = GENMASK(8, 8), .sram_pdn_ack_bits = GENMASK(12, 12), }, [MT8183_POWER_DOMAIN_MFG_CORE1] = { + .name = "mfg_core1", .sta_mask = BIT(20), .ctl_offs = 0x0310, .sram_pdn_bits = GENMASK(8, 8), .sram_pdn_ack_bits = GENMASK(12, 12), }, [MT8183_POWER_DOMAIN_MFG_2D] = { + .name = "mfg_2d", .sta_mask = PWR_STATUS_MFG_2D, .ctl_offs = 0x0348, .sram_pdn_bits = GENMASK(8, 8), @@ -65,6 +72,7 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8183[] = { }, }, [MT8183_POWER_DOMAIN_DISP] = { + .name = "disp", .sta_mask = PWR_STATUS_DISP, .ctl_offs = 0x030c, .sram_pdn_bits = GENMASK(8, 8), @@ -83,6 +91,7 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8183[] = { }, }, [MT8183_POWER_DOMAIN_CAM] = { + .name = "cam", .sta_mask = BIT(25), .ctl_offs = 0x0344, .sram_pdn_bits = GENMASK(9, 8), @@ -105,6 +114,7 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8183[] = { }, }, [MT8183_POWER_DOMAIN_ISP] = { + .name = "isp", .sta_mask = PWR_STATUS_ISP, .ctl_offs = 0x0308, .sram_pdn_bits = GENMASK(9, 8), @@ -127,6 +137,7 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8183[] = { }, }, [MT8183_POWER_DOMAIN_VDEC] = { + .name = "vdec", .sta_mask = BIT(31), .ctl_offs = 0x0300, .sram_pdn_bits = GENMASK(8, 8), @@ -139,6 +150,7 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8183[] = { }, }, [MT8183_POWER_DOMAIN_VENC] = { + .name = "venc", .sta_mask = PWR_STATUS_VENC, .ctl_offs = 0x0304, .sram_pdn_bits = GENMASK(11, 8), @@ -151,6 +163,7 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8183[] = { }, }, [MT8183_POWER_DOMAIN_VPU_TOP] = { + .name = "vpu_top", .sta_mask = BIT(26), .ctl_offs = 0x0324, .sram_pdn_bits = GENMASK(8, 8), @@ -177,6 +190,7 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8183[] = { }, }, [MT8183_POWER_DOMAIN_VPU_CORE0] = { + .name = "vpu_core0", .sta_mask = BIT(27), .ctl_offs = 0x33c, .sram_pdn_bits = GENMASK(11, 8), @@ -194,6 +208,7 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8183[] = { .caps = MTK_SCPD_SRAM_ISO, }, [MT8183_POWER_DOMAIN_VPU_CORE1] = { + .name = "vpu_core1", .sta_mask = BIT(28), .ctl_offs = 0x0340, .sram_pdn_bits = GENMASK(11, 8), diff --git a/drivers/soc/mediatek/mt8192-pm-domains.h b/drivers/soc/mediatek/mt8192-pm-domains.h index 0fdf6dc6231f..543dda70de01 100644 --- a/drivers/soc/mediatek/mt8192-pm-domains.h +++ b/drivers/soc/mediatek/mt8192-pm-domains.h @@ -12,6 +12,7 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8192[] = { [MT8192_POWER_DOMAIN_AUDIO] = { + .name = "audio", .sta_mask = BIT(21), .ctl_offs = 0x0354, .sram_pdn_bits = GENMASK(8, 8), @@ -24,6 +25,7 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8192[] = { }, }, [MT8192_POWER_DOMAIN_CONN] = { + .name = "conn", .sta_mask = PWR_STATUS_CONN, .ctl_offs = 0x0304, .sram_pdn_bits = 0, @@ -45,12 +47,14 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8192[] = { .caps = MTK_SCPD_KEEP_DEFAULT_OFF, }, [MT8192_POWER_DOMAIN_MFG0] = { + .name = "mfg0", .sta_mask = BIT(2), .ctl_offs = 0x0308, .sram_pdn_bits = GENMASK(8, 8), .sram_pdn_ack_bits = GENMASK(12, 12), }, [MT8192_POWER_DOMAIN_MFG1] = { + .name = "mfg1", .sta_mask = BIT(3), .ctl_offs = 0x030c, .sram_pdn_bits = GENMASK(8, 8), @@ -75,36 +79,42 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8192[] = { }, }, [MT8192_POWER_DOMAIN_MFG2] = { + .name = "mfg2", .sta_mask = BIT(4), .ctl_offs = 0x0310, .sram_pdn_bits = GENMASK(8, 8), .sram_pdn_ack_bits = GENMASK(12, 12), }, [MT8192_POWER_DOMAIN_MFG3] = { + .name = "mfg3", .sta_mask = BIT(5), .ctl_offs = 0x0314, .sram_pdn_bits = GENMASK(8, 8), .sram_pdn_ack_bits = GENMASK(12, 12), }, [MT8192_POWER_DOMAIN_MFG4] = { + .name = "mfg4", .sta_mask = BIT(6), .ctl_offs = 0x0318, .sram_pdn_bits = GENMASK(8, 8), .sram_pdn_ack_bits = GENMASK(12, 12), }, [MT8192_POWER_DOMAIN_MFG5] = { + .name = "mfg5", .sta_mask = BIT(7), .ctl_offs = 0x031c, .sram_pdn_bits = GENMASK(8, 8), .sram_pdn_ack_bits = GENMASK(12, 12), }, [MT8192_POWER_DOMAIN_MFG6] = { + .name = "mfg6", .sta_mask = BIT(8), .ctl_offs = 0x0320, .sram_pdn_bits = GENMASK(8, 8), .sram_pdn_ack_bits = GENMASK(12, 12), }, [MT8192_POWER_DOMAIN_DISP] = { + .name = "disp", .sta_mask = BIT(20), .ctl_offs = 0x0350, .sram_pdn_bits = GENMASK(8, 8), @@ -133,6 +143,7 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8192[] = { }, }, [MT8192_POWER_DOMAIN_IPE] = { + .name = "ipe", .sta_mask = BIT(14), .ctl_offs = 0x0338, .sram_pdn_bits = GENMASK(8, 8), @@ -149,6 +160,7 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8192[] = { }, }, [MT8192_POWER_DOMAIN_ISP] = { + .name = "isp", .sta_mask = BIT(12), .ctl_offs = 0x0330, .sram_pdn_bits = GENMASK(8, 8), @@ -165,6 +177,7 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8192[] = { }, }, [MT8192_POWER_DOMAIN_ISP2] = { + .name = "isp2", .sta_mask = BIT(13), .ctl_offs = 0x0334, .sram_pdn_bits = GENMASK(8, 8), @@ -181,6 +194,7 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8192[] = { }, }, [MT8192_POWER_DOMAIN_MDP] = { + .name = "mdp", .sta_mask = BIT(19), .ctl_offs = 0x034c, .sram_pdn_bits = GENMASK(8, 8), @@ -197,6 +211,7 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8192[] = { }, }, [MT8192_POWER_DOMAIN_VENC] = { + .name = "venc", .sta_mask = BIT(17), .ctl_offs = 0x0344, .sram_pdn_bits = GENMASK(8, 8), @@ -213,6 +228,7 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8192[] = { }, }, [MT8192_POWER_DOMAIN_VDEC] = { + .name = "vdec", .sta_mask = BIT(15), .ctl_offs = 0x033c, .sram_pdn_bits = GENMASK(8, 8), @@ -229,12 +245,14 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8192[] = { }, }, [MT8192_POWER_DOMAIN_VDEC2] = { + .name = "vdec2", .sta_mask = BIT(16), .ctl_offs = 0x0340, .sram_pdn_bits = GENMASK(8, 8), .sram_pdn_ack_bits = GENMASK(12, 12), }, [MT8192_POWER_DOMAIN_CAM] = { + .name = "cam", .sta_mask = BIT(23), .ctl_offs = 0x035c, .sram_pdn_bits = GENMASK(8, 8), @@ -263,18 +281,21 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8192[] = { }, }, [MT8192_POWER_DOMAIN_CAM_RAWA] = { + .name = "cam_rawa", .sta_mask = BIT(24), .ctl_offs = 0x0360, .sram_pdn_bits = GENMASK(8, 8), .sram_pdn_ack_bits = GENMASK(12, 12), }, [MT8192_POWER_DOMAIN_CAM_RAWB] = { + .name = "cam_rawb", .sta_mask = BIT(25), .ctl_offs = 0x0364, .sram_pdn_bits = GENMASK(8, 8), .sram_pdn_ack_bits = GENMASK(12, 12), }, [MT8192_POWER_DOMAIN_CAM_RAWC] = { + .name = "cam_rawc", .sta_mask = BIT(26), .ctl_offs = 0x0368, .sram_pdn_bits = GENMASK(8, 8), diff --git a/drivers/soc/mediatek/mtk-pm-domains.c b/drivers/soc/mediatek/mtk-pm-domains.c index b7f697666bdd..0af00efa0ef8 100644 --- a/drivers/soc/mediatek/mtk-pm-domains.c +++ b/drivers/soc/mediatek/mtk-pm-domains.c @@ -438,7 +438,11 @@ generic_pm_domain *scpsys_add_one_domain(struct scpsys *scpsys, struct device_no goto err_unprepare_subsys_clocks; } - pd->genpd.name = node->name; + if (!pd->data->name) + pd->genpd.name = node->name; + else + pd->genpd.name = pd->data->name; + pd->genpd.power_off = scpsys_power_off; pd->genpd.power_on = scpsys_power_on; @@ -487,8 +491,9 @@ static int scpsys_add_subdomain(struct scpsys *scpsys, struct device_node *paren child_pd = scpsys_add_one_domain(scpsys, child); if (IS_ERR(child_pd)) { - dev_err_probe(scpsys->dev, PTR_ERR(child_pd), - "%pOF: failed to get child domain id\n", child); + ret = PTR_ERR(child_pd); + dev_err_probe(scpsys->dev, ret, "%pOF: failed to get child domain id\n", + child); goto err_put_node; } diff --git a/drivers/soc/mediatek/mtk-pm-domains.h b/drivers/soc/mediatek/mtk-pm-domains.h index 141dc76054e6..21a4e113bbec 100644 --- a/drivers/soc/mediatek/mtk-pm-domains.h +++ b/drivers/soc/mediatek/mtk-pm-domains.h @@ -76,6 +76,7 @@ struct scpsys_bus_prot_data { /** * struct scpsys_domain_data - scp domain data for power on/off flow + * @name: The name of the power domain. * @sta_mask: The mask for power on/off status bit. * @ctl_offs: The offset for main power control register. * @sram_pdn_bits: The mask for sram power control bits. @@ -85,6 +86,7 @@ struct scpsys_bus_prot_data { * @bp_smi: bus protection for smi subsystem */ struct scpsys_domain_data { + const char *name; u32 sta_mask; int ctl_offs; u32 sram_pdn_bits; diff --git a/drivers/soc/qcom/mdt_loader.c b/drivers/soc/qcom/mdt_loader.c index 24cd193dec55..eba7f76f9d61 100644 --- a/drivers/soc/qcom/mdt_loader.c +++ b/drivers/soc/qcom/mdt_loader.c @@ -230,6 +230,14 @@ static int __qcom_mdt_load(struct device *dev, const struct firmware *fw, break; } + if (phdr->p_filesz > phdr->p_memsz) { + dev_err(dev, + "refusing to load segment %d with p_filesz > p_memsz\n", + i); + ret = -EINVAL; + break; + } + ptr = mem_region + offset; if (phdr->p_filesz && phdr->p_offset < fw->size) { @@ -253,6 +261,15 @@ static int __qcom_mdt_load(struct device *dev, const struct firmware *fw, break; } + if (seg_fw->size != phdr->p_filesz) { + dev_err(dev, + "failed to load segment %d from truncated file %s\n", + i, fw_name); + release_firmware(seg_fw); + ret = -EINVAL; + break; + } + release_firmware(seg_fw); } diff --git a/drivers/soc/qcom/pdr_interface.c b/drivers/soc/qcom/pdr_interface.c index 209dcdca923f..915d5bc3d46e 100644 --- a/drivers/soc/qcom/pdr_interface.c +++ b/drivers/soc/qcom/pdr_interface.c @@ -153,7 +153,7 @@ static int pdr_register_listener(struct pdr_handle *pdr, if (resp.resp.result != QMI_RESULT_SUCCESS_V01) { pr_err("PDR: %s register listener failed: 0x%x\n", pds->service_path, resp.resp.error); - return ret; + return -EREMOTEIO; } pds->state = resp.curr_state; diff --git a/drivers/soc/tegra/pmc.c b/drivers/soc/tegra/pmc.c index df9a5ca8c99c..0118bd986f90 100644 --- a/drivers/soc/tegra/pmc.c +++ b/drivers/soc/tegra/pmc.c @@ -317,6 +317,8 @@ struct tegra_pmc_soc { bool invert); int (*irq_set_wake)(struct irq_data *data, unsigned int on); int (*irq_set_type)(struct irq_data *data, unsigned int type); + int (*powergate_set)(struct tegra_pmc *pmc, unsigned int id, + bool new_state); const char * const *reset_sources; unsigned int num_reset_sources; @@ -517,6 +519,63 @@ static int tegra_powergate_lookup(struct tegra_pmc *pmc, const char *name) return -ENODEV; } +static int tegra20_powergate_set(struct tegra_pmc *pmc, unsigned int id, + bool new_state) +{ + unsigned int retries = 100; + bool status; + int ret; + + /* + * As per TRM documentation, the toggle command will be dropped by PMC + * if there is contention with a HW-initiated toggling (i.e. CPU core + * power-gated), the command should be retried in that case. + */ + do { + tegra_pmc_writel(pmc, PWRGATE_TOGGLE_START | id, PWRGATE_TOGGLE); + + /* wait for PMC to execute the command */ + ret = readx_poll_timeout(tegra_powergate_state, id, status, + status == new_state, 1, 10); + } while (ret == -ETIMEDOUT && retries--); + + return ret; +} + +static inline bool tegra_powergate_toggle_ready(struct tegra_pmc *pmc) +{ + return !(tegra_pmc_readl(pmc, PWRGATE_TOGGLE) & PWRGATE_TOGGLE_START); +} + +static int tegra114_powergate_set(struct tegra_pmc *pmc, unsigned int id, + bool new_state) +{ + bool status; + int err; + + /* wait while PMC power gating is contended */ + err = readx_poll_timeout(tegra_powergate_toggle_ready, pmc, status, + status == true, 1, 100); + if (err) + return err; + + tegra_pmc_writel(pmc, PWRGATE_TOGGLE_START | id, PWRGATE_TOGGLE); + + /* wait for PMC to accept the command */ + err = readx_poll_timeout(tegra_powergate_toggle_ready, pmc, status, + status == true, 1, 100); + if (err) + return err; + + /* wait for PMC to execute the command */ + err = readx_poll_timeout(tegra_powergate_state, id, status, + status == new_state, 10, 100000); + if (err) + return err; + + return 0; +} + /** * tegra_powergate_set() - set the state of a partition * @pmc: power management controller @@ -526,7 +585,6 @@ static int tegra_powergate_lookup(struct tegra_pmc *pmc, const char *name) static int tegra_powergate_set(struct tegra_pmc *pmc, unsigned int id, bool new_state) { - bool status; int err; if (id == TEGRA_POWERGATE_3D && pmc->soc->has_gpu_clamps) @@ -539,10 +597,7 @@ static int tegra_powergate_set(struct tegra_pmc *pmc, unsigned int id, return 0; } - tegra_pmc_writel(pmc, PWRGATE_TOGGLE_START | id, PWRGATE_TOGGLE); - - err = readx_poll_timeout(tegra_powergate_state, id, status, - status == new_state, 10, 100000); + err = pmc->soc->powergate_set(pmc, id, new_state); mutex_unlock(&pmc->powergates_lock); @@ -2699,6 +2754,7 @@ static const struct tegra_pmc_soc tegra20_pmc_soc = { .regs = &tegra20_pmc_regs, .init = tegra20_pmc_init, .setup_irq_polarity = tegra20_pmc_setup_irq_polarity, + .powergate_set = tegra20_powergate_set, .reset_sources = NULL, .num_reset_sources = 0, .reset_levels = NULL, @@ -2757,6 +2813,7 @@ static const struct tegra_pmc_soc tegra30_pmc_soc = { .regs = &tegra20_pmc_regs, .init = tegra20_pmc_init, .setup_irq_polarity = tegra20_pmc_setup_irq_polarity, + .powergate_set = tegra20_powergate_set, .reset_sources = tegra30_reset_sources, .num_reset_sources = ARRAY_SIZE(tegra30_reset_sources), .reset_levels = NULL, @@ -2811,6 +2868,7 @@ static const struct tegra_pmc_soc tegra114_pmc_soc = { .regs = &tegra20_pmc_regs, .init = tegra20_pmc_init, .setup_irq_polarity = tegra20_pmc_setup_irq_polarity, + .powergate_set = tegra114_powergate_set, .reset_sources = tegra30_reset_sources, .num_reset_sources = ARRAY_SIZE(tegra30_reset_sources), .reset_levels = NULL, @@ -2925,6 +2983,7 @@ static const struct tegra_pmc_soc tegra124_pmc_soc = { .regs = &tegra20_pmc_regs, .init = tegra20_pmc_init, .setup_irq_polarity = tegra20_pmc_setup_irq_polarity, + .powergate_set = tegra114_powergate_set, .reset_sources = tegra30_reset_sources, .num_reset_sources = ARRAY_SIZE(tegra30_reset_sources), .reset_levels = NULL, @@ -3048,6 +3107,7 @@ static const struct tegra_pmc_soc tegra210_pmc_soc = { .regs = &tegra20_pmc_regs, .init = tegra20_pmc_init, .setup_irq_polarity = tegra20_pmc_setup_irq_polarity, + .powergate_set = tegra114_powergate_set, .irq_set_wake = tegra210_pmc_irq_set_wake, .irq_set_type = tegra210_pmc_irq_set_type, .reset_sources = tegra210_reset_sources, diff --git a/drivers/soc/tegra/regulators-tegra30.c b/drivers/soc/tegra/regulators-tegra30.c index 7f21f31de09d..0e776b20f625 100644 --- a/drivers/soc/tegra/regulators-tegra30.c +++ b/drivers/soc/tegra/regulators-tegra30.c @@ -178,7 +178,7 @@ static int tegra30_voltage_update(struct tegra_regulator_coupler *tegra, * survive the voltage drop if it's running on a higher frequency. */ if (!cpu_min_uV_consumers) - cpu_min_uV = cpu_uV; + cpu_min_uV = max(cpu_uV, cpu_min_uV); /* * Bootloader shall set up voltages correctly, but if it diff --git a/drivers/soundwire/bus.c b/drivers/soundwire/bus.c index 46885429928a..4ec29338ce9a 100644 --- a/drivers/soundwire/bus.c +++ b/drivers/soundwire/bus.c @@ -705,7 +705,7 @@ static int sdw_program_device_num(struct sdw_bus *bus) struct sdw_slave *slave, *_s; struct sdw_slave_id id; struct sdw_msg msg; - bool found = false; + bool found; int count = 0, ret; u64 addr; @@ -737,6 +737,7 @@ static int sdw_program_device_num(struct sdw_bus *bus) sdw_extract_slave_id(bus, addr, &id); + found = false; /* Now compare with entries */ list_for_each_entry_safe(slave, _s, &bus->slaves, node) { if (sdw_compare_devid(slave, id) == 0) { diff --git a/drivers/soundwire/cadence_master.c b/drivers/soundwire/cadence_master.c index d05442e646a3..57c59a33ce61 100644 --- a/drivers/soundwire/cadence_master.c +++ b/drivers/soundwire/cadence_master.c @@ -1450,10 +1450,12 @@ int sdw_cdns_clock_stop(struct sdw_cdns *cdns, bool block_wake) } /* Prepare slaves for clock stop */ - ret = sdw_bus_prep_clk_stop(&cdns->bus); - if (ret < 0) { - dev_err(cdns->dev, "prepare clock stop failed %d", ret); - return ret; + if (slave_present) { + ret = sdw_bus_prep_clk_stop(&cdns->bus); + if (ret < 0 && ret != -ENODATA) { + dev_err(cdns->dev, "prepare clock stop failed %d\n", ret); + return ret; + } } /* diff --git a/drivers/soundwire/stream.c b/drivers/soundwire/stream.c index 1099b5d1262b..a418c3c7001c 100644 --- a/drivers/soundwire/stream.c +++ b/drivers/soundwire/stream.c @@ -1375,8 +1375,16 @@ int sdw_stream_add_slave(struct sdw_slave *slave, } ret = sdw_config_stream(&slave->dev, stream, stream_config, true); - if (ret) + if (ret) { + /* + * sdw_release_master_stream will release s_rt in slave_rt_list in + * stream_error case, but s_rt is only added to slave_rt_list + * when sdw_config_stream is successful, so free s_rt explicitly + * when sdw_config_stream is failed. + */ + kfree(s_rt); goto stream_error; + } list_add_tail(&s_rt->m_rt_node, &m_rt->slave_rt_list); diff --git a/drivers/spi/spi-ath79.c b/drivers/spi/spi-ath79.c index eb9a243e9526..98ace748cd98 100644 --- a/drivers/spi/spi-ath79.c +++ b/drivers/spi/spi-ath79.c @@ -156,8 +156,7 @@ static int ath79_spi_probe(struct platform_device *pdev) master->use_gpio_descriptors = true; master->bits_per_word_mask = SPI_BPW_RANGE_MASK(1, 32); - master->setup = spi_bitbang_setup; - master->cleanup = spi_bitbang_cleanup; + master->flags = SPI_MASTER_GPIO_SS; if (pdata) { master->bus_num = pdata->bus_num; master->num_chipselect = pdata->num_chipselect; diff --git a/drivers/spi/spi-dln2.c b/drivers/spi/spi-dln2.c index 75b33d7d14b0..9a4d942fafcf 100644 --- a/drivers/spi/spi-dln2.c +++ b/drivers/spi/spi-dln2.c @@ -780,7 +780,7 @@ static int dln2_spi_probe(struct platform_device *pdev) static int dln2_spi_remove(struct platform_device *pdev) { - struct spi_master *master = spi_master_get(platform_get_drvdata(pdev)); + struct spi_master *master = platform_get_drvdata(pdev); struct dln2_spi *dln2 = spi_master_get_devdata(master); pm_runtime_disable(&pdev->dev); diff --git a/drivers/spi/spi-fsl-lpspi.c b/drivers/spi/spi-fsl-lpspi.c index a2886ee44e4c..5d98611dd999 100644 --- a/drivers/spi/spi-fsl-lpspi.c +++ b/drivers/spi/spi-fsl-lpspi.c @@ -200,7 +200,7 @@ static int lpspi_prepare_xfer_hardware(struct spi_controller *controller) spi_controller_get_devdata(controller); int ret; - ret = pm_runtime_get_sync(fsl_lpspi->dev); + ret = pm_runtime_resume_and_get(fsl_lpspi->dev); if (ret < 0) { dev_err(fsl_lpspi->dev, "failed to enable clock\n"); return ret; diff --git a/drivers/spi/spi-fsl-spi.c b/drivers/spi/spi-fsl-spi.c index e4a8d203f940..d0e5aa18b7ba 100644 --- a/drivers/spi/spi-fsl-spi.c +++ b/drivers/spi/spi-fsl-spi.c @@ -707,6 +707,11 @@ static int of_fsl_spi_probe(struct platform_device *ofdev) struct resource mem; int irq, type; int ret; + bool spisel_boot = false; +#if IS_ENABLED(CONFIG_FSL_SOC) + struct mpc8xxx_spi_probe_info *pinfo = NULL; +#endif + ret = of_mpc8xxx_spi_probe(ofdev); if (ret) @@ -715,9 +720,8 @@ static int of_fsl_spi_probe(struct platform_device *ofdev) type = fsl_spi_get_type(&ofdev->dev); if (type == TYPE_FSL) { struct fsl_spi_platform_data *pdata = dev_get_platdata(dev); - bool spisel_boot = false; #if IS_ENABLED(CONFIG_FSL_SOC) - struct mpc8xxx_spi_probe_info *pinfo = to_of_pinfo(pdata); + pinfo = to_of_pinfo(pdata); spisel_boot = of_property_read_bool(np, "fsl,spisel_boot"); if (spisel_boot) { @@ -746,15 +750,24 @@ static int of_fsl_spi_probe(struct platform_device *ofdev) ret = of_address_to_resource(np, 0, &mem); if (ret) - return ret; + goto unmap_out; irq = platform_get_irq(ofdev, 0); - if (irq < 0) - return irq; + if (irq < 0) { + ret = irq; + goto unmap_out; + } master = fsl_spi_probe(dev, &mem, irq); return PTR_ERR_OR_ZERO(master); + +unmap_out: +#if IS_ENABLED(CONFIG_FSL_SOC) + if (spisel_boot) + iounmap(pinfo->immr_spi_cs); +#endif + return ret; } static int of_fsl_spi_remove(struct platform_device *ofdev) diff --git a/drivers/spi/spi-omap-100k.c b/drivers/spi/spi-omap-100k.c index 36a4922a134a..ccd817ee4917 100644 --- a/drivers/spi/spi-omap-100k.c +++ b/drivers/spi/spi-omap-100k.c @@ -424,7 +424,7 @@ static int omap1_spi100k_probe(struct platform_device *pdev) static int omap1_spi100k_remove(struct platform_device *pdev) { - struct spi_master *master = spi_master_get(platform_get_drvdata(pdev)); + struct spi_master *master = platform_get_drvdata(pdev); struct omap1_spi100k *spi100k = spi_master_get_devdata(master); pm_runtime_disable(&pdev->dev); @@ -438,7 +438,7 @@ static int omap1_spi100k_remove(struct platform_device *pdev) #ifdef CONFIG_PM static int omap1_spi100k_runtime_suspend(struct device *dev) { - struct spi_master *master = spi_master_get(dev_get_drvdata(dev)); + struct spi_master *master = dev_get_drvdata(dev); struct omap1_spi100k *spi100k = spi_master_get_devdata(master); clk_disable_unprepare(spi100k->ick); @@ -449,7 +449,7 @@ static int omap1_spi100k_runtime_suspend(struct device *dev) static int omap1_spi100k_runtime_resume(struct device *dev) { - struct spi_master *master = spi_master_get(dev_get_drvdata(dev)); + struct spi_master *master = dev_get_drvdata(dev); struct omap1_spi100k *spi100k = spi_master_get_devdata(master); int ret; diff --git a/drivers/spi/spi-qup.c b/drivers/spi/spi-qup.c index 8dcb2e70735c..d39dec6d1c91 100644 --- a/drivers/spi/spi-qup.c +++ b/drivers/spi/spi-qup.c @@ -1263,7 +1263,7 @@ static int spi_qup_remove(struct platform_device *pdev) struct spi_qup *controller = spi_master_get_devdata(master); int ret; - ret = pm_runtime_get_sync(&pdev->dev); + ret = pm_runtime_resume_and_get(&pdev->dev); if (ret < 0) return ret; diff --git a/drivers/spi/spi-rockchip.c b/drivers/spi/spi-rockchip.c index 936ef54e0903..0d75080da648 100644 --- a/drivers/spi/spi-rockchip.c +++ b/drivers/spi/spi-rockchip.c @@ -476,7 +476,7 @@ static int rockchip_spi_prepare_dma(struct rockchip_spi *rs, return 1; } -static void rockchip_spi_config(struct rockchip_spi *rs, +static int rockchip_spi_config(struct rockchip_spi *rs, struct spi_device *spi, struct spi_transfer *xfer, bool use_dma, bool slave_mode) { @@ -521,7 +521,9 @@ static void rockchip_spi_config(struct rockchip_spi *rs, * ctlr->bits_per_word_mask, so this shouldn't * happen */ - unreachable(); + dev_err(rs->dev, "unknown bits per word: %d\n", + xfer->bits_per_word); + return -EINVAL; } if (use_dma) { @@ -554,6 +556,8 @@ static void rockchip_spi_config(struct rockchip_spi *rs, */ writel_relaxed(2 * DIV_ROUND_UP(rs->freq, 2 * xfer->speed_hz), rs->regs + ROCKCHIP_SPI_BAUDR); + + return 0; } static size_t rockchip_spi_max_transfer_size(struct spi_device *spi) @@ -577,6 +581,7 @@ static int rockchip_spi_transfer_one( struct spi_transfer *xfer) { struct rockchip_spi *rs = spi_controller_get_devdata(ctlr); + int ret; bool use_dma; WARN_ON(readl_relaxed(rs->regs + ROCKCHIP_SPI_SSIENR) && @@ -596,7 +601,9 @@ static int rockchip_spi_transfer_one( use_dma = ctlr->can_dma ? ctlr->can_dma(ctlr, spi, xfer) : false; - rockchip_spi_config(rs, spi, xfer, use_dma, ctlr->slave); + ret = rockchip_spi_config(rs, spi, xfer, use_dma, ctlr->slave); + if (ret) + return ret; if (use_dma) return rockchip_spi_prepare_dma(rs, ctlr, xfer); diff --git a/drivers/spi/spi-stm32-qspi.c b/drivers/spi/spi-stm32-qspi.c index 947e6b9dc9f4..2786470a5201 100644 --- a/drivers/spi/spi-stm32-qspi.c +++ b/drivers/spi/spi-stm32-qspi.c @@ -727,21 +727,31 @@ static int __maybe_unused stm32_qspi_suspend(struct device *dev) { pinctrl_pm_select_sleep_state(dev); - return 0; + return pm_runtime_force_suspend(dev); } static int __maybe_unused stm32_qspi_resume(struct device *dev) { struct stm32_qspi *qspi = dev_get_drvdata(dev); + int ret; + + ret = pm_runtime_force_resume(dev); + if (ret < 0) + return ret; pinctrl_pm_select_default_state(dev); - clk_prepare_enable(qspi->clk); + + ret = pm_runtime_get_sync(dev); + if (ret < 0) { + pm_runtime_put_noidle(dev); + return ret; + } writel_relaxed(qspi->cr_reg, qspi->io_base + QSPI_CR); writel_relaxed(qspi->dcr_reg, qspi->io_base + QSPI_DCR); - pm_runtime_mark_last_busy(qspi->dev); - pm_runtime_put_autosuspend(qspi->dev); + pm_runtime_mark_last_busy(dev); + pm_runtime_put_autosuspend(dev); return 0; } diff --git a/drivers/spi/spi-stm32.c b/drivers/spi/spi-stm32.c index 25c076461011..7f0244a246e9 100644 --- a/drivers/spi/spi-stm32.c +++ b/drivers/spi/spi-stm32.c @@ -1803,7 +1803,7 @@ static int stm32_spi_probe(struct platform_device *pdev) struct reset_control *rst; int ret; - master = spi_alloc_master(&pdev->dev, sizeof(struct stm32_spi)); + master = devm_spi_alloc_master(&pdev->dev, sizeof(struct stm32_spi)); if (!master) { dev_err(&pdev->dev, "spi master allocation failed\n"); return -ENOMEM; @@ -1821,18 +1821,16 @@ static int stm32_spi_probe(struct platform_device *pdev) res = platform_get_resource(pdev, IORESOURCE_MEM, 0); spi->base = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(spi->base)) { - ret = PTR_ERR(spi->base); - goto err_master_put; - } + if (IS_ERR(spi->base)) + return PTR_ERR(spi->base); spi->phys_addr = (dma_addr_t)res->start; spi->irq = platform_get_irq(pdev, 0); - if (spi->irq <= 0) { - ret = dev_err_probe(&pdev->dev, spi->irq, "failed to get irq\n"); - goto err_master_put; - } + if (spi->irq <= 0) + return dev_err_probe(&pdev->dev, spi->irq, + "failed to get irq\n"); + ret = devm_request_threaded_irq(&pdev->dev, spi->irq, spi->cfg->irq_handler_event, spi->cfg->irq_handler_thread, @@ -1840,20 +1838,20 @@ static int stm32_spi_probe(struct platform_device *pdev) if (ret) { dev_err(&pdev->dev, "irq%d request failed: %d\n", spi->irq, ret); - goto err_master_put; + return ret; } spi->clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(spi->clk)) { ret = PTR_ERR(spi->clk); dev_err(&pdev->dev, "clk get failed: %d\n", ret); - goto err_master_put; + return ret; } ret = clk_prepare_enable(spi->clk); if (ret) { dev_err(&pdev->dev, "clk enable failed: %d\n", ret); - goto err_master_put; + return ret; } spi->clk_rate = clk_get_rate(spi->clk); if (!spi->clk_rate) { @@ -1929,7 +1927,7 @@ static int stm32_spi_probe(struct platform_device *pdev) pm_runtime_set_active(&pdev->dev); pm_runtime_enable(&pdev->dev); - ret = devm_spi_register_master(&pdev->dev, master); + ret = spi_register_master(master); if (ret) { dev_err(&pdev->dev, "spi master registration failed: %d\n", ret); @@ -1949,8 +1947,6 @@ static int stm32_spi_probe(struct platform_device *pdev) dma_release_channel(spi->dma_rx); err_clk_disable: clk_disable_unprepare(spi->clk); -err_master_put: - spi_master_put(master); return ret; } @@ -1960,6 +1956,7 @@ static int stm32_spi_remove(struct platform_device *pdev) struct spi_master *master = platform_get_drvdata(pdev); struct stm32_spi *spi = spi_master_get_devdata(master); + spi_unregister_master(master); spi->cfg->disable(spi); if (master->dma_tx) diff --git a/drivers/spi/spi-ti-qspi.c b/drivers/spi/spi-ti-qspi.c index 9417385c0921..e06aafe169e0 100644 --- a/drivers/spi/spi-ti-qspi.c +++ b/drivers/spi/spi-ti-qspi.c @@ -733,6 +733,17 @@ static int ti_qspi_runtime_resume(struct device *dev) return 0; } +static void ti_qspi_dma_cleanup(struct ti_qspi *qspi) +{ + if (qspi->rx_bb_addr) + dma_free_coherent(qspi->dev, QSPI_DMA_BUFFER_SIZE, + qspi->rx_bb_addr, + qspi->rx_bb_dma_addr); + + if (qspi->rx_chan) + dma_release_channel(qspi->rx_chan); +} + static const struct of_device_id ti_qspi_match[] = { {.compatible = "ti,dra7xxx-qspi" }, {.compatible = "ti,am4372-qspi" }, @@ -886,6 +897,8 @@ static int ti_qspi_probe(struct platform_device *pdev) if (!ret) return 0; + ti_qspi_dma_cleanup(qspi); + pm_runtime_disable(&pdev->dev); free_master: spi_master_put(master); @@ -904,12 +917,7 @@ static int ti_qspi_remove(struct platform_device *pdev) pm_runtime_put_sync(&pdev->dev); pm_runtime_disable(&pdev->dev); - if (qspi->rx_bb_addr) - dma_free_coherent(qspi->dev, QSPI_DMA_BUFFER_SIZE, - qspi->rx_bb_addr, - qspi->rx_bb_dma_addr); - if (qspi->rx_chan) - dma_release_channel(qspi->rx_chan); + ti_qspi_dma_cleanup(qspi); return 0; } diff --git a/drivers/spi/spi-zynqmp-gqspi.c b/drivers/spi/spi-zynqmp-gqspi.c index c8fa6ee18ae7..7162387b9f96 100644 --- a/drivers/spi/spi-zynqmp-gqspi.c +++ b/drivers/spi/spi-zynqmp-gqspi.c @@ -157,6 +157,7 @@ enum mode_type {GQSPI_MODE_IO, GQSPI_MODE_DMA}; * @data_completion: completion structure */ struct zynqmp_qspi { + struct spi_controller *ctlr; void __iomem *regs; struct clk *refclk; struct clk *pclk; @@ -173,6 +174,7 @@ struct zynqmp_qspi { u32 genfifoentry; enum mode_type mode; struct completion data_completion; + struct mutex op_lock; }; /** @@ -486,24 +488,10 @@ static int zynqmp_qspi_setup_op(struct spi_device *qspi) { struct spi_controller *ctlr = qspi->master; struct zynqmp_qspi *xqspi = spi_controller_get_devdata(ctlr); - struct device *dev = &ctlr->dev; - int ret; if (ctlr->busy) return -EBUSY; - ret = clk_enable(xqspi->refclk); - if (ret) { - dev_err(dev, "Cannot enable device clock.\n"); - return ret; - } - - ret = clk_enable(xqspi->pclk); - if (ret) { - dev_err(dev, "Cannot enable APB clock.\n"); - clk_disable(xqspi->refclk); - return ret; - } zynqmp_gqspi_write(xqspi, GQSPI_EN_OFST, GQSPI_EN_MASK); return 0; @@ -520,7 +508,7 @@ static void zynqmp_qspi_filltxfifo(struct zynqmp_qspi *xqspi, int size) { u32 count = 0, intermediate; - while ((xqspi->bytes_to_transfer > 0) && (count < size)) { + while ((xqspi->bytes_to_transfer > 0) && (count < size) && (xqspi->txbuf)) { memcpy(&intermediate, xqspi->txbuf, 4); zynqmp_gqspi_write(xqspi, GQSPI_TXD_OFST, intermediate); @@ -579,7 +567,7 @@ static void zynqmp_qspi_fillgenfifo(struct zynqmp_qspi *xqspi, u8 nbits, genfifoentry |= GQSPI_GENFIFO_DATA_XFER; genfifoentry |= GQSPI_GENFIFO_TX; transfer_len = xqspi->bytes_to_transfer; - } else { + } else if (xqspi->rxbuf) { genfifoentry &= ~GQSPI_GENFIFO_TX; genfifoentry |= GQSPI_GENFIFO_DATA_XFER; genfifoentry |= GQSPI_GENFIFO_RX; @@ -587,6 +575,11 @@ static void zynqmp_qspi_fillgenfifo(struct zynqmp_qspi *xqspi, u8 nbits, transfer_len = xqspi->dma_rx_bytes; else transfer_len = xqspi->bytes_to_receive; + } else { + /* Sending dummy circles here */ + genfifoentry &= ~(GQSPI_GENFIFO_TX | GQSPI_GENFIFO_RX); + genfifoentry |= GQSPI_GENFIFO_DATA_XFER; + transfer_len = xqspi->bytes_to_transfer; } genfifoentry |= zynqmp_qspi_selectspimode(xqspi, nbits); xqspi->genfifoentry = genfifoentry; @@ -738,7 +731,7 @@ static irqreturn_t zynqmp_qspi_irq(int irq, void *dev_id) * zynqmp_qspi_setuprxdma - This function sets up the RX DMA operation * @xqspi: xqspi is a pointer to the GQSPI instance. */ -static void zynqmp_qspi_setuprxdma(struct zynqmp_qspi *xqspi) +static int zynqmp_qspi_setuprxdma(struct zynqmp_qspi *xqspi) { u32 rx_bytes, rx_rem, config_reg; dma_addr_t addr; @@ -752,7 +745,7 @@ static void zynqmp_qspi_setuprxdma(struct zynqmp_qspi *xqspi) zynqmp_gqspi_write(xqspi, GQSPI_CONFIG_OFST, config_reg); xqspi->mode = GQSPI_MODE_IO; xqspi->dma_rx_bytes = 0; - return; + return 0; } rx_rem = xqspi->bytes_to_receive % 4; @@ -760,8 +753,10 @@ static void zynqmp_qspi_setuprxdma(struct zynqmp_qspi *xqspi) addr = dma_map_single(xqspi->dev, (void *)xqspi->rxbuf, rx_bytes, DMA_FROM_DEVICE); - if (dma_mapping_error(xqspi->dev, addr)) + if (dma_mapping_error(xqspi->dev, addr)) { dev_err(xqspi->dev, "ERR:rxdma:memory not mapped\n"); + return -ENOMEM; + } xqspi->dma_rx_bytes = rx_bytes; xqspi->dma_addr = addr; @@ -782,6 +777,8 @@ static void zynqmp_qspi_setuprxdma(struct zynqmp_qspi *xqspi) /* Write the number of bytes to transfer */ zynqmp_gqspi_write(xqspi, GQSPI_QSPIDMA_DST_SIZE_OFST, rx_bytes); + + return 0; } /** @@ -818,11 +815,17 @@ static void zynqmp_qspi_write_op(struct zynqmp_qspi *xqspi, u8 tx_nbits, * @genfifoentry: genfifoentry is pointer to the variable in which * GENFIFO mask is returned to calling function */ -static void zynqmp_qspi_read_op(struct zynqmp_qspi *xqspi, u8 rx_nbits, +static int zynqmp_qspi_read_op(struct zynqmp_qspi *xqspi, u8 rx_nbits, u32 genfifoentry) { + int ret; + + ret = zynqmp_qspi_setuprxdma(xqspi); + if (ret) + return ret; zynqmp_qspi_fillgenfifo(xqspi, rx_nbits, genfifoentry); - zynqmp_qspi_setuprxdma(xqspi); + + return 0; } /** @@ -835,10 +838,13 @@ static void zynqmp_qspi_read_op(struct zynqmp_qspi *xqspi, u8 rx_nbits, */ static int __maybe_unused zynqmp_qspi_suspend(struct device *dev) { - struct spi_controller *ctlr = dev_get_drvdata(dev); - struct zynqmp_qspi *xqspi = spi_controller_get_devdata(ctlr); + struct zynqmp_qspi *xqspi = dev_get_drvdata(dev); + struct spi_controller *ctlr = xqspi->ctlr; + int ret; - spi_controller_suspend(ctlr); + ret = spi_controller_suspend(ctlr); + if (ret) + return ret; zynqmp_gqspi_write(xqspi, GQSPI_EN_OFST, 0x0); @@ -856,27 +862,13 @@ static int __maybe_unused zynqmp_qspi_suspend(struct device *dev) */ static int __maybe_unused zynqmp_qspi_resume(struct device *dev) { - struct spi_controller *ctlr = dev_get_drvdata(dev); - struct zynqmp_qspi *xqspi = spi_controller_get_devdata(ctlr); - int ret = 0; - - ret = clk_enable(xqspi->pclk); - if (ret) { - dev_err(dev, "Cannot enable APB clock.\n"); - return ret; - } + struct zynqmp_qspi *xqspi = dev_get_drvdata(dev); + struct spi_controller *ctlr = xqspi->ctlr; - ret = clk_enable(xqspi->refclk); - if (ret) { - dev_err(dev, "Cannot enable device clock.\n"); - clk_disable(xqspi->pclk); - return ret; - } + zynqmp_gqspi_write(xqspi, GQSPI_EN_OFST, GQSPI_EN_MASK); spi_controller_resume(ctlr); - clk_disable(xqspi->refclk); - clk_disable(xqspi->pclk); return 0; } @@ -890,10 +882,10 @@ static int __maybe_unused zynqmp_qspi_resume(struct device *dev) */ static int __maybe_unused zynqmp_runtime_suspend(struct device *dev) { - struct zynqmp_qspi *xqspi = (struct zynqmp_qspi *)dev_get_drvdata(dev); + struct zynqmp_qspi *xqspi = dev_get_drvdata(dev); - clk_disable(xqspi->refclk); - clk_disable(xqspi->pclk); + clk_disable_unprepare(xqspi->refclk); + clk_disable_unprepare(xqspi->pclk); return 0; } @@ -908,19 +900,19 @@ static int __maybe_unused zynqmp_runtime_suspend(struct device *dev) */ static int __maybe_unused zynqmp_runtime_resume(struct device *dev) { - struct zynqmp_qspi *xqspi = (struct zynqmp_qspi *)dev_get_drvdata(dev); + struct zynqmp_qspi *xqspi = dev_get_drvdata(dev); int ret; - ret = clk_enable(xqspi->pclk); + ret = clk_prepare_enable(xqspi->pclk); if (ret) { dev_err(dev, "Cannot enable APB clock.\n"); return ret; } - ret = clk_enable(xqspi->refclk); + ret = clk_prepare_enable(xqspi->refclk); if (ret) { dev_err(dev, "Cannot enable device clock.\n"); - clk_disable(xqspi->pclk); + clk_disable_unprepare(xqspi->pclk); return ret; } @@ -944,25 +936,23 @@ static int zynqmp_qspi_exec_op(struct spi_mem *mem, struct zynqmp_qspi *xqspi = spi_controller_get_devdata (mem->spi->master); int err = 0, i; - u8 *tmpbuf; u32 genfifoentry = 0; + u16 opcode = op->cmd.opcode; + u64 opaddr; dev_dbg(xqspi->dev, "cmd:%#x mode:%d.%d.%d.%d\n", op->cmd.opcode, op->cmd.buswidth, op->addr.buswidth, op->dummy.buswidth, op->data.buswidth); + mutex_lock(&xqspi->op_lock); zynqmp_qspi_config_op(xqspi, mem->spi); zynqmp_qspi_chipselect(mem->spi, false); genfifoentry |= xqspi->genfifocs; genfifoentry |= xqspi->genfifobus; if (op->cmd.opcode) { - tmpbuf = kzalloc(op->cmd.nbytes, GFP_KERNEL | GFP_DMA); - if (!tmpbuf) - return -ENOMEM; - tmpbuf[0] = op->cmd.opcode; reinit_completion(&xqspi->data_completion); - xqspi->txbuf = tmpbuf; + xqspi->txbuf = &opcode; xqspi->rxbuf = NULL; xqspi->bytes_to_transfer = op->cmd.nbytes; xqspi->bytes_to_receive = 0; @@ -973,16 +963,15 @@ static int zynqmp_qspi_exec_op(struct spi_mem *mem, zynqmp_gqspi_write(xqspi, GQSPI_IER_OFST, GQSPI_IER_GENFIFOEMPTY_MASK | GQSPI_IER_TXNOT_FULL_MASK); - if (!wait_for_completion_interruptible_timeout + if (!wait_for_completion_timeout (&xqspi->data_completion, msecs_to_jiffies(1000))) { err = -ETIMEDOUT; - kfree(tmpbuf); goto return_err; } - kfree(tmpbuf); } if (op->addr.nbytes) { + xqspi->txbuf = &opaddr; for (i = 0; i < op->addr.nbytes; i++) { *(((u8 *)xqspi->txbuf) + i) = op->addr.val >> (8 * (op->addr.nbytes - i - 1)); @@ -1001,7 +990,7 @@ static int zynqmp_qspi_exec_op(struct spi_mem *mem, GQSPI_IER_TXEMPTY_MASK | GQSPI_IER_GENFIFOEMPTY_MASK | GQSPI_IER_TXNOT_FULL_MASK); - if (!wait_for_completion_interruptible_timeout + if (!wait_for_completion_timeout (&xqspi->data_completion, msecs_to_jiffies(1000))) { err = -ETIMEDOUT; goto return_err; @@ -1009,32 +998,23 @@ static int zynqmp_qspi_exec_op(struct spi_mem *mem, } if (op->dummy.nbytes) { - tmpbuf = kzalloc(op->dummy.nbytes, GFP_KERNEL | GFP_DMA); - if (!tmpbuf) - return -ENOMEM; - memset(tmpbuf, 0xff, op->dummy.nbytes); - reinit_completion(&xqspi->data_completion); - xqspi->txbuf = tmpbuf; + xqspi->txbuf = NULL; xqspi->rxbuf = NULL; - xqspi->bytes_to_transfer = op->dummy.nbytes; + /* + * xqspi->bytes_to_transfer here represents the dummy circles + * which need to be sent. + */ + xqspi->bytes_to_transfer = op->dummy.nbytes * 8 / op->dummy.buswidth; xqspi->bytes_to_receive = 0; - zynqmp_qspi_write_op(xqspi, op->dummy.buswidth, + /* + * Using op->data.buswidth instead of op->dummy.buswidth here because + * we need to use it to configure the correct SPI mode. + */ + zynqmp_qspi_write_op(xqspi, op->data.buswidth, genfifoentry); zynqmp_gqspi_write(xqspi, GQSPI_CONFIG_OFST, zynqmp_gqspi_read(xqspi, GQSPI_CONFIG_OFST) | GQSPI_CFG_START_GEN_FIFO_MASK); - zynqmp_gqspi_write(xqspi, GQSPI_IER_OFST, - GQSPI_IER_TXEMPTY_MASK | - GQSPI_IER_GENFIFOEMPTY_MASK | - GQSPI_IER_TXNOT_FULL_MASK); - if (!wait_for_completion_interruptible_timeout - (&xqspi->data_completion, msecs_to_jiffies(1000))) { - err = -ETIMEDOUT; - kfree(tmpbuf); - goto return_err; - } - - kfree(tmpbuf); } if (op->data.nbytes) { @@ -1059,8 +1039,11 @@ static int zynqmp_qspi_exec_op(struct spi_mem *mem, xqspi->rxbuf = (u8 *)op->data.buf.in; xqspi->bytes_to_receive = op->data.nbytes; xqspi->bytes_to_transfer = 0; - zynqmp_qspi_read_op(xqspi, op->data.buswidth, + err = zynqmp_qspi_read_op(xqspi, op->data.buswidth, genfifoentry); + if (err) + goto return_err; + zynqmp_gqspi_write(xqspi, GQSPI_CONFIG_OFST, zynqmp_gqspi_read (xqspi, GQSPI_CONFIG_OFST) | @@ -1076,7 +1059,7 @@ static int zynqmp_qspi_exec_op(struct spi_mem *mem, GQSPI_IER_RXEMPTY_MASK); } } - if (!wait_for_completion_interruptible_timeout + if (!wait_for_completion_timeout (&xqspi->data_completion, msecs_to_jiffies(1000))) err = -ETIMEDOUT; } @@ -1084,6 +1067,7 @@ static int zynqmp_qspi_exec_op(struct spi_mem *mem, return_err: zynqmp_qspi_chipselect(mem->spi, true); + mutex_unlock(&xqspi->op_lock); return err; } @@ -1120,6 +1104,7 @@ static int zynqmp_qspi_probe(struct platform_device *pdev) xqspi = spi_controller_get_devdata(ctlr); xqspi->dev = dev; + xqspi->ctlr = ctlr; platform_set_drvdata(pdev, xqspi); xqspi->regs = devm_platform_ioremap_resource(pdev, 0); @@ -1135,13 +1120,11 @@ static int zynqmp_qspi_probe(struct platform_device *pdev) goto remove_master; } - init_completion(&xqspi->data_completion); - xqspi->refclk = devm_clk_get(&pdev->dev, "ref_clk"); if (IS_ERR(xqspi->refclk)) { dev_err(dev, "ref_clk clock not found.\n"); ret = PTR_ERR(xqspi->refclk); - goto clk_dis_pclk; + goto remove_master; } ret = clk_prepare_enable(xqspi->pclk); @@ -1156,15 +1139,24 @@ static int zynqmp_qspi_probe(struct platform_device *pdev) goto clk_dis_pclk; } + init_completion(&xqspi->data_completion); + + mutex_init(&xqspi->op_lock); + pm_runtime_use_autosuspend(&pdev->dev); pm_runtime_set_autosuspend_delay(&pdev->dev, SPI_AUTOSUSPEND_TIMEOUT); pm_runtime_set_active(&pdev->dev); pm_runtime_enable(&pdev->dev); + + ret = pm_runtime_get_sync(&pdev->dev); + if (ret < 0) { + dev_err(&pdev->dev, "Failed to pm_runtime_get_sync: %d\n", ret); + goto clk_dis_all; + } + /* QSPI controller initializations */ zynqmp_qspi_init_hw(xqspi); - pm_runtime_mark_last_busy(&pdev->dev); - pm_runtime_put_autosuspend(&pdev->dev); xqspi->irq = platform_get_irq(pdev, 0); if (xqspi->irq <= 0) { ret = -ENXIO; @@ -1178,6 +1170,7 @@ static int zynqmp_qspi_probe(struct platform_device *pdev) goto clk_dis_all; } + dma_set_mask(&pdev->dev, DMA_BIT_MASK(44)); ctlr->bits_per_word_mask = SPI_BPW_MASK(8); ctlr->num_chipselect = GQSPI_DEFAULT_NUM_CS; ctlr->mem_ops = &zynqmp_qspi_mem_ops; @@ -1187,6 +1180,7 @@ static int zynqmp_qspi_probe(struct platform_device *pdev) ctlr->mode_bits = SPI_CPOL | SPI_CPHA | SPI_RX_DUAL | SPI_RX_QUAD | SPI_TX_DUAL | SPI_TX_QUAD; ctlr->dev.of_node = np; + ctlr->auto_runtime_pm = true; ret = devm_spi_register_controller(&pdev->dev, ctlr); if (ret) { @@ -1194,9 +1188,13 @@ static int zynqmp_qspi_probe(struct platform_device *pdev) goto clk_dis_all; } + pm_runtime_mark_last_busy(&pdev->dev); + pm_runtime_put_autosuspend(&pdev->dev); + return 0; clk_dis_all: + pm_runtime_put_sync(&pdev->dev); pm_runtime_set_suspended(&pdev->dev); pm_runtime_disable(&pdev->dev); clk_disable_unprepare(xqspi->refclk); diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index b08efe88ccd6..8da4fe475b84 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -795,7 +795,7 @@ int spi_register_board_info(struct spi_board_info const *info, unsigned n) /*-------------------------------------------------------------------------*/ -static void spi_set_cs(struct spi_device *spi, bool enable) +static void spi_set_cs(struct spi_device *spi, bool enable, bool force) { bool enable1 = enable; @@ -803,7 +803,7 @@ static void spi_set_cs(struct spi_device *spi, bool enable) * Avoid calling into the driver (or doing delays) if the chip select * isn't actually changing from the last time this was called. */ - if ((spi->controller->last_cs_enable == enable) && + if (!force && (spi->controller->last_cs_enable == enable) && (spi->controller->last_cs_mode_high == (spi->mode & SPI_CS_HIGH))) return; @@ -1253,7 +1253,7 @@ static int spi_transfer_one_message(struct spi_controller *ctlr, struct spi_statistics *statm = &ctlr->statistics; struct spi_statistics *stats = &msg->spi->statistics; - spi_set_cs(msg->spi, true); + spi_set_cs(msg->spi, true, false); SPI_STATISTICS_INCREMENT_FIELD(statm, messages); SPI_STATISTICS_INCREMENT_FIELD(stats, messages); @@ -1321,9 +1321,9 @@ static int spi_transfer_one_message(struct spi_controller *ctlr, &msg->transfers)) { keep_cs = true; } else { - spi_set_cs(msg->spi, false); + spi_set_cs(msg->spi, false, false); _spi_transfer_cs_change_delay(msg, xfer); - spi_set_cs(msg->spi, true); + spi_set_cs(msg->spi, true, false); } } @@ -1332,7 +1332,7 @@ static int spi_transfer_one_message(struct spi_controller *ctlr, out: if (ret != 0 || !keep_cs) - spi_set_cs(msg->spi, false); + spi_set_cs(msg->spi, false, false); if (msg->status == -EINPROGRESS) msg->status = ret; @@ -2496,6 +2496,7 @@ struct spi_controller *__devm_spi_alloc_controller(struct device *dev, ctlr = __spi_alloc_controller(dev, size, slave); if (ctlr) { + ctlr->devm_allocated = true; *ptr = ctlr; devres_add(dev, ptr); } else { @@ -2842,11 +2843,6 @@ int devm_spi_register_controller(struct device *dev, } EXPORT_SYMBOL_GPL(devm_spi_register_controller); -static int devm_spi_match_controller(struct device *dev, void *res, void *ctlr) -{ - return *(struct spi_controller **)res == ctlr; -} - static int __unregister(struct device *dev, void *null) { spi_unregister_device(to_spi_device(dev)); @@ -2893,8 +2889,7 @@ void spi_unregister_controller(struct spi_controller *ctlr) /* Release the last reference on the controller if its driver * has not yet been converted to devm_spi_alloc_master/slave(). */ - if (!devres_find(ctlr->dev.parent, devm_spi_release_controller, - devm_spi_match_controller, ctlr)) + if (!ctlr->devm_allocated) put_device(&ctlr->dev); /* free bus id */ @@ -3423,11 +3418,11 @@ int spi_setup(struct spi_device *spi) */ status = 0; - spi_set_cs(spi, false); + spi_set_cs(spi, false, true); pm_runtime_mark_last_busy(spi->controller->dev.parent); pm_runtime_put_autosuspend(spi->controller->dev.parent); } else { - spi_set_cs(spi, false); + spi_set_cs(spi, false, true); } mutex_unlock(&spi->controller->io_mutex); diff --git a/drivers/staging/comedi/drivers/tests/ni_routes_test.c b/drivers/staging/comedi/drivers/tests/ni_routes_test.c index 4061b3b5f8e9..68defeb53de4 100644 --- a/drivers/staging/comedi/drivers/tests/ni_routes_test.c +++ b/drivers/staging/comedi/drivers/tests/ni_routes_test.c @@ -217,7 +217,8 @@ void test_ni_assign_device_routes(void) const u8 *table, *oldtable; init_pci_6070e(); - ni_assign_device_routes(ni_eseries, pci_6070e, &private.routing_tables); + ni_assign_device_routes(ni_eseries, pci_6070e, NULL, + &private.routing_tables); devroutes = private.routing_tables.valid_routes; table = private.routing_tables.route_values; @@ -253,7 +254,8 @@ void test_ni_assign_device_routes(void) olddevroutes = devroutes; oldtable = table; init_pci_6220(); - ni_assign_device_routes(ni_mseries, pci_6220, &private.routing_tables); + ni_assign_device_routes(ni_mseries, pci_6220, NULL, + &private.routing_tables); devroutes = private.routing_tables.valid_routes; table = private.routing_tables.route_values; diff --git a/drivers/staging/fwserial/fwserial.c b/drivers/staging/fwserial/fwserial.c index c368082aae1a..0f4655d7d520 100644 --- a/drivers/staging/fwserial/fwserial.c +++ b/drivers/staging/fwserial/fwserial.c @@ -1218,13 +1218,12 @@ static int get_serial_info(struct tty_struct *tty, struct fwtty_port *port = tty->driver_data; mutex_lock(&port->port.mutex); - ss->type = PORT_UNKNOWN; - ss->line = port->port.tty->index; - ss->flags = port->port.flags; - ss->xmit_fifo_size = FWTTY_PORT_TXFIFO_LEN; + ss->line = port->index; ss->baud_base = 400000000; - ss->close_delay = port->port.close_delay; + ss->close_delay = jiffies_to_msecs(port->port.close_delay) / 10; + ss->closing_wait = 3000; mutex_unlock(&port->port.mutex); + return 0; } @@ -1232,20 +1231,20 @@ static int set_serial_info(struct tty_struct *tty, struct serial_struct *ss) { struct fwtty_port *port = tty->driver_data; + unsigned int cdelay; - if (ss->irq != 0 || ss->port != 0 || ss->custom_divisor != 0 || - ss->baud_base != 400000000) - return -EPERM; + cdelay = msecs_to_jiffies(ss->close_delay * 10); mutex_lock(&port->port.mutex); if (!capable(CAP_SYS_ADMIN)) { - if (((ss->flags & ~ASYNC_USR_MASK) != + if (cdelay != port->port.close_delay || + ((ss->flags & ~ASYNC_USR_MASK) != (port->port.flags & ~ASYNC_USR_MASK))) { mutex_unlock(&port->port.mutex); return -EPERM; } } - port->port.close_delay = ss->close_delay * HZ / 100; + port->port.close_delay = cdelay; mutex_unlock(&port->port.mutex); return 0; diff --git a/drivers/staging/greybus/uart.c b/drivers/staging/greybus/uart.c index 607378bfebb7..a520f7f213db 100644 --- a/drivers/staging/greybus/uart.c +++ b/drivers/staging/greybus/uart.c @@ -614,10 +614,12 @@ static int get_serial_info(struct tty_struct *tty, ss->line = gb_tty->minor; ss->xmit_fifo_size = 16; ss->baud_base = 9600; - ss->close_delay = gb_tty->port.close_delay / 10; + ss->close_delay = jiffies_to_msecs(gb_tty->port.close_delay) / 10; ss->closing_wait = gb_tty->port.closing_wait == ASYNC_CLOSING_WAIT_NONE ? - ASYNC_CLOSING_WAIT_NONE : gb_tty->port.closing_wait / 10; + ASYNC_CLOSING_WAIT_NONE : + jiffies_to_msecs(gb_tty->port.closing_wait) / 10; + return 0; } @@ -629,17 +631,16 @@ static int set_serial_info(struct tty_struct *tty, unsigned int close_delay; int retval = 0; - close_delay = ss->close_delay * 10; + close_delay = msecs_to_jiffies(ss->close_delay * 10); closing_wait = ss->closing_wait == ASYNC_CLOSING_WAIT_NONE ? - ASYNC_CLOSING_WAIT_NONE : ss->closing_wait * 10; + ASYNC_CLOSING_WAIT_NONE : + msecs_to_jiffies(ss->closing_wait * 10); mutex_lock(&gb_tty->port.mutex); if (!capable(CAP_SYS_ADMIN)) { if ((close_delay != gb_tty->port.close_delay) || (closing_wait != gb_tty->port.closing_wait)) retval = -EPERM; - else - retval = -EOPNOTSUPP; } else { gb_tty->port.close_delay = close_delay; gb_tty->port.closing_wait = closing_wait; diff --git a/drivers/staging/media/atomisp/i2c/atomisp-lm3554.c b/drivers/staging/media/atomisp/i2c/atomisp-lm3554.c index 7ca7378b1859..0ab67b2aec67 100644 --- a/drivers/staging/media/atomisp/i2c/atomisp-lm3554.c +++ b/drivers/staging/media/atomisp/i2c/atomisp-lm3554.c @@ -843,8 +843,10 @@ static int lm3554_probe(struct i2c_client *client) return -ENOMEM; flash->pdata = lm3554_platform_data_func(client); - if (IS_ERR(flash->pdata)) - return PTR_ERR(flash->pdata); + if (IS_ERR(flash->pdata)) { + err = PTR_ERR(flash->pdata); + goto fail1; + } v4l2_i2c_subdev_init(&flash->sd, client, &lm3554_ops); flash->sd.internal_ops = &lm3554_internal_ops; @@ -856,7 +858,7 @@ static int lm3554_probe(struct i2c_client *client) ARRAY_SIZE(lm3554_controls)); if (ret) { dev_err(&client->dev, "error initialize a ctrl_handler.\n"); - goto fail2; + goto fail3; } for (i = 0; i < ARRAY_SIZE(lm3554_controls); i++) @@ -865,14 +867,14 @@ static int lm3554_probe(struct i2c_client *client) if (flash->ctrl_handler.error) { dev_err(&client->dev, "ctrl_handler error.\n"); - goto fail2; + goto fail3; } flash->sd.ctrl_handler = &flash->ctrl_handler; err = media_entity_pads_init(&flash->sd.entity, 0, NULL); if (err) { dev_err(&client->dev, "error initialize a media entity.\n"); - goto fail1; + goto fail2; } flash->sd.entity.function = MEDIA_ENT_F_FLASH; @@ -884,14 +886,15 @@ static int lm3554_probe(struct i2c_client *client) err = lm3554_gpio_init(client); if (err) { dev_err(&client->dev, "gpio request/direction_output fail"); - goto fail2; + goto fail3; } return atomisp_register_i2c_module(&flash->sd, NULL, LED_FLASH); -fail2: +fail3: media_entity_cleanup(&flash->sd.entity); v4l2_ctrl_handler_free(&flash->ctrl_handler); -fail1: +fail2: v4l2_device_unregister_subdev(&flash->sd); +fail1: kfree(flash); return err; diff --git a/drivers/staging/media/atomisp/pci/atomisp_fops.c b/drivers/staging/media/atomisp/pci/atomisp_fops.c index 453bb6913550..f1e6b2597853 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_fops.c +++ b/drivers/staging/media/atomisp/pci/atomisp_fops.c @@ -221,6 +221,9 @@ int atomisp_q_video_buffers_to_css(struct atomisp_sub_device *asd, unsigned long irqflags; int err = 0; + if (WARN_ON(css_pipe_id >= IA_CSS_PIPE_ID_NUM)) + return -EINVAL; + while (pipe->buffers_in_css < ATOMISP_CSS_Q_DEPTH) { struct videobuf_buffer *vb; diff --git a/drivers/staging/media/atomisp/pci/atomisp_ioctl.c b/drivers/staging/media/atomisp/pci/atomisp_ioctl.c index 2ae50decfc8b..9da82855552d 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_ioctl.c +++ b/drivers/staging/media/atomisp/pci/atomisp_ioctl.c @@ -948,10 +948,8 @@ int atomisp_alloc_css_stat_bufs(struct atomisp_sub_device *asd, dev_dbg(isp->dev, "allocating %d dis buffers\n", count); while (count--) { dis_buf = kzalloc(sizeof(struct atomisp_dis_buf), GFP_KERNEL); - if (!dis_buf) { - kfree(s3a_buf); + if (!dis_buf) goto error; - } if (atomisp_css_allocate_stat_buffers( asd, stream_id, NULL, dis_buf, NULL)) { kfree(dis_buf); diff --git a/drivers/staging/media/atomisp/pci/hmm/hmm_bo.c b/drivers/staging/media/atomisp/pci/hmm/hmm_bo.c index f13af2329f48..0168f9839c90 100644 --- a/drivers/staging/media/atomisp/pci/hmm/hmm_bo.c +++ b/drivers/staging/media/atomisp/pci/hmm/hmm_bo.c @@ -857,16 +857,17 @@ static void free_private_pages(struct hmm_buffer_object *bo, kfree(bo->page_obj); } -static void free_user_pages(struct hmm_buffer_object *bo) +static void free_user_pages(struct hmm_buffer_object *bo, + unsigned int page_nr) { int i; hmm_mem_stat.usr_size -= bo->pgnr; if (bo->mem_type == HMM_BO_MEM_TYPE_PFN) { - unpin_user_pages(bo->pages, bo->pgnr); + unpin_user_pages(bo->pages, page_nr); } else { - for (i = 0; i < bo->pgnr; i++) + for (i = 0; i < page_nr; i++) put_page(bo->pages[i]); } kfree(bo->pages); @@ -942,6 +943,8 @@ static int alloc_user_pages(struct hmm_buffer_object *bo, dev_err(atomisp_dev, "get_user_pages err: bo->pgnr = %d, pgnr actually pinned = %d.\n", bo->pgnr, page_nr); + if (page_nr < 0) + page_nr = 0; goto out_of_mem; } @@ -954,7 +957,7 @@ static int alloc_user_pages(struct hmm_buffer_object *bo, out_of_mem: - free_user_pages(bo); + free_user_pages(bo, page_nr); return -ENOMEM; } @@ -1037,7 +1040,7 @@ void hmm_bo_free_pages(struct hmm_buffer_object *bo) if (bo->type == HMM_BO_PRIVATE) free_private_pages(bo, &dynamic_pool, &reserved_pool); else if (bo->type == HMM_BO_USER) - free_user_pages(bo); + free_user_pages(bo, bo->pgnr); else dev_err(atomisp_dev, "invalid buffer type.\n"); mutex_unlock(&bo->mutex); diff --git a/drivers/staging/media/imx/imx-media-capture.c b/drivers/staging/media/imx/imx-media-capture.c index e10ce103a5b4..94a0467d673b 100644 --- a/drivers/staging/media/imx/imx-media-capture.c +++ b/drivers/staging/media/imx/imx-media-capture.c @@ -557,7 +557,7 @@ static int capture_validate_fmt(struct capture_priv *priv) priv->vdev.fmt.fmt.pix.height != f.fmt.pix.height || priv->vdev.cc->cs != cc->cs || priv->vdev.compose.width != compose.width || - priv->vdev.compose.height != compose.height) ? -EINVAL : 0; + priv->vdev.compose.height != compose.height) ? -EPIPE : 0; } static int capture_start_streaming(struct vb2_queue *vq, unsigned int count) diff --git a/drivers/staging/media/ipu3/ipu3-v4l2.c b/drivers/staging/media/ipu3/ipu3-v4l2.c index 60aa02eb7d2a..6d9c49b39531 100644 --- a/drivers/staging/media/ipu3/ipu3-v4l2.c +++ b/drivers/staging/media/ipu3/ipu3-v4l2.c @@ -686,6 +686,7 @@ static int imgu_fmt(struct imgu_device *imgu, unsigned int pipe, int node, dev_dbg(dev, "IPU3 pipe %u pipe_id = %u", pipe, css_pipe->pipe_id); + css_q = imgu_node_to_queue(node); for (i = 0; i < IPU3_CSS_QUEUES; i++) { unsigned int inode = imgu_map_node(imgu, i); @@ -693,6 +694,18 @@ static int imgu_fmt(struct imgu_device *imgu, unsigned int pipe, int node, if (inode == IMGU_NODE_STAT_3A || inode == IMGU_NODE_PARAMS) continue; + /* CSS expects some format on OUT queue */ + if (i != IPU3_CSS_QUEUE_OUT && + !imgu_pipe->nodes[inode].enabled) { + fmts[i] = NULL; + continue; + } + + if (i == css_q) { + fmts[i] = &f->fmt.pix_mp; + continue; + } + if (try) { fmts[i] = kmemdup(&imgu_pipe->nodes[inode].vdev_fmt.fmt.pix_mp, sizeof(struct v4l2_pix_format_mplane), @@ -705,10 +718,6 @@ static int imgu_fmt(struct imgu_device *imgu, unsigned int pipe, int node, fmts[i] = &imgu_pipe->nodes[inode].vdev_fmt.fmt.pix_mp; } - /* CSS expects some format on OUT queue */ - if (i != IPU3_CSS_QUEUE_OUT && - !imgu_pipe->nodes[inode].enabled) - fmts[i] = NULL; } if (!try) { @@ -725,16 +734,10 @@ static int imgu_fmt(struct imgu_device *imgu, unsigned int pipe, int node, rects[IPU3_CSS_RECT_GDC]->height = pad_fmt.height; } - /* - * imgu doesn't set the node to the value given by user - * before we return success from this function, so set it here. - */ - css_q = imgu_node_to_queue(node); if (!fmts[css_q]) { ret = -EINVAL; goto out; } - *fmts[css_q] = f->fmt.pix_mp; if (try) ret = imgu_css_fmt_try(&imgu->css, fmts, rects, pipe); @@ -745,15 +748,18 @@ static int imgu_fmt(struct imgu_device *imgu, unsigned int pipe, int node, if (ret < 0) goto out; - if (try) - f->fmt.pix_mp = *fmts[css_q]; - else - f->fmt = imgu_pipe->nodes[node].vdev_fmt.fmt; + /* + * imgu doesn't set the node to the value given by user + * before we return success from this function, so set it here. + */ + if (!try) + imgu_pipe->nodes[node].vdev_fmt.fmt.pix_mp = f->fmt.pix_mp; out: if (try) { for (i = 0; i < IPU3_CSS_QUEUES; i++) - kfree(fmts[i]); + if (i != css_q) + kfree(fmts[i]); } return ret; diff --git a/drivers/staging/media/omap4iss/iss.c b/drivers/staging/media/omap4iss/iss.c index dae9073e7d3c..085397045b36 100644 --- a/drivers/staging/media/omap4iss/iss.c +++ b/drivers/staging/media/omap4iss/iss.c @@ -1236,8 +1236,10 @@ static int iss_probe(struct platform_device *pdev) if (ret < 0) goto error; - if (!omap4iss_get(iss)) + if (!omap4iss_get(iss)) { + ret = -EINVAL; goto error; + } ret = iss_reset(iss); if (ret < 0) diff --git a/drivers/staging/media/rkvdec/rkvdec.c b/drivers/staging/media/rkvdec/rkvdec.c index d3eb81ee8dc2..d821661d30f3 100644 --- a/drivers/staging/media/rkvdec/rkvdec.c +++ b/drivers/staging/media/rkvdec/rkvdec.c @@ -55,16 +55,13 @@ static const struct v4l2_ctrl_ops rkvdec_ctrl_ops = { static const struct rkvdec_ctrl_desc rkvdec_h264_ctrl_descs[] = { { - .mandatory = true, .cfg.id = V4L2_CID_STATELESS_H264_DECODE_PARAMS, }, { - .mandatory = true, .cfg.id = V4L2_CID_STATELESS_H264_SPS, .cfg.ops = &rkvdec_ctrl_ops, }, { - .mandatory = true, .cfg.id = V4L2_CID_STATELESS_H264_PPS, }, { @@ -585,25 +582,7 @@ static const struct vb2_ops rkvdec_queue_ops = { static int rkvdec_request_validate(struct media_request *req) { - struct media_request_object *obj; - const struct rkvdec_ctrls *ctrls; - struct v4l2_ctrl_handler *hdl; - struct rkvdec_ctx *ctx = NULL; - unsigned int count, i; - int ret; - - list_for_each_entry(obj, &req->objects, list) { - if (vb2_request_object_is_buffer(obj)) { - struct vb2_buffer *vb; - - vb = container_of(obj, struct vb2_buffer, req_obj); - ctx = vb2_get_drv_priv(vb->vb2_queue); - break; - } - } - - if (!ctx) - return -EINVAL; + unsigned int count; count = vb2_request_buffer_cnt(req); if (!count) @@ -611,31 +590,6 @@ static int rkvdec_request_validate(struct media_request *req) else if (count > 1) return -EINVAL; - hdl = v4l2_ctrl_request_hdl_find(req, &ctx->ctrl_hdl); - if (!hdl) - return -ENOENT; - - ret = 0; - ctrls = ctx->coded_fmt_desc->ctrls; - for (i = 0; ctrls && i < ctrls->num_ctrls; i++) { - u32 id = ctrls->ctrls[i].cfg.id; - struct v4l2_ctrl *ctrl; - - if (!ctrls->ctrls[i].mandatory) - continue; - - ctrl = v4l2_ctrl_request_hdl_ctrl_find(hdl, id); - if (!ctrl) { - ret = -ENOENT; - break; - } - } - - v4l2_ctrl_request_hdl_put(hdl); - - if (ret) - return ret; - return vb2_request_validate(req); } @@ -1118,7 +1072,7 @@ static struct platform_driver rkvdec_driver = { .remove = rkvdec_remove, .driver = { .name = "rkvdec", - .of_match_table = of_match_ptr(of_rkvdec_match), + .of_match_table = of_rkvdec_match, .pm = &rkvdec_pm_ops, }, }; diff --git a/drivers/staging/media/rkvdec/rkvdec.h b/drivers/staging/media/rkvdec/rkvdec.h index 77a137cca88e..52ac3874c5e5 100644 --- a/drivers/staging/media/rkvdec/rkvdec.h +++ b/drivers/staging/media/rkvdec/rkvdec.h @@ -25,7 +25,6 @@ struct rkvdec_ctx; struct rkvdec_ctrl_desc { - u32 mandatory : 1; struct v4l2_ctrl_config cfg; }; diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_regs.h b/drivers/staging/media/sunxi/cedrus/cedrus_regs.h index 7718c561823f..92ace87c1c7d 100644 --- a/drivers/staging/media/sunxi/cedrus/cedrus_regs.h +++ b/drivers/staging/media/sunxi/cedrus/cedrus_regs.h @@ -443,16 +443,17 @@ #define VE_DEC_H265_STATUS_STCD_BUSY BIT(21) #define VE_DEC_H265_STATUS_WB_BUSY BIT(20) #define VE_DEC_H265_STATUS_BS_DMA_BUSY BIT(19) -#define VE_DEC_H265_STATUS_IQIT_BUSY BIT(18) +#define VE_DEC_H265_STATUS_IT_BUSY BIT(18) #define VE_DEC_H265_STATUS_INTER_BUSY BIT(17) #define VE_DEC_H265_STATUS_MORE_DATA BIT(16) -#define VE_DEC_H265_STATUS_VLD_BUSY BIT(14) -#define VE_DEC_H265_STATUS_DEBLOCKING_BUSY BIT(13) -#define VE_DEC_H265_STATUS_DEBLOCKING_DRAM_BUSY BIT(12) -#define VE_DEC_H265_STATUS_INTRA_BUSY BIT(11) -#define VE_DEC_H265_STATUS_SAO_BUSY BIT(10) -#define VE_DEC_H265_STATUS_MVP_BUSY BIT(9) -#define VE_DEC_H265_STATUS_SWDEC_BUSY BIT(8) +#define VE_DEC_H265_STATUS_DBLK_BUSY BIT(15) +#define VE_DEC_H265_STATUS_IREC_BUSY BIT(14) +#define VE_DEC_H265_STATUS_INTRA_BUSY BIT(13) +#define VE_DEC_H265_STATUS_MCRI_BUSY BIT(12) +#define VE_DEC_H265_STATUS_IQIT_BUSY BIT(11) +#define VE_DEC_H265_STATUS_MVP_BUSY BIT(10) +#define VE_DEC_H265_STATUS_IS_BUSY BIT(9) +#define VE_DEC_H265_STATUS_VLD_BUSY BIT(8) #define VE_DEC_H265_STATUS_OVER_TIME BIT(3) #define VE_DEC_H265_STATUS_VLD_DATA_REQ BIT(2) #define VE_DEC_H265_STATUS_ERROR BIT(1) diff --git a/drivers/staging/qlge/qlge_main.c b/drivers/staging/qlge/qlge_main.c index 5516be3af898..c1d52190e1bd 100644 --- a/drivers/staging/qlge/qlge_main.c +++ b/drivers/staging/qlge/qlge_main.c @@ -4550,7 +4550,7 @@ static int qlge_probe(struct pci_dev *pdev, struct net_device *ndev = NULL; struct devlink *devlink; static int cards_found; - int err = 0; + int err; devlink = devlink_alloc(&qlge_devlink_ops, sizeof(struct qlge_adapter)); if (!devlink) @@ -4561,8 +4561,10 @@ static int qlge_probe(struct pci_dev *pdev, ndev = alloc_etherdev_mq(sizeof(struct qlge_netdev_priv), min(MAX_CPUS, netif_get_num_default_rss_queues())); - if (!ndev) + if (!ndev) { + err = -ENOMEM; goto devlink_free; + } ndev_priv = netdev_priv(ndev); ndev_priv->qdev = qdev; diff --git a/drivers/staging/rtl8192u/r8192U_core.c b/drivers/staging/rtl8192u/r8192U_core.c index 9fc4adc83d77..b5a313649f44 100644 --- a/drivers/staging/rtl8192u/r8192U_core.c +++ b/drivers/staging/rtl8192u/r8192U_core.c @@ -3210,7 +3210,7 @@ static void rtl819x_update_rxcounts(struct r8192_priv *priv, u32 *TotalRxBcnNum, u32 *TotalRxDataNum) { u16 SlotIndex; - u8 i; + u16 i; *TotalRxBcnNum = 0; *TotalRxDataNum = 0; diff --git a/drivers/staging/wimax/i2400m/op-rfkill.c b/drivers/staging/wimax/i2400m/op-rfkill.c index fbddf2e18c14..44698a1aae87 100644 --- a/drivers/staging/wimax/i2400m/op-rfkill.c +++ b/drivers/staging/wimax/i2400m/op-rfkill.c @@ -86,7 +86,7 @@ int i2400m_op_rfkill_sw_toggle(struct wimax_dev *wimax_dev, if (cmd == NULL) goto error_alloc; cmd->hdr.type = cpu_to_le16(I2400M_MT_CMD_RF_CONTROL); - cmd->hdr.length = sizeof(cmd->sw_rf); + cmd->hdr.length = cpu_to_le16(sizeof(cmd->sw_rf)); cmd->hdr.version = cpu_to_le16(I2400M_L3L4_VERSION); cmd->sw_rf.hdr.type = cpu_to_le16(I2400M_TLV_RF_OPERATION); cmd->sw_rf.hdr.length = cpu_to_le16(sizeof(cmd->sw_rf.status)); diff --git a/drivers/target/target_core_pscsi.c b/drivers/target/target_core_pscsi.c index 9ee797b8cb7e..508b49b0eaf5 100644 --- a/drivers/target/target_core_pscsi.c +++ b/drivers/target/target_core_pscsi.c @@ -620,8 +620,9 @@ static void pscsi_complete_cmd(struct se_cmd *cmd, u8 scsi_status, unsigned char *buf; buf = transport_kmap_data_sg(cmd); - if (!buf) + if (!buf) { ; /* XXX: TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE */ + } if (cdb[0] == MODE_SENSE_10) { if (!(buf[3] & 0x80)) diff --git a/drivers/target/target_core_user.c b/drivers/target/target_core_user.c index bf73cd5f4b04..6809c970be03 100644 --- a/drivers/target/target_core_user.c +++ b/drivers/target/target_core_user.c @@ -1377,7 +1377,7 @@ static int tcmu_run_tmr_queue(struct tcmu_dev *udev) return 1; } -static unsigned int tcmu_handle_completions(struct tcmu_dev *udev) +static bool tcmu_handle_completions(struct tcmu_dev *udev) { struct tcmu_mailbox *mb; struct tcmu_cmd *cmd; @@ -1420,7 +1420,7 @@ static unsigned int tcmu_handle_completions(struct tcmu_dev *udev) pr_err("cmd_id %u not found, ring is broken\n", entry->hdr.cmd_id); set_bit(TCMU_DEV_BIT_BROKEN, &udev->flags); - break; + return false; } tcmu_handle_completion(cmd, entry); diff --git a/drivers/tee/optee/core.c b/drivers/tee/optee/core.c index 319a1e701163..ddb8f9ecf307 100644 --- a/drivers/tee/optee/core.c +++ b/drivers/tee/optee/core.c @@ -79,16 +79,6 @@ int optee_from_msg_param(struct tee_param *params, size_t num_params, return rc; p->u.memref.shm_offs = mp->u.tmem.buf_ptr - pa; p->u.memref.shm = shm; - - /* Check that the memref is covered by the shm object */ - if (p->u.memref.size) { - size_t o = p->u.memref.shm_offs + - p->u.memref.size - 1; - - rc = tee_shm_get_pa(shm, o, NULL); - if (rc) - return rc; - } break; case OPTEE_MSG_ATTR_TYPE_RMEM_INPUT: case OPTEE_MSG_ATTR_TYPE_RMEM_OUTPUT: diff --git a/drivers/thermal/cpufreq_cooling.c b/drivers/thermal/cpufreq_cooling.c index 10af3341e5ea..6956581ed7a4 100644 --- a/drivers/thermal/cpufreq_cooling.c +++ b/drivers/thermal/cpufreq_cooling.c @@ -125,7 +125,7 @@ static u32 cpu_power_to_freq(struct cpufreq_cooling_device *cpufreq_cdev, { int i; - for (i = cpufreq_cdev->max_level; i >= 0; i--) { + for (i = cpufreq_cdev->max_level; i > 0; i--) { if (power >= cpufreq_cdev->em->table[i].power) break; } diff --git a/drivers/thermal/gov_fair_share.c b/drivers/thermal/gov_fair_share.c index aaa07180ab48..645432ce6365 100644 --- a/drivers/thermal/gov_fair_share.c +++ b/drivers/thermal/gov_fair_share.c @@ -82,6 +82,8 @@ static int fair_share_throttle(struct thermal_zone_device *tz, int trip) int total_instance = 0; int cur_trip_level = get_trip_level(tz); + mutex_lock(&tz->lock); + list_for_each_entry(instance, &tz->thermal_instances, tz_node) { if (instance->trip != trip) continue; @@ -110,6 +112,8 @@ static int fair_share_throttle(struct thermal_zone_device *tz, int trip) mutex_unlock(&instance->cdev->lock); thermal_cdev_update(cdev); } + + mutex_unlock(&tz->lock); return 0; } diff --git a/drivers/thermal/qcom/tsens.c b/drivers/thermal/qcom/tsens.c index d8ce3a687b80..3c4c0516e58a 100644 --- a/drivers/thermal/qcom/tsens.c +++ b/drivers/thermal/qcom/tsens.c @@ -755,8 +755,10 @@ int __init init_common(struct tsens_priv *priv) for (i = VER_MAJOR; i <= VER_STEP; i++) { priv->rf[i] = devm_regmap_field_alloc(dev, priv->srot_map, priv->fields[i]); - if (IS_ERR(priv->rf[i])) - return PTR_ERR(priv->rf[i]); + if (IS_ERR(priv->rf[i])) { + ret = PTR_ERR(priv->rf[i]); + goto err_put_device; + } } ret = regmap_field_read(priv->rf[VER_MINOR], &ver_minor); if (ret) diff --git a/drivers/thermal/thermal_of.c b/drivers/thermal/thermal_of.c index 69ef12f852b7..5b76f9a1280d 100644 --- a/drivers/thermal/thermal_of.c +++ b/drivers/thermal/thermal_of.c @@ -704,14 +704,17 @@ static int thermal_of_populate_bind_params(struct device_node *np, count = of_count_phandle_with_args(np, "cooling-device", "#cooling-cells"); - if (!count) { + if (count <= 0) { pr_err("Add a cooling_device property with at least one device\n"); + ret = -ENOENT; goto end; } __tcbp = kcalloc(count, sizeof(*__tcbp), GFP_KERNEL); - if (!__tcbp) + if (!__tcbp) { + ret = -ENOMEM; goto end; + } for (i = 0; i < count; i++) { ret = of_parse_phandle_with_args(np, "cooling-device", diff --git a/drivers/tty/amiserial.c b/drivers/tty/amiserial.c index 18b78ea110ef..ecda5e18d23f 100644 --- a/drivers/tty/amiserial.c +++ b/drivers/tty/amiserial.c @@ -970,6 +970,7 @@ static int set_serial_info(struct tty_struct *tty, struct serial_struct *ss) if (!serial_isroot()) { if ((ss->baud_base != state->baud_base) || (ss->close_delay != port->close_delay) || + (ss->closing_wait != port->closing_wait) || (ss->xmit_fifo_size != state->xmit_fifo_size) || ((ss->flags & ~ASYNC_USR_MASK) != (port->flags & ~ASYNC_USR_MASK))) { diff --git a/drivers/tty/moxa.c b/drivers/tty/moxa.c index 9f13f7d49dd7..f9f14104bd2c 100644 --- a/drivers/tty/moxa.c +++ b/drivers/tty/moxa.c @@ -2040,7 +2040,7 @@ static int moxa_get_serial_info(struct tty_struct *tty, ss->line = info->port.tty->index, ss->flags = info->port.flags, ss->baud_base = 921600, - ss->close_delay = info->port.close_delay; + ss->close_delay = jiffies_to_msecs(info->port.close_delay) / 10; mutex_unlock(&info->port.mutex); return 0; } @@ -2050,6 +2050,7 @@ static int moxa_set_serial_info(struct tty_struct *tty, struct serial_struct *ss) { struct moxa_port *info = tty->driver_data; + unsigned int close_delay; if (tty->index == MAX_PORTS) return -EINVAL; @@ -2061,19 +2062,24 @@ static int moxa_set_serial_info(struct tty_struct *tty, ss->baud_base != 921600) return -EPERM; + close_delay = msecs_to_jiffies(ss->close_delay * 10); + mutex_lock(&info->port.mutex); if (!capable(CAP_SYS_ADMIN)) { - if (((ss->flags & ~ASYNC_USR_MASK) != + if (close_delay != info->port.close_delay || + ss->type != info->type || + ((ss->flags & ~ASYNC_USR_MASK) != (info->port.flags & ~ASYNC_USR_MASK))) { mutex_unlock(&info->port.mutex); return -EPERM; } - } - info->port.close_delay = ss->close_delay * HZ / 100; + } else { + info->port.close_delay = close_delay; - MoxaSetFifo(info, ss->type == PORT_16550A); + MoxaSetFifo(info, ss->type == PORT_16550A); - info->type = ss->type; + info->type = ss->type; + } mutex_unlock(&info->port.mutex); return 0; } diff --git a/drivers/tty/mxser.c b/drivers/tty/mxser.c index 4203b64bccdb..2d8e76263a25 100644 --- a/drivers/tty/mxser.c +++ b/drivers/tty/mxser.c @@ -1208,19 +1208,26 @@ static int mxser_get_serial_info(struct tty_struct *tty, { struct mxser_port *info = tty->driver_data; struct tty_port *port = &info->port; + unsigned int closing_wait, close_delay; if (tty->index == MXSER_PORTS) return -ENOTTY; mutex_lock(&port->mutex); + + close_delay = jiffies_to_msecs(info->port.close_delay) / 10; + closing_wait = info->port.closing_wait; + if (closing_wait != ASYNC_CLOSING_WAIT_NONE) + closing_wait = jiffies_to_msecs(closing_wait) / 10; + ss->type = info->type, ss->line = tty->index, ss->port = info->ioaddr, ss->irq = info->board->irq, ss->flags = info->port.flags, ss->baud_base = info->baud_base, - ss->close_delay = info->port.close_delay, - ss->closing_wait = info->port.closing_wait, + ss->close_delay = close_delay; + ss->closing_wait = closing_wait; ss->custom_divisor = info->custom_divisor, mutex_unlock(&port->mutex); return 0; @@ -1233,7 +1240,7 @@ static int mxser_set_serial_info(struct tty_struct *tty, struct tty_port *port = &info->port; speed_t baud; unsigned long sl_flags; - unsigned int flags; + unsigned int flags, close_delay, closing_wait; int retval = 0; if (tty->index == MXSER_PORTS) @@ -1255,9 +1262,15 @@ static int mxser_set_serial_info(struct tty_struct *tty, flags = port->flags & ASYNC_SPD_MASK; + close_delay = msecs_to_jiffies(ss->close_delay * 10); + closing_wait = ss->closing_wait; + if (closing_wait != ASYNC_CLOSING_WAIT_NONE) + closing_wait = msecs_to_jiffies(closing_wait * 10); + if (!capable(CAP_SYS_ADMIN)) { if ((ss->baud_base != info->baud_base) || - (ss->close_delay != info->port.close_delay) || + (close_delay != info->port.close_delay) || + (closing_wait != info->port.closing_wait) || ((ss->flags & ~ASYNC_USR_MASK) != (info->port.flags & ~ASYNC_USR_MASK))) { mutex_unlock(&port->mutex); return -EPERM; @@ -1271,8 +1284,8 @@ static int mxser_set_serial_info(struct tty_struct *tty, */ port->flags = ((port->flags & ~ASYNC_FLAGS) | (ss->flags & ASYNC_FLAGS)); - port->close_delay = ss->close_delay * HZ / 100; - port->closing_wait = ss->closing_wait * HZ / 100; + port->close_delay = close_delay; + port->closing_wait = closing_wait; if ((port->flags & ASYNC_SPD_MASK) == ASYNC_SPD_CUST && (ss->baud_base != info->baud_base || ss->custom_divisor != @@ -1284,11 +1297,11 @@ static int mxser_set_serial_info(struct tty_struct *tty, baud = ss->baud_base / ss->custom_divisor; tty_encode_baud_rate(tty, baud, baud); } - } - info->type = ss->type; + info->type = ss->type; - process_txrx_fifo(info); + process_txrx_fifo(info); + } if (tty_port_initialized(port)) { if (flags != (port->flags & ASYNC_SPD_MASK)) { diff --git a/drivers/tty/n_gsm.c b/drivers/tty/n_gsm.c index 51dafc06f541..2406653d38b7 100644 --- a/drivers/tty/n_gsm.c +++ b/drivers/tty/n_gsm.c @@ -2384,8 +2384,18 @@ static int gsmld_attach_gsm(struct tty_struct *tty, struct gsm_mux *gsm) /* Don't register device 0 - this is the control channel and not a usable tty interface */ base = mux_num_to_base(gsm); /* Base for this MUX */ - for (i = 1; i < NUM_DLCI; i++) - tty_register_device(gsm_tty_driver, base + i, NULL); + for (i = 1; i < NUM_DLCI; i++) { + struct device *dev; + + dev = tty_register_device(gsm_tty_driver, + base + i, NULL); + if (IS_ERR(dev)) { + for (i--; i >= 1; i--) + tty_unregister_device(gsm_tty_driver, + base + i); + return PTR_ERR(dev); + } + } } return ret; } diff --git a/drivers/tty/serial/liteuart.c b/drivers/tty/serial/liteuart.c index 64842f3539e1..0b06770642cb 100644 --- a/drivers/tty/serial/liteuart.c +++ b/drivers/tty/serial/liteuart.c @@ -270,8 +270,8 @@ static int liteuart_probe(struct platform_device *pdev) /* get membase */ port->membase = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); - if (!port->membase) - return -ENXIO; + if (IS_ERR(port->membase)) + return PTR_ERR(port->membase); /* values not from device tree */ port->dev = &pdev->dev; diff --git a/drivers/tty/serial/omap-serial.c b/drivers/tty/serial/omap-serial.c index 76b94d0ff586..84e8158088cd 100644 --- a/drivers/tty/serial/omap-serial.c +++ b/drivers/tty/serial/omap-serial.c @@ -159,6 +159,8 @@ struct uart_omap_port { u32 calc_latency; struct work_struct qos_work; bool is_suspending; + + unsigned int rs485_tx_filter_count; }; #define to_uart_omap_port(p) ((container_of((p), struct uart_omap_port, port))) @@ -302,7 +304,8 @@ static void serial_omap_stop_tx(struct uart_port *port) serial_out(up, UART_OMAP_SCR, up->scr); res = (port->rs485.flags & SER_RS485_RTS_AFTER_SEND) ? 1 : 0; - if (gpiod_get_value(up->rts_gpiod) != res) { + if (up->rts_gpiod && + gpiod_get_value(up->rts_gpiod) != res) { if (port->rs485.delay_rts_after_send > 0) mdelay( port->rs485.delay_rts_after_send); @@ -328,19 +331,6 @@ static void serial_omap_stop_tx(struct uart_port *port) serial_out(up, UART_IER, up->ier); } - if ((port->rs485.flags & SER_RS485_ENABLED) && - !(port->rs485.flags & SER_RS485_RX_DURING_TX)) { - /* - * Empty the RX FIFO, we are not interested in anything - * received during the half-duplex transmission. - */ - serial_out(up, UART_FCR, up->fcr | UART_FCR_CLEAR_RCVR); - /* Re-enable RX interrupts */ - up->ier |= UART_IER_RLSI | UART_IER_RDI; - up->port.read_status_mask |= UART_LSR_DR; - serial_out(up, UART_IER, up->ier); - } - pm_runtime_mark_last_busy(up->dev); pm_runtime_put_autosuspend(up->dev); } @@ -366,6 +356,10 @@ static void transmit_chars(struct uart_omap_port *up, unsigned int lsr) serial_out(up, UART_TX, up->port.x_char); up->port.icount.tx++; up->port.x_char = 0; + if ((up->port.rs485.flags & SER_RS485_ENABLED) && + !(up->port.rs485.flags & SER_RS485_RX_DURING_TX)) + up->rs485_tx_filter_count++; + return; } if (uart_circ_empty(xmit) || uart_tx_stopped(&up->port)) { @@ -377,6 +371,10 @@ static void transmit_chars(struct uart_omap_port *up, unsigned int lsr) serial_out(up, UART_TX, xmit->buf[xmit->tail]); xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1); up->port.icount.tx++; + if ((up->port.rs485.flags & SER_RS485_ENABLED) && + !(up->port.rs485.flags & SER_RS485_RX_DURING_TX)) + up->rs485_tx_filter_count++; + if (uart_circ_empty(xmit)) break; } while (--count > 0); @@ -411,7 +409,7 @@ static void serial_omap_start_tx(struct uart_port *port) /* if rts not already enabled */ res = (port->rs485.flags & SER_RS485_RTS_ON_SEND) ? 1 : 0; - if (gpiod_get_value(up->rts_gpiod) != res) { + if (up->rts_gpiod && gpiod_get_value(up->rts_gpiod) != res) { gpiod_set_value(up->rts_gpiod, res); if (port->rs485.delay_rts_before_send > 0) mdelay(port->rs485.delay_rts_before_send); @@ -420,7 +418,7 @@ static void serial_omap_start_tx(struct uart_port *port) if ((port->rs485.flags & SER_RS485_ENABLED) && !(port->rs485.flags & SER_RS485_RX_DURING_TX)) - serial_omap_stop_rx(port); + up->rs485_tx_filter_count = 0; serial_omap_enable_ier_thri(up); pm_runtime_mark_last_busy(up->dev); @@ -491,8 +489,13 @@ static void serial_omap_rlsi(struct uart_omap_port *up, unsigned int lsr) * Read one data character out to avoid stalling the receiver according * to the table 23-246 of the omap4 TRM. */ - if (likely(lsr & UART_LSR_DR)) + if (likely(lsr & UART_LSR_DR)) { serial_in(up, UART_RX); + if ((up->port.rs485.flags & SER_RS485_ENABLED) && + !(up->port.rs485.flags & SER_RS485_RX_DURING_TX) && + up->rs485_tx_filter_count) + up->rs485_tx_filter_count--; + } up->port.icount.rx++; flag = TTY_NORMAL; @@ -543,6 +546,13 @@ static void serial_omap_rdi(struct uart_omap_port *up, unsigned int lsr) return; ch = serial_in(up, UART_RX); + if ((up->port.rs485.flags & SER_RS485_ENABLED) && + !(up->port.rs485.flags & SER_RS485_RX_DURING_TX) && + up->rs485_tx_filter_count) { + up->rs485_tx_filter_count--; + return; + } + flag = TTY_NORMAL; up->port.icount.rx++; @@ -1407,18 +1417,13 @@ serial_omap_config_rs485(struct uart_port *port, struct serial_rs485 *rs485) /* store new config */ port->rs485 = *rs485; - /* - * Just as a precaution, only allow rs485 - * to be enabled if the gpio pin is valid - */ if (up->rts_gpiod) { /* enable / disable rts */ val = (port->rs485.flags & SER_RS485_ENABLED) ? SER_RS485_RTS_AFTER_SEND : SER_RS485_RTS_ON_SEND; val = (port->rs485.flags & val) ? 1 : 0; gpiod_set_value(up->rts_gpiod, val); - } else - port->rs485.flags &= ~SER_RS485_ENABLED; + } /* Enable interrupts */ up->ier = mode; diff --git a/drivers/tty/serial/sc16is7xx.c b/drivers/tty/serial/sc16is7xx.c index f86ec2d2635b..9adb8362578c 100644 --- a/drivers/tty/serial/sc16is7xx.c +++ b/drivers/tty/serial/sc16is7xx.c @@ -1196,7 +1196,7 @@ static int sc16is7xx_probe(struct device *dev, ret = regmap_read(regmap, SC16IS7XX_LSR_REG << SC16IS7XX_REG_SHIFT, &val); if (ret < 0) - return ret; + return -EPROBE_DEFER; /* Alloc port structure */ s = devm_kzalloc(dev, struct_size(s, p, devtype->nr_uart), GFP_KERNEL); diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c index ba31e97d3d96..43f02ed055d5 100644 --- a/drivers/tty/serial/serial_core.c +++ b/drivers/tty/serial/serial_core.c @@ -1305,7 +1305,7 @@ static int uart_set_rs485_config(struct uart_port *port, unsigned long flags; if (!port->rs485_config) - return -ENOIOCTLCMD; + return -ENOTTY; if (copy_from_user(&rs485, rs485_user, sizeof(*rs485_user))) return -EFAULT; @@ -1329,7 +1329,7 @@ static int uart_get_iso7816_config(struct uart_port *port, struct serial_iso7816 aux; if (!port->iso7816_config) - return -ENOIOCTLCMD; + return -ENOTTY; spin_lock_irqsave(&port->lock, flags); aux = port->iso7816; @@ -1349,7 +1349,7 @@ static int uart_set_iso7816_config(struct uart_port *port, unsigned long flags; if (!port->iso7816_config) - return -ENOIOCTLCMD; + return -ENOTTY; if (copy_from_user(&iso7816, iso7816_user, sizeof(*iso7816_user))) return -EFAULT; diff --git a/drivers/tty/serial/stm32-usart.c b/drivers/tty/serial/stm32-usart.c index b3675cf25a69..99dfa884cbef 100644 --- a/drivers/tty/serial/stm32-usart.c +++ b/drivers/tty/serial/stm32-usart.c @@ -214,12 +214,14 @@ static void stm32_usart_receive_chars(struct uart_port *port, bool threaded) struct tty_port *tport = &port->state->port; struct stm32_port *stm32_port = to_stm32_port(port); const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs; - unsigned long c; + unsigned long c, flags; u32 sr; char flag; - if (irqd_is_wakeup_set(irq_get_irq_data(port->irq))) - pm_wakeup_event(tport->tty->dev, 0); + if (threaded) + spin_lock_irqsave(&port->lock, flags); + else + spin_lock(&port->lock); while (stm32_usart_pending_rx(port, &sr, &stm32_port->last_res, threaded)) { @@ -276,9 +278,12 @@ static void stm32_usart_receive_chars(struct uart_port *port, bool threaded) uart_insert_char(port, sr, USART_SR_ORE, c, flag); } - spin_unlock(&port->lock); + if (threaded) + spin_unlock_irqrestore(&port->lock, flags); + else + spin_unlock(&port->lock); + tty_flip_buffer_push(tport); - spin_lock(&port->lock); } static void stm32_usart_tx_dma_complete(void *arg) @@ -286,12 +291,16 @@ static void stm32_usart_tx_dma_complete(void *arg) struct uart_port *port = arg; struct stm32_port *stm32port = to_stm32_port(port); const struct stm32_usart_offsets *ofs = &stm32port->info->ofs; + unsigned long flags; + dmaengine_terminate_async(stm32port->tx_ch); stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAT); stm32port->tx_dma_busy = false; /* Let's see if we have pending data to send */ + spin_lock_irqsave(&port->lock, flags); stm32_usart_transmit_chars(port); + spin_unlock_irqrestore(&port->lock, flags); } static void stm32_usart_tx_interrupt_enable(struct uart_port *port) @@ -455,29 +464,34 @@ static void stm32_usart_transmit_chars(struct uart_port *port) static irqreturn_t stm32_usart_interrupt(int irq, void *ptr) { struct uart_port *port = ptr; + struct tty_port *tport = &port->state->port; struct stm32_port *stm32_port = to_stm32_port(port); const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs; u32 sr; - spin_lock(&port->lock); - sr = readl_relaxed(port->membase + ofs->isr); if ((sr & USART_SR_RTOF) && ofs->icr != UNDEF_REG) writel_relaxed(USART_ICR_RTOCF, port->membase + ofs->icr); - if ((sr & USART_SR_WUF) && ofs->icr != UNDEF_REG) + if ((sr & USART_SR_WUF) && ofs->icr != UNDEF_REG) { + /* Clear wake up flag and disable wake up interrupt */ writel_relaxed(USART_ICR_WUCF, port->membase + ofs->icr); + stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_WUFIE); + if (irqd_is_wakeup_set(irq_get_irq_data(port->irq))) + pm_wakeup_event(tport->tty->dev, 0); + } if ((sr & USART_SR_RXNE) && !(stm32_port->rx_ch)) stm32_usart_receive_chars(port, false); - if ((sr & USART_SR_TXE) && !(stm32_port->tx_ch)) + if ((sr & USART_SR_TXE) && !(stm32_port->tx_ch)) { + spin_lock(&port->lock); stm32_usart_transmit_chars(port); - - spin_unlock(&port->lock); + spin_unlock(&port->lock); + } if (stm32_port->rx_ch) return IRQ_WAKE_THREAD; @@ -490,13 +504,9 @@ static irqreturn_t stm32_usart_threaded_interrupt(int irq, void *ptr) struct uart_port *port = ptr; struct stm32_port *stm32_port = to_stm32_port(port); - spin_lock(&port->lock); - if (stm32_port->rx_ch) stm32_usart_receive_chars(port, true); - spin_unlock(&port->lock); - return IRQ_HANDLED; } @@ -505,7 +515,10 @@ static unsigned int stm32_usart_tx_empty(struct uart_port *port) struct stm32_port *stm32_port = to_stm32_port(port); const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs; - return readl_relaxed(port->membase + ofs->isr) & USART_SR_TXE; + if (readl_relaxed(port->membase + ofs->isr) & USART_SR_TC) + return TIOCSER_TEMT; + + return 0; } static void stm32_usart_set_mctrl(struct uart_port *port, unsigned int mctrl) @@ -634,6 +647,7 @@ static int stm32_usart_startup(struct uart_port *port) { struct stm32_port *stm32_port = to_stm32_port(port); const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs; + const struct stm32_usart_config *cfg = &stm32_port->info->cfg; const char *name = to_platform_device(port->dev)->name; u32 val; int ret; @@ -646,21 +660,10 @@ static int stm32_usart_startup(struct uart_port *port) /* RX FIFO Flush */ if (ofs->rqr != UNDEF_REG) - stm32_usart_set_bits(port, ofs->rqr, USART_RQR_RXFRQ); - - /* Tx and RX FIFO configuration */ - if (stm32_port->fifoen) { - val = readl_relaxed(port->membase + ofs->cr3); - val &= ~(USART_CR3_TXFTCFG_MASK | USART_CR3_RXFTCFG_MASK); - val |= USART_CR3_TXFTCFG_HALF << USART_CR3_TXFTCFG_SHIFT; - val |= USART_CR3_RXFTCFG_HALF << USART_CR3_RXFTCFG_SHIFT; - writel_relaxed(val, port->membase + ofs->cr3); - } + writel_relaxed(USART_RQR_RXFRQ, port->membase + ofs->rqr); - /* RX FIFO enabling */ - val = stm32_port->cr1_irq | USART_CR1_RE; - if (stm32_port->fifoen) - val |= USART_CR1_FIFOEN; + /* RX enabling */ + val = stm32_port->cr1_irq | USART_CR1_RE | BIT(cfg->uart_enable_bit); stm32_usart_set_bits(port, ofs->cr1, val); return 0; @@ -691,6 +694,11 @@ static void stm32_usart_shutdown(struct uart_port *port) if (ret) dev_err(port->dev, "Transmission is not complete\n"); + /* flush RX & TX FIFO */ + if (ofs->rqr != UNDEF_REG) + writel_relaxed(USART_RQR_TXFRQ | USART_RQR_RXFRQ, + port->membase + ofs->rqr); + stm32_usart_clr_bits(port, ofs->cr1, val); free_irq(port->irq, port); @@ -737,8 +745,9 @@ static void stm32_usart_set_termios(struct uart_port *port, unsigned int baud, bits; u32 usartdiv, mantissa, fraction, oversampling; tcflag_t cflag = termios->c_cflag; - u32 cr1, cr2, cr3; + u32 cr1, cr2, cr3, isr; unsigned long flags; + int ret; if (!stm32_port->hw_flow_control) cflag &= ~CRTSCTS; @@ -747,21 +756,36 @@ static void stm32_usart_set_termios(struct uart_port *port, spin_lock_irqsave(&port->lock, flags); + ret = readl_relaxed_poll_timeout_atomic(port->membase + ofs->isr, + isr, + (isr & USART_SR_TC), + 10, 100000); + + /* Send the TC error message only when ISR_TC is not set. */ + if (ret) + dev_err(port->dev, "Transmission is not complete\n"); + /* Stop serial port and reset value */ writel_relaxed(0, port->membase + ofs->cr1); /* flush RX & TX FIFO */ if (ofs->rqr != UNDEF_REG) - stm32_usart_set_bits(port, ofs->rqr, - USART_RQR_TXFRQ | USART_RQR_RXFRQ); + writel_relaxed(USART_RQR_TXFRQ | USART_RQR_RXFRQ, + port->membase + ofs->rqr); cr1 = USART_CR1_TE | USART_CR1_RE; if (stm32_port->fifoen) cr1 |= USART_CR1_FIFOEN; cr2 = 0; + + /* Tx and RX FIFO configuration */ cr3 = readl_relaxed(port->membase + ofs->cr3); - cr3 &= USART_CR3_TXFTIE | USART_CR3_RXFTCFG_MASK | USART_CR3_RXFTIE - | USART_CR3_TXFTCFG_MASK; + cr3 &= USART_CR3_TXFTIE | USART_CR3_RXFTIE; + if (stm32_port->fifoen) { + cr3 &= ~(USART_CR3_TXFTCFG_MASK | USART_CR3_RXFTCFG_MASK); + cr3 |= USART_CR3_TXFTCFG_HALF << USART_CR3_TXFTCFG_SHIFT; + cr3 |= USART_CR3_RXFTCFG_HALF << USART_CR3_RXFTCFG_SHIFT; + } if (cflag & CSTOPB) cr2 |= USART_CR2_STOP_2B; @@ -817,12 +841,6 @@ static void stm32_usart_set_termios(struct uart_port *port, cr3 |= USART_CR3_CTSE | USART_CR3_RTSE; } - /* Handle modem control interrupts */ - if (UART_ENABLE_MS(port, termios->c_cflag)) - stm32_usart_enable_ms(port); - else - stm32_usart_disable_ms(port); - usartdiv = DIV_ROUND_CLOSEST(port->uartclk, baud); /* @@ -892,12 +910,24 @@ static void stm32_usart_set_termios(struct uart_port *port, cr1 &= ~(USART_CR1_DEDT_MASK | USART_CR1_DEAT_MASK); } + /* Configure wake up from low power on start bit detection */ + if (stm32_port->wakeirq > 0) { + cr3 &= ~USART_CR3_WUS_MASK; + cr3 |= USART_CR3_WUS_START_BIT; + } + writel_relaxed(cr3, port->membase + ofs->cr3); writel_relaxed(cr2, port->membase + ofs->cr2); writel_relaxed(cr1, port->membase + ofs->cr1); stm32_usart_set_bits(port, ofs->cr1, BIT(cfg->uart_enable_bit)); spin_unlock_irqrestore(&port->lock, flags); + + /* Handle modem control interrupts */ + if (UART_ENABLE_MS(port, termios->c_cflag)) + stm32_usart_enable_ms(port); + else + stm32_usart_disable_ms(port); } static const char *stm32_usart_type(struct uart_port *port) @@ -1252,10 +1282,6 @@ static int stm32_usart_serial_probe(struct platform_device *pdev) device_set_wakeup_enable(&pdev->dev, false); } - ret = uart_add_one_port(&stm32_usart_driver, &stm32port->port); - if (ret) - goto err_wirq; - ret = stm32_usart_of_dma_rx_probe(stm32port, pdev); if (ret) dev_info(&pdev->dev, "interrupt mode used for rx (no dma)\n"); @@ -1269,11 +1295,40 @@ static int stm32_usart_serial_probe(struct platform_device *pdev) pm_runtime_get_noresume(&pdev->dev); pm_runtime_set_active(&pdev->dev); pm_runtime_enable(&pdev->dev); + + ret = uart_add_one_port(&stm32_usart_driver, &stm32port->port); + if (ret) + goto err_port; + pm_runtime_put_sync(&pdev->dev); return 0; -err_wirq: +err_port: + pm_runtime_disable(&pdev->dev); + pm_runtime_set_suspended(&pdev->dev); + pm_runtime_put_noidle(&pdev->dev); + + if (stm32port->rx_ch) { + dmaengine_terminate_async(stm32port->rx_ch); + dma_release_channel(stm32port->rx_ch); + } + + if (stm32port->rx_dma_buf) + dma_free_coherent(&pdev->dev, + RX_BUF_L, stm32port->rx_buf, + stm32port->rx_dma_buf); + + if (stm32port->tx_ch) { + dmaengine_terminate_async(stm32port->tx_ch); + dma_release_channel(stm32port->tx_ch); + } + + if (stm32port->tx_dma_buf) + dma_free_coherent(&pdev->dev, + TX_BUF_L, stm32port->tx_buf, + stm32port->tx_dma_buf); + if (stm32port->wakeirq > 0) dev_pm_clear_wake_irq(&pdev->dev); @@ -1295,11 +1350,20 @@ static int stm32_usart_serial_remove(struct platform_device *pdev) int err; pm_runtime_get_sync(&pdev->dev); + err = uart_remove_one_port(&stm32_usart_driver, port); + if (err) + return(err); + + pm_runtime_disable(&pdev->dev); + pm_runtime_set_suspended(&pdev->dev); + pm_runtime_put_noidle(&pdev->dev); stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAR); - if (stm32_port->rx_ch) + if (stm32_port->rx_ch) { + dmaengine_terminate_async(stm32_port->rx_ch); dma_release_channel(stm32_port->rx_ch); + } if (stm32_port->rx_dma_buf) dma_free_coherent(&pdev->dev, @@ -1308,8 +1372,10 @@ static int stm32_usart_serial_remove(struct platform_device *pdev) stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAT); - if (stm32_port->tx_ch) + if (stm32_port->tx_ch) { + dmaengine_terminate_async(stm32_port->tx_ch); dma_release_channel(stm32_port->tx_ch); + } if (stm32_port->tx_dma_buf) dma_free_coherent(&pdev->dev, @@ -1323,12 +1389,7 @@ static int stm32_usart_serial_remove(struct platform_device *pdev) stm32_usart_deinit_port(stm32_port); - err = uart_remove_one_port(&stm32_usart_driver, port); - - pm_runtime_disable(&pdev->dev); - pm_runtime_put_noidle(&pdev->dev); - - return err; + return 0; } #ifdef CONFIG_SERIAL_STM32_CONSOLE @@ -1436,23 +1497,20 @@ static void __maybe_unused stm32_usart_serial_en_wakeup(struct uart_port *port, { struct stm32_port *stm32_port = to_stm32_port(port); const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs; - const struct stm32_usart_config *cfg = &stm32_port->info->cfg; - u32 val; if (stm32_port->wakeirq <= 0) return; + /* + * Enable low-power wake-up and wake-up irq if argument is set to + * "enable", disable low-power wake-up and wake-up irq otherwise + */ if (enable) { - stm32_usart_clr_bits(port, ofs->cr1, BIT(cfg->uart_enable_bit)); stm32_usart_set_bits(port, ofs->cr1, USART_CR1_UESM); - val = readl_relaxed(port->membase + ofs->cr3); - val &= ~USART_CR3_WUS_MASK; - /* Enable Wake up interrupt from low power on start bit */ - val |= USART_CR3_WUS_START_BIT | USART_CR3_WUFIE; - writel_relaxed(val, port->membase + ofs->cr3); - stm32_usart_set_bits(port, ofs->cr1, BIT(cfg->uart_enable_bit)); + stm32_usart_set_bits(port, ofs->cr3, USART_CR3_WUFIE); } else { stm32_usart_clr_bits(port, ofs->cr1, USART_CR1_UESM); + stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_WUFIE); } } diff --git a/drivers/tty/serial/stm32-usart.h b/drivers/tty/serial/stm32-usart.h index cb4f327c46db..94b568aa46bb 100644 --- a/drivers/tty/serial/stm32-usart.h +++ b/drivers/tty/serial/stm32-usart.h @@ -127,9 +127,6 @@ struct stm32_usart_info stm32h7_info = { /* Dummy bits */ #define USART_SR_DUMMY_RX BIT(16) -/* USART_ICR (F7) */ -#define USART_CR_TC BIT(6) - /* USART_DR */ #define USART_DR_MASK GENMASK(8, 0) diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c index 391bada4cedb..adbcbfa11b29 100644 --- a/drivers/tty/tty_io.c +++ b/drivers/tty/tty_io.c @@ -2530,14 +2530,14 @@ static int send_break(struct tty_struct *tty, unsigned int duration) * @p: pointer to result * * Obtain the modem status bits from the tty driver if the feature - * is supported. Return -EINVAL if it is not available. + * is supported. Return -ENOTTY if it is not available. * * Locking: none (up to the driver) */ static int tty_tiocmget(struct tty_struct *tty, int __user *p) { - int retval = -EINVAL; + int retval = -ENOTTY; if (tty->ops->tiocmget) { retval = tty->ops->tiocmget(tty); @@ -2555,7 +2555,7 @@ static int tty_tiocmget(struct tty_struct *tty, int __user *p) * @p: pointer to desired bits * * Set the modem status bits from the tty driver if the feature - * is supported. Return -EINVAL if it is not available. + * is supported. Return -ENOTTY if it is not available. * * Locking: none (up to the driver) */ @@ -2567,7 +2567,7 @@ static int tty_tiocmset(struct tty_struct *tty, unsigned int cmd, unsigned int set, clear, val; if (tty->ops->tiocmset == NULL) - return -EINVAL; + return -ENOTTY; retval = get_user(val, p); if (retval) @@ -2607,7 +2607,7 @@ int tty_get_icount(struct tty_struct *tty, if (tty->ops->get_icount) return tty->ops->get_icount(tty, icount); else - return -EINVAL; + return -ENOTTY; } EXPORT_SYMBOL_GPL(tty_get_icount); diff --git a/drivers/tty/tty_ioctl.c b/drivers/tty/tty_ioctl.c index 4de1c6ddb8ff..803da2d111c8 100644 --- a/drivers/tty/tty_ioctl.c +++ b/drivers/tty/tty_ioctl.c @@ -774,8 +774,8 @@ int tty_mode_ioctl(struct tty_struct *tty, struct file *file, case TCSETX: case TCSETXW: case TCSETXF: - return -EINVAL; -#endif + return -ENOTTY; +#endif case TIOCGSOFTCAR: copy_termios(real_tty, &kterm); ret = put_user((kterm.c_cflag & CLOCAL) ? 1 : 0, diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index 284b07224c55..0cc360da5426 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -1381,6 +1381,7 @@ struct vc_data *vc_deallocate(unsigned int currcons) atomic_notifier_call_chain(&vt_notifier_list, VT_DEALLOCATE, ¶m); vcs_remove_sysfs(currcons); visual_deinit(vc); + con_free_unimap(vc); put_pid(vc->vt_pid); vc_uniscr_set(vc, NULL); kfree(vc->vc_screenbuf); diff --git a/drivers/usb/cdns3/cdnsp-gadget.c b/drivers/usb/cdns3/cdnsp-gadget.c index d7d4bdd57f46..56707b6b0f57 100644 --- a/drivers/usb/cdns3/cdnsp-gadget.c +++ b/drivers/usb/cdns3/cdnsp-gadget.c @@ -727,7 +727,7 @@ int cdnsp_reset_device(struct cdnsp_device *pdev) * are in Disabled state. */ for (i = 1; i < CDNSP_ENDPOINTS_NUM; ++i) - pdev->eps[i].ep_state |= EP_STOPPED; + pdev->eps[i].ep_state |= EP_STOPPED | EP_UNCONFIGURED; trace_cdnsp_handle_cmd_reset_dev(slot_ctx); @@ -942,6 +942,7 @@ static int cdnsp_gadget_ep_enable(struct usb_ep *ep, pep = to_cdnsp_ep(ep); pdev = pep->pdev; + pep->ep_state &= ~EP_UNCONFIGURED; if (dev_WARN_ONCE(pdev->dev, pep->ep_state & EP_ENABLED, "%s is already enabled\n", pep->name)) @@ -1023,9 +1024,13 @@ static int cdnsp_gadget_ep_disable(struct usb_ep *ep) goto finish; } - cdnsp_cmd_stop_ep(pdev, pep); pep->ep_state |= EP_DIS_IN_RROGRESS; - cdnsp_cmd_flush_ep(pdev, pep); + + /* Endpoint was unconfigured by Reset Device command. */ + if (!(pep->ep_state & EP_UNCONFIGURED)) { + cdnsp_cmd_stop_ep(pdev, pep); + cdnsp_cmd_flush_ep(pdev, pep); + } /* Remove all queued USB requests. */ while (!list_empty(&pep->pending_list)) { @@ -1043,10 +1048,12 @@ static int cdnsp_gadget_ep_disable(struct usb_ep *ep) cdnsp_endpoint_zero(pdev, pep); - ret = cdnsp_update_eps_configuration(pdev, pep); + if (!(pep->ep_state & EP_UNCONFIGURED)) + ret = cdnsp_update_eps_configuration(pdev, pep); + cdnsp_free_endpoint_rings(pdev, pep); - pep->ep_state &= ~EP_ENABLED; + pep->ep_state &= ~(EP_ENABLED | EP_UNCONFIGURED); pep->ep_state |= EP_STOPPED; finish: diff --git a/drivers/usb/cdns3/cdnsp-gadget.h b/drivers/usb/cdns3/cdnsp-gadget.h index 6bbb26548c04..783ca8ffde00 100644 --- a/drivers/usb/cdns3/cdnsp-gadget.h +++ b/drivers/usb/cdns3/cdnsp-gadget.h @@ -835,6 +835,7 @@ struct cdnsp_ep { #define EP_WEDGE BIT(4) #define EP0_HALTED_STATUS BIT(5) #define EP_HAS_STREAMS BIT(6) +#define EP_UNCONFIGURED BIT(7) bool skip; }; diff --git a/drivers/usb/class/cdc-acm.c b/drivers/usb/class/cdc-acm.c index 3fda1ec961d7..c103961c3fae 100644 --- a/drivers/usb/class/cdc-acm.c +++ b/drivers/usb/class/cdc-acm.c @@ -929,8 +929,7 @@ static int get_serial_info(struct tty_struct *tty, struct serial_struct *ss) { struct acm *acm = tty->driver_data; - ss->xmit_fifo_size = acm->writesize; - ss->baud_base = le32_to_cpu(acm->line.dwDTERate); + ss->line = acm->minor; ss->close_delay = jiffies_to_msecs(acm->port.close_delay) / 10; ss->closing_wait = acm->port.closing_wait == ASYNC_CLOSING_WAIT_NONE ? ASYNC_CLOSING_WAIT_NONE : @@ -942,7 +941,6 @@ static int set_serial_info(struct tty_struct *tty, struct serial_struct *ss) { struct acm *acm = tty->driver_data; unsigned int closing_wait, close_delay; - unsigned int old_closing_wait, old_close_delay; int retval = 0; close_delay = msecs_to_jiffies(ss->close_delay * 10); @@ -950,20 +948,12 @@ static int set_serial_info(struct tty_struct *tty, struct serial_struct *ss) ASYNC_CLOSING_WAIT_NONE : msecs_to_jiffies(ss->closing_wait * 10); - /* we must redo the rounding here, so that the values match */ - old_close_delay = jiffies_to_msecs(acm->port.close_delay) / 10; - old_closing_wait = acm->port.closing_wait == ASYNC_CLOSING_WAIT_NONE ? - ASYNC_CLOSING_WAIT_NONE : - jiffies_to_msecs(acm->port.closing_wait) / 10; - mutex_lock(&acm->port.mutex); if (!capable(CAP_SYS_ADMIN)) { - if ((ss->close_delay != old_close_delay) || - (ss->closing_wait != old_closing_wait)) + if ((close_delay != acm->port.close_delay) || + (closing_wait != acm->port.closing_wait)) retval = -EPERM; - else - retval = -EOPNOTSUPP; } else { acm->port.close_delay = close_delay; acm->port.closing_wait = closing_wait; @@ -1634,12 +1624,13 @@ static int acm_resume(struct usb_interface *intf) struct urb *urb; int rv = 0; - acm_unpoison_urbs(acm); spin_lock_irq(&acm->write_lock); if (--acm->susp_count) goto out; + acm_unpoison_urbs(acm); + if (tty_port_initialized(&acm->port)) { rv = usb_submit_urb(acm->ctrlurb, GFP_ATOMIC); diff --git a/drivers/usb/class/cdc-wdm.c b/drivers/usb/class/cdc-wdm.c index 508b1c3f8b73..d1e4a7379beb 100644 --- a/drivers/usb/class/cdc-wdm.c +++ b/drivers/usb/class/cdc-wdm.c @@ -321,12 +321,23 @@ static void wdm_int_callback(struct urb *urb) } -static void kill_urbs(struct wdm_device *desc) +static void poison_urbs(struct wdm_device *desc) { /* the order here is essential */ - usb_kill_urb(desc->command); - usb_kill_urb(desc->validity); - usb_kill_urb(desc->response); + usb_poison_urb(desc->command); + usb_poison_urb(desc->validity); + usb_poison_urb(desc->response); +} + +static void unpoison_urbs(struct wdm_device *desc) +{ + /* + * the order here is not essential + * it is symmetrical just to be nice + */ + usb_unpoison_urb(desc->response); + usb_unpoison_urb(desc->validity); + usb_unpoison_urb(desc->command); } static void free_urbs(struct wdm_device *desc) @@ -741,11 +752,12 @@ static int wdm_release(struct inode *inode, struct file *file) if (!desc->count) { if (!test_bit(WDM_DISCONNECTING, &desc->flags)) { dev_dbg(&desc->intf->dev, "wdm_release: cleanup\n"); - kill_urbs(desc); + poison_urbs(desc); spin_lock_irq(&desc->iuspin); desc->resp_count = 0; spin_unlock_irq(&desc->iuspin); desc->manage_power(desc->intf, 0); + unpoison_urbs(desc); } else { /* must avoid dev_printk here as desc->intf is invalid */ pr_debug(KBUILD_MODNAME " %s: device gone - cleaning up\n", __func__); @@ -1037,9 +1049,9 @@ static void wdm_disconnect(struct usb_interface *intf) wake_up_all(&desc->wait); mutex_lock(&desc->rlock); mutex_lock(&desc->wlock); + poison_urbs(desc); cancel_work_sync(&desc->rxwork); cancel_work_sync(&desc->service_outs_intr); - kill_urbs(desc); mutex_unlock(&desc->wlock); mutex_unlock(&desc->rlock); @@ -1080,9 +1092,10 @@ static int wdm_suspend(struct usb_interface *intf, pm_message_t message) set_bit(WDM_SUSPENDING, &desc->flags); spin_unlock_irq(&desc->iuspin); /* callback submits work - order is essential */ - kill_urbs(desc); + poison_urbs(desc); cancel_work_sync(&desc->rxwork); cancel_work_sync(&desc->service_outs_intr); + unpoison_urbs(desc); } if (!PMSG_IS_AUTO(message)) { mutex_unlock(&desc->wlock); @@ -1140,7 +1153,7 @@ static int wdm_pre_reset(struct usb_interface *intf) wake_up_all(&desc->wait); mutex_lock(&desc->rlock); mutex_lock(&desc->wlock); - kill_urbs(desc); + poison_urbs(desc); cancel_work_sync(&desc->rxwork); cancel_work_sync(&desc->service_outs_intr); return 0; @@ -1151,6 +1164,7 @@ static int wdm_post_reset(struct usb_interface *intf) struct wdm_device *desc = wdm_find_device(intf); int rv; + unpoison_urbs(desc); clear_bit(WDM_OVERFLOW, &desc->flags); clear_bit(WDM_RESETTING, &desc->flags); rv = recover_from_urb_loss(desc); diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c index 7f71218cc1e5..13fe37fbbd2c 100644 --- a/drivers/usb/core/hub.c +++ b/drivers/usb/core/hub.c @@ -3556,7 +3556,7 @@ int usb_port_resume(struct usb_device *udev, pm_message_t msg) u16 portchange, portstatus; if (!test_and_set_bit(port1, hub->child_usage_bits)) { - status = pm_runtime_get_sync(&port_dev->dev); + status = pm_runtime_resume_and_get(&port_dev->dev); if (status < 0) { dev_dbg(&udev->dev, "can't resume usb port, status %d\n", status); @@ -3593,9 +3593,6 @@ int usb_port_resume(struct usb_device *udev, pm_message_t msg) * sequence. */ status = hub_port_status(hub, port1, &portstatus, &portchange); - - /* TRSMRCY = 10 msec */ - msleep(10); } SuspendCleared: @@ -3610,6 +3607,9 @@ int usb_port_resume(struct usb_device *udev, pm_message_t msg) usb_clear_port_feature(hub->hdev, port1, USB_PORT_FEAT_C_SUSPEND); } + + /* TRSMRCY = 10 msec */ + msleep(10); } if (udev->persist_enabled) diff --git a/drivers/usb/core/quirks.c b/drivers/usb/core/quirks.c index 76ac5d6555ae..21e7522655ac 100644 --- a/drivers/usb/core/quirks.c +++ b/drivers/usb/core/quirks.c @@ -406,6 +406,7 @@ static const struct usb_device_id usb_quirk_list[] = { /* Realtek hub in Dell WD19 (Type-C) */ { USB_DEVICE(0x0bda, 0x0487), .driver_info = USB_QUIRK_NO_LPM }, + { USB_DEVICE(0x0bda, 0x5487), .driver_info = USB_QUIRK_RESET_RESUME }, /* Generic RTL8153 based ethernet adapters */ { USB_DEVICE(0x0bda, 0x8153), .driver_info = USB_QUIRK_NO_LPM }, @@ -438,6 +439,9 @@ static const struct usb_device_id usb_quirk_list[] = { { USB_DEVICE(0x17ef, 0xa012), .driver_info = USB_QUIRK_DISCONNECT_SUSPEND }, + /* Lenovo ThinkPad USB-C Dock Gen2 Ethernet (RTL8153 GigE) */ + { USB_DEVICE(0x17ef, 0xa387), .driver_info = USB_QUIRK_NO_LPM }, + /* BUILDWIN Photo Frame */ { USB_DEVICE(0x1908, 0x1315), .driver_info = USB_QUIRK_HONOR_BNUMINTERFACES }, diff --git a/drivers/usb/dwc2/core.h b/drivers/usb/dwc2/core.h index 7161344c6522..641e4251cb7f 100644 --- a/drivers/usb/dwc2/core.h +++ b/drivers/usb/dwc2/core.h @@ -112,6 +112,7 @@ struct dwc2_hsotg_req; * @debugfs: File entry for debugfs file for this endpoint. * @dir_in: Set to true if this endpoint is of the IN direction, which * means that it is sending data to the Host. + * @map_dir: Set to the value of dir_in when the DMA buffer is mapped. * @index: The index for the endpoint registers. * @mc: Multi Count - number of transactions per microframe * @interval: Interval for periodic endpoints, in frames or microframes. @@ -161,6 +162,7 @@ struct dwc2_hsotg_ep { unsigned short fifo_index; unsigned char dir_in; + unsigned char map_dir; unsigned char index; unsigned char mc; u16 interval; diff --git a/drivers/usb/dwc2/core_intr.c b/drivers/usb/dwc2/core_intr.c index 55f1d14fc414..510fd0572feb 100644 --- a/drivers/usb/dwc2/core_intr.c +++ b/drivers/usb/dwc2/core_intr.c @@ -307,6 +307,7 @@ static void dwc2_handle_conn_id_status_change_intr(struct dwc2_hsotg *hsotg) static void dwc2_handle_session_req_intr(struct dwc2_hsotg *hsotg) { int ret; + u32 hprt0; /* Clear interrupt */ dwc2_writel(hsotg, GINTSTS_SESSREQINT, GINTSTS); @@ -327,6 +328,13 @@ static void dwc2_handle_session_req_intr(struct dwc2_hsotg *hsotg) * established */ dwc2_hsotg_disconnect(hsotg); + } else { + /* Turn on the port power bit. */ + hprt0 = dwc2_read_hprt0(hsotg); + hprt0 |= HPRT0_PWR; + dwc2_writel(hsotg, hprt0, HPRT0); + /* Connect hcd after port power is set. */ + dwc2_hcd_connect(hsotg); } } @@ -652,6 +660,71 @@ static u32 dwc2_read_common_intr(struct dwc2_hsotg *hsotg) return 0; } +/** + * dwc_handle_gpwrdn_disc_det() - Handles the gpwrdn disconnect detect. + * Exits hibernation without restoring registers. + * + * @hsotg: Programming view of DWC_otg controller + * @gpwrdn: GPWRDN register + */ +static inline void dwc_handle_gpwrdn_disc_det(struct dwc2_hsotg *hsotg, + u32 gpwrdn) +{ + u32 gpwrdn_tmp; + + /* Switch-on voltage to the core */ + gpwrdn_tmp = dwc2_readl(hsotg, GPWRDN); + gpwrdn_tmp &= ~GPWRDN_PWRDNSWTCH; + dwc2_writel(hsotg, gpwrdn_tmp, GPWRDN); + udelay(5); + + /* Reset core */ + gpwrdn_tmp = dwc2_readl(hsotg, GPWRDN); + gpwrdn_tmp &= ~GPWRDN_PWRDNRSTN; + dwc2_writel(hsotg, gpwrdn_tmp, GPWRDN); + udelay(5); + + /* Disable Power Down Clamp */ + gpwrdn_tmp = dwc2_readl(hsotg, GPWRDN); + gpwrdn_tmp &= ~GPWRDN_PWRDNCLMP; + dwc2_writel(hsotg, gpwrdn_tmp, GPWRDN); + udelay(5); + + /* Deassert reset core */ + gpwrdn_tmp = dwc2_readl(hsotg, GPWRDN); + gpwrdn_tmp |= GPWRDN_PWRDNRSTN; + dwc2_writel(hsotg, gpwrdn_tmp, GPWRDN); + udelay(5); + + /* Disable PMU interrupt */ + gpwrdn_tmp = dwc2_readl(hsotg, GPWRDN); + gpwrdn_tmp &= ~GPWRDN_PMUINTSEL; + dwc2_writel(hsotg, gpwrdn_tmp, GPWRDN); + + /* De-assert Wakeup Logic */ + gpwrdn_tmp = dwc2_readl(hsotg, GPWRDN); + gpwrdn_tmp &= ~GPWRDN_PMUACTV; + dwc2_writel(hsotg, gpwrdn_tmp, GPWRDN); + + hsotg->hibernated = 0; + hsotg->bus_suspended = 0; + + if (gpwrdn & GPWRDN_IDSTS) { + hsotg->op_state = OTG_STATE_B_PERIPHERAL; + dwc2_core_init(hsotg, false); + dwc2_enable_global_interrupts(hsotg); + dwc2_hsotg_core_init_disconnected(hsotg, false); + dwc2_hsotg_core_connect(hsotg); + } else { + hsotg->op_state = OTG_STATE_A_HOST; + + /* Initialize the Core for Host mode */ + dwc2_core_init(hsotg, false); + dwc2_enable_global_interrupts(hsotg); + dwc2_hcd_start(hsotg); + } +} + /* * GPWRDN interrupt handler. * @@ -673,64 +746,14 @@ static void dwc2_handle_gpwrdn_intr(struct dwc2_hsotg *hsotg) if ((gpwrdn & GPWRDN_DISCONN_DET) && (gpwrdn & GPWRDN_DISCONN_DET_MSK) && !linestate) { - u32 gpwrdn_tmp; - dev_dbg(hsotg->dev, "%s: GPWRDN_DISCONN_DET\n", __func__); - - /* Switch-on voltage to the core */ - gpwrdn_tmp = dwc2_readl(hsotg, GPWRDN); - gpwrdn_tmp &= ~GPWRDN_PWRDNSWTCH; - dwc2_writel(hsotg, gpwrdn_tmp, GPWRDN); - udelay(10); - - /* Reset core */ - gpwrdn_tmp = dwc2_readl(hsotg, GPWRDN); - gpwrdn_tmp &= ~GPWRDN_PWRDNRSTN; - dwc2_writel(hsotg, gpwrdn_tmp, GPWRDN); - udelay(10); - - /* Disable Power Down Clamp */ - gpwrdn_tmp = dwc2_readl(hsotg, GPWRDN); - gpwrdn_tmp &= ~GPWRDN_PWRDNCLMP; - dwc2_writel(hsotg, gpwrdn_tmp, GPWRDN); - udelay(10); - - /* Deassert reset core */ - gpwrdn_tmp = dwc2_readl(hsotg, GPWRDN); - gpwrdn_tmp |= GPWRDN_PWRDNRSTN; - dwc2_writel(hsotg, gpwrdn_tmp, GPWRDN); - udelay(10); - - /* Disable PMU interrupt */ - gpwrdn_tmp = dwc2_readl(hsotg, GPWRDN); - gpwrdn_tmp &= ~GPWRDN_PMUINTSEL; - dwc2_writel(hsotg, gpwrdn_tmp, GPWRDN); - - /* De-assert Wakeup Logic */ - gpwrdn_tmp = dwc2_readl(hsotg, GPWRDN); - gpwrdn_tmp &= ~GPWRDN_PMUACTV; - dwc2_writel(hsotg, gpwrdn_tmp, GPWRDN); - - hsotg->hibernated = 0; - - if (gpwrdn & GPWRDN_IDSTS) { - hsotg->op_state = OTG_STATE_B_PERIPHERAL; - dwc2_core_init(hsotg, false); - dwc2_enable_global_interrupts(hsotg); - dwc2_hsotg_core_init_disconnected(hsotg, false); - dwc2_hsotg_core_connect(hsotg); - } else { - hsotg->op_state = OTG_STATE_A_HOST; - - /* Initialize the Core for Host mode */ - dwc2_core_init(hsotg, false); - dwc2_enable_global_interrupts(hsotg); - dwc2_hcd_start(hsotg); - } - } - - if ((gpwrdn & GPWRDN_LNSTSCHG) && - (gpwrdn & GPWRDN_LNSTSCHG_MSK) && linestate) { + /* + * Call disconnect detect function to exit from + * hibernation + */ + dwc_handle_gpwrdn_disc_det(hsotg, gpwrdn); + } else if ((gpwrdn & GPWRDN_LNSTSCHG) && + (gpwrdn & GPWRDN_LNSTSCHG_MSK) && linestate) { dev_dbg(hsotg->dev, "%s: GPWRDN_LNSTSCHG\n", __func__); if (hsotg->hw_params.hibernation && hsotg->hibernated) { @@ -741,24 +764,21 @@ static void dwc2_handle_gpwrdn_intr(struct dwc2_hsotg *hsotg) dwc2_exit_hibernation(hsotg, 1, 0, 1); } } - } - if ((gpwrdn & GPWRDN_RST_DET) && (gpwrdn & GPWRDN_RST_DET_MSK)) { + } else if ((gpwrdn & GPWRDN_RST_DET) && + (gpwrdn & GPWRDN_RST_DET_MSK)) { dev_dbg(hsotg->dev, "%s: GPWRDN_RST_DET\n", __func__); if (!linestate && (gpwrdn & GPWRDN_BSESSVLD)) dwc2_exit_hibernation(hsotg, 0, 1, 0); - } - if ((gpwrdn & GPWRDN_STS_CHGINT) && - (gpwrdn & GPWRDN_STS_CHGINT_MSK) && linestate) { + } else if ((gpwrdn & GPWRDN_STS_CHGINT) && + (gpwrdn & GPWRDN_STS_CHGINT_MSK)) { dev_dbg(hsotg->dev, "%s: GPWRDN_STS_CHGINT\n", __func__); - if (hsotg->hw_params.hibernation && - hsotg->hibernated) { - if (gpwrdn & GPWRDN_IDSTS) { - dwc2_exit_hibernation(hsotg, 0, 0, 0); - call_gadget(hsotg, resume); - } else { - dwc2_exit_hibernation(hsotg, 1, 0, 1); - } - } + /* + * As GPWRDN_STS_CHGINT exit from hibernation flow is + * the same as in GPWRDN_DISCONN_DET flow. Call + * disconnect detect helper function to exit from + * hibernation. + */ + dwc_handle_gpwrdn_disc_det(hsotg, gpwrdn); } } diff --git a/drivers/usb/dwc2/gadget.c b/drivers/usb/dwc2/gadget.c index ad4c94366dad..d2f623d83bf7 100644 --- a/drivers/usb/dwc2/gadget.c +++ b/drivers/usb/dwc2/gadget.c @@ -422,7 +422,7 @@ static void dwc2_hsotg_unmap_dma(struct dwc2_hsotg *hsotg, { struct usb_request *req = &hs_req->req; - usb_gadget_unmap_request(&hsotg->gadget, req, hs_ep->dir_in); + usb_gadget_unmap_request(&hsotg->gadget, req, hs_ep->map_dir); } /* @@ -1242,6 +1242,7 @@ static int dwc2_hsotg_map_dma(struct dwc2_hsotg *hsotg, { int ret; + hs_ep->map_dir = hs_ep->dir_in; ret = usb_gadget_map_request(&hsotg->gadget, req, hs_ep->dir_in); if (ret) goto dma_error; diff --git a/drivers/usb/dwc2/hcd.c b/drivers/usb/dwc2/hcd.c index 1a9789ec5847..6af1dcbc3656 100644 --- a/drivers/usb/dwc2/hcd.c +++ b/drivers/usb/dwc2/hcd.c @@ -5580,7 +5580,15 @@ int dwc2_host_exit_hibernation(struct dwc2_hsotg *hsotg, int rem_wakeup, return ret; } - dwc2_hcd_rem_wakeup(hsotg); + if (rem_wakeup) { + dwc2_hcd_rem_wakeup(hsotg); + /* + * Change "port_connect_status_change" flag to re-enumerate, + * because after exit from hibernation port connection status + * is not detected. + */ + hsotg->flags.b.port_connect_status_change = 1; + } hsotg->hibernated = 0; hsotg->bus_suspended = 0; diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c index f2448d0a9d39..126f0e10b3ef 100644 --- a/drivers/usb/dwc3/core.c +++ b/drivers/usb/dwc3/core.c @@ -114,6 +114,8 @@ void dwc3_set_prtcap(struct dwc3 *dwc, u32 mode) dwc->current_dr_role = mode; } +static int dwc3_core_soft_reset(struct dwc3 *dwc); + static void __dwc3_set_mode(struct work_struct *work) { struct dwc3 *dwc = work_to_dwc(work); @@ -121,6 +123,8 @@ static void __dwc3_set_mode(struct work_struct *work) int ret; u32 reg; + mutex_lock(&dwc->mutex); + pm_runtime_get_sync(dwc->dev); if (dwc->current_dr_role == DWC3_GCTL_PRTCAP_OTG) @@ -154,6 +158,25 @@ static void __dwc3_set_mode(struct work_struct *work) break; } + /* For DRD host or device mode only */ + if (dwc->desired_dr_role != DWC3_GCTL_PRTCAP_OTG) { + reg = dwc3_readl(dwc->regs, DWC3_GCTL); + reg |= DWC3_GCTL_CORESOFTRESET; + dwc3_writel(dwc->regs, DWC3_GCTL, reg); + + /* + * Wait for internal clocks to synchronized. DWC_usb31 and + * DWC_usb32 may need at least 50ms (less for DWC_usb3). To + * keep it consistent across different IPs, let's wait up to + * 100ms before clearing GCTL.CORESOFTRESET. + */ + msleep(100); + + reg = dwc3_readl(dwc->regs, DWC3_GCTL); + reg &= ~DWC3_GCTL_CORESOFTRESET; + dwc3_writel(dwc->regs, DWC3_GCTL, reg); + } + spin_lock_irqsave(&dwc->lock, flags); dwc3_set_prtcap(dwc, dwc->desired_dr_role); @@ -178,6 +201,8 @@ static void __dwc3_set_mode(struct work_struct *work) } break; case DWC3_GCTL_PRTCAP_DEVICE: + dwc3_core_soft_reset(dwc); + dwc3_event_buffers_setup(dwc); if (dwc->usb2_phy) @@ -200,6 +225,7 @@ static void __dwc3_set_mode(struct work_struct *work) out: pm_runtime_mark_last_busy(dwc->dev); pm_runtime_put_autosuspend(dwc->dev); + mutex_unlock(&dwc->mutex); } void dwc3_set_mode(struct dwc3 *dwc, u32 mode) @@ -1277,6 +1303,8 @@ static void dwc3_get_properties(struct dwc3 *dwc) "snps,usb3_lpm_capable"); dwc->usb2_lpm_disable = device_property_read_bool(dev, "snps,usb2-lpm-disable"); + dwc->usb2_gadget_lpm_disable = device_property_read_bool(dev, + "snps,usb2-gadget-lpm-disable"); device_property_read_u8(dev, "snps,rx-thr-num-pkt-prd", &rx_thr_num_pkt_prd); device_property_read_u8(dev, "snps,rx-max-burst-prd", @@ -1543,6 +1571,7 @@ static int dwc3_probe(struct platform_device *pdev) dwc3_cache_hwparams(dwc); spin_lock_init(&dwc->lock); + mutex_init(&dwc->mutex); pm_runtime_set_active(dev); pm_runtime_use_autosuspend(dev); diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h index 052b20d52651..453cfebd4d04 100644 --- a/drivers/usb/dwc3/core.h +++ b/drivers/usb/dwc3/core.h @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -946,6 +947,7 @@ struct dwc3_scratchpad_array { * @scratch_addr: dma address of scratchbuf * @ep0_in_setup: one control transfer is completed and enter setup phase * @lock: for synchronizing + * @mutex: for mode switching * @dev: pointer to our struct device * @sysdev: pointer to the DMA-capable device * @xhci: pointer to our xHCI child @@ -1034,7 +1036,8 @@ struct dwc3_scratchpad_array { * @dis_start_transfer_quirk: set if start_transfer failure SW workaround is * not needed for DWC_usb31 version 1.70a-ea06 and below * @usb3_lpm_capable: set if hadrware supports Link Power Management - * @usb2_lpm_disable: set to disable usb2 lpm + * @usb2_lpm_disable: set to disable usb2 lpm for host + * @usb2_gadget_lpm_disable: set to disable usb2 lpm for gadget * @disable_scramble_quirk: set if we enable the disable scramble quirk * @u2exit_lfps_quirk: set if we enable u2exit lfps quirk * @u2ss_inp3_quirk: set if we enable P3 OK for U2/SS Inactive quirk @@ -1085,6 +1088,9 @@ struct dwc3 { /* device lock */ spinlock_t lock; + /* mode switching lock */ + struct mutex mutex; + struct device *dev; struct device *sysdev; @@ -1238,6 +1244,7 @@ struct dwc3 { unsigned dis_start_transfer_quirk:1; unsigned usb3_lpm_capable:1; unsigned usb2_lpm_disable:1; + unsigned usb2_gadget_lpm_disable:1; unsigned disable_scramble_quirk:1; unsigned u2exit_lfps_quirk:1; diff --git a/drivers/usb/dwc3/dwc3-imx8mp.c b/drivers/usb/dwc3/dwc3-imx8mp.c index 75f0042b998b..84c1a4ac2444 100644 --- a/drivers/usb/dwc3/dwc3-imx8mp.c +++ b/drivers/usb/dwc3/dwc3-imx8mp.c @@ -167,6 +167,7 @@ static int dwc3_imx8mp_probe(struct platform_device *pdev) dwc3_np = of_get_child_by_name(node, "dwc3"); if (!dwc3_np) { + err = -ENODEV; dev_err(dev, "failed to find dwc3 core child\n"); goto disable_rpm; } diff --git a/drivers/usb/dwc3/dwc3-omap.c b/drivers/usb/dwc3/dwc3-omap.c index 3db17806e92e..e196673f5c64 100644 --- a/drivers/usb/dwc3/dwc3-omap.c +++ b/drivers/usb/dwc3/dwc3-omap.c @@ -437,8 +437,13 @@ static int dwc3_omap_extcon_register(struct dwc3_omap *omap) if (extcon_get_state(edev, EXTCON_USB) == true) dwc3_omap_set_mailbox(omap, OMAP_DWC3_VBUS_VALID); + else + dwc3_omap_set_mailbox(omap, OMAP_DWC3_VBUS_OFF); + if (extcon_get_state(edev, EXTCON_USB_HOST) == true) dwc3_omap_set_mailbox(omap, OMAP_DWC3_ID_GROUND); + else + dwc3_omap_set_mailbox(omap, OMAP_DWC3_ID_FLOAT); omap->edev = edev; } diff --git a/drivers/usb/dwc3/dwc3-pci.c b/drivers/usb/dwc3/dwc3-pci.c index 4c5c6972124a..19789e94bbd0 100644 --- a/drivers/usb/dwc3/dwc3-pci.c +++ b/drivers/usb/dwc3/dwc3-pci.c @@ -41,6 +41,7 @@ #define PCI_DEVICE_ID_INTEL_TGPH 0x43ee #define PCI_DEVICE_ID_INTEL_JSP 0x4dee #define PCI_DEVICE_ID_INTEL_ADLP 0x51ee +#define PCI_DEVICE_ID_INTEL_ADLM 0x54ee #define PCI_DEVICE_ID_INTEL_ADLS 0x7ae1 #define PCI_DEVICE_ID_INTEL_TGL 0x9a15 @@ -122,6 +123,7 @@ static const struct property_entry dwc3_pci_mrfld_properties[] = { PROPERTY_ENTRY_STRING("linux,extcon-name", "mrfld_bcove_pwrsrc"), PROPERTY_ENTRY_BOOL("snps,dis_u3_susphy_quirk"), PROPERTY_ENTRY_BOOL("snps,dis_u2_susphy_quirk"), + PROPERTY_ENTRY_BOOL("snps,usb2-gadget-lpm-disable"), PROPERTY_ENTRY_BOOL("linux,sysdev_is_parent"), {} }; @@ -388,6 +390,9 @@ static const struct pci_device_id dwc3_pci_id_table[] = { { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ADLP), (kernel_ulong_t) &dwc3_pci_intel_swnode, }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ADLM), + (kernel_ulong_t) &dwc3_pci_intel_swnode, }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ADLS), (kernel_ulong_t) &dwc3_pci_intel_swnode, }, diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index c7ef218e7a8c..8585b56d9f2d 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -308,13 +308,12 @@ int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd, } if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) { - int needs_wakeup; + int link_state; - needs_wakeup = (dwc->link_state == DWC3_LINK_STATE_U1 || - dwc->link_state == DWC3_LINK_STATE_U2 || - dwc->link_state == DWC3_LINK_STATE_U3); - - if (unlikely(needs_wakeup)) { + link_state = dwc3_gadget_get_link_state(dwc); + if (link_state == DWC3_LINK_STATE_U1 || + link_state == DWC3_LINK_STATE_U2 || + link_state == DWC3_LINK_STATE_U3) { ret = __dwc3_gadget_wakeup(dwc); dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n", ret); @@ -608,12 +607,14 @@ static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action) u8 bInterval_m1; /* - * Valid range for DEPCFG.bInterval_m1 is from 0 to 13, and it - * must be set to 0 when the controller operates in full-speed. + * Valid range for DEPCFG.bInterval_m1 is from 0 to 13. + * + * NOTE: The programming guide incorrectly stated bInterval_m1 + * must be set to 0 when operating in fullspeed. Internally the + * controller does not have this limitation. See DWC_usb3x + * programming guide section 3.2.2.1. */ bInterval_m1 = min_t(u8, desc->bInterval - 1, 13); - if (dwc->gadget->speed == USB_SPEED_FULL) - bInterval_m1 = 0; if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT && dwc->gadget->speed == USB_SPEED_FULL) @@ -1675,7 +1676,9 @@ static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req) } } - return __dwc3_gadget_kick_transfer(dep); + __dwc3_gadget_kick_transfer(dep); + + return 0; } static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request, @@ -1973,6 +1976,8 @@ static int __dwc3_gadget_wakeup(struct dwc3 *dwc) case DWC3_LINK_STATE_RESET: case DWC3_LINK_STATE_RX_DET: /* in HS, means Early Suspend */ case DWC3_LINK_STATE_U3: /* in HS, means SUSPEND */ + case DWC3_LINK_STATE_U2: /* in HS, means Sleep (L1) */ + case DWC3_LINK_STATE_U1: case DWC3_LINK_STATE_RESUME: break; default: @@ -2299,6 +2304,10 @@ static void dwc3_gadget_enable_irq(struct dwc3 *dwc) if (DWC3_VER_IS_PRIOR(DWC3, 250A)) reg |= DWC3_DEVTEN_ULSTCNGEN; + /* On 2.30a and above this bit enables U3/L2-L1 Suspend Events */ + if (!DWC3_VER_IS_PRIOR(DWC3, 230A)) + reg |= DWC3_DEVTEN_EOPFEN; + dwc3_writel(dwc->regs, DWC3_DEVTEN, reg); } @@ -3322,6 +3331,15 @@ static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc) { u32 reg; + /* + * Ideally, dwc3_reset_gadget() would trigger the function + * drivers to stop any active transfers through ep disable. + * However, for functions which defer ep disable, such as mass + * storage, we will need to rely on the call to stop active + * transfers here, and avoid allowing of request queuing. + */ + dwc->connected = false; + /* * WORKAROUND: DWC3 revisions <1.88a have an issue which * would cause a missing Disconnect Event if there's a @@ -3460,6 +3478,7 @@ static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc) /* Enable USB2 LPM Capability */ if (!DWC3_VER_IS_WITHIN(DWC3, ANY, 194A) && + !dwc->usb2_gadget_lpm_disable && (speed != DWC3_DSTS_SUPERSPEED) && (speed != DWC3_DSTS_SUPERSPEED_PLUS)) { reg = dwc3_readl(dwc->regs, DWC3_DCFG); @@ -3486,6 +3505,12 @@ static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc) dwc3_gadget_dctl_write_safe(dwc, reg); } else { + if (dwc->usb2_gadget_lpm_disable) { + reg = dwc3_readl(dwc->regs, DWC3_DCFG); + reg &= ~DWC3_DCFG_LPM_CAP; + dwc3_writel(dwc->regs, DWC3_DCFG, reg); + } + reg = dwc3_readl(dwc->regs, DWC3_DCTL); reg &= ~DWC3_DCTL_HIRD_THRES_MASK; dwc3_gadget_dctl_write_safe(dwc, reg); @@ -3934,7 +3959,7 @@ int dwc3_gadget_init(struct dwc3 *dwc) dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN; dwc->gadget->sg_supported = true; dwc->gadget->name = "dwc3-gadget"; - dwc->gadget->lpm_capable = true; + dwc->gadget->lpm_capable = !dwc->usb2_gadget_lpm_disable; /* * FIXME We might be setting max_speed to gadget); + usb_del_gadget(dwc->gadget); dwc3_gadget_free_endpoints(dwc); + usb_put_gadget(dwc->gadget); dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce, dwc->bounce_addr); kfree(dwc->setup_buf); diff --git a/drivers/usb/gadget/config.c b/drivers/usb/gadget/config.c index 2d115353424c..8bb25773b61e 100644 --- a/drivers/usb/gadget/config.c +++ b/drivers/usb/gadget/config.c @@ -194,9 +194,13 @@ EXPORT_SYMBOL_GPL(usb_assign_descriptors); void usb_free_all_descriptors(struct usb_function *f) { usb_free_descriptors(f->fs_descriptors); + f->fs_descriptors = NULL; usb_free_descriptors(f->hs_descriptors); + f->hs_descriptors = NULL; usb_free_descriptors(f->ss_descriptors); + f->ss_descriptors = NULL; usb_free_descriptors(f->ssp_descriptors); + f->ssp_descriptors = NULL; } EXPORT_SYMBOL_GPL(usb_free_all_descriptors); diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c index 801a8b668a35..10a5d9f0f2b9 100644 --- a/drivers/usb/gadget/function/f_fs.c +++ b/drivers/usb/gadget/function/f_fs.c @@ -2640,6 +2640,7 @@ static int __ffs_data_got_strings(struct ffs_data *ffs, do { /* lang_count > 0 so we can use do-while */ unsigned needed = needed_count; + u32 str_per_lang = str_count; if (len < 3) goto error_free; @@ -2675,7 +2676,7 @@ static int __ffs_data_got_strings(struct ffs_data *ffs, data += length + 1; len -= length + 1; - } while (--str_count); + } while (--str_per_lang); s->id = 0; /* terminator */ s->s = NULL; diff --git a/drivers/usb/gadget/function/f_uac1.c b/drivers/usb/gadget/function/f_uac1.c index 560382e0a8f3..e65f474ad7b3 100644 --- a/drivers/usb/gadget/function/f_uac1.c +++ b/drivers/usb/gadget/function/f_uac1.c @@ -19,6 +19,9 @@ #include "u_audio.h" #include "u_uac1.h" +/* UAC1 spec: 3.7.2.3 Audio Channel Cluster Format */ +#define UAC1_CHANNEL_MASK 0x0FFF + struct f_uac1 { struct g_audio g_audio; u8 ac_intf, as_in_intf, as_out_intf; @@ -30,6 +33,11 @@ static inline struct f_uac1 *func_to_uac1(struct usb_function *f) return container_of(f, struct f_uac1, g_audio.func); } +static inline struct f_uac1_opts *g_audio_to_uac1_opts(struct g_audio *audio) +{ + return container_of(audio->func.fi, struct f_uac1_opts, func_inst); +} + /* * DESCRIPTORS ... most are static, but strings and full * configuration descriptors are built on demand. @@ -505,11 +513,42 @@ static void f_audio_disable(struct usb_function *f) /*-------------------------------------------------------------------------*/ +static int f_audio_validate_opts(struct g_audio *audio, struct device *dev) +{ + struct f_uac1_opts *opts = g_audio_to_uac1_opts(audio); + + if (!opts->p_chmask && !opts->c_chmask) { + dev_err(dev, "Error: no playback and capture channels\n"); + return -EINVAL; + } else if (opts->p_chmask & ~UAC1_CHANNEL_MASK) { + dev_err(dev, "Error: unsupported playback channels mask\n"); + return -EINVAL; + } else if (opts->c_chmask & ~UAC1_CHANNEL_MASK) { + dev_err(dev, "Error: unsupported capture channels mask\n"); + return -EINVAL; + } else if ((opts->p_ssize < 1) || (opts->p_ssize > 4)) { + dev_err(dev, "Error: incorrect playback sample size\n"); + return -EINVAL; + } else if ((opts->c_ssize < 1) || (opts->c_ssize > 4)) { + dev_err(dev, "Error: incorrect capture sample size\n"); + return -EINVAL; + } else if (!opts->p_srate) { + dev_err(dev, "Error: incorrect playback sampling rate\n"); + return -EINVAL; + } else if (!opts->c_srate) { + dev_err(dev, "Error: incorrect capture sampling rate\n"); + return -EINVAL; + } + + return 0; +} + /* audio function driver setup/binding */ static int f_audio_bind(struct usb_configuration *c, struct usb_function *f) { struct usb_composite_dev *cdev = c->cdev; struct usb_gadget *gadget = cdev->gadget; + struct device *dev = &gadget->dev; struct f_uac1 *uac1 = func_to_uac1(f); struct g_audio *audio = func_to_g_audio(f); struct f_uac1_opts *audio_opts; @@ -519,6 +558,10 @@ static int f_audio_bind(struct usb_configuration *c, struct usb_function *f) int rate; int status; + status = f_audio_validate_opts(audio, dev); + if (status) + return status; + audio_opts = container_of(f->fi, struct f_uac1_opts, func_inst); us = usb_gstrings_attach(cdev, uac1_strings, ARRAY_SIZE(strings_uac1)); diff --git a/drivers/usb/gadget/function/f_uac2.c b/drivers/usb/gadget/function/f_uac2.c index 6f03e944e0e3..dd960cea642f 100644 --- a/drivers/usb/gadget/function/f_uac2.c +++ b/drivers/usb/gadget/function/f_uac2.c @@ -14,6 +14,9 @@ #include "u_audio.h" #include "u_uac2.h" +/* UAC2 spec: 4.1 Audio Channel Cluster Descriptor */ +#define UAC2_CHANNEL_MASK 0x07FFFFFF + /* * The driver implements a simple UAC_2 topology. * USB-OUT -> IT_1 -> OT_3 -> ALSA_Capture @@ -604,6 +607,36 @@ static void setup_descriptor(struct f_uac2_opts *opts) hs_audio_desc[i] = NULL; } +static int afunc_validate_opts(struct g_audio *agdev, struct device *dev) +{ + struct f_uac2_opts *opts = g_audio_to_uac2_opts(agdev); + + if (!opts->p_chmask && !opts->c_chmask) { + dev_err(dev, "Error: no playback and capture channels\n"); + return -EINVAL; + } else if (opts->p_chmask & ~UAC2_CHANNEL_MASK) { + dev_err(dev, "Error: unsupported playback channels mask\n"); + return -EINVAL; + } else if (opts->c_chmask & ~UAC2_CHANNEL_MASK) { + dev_err(dev, "Error: unsupported capture channels mask\n"); + return -EINVAL; + } else if ((opts->p_ssize < 1) || (opts->p_ssize > 4)) { + dev_err(dev, "Error: incorrect playback sample size\n"); + return -EINVAL; + } else if ((opts->c_ssize < 1) || (opts->c_ssize > 4)) { + dev_err(dev, "Error: incorrect capture sample size\n"); + return -EINVAL; + } else if (!opts->p_srate) { + dev_err(dev, "Error: incorrect playback sampling rate\n"); + return -EINVAL; + } else if (!opts->c_srate) { + dev_err(dev, "Error: incorrect capture sampling rate\n"); + return -EINVAL; + } + + return 0; +} + static int afunc_bind(struct usb_configuration *cfg, struct usb_function *fn) { @@ -612,11 +645,13 @@ afunc_bind(struct usb_configuration *cfg, struct usb_function *fn) struct usb_composite_dev *cdev = cfg->cdev; struct usb_gadget *gadget = cdev->gadget; struct device *dev = &gadget->dev; - struct f_uac2_opts *uac2_opts; + struct f_uac2_opts *uac2_opts = g_audio_to_uac2_opts(agdev); struct usb_string *us; int ret; - uac2_opts = container_of(fn->fi, struct f_uac2_opts, func_inst); + ret = afunc_validate_opts(agdev, dev); + if (ret) + return ret; us = usb_gstrings_attach(cdev, fn_strings, ARRAY_SIZE(strings_fn)); if (IS_ERR(us)) diff --git a/drivers/usb/gadget/function/f_uvc.c b/drivers/usb/gadget/function/f_uvc.c index 44b4352a2676..f48a00e49794 100644 --- a/drivers/usb/gadget/function/f_uvc.c +++ b/drivers/usb/gadget/function/f_uvc.c @@ -633,7 +633,12 @@ uvc_function_bind(struct usb_configuration *c, struct usb_function *f) uvc_hs_streaming_ep.wMaxPacketSize = cpu_to_le16(max_packet_size | ((max_packet_mult - 1) << 11)); - uvc_hs_streaming_ep.bInterval = opts->streaming_interval; + + /* A high-bandwidth endpoint must specify a bInterval value of 1 */ + if (max_packet_mult > 1) + uvc_hs_streaming_ep.bInterval = 1; + else + uvc_hs_streaming_ep.bInterval = opts->streaming_interval; uvc_ss_streaming_ep.wMaxPacketSize = cpu_to_le16(max_packet_size); uvc_ss_streaming_ep.bInterval = opts->streaming_interval; @@ -817,6 +822,7 @@ static struct usb_function_instance *uvc_alloc_inst(void) pd->bmControls[0] = 1; pd->bmControls[1] = 0; pd->iProcessing = 0; + pd->bmVideoStandards = 0; od = &opts->uvc_output_terminal; od->bLength = UVC_DT_OUTPUT_TERMINAL_SIZE; diff --git a/drivers/usb/gadget/legacy/webcam.c b/drivers/usb/gadget/legacy/webcam.c index a9f8eb8e1c76..2c9eab2b863d 100644 --- a/drivers/usb/gadget/legacy/webcam.c +++ b/drivers/usb/gadget/legacy/webcam.c @@ -125,6 +125,7 @@ static const struct uvc_processing_unit_descriptor uvc_processing = { .bmControls[0] = 1, .bmControls[1] = 0, .iProcessing = 0, + .bmVideoStandards = 0, }; static const struct uvc_output_terminal_descriptor uvc_output_terminal = { diff --git a/drivers/usb/gadget/udc/aspeed-vhub/core.c b/drivers/usb/gadget/udc/aspeed-vhub/core.c index be7bb64e3594..d11d3d14313f 100644 --- a/drivers/usb/gadget/udc/aspeed-vhub/core.c +++ b/drivers/usb/gadget/udc/aspeed-vhub/core.c @@ -36,6 +36,7 @@ void ast_vhub_done(struct ast_vhub_ep *ep, struct ast_vhub_req *req, int status) { bool internal = req->internal; + struct ast_vhub *vhub = ep->vhub; EPVDBG(ep, "completing request @%p, status %d\n", req, status); @@ -46,7 +47,7 @@ void ast_vhub_done(struct ast_vhub_ep *ep, struct ast_vhub_req *req, if (req->req.dma) { if (!WARN_ON(!ep->dev)) - usb_gadget_unmap_request(&ep->dev->gadget, + usb_gadget_unmap_request_by_dev(&vhub->pdev->dev, &req->req, ep->epn.is_in); req->req.dma = 0; } diff --git a/drivers/usb/gadget/udc/aspeed-vhub/epn.c b/drivers/usb/gadget/udc/aspeed-vhub/epn.c index 02d8bfae58fb..cb164c615e6f 100644 --- a/drivers/usb/gadget/udc/aspeed-vhub/epn.c +++ b/drivers/usb/gadget/udc/aspeed-vhub/epn.c @@ -376,7 +376,7 @@ static int ast_vhub_epn_queue(struct usb_ep* u_ep, struct usb_request *u_req, if (ep->epn.desc_mode || ((((unsigned long)u_req->buf & 7) == 0) && (ep->epn.is_in || !(u_req->length & (u_ep->maxpacket - 1))))) { - rc = usb_gadget_map_request(&ep->dev->gadget, u_req, + rc = usb_gadget_map_request_by_dev(&vhub->pdev->dev, u_req, ep->epn.is_in); if (rc) { dev_warn(&vhub->pdev->dev, diff --git a/drivers/usb/gadget/udc/dummy_hcd.c b/drivers/usb/gadget/udc/dummy_hcd.c index 57067763b100..5f474ffe2be1 100644 --- a/drivers/usb/gadget/udc/dummy_hcd.c +++ b/drivers/usb/gadget/udc/dummy_hcd.c @@ -903,6 +903,21 @@ static int dummy_pullup(struct usb_gadget *_gadget, int value) spin_lock_irqsave(&dum->lock, flags); dum->pullup = (value != 0); set_link_state(dum_hcd); + if (value == 0) { + /* + * Emulate synchronize_irq(): wait for callbacks to finish. + * This seems to be the best place to emulate the call to + * synchronize_irq() that's in usb_gadget_remove_driver(). + * Doing it in dummy_udc_stop() would be too late since it + * is called after the unbind callback and unbind shouldn't + * be invoked until all the other callbacks are finished. + */ + while (dum->callback_usage > 0) { + spin_unlock_irqrestore(&dum->lock, flags); + usleep_range(1000, 2000); + spin_lock_irqsave(&dum->lock, flags); + } + } spin_unlock_irqrestore(&dum->lock, flags); usb_hcd_poll_rh_status(dummy_hcd_to_hcd(dum_hcd)); @@ -1004,14 +1019,6 @@ static int dummy_udc_stop(struct usb_gadget *g) spin_lock_irq(&dum->lock); dum->ints_enabled = 0; stop_activity(dum); - - /* emulate synchronize_irq(): wait for callbacks to finish */ - while (dum->callback_usage > 0) { - spin_unlock_irq(&dum->lock); - usleep_range(1000, 2000); - spin_lock_irq(&dum->lock); - } - dum->driver = NULL; spin_unlock_irq(&dum->lock); diff --git a/drivers/usb/gadget/udc/fotg210-udc.c b/drivers/usb/gadget/udc/fotg210-udc.c index d6ca50f01985..75bf446f4a66 100644 --- a/drivers/usb/gadget/udc/fotg210-udc.c +++ b/drivers/usb/gadget/udc/fotg210-udc.c @@ -338,15 +338,16 @@ static void fotg210_start_dma(struct fotg210_ep *ep, } else { buffer = req->req.buf + req->req.actual; length = ioread32(ep->fotg210->reg + - FOTG210_FIBCR(ep->epnum - 1)); - length &= FIBCR_BCFX; + FOTG210_FIBCR(ep->epnum - 1)) & FIBCR_BCFX; + if (length > req->req.length - req->req.actual) + length = req->req.length - req->req.actual; } } else { buffer = req->req.buf + req->req.actual; if (req->req.length - req->req.actual > ep->ep.maxpacket) length = ep->ep.maxpacket; else - length = req->req.length; + length = req->req.length - req->req.actual; } d = dma_map_single(dev, buffer, length, @@ -379,8 +380,7 @@ static void fotg210_ep0_queue(struct fotg210_ep *ep, } if (ep->dir_in) { /* if IN */ fotg210_start_dma(ep, req); - if ((req->req.length == req->req.actual) || - (req->req.actual < ep->ep.maxpacket)) + if (req->req.length == req->req.actual) fotg210_done(ep, req, 0); } else { /* OUT */ u32 value = ioread32(ep->fotg210->reg + FOTG210_DMISGR0); @@ -820,7 +820,7 @@ static void fotg210_ep0in(struct fotg210_udc *fotg210) if (req->req.length) fotg210_start_dma(ep, req); - if ((req->req.length - req->req.actual) < ep->ep.maxpacket) + if (req->req.actual == req->req.length) fotg210_done(ep, req, 0); } else { fotg210_set_cxdone(fotg210); @@ -849,12 +849,16 @@ static void fotg210_out_fifo_handler(struct fotg210_ep *ep) { struct fotg210_request *req = list_entry(ep->queue.next, struct fotg210_request, queue); + int disgr1 = ioread32(ep->fotg210->reg + FOTG210_DISGR1); fotg210_start_dma(ep, req); - /* finish out transfer */ + /* Complete the request when it's full or a short packet arrived. + * Like other drivers, short_not_ok isn't handled. + */ + if (req->req.length == req->req.actual || - req->req.actual < ep->ep.maxpacket) + (disgr1 & DISGR1_SPK_INT(ep->epnum - 1))) fotg210_done(ep, req, 0); } @@ -1027,6 +1031,12 @@ static void fotg210_init(struct fotg210_udc *fotg210) value &= ~DMCR_GLINT_EN; iowrite32(value, fotg210->reg + FOTG210_DMCR); + /* enable only grp2 irqs we handle */ + iowrite32(~(DISGR2_DMA_ERROR | DISGR2_RX0BYTE_INT | DISGR2_TX0BYTE_INT + | DISGR2_ISO_SEQ_ABORT_INT | DISGR2_ISO_SEQ_ERR_INT + | DISGR2_RESM_INT | DISGR2_SUSP_INT | DISGR2_USBRST_INT), + fotg210->reg + FOTG210_DMISGR2); + /* disable all fifo interrupt */ iowrite32(~(u32)0, fotg210->reg + FOTG210_DMISGR1); diff --git a/drivers/usb/gadget/udc/pch_udc.c b/drivers/usb/gadget/udc/pch_udc.c index a3c1fc924268..fd3656d0f760 100644 --- a/drivers/usb/gadget/udc/pch_udc.c +++ b/drivers/usb/gadget/udc/pch_udc.c @@ -7,12 +7,14 @@ #include #include #include +#include #include +#include +#include #include #include #include #include -#include #include #define PCH_VBUS_PERIOD 3000 /* VBUS polling period (msec) */ @@ -596,18 +598,22 @@ static void pch_udc_reconnect(struct pch_udc_dev *dev) static inline void pch_udc_vbus_session(struct pch_udc_dev *dev, int is_active) { + unsigned long iflags; + + spin_lock_irqsave(&dev->lock, iflags); if (is_active) { pch_udc_reconnect(dev); dev->vbus_session = 1; } else { if (dev->driver && dev->driver->disconnect) { - spin_lock(&dev->lock); + spin_unlock_irqrestore(&dev->lock, iflags); dev->driver->disconnect(&dev->gadget); - spin_unlock(&dev->lock); + spin_lock_irqsave(&dev->lock, iflags); } pch_udc_set_disconnect(dev); dev->vbus_session = 0; } + spin_unlock_irqrestore(&dev->lock, iflags); } /** @@ -1166,20 +1172,25 @@ static int pch_udc_pcd_selfpowered(struct usb_gadget *gadget, int value) static int pch_udc_pcd_pullup(struct usb_gadget *gadget, int is_on) { struct pch_udc_dev *dev; + unsigned long iflags; if (!gadget) return -EINVAL; + dev = container_of(gadget, struct pch_udc_dev, gadget); + + spin_lock_irqsave(&dev->lock, iflags); if (is_on) { pch_udc_reconnect(dev); } else { if (dev->driver && dev->driver->disconnect) { - spin_lock(&dev->lock); + spin_unlock_irqrestore(&dev->lock, iflags); dev->driver->disconnect(&dev->gadget); - spin_unlock(&dev->lock); + spin_lock_irqsave(&dev->lock, iflags); } pch_udc_set_disconnect(dev); } + spin_unlock_irqrestore(&dev->lock, iflags); return 0; } @@ -1350,6 +1361,43 @@ static irqreturn_t pch_vbus_gpio_irq(int irq, void *data) return IRQ_HANDLED; } +static struct gpiod_lookup_table minnowboard_udc_gpios = { + .dev_id = "0000:02:02.4", + .table = { + GPIO_LOOKUP("sch_gpio.33158", 12, NULL, GPIO_ACTIVE_HIGH), + {} + }, +}; + +static const struct dmi_system_id pch_udc_gpio_dmi_table[] = { + { + .ident = "MinnowBoard", + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "MinnowBoard"), + }, + .driver_data = &minnowboard_udc_gpios, + }, + { } +}; + +static void pch_vbus_gpio_remove_table(void *table) +{ + gpiod_remove_lookup_table(table); +} + +static int pch_vbus_gpio_add_table(struct pch_udc_dev *dev) +{ + struct device *d = &dev->pdev->dev; + const struct dmi_system_id *dmi; + + dmi = dmi_first_match(pch_udc_gpio_dmi_table); + if (!dmi) + return 0; + + gpiod_add_lookup_table(dmi->driver_data); + return devm_add_action_or_reset(d, pch_vbus_gpio_remove_table, dmi->driver_data); +} + /** * pch_vbus_gpio_init() - This API initializes GPIO port detecting VBUS. * @dev: Reference to the driver structure @@ -1360,6 +1408,7 @@ static irqreturn_t pch_vbus_gpio_irq(int irq, void *data) */ static int pch_vbus_gpio_init(struct pch_udc_dev *dev) { + struct device *d = &dev->pdev->dev; int err; int irq_num = 0; struct gpio_desc *gpiod; @@ -1367,8 +1416,12 @@ static int pch_vbus_gpio_init(struct pch_udc_dev *dev) dev->vbus_gpio.port = NULL; dev->vbus_gpio.intr = 0; + err = pch_vbus_gpio_add_table(dev); + if (err) + return err; + /* Retrieve the GPIO line from the USB gadget device */ - gpiod = devm_gpiod_get(dev->gadget.dev.parent, NULL, GPIOD_IN); + gpiod = devm_gpiod_get_optional(d, NULL, GPIOD_IN); if (IS_ERR(gpiod)) return PTR_ERR(gpiod); gpiod_set_consumer_name(gpiod, "pch_vbus"); @@ -1756,7 +1809,7 @@ static struct usb_request *pch_udc_alloc_request(struct usb_ep *usbep, } /* prevent from using desc. - set HOST BUSY */ dma_desc->status |= PCH_UDC_BS_HST_BSY; - dma_desc->dataptr = cpu_to_le32(DMA_ADDR_INVALID); + dma_desc->dataptr = lower_32_bits(DMA_ADDR_INVALID); req->td_data = dma_desc; req->td_data_last = dma_desc; req->chain_len = 1; @@ -2298,6 +2351,21 @@ static void pch_udc_svc_data_out(struct pch_udc_dev *dev, int ep_num) pch_udc_set_dma(dev, DMA_DIR_RX); } +static int pch_udc_gadget_setup(struct pch_udc_dev *dev) + __must_hold(&dev->lock) +{ + int rc; + + /* In some cases we can get an interrupt before driver gets setup */ + if (!dev->driver) + return -ESHUTDOWN; + + spin_unlock(&dev->lock); + rc = dev->driver->setup(&dev->gadget, &dev->setup_data); + spin_lock(&dev->lock); + return rc; +} + /** * pch_udc_svc_control_in() - Handle Control IN endpoint interrupts * @dev: Reference to the device structure @@ -2369,15 +2437,12 @@ static void pch_udc_svc_control_out(struct pch_udc_dev *dev) dev->gadget.ep0 = &dev->ep[UDC_EP0IN_IDX].ep; else /* OUT */ dev->gadget.ep0 = &ep->ep; - spin_lock(&dev->lock); /* If Mass storage Reset */ if ((dev->setup_data.bRequestType == 0x21) && (dev->setup_data.bRequest == 0xFF)) dev->prot_stall = 0; /* call gadget with setup data received */ - setup_supported = dev->driver->setup(&dev->gadget, - &dev->setup_data); - spin_unlock(&dev->lock); + setup_supported = pch_udc_gadget_setup(dev); if (dev->setup_data.bRequestType & USB_DIR_IN) { ep->td_data->status = (ep->td_data->status & @@ -2625,9 +2690,7 @@ static void pch_udc_svc_intf_interrupt(struct pch_udc_dev *dev) dev->ep[i].halted = 0; } dev->stall = 0; - spin_unlock(&dev->lock); - dev->driver->setup(&dev->gadget, &dev->setup_data); - spin_lock(&dev->lock); + pch_udc_gadget_setup(dev); } /** @@ -2662,9 +2725,7 @@ static void pch_udc_svc_cfg_interrupt(struct pch_udc_dev *dev) dev->stall = 0; /* call gadget zero with setup data received */ - spin_unlock(&dev->lock); - dev->driver->setup(&dev->gadget, &dev->setup_data); - spin_lock(&dev->lock); + pch_udc_gadget_setup(dev); } /** @@ -2870,14 +2931,20 @@ static void pch_udc_pcd_reinit(struct pch_udc_dev *dev) * @dev: Reference to the driver structure * * Return codes: - * 0: Success + * 0: Success + * -%ERRNO: All kind of errors when retrieving VBUS GPIO */ static int pch_udc_pcd_init(struct pch_udc_dev *dev) { + int ret; + pch_udc_init(dev); pch_udc_pcd_reinit(dev); - pch_vbus_gpio_init(dev); - return 0; + + ret = pch_vbus_gpio_init(dev); + if (ret) + pch_udc_exit(dev); + return ret; } /** @@ -2938,7 +3005,7 @@ static int init_dma_pools(struct pch_udc_dev *dev) dev->dma_addr = dma_map_single(&dev->pdev->dev, ep0out_buf, UDC_EP0OUT_BUFF_SIZE * 4, DMA_FROM_DEVICE); - return 0; + return dma_mapping_error(&dev->pdev->dev, dev->dma_addr); } static int pch_udc_start(struct usb_gadget *g, @@ -3063,6 +3130,7 @@ static int pch_udc_probe(struct pci_dev *pdev, if (retval) return retval; + dev->pdev = pdev; pci_set_drvdata(pdev, dev); /* Determine BAR based on PCI ID */ @@ -3078,16 +3146,10 @@ static int pch_udc_probe(struct pci_dev *pdev, dev->base_addr = pcim_iomap_table(pdev)[bar]; - /* - * FIXME: add a GPIO descriptor table to pdev.dev using - * gpiod_add_descriptor_table() from based on - * the PCI subsystem ID. The system-dependent GPIO is necessary for - * VBUS operation. - */ - /* initialize the hardware */ - if (pch_udc_pcd_init(dev)) - return -ENODEV; + retval = pch_udc_pcd_init(dev); + if (retval) + return retval; pci_enable_msi(pdev); @@ -3104,7 +3166,6 @@ static int pch_udc_probe(struct pci_dev *pdev, /* device struct setup */ spin_lock_init(&dev->lock); - dev->pdev = pdev; dev->gadget.ops = &pch_udc_ops; retval = init_dma_pools(dev); diff --git a/drivers/usb/gadget/udc/r8a66597-udc.c b/drivers/usb/gadget/udc/r8a66597-udc.c index 896c1a016d55..65cae4883454 100644 --- a/drivers/usb/gadget/udc/r8a66597-udc.c +++ b/drivers/usb/gadget/udc/r8a66597-udc.c @@ -1849,6 +1849,8 @@ static int r8a66597_probe(struct platform_device *pdev) return PTR_ERR(reg); ires = platform_get_resource(pdev, IORESOURCE_IRQ, 0); + if (!ires) + return -EINVAL; irq = ires->start; irq_trigger = ires->flags & IRQF_TRIGGER_MASK; diff --git a/drivers/usb/gadget/udc/s3c2410_udc.c b/drivers/usb/gadget/udc/s3c2410_udc.c index 1d3ebb07ccd4..b154b62abefa 100644 --- a/drivers/usb/gadget/udc/s3c2410_udc.c +++ b/drivers/usb/gadget/udc/s3c2410_udc.c @@ -54,8 +54,6 @@ static struct clk *udc_clock; static struct clk *usb_bus_clock; static void __iomem *base_addr; static int irq_usbd; -static u64 rsrc_start; -static u64 rsrc_len; static struct dentry *s3c2410_udc_debugfs_root; static inline u32 udc_read(u32 reg) @@ -1752,7 +1750,8 @@ static int s3c2410_udc_probe(struct platform_device *pdev) udc_clock = clk_get(NULL, "usb-device"); if (IS_ERR(udc_clock)) { dev_err(dev, "failed to get udc clock source\n"); - return PTR_ERR(udc_clock); + retval = PTR_ERR(udc_clock); + goto err_usb_bus_clk; } clk_prepare_enable(udc_clock); @@ -1775,7 +1774,7 @@ static int s3c2410_udc_probe(struct platform_device *pdev) base_addr = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(base_addr)) { retval = PTR_ERR(base_addr); - goto err_mem; + goto err_udc_clk; } the_controller = udc; @@ -1793,7 +1792,7 @@ static int s3c2410_udc_probe(struct platform_device *pdev) if (retval != 0) { dev_err(dev, "cannot get irq %i, err %d\n", irq_usbd, retval); retval = -EBUSY; - goto err_map; + goto err_udc_clk; } dev_dbg(dev, "got irq %i\n", irq_usbd); @@ -1864,10 +1863,14 @@ static int s3c2410_udc_probe(struct platform_device *pdev) gpio_free(udc_info->vbus_pin); err_int: free_irq(irq_usbd, udc); -err_map: - iounmap(base_addr); -err_mem: - release_mem_region(rsrc_start, rsrc_len); +err_udc_clk: + clk_disable_unprepare(udc_clock); + clk_put(udc_clock); + udc_clock = NULL; +err_usb_bus_clk: + clk_disable_unprepare(usb_bus_clock); + clk_put(usb_bus_clock); + usb_bus_clock = NULL; return retval; } @@ -1899,9 +1902,6 @@ static int s3c2410_udc_remove(struct platform_device *pdev) free_irq(irq_usbd, udc); - iounmap(base_addr); - release_mem_region(rsrc_start, rsrc_len); - if (!IS_ERR(udc_clock) && udc_clock != NULL) { clk_disable_unprepare(udc_clock); clk_put(udc_clock); diff --git a/drivers/usb/gadget/udc/snps_udc_plat.c b/drivers/usb/gadget/udc/snps_udc_plat.c index 32f1d3e90c26..99805d60a7ab 100644 --- a/drivers/usb/gadget/udc/snps_udc_plat.c +++ b/drivers/usb/gadget/udc/snps_udc_plat.c @@ -114,8 +114,8 @@ static int udc_plat_probe(struct platform_device *pdev) res = platform_get_resource(pdev, IORESOURCE_MEM, 0); udc->virt_addr = devm_ioremap_resource(dev, res); - if (IS_ERR(udc->regs)) - return PTR_ERR(udc->regs); + if (IS_ERR(udc->virt_addr)) + return PTR_ERR(udc->virt_addr); /* udc csr registers base */ udc->csr = udc->virt_addr + UDC_CSR_ADDR; diff --git a/drivers/usb/gadget/udc/tegra-xudc.c b/drivers/usb/gadget/udc/tegra-xudc.c index 580bef8eb4cb..2319c9737c2b 100644 --- a/drivers/usb/gadget/udc/tegra-xudc.c +++ b/drivers/usb/gadget/udc/tegra-xudc.c @@ -3883,7 +3883,7 @@ static int tegra_xudc_remove(struct platform_device *pdev) pm_runtime_get_sync(xudc->dev); - cancel_delayed_work(&xudc->plc_reset_work); + cancel_delayed_work_sync(&xudc->plc_reset_work); cancel_work_sync(&xudc->usb_role_sw_work); usb_del_gadget_udc(&xudc->gadget); diff --git a/drivers/usb/host/Kconfig b/drivers/usb/host/Kconfig index b94f2a070c05..df9428f1dc5e 100644 --- a/drivers/usb/host/Kconfig +++ b/drivers/usb/host/Kconfig @@ -272,6 +272,7 @@ config USB_EHCI_TEGRA select USB_CHIPIDEA select USB_CHIPIDEA_HOST select USB_CHIPIDEA_TEGRA + select USB_GADGET help This option is deprecated now and the driver was removed, use USB_CHIPIDEA_TEGRA instead. diff --git a/drivers/usb/host/fotg210-hcd.c b/drivers/usb/host/fotg210-hcd.c index 5617ef30530a..f0e4a315cc81 100644 --- a/drivers/usb/host/fotg210-hcd.c +++ b/drivers/usb/host/fotg210-hcd.c @@ -5568,7 +5568,7 @@ static int fotg210_hcd_probe(struct platform_device *pdev) struct usb_hcd *hcd; struct resource *res; int irq; - int retval = -ENODEV; + int retval; struct fotg210_hcd *fotg210; if (usb_disabled()) @@ -5588,7 +5588,7 @@ static int fotg210_hcd_probe(struct platform_device *pdev) hcd = usb_create_hcd(&fotg210_fotg210_hc_driver, dev, dev_name(dev)); if (!hcd) { - dev_err(dev, "failed to create hcd with err %d\n", retval); + dev_err(dev, "failed to create hcd\n"); retval = -ENOMEM; goto fail_create_hcd; } diff --git a/drivers/usb/host/sl811-hcd.c b/drivers/usb/host/sl811-hcd.c index 115ced0d93e1..1be692d3cf90 100644 --- a/drivers/usb/host/sl811-hcd.c +++ b/drivers/usb/host/sl811-hcd.c @@ -1287,11 +1287,10 @@ sl811h_hub_control( goto error; put_unaligned_le32(sl811->port1, buf); -#ifndef VERBOSE - if (*(u16*)(buf+2)) /* only if wPortChange is interesting */ -#endif - dev_dbg(hcd->self.controller, "GetPortStatus %08x\n", - sl811->port1); + if (__is_defined(VERBOSE) || + *(u16*)(buf+2)) /* only if wPortChange is interesting */ + dev_dbg(hcd->self.controller, "GetPortStatus %08x\n", + sl811->port1); break; case SetPortFeature: if (wIndex != 1 || wLength != 0) diff --git a/drivers/usb/host/xhci-ext-caps.h b/drivers/usb/host/xhci-ext-caps.h index fa59b242cd51..e8af0a125f84 100644 --- a/drivers/usb/host/xhci-ext-caps.h +++ b/drivers/usb/host/xhci-ext-caps.h @@ -7,8 +7,9 @@ * Author: Sarah Sharp * Some code borrowed from the Linux EHCI driver. */ -/* Up to 16 ms to halt an HC */ -#define XHCI_MAX_HALT_USEC (16*1000) + +/* HC should halt within 16 ms, but use 32 ms as some hosts take longer */ +#define XHCI_MAX_HALT_USEC (32 * 1000) /* HC not running - set to 1 when run/stop bit is cleared. */ #define XHCI_STS_HALT (1<<0) diff --git a/drivers/usb/host/xhci-mem.c b/drivers/usb/host/xhci-mem.c index f2c4ee7c4786..717c122f9449 100644 --- a/drivers/usb/host/xhci-mem.c +++ b/drivers/usb/host/xhci-mem.c @@ -2129,6 +2129,15 @@ static void xhci_add_in_port(struct xhci_hcd *xhci, unsigned int num_ports, if (major_revision == 0x03) { rhub = &xhci->usb3_rhub; + /* + * Some hosts incorrectly use sub-minor version for minor + * version (i.e. 0x02 instead of 0x20 for bcdUSB 0x320 and 0x01 + * for bcdUSB 0x310). Since there is no USB release with sub + * minor version 0x301 to 0x309, we can assume that they are + * incorrect and fix it here. + */ + if (minor_revision > 0x00 && minor_revision < 0x10) + minor_revision <<= 4; } else if (major_revision <= 0x02) { rhub = &xhci->usb2_rhub; } else { @@ -2240,6 +2249,9 @@ static void xhci_create_rhub_port_array(struct xhci_hcd *xhci, return; rhub->ports = kcalloc_node(rhub->num_ports, sizeof(*rhub->ports), flags, dev_to_node(dev)); + if (!rhub->ports) + return; + for (i = 0; i < HCS_MAX_PORTS(xhci->hcs_params1); i++) { if (xhci->hw_ports[i].rhub != rhub || xhci->hw_ports[i].hcd_portnum == DUPLICATE_ENTRY) diff --git a/drivers/usb/host/xhci-mtk-sch.c b/drivers/usb/host/xhci-mtk-sch.c index b45e5bf08997..8950d1f10a7f 100644 --- a/drivers/usb/host/xhci-mtk-sch.c +++ b/drivers/usb/host/xhci-mtk-sch.c @@ -378,6 +378,31 @@ static void update_bus_bw(struct mu3h_sch_bw_info *sch_bw, sch_ep->allocated = used; } +static int check_fs_bus_bw(struct mu3h_sch_ep_info *sch_ep, int offset) +{ + struct mu3h_sch_tt *tt = sch_ep->sch_tt; + u32 num_esit, tmp; + int base; + int i, j; + + num_esit = XHCI_MTK_MAX_ESIT / sch_ep->esit; + for (i = 0; i < num_esit; i++) { + base = offset + i * sch_ep->esit; + + /* + * Compared with hs bus, no matter what ep type, + * the hub will always delay one uframe to send data + */ + for (j = 0; j < sch_ep->cs_count; j++) { + tmp = tt->fs_bus_bw[base + j] + sch_ep->bw_cost_per_microframe; + if (tmp > FS_PAYLOAD_MAX) + return -ERANGE; + } + } + + return 0; +} + static int check_sch_tt(struct usb_device *udev, struct mu3h_sch_ep_info *sch_ep, u32 offset) { @@ -402,7 +427,7 @@ static int check_sch_tt(struct usb_device *udev, return -ERANGE; for (i = 0; i < sch_ep->cs_count; i++) - if (test_bit(offset + i, tt->split_bit_map)) + if (test_bit(offset + i, tt->ss_bit_map)) return -ERANGE; } else { @@ -432,7 +457,7 @@ static int check_sch_tt(struct usb_device *udev, cs_count = 7; /* HW limit */ for (i = 0; i < cs_count + 2; i++) { - if (test_bit(offset + i, tt->split_bit_map)) + if (test_bit(offset + i, tt->ss_bit_map)) return -ERANGE; } @@ -448,24 +473,44 @@ static int check_sch_tt(struct usb_device *udev, sch_ep->num_budget_microframes = sch_ep->esit; } - return 0; + return check_fs_bus_bw(sch_ep, offset); } static void update_sch_tt(struct usb_device *udev, - struct mu3h_sch_ep_info *sch_ep) + struct mu3h_sch_ep_info *sch_ep, bool used) { struct mu3h_sch_tt *tt = sch_ep->sch_tt; u32 base, num_esit; + int bw_updated; + int bits; int i, j; num_esit = XHCI_MTK_MAX_ESIT / sch_ep->esit; + bits = (sch_ep->ep_type == ISOC_OUT_EP) ? sch_ep->cs_count : 1; + + if (used) + bw_updated = sch_ep->bw_cost_per_microframe; + else + bw_updated = -sch_ep->bw_cost_per_microframe; + for (i = 0; i < num_esit; i++) { base = sch_ep->offset + i * sch_ep->esit; - for (j = 0; j < sch_ep->num_budget_microframes; j++) - set_bit(base + j, tt->split_bit_map); + + for (j = 0; j < bits; j++) { + if (used) + set_bit(base + j, tt->ss_bit_map); + else + clear_bit(base + j, tt->ss_bit_map); + } + + for (j = 0; j < sch_ep->cs_count; j++) + tt->fs_bus_bw[base + j] += bw_updated; } - list_add_tail(&sch_ep->tt_endpoint, &tt->ep_list); + if (used) + list_add_tail(&sch_ep->tt_endpoint, &tt->ep_list); + else + list_del(&sch_ep->tt_endpoint); } static int check_sch_bw(struct usb_device *udev, @@ -535,7 +580,7 @@ static int check_sch_bw(struct usb_device *udev, if (!tt_offset_ok) return -ERANGE; - update_sch_tt(udev, sch_ep); + update_sch_tt(udev, sch_ep, 1); } /* update bus bandwidth info */ @@ -548,15 +593,16 @@ static void destroy_sch_ep(struct usb_device *udev, struct mu3h_sch_bw_info *sch_bw, struct mu3h_sch_ep_info *sch_ep) { /* only release ep bw check passed by check_sch_bw() */ - if (sch_ep->allocated) + if (sch_ep->allocated) { update_bus_bw(sch_bw, sch_ep, 0); + if (sch_ep->sch_tt) + update_sch_tt(udev, sch_ep, 0); + } - list_del(&sch_ep->endpoint); - - if (sch_ep->sch_tt) { - list_del(&sch_ep->tt_endpoint); + if (sch_ep->sch_tt) drop_tt(udev); - } + + list_del(&sch_ep->endpoint); kfree(sch_ep); } @@ -643,7 +689,7 @@ int xhci_mtk_add_ep_quirk(struct usb_hcd *hcd, struct usb_device *udev, */ if (usb_endpoint_xfer_int(&ep->desc) || usb_endpoint_xfer_isoc(&ep->desc)) - ep_ctx->reserved[0] |= cpu_to_le32(EP_BPKTS(1)); + ep_ctx->reserved[0] = cpu_to_le32(EP_BPKTS(1)); return 0; } @@ -730,10 +776,10 @@ int xhci_mtk_check_bandwidth(struct usb_hcd *hcd, struct usb_device *udev) list_move_tail(&sch_ep->endpoint, &sch_bw->bw_ep_list); ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index); - ep_ctx->reserved[0] |= cpu_to_le32(EP_BPKTS(sch_ep->pkts) + ep_ctx->reserved[0] = cpu_to_le32(EP_BPKTS(sch_ep->pkts) | EP_BCSCOUNT(sch_ep->cs_count) | EP_BBM(sch_ep->burst_mode)); - ep_ctx->reserved[1] |= cpu_to_le32(EP_BOFFSET(sch_ep->offset) + ep_ctx->reserved[1] = cpu_to_le32(EP_BOFFSET(sch_ep->offset) | EP_BREPEAT(sch_ep->repeat)); xhci_dbg(xhci, " PKTS:%x, CSCOUNT:%x, BM:%x, OFFSET:%x, REPEAT:%x\n", diff --git a/drivers/usb/host/xhci-mtk.c b/drivers/usb/host/xhci-mtk.c index 2f27dc0d9c6b..1c331577fca9 100644 --- a/drivers/usb/host/xhci-mtk.c +++ b/drivers/usb/host/xhci-mtk.c @@ -397,6 +397,8 @@ static void xhci_mtk_quirks(struct device *dev, struct xhci_hcd *xhci) xhci->quirks |= XHCI_SPURIOUS_SUCCESS; if (mtk->lpm_support) xhci->quirks |= XHCI_LPM_SUPPORT; + if (mtk->u2_lpm_disable) + xhci->quirks |= XHCI_HW_LPM_DISABLE; /* * MTK xHCI 0.96: PSA is 1 by default even if doesn't support stream, @@ -469,6 +471,7 @@ static int xhci_mtk_probe(struct platform_device *pdev) return ret; mtk->lpm_support = of_property_read_bool(node, "usb3-lpm-capable"); + mtk->u2_lpm_disable = of_property_read_bool(node, "usb2-lpm-disable"); /* optional property, ignore the error if it does not exist */ of_property_read_u32(node, "mediatek,u3p-dis-msk", &mtk->u3p_dis_msk); diff --git a/drivers/usb/host/xhci-mtk.h b/drivers/usb/host/xhci-mtk.h index cbb09dfea62e..2fc0568ba054 100644 --- a/drivers/usb/host/xhci-mtk.h +++ b/drivers/usb/host/xhci-mtk.h @@ -20,13 +20,15 @@ #define XHCI_MTK_MAX_ESIT 64 /** - * @split_bit_map: used to avoid split microframes overlay + * @ss_bit_map: used to avoid start split microframes overlay + * @fs_bus_bw: array to keep track of bandwidth already used for FS * @ep_list: Endpoints using this TT * @usb_tt: usb TT related * @tt_port: TT port number */ struct mu3h_sch_tt { - DECLARE_BITMAP(split_bit_map, XHCI_MTK_MAX_ESIT); + DECLARE_BITMAP(ss_bit_map, XHCI_MTK_MAX_ESIT); + u32 fs_bus_bw[XHCI_MTK_MAX_ESIT]; struct list_head ep_list; struct usb_tt *usb_tt; int tt_port; @@ -150,6 +152,7 @@ struct xhci_hcd_mtk { struct phy **phys; int num_phys; bool lpm_support; + bool u2_lpm_disable; /* usb remote wakeup */ bool uwk_en; struct regmap *uwk; diff --git a/drivers/usb/host/xhci-pci.c b/drivers/usb/host/xhci-pci.c index 5bbccc9a0179..7bc18cf8042c 100644 --- a/drivers/usb/host/xhci-pci.c +++ b/drivers/usb/host/xhci-pci.c @@ -57,6 +57,7 @@ #define PCI_DEVICE_ID_INTEL_CML_XHCI 0xa3af #define PCI_DEVICE_ID_INTEL_TIGER_LAKE_XHCI 0x9a13 #define PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_XHCI 0x1138 +#define PCI_DEVICE_ID_INTEL_ALDER_LAKE_XHCI 0x461e #define PCI_DEVICE_ID_AMD_PROMONTORYA_4 0x43b9 #define PCI_DEVICE_ID_AMD_PROMONTORYA_3 0x43ba @@ -166,8 +167,10 @@ static void xhci_pci_quirks(struct device *dev, struct xhci_hcd *xhci) (pdev->device == 0x15e0 || pdev->device == 0x15e1)) xhci->quirks |= XHCI_SNPS_BROKEN_SUSPEND; - if (pdev->vendor == PCI_VENDOR_ID_AMD && pdev->device == 0x15e5) + if (pdev->vendor == PCI_VENDOR_ID_AMD && pdev->device == 0x15e5) { xhci->quirks |= XHCI_DISABLE_SPARSE; + xhci->quirks |= XHCI_RESET_ON_RESUME; + } if (pdev->vendor == PCI_VENDOR_ID_AMD) xhci->quirks |= XHCI_TRUST_TX_LENGTH; @@ -243,7 +246,8 @@ static void xhci_pci_quirks(struct device *dev, struct xhci_hcd *xhci) pdev->device == PCI_DEVICE_ID_INTEL_TITAN_RIDGE_DD_XHCI || pdev->device == PCI_DEVICE_ID_INTEL_ICE_LAKE_XHCI || pdev->device == PCI_DEVICE_ID_INTEL_TIGER_LAKE_XHCI || - pdev->device == PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_XHCI)) + pdev->device == PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_XHCI || + pdev->device == PCI_DEVICE_ID_INTEL_ALDER_LAKE_XHCI)) xhci->quirks |= XHCI_DEFAULT_PM_RUNTIME_ALLOW; if (pdev->vendor == PCI_VENDOR_ID_ETRON && diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c index ce38076901e2..6cdea0d00d19 100644 --- a/drivers/usb/host/xhci-ring.c +++ b/drivers/usb/host/xhci-ring.c @@ -863,7 +863,7 @@ static int xhci_reset_halted_ep(struct xhci_hcd *xhci, unsigned int slot_id, return ret; } -static void xhci_handle_halted_endpoint(struct xhci_hcd *xhci, +static int xhci_handle_halted_endpoint(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, unsigned int stream_id, struct xhci_td *td, enum xhci_ep_reset_type reset_type) @@ -876,7 +876,7 @@ static void xhci_handle_halted_endpoint(struct xhci_hcd *xhci, * Device will be reset soon to recover the link so don't do anything */ if (ep->vdev->flags & VDEV_PORT_ERROR) - return; + return -ENODEV; /* add td to cancelled list and let reset ep handler take care of it */ if (reset_type == EP_HARD_RESET) { @@ -889,16 +889,18 @@ static void xhci_handle_halted_endpoint(struct xhci_hcd *xhci, if (ep->ep_state & EP_HALTED) { xhci_dbg(xhci, "Reset ep command already pending\n"); - return; + return 0; } err = xhci_reset_halted_ep(xhci, slot_id, ep->ep_index, reset_type); if (err) - return; + return err; ep->ep_state |= EP_HALTED; xhci_ring_cmd_db(xhci); + + return 0; } /* @@ -1015,6 +1017,7 @@ static void xhci_handle_cmd_stop_ep(struct xhci_hcd *xhci, int slot_id, struct xhci_td *td = NULL; enum xhci_ep_reset_type reset_type; struct xhci_command *command; + int err; if (unlikely(TRB_TO_SUSPEND_PORT(le32_to_cpu(trb->generic.field[3])))) { if (!xhci->devs[slot_id]) @@ -1059,7 +1062,10 @@ static void xhci_handle_cmd_stop_ep(struct xhci_hcd *xhci, int slot_id, td->status = -EPROTO; } /* reset ep, reset handler cleans up cancelled tds */ - xhci_handle_halted_endpoint(xhci, ep, 0, td, reset_type); + err = xhci_handle_halted_endpoint(xhci, ep, 0, td, + reset_type); + if (err) + break; xhci_stop_watchdog_timer_in_irq(xhci, ep); return; case EP_STATE_RUNNING: @@ -2129,16 +2135,13 @@ int xhci_is_vendor_info_code(struct xhci_hcd *xhci, unsigned int trb_comp_code) return 0; } -static int finish_td(struct xhci_hcd *xhci, struct xhci_td *td, - struct xhci_transfer_event *event, struct xhci_virt_ep *ep) +static int finish_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, + struct xhci_ring *ep_ring, struct xhci_td *td, + u32 trb_comp_code) { struct xhci_ep_ctx *ep_ctx; - struct xhci_ring *ep_ring; - u32 trb_comp_code; - ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer)); ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep->ep_index); - trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); switch (trb_comp_code) { case COMP_STOPPED_LENGTH_INVALID: @@ -2234,9 +2237,9 @@ static int sum_trb_lengths(struct xhci_hcd *xhci, struct xhci_ring *ring, /* * Process control tds, update urb status and actual_length. */ -static int process_ctrl_td(struct xhci_hcd *xhci, struct xhci_td *td, - union xhci_trb *ep_trb, struct xhci_transfer_event *event, - struct xhci_virt_ep *ep) +static int process_ctrl_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, + struct xhci_ring *ep_ring, struct xhci_td *td, + union xhci_trb *ep_trb, struct xhci_transfer_event *event) { struct xhci_ep_ctx *ep_ctx; u32 trb_comp_code; @@ -2324,15 +2327,15 @@ static int process_ctrl_td(struct xhci_hcd *xhci, struct xhci_td *td, td->urb->actual_length = requested; finish_td: - return finish_td(xhci, td, event, ep); + return finish_td(xhci, ep, ep_ring, td, trb_comp_code); } /* * Process isochronous tds, update urb packet status and actual_length. */ -static int process_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td, - union xhci_trb *ep_trb, struct xhci_transfer_event *event, - struct xhci_virt_ep *ep) +static int process_isoc_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, + struct xhci_ring *ep_ring, struct xhci_td *td, + union xhci_trb *ep_trb, struct xhci_transfer_event *event) { struct urb_priv *urb_priv; int idx; @@ -2409,7 +2412,7 @@ static int process_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td, td->urb->actual_length += frame->actual_length; - return finish_td(xhci, td, event, ep); + return finish_td(xhci, ep, ep_ring, td, trb_comp_code); } static int skip_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td, @@ -2441,17 +2444,15 @@ static int skip_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td, /* * Process bulk and interrupt tds, update urb status and actual_length. */ -static int process_bulk_intr_td(struct xhci_hcd *xhci, struct xhci_td *td, - union xhci_trb *ep_trb, struct xhci_transfer_event *event, - struct xhci_virt_ep *ep) +static int process_bulk_intr_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, + struct xhci_ring *ep_ring, struct xhci_td *td, + union xhci_trb *ep_trb, struct xhci_transfer_event *event) { struct xhci_slot_ctx *slot_ctx; - struct xhci_ring *ep_ring; u32 trb_comp_code; u32 remaining, requested, ep_trb_len; slot_ctx = xhci_get_slot_ctx(xhci, ep->vdev->out_ctx); - ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer)); trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2])); @@ -2511,7 +2512,8 @@ static int process_bulk_intr_td(struct xhci_hcd *xhci, struct xhci_td *td, remaining); td->urb->actual_length = 0; } - return finish_td(xhci, td, event, ep); + + return finish_td(xhci, ep, ep_ring, td, trb_comp_code); } /* @@ -2854,11 +2856,11 @@ static int handle_tx_event(struct xhci_hcd *xhci, /* update the urb's actual_length and give back to the core */ if (usb_endpoint_xfer_control(&td->urb->ep->desc)) - process_ctrl_td(xhci, td, ep_trb, event, ep); + process_ctrl_td(xhci, ep, ep_ring, td, ep_trb, event); else if (usb_endpoint_xfer_isoc(&td->urb->ep->desc)) - process_isoc_td(xhci, td, ep_trb, event, ep); + process_isoc_td(xhci, ep, ep_ring, td, ep_trb, event); else - process_bulk_intr_td(xhci, td, ep_trb, event, ep); + process_bulk_intr_td(xhci, ep, ep_ring, td, ep_trb, event); cleanup: handling_skipped_tds = ep->skip && trb_comp_code != COMP_MISSED_SERVICE_ERROR && diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c index 1975016f46bf..0d2f1c37ab74 100644 --- a/drivers/usb/host/xhci.c +++ b/drivers/usb/host/xhci.c @@ -228,6 +228,7 @@ static void xhci_zero_64b_regs(struct xhci_hcd *xhci) struct device *dev = xhci_to_hcd(xhci)->self.sysdev; int err, i; u64 val; + u32 intrs; /* * Some Renesas controllers get into a weird state if they are @@ -266,7 +267,10 @@ static void xhci_zero_64b_regs(struct xhci_hcd *xhci) if (upper_32_bits(val)) xhci_write_64(xhci, 0, &xhci->op_regs->cmd_ring); - for (i = 0; i < HCS_MAX_INTRS(xhci->hcs_params1); i++) { + intrs = min_t(u32, HCS_MAX_INTRS(xhci->hcs_params1), + ARRAY_SIZE(xhci->run_regs->ir_set)); + + for (i = 0; i < intrs; i++) { struct xhci_intr_reg __iomem *ir; ir = &xhci->run_regs->ir_set[i]; @@ -1510,7 +1514,7 @@ static int xhci_configure_endpoint(struct xhci_hcd *xhci, * we need to issue an evaluate context command and wait on it. */ static int xhci_check_maxpacket(struct xhci_hcd *xhci, unsigned int slot_id, - unsigned int ep_index, struct urb *urb) + unsigned int ep_index, struct urb *urb, gfp_t mem_flags) { struct xhci_container_ctx *out_ctx; struct xhci_input_control_ctx *ctrl_ctx; @@ -1541,7 +1545,7 @@ static int xhci_check_maxpacket(struct xhci_hcd *xhci, unsigned int slot_id, * changes max packet sizes. */ - command = xhci_alloc_command(xhci, true, GFP_KERNEL); + command = xhci_alloc_command(xhci, true, mem_flags); if (!command) return -ENOMEM; @@ -1635,7 +1639,7 @@ static int xhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb, gfp_t mem_flag */ if (urb->dev->speed == USB_SPEED_FULL) { ret = xhci_check_maxpacket(xhci, slot_id, - ep_index, urb); + ep_index, urb, mem_flags); if (ret < 0) { xhci_urb_free_priv(urb_priv); urb->hcpriv = NULL; @@ -3269,6 +3273,14 @@ static void xhci_endpoint_reset(struct usb_hcd *hcd, /* config ep command clears toggle if add and drop ep flags are set */ ctrl_ctx = xhci_get_input_control_ctx(cfg_cmd->in_ctx); + if (!ctrl_ctx) { + spin_unlock_irqrestore(&xhci->lock, flags); + xhci_free_command(xhci, cfg_cmd); + xhci_warn(xhci, "%s: Could not get input context, bad type.\n", + __func__); + goto cleanup; + } + xhci_setup_input_ctx_for_config_ep(xhci, cfg_cmd->in_ctx, vdev->out_ctx, ctrl_ctx, ep_flag, ep_flag); xhci_endpoint_copy(xhci, cfg_cmd->in_ctx, vdev->out_ctx, ep_index); diff --git a/drivers/usb/musb/mediatek.c b/drivers/usb/musb/mediatek.c index eebeadd26946..6b92d037d8fc 100644 --- a/drivers/usb/musb/mediatek.c +++ b/drivers/usb/musb/mediatek.c @@ -518,8 +518,8 @@ static int mtk_musb_probe(struct platform_device *pdev) glue->xceiv = devm_usb_get_phy(dev, USB_PHY_TYPE_USB2); if (IS_ERR(glue->xceiv)) { - dev_err(dev, "fail to getting usb-phy %d\n", ret); ret = PTR_ERR(glue->xceiv); + dev_err(dev, "fail to getting usb-phy %d\n", ret); goto err_unregister_usb_phy; } diff --git a/drivers/usb/musb/musb_core.c b/drivers/usb/musb/musb_core.c index fc0457db62e1..8f09a387b773 100644 --- a/drivers/usb/musb/musb_core.c +++ b/drivers/usb/musb/musb_core.c @@ -2070,7 +2070,7 @@ static void musb_irq_work(struct work_struct *data) struct musb *musb = container_of(data, struct musb, irq_work.work); int error; - error = pm_runtime_get_sync(musb->controller); + error = pm_runtime_resume_and_get(musb->controller); if (error < 0) { dev_err(musb->controller, "Could not enable: %i\n", error); diff --git a/drivers/usb/roles/class.c b/drivers/usb/roles/class.c index 97f37077b7f9..33b637d0d8d9 100644 --- a/drivers/usb/roles/class.c +++ b/drivers/usb/roles/class.c @@ -189,6 +189,8 @@ usb_role_switch_find_by_fwnode(const struct fwnode_handle *fwnode) return NULL; dev = class_find_device_by_fwnode(role_class, fwnode); + if (dev) + WARN_ON(!try_module_get(dev->parent->driver->owner)); return dev ? to_role_switch(dev) : NULL; } diff --git a/drivers/usb/serial/ti_usb_3410_5052.c b/drivers/usb/serial/ti_usb_3410_5052.c index 7252b0ce75a6..fe1c13a8849c 100644 --- a/drivers/usb/serial/ti_usb_3410_5052.c +++ b/drivers/usb/serial/ti_usb_3410_5052.c @@ -1418,14 +1418,19 @@ static int ti_set_serial_info(struct tty_struct *tty, struct serial_struct *ss) { struct usb_serial_port *port = tty->driver_data; - struct ti_port *tport = usb_get_serial_port_data(port); + struct tty_port *tport = &port->port; unsigned cwait; cwait = ss->closing_wait; if (cwait != ASYNC_CLOSING_WAIT_NONE) cwait = msecs_to_jiffies(10 * ss->closing_wait); - tport->tp_port->port.closing_wait = cwait; + if (!capable(CAP_SYS_ADMIN)) { + if (cwait != tport->closing_wait) + return -EPERM; + } + + tport->closing_wait = cwait; return 0; } diff --git a/drivers/usb/serial/usb_wwan.c b/drivers/usb/serial/usb_wwan.c index 46d46a4f99c9..4e9c994a972a 100644 --- a/drivers/usb/serial/usb_wwan.c +++ b/drivers/usb/serial/usb_wwan.c @@ -140,10 +140,10 @@ int usb_wwan_get_serial_info(struct tty_struct *tty, ss->line = port->minor; ss->port = port->port_number; ss->baud_base = tty_get_baud_rate(port->port.tty); - ss->close_delay = port->port.close_delay / 10; + ss->close_delay = jiffies_to_msecs(port->port.close_delay) / 10; ss->closing_wait = port->port.closing_wait == ASYNC_CLOSING_WAIT_NONE ? ASYNC_CLOSING_WAIT_NONE : - port->port.closing_wait / 10; + jiffies_to_msecs(port->port.closing_wait) / 10; return 0; } EXPORT_SYMBOL(usb_wwan_get_serial_info); @@ -155,9 +155,10 @@ int usb_wwan_set_serial_info(struct tty_struct *tty, unsigned int closing_wait, close_delay; int retval = 0; - close_delay = ss->close_delay * 10; + close_delay = msecs_to_jiffies(ss->close_delay * 10); closing_wait = ss->closing_wait == ASYNC_CLOSING_WAIT_NONE ? - ASYNC_CLOSING_WAIT_NONE : ss->closing_wait * 10; + ASYNC_CLOSING_WAIT_NONE : + msecs_to_jiffies(ss->closing_wait * 10); mutex_lock(&port->port.mutex); diff --git a/drivers/usb/serial/xr_serial.c b/drivers/usb/serial/xr_serial.c index 0ca04906da4b..c59c8b47a120 100644 --- a/drivers/usb/serial/xr_serial.c +++ b/drivers/usb/serial/xr_serial.c @@ -467,6 +467,11 @@ static void xr_set_termios(struct tty_struct *tty, termios->c_cflag &= ~CSIZE; if (old_termios) termios->c_cflag |= old_termios->c_cflag & CSIZE; + else + termios->c_cflag |= CS8; + + if (C_CSIZE(tty) == CS7) + bits |= XR21V141X_UART_DATA_7; else bits |= XR21V141X_UART_DATA_8; break; diff --git a/drivers/usb/typec/stusb160x.c b/drivers/usb/typec/stusb160x.c index d21750bbbb44..6eaeba9b096e 100644 --- a/drivers/usb/typec/stusb160x.c +++ b/drivers/usb/typec/stusb160x.c @@ -682,8 +682,8 @@ static int stusb160x_probe(struct i2c_client *client) } fwnode = device_get_named_child_node(chip->dev, "connector"); - if (IS_ERR(fwnode)) - return PTR_ERR(fwnode); + if (!fwnode) + return -ENODEV; /* * When both VDD and VSYS power supplies are present, the low power diff --git a/drivers/usb/typec/tcpm/tcpci.c b/drivers/usb/typec/tcpm/tcpci.c index a27deb0b5f03..027afd7dfdce 100644 --- a/drivers/usb/typec/tcpm/tcpci.c +++ b/drivers/usb/typec/tcpm/tcpci.c @@ -24,6 +24,15 @@ #define AUTO_DISCHARGE_PD_HEADROOM_MV 850 #define AUTO_DISCHARGE_PPS_HEADROOM_MV 1250 +#define tcpc_presenting_cc1_rd(reg) \ + (!(TCPC_ROLE_CTRL_DRP & (reg)) && \ + (((reg) & (TCPC_ROLE_CTRL_CC1_MASK << TCPC_ROLE_CTRL_CC1_SHIFT)) == \ + (TCPC_ROLE_CTRL_CC_RD << TCPC_ROLE_CTRL_CC1_SHIFT))) +#define tcpc_presenting_cc2_rd(reg) \ + (!(TCPC_ROLE_CTRL_DRP & (reg)) && \ + (((reg) & (TCPC_ROLE_CTRL_CC2_MASK << TCPC_ROLE_CTRL_CC2_SHIFT)) == \ + (TCPC_ROLE_CTRL_CC_RD << TCPC_ROLE_CTRL_CC2_SHIFT))) + struct tcpci { struct device *dev; @@ -178,19 +187,25 @@ static int tcpci_get_cc(struct tcpc_dev *tcpc, enum typec_cc_status *cc1, enum typec_cc_status *cc2) { struct tcpci *tcpci = tcpc_to_tcpci(tcpc); - unsigned int reg; + unsigned int reg, role_control; int ret; + ret = regmap_read(tcpci->regmap, TCPC_ROLE_CTRL, &role_control); + if (ret < 0) + return ret; + ret = regmap_read(tcpci->regmap, TCPC_CC_STATUS, ®); if (ret < 0) return ret; *cc1 = tcpci_to_typec_cc((reg >> TCPC_CC_STATUS_CC1_SHIFT) & TCPC_CC_STATUS_CC1_MASK, - reg & TCPC_CC_STATUS_TERM); + reg & TCPC_CC_STATUS_TERM || + tcpc_presenting_cc1_rd(role_control)); *cc2 = tcpci_to_typec_cc((reg >> TCPC_CC_STATUS_CC2_SHIFT) & TCPC_CC_STATUS_CC2_MASK, - reg & TCPC_CC_STATUS_TERM); + reg & TCPC_CC_STATUS_TERM || + tcpc_presenting_cc2_rd(role_control)); return 0; } diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index ce7af398c7c1..52acc884a61f 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -268,12 +268,27 @@ struct pd_mode_data { struct typec_altmode_desc altmode_desc[ALTMODE_DISCOVERY_MAX]; }; +/* + * @min_volt: Actual min voltage at the local port + * @req_min_volt: Requested min voltage to the port partner + * @max_volt: Actual max voltage at the local port + * @req_max_volt: Requested max voltage to the port partner + * @max_curr: Actual max current at the local port + * @req_max_curr: Requested max current of the port partner + * @req_out_volt: Requested output voltage to the port partner + * @req_op_curr: Requested operating current to the port partner + * @supported: Parter has atleast one APDO hence supports PPS + * @active: PPS mode is active + */ struct pd_pps_data { u32 min_volt; + u32 req_min_volt; u32 max_volt; + u32 req_max_volt; u32 max_curr; - u32 out_volt; - u32 op_curr; + u32 req_max_curr; + u32 req_out_volt; + u32 req_op_curr; bool supported; bool active; }; @@ -389,7 +404,10 @@ struct tcpm_port { unsigned int operating_snk_mw; bool update_sink_caps; - /* Requested current / voltage */ + /* Requested current / voltage to the port partner */ + u32 req_current_limit; + u32 req_supply_voltage; + /* Actual current / voltage limit of the local port */ u32 current_limit; u32 supply_voltage; @@ -438,6 +456,9 @@ struct tcpm_port { enum tcpm_ams next_ams; bool in_ams; + /* Auto vbus discharge status */ + bool auto_vbus_discharge_enabled; + #ifdef CONFIG_DEBUG_FS struct dentry *dentry; struct mutex logbuffer_lock; /* log buffer access lock */ @@ -507,6 +528,9 @@ static const char * const pd_rev[] = { (tcpm_port_is_sink(port) && \ ((port)->cc1 == TYPEC_CC_RP_3_0 || (port)->cc2 == TYPEC_CC_RP_3_0)) +#define tcpm_wait_for_discharge(port) \ + (((port)->auto_vbus_discharge_enabled && !(port)->vbus_vsafe0v) ? PD_T_SAFE_0V : 0) + static enum tcpm_state tcpm_default_state(struct tcpm_port *port) { if (port->port_type == TYPEC_PORT_DRP) { @@ -1853,7 +1877,6 @@ static void vdm_run_state_machine(struct tcpm_port *port) } if (res < 0) { - port->vdm_sm_running = false; return; } } @@ -1869,6 +1892,7 @@ static void vdm_run_state_machine(struct tcpm_port *port) port->vdo_data[0] = port->vdo_retry; port->vdo_count = 1; port->vdm_state = VDM_STATE_READY; + tcpm_ams_finish(port); break; case VDM_STATE_BUSY: port->vdm_state = VDM_STATE_ERR_TMOUT; @@ -1934,7 +1958,7 @@ static void vdm_state_machine_work(struct kthread_work *work) port->vdm_state != VDM_STATE_BUSY && port->vdm_state != VDM_STATE_SEND_MESSAGE); - if (port->vdm_state == VDM_STATE_ERR_TMOUT) + if (port->vdm_state < VDM_STATE_READY) port->vdm_sm_running = false; mutex_unlock(&port->lock); @@ -2363,7 +2387,7 @@ static void tcpm_pd_data_request(struct tcpm_port *port, port->nr_sink_caps = cnt; port->sink_cap_done = true; if (port->ams == GET_SINK_CAPABILITIES) - tcpm_pd_handle_state(port, ready_state(port), NONE_AMS, 0); + tcpm_set_state(port, ready_state(port), 0); /* Unexpected Sink Capabilities */ else tcpm_pd_handle_msg(port, @@ -2432,8 +2456,8 @@ static void tcpm_pd_ctrl_request(struct tcpm_port *port, case SNK_TRANSITION_SINK: if (port->vbus_present) { tcpm_set_current_limit(port, - port->current_limit, - port->supply_voltage); + port->req_current_limit, + port->req_supply_voltage); port->explicit_contract = true; tcpm_set_auto_vbus_discharge_threshold(port, TYPEC_PWR_MODE_PD, @@ -2492,8 +2516,8 @@ static void tcpm_pd_ctrl_request(struct tcpm_port *port, break; case SNK_NEGOTIATE_PPS_CAPABILITIES: /* Revert data back from any requested PPS updates */ - port->pps_data.out_volt = port->supply_voltage; - port->pps_data.op_curr = port->current_limit; + port->pps_data.req_out_volt = port->supply_voltage; + port->pps_data.req_op_curr = port->current_limit; port->pps_status = (type == PD_CTRL_WAIT ? -EAGAIN : -EOPNOTSUPP); @@ -2525,6 +2549,16 @@ static void tcpm_pd_ctrl_request(struct tcpm_port *port, port->sink_cap_done = true; tcpm_set_state(port, ready_state(port), 0); break; + case SRC_READY: + case SNK_READY: + if (port->vdm_state > VDM_STATE_READY) { + port->vdm_state = VDM_STATE_DONE; + if (tcpm_vdm_ams(port)) + tcpm_ams_finish(port); + mod_vdm_delayed_work(port, 0); + break; + } + fallthrough; default: tcpm_pd_handle_state(port, port->pwr_role == TYPEC_SOURCE ? @@ -2542,8 +2576,12 @@ static void tcpm_pd_ctrl_request(struct tcpm_port *port, break; case SNK_NEGOTIATE_PPS_CAPABILITIES: port->pps_data.active = true; - port->supply_voltage = port->pps_data.out_volt; - port->current_limit = port->pps_data.op_curr; + port->pps_data.min_volt = port->pps_data.req_min_volt; + port->pps_data.max_volt = port->pps_data.req_max_volt; + port->pps_data.max_curr = port->pps_data.req_max_curr; + port->req_supply_voltage = port->pps_data.req_out_volt; + port->req_current_limit = port->pps_data.req_op_curr; + power_supply_changed(port->psy); tcpm_set_state(port, SNK_TRANSITION_SINK, 0); break; case SOFT_RESET_SEND: @@ -3102,17 +3140,16 @@ static unsigned int tcpm_pd_select_pps_apdo(struct tcpm_port *port) src = port->source_caps[src_pdo]; snk = port->snk_pdo[snk_pdo]; - port->pps_data.min_volt = max(pdo_pps_apdo_min_voltage(src), - pdo_pps_apdo_min_voltage(snk)); - port->pps_data.max_volt = min(pdo_pps_apdo_max_voltage(src), - pdo_pps_apdo_max_voltage(snk)); - port->pps_data.max_curr = min_pps_apdo_current(src, snk); - port->pps_data.out_volt = min(port->pps_data.max_volt, - max(port->pps_data.min_volt, - port->pps_data.out_volt)); - port->pps_data.op_curr = min(port->pps_data.max_curr, - port->pps_data.op_curr); - power_supply_changed(port->psy); + port->pps_data.req_min_volt = max(pdo_pps_apdo_min_voltage(src), + pdo_pps_apdo_min_voltage(snk)); + port->pps_data.req_max_volt = min(pdo_pps_apdo_max_voltage(src), + pdo_pps_apdo_max_voltage(snk)); + port->pps_data.req_max_curr = min_pps_apdo_current(src, snk); + port->pps_data.req_out_volt = min(port->pps_data.req_max_volt, + max(port->pps_data.req_min_volt, + port->pps_data.req_out_volt)); + port->pps_data.req_op_curr = min(port->pps_data.req_max_curr, + port->pps_data.req_op_curr); } return src_pdo; @@ -3192,8 +3229,8 @@ static int tcpm_pd_build_request(struct tcpm_port *port, u32 *rdo) flags & RDO_CAP_MISMATCH ? " [mismatch]" : ""); } - port->current_limit = ma; - port->supply_voltage = mv; + port->req_current_limit = ma; + port->req_supply_voltage = mv; return 0; } @@ -3239,10 +3276,10 @@ static int tcpm_pd_build_pps_request(struct tcpm_port *port, u32 *rdo) tcpm_log(port, "Invalid APDO selected!"); return -EINVAL; } - max_mv = port->pps_data.max_volt; - max_ma = port->pps_data.max_curr; - out_mv = port->pps_data.out_volt; - op_ma = port->pps_data.op_curr; + max_mv = port->pps_data.req_max_volt; + max_ma = port->pps_data.req_max_curr; + out_mv = port->pps_data.req_out_volt; + op_ma = port->pps_data.req_op_curr; break; default: tcpm_log(port, "Invalid PDO selected!"); @@ -3289,8 +3326,8 @@ static int tcpm_pd_build_pps_request(struct tcpm_port *port, u32 *rdo) tcpm_log(port, "Requesting APDO %d: %u mV, %u mA", src_pdo_index, out_mv, op_ma); - port->pps_data.op_curr = op_ma; - port->pps_data.out_volt = out_mv; + port->pps_data.req_op_curr = op_ma; + port->pps_data.req_out_volt = out_mv; return 0; } @@ -3418,6 +3455,8 @@ static int tcpm_src_attach(struct tcpm_port *port) if (port->tcpc->enable_auto_vbus_discharge) { ret = port->tcpc->enable_auto_vbus_discharge(port->tcpc, true); tcpm_log_force(port, "enable vbus discharge ret:%d", ret); + if (!ret) + port->auto_vbus_discharge_enabled = true; } ret = tcpm_set_roles(port, true, TYPEC_SOURCE, tcpm_data_role_for_source(port)); @@ -3500,6 +3539,8 @@ static void tcpm_reset_port(struct tcpm_port *port) if (port->tcpc->enable_auto_vbus_discharge) { ret = port->tcpc->enable_auto_vbus_discharge(port->tcpc, false); tcpm_log_force(port, "Disable vbus discharge ret:%d", ret); + if (!ret) + port->auto_vbus_discharge_enabled = false; } port->in_ams = false; port->ams = NONE_AMS; @@ -3533,8 +3574,6 @@ static void tcpm_reset_port(struct tcpm_port *port) port->sink_cap_done = false; if (port->tcpc->enable_frs) port->tcpc->enable_frs(port->tcpc, false); - - power_supply_changed(port->psy); } static void tcpm_detach(struct tcpm_port *port) @@ -3574,6 +3613,8 @@ static int tcpm_snk_attach(struct tcpm_port *port) tcpm_set_auto_vbus_discharge_threshold(port, TYPEC_PWR_MODE_USB, false, VSAFE5V); ret = port->tcpc->enable_auto_vbus_discharge(port->tcpc, true); tcpm_log_force(port, "enable vbus discharge ret:%d", ret); + if (!ret) + port->auto_vbus_discharge_enabled = true; } ret = tcpm_set_roles(port, true, TYPEC_SINK, tcpm_data_role_for_sink(port)); @@ -4103,6 +4144,23 @@ static void run_state_machine(struct tcpm_port *port) } break; case SNK_TRANSITION_SINK: + /* From the USB PD spec: + * "The Sink Shall transition to Sink Standby before a positive or + * negative voltage transition of VBUS. During Sink Standby + * the Sink Shall reduce its power draw to pSnkStdby." + * + * This is not applicable to PPS though as the port can continue + * to draw negotiated power without switching to standby. + */ + if (port->supply_voltage != port->req_supply_voltage && !port->pps_data.active && + port->current_limit * port->supply_voltage / 1000 > PD_P_SNK_STDBY_MW) { + u32 stdby_ma = PD_P_SNK_STDBY_MW * 1000 / port->supply_voltage; + + tcpm_log(port, "Setting standby current %u mV @ %u mA", + port->supply_voltage, stdby_ma); + tcpm_set_current_limit(port, stdby_ma, port->supply_voltage); + } + fallthrough; case SNK_TRANSITION_SINK_VBUS: tcpm_set_state(port, hard_reset_state(port), PD_T_PS_TRANSITION); @@ -4676,9 +4734,9 @@ static void _tcpm_cc_change(struct tcpm_port *port, enum typec_cc_status cc1, if (tcpm_port_is_disconnected(port) || !tcpm_port_is_source(port)) { if (port->port_type == TYPEC_PORT_SRC) - tcpm_set_state(port, SRC_UNATTACHED, 0); + tcpm_set_state(port, SRC_UNATTACHED, tcpm_wait_for_discharge(port)); else - tcpm_set_state(port, SNK_UNATTACHED, 0); + tcpm_set_state(port, SNK_UNATTACHED, tcpm_wait_for_discharge(port)); } break; case SNK_UNATTACHED: @@ -4709,7 +4767,23 @@ static void _tcpm_cc_change(struct tcpm_port *port, enum typec_cc_status cc1, tcpm_set_state(port, SNK_DEBOUNCED, 0); break; case SNK_READY: - if (tcpm_port_is_disconnected(port)) + /* + * EXIT condition is based primarily on vbus disconnect and CC is secondary. + * "A port that has entered into USB PD communications with the Source and + * has seen the CC voltage exceed vRd-USB may monitor the CC pin to detect + * cable disconnect in addition to monitoring VBUS. + * + * A port that is monitoring the CC voltage for disconnect (but is not in + * the process of a USB PD PR_Swap or USB PD FR_Swap) shall transition to + * Unattached.SNK within tSinkDisconnect after the CC voltage remains below + * vRd-USB for tPDDebounce." + * + * When set_auto_vbus_discharge_threshold is enabled, CC pins go + * away before vbus decays to disconnect threshold. Allow + * disconnect to be driven by vbus disconnect when auto vbus + * discharge is enabled. + */ + if (!port->auto_vbus_discharge_enabled && tcpm_port_is_disconnected(port)) tcpm_set_state(port, unattached_state(port), 0); else if (!port->pd_capable && (cc1 != old_cc1 || cc2 != old_cc2)) @@ -4808,9 +4882,13 @@ static void _tcpm_cc_change(struct tcpm_port *port, enum typec_cc_status cc1, * Ignore CC changes here. */ break; - default: - if (tcpm_port_is_disconnected(port)) + /* + * While acting as sink and auto vbus discharge is enabled, Allow disconnect + * to be driven by vbus disconnect. + */ + if (tcpm_port_is_disconnected(port) && !(port->pwr_role == TYPEC_SINK && + port->auto_vbus_discharge_enabled)) tcpm_set_state(port, unattached_state(port), 0); break; } @@ -4974,8 +5052,16 @@ static void _tcpm_pd_vbus_off(struct tcpm_port *port) case SRC_TRANSITION_SUPPLY: case SRC_READY: case SRC_WAIT_NEW_CAPABILITIES: - /* Force to unattached state to re-initiate connection */ - tcpm_set_state(port, SRC_UNATTACHED, 0); + /* + * Force to unattached state to re-initiate connection. + * DRP port should move to Unattached.SNK instead of Unattached.SRC if + * sink removed. Although sink removal here is due to source's vbus collapse, + * treat it the same way for consistency. + */ + if (port->port_type == TYPEC_PORT_SRC) + tcpm_set_state(port, SRC_UNATTACHED, tcpm_wait_for_discharge(port)); + else + tcpm_set_state(port, SNK_UNATTACHED, tcpm_wait_for_discharge(port)); break; case PORT_RESET: @@ -4994,9 +5080,8 @@ static void _tcpm_pd_vbus_off(struct tcpm_port *port) break; default: - if (port->pwr_role == TYPEC_SINK && - port->attached) - tcpm_set_state(port, SNK_UNATTACHED, 0); + if (port->pwr_role == TYPEC_SINK && port->attached) + tcpm_set_state(port, SNK_UNATTACHED, tcpm_wait_for_discharge(port)); break; } } @@ -5018,7 +5103,23 @@ static void _tcpm_pd_vbus_vsafe0v(struct tcpm_port *port) tcpm_set_state(port, tcpm_try_snk(port) ? SNK_TRY : SRC_ATTACHED, PD_T_CC_DEBOUNCE); break; + case SRC_STARTUP: + case SRC_SEND_CAPABILITIES: + case SRC_SEND_CAPABILITIES_TIMEOUT: + case SRC_NEGOTIATE_CAPABILITIES: + case SRC_TRANSITION_SUPPLY: + case SRC_READY: + case SRC_WAIT_NEW_CAPABILITIES: + if (port->auto_vbus_discharge_enabled) { + if (port->port_type == TYPEC_PORT_SRC) + tcpm_set_state(port, SRC_UNATTACHED, 0); + else + tcpm_set_state(port, SNK_UNATTACHED, 0); + } + break; default: + if (port->pwr_role == TYPEC_SINK && port->auto_vbus_discharge_enabled) + tcpm_set_state(port, SNK_UNATTACHED, 0); break; } } @@ -5374,7 +5475,7 @@ static int tcpm_try_role(struct typec_port *p, int role) return ret; } -static int tcpm_pps_set_op_curr(struct tcpm_port *port, u16 op_curr) +static int tcpm_pps_set_op_curr(struct tcpm_port *port, u16 req_op_curr) { unsigned int target_mw; int ret; @@ -5392,12 +5493,12 @@ static int tcpm_pps_set_op_curr(struct tcpm_port *port, u16 op_curr) goto port_unlock; } - if (op_curr > port->pps_data.max_curr) { + if (req_op_curr > port->pps_data.max_curr) { ret = -EINVAL; goto port_unlock; } - target_mw = (op_curr * port->pps_data.out_volt) / 1000; + target_mw = (req_op_curr * port->supply_voltage) / 1000; if (target_mw < port->operating_snk_mw) { ret = -EINVAL; goto port_unlock; @@ -5411,10 +5512,10 @@ static int tcpm_pps_set_op_curr(struct tcpm_port *port, u16 op_curr) } /* Round down operating current to align with PPS valid steps */ - op_curr = op_curr - (op_curr % RDO_PROG_CURR_MA_STEP); + req_op_curr = req_op_curr - (req_op_curr % RDO_PROG_CURR_MA_STEP); reinit_completion(&port->pps_complete); - port->pps_data.op_curr = op_curr; + port->pps_data.req_op_curr = req_op_curr; port->pps_status = 0; port->pps_pending = true; mutex_unlock(&port->lock); @@ -5435,7 +5536,7 @@ static int tcpm_pps_set_op_curr(struct tcpm_port *port, u16 op_curr) return ret; } -static int tcpm_pps_set_out_volt(struct tcpm_port *port, u16 out_volt) +static int tcpm_pps_set_out_volt(struct tcpm_port *port, u16 req_out_volt) { unsigned int target_mw; int ret; @@ -5453,13 +5554,13 @@ static int tcpm_pps_set_out_volt(struct tcpm_port *port, u16 out_volt) goto port_unlock; } - if (out_volt < port->pps_data.min_volt || - out_volt > port->pps_data.max_volt) { + if (req_out_volt < port->pps_data.min_volt || + req_out_volt > port->pps_data.max_volt) { ret = -EINVAL; goto port_unlock; } - target_mw = (port->pps_data.op_curr * out_volt) / 1000; + target_mw = (port->current_limit * req_out_volt) / 1000; if (target_mw < port->operating_snk_mw) { ret = -EINVAL; goto port_unlock; @@ -5473,10 +5574,10 @@ static int tcpm_pps_set_out_volt(struct tcpm_port *port, u16 out_volt) } /* Round down output voltage to align with PPS valid steps */ - out_volt = out_volt - (out_volt % RDO_PROG_VOLT_MV_STEP); + req_out_volt = req_out_volt - (req_out_volt % RDO_PROG_VOLT_MV_STEP); reinit_completion(&port->pps_complete); - port->pps_data.out_volt = out_volt; + port->pps_data.req_out_volt = req_out_volt; port->pps_status = 0; port->pps_pending = true; mutex_unlock(&port->lock); @@ -5534,8 +5635,8 @@ static int tcpm_pps_activate(struct tcpm_port *port, bool activate) /* Trigger PPS request or move back to standard PDO contract */ if (activate) { - port->pps_data.out_volt = port->supply_voltage; - port->pps_data.op_curr = port->current_limit; + port->pps_data.req_out_volt = port->supply_voltage; + port->pps_data.req_op_curr = port->current_limit; } mutex_unlock(&port->lock); diff --git a/drivers/usb/typec/tps6598x.c b/drivers/usb/typec/tps6598x.c index 29bd1c5a283c..4038104568f5 100644 --- a/drivers/usb/typec/tps6598x.c +++ b/drivers/usb/typec/tps6598x.c @@ -614,8 +614,8 @@ static int tps6598x_probe(struct i2c_client *client) return ret; fwnode = device_get_named_child_node(&client->dev, "connector"); - if (IS_ERR(fwnode)) - return PTR_ERR(fwnode); + if (!fwnode) + return -ENODEV; tps->role_sw = fwnode_usb_role_switch_get(fwnode); if (IS_ERR(tps->role_sw)) { diff --git a/drivers/usb/typec/ucsi/ucsi.c b/drivers/usb/typec/ucsi/ucsi.c index 244270755ae6..1e266f083bf8 100644 --- a/drivers/usb/typec/ucsi/ucsi.c +++ b/drivers/usb/typec/ucsi/ucsi.c @@ -495,7 +495,8 @@ static void ucsi_unregister_altmodes(struct ucsi_connector *con, u8 recipient) } } -static void ucsi_get_pdos(struct ucsi_connector *con, int is_partner) +static int ucsi_get_pdos(struct ucsi_connector *con, int is_partner, + u32 *pdos, int offset, int num_pdos) { struct ucsi *ucsi = con->ucsi; u64 command; @@ -503,17 +504,39 @@ static void ucsi_get_pdos(struct ucsi_connector *con, int is_partner) command = UCSI_COMMAND(UCSI_GET_PDOS) | UCSI_CONNECTOR_NUMBER(con->num); command |= UCSI_GET_PDOS_PARTNER_PDO(is_partner); - command |= UCSI_GET_PDOS_NUM_PDOS(UCSI_MAX_PDOS - 1); + command |= UCSI_GET_PDOS_PDO_OFFSET(offset); + command |= UCSI_GET_PDOS_NUM_PDOS(num_pdos - 1); command |= UCSI_GET_PDOS_SRC_PDOS; - ret = ucsi_send_command(ucsi, command, con->src_pdos, - sizeof(con->src_pdos)); - if (ret < 0) { + ret = ucsi_send_command(ucsi, command, pdos + offset, + num_pdos * sizeof(u32)); + if (ret < 0) dev_err(ucsi->dev, "UCSI_GET_PDOS failed (%d)\n", ret); + if (ret == 0 && offset == 0) + dev_warn(ucsi->dev, "UCSI_GET_PDOS returned 0 bytes\n"); + + return ret; +} + +static void ucsi_get_src_pdos(struct ucsi_connector *con, int is_partner) +{ + int ret; + + /* UCSI max payload means only getting at most 4 PDOs at a time */ + ret = ucsi_get_pdos(con, 1, con->src_pdos, 0, UCSI_MAX_PDOS); + if (ret < 0) return; - } + con->num_pdos = ret / sizeof(u32); /* number of bytes to 32-bit PDOs */ - if (ret == 0) - dev_warn(ucsi->dev, "UCSI_GET_PDOS returned 0 bytes\n"); + if (con->num_pdos < UCSI_MAX_PDOS) + return; + + /* get the remaining PDOs, if any */ + ret = ucsi_get_pdos(con, 1, con->src_pdos, UCSI_MAX_PDOS, + PDO_MAX_OBJECTS - UCSI_MAX_PDOS); + if (ret < 0) + return; + + con->num_pdos += ret / sizeof(u32); } static void ucsi_pwr_opmode_change(struct ucsi_connector *con) @@ -522,7 +545,7 @@ static void ucsi_pwr_opmode_change(struct ucsi_connector *con) case UCSI_CONSTAT_PWR_OPMODE_PD: con->rdo = con->status.request_data_obj; typec_set_pwr_opmode(con->port, TYPEC_PWR_MODE_PD); - ucsi_get_pdos(con, 1); + ucsi_get_src_pdos(con, 1); break; case UCSI_CONSTAT_PWR_OPMODE_TYPEC1_5: con->rdo = 0; @@ -999,6 +1022,7 @@ static const struct typec_operations ucsi_ops = { .pr_set = ucsi_pr_swap }; +/* Caller must call fwnode_handle_put() after use */ static struct fwnode_handle *ucsi_find_fwnode(struct ucsi_connector *con) { struct fwnode_handle *fwnode; @@ -1033,7 +1057,7 @@ static int ucsi_register_port(struct ucsi *ucsi, int index) command |= UCSI_CONNECTOR_NUMBER(con->num); ret = ucsi_send_command(ucsi, command, &con->cap, sizeof(con->cap)); if (ret < 0) - goto out; + goto out_unlock; if (con->cap.op_mode & UCSI_CONCAP_OPMODE_DRP) cap->data = TYPEC_PORT_DRD; @@ -1151,6 +1175,8 @@ static int ucsi_register_port(struct ucsi *ucsi, int index) trace_ucsi_register_port(con->num, &con->status); out: + fwnode_handle_put(cap->fwnode); +out_unlock: mutex_unlock(&con->lock); return ret; } diff --git a/drivers/usb/typec/ucsi/ucsi.h b/drivers/usb/typec/ucsi/ucsi.h index 3920e20a9e9e..cee666790907 100644 --- a/drivers/usb/typec/ucsi/ucsi.h +++ b/drivers/usb/typec/ucsi/ucsi.h @@ -8,6 +8,7 @@ #include #include #include +#include #include /* -------------------------------------------------------------------------- */ @@ -134,7 +135,9 @@ void ucsi_connector_change(struct ucsi *ucsi, u8 num); /* GET_PDOS command bits */ #define UCSI_GET_PDOS_PARTNER_PDO(_r_) ((u64)(_r_) << 23) +#define UCSI_GET_PDOS_PDO_OFFSET(_r_) ((u64)(_r_) << 24) #define UCSI_GET_PDOS_NUM_PDOS(_r_) ((u64)(_r_) << 32) +#define UCSI_MAX_PDOS (4) #define UCSI_GET_PDOS_SRC_PDOS ((u64)1 << 34) /* -------------------------------------------------------------------------- */ @@ -302,7 +305,6 @@ struct ucsi { #define UCSI_MAX_SVID 5 #define UCSI_MAX_ALTMODES (UCSI_MAX_SVID * 6) -#define UCSI_MAX_PDOS (4) #define UCSI_TYPEC_VSAFE5V 5000 #define UCSI_TYPEC_1_5_CURRENT 1500 @@ -330,7 +332,7 @@ struct ucsi_connector { struct power_supply *psy; struct power_supply_desc psy_desc; u32 rdo; - u32 src_pdos[UCSI_MAX_PDOS]; + u32 src_pdos[PDO_MAX_OBJECTS]; int num_pdos; struct usb_role_switch *usb_role_sw; diff --git a/drivers/usb/usbip/vudc_sysfs.c b/drivers/usb/usbip/vudc_sysfs.c index f7633ee655a1..d1cf6b51bf85 100644 --- a/drivers/usb/usbip/vudc_sysfs.c +++ b/drivers/usb/usbip/vudc_sysfs.c @@ -156,12 +156,14 @@ static ssize_t usbip_sockfd_store(struct device *dev, tcp_rx = kthread_create(&v_rx_loop, &udc->ud, "vudc_rx"); if (IS_ERR(tcp_rx)) { sockfd_put(socket); + mutex_unlock(&udc->ud.sysfs_lock); return -EINVAL; } tcp_tx = kthread_create(&v_tx_loop, &udc->ud, "vudc_tx"); if (IS_ERR(tcp_tx)) { kthread_stop(tcp_rx); sockfd_put(socket); + mutex_unlock(&udc->ud.sysfs_lock); return -EINVAL; } diff --git a/drivers/vfio/fsl-mc/vfio_fsl_mc.c b/drivers/vfio/fsl-mc/vfio_fsl_mc.c index f27e25112c40..8722f5effacd 100644 --- a/drivers/vfio/fsl-mc/vfio_fsl_mc.c +++ b/drivers/vfio/fsl-mc/vfio_fsl_mc.c @@ -568,23 +568,39 @@ static int vfio_fsl_mc_init_device(struct vfio_fsl_mc_device *vdev) dev_err(&mc_dev->dev, "VFIO_FSL_MC: Failed to setup DPRC (%d)\n", ret); goto out_nc_unreg; } + return 0; + +out_nc_unreg: + bus_unregister_notifier(&fsl_mc_bus_type, &vdev->nb); + return ret; +} +static int vfio_fsl_mc_scan_container(struct fsl_mc_device *mc_dev) +{ + int ret; + + /* non dprc devices do not scan for other devices */ + if (!is_fsl_mc_bus_dprc(mc_dev)) + return 0; ret = dprc_scan_container(mc_dev, false); if (ret) { - dev_err(&mc_dev->dev, "VFIO_FSL_MC: Container scanning failed (%d)\n", ret); - goto out_dprc_cleanup; + dev_err(&mc_dev->dev, + "VFIO_FSL_MC: Container scanning failed (%d)\n", ret); + dprc_remove_devices(mc_dev, NULL, 0); + return ret; } - return 0; +} + +static void vfio_fsl_uninit_device(struct vfio_fsl_mc_device *vdev) +{ + struct fsl_mc_device *mc_dev = vdev->mc_dev; + + if (!is_fsl_mc_bus_dprc(mc_dev)) + return; -out_dprc_cleanup: - dprc_remove_devices(mc_dev, NULL, 0); dprc_cleanup(mc_dev); -out_nc_unreg: bus_unregister_notifier(&fsl_mc_bus_type, &vdev->nb); - vdev->nb.notifier_call = NULL; - - return ret; } static int vfio_fsl_mc_probe(struct fsl_mc_device *mc_dev) @@ -607,29 +623,39 @@ static int vfio_fsl_mc_probe(struct fsl_mc_device *mc_dev) } vdev->mc_dev = mc_dev; - - ret = vfio_add_group_dev(dev, &vfio_fsl_mc_ops, vdev); - if (ret) { - dev_err(dev, "VFIO_FSL_MC: Failed to add to vfio group\n"); - goto out_group_put; - } + mutex_init(&vdev->igate); ret = vfio_fsl_mc_reflck_attach(vdev); if (ret) - goto out_group_dev; + goto out_group_put; ret = vfio_fsl_mc_init_device(vdev); if (ret) goto out_reflck; - mutex_init(&vdev->igate); + ret = vfio_add_group_dev(dev, &vfio_fsl_mc_ops, vdev); + if (ret) { + dev_err(dev, "VFIO_FSL_MC: Failed to add to vfio group\n"); + goto out_device; + } + /* + * This triggers recursion into vfio_fsl_mc_probe() on another device + * and the vfio_fsl_mc_reflck_attach() must succeed, which relies on the + * vfio_add_group_dev() above. It has no impact on this vdev, so it is + * safe to be after the vfio device is made live. + */ + ret = vfio_fsl_mc_scan_container(mc_dev); + if (ret) + goto out_group_dev; return 0; -out_reflck: - vfio_fsl_mc_reflck_put(vdev->reflck); out_group_dev: vfio_del_group_dev(dev); +out_device: + vfio_fsl_uninit_device(vdev); +out_reflck: + vfio_fsl_mc_reflck_put(vdev->reflck); out_group_put: vfio_iommu_group_put(group, dev); return ret; @@ -646,16 +672,10 @@ static int vfio_fsl_mc_remove(struct fsl_mc_device *mc_dev) mutex_destroy(&vdev->igate); + dprc_remove_devices(mc_dev, NULL, 0); + vfio_fsl_uninit_device(vdev); vfio_fsl_mc_reflck_put(vdev->reflck); - if (is_fsl_mc_bus_dprc(mc_dev)) { - dprc_remove_devices(mc_dev, NULL, 0); - dprc_cleanup(mc_dev); - } - - if (vdev->nb.notifier_call) - bus_unregister_notifier(&fsl_mc_bus_type, &vdev->nb); - vfio_iommu_group_put(mc_dev->dev.iommu_group, dev); return 0; diff --git a/drivers/vfio/mdev/mdev_sysfs.c b/drivers/vfio/mdev/mdev_sysfs.c index 917fd84c1c6f..367ff5412a38 100644 --- a/drivers/vfio/mdev/mdev_sysfs.c +++ b/drivers/vfio/mdev/mdev_sysfs.c @@ -105,6 +105,7 @@ static struct mdev_type *add_mdev_supported_type(struct mdev_parent *parent, return ERR_PTR(-ENOMEM); type->kobj.kset = parent->mdev_types_kset; + type->parent = parent; ret = kobject_init_and_add(&type->kobj, &mdev_type_ktype, NULL, "%s-%s", dev_driver_string(parent->dev), @@ -132,7 +133,6 @@ static struct mdev_type *add_mdev_supported_type(struct mdev_parent *parent, } type->group = group; - type->parent = parent; return type; attrs_failed: diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c index 5023e23db3bc..cb7f2dc09e9d 100644 --- a/drivers/vfio/pci/vfio_pci.c +++ b/drivers/vfio/pci/vfio_pci.c @@ -1924,6 +1924,68 @@ static int vfio_pci_bus_notifier(struct notifier_block *nb, return 0; } +static int vfio_pci_vf_init(struct vfio_pci_device *vdev) +{ + struct pci_dev *pdev = vdev->pdev; + int ret; + + if (!pdev->is_physfn) + return 0; + + vdev->vf_token = kzalloc(sizeof(*vdev->vf_token), GFP_KERNEL); + if (!vdev->vf_token) + return -ENOMEM; + + mutex_init(&vdev->vf_token->lock); + uuid_gen(&vdev->vf_token->uuid); + + vdev->nb.notifier_call = vfio_pci_bus_notifier; + ret = bus_register_notifier(&pci_bus_type, &vdev->nb); + if (ret) { + kfree(vdev->vf_token); + return ret; + } + return 0; +} + +static void vfio_pci_vf_uninit(struct vfio_pci_device *vdev) +{ + if (!vdev->vf_token) + return; + + bus_unregister_notifier(&pci_bus_type, &vdev->nb); + WARN_ON(vdev->vf_token->users); + mutex_destroy(&vdev->vf_token->lock); + kfree(vdev->vf_token); +} + +static int vfio_pci_vga_init(struct vfio_pci_device *vdev) +{ + struct pci_dev *pdev = vdev->pdev; + int ret; + + if (!vfio_pci_is_vga(pdev)) + return 0; + + ret = vga_client_register(pdev, vdev, NULL, vfio_pci_set_vga_decode); + if (ret) + return ret; + vga_set_legacy_decoding(pdev, vfio_pci_set_vga_decode(vdev, false)); + return 0; +} + +static void vfio_pci_vga_uninit(struct vfio_pci_device *vdev) +{ + struct pci_dev *pdev = vdev->pdev; + + if (!vfio_pci_is_vga(pdev)) + return; + vga_client_register(pdev, NULL, NULL, NULL); + vga_set_legacy_decoding(pdev, VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM | + VGA_RSRC_LEGACY_IO | + VGA_RSRC_LEGACY_MEM); +} + static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { struct vfio_pci_device *vdev; @@ -1970,35 +2032,15 @@ static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) INIT_LIST_HEAD(&vdev->vma_list); init_rwsem(&vdev->memory_lock); - ret = vfio_add_group_dev(&pdev->dev, &vfio_pci_ops, vdev); + ret = vfio_pci_reflck_attach(vdev); if (ret) goto out_free; - - ret = vfio_pci_reflck_attach(vdev); + ret = vfio_pci_vf_init(vdev); if (ret) - goto out_del_group_dev; - - if (pdev->is_physfn) { - vdev->vf_token = kzalloc(sizeof(*vdev->vf_token), GFP_KERNEL); - if (!vdev->vf_token) { - ret = -ENOMEM; - goto out_reflck; - } - - mutex_init(&vdev->vf_token->lock); - uuid_gen(&vdev->vf_token->uuid); - - vdev->nb.notifier_call = vfio_pci_bus_notifier; - ret = bus_register_notifier(&pci_bus_type, &vdev->nb); - if (ret) - goto out_vf_token; - } - - if (vfio_pci_is_vga(pdev)) { - vga_client_register(pdev, vdev, NULL, vfio_pci_set_vga_decode); - vga_set_legacy_decoding(pdev, - vfio_pci_set_vga_decode(vdev, false)); - } + goto out_reflck; + ret = vfio_pci_vga_init(vdev); + if (ret) + goto out_vf; vfio_pci_probe_power_state(vdev); @@ -2016,15 +2058,20 @@ static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) vfio_pci_set_power_state(vdev, PCI_D3hot); } - return ret; + ret = vfio_add_group_dev(&pdev->dev, &vfio_pci_ops, vdev); + if (ret) + goto out_power; + return 0; -out_vf_token: - kfree(vdev->vf_token); +out_power: + if (!disable_idle_d3) + vfio_pci_set_power_state(vdev, PCI_D0); +out_vf: + vfio_pci_vf_uninit(vdev); out_reflck: vfio_pci_reflck_put(vdev->reflck); -out_del_group_dev: - vfio_del_group_dev(&pdev->dev); out_free: + kfree(vdev->pm_save); kfree(vdev); out_group_put: vfio_iommu_group_put(group, &pdev->dev); @@ -2041,33 +2088,19 @@ static void vfio_pci_remove(struct pci_dev *pdev) if (!vdev) return; - if (vdev->vf_token) { - WARN_ON(vdev->vf_token->users); - mutex_destroy(&vdev->vf_token->lock); - kfree(vdev->vf_token); - } - - if (vdev->nb.notifier_call) - bus_unregister_notifier(&pci_bus_type, &vdev->nb); - + vfio_pci_vf_uninit(vdev); vfio_pci_reflck_put(vdev->reflck); + vfio_pci_vga_uninit(vdev); vfio_iommu_group_put(pdev->dev.iommu_group, &pdev->dev); - kfree(vdev->region); - mutex_destroy(&vdev->ioeventfds_lock); if (!disable_idle_d3) vfio_pci_set_power_state(vdev, PCI_D0); + mutex_destroy(&vdev->ioeventfds_lock); + kfree(vdev->region); kfree(vdev->pm_save); kfree(vdev); - - if (vfio_pci_is_vga(pdev)) { - vga_client_register(pdev, NULL, NULL, NULL); - vga_set_legacy_decoding(pdev, - VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM | - VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM); - } } static pci_ers_result_t vfio_pci_aer_err_detected(struct pci_dev *pdev, diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index bfa4c6ef554e..c79d2f2387aa 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -993,6 +993,7 @@ static int vhost_vdpa_mmap(struct file *file, struct vm_area_struct *vma) if (vma->vm_end - vma->vm_start != notify.size) return -ENOTSUPP; + vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP; vma->vm_ops = &vhost_vdpa_vm_ops; return 0; } diff --git a/drivers/video/backlight/qcom-wled.c b/drivers/video/backlight/qcom-wled.c index 091f07e7c145..e9fbe2483844 100644 --- a/drivers/video/backlight/qcom-wled.c +++ b/drivers/video/backlight/qcom-wled.c @@ -336,19 +336,19 @@ static int wled3_sync_toggle(struct wled *wled) unsigned int mask = GENMASK(wled->max_string_count - 1, 0); rc = regmap_update_bits(wled->regmap, - wled->ctrl_addr + WLED3_SINK_REG_SYNC, + wled->sink_addr + WLED3_SINK_REG_SYNC, mask, mask); if (rc < 0) return rc; rc = regmap_update_bits(wled->regmap, - wled->ctrl_addr + WLED3_SINK_REG_SYNC, + wled->sink_addr + WLED3_SINK_REG_SYNC, mask, WLED3_SINK_REG_SYNC_CLEAR); return rc; } -static int wled5_sync_toggle(struct wled *wled) +static int wled5_mod_sync_toggle(struct wled *wled) { int rc; u8 val; @@ -445,10 +445,23 @@ static int wled_update_status(struct backlight_device *bl) goto unlock_mutex; } - rc = wled->wled_sync_toggle(wled); - if (rc < 0) { - dev_err(wled->dev, "wled sync failed rc:%d\n", rc); - goto unlock_mutex; + if (wled->version < 5) { + rc = wled->wled_sync_toggle(wled); + if (rc < 0) { + dev_err(wled->dev, "wled sync failed rc:%d\n", rc); + goto unlock_mutex; + } + } else { + /* + * For WLED5 toggling the MOD_SYNC_BIT updates the + * brightness + */ + rc = wled5_mod_sync_toggle(wled); + if (rc < 0) { + dev_err(wled->dev, "wled mod sync failed rc:%d\n", + rc); + goto unlock_mutex; + } } } @@ -1459,7 +1472,7 @@ static int wled_configure(struct wled *wled) size = ARRAY_SIZE(wled5_opts); *cfg = wled5_config_defaults; wled->wled_set_brightness = wled5_set_brightness; - wled->wled_sync_toggle = wled5_sync_toggle; + wled->wled_sync_toggle = wled3_sync_toggle; wled->wled_cabc_config = wled5_cabc_config; wled->wled_ovp_delay = wled5_ovp_delay; wled->wled_auto_detection_required = diff --git a/drivers/video/fbdev/core/fbcmap.c b/drivers/video/fbdev/core/fbcmap.c index 757d5c3f620b..ff09e57f3c38 100644 --- a/drivers/video/fbdev/core/fbcmap.c +++ b/drivers/video/fbdev/core/fbcmap.c @@ -101,17 +101,17 @@ int fb_alloc_cmap_gfp(struct fb_cmap *cmap, int len, int transp, gfp_t flags) if (!len) return 0; - cmap->red = kmalloc(size, flags); + cmap->red = kzalloc(size, flags); if (!cmap->red) goto fail; - cmap->green = kmalloc(size, flags); + cmap->green = kzalloc(size, flags); if (!cmap->green) goto fail; - cmap->blue = kmalloc(size, flags); + cmap->blue = kzalloc(size, flags); if (!cmap->blue) goto fail; if (transp) { - cmap->transp = kmalloc(size, flags); + cmap->transp = kzalloc(size, flags); if (!cmap->transp) goto fail; } else { diff --git a/drivers/virt/nitro_enclaves/ne_misc_dev.c b/drivers/virt/nitro_enclaves/ne_misc_dev.c index f1964ea4b826..e21e1e86ad15 100644 --- a/drivers/virt/nitro_enclaves/ne_misc_dev.c +++ b/drivers/virt/nitro_enclaves/ne_misc_dev.c @@ -1524,7 +1524,8 @@ static const struct file_operations ne_enclave_fops = { * enclave file descriptor to be further used for enclave * resources handling e.g. memory regions and CPUs. * @ne_pci_dev : Private data associated with the PCI device. - * @slot_uid: Generated unique slot id associated with an enclave. + * @slot_uid: User pointer to store the generated unique slot id + * associated with an enclave to. * * Context: Process context. This function is called with the ne_pci_dev enclave * mutex held. @@ -1532,7 +1533,7 @@ static const struct file_operations ne_enclave_fops = { * * Enclave fd on success. * * Negative return value on failure. */ -static int ne_create_vm_ioctl(struct ne_pci_dev *ne_pci_dev, u64 *slot_uid) +static int ne_create_vm_ioctl(struct ne_pci_dev *ne_pci_dev, u64 __user *slot_uid) { struct ne_pci_dev_cmd_reply cmd_reply = {}; int enclave_fd = -1; @@ -1634,7 +1635,18 @@ static int ne_create_vm_ioctl(struct ne_pci_dev *ne_pci_dev, u64 *slot_uid) list_add(&ne_enclave->enclave_list_entry, &ne_pci_dev->enclaves_list); - *slot_uid = ne_enclave->slot_uid; + if (copy_to_user(slot_uid, &ne_enclave->slot_uid, sizeof(ne_enclave->slot_uid))) { + /* + * As we're holding the only reference to 'enclave_file', fput() + * will call ne_enclave_release() which will do a proper cleanup + * of all so far allocated resources, leaving only the unused fd + * for us to free. + */ + fput(enclave_file); + put_unused_fd(enclave_fd); + + return -EFAULT; + } fd_install(enclave_fd, enclave_file); @@ -1671,34 +1683,13 @@ static long ne_ioctl(struct file *file, unsigned int cmd, unsigned long arg) switch (cmd) { case NE_CREATE_VM: { int enclave_fd = -1; - struct file *enclave_file = NULL; struct ne_pci_dev *ne_pci_dev = ne_devs.ne_pci_dev; - int rc = -EINVAL; - u64 slot_uid = 0; + u64 __user *slot_uid = (void __user *)arg; mutex_lock(&ne_pci_dev->enclaves_list_mutex); - - enclave_fd = ne_create_vm_ioctl(ne_pci_dev, &slot_uid); - if (enclave_fd < 0) { - rc = enclave_fd; - - mutex_unlock(&ne_pci_dev->enclaves_list_mutex); - - return rc; - } - + enclave_fd = ne_create_vm_ioctl(ne_pci_dev, slot_uid); mutex_unlock(&ne_pci_dev->enclaves_list_mutex); - if (copy_to_user((void __user *)arg, &slot_uid, sizeof(slot_uid))) { - enclave_file = fget(enclave_fd); - /* Decrement file refs to have release() called. */ - fput(enclave_file); - fput(enclave_file); - put_unused_fd(enclave_fd); - - return -EFAULT; - } - return enclave_fd; } diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c index f01d58c7a042..a3e7be96527d 100644 --- a/drivers/xen/gntdev.c +++ b/drivers/xen/gntdev.c @@ -1017,8 +1017,10 @@ static int gntdev_mmap(struct file *flip, struct vm_area_struct *vma) err = mmu_interval_notifier_insert_locked( &map->notifier, vma->vm_mm, vma->vm_start, vma->vm_end - vma->vm_start, &gntdev_mmu_ops); - if (err) + if (err) { + map->vma = NULL; goto out_unlock_put; + } } mutex_unlock(&priv->lock); diff --git a/drivers/xen/unpopulated-alloc.c b/drivers/xen/unpopulated-alloc.c index e64e6befc63b..87e6b7db892f 100644 --- a/drivers/xen/unpopulated-alloc.c +++ b/drivers/xen/unpopulated-alloc.c @@ -39,8 +39,10 @@ static int fill_list(unsigned int nr_pages) } pgmap = kzalloc(sizeof(*pgmap), GFP_KERNEL); - if (!pgmap) + if (!pgmap) { + ret = -ENOMEM; goto err_pgmap; + } pgmap->type = MEMORY_DEVICE_GENERIC; pgmap->range = (struct range) { diff --git a/fs/9p/vfs_file.c b/fs/9p/vfs_file.c index 649f04f112dc..59c32c9b799f 100644 --- a/fs/9p/vfs_file.c +++ b/fs/9p/vfs_file.c @@ -86,8 +86,8 @@ int v9fs_file_open(struct inode *inode, struct file *file) * to work. */ writeback_fid = v9fs_writeback_fid(file_dentry(file)); - if (IS_ERR(fid)) { - err = PTR_ERR(fid); + if (IS_ERR(writeback_fid)) { + err = PTR_ERR(writeback_fid); mutex_unlock(&v9inode->v_mutex); goto out_error; } diff --git a/fs/Kconfig b/fs/Kconfig index a55bda4233bb..f61330e4efc0 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -145,6 +145,7 @@ menu "DOS/FAT/EXFAT/NT Filesystems" source "fs/fat/Kconfig" source "fs/exfat/Kconfig" source "fs/ntfs/Kconfig" +source "fs/ntfs3/Kconfig" endmenu endif # BLOCK diff --git a/fs/Makefile b/fs/Makefile index 3215fe205256..6bdfcf712cb1 100644 --- a/fs/Makefile +++ b/fs/Makefile @@ -99,6 +99,7 @@ obj-$(CONFIG_SYSV_FS) += sysv/ obj-$(CONFIG_CIFS) += cifs/ obj-$(CONFIG_HPFS_FS) += hpfs/ obj-$(CONFIG_NTFS_FS) += ntfs/ +obj-$(CONFIG_NTFS3_FS) += ntfs3/ obj-$(CONFIG_UFS_FS) += ufs/ obj-$(CONFIG_EFS_FS) += efs/ obj-$(CONFIG_JFFS2_FS) += jffs2/ diff --git a/fs/afs/dir.c b/fs/afs/dir.c index 17548c1faf02..31251d11d576 100644 --- a/fs/afs/dir.c +++ b/fs/afs/dir.c @@ -1342,6 +1342,7 @@ static int afs_mkdir(struct user_namespace *mnt_userns, struct inode *dir, afs_op_set_vnode(op, 0, dvnode); op->file[0].dv_delta = 1; + op->file[0].modification = true; op->file[0].update_ctime = true; op->dentry = dentry; op->create.mode = S_IFDIR | mode; @@ -1423,6 +1424,7 @@ static int afs_rmdir(struct inode *dir, struct dentry *dentry) afs_op_set_vnode(op, 0, dvnode); op->file[0].dv_delta = 1; + op->file[0].modification = true; op->file[0].update_ctime = true; op->dentry = dentry; @@ -1559,6 +1561,7 @@ static int afs_unlink(struct inode *dir, struct dentry *dentry) afs_op_set_vnode(op, 0, dvnode); op->file[0].dv_delta = 1; + op->file[0].modification = true; op->file[0].update_ctime = true; /* Try to make sure we have a callback promise on the victim. */ @@ -1641,6 +1644,7 @@ static int afs_create(struct user_namespace *mnt_userns, struct inode *dir, afs_op_set_vnode(op, 0, dvnode); op->file[0].dv_delta = 1; + op->file[0].modification = true; op->file[0].update_ctime = true; op->dentry = dentry; @@ -1715,6 +1719,7 @@ static int afs_link(struct dentry *from, struct inode *dir, afs_op_set_vnode(op, 0, dvnode); afs_op_set_vnode(op, 1, vnode); op->file[0].dv_delta = 1; + op->file[0].modification = true; op->file[0].update_ctime = true; op->file[1].update_ctime = true; @@ -1910,6 +1915,8 @@ static int afs_rename(struct user_namespace *mnt_userns, struct inode *old_dir, afs_op_set_vnode(op, 1, new_dvnode); /* May be same as orig_dvnode */ op->file[0].dv_delta = 1; op->file[1].dv_delta = 1; + op->file[0].modification = true; + op->file[1].modification = true; op->file[0].update_ctime = true; op->file[1].update_ctime = true; diff --git a/fs/afs/dir_silly.c b/fs/afs/dir_silly.c index 04f75a44f243..dae9a57d7ec0 100644 --- a/fs/afs/dir_silly.c +++ b/fs/afs/dir_silly.c @@ -73,6 +73,8 @@ static int afs_do_silly_rename(struct afs_vnode *dvnode, struct afs_vnode *vnode afs_op_set_vnode(op, 1, dvnode); op->file[0].dv_delta = 1; op->file[1].dv_delta = 1; + op->file[0].modification = true; + op->file[1].modification = true; op->file[0].update_ctime = true; op->file[1].update_ctime = true; @@ -201,6 +203,7 @@ static int afs_do_silly_unlink(struct afs_vnode *dvnode, struct afs_vnode *vnode afs_op_set_vnode(op, 0, dvnode); afs_op_set_vnode(op, 1, vnode); op->file[0].dv_delta = 1; + op->file[0].modification = true; op->file[0].update_ctime = true; op->file[1].op_unlinked = true; op->file[1].update_ctime = true; diff --git a/fs/afs/fs_operation.c b/fs/afs/fs_operation.c index 71c58723763d..a82515b47350 100644 --- a/fs/afs/fs_operation.c +++ b/fs/afs/fs_operation.c @@ -118,6 +118,8 @@ static void afs_prepare_vnode(struct afs_operation *op, struct afs_vnode_param * vp->cb_break_before = afs_calc_vnode_cb_break(vnode); if (vnode->lock_state != AFS_VNODE_LOCK_NONE) op->flags |= AFS_OPERATION_CUR_ONLY; + if (vp->modification) + set_bit(AFS_VNODE_MODIFYING, &vnode->flags); } if (vp->fid.vnode) @@ -223,6 +225,10 @@ int afs_put_operation(struct afs_operation *op) if (op->ops && op->ops->put) op->ops->put(op); + if (op->file[0].modification) + clear_bit(AFS_VNODE_MODIFYING, &op->file[0].vnode->flags); + if (op->file[1].modification && op->file[1].vnode != op->file[0].vnode) + clear_bit(AFS_VNODE_MODIFYING, &op->file[1].vnode->flags); if (op->file[0].put_vnode) iput(&op->file[0].vnode->vfs_inode); if (op->file[1].put_vnode) diff --git a/fs/afs/inode.c b/fs/afs/inode.c index 12be88716e4c..fddf7d54e0b7 100644 --- a/fs/afs/inode.c +++ b/fs/afs/inode.c @@ -102,13 +102,13 @@ static int afs_inode_init_from_status(struct afs_operation *op, switch (status->type) { case AFS_FTYPE_FILE: - inode->i_mode = S_IFREG | status->mode; + inode->i_mode = S_IFREG | (status->mode & S_IALLUGO); inode->i_op = &afs_file_inode_operations; inode->i_fop = &afs_file_operations; inode->i_mapping->a_ops = &afs_fs_aops; break; case AFS_FTYPE_DIR: - inode->i_mode = S_IFDIR | status->mode; + inode->i_mode = S_IFDIR | (status->mode & S_IALLUGO); inode->i_op = &afs_dir_inode_operations; inode->i_fop = &afs_dir_file_operations; inode->i_mapping->a_ops = &afs_dir_aops; @@ -198,7 +198,7 @@ static void afs_apply_status(struct afs_operation *op, if (status->mode != vnode->status.mode) { mode = inode->i_mode; mode &= ~S_IALLUGO; - mode |= status->mode; + mode |= status->mode & S_IALLUGO; WRITE_ONCE(inode->i_mode, mode); } @@ -293,8 +293,9 @@ void afs_vnode_commit_status(struct afs_operation *op, struct afs_vnode_param *v op->flags &= ~AFS_OPERATION_DIR_CONFLICT; } } else if (vp->scb.have_status) { - if (vp->dv_before + vp->dv_delta != vp->scb.status.data_version && - vp->speculative) + if (vp->speculative && + (test_bit(AFS_VNODE_MODIFYING, &vnode->flags) || + vp->dv_before != vnode->status.data_version)) /* Ignore the result of a speculative bulk status fetch * if it splits around a modification op, thereby * appearing to regress the data version. @@ -910,6 +911,7 @@ int afs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, } op->ctime = attr->ia_ctime; op->file[0].update_ctime = 1; + op->file[0].modification = true; op->ops = &afs_setattr_operation; ret = afs_do_sync_operation(op); diff --git a/fs/afs/internal.h b/fs/afs/internal.h index 1627b1872812..be981a9a1add 100644 --- a/fs/afs/internal.h +++ b/fs/afs/internal.h @@ -640,6 +640,7 @@ struct afs_vnode { #define AFS_VNODE_PSEUDODIR 7 /* set if Vnode is a pseudo directory */ #define AFS_VNODE_NEW_CONTENT 8 /* Set if file has new content (create/trunc-0) */ #define AFS_VNODE_SILLY_DELETED 9 /* Set if file has been silly-deleted */ +#define AFS_VNODE_MODIFYING 10 /* Set if we're performing a modification op */ struct list_head wb_keys; /* List of keys available for writeback */ struct list_head pending_locks; /* locks waiting to be granted */ @@ -756,6 +757,7 @@ struct afs_vnode_param { bool set_size:1; /* Must update i_size */ bool op_unlinked:1; /* True if file was unlinked by op */ bool speculative:1; /* T if speculative status fetch (no vnode lock) */ + bool modification:1; /* Set if the content gets modified */ }; /* diff --git a/fs/afs/write.c b/fs/afs/write.c index eb737ed63afb..ebe3b6493fce 100644 --- a/fs/afs/write.c +++ b/fs/afs/write.c @@ -450,6 +450,7 @@ static int afs_store_data(struct address_space *mapping, afs_op_set_vnode(op, 0, vnode); op->file[0].dv_delta = 1; op->store.mapping = mapping; + op->file[0].modification = true; op->store.first = first; op->store.last = last; op->store.first_offset = offset; diff --git a/fs/block_dev.c b/fs/block_dev.c index 09d6f7229db9..a5a6a7930e5e 100644 --- a/fs/block_dev.c +++ b/fs/block_dev.c @@ -1684,6 +1684,7 @@ ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from) struct inode *bd_inode = bdev_file_inode(file); loff_t size = i_size_read(bd_inode); struct blk_plug plug; + size_t shorted = 0; ssize_t ret; if (bdev_read_only(I_BDEV(bd_inode))) @@ -1701,12 +1702,17 @@ ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from) if ((iocb->ki_flags & (IOCB_NOWAIT | IOCB_DIRECT)) == IOCB_NOWAIT) return -EOPNOTSUPP; - iov_iter_truncate(from, size - iocb->ki_pos); + size -= iocb->ki_pos; + if (iov_iter_count(from) > size) { + shorted = iov_iter_count(from) - size; + iov_iter_truncate(from, size); + } blk_start_plug(&plug); ret = __generic_file_write_iter(iocb, from); if (ret > 0) ret = generic_write_sync(iocb, ret); + iov_iter_reexpand(from, iov_iter_count(from) + shorted); blk_finish_plug(&plug); return ret; } @@ -1718,13 +1724,21 @@ ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to) struct inode *bd_inode = bdev_file_inode(file); loff_t size = i_size_read(bd_inode); loff_t pos = iocb->ki_pos; + size_t shorted = 0; + ssize_t ret; if (pos >= size) return 0; size -= pos; - iov_iter_truncate(to, size); - return generic_file_read_iter(iocb, to); + if (iov_iter_count(to) > size) { + shorted = iov_iter_count(to) - size; + iov_iter_truncate(to, size); + } + + ret = generic_file_read_iter(iocb, to); + iov_iter_reexpand(to, iov_iter_count(to) + shorted); + return ret; } EXPORT_SYMBOL_GPL(blkdev_read_iter); diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 744b99ddc28c..a7d9e147dee6 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -3269,6 +3269,7 @@ static u64 get_profile_num_devs(struct btrfs_fs_info *fs_info, u64 type) */ void check_system_chunk(struct btrfs_trans_handle *trans, u64 type) { + struct btrfs_transaction *cur_trans = trans->transaction; struct btrfs_fs_info *fs_info = trans->fs_info; struct btrfs_space_info *info; u64 left; @@ -3283,6 +3284,7 @@ void check_system_chunk(struct btrfs_trans_handle *trans, u64 type) lockdep_assert_held(&fs_info->chunk_mutex); info = btrfs_find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM); +again: spin_lock(&info->lock); left = info->total_bytes - btrfs_space_info_used(info, true); spin_unlock(&info->lock); @@ -3301,6 +3303,58 @@ void check_system_chunk(struct btrfs_trans_handle *trans, u64 type) if (left < thresh) { u64 flags = btrfs_system_alloc_profile(fs_info); + u64 reserved = atomic64_read(&cur_trans->chunk_bytes_reserved); + + /* + * If there's not available space for the chunk tree (system + * space) and there are other tasks that reserved space for + * creating a new system block group, wait for them to complete + * the creation of their system block group and release excess + * reserved space. We do this because: + * + * *) We can end up allocating more system chunks than necessary + * when there are multiple tasks that are concurrently + * allocating block groups, which can lead to exhaustion of + * the system array in the superblock; + * + * *) If we allocate extra and unnecessary system block groups, + * despite being empty for a long time, and possibly forever, + * they end not being added to the list of unused block groups + * because that typically happens only when deallocating the + * last extent from a block group - which never happens since + * we never allocate from them in the first place. The few + * exceptions are when mounting a filesystem or running scrub, + * which add unused block groups to the list of unused block + * groups, to be deleted by the cleaner kthread. + * And even when they are added to the list of unused block + * groups, it can take a long time until they get deleted, + * since the cleaner kthread might be sleeping or busy with + * other work (deleting subvolumes, running delayed iputs, + * defrag scheduling, etc); + * + * This is rare in practice, but can happen when too many tasks + * are allocating blocks groups in parallel (via fallocate()) + * and before the one that reserved space for a new system block + * group finishes the block group creation and releases the space + * reserved in excess (at btrfs_create_pending_block_groups()), + * other tasks end up here and see free system space temporarily + * not enough for updating the chunk tree. + * + * We unlock the chunk mutex before waiting for such tasks and + * lock it again after the wait, otherwise we would deadlock. + * It is safe to do so because allocating a system chunk is the + * first thing done while allocating a new block group. + */ + if (reserved > trans->chunk_bytes_reserved) { + const u64 min_needed = reserved - thresh; + + mutex_unlock(&fs_info->chunk_mutex); + wait_event(cur_trans->chunk_reserve_wait, + atomic64_read(&cur_trans->chunk_bytes_reserved) <= + min_needed); + mutex_lock(&fs_info->chunk_mutex); + goto again; + } /* * Ignore failure to create system chunk. We might end up not @@ -3315,8 +3369,10 @@ void check_system_chunk(struct btrfs_trans_handle *trans, u64 type) ret = btrfs_block_rsv_add(fs_info->chunk_root, &fs_info->chunk_block_rsv, thresh, BTRFS_RESERVE_NO_FLUSH); - if (!ret) + if (!ret) { + atomic64_add(thresh, &cur_trans->chunk_bytes_reserved); trans->chunk_bytes_reserved += thresh; + } } } diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h index 28e202e89660..418903604936 100644 --- a/fs/btrfs/btrfs_inode.h +++ b/fs/btrfs/btrfs_inode.h @@ -299,6 +299,21 @@ static inline void btrfs_mod_outstanding_extents(struct btrfs_inode *inode, mod); } +/* + * Called every time after doing a buffered, direct IO or memory mapped write. + * + * This is to ensure that if we write to a file that was previously fsynced in + * the current transaction, then try to fsync it again in the same transaction, + * we will know that there were changes in the file and that it needs to be + * logged. + */ +static inline void btrfs_set_inode_last_sub_trans(struct btrfs_inode *inode) +{ + spin_lock(&inode->lock); + inode->last_sub_trans = inode->root->log_transid; + spin_unlock(&inode->lock); +} + static inline int btrfs_inode_in_log(struct btrfs_inode *inode, u64 generation) { int ret = 0; diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 3f4c832abfed..81387cdf334d 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -80,10 +80,15 @@ static int compression_compress_pages(int type, struct list_head *ws, case BTRFS_COMPRESS_NONE: default: /* - * This can't happen, the type is validated several times - * before we get here. As a sane fallback, return what the - * callers will understand as 'no compression happened'. + * This can happen when compression races with remount setting + * it to 'no compress', while caller doesn't call + * inode_need_compress() to check if we really need to + * compress. + * + * Not a big deal, just need to inform caller that we + * haven't allocated any pages yet. */ + *out_pages = 0; return -E2BIG; } } diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c index 34b929bd5c1a..f43ce82a6aed 100644 --- a/fs/btrfs/ctree.c +++ b/fs/btrfs/ctree.c @@ -1365,10 +1365,30 @@ get_old_root(struct btrfs_root *root, u64 time_seq) "failed to read tree block %llu from get_old_root", logical); } else { + struct tree_mod_elem *tm2; + btrfs_tree_read_lock(old); eb = btrfs_clone_extent_buffer(old); + /* + * After the lookup for the most recent tree mod operation + * above and before we locked and cloned the extent buffer + * 'old', a new tree mod log operation may have been added. + * So lookup for a more recent one to make sure the number + * of mod log operations we replay is consistent with the + * number of items we have in the cloned extent buffer, + * otherwise we can hit a BUG_ON when rewinding the extent + * buffer. + */ + tm2 = tree_mod_log_search(fs_info, logical, time_seq); btrfs_tree_read_unlock(old); free_extent_buffer(old); + ASSERT(tm2); + ASSERT(tm2 == tm || tm2->seq > tm->seq); + if (!tm2 || tm2->seq < tm->seq) { + free_extent_buffer(eb); + return NULL; + } + tm = tm2; } } else if (old_root) { eb_root_owner = btrfs_header_owner(eb_root); diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 9ae776ab3967..29ef969035df 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -3110,7 +3110,7 @@ int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans, struct btrfs_inode *inode, u64 new_size, u32 min_type); -int btrfs_start_delalloc_snapshot(struct btrfs_root *root); +int btrfs_start_delalloc_snapshot(struct btrfs_root *root, bool in_reclaim_context); int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, long nr, bool in_reclaim_context); int btrfs_set_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end, diff --git a/fs/btrfs/delayed-inode.c b/fs/btrfs/delayed-inode.c index bf25401c9768..c1d2b6786129 100644 --- a/fs/btrfs/delayed-inode.c +++ b/fs/btrfs/delayed-inode.c @@ -1589,8 +1589,8 @@ bool btrfs_readdir_get_delayed_items(struct inode *inode, * We can only do one readdir with delayed items at a time because of * item->readdir_list. */ - inode_unlock_shared(inode); - inode_lock(inode); + btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); + btrfs_inode_lock(inode, 0); mutex_lock(&delayed_node->mutex); item = __btrfs_first_delayed_insertion_item(delayed_node); diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 36a3c973fda1..5b82050b871a 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -1340,12 +1340,16 @@ int btrfs_discard_extent(struct btrfs_fs_info *fs_info, u64 bytenr, stripe = bbio->stripes; for (i = 0; i < bbio->num_stripes; i++, stripe++) { u64 bytes; + struct btrfs_device *device = stripe->dev; - if (!stripe->dev->bdev) { + if (!device->bdev) { ASSERT(btrfs_test_opt(fs_info, DEGRADED)); continue; } + if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) + continue; + ret = do_discard_extent(stripe, &bytes); if (!ret) { discarded_bytes += bytes; diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index 0e155f013839..abee4b62741d 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -2014,14 +2014,8 @@ static ssize_t btrfs_file_write_iter(struct kiocb *iocb, else num_written = btrfs_buffered_write(iocb, from); - /* - * We also have to set last_sub_trans to the current log transid, - * otherwise subsequent syncs to a file that's been synced in this - * transaction will appear to have already occurred. - */ - spin_lock(&inode->lock); - inode->last_sub_trans = inode->root->log_transid; - spin_unlock(&inode->lock); + btrfs_set_inode_last_sub_trans(inode); + if (num_written > 0) num_written = generic_write_sync(iocb, num_written); @@ -2073,6 +2067,30 @@ static int start_ordered_ops(struct inode *inode, loff_t start, loff_t end) return ret; } +static inline bool skip_inode_logging(const struct btrfs_log_ctx *ctx) +{ + struct btrfs_inode *inode = BTRFS_I(ctx->inode); + struct btrfs_fs_info *fs_info = inode->root->fs_info; + + if (btrfs_inode_in_log(inode, fs_info->generation) && + list_empty(&ctx->ordered_extents)) + return true; + + /* + * If we are doing a fast fsync we can not bail out if the inode's + * last_trans is <= then the last committed transaction, because we only + * update the last_trans of the inode during ordered extent completion, + * and for a fast fsync we don't wait for that, we only wait for the + * writeback to complete. + */ + if (inode->last_trans <= fs_info->last_trans_committed && + (test_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags) || + list_empty(&ctx->ordered_extents))) + return true; + + return false; +} + /* * fsync call for both files and directories. This logs the inode into * the tree log instead of forcing full commits whenever possible. @@ -2122,7 +2140,7 @@ int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync) if (ret) goto out; - inode_lock(inode); + btrfs_inode_lock(inode, 0); atomic_inc(&root->log_batch); @@ -2154,7 +2172,7 @@ int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync) */ ret = start_ordered_ops(inode, start, end); if (ret) { - inode_unlock(inode); + btrfs_inode_unlock(inode, 0); goto out; } @@ -2191,17 +2209,8 @@ int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync) atomic_inc(&root->log_batch); - /* - * If we are doing a fast fsync we can not bail out if the inode's - * last_trans is <= then the last committed transaction, because we only - * update the last_trans of the inode during ordered extent completion, - * and for a fast fsync we don't wait for that, we only wait for the - * writeback to complete. - */ smp_mb(); - if (btrfs_inode_in_log(BTRFS_I(inode), fs_info->generation) || - (BTRFS_I(inode)->last_trans <= fs_info->last_trans_committed && - (full_sync || list_empty(&ctx.ordered_extents)))) { + if (skip_inode_logging(&ctx)) { /* * We've had everything committed since the last time we were * modified so clear this flag in case it was set for whatever @@ -2255,7 +2264,7 @@ int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync) * file again, but that will end up using the synchronization * inside btrfs_sync_log to keep things safe. */ - inode_unlock(inode); + btrfs_inode_unlock(inode, 0); if (ret != BTRFS_NO_LOG_SYNC) { if (!ret) { @@ -2285,7 +2294,7 @@ int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync) out_release_extents: btrfs_release_log_ctx_extents(&ctx); - inode_unlock(inode); + btrfs_inode_unlock(inode, 0); goto out; } @@ -2735,8 +2744,6 @@ int btrfs_replace_file_extents(struct inode *inode, struct btrfs_path *path, extent_info->file_offset += replace_len; } - cur_offset = drop_args.drop_end; - ret = btrfs_update_inode(trans, root, BTRFS_I(inode)); if (ret) break; @@ -2756,7 +2763,9 @@ int btrfs_replace_file_extents(struct inode *inode, struct btrfs_path *path, BUG_ON(ret); /* shouldn't happen */ trans->block_rsv = rsv; - if (!extent_info) { + cur_offset = drop_args.drop_end; + len = end - cur_offset; + if (!extent_info && len) { ret = find_first_non_hole(BTRFS_I(inode), &cur_offset, &len); if (unlikely(ret < 0)) @@ -2868,7 +2877,7 @@ static int btrfs_punch_hole(struct inode *inode, loff_t offset, loff_t len) if (ret) return ret; - inode_lock(inode); + btrfs_inode_lock(inode, 0); ino_size = round_up(inode->i_size, fs_info->sectorsize); ret = find_first_non_hole(BTRFS_I(inode), &offset, &len); if (ret < 0) @@ -2908,7 +2917,7 @@ static int btrfs_punch_hole(struct inode *inode, loff_t offset, loff_t len) truncated_block = true; ret = btrfs_truncate_block(BTRFS_I(inode), offset, 0, 0); if (ret) { - inode_unlock(inode); + btrfs_inode_unlock(inode, 0); return ret; } } @@ -3009,7 +3018,7 @@ static int btrfs_punch_hole(struct inode *inode, loff_t offset, loff_t len) ret = ret2; } } - inode_unlock(inode); + btrfs_inode_unlock(inode, 0); return ret; } @@ -3377,7 +3386,7 @@ static long btrfs_fallocate(struct file *file, int mode, if (mode & FALLOC_FL_ZERO_RANGE) { ret = btrfs_zero_range(inode, offset, len, mode); - inode_unlock(inode); + btrfs_inode_unlock(inode, 0); return ret; } @@ -3487,7 +3496,7 @@ static long btrfs_fallocate(struct file *file, int mode, unlock_extent_cached(&BTRFS_I(inode)->io_tree, alloc_start, locked_end, &cached_state); out: - inode_unlock(inode); + btrfs_inode_unlock(inode, 0); /* Let go of our reservation. */ if (ret != 0 && !(mode & FALLOC_FL_ZERO_RANGE)) btrfs_free_reserved_data_space(BTRFS_I(inode), data_reserved, diff --git a/fs/btrfs/free-space-cache.c b/fs/btrfs/free-space-cache.c index 9988decd5717..ac9c2691376d 100644 --- a/fs/btrfs/free-space-cache.c +++ b/fs/btrfs/free-space-cache.c @@ -3942,7 +3942,7 @@ static int cleanup_free_space_cache_v1(struct btrfs_fs_info *fs_info, { struct btrfs_block_group *block_group; struct rb_node *node; - int ret; + int ret = 0; btrfs_info(fs_info, "cleaning free space cache v1"); diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index a520775949a0..8c4d2eaa5d58 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -8619,9 +8619,7 @@ vm_fault_t btrfs_page_mkwrite(struct vm_fault *vmf) set_page_dirty(page); SetPageUptodate(page); - BTRFS_I(inode)->last_trans = fs_info->generation; - BTRFS_I(inode)->last_sub_trans = BTRFS_I(inode)->root->log_transid; - BTRFS_I(inode)->last_log_commit = BTRFS_I(inode)->root->last_log_commit; + btrfs_set_inode_last_sub_trans(BTRFS_I(inode)); unlock_extent_cached(io_tree, page_start, page_end, &cached_state); @@ -9674,7 +9672,7 @@ static int start_delalloc_inodes(struct btrfs_root *root, return ret; } -int btrfs_start_delalloc_snapshot(struct btrfs_root *root) +int btrfs_start_delalloc_snapshot(struct btrfs_root *root, bool in_reclaim_context) { struct writeback_control wbc = { .nr_to_write = LONG_MAX, @@ -9687,7 +9685,7 @@ int btrfs_start_delalloc_snapshot(struct btrfs_root *root) if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) return -EROFS; - return start_delalloc_inodes(root, &wbc, true, false); + return start_delalloc_inodes(root, &wbc, true, in_reclaim_context); } int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, long nr, diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index e8d53fea4c61..f9ecb6c0bf15 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -226,7 +226,7 @@ static int btrfs_ioctl_setflags(struct file *file, void __user *arg) if (ret) return ret; - inode_lock(inode); + btrfs_inode_lock(inode, 0); fsflags = btrfs_mask_fsflags_for_type(inode, fsflags); old_fsflags = btrfs_inode_flags_to_fsflags(binode->flags); @@ -353,7 +353,7 @@ static int btrfs_ioctl_setflags(struct file *file, void __user *arg) out_end_trans: btrfs_end_transaction(trans); out_unlock: - inode_unlock(inode); + btrfs_inode_unlock(inode, 0); mnt_drop_write_file(file); return ret; } @@ -449,7 +449,7 @@ static int btrfs_ioctl_fssetxattr(struct file *file, void __user *arg) if (ret) return ret; - inode_lock(inode); + btrfs_inode_lock(inode, 0); old_flags = binode->flags; old_i_flags = inode->i_flags; @@ -501,7 +501,7 @@ static int btrfs_ioctl_fssetxattr(struct file *file, void __user *arg) inode->i_flags = old_i_flags; } - inode_unlock(inode); + btrfs_inode_unlock(inode, 0); mnt_drop_write_file(file); return ret; @@ -697,8 +697,6 @@ static noinline int create_subvol(struct inode *dir, btrfs_set_root_otransid(root_item, trans->transid); btrfs_tree_unlock(leaf); - free_extent_buffer(leaf); - leaf = NULL; btrfs_set_root_dirid(root_item, BTRFS_FIRST_FREE_OBJECTID); @@ -707,8 +705,22 @@ static noinline int create_subvol(struct inode *dir, key.type = BTRFS_ROOT_ITEM_KEY; ret = btrfs_insert_root(trans, fs_info->tree_root, &key, root_item); - if (ret) + if (ret) { + /* + * Since we don't abort the transaction in this case, free the + * tree block so that we don't leak space and leave the + * filesystem in an inconsistent state (an extent item in the + * extent tree without backreferences). Also no need to have + * the tree block locked since it is not in any tree at this + * point, so no other task can find it and use it. + */ + btrfs_free_tree_block(trans, root, leaf, 0, 1); + free_extent_buffer(leaf); goto fail; + } + + free_extent_buffer(leaf); + leaf = NULL; key.offset = (u64)-1; new_root = btrfs_get_new_fs_root(fs_info, objectid, anon_dev); @@ -1014,7 +1026,7 @@ static noinline int btrfs_mksubvol(const struct path *parent, out_dput: dput(dentry); out_unlock: - inode_unlock(dir); + btrfs_inode_unlock(dir, 0); return error; } @@ -1034,7 +1046,7 @@ static noinline int btrfs_mksnapshot(const struct path *parent, */ btrfs_drew_read_lock(&root->snapshot_lock); - ret = btrfs_start_delalloc_snapshot(root); + ret = btrfs_start_delalloc_snapshot(root, false); if (ret) goto out; @@ -1612,7 +1624,7 @@ int btrfs_defrag_file(struct inode *inode, struct file *file, ra_index += cluster; } - inode_lock(inode); + btrfs_inode_lock(inode, 0); if (IS_SWAPFILE(inode)) { ret = -ETXTBSY; } else { @@ -1621,13 +1633,13 @@ int btrfs_defrag_file(struct inode *inode, struct file *file, ret = cluster_pages_for_defrag(inode, pages, i, cluster); } if (ret < 0) { - inode_unlock(inode); + btrfs_inode_unlock(inode, 0); goto out_ra; } defrag_count += ret; balance_dirty_pages_ratelimited(inode->i_mapping); - inode_unlock(inode); + btrfs_inode_unlock(inode, 0); if (newer_than) { if (newer_off == (u64)-1) @@ -1675,9 +1687,9 @@ int btrfs_defrag_file(struct inode *inode, struct file *file, out_ra: if (do_compress) { - inode_lock(inode); + btrfs_inode_lock(inode, 0); BTRFS_I(inode)->defrag_compress = BTRFS_COMPRESS_NONE; - inode_unlock(inode); + btrfs_inode_unlock(inode, 0); } if (!file) kfree(ra); @@ -3112,9 +3124,9 @@ static noinline int btrfs_ioctl_snap_destroy(struct file *file, goto out_dput; } - inode_lock(inode); + btrfs_inode_lock(inode, 0); err = btrfs_delete_subvolume(dir, dentry); - inode_unlock(inode); + btrfs_inode_unlock(inode, 0); if (!err) { fsnotify_rmdir(dir, dentry); d_delete(dentry); @@ -3123,7 +3135,7 @@ static noinline int btrfs_ioctl_snap_destroy(struct file *file, out_dput: dput(dentry); out_unlock_dir: - inode_unlock(dir); + btrfs_inode_unlock(dir, 0); free_subvol_name: kfree(subvol_name_ptr); free_parent: diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c index 985a21558437..043e3fa961e0 100644 --- a/fs/btrfs/ordered-data.c +++ b/fs/btrfs/ordered-data.c @@ -995,7 +995,7 @@ int btrfs_split_ordered_extent(struct btrfs_ordered_extent *ordered, u64 pre, if (pre) ret = clone_ordered_extent(ordered, 0, pre); - if (post) + if (ret == 0 && post) ret = clone_ordered_extent(ordered, pre + ordered->disk_num_bytes, post); diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c index f0b9ef13153a..2991287a71a8 100644 --- a/fs/btrfs/qgroup.c +++ b/fs/btrfs/qgroup.c @@ -3579,7 +3579,7 @@ static int try_flush_qgroup(struct btrfs_root *root) return 0; } - ret = btrfs_start_delalloc_snapshot(root); + ret = btrfs_start_delalloc_snapshot(root, true); if (ret < 0) goto out; btrfs_wait_ordered_extents(root, U64_MAX, 0, (u64)-1); diff --git a/fs/btrfs/reflink.c b/fs/btrfs/reflink.c index 762881b777b3..0abbf050580d 100644 --- a/fs/btrfs/reflink.c +++ b/fs/btrfs/reflink.c @@ -833,7 +833,7 @@ loff_t btrfs_remap_file_range(struct file *src_file, loff_t off, return -EINVAL; if (same_inode) - inode_lock(src_inode); + btrfs_inode_lock(src_inode, 0); else lock_two_nondirectories(src_inode, dst_inode); @@ -849,7 +849,7 @@ loff_t btrfs_remap_file_range(struct file *src_file, loff_t off, out_unlock: if (same_inode) - inode_unlock(src_inode); + btrfs_inode_unlock(src_inode, 0); else unlock_two_nondirectories(src_inode, dst_inode); diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 232d5da7b7be..829dc8dcc151 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -733,10 +733,12 @@ static struct btrfs_root *create_reloc_root(struct btrfs_trans_handle *trans, struct extent_buffer *eb; struct btrfs_root_item *root_item; struct btrfs_key root_key; - int ret; + int ret = 0; + bool must_abort = false; root_item = kmalloc(sizeof(*root_item), GFP_NOFS); - BUG_ON(!root_item); + if (!root_item) + return ERR_PTR(-ENOMEM); root_key.objectid = BTRFS_TREE_RELOC_OBJECTID; root_key.type = BTRFS_ROOT_ITEM_KEY; @@ -748,7 +750,9 @@ static struct btrfs_root *create_reloc_root(struct btrfs_trans_handle *trans, /* called by btrfs_init_reloc_root */ ret = btrfs_copy_root(trans, root, root->commit_root, &eb, BTRFS_TREE_RELOC_OBJECTID); - BUG_ON(ret); + if (ret) + goto fail; + /* * Set the last_snapshot field to the generation of the commit * root - like this ctree.c:btrfs_block_can_be_shared() behaves @@ -769,9 +773,16 @@ static struct btrfs_root *create_reloc_root(struct btrfs_trans_handle *trans, */ ret = btrfs_copy_root(trans, root, root->node, &eb, BTRFS_TREE_RELOC_OBJECTID); - BUG_ON(ret); + if (ret) + goto fail; } + /* + * We have changed references at this point, we must abort the + * transaction if anything fails. + */ + must_abort = true; + memcpy(root_item, &root->root_item, sizeof(*root_item)); btrfs_set_root_bytenr(root_item, eb->start); btrfs_set_root_level(root_item, btrfs_header_level(eb)); @@ -789,14 +800,25 @@ static struct btrfs_root *create_reloc_root(struct btrfs_trans_handle *trans, ret = btrfs_insert_root(trans, fs_info->tree_root, &root_key, root_item); - BUG_ON(ret); + if (ret) + goto fail; + kfree(root_item); reloc_root = btrfs_read_tree_root(fs_info->tree_root, &root_key); - BUG_ON(IS_ERR(reloc_root)); + if (IS_ERR(reloc_root)) { + ret = PTR_ERR(reloc_root); + goto abort; + } set_bit(BTRFS_ROOT_SHAREABLE, &reloc_root->state); reloc_root->last_trans = trans->transid; return reloc_root; +fail: + kfree(root_item); +abort: + if (must_abort) + btrfs_abort_transaction(trans, ret); + return ERR_PTR(ret); } /* @@ -875,7 +897,7 @@ int btrfs_update_reloc_root(struct btrfs_trans_handle *trans, int ret; if (!have_reloc_root(root)) - goto out; + return 0; reloc_root = root->reloc_root; root_item = &reloc_root->root_item; @@ -908,10 +930,8 @@ int btrfs_update_reloc_root(struct btrfs_trans_handle *trans, ret = btrfs_update_root(trans, fs_info->tree_root, &reloc_root->root_key, root_item); - BUG_ON(ret); btrfs_put_root(reloc_root); -out: - return 0; + return ret; } /* @@ -1185,8 +1205,8 @@ int replace_path(struct btrfs_trans_handle *trans, struct reloc_control *rc, int ret; int slot; - BUG_ON(src->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID); - BUG_ON(dest->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID); + ASSERT(src->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID); + ASSERT(dest->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID); last_snapshot = btrfs_root_last_snapshot(&src->root_item); again: @@ -1217,7 +1237,7 @@ int replace_path(struct btrfs_trans_handle *trans, struct reloc_control *rc, parent = eb; while (1) { level = btrfs_header_level(parent); - BUG_ON(level < lowest_level); + ASSERT(level >= lowest_level); ret = btrfs_bin_search(parent, &key, &slot); if (ret < 0) @@ -2578,7 +2598,7 @@ static noinline_for_stack int prealloc_file_extent_cluster( return btrfs_end_transaction(trans); } - inode_lock(&inode->vfs_inode); + btrfs_inode_lock(&inode->vfs_inode, 0); for (nr = 0; nr < cluster->nr; nr++) { start = cluster->boundary[nr] - offset; if (nr + 1 < cluster->nr) @@ -2596,7 +2616,7 @@ static noinline_for_stack int prealloc_file_extent_cluster( if (ret) break; } - inode_unlock(&inode->vfs_inode); + btrfs_inode_unlock(&inode->vfs_inode, 0); if (cur_offset < prealloc_end) btrfs_free_reserved_data_space_noquota(inode->root->fs_info, diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index 3d9088eab2fc..b9202a1f1af1 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -3682,8 +3682,8 @@ int scrub_enumerate_chunks(struct scrub_ctx *sctx, spin_lock(&cache->lock); if (!cache->to_copy) { spin_unlock(&cache->lock); - ro_set = 0; - goto done; + btrfs_put_block_group(cache); + goto skip; } spin_unlock(&cache->lock); } @@ -3841,7 +3841,6 @@ int scrub_enumerate_chunks(struct scrub_ctx *sctx, cache, found_key.offset)) ro_set = 0; -done: down_write(&dev_replace->rwsem); dev_replace->cursor_left = dev_replace->cursor_right; dev_replace->item_needs_writeback = 1; diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 8f323859156b..8ae8f1732fd2 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -7139,7 +7139,7 @@ static int flush_delalloc_roots(struct send_ctx *sctx) int i; if (root) { - ret = btrfs_start_delalloc_snapshot(root); + ret = btrfs_start_delalloc_snapshot(root, false); if (ret) return ret; btrfs_wait_ordered_extents(root, U64_MAX, 0, U64_MAX); @@ -7147,7 +7147,7 @@ static int flush_delalloc_roots(struct send_ctx *sctx) for (i = 0; i < sctx->clone_roots_cnt; i++) { root = sctx->clone_roots[i].root; - ret = btrfs_start_delalloc_snapshot(root); + ret = btrfs_start_delalloc_snapshot(root, false); if (ret) return ret; btrfs_wait_ordered_extents(root, U64_MAX, 0, U64_MAX); diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index acff6bb49a97..d56d3e7ca324 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -260,6 +260,7 @@ static inline int extwriter_counter_read(struct btrfs_transaction *trans) void btrfs_trans_release_chunk_metadata(struct btrfs_trans_handle *trans) { struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_transaction *cur_trans = trans->transaction; if (!trans->chunk_bytes_reserved) return; @@ -268,6 +269,8 @@ void btrfs_trans_release_chunk_metadata(struct btrfs_trans_handle *trans) btrfs_block_rsv_release(fs_info, &fs_info->chunk_block_rsv, trans->chunk_bytes_reserved, NULL); + atomic64_sub(trans->chunk_bytes_reserved, &cur_trans->chunk_bytes_reserved); + cond_wake_up(&cur_trans->chunk_reserve_wait); trans->chunk_bytes_reserved = 0; } @@ -383,6 +386,8 @@ static noinline int join_transaction(struct btrfs_fs_info *fs_info, spin_lock_init(&cur_trans->dropped_roots_lock); INIT_LIST_HEAD(&cur_trans->releasing_ebs); spin_lock_init(&cur_trans->releasing_ebs_lock); + atomic64_set(&cur_trans->chunk_bytes_reserved, 0); + init_waitqueue_head(&cur_trans->chunk_reserve_wait); list_add_tail(&cur_trans->list, &fs_info->trans_list); extent_io_tree_init(fs_info, &cur_trans->dirty_pages, IO_TREE_TRANS_DIRTY_PAGES, fs_info->btree_inode); @@ -1961,7 +1966,6 @@ static void cleanup_transaction(struct btrfs_trans_handle *trans, int err) */ BUG_ON(list_empty(&cur_trans->list)); - list_del_init(&cur_trans->list); if (cur_trans == fs_info->running_transaction) { cur_trans->state = TRANS_STATE_COMMIT_DOING; spin_unlock(&fs_info->trans_lock); @@ -1970,6 +1974,17 @@ static void cleanup_transaction(struct btrfs_trans_handle *trans, int err) spin_lock(&fs_info->trans_lock); } + + /* + * Now that we know no one else is still using the transaction we can + * remove the transaction from the list of transactions. This avoids + * the transaction kthread from cleaning up the transaction while some + * other task is still using it, which could result in a use-after-free + * on things like log trees, as it forces the transaction kthread to + * wait for this transaction to be cleaned up by us. + */ + list_del_init(&cur_trans->list); + spin_unlock(&fs_info->trans_lock); btrfs_cleanup_one_transaction(trans->transaction, fs_info); diff --git a/fs/btrfs/transaction.h b/fs/btrfs/transaction.h index 6335716e513f..364cfbb4c5c5 100644 --- a/fs/btrfs/transaction.h +++ b/fs/btrfs/transaction.h @@ -96,6 +96,13 @@ struct btrfs_transaction { spinlock_t releasing_ebs_lock; struct list_head releasing_ebs; + + /* + * The number of bytes currently reserved, by all transaction handles + * attached to this transaction, for metadata extents of the chunk tree. + */ + atomic64_t chunk_bytes_reserved; + wait_queue_head_t chunk_reserve_wait; }; #define __TRANS_FREEZABLE (1U << 0) @@ -175,7 +182,7 @@ static inline void btrfs_set_inode_last_trans(struct btrfs_trans_handle *trans, spin_lock(&inode->lock); inode->last_trans = trans->transaction->transid; inode->last_sub_trans = inode->root->log_transid; - inode->last_log_commit = inode->root->last_log_commit; + inode->last_log_commit = inode->last_sub_trans - 1; spin_unlock(&inode->lock); } diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 92a368627791..47e76e79b3d6 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -3165,20 +3165,22 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, */ mutex_unlock(&root->log_mutex); - btrfs_init_log_ctx(&root_log_ctx, NULL); - - mutex_lock(&log_root_tree->log_mutex); - if (btrfs_is_zoned(fs_info)) { + mutex_lock(&fs_info->tree_root->log_mutex); if (!log_root_tree->node) { ret = btrfs_alloc_log_tree_node(trans, log_root_tree); if (ret) { - mutex_unlock(&log_root_tree->log_mutex); + mutex_unlock(&fs_info->tree_log_mutex); goto out; } } + mutex_unlock(&fs_info->tree_root->log_mutex); } + btrfs_init_log_ctx(&root_log_ctx, NULL); + + mutex_lock(&log_root_tree->log_mutex); + index2 = log_root_tree->log_transid % 2; list_add_tail(&root_log_ctx.list, &log_root_tree->log_ctxs[index2]); root_log_ctx.log_transid = log_root_tree->log_transid; @@ -6058,7 +6060,8 @@ static int btrfs_log_inode_parent(struct btrfs_trans_handle *trans, * (since logging them is pointless, a link count of 0 means they * will never be accessible). */ - if (btrfs_inode_in_log(inode, trans->transid) || + if ((btrfs_inode_in_log(inode, trans->transid) && + list_empty(&ctx->ordered_extents)) || inode->vfs_inode.i_nlink == 0) { ret = BTRFS_NO_LOG_SYNC; goto end_no_trans; diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index 1c6810bbaf8b..3912eda7905f 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -4989,6 +4989,8 @@ static void init_alloc_chunk_ctl_policy_zoned( ctl->max_chunk_size = 2 * ctl->max_stripe_size; ctl->devs_max = min_t(int, ctl->devs_max, BTRFS_MAX_DEVS_SYS_CHUNK); + } else { + BUG(); } /* We don't want a chunk larger than 10% of writable space */ diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index eeb3ebe11d7a..304ce64c70a4 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -342,6 +342,13 @@ int btrfs_get_dev_zone_info(struct btrfs_device *device) if (!IS_ALIGNED(nr_sectors, zone_sectors)) zone_info->nr_zones++; + if (bdev_is_zoned(bdev) && zone_info->max_zone_append_size == 0) { + btrfs_err(fs_info, "zoned: device %pg does not support zone append", + bdev); + ret = -EINVAL; + goto out; + } + zone_info->seq_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL); if (!zone_info->seq_zones) { ret = -ENOMEM; @@ -1119,6 +1126,11 @@ int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache, bool new) goto out; } + if (zone.type == BLK_ZONE_TYPE_CONVENTIONAL) { + ret = -EIO; + goto out; + } + switch (zone.cond) { case BLK_ZONE_COND_OFFLINE: case BLK_ZONE_COND_READONLY: diff --git a/fs/btrfs/zstd.c b/fs/btrfs/zstd.c index 8e9626d63976..14418b02c189 100644 --- a/fs/btrfs/zstd.c +++ b/fs/btrfs/zstd.c @@ -28,10 +28,10 @@ /* 307s to avoid pathologically clashing with transaction commit */ #define ZSTD_BTRFS_RECLAIM_JIFFIES (307 * HZ) -static ZSTD_parameters zstd_get_btrfs_parameters(unsigned int level, +static zstd_parameters zstd_get_btrfs_parameters(unsigned int level, size_t src_len) { - ZSTD_parameters params = ZSTD_getParams(level, src_len, 0); + zstd_parameters params = zstd_get_params(level, src_len); if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG) params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG; @@ -48,8 +48,8 @@ struct workspace { unsigned long last_used; /* jiffies */ struct list_head list; struct list_head lru_list; - ZSTD_inBuffer in_buf; - ZSTD_outBuffer out_buf; + zstd_in_buffer in_buf; + zstd_out_buffer out_buf; }; /* @@ -155,12 +155,12 @@ static void zstd_calc_ws_mem_sizes(void) unsigned int level; for (level = 1; level <= ZSTD_BTRFS_MAX_LEVEL; level++) { - ZSTD_parameters params = + zstd_parameters params = zstd_get_btrfs_parameters(level, ZSTD_BTRFS_MAX_INPUT); size_t level_size = max_t(size_t, - ZSTD_CStreamWorkspaceBound(params.cParams), - ZSTD_DStreamWorkspaceBound(ZSTD_BTRFS_MAX_INPUT)); + zstd_cstream_workspace_bound(¶ms.cParams), + zstd_dstream_workspace_bound(ZSTD_BTRFS_MAX_INPUT)); max_size = max_t(size_t, max_size, level_size); zstd_ws_mem_sizes[level - 1] = max_size; @@ -371,7 +371,7 @@ int zstd_compress_pages(struct list_head *ws, struct address_space *mapping, unsigned long *total_in, unsigned long *total_out) { struct workspace *workspace = list_entry(ws, struct workspace, list); - ZSTD_CStream *stream; + zstd_cstream *stream; int ret = 0; int nr_pages = 0; struct page *in_page = NULL; /* The current page to read */ @@ -381,7 +381,7 @@ int zstd_compress_pages(struct list_head *ws, struct address_space *mapping, unsigned long len = *total_out; const unsigned long nr_dest_pages = *out_pages; unsigned long max_out = nr_dest_pages * PAGE_SIZE; - ZSTD_parameters params = zstd_get_btrfs_parameters(workspace->req_level, + zstd_parameters params = zstd_get_btrfs_parameters(workspace->req_level, len); *out_pages = 0; @@ -389,10 +389,10 @@ int zstd_compress_pages(struct list_head *ws, struct address_space *mapping, *total_in = 0; /* Initialize the stream */ - stream = ZSTD_initCStream(params, len, workspace->mem, + stream = zstd_init_cstream(¶ms, len, workspace->mem, workspace->size); if (!stream) { - pr_warn("BTRFS: ZSTD_initCStream failed\n"); + pr_warn("BTRFS: zstd_init_cstream failed\n"); ret = -EIO; goto out; } @@ -418,11 +418,11 @@ int zstd_compress_pages(struct list_head *ws, struct address_space *mapping, while (1) { size_t ret2; - ret2 = ZSTD_compressStream(stream, &workspace->out_buf, + ret2 = zstd_compress_stream(stream, &workspace->out_buf, &workspace->in_buf); - if (ZSTD_isError(ret2)) { - pr_debug("BTRFS: ZSTD_compressStream returned %d\n", - ZSTD_getErrorCode(ret2)); + if (zstd_is_error(ret2)) { + pr_debug("BTRFS: zstd_compress_stream returned %d\n", + zstd_get_error_code(ret2)); ret = -EIO; goto out; } @@ -487,10 +487,10 @@ int zstd_compress_pages(struct list_head *ws, struct address_space *mapping, while (1) { size_t ret2; - ret2 = ZSTD_endStream(stream, &workspace->out_buf); - if (ZSTD_isError(ret2)) { - pr_debug("BTRFS: ZSTD_endStream returned %d\n", - ZSTD_getErrorCode(ret2)); + ret2 = zstd_end_stream(stream, &workspace->out_buf); + if (zstd_is_error(ret2)) { + pr_debug("BTRFS: zstd_end_stream returned %d\n", + zstd_get_error_code(ret2)); ret = -EIO; goto out; } @@ -550,17 +550,17 @@ int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb) u64 disk_start = cb->start; struct bio *orig_bio = cb->orig_bio; size_t srclen = cb->compressed_len; - ZSTD_DStream *stream; + zstd_dstream *stream; int ret = 0; unsigned long page_in_index = 0; unsigned long total_pages_in = DIV_ROUND_UP(srclen, PAGE_SIZE); unsigned long buf_start; unsigned long total_out = 0; - stream = ZSTD_initDStream( + stream = zstd_init_dstream( ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size); if (!stream) { - pr_debug("BTRFS: ZSTD_initDStream failed\n"); + pr_debug("BTRFS: zstd_init_dstream failed\n"); ret = -EIO; goto done; } @@ -576,11 +576,11 @@ int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb) while (1) { size_t ret2; - ret2 = ZSTD_decompressStream(stream, &workspace->out_buf, + ret2 = zstd_decompress_stream(stream, &workspace->out_buf, &workspace->in_buf); - if (ZSTD_isError(ret2)) { - pr_debug("BTRFS: ZSTD_decompressStream returned %d\n", - ZSTD_getErrorCode(ret2)); + if (zstd_is_error(ret2)) { + pr_debug("BTRFS: zstd_decompress_stream returned %d\n", + zstd_get_error_code(ret2)); ret = -EIO; goto done; } @@ -626,17 +626,17 @@ int zstd_decompress(struct list_head *ws, unsigned char *data_in, size_t destlen) { struct workspace *workspace = list_entry(ws, struct workspace, list); - ZSTD_DStream *stream; + zstd_dstream *stream; int ret = 0; size_t ret2; unsigned long total_out = 0; unsigned long pg_offset = 0; char *kaddr; - stream = ZSTD_initDStream( + stream = zstd_init_dstream( ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size); if (!stream) { - pr_warn("BTRFS: ZSTD_initDStream failed\n"); + pr_warn("BTRFS: zstd_init_dstream failed\n"); ret = -EIO; goto finish; } @@ -660,15 +660,15 @@ int zstd_decompress(struct list_head *ws, unsigned char *data_in, /* Check if the frame is over and we still need more input */ if (ret2 == 0) { - pr_debug("BTRFS: ZSTD_decompressStream ended early\n"); + pr_debug("BTRFS: zstd_decompress_stream ended early\n"); ret = -EIO; goto finish; } - ret2 = ZSTD_decompressStream(stream, &workspace->out_buf, + ret2 = zstd_decompress_stream(stream, &workspace->out_buf, &workspace->in_buf); - if (ZSTD_isError(ret2)) { - pr_debug("BTRFS: ZSTD_decompressStream returned %d\n", - ZSTD_getErrorCode(ret2)); + if (zstd_is_error(ret2)) { + pr_debug("BTRFS: zstd_decompress_stream returned %d\n", + zstd_get_error_code(ret2)); ret = -EIO; goto finish; } diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index 570731c4d019..d405ba801492 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -1867,6 +1867,7 @@ static int try_nonblocking_invalidate(struct inode *inode) u32 invalidating_gen = ci->i_rdcache_gen; spin_unlock(&ci->i_ceph_lock); + ceph_fscache_invalidate(inode); invalidate_mapping_pages(&inode->i_data, 0, -1); spin_lock(&ci->i_ceph_lock); diff --git a/fs/ceph/export.c b/fs/ceph/export.c index e088843a7734..042bb4a02c0a 100644 --- a/fs/ceph/export.c +++ b/fs/ceph/export.c @@ -129,6 +129,10 @@ static struct inode *__lookup_inode(struct super_block *sb, u64 ino) vino.ino = ino; vino.snap = CEPH_NOSNAP; + + if (ceph_vino_is_reserved(vino)) + return ERR_PTR(-ESTALE); + inode = ceph_find_inode(sb, vino); if (!inode) { struct ceph_mds_request *req; @@ -178,8 +182,10 @@ static struct dentry *__fh_to_dentry(struct super_block *sb, u64 ino) return ERR_CAST(inode); /* We need LINK caps to reliably check i_nlink */ err = ceph_do_getattr(inode, CEPH_CAP_LINK_SHARED, false); - if (err) + if (err) { + iput(inode); return ERR_PTR(err); + } /* -ESTALE if inode as been unlinked and no file is open */ if ((inode->i_nlink == 0) && (atomic_read(&inode->i_count) == 1)) { iput(inode); @@ -212,6 +218,10 @@ static struct dentry *__snapfh_to_dentry(struct super_block *sb, vino.ino = sfh->ino; vino.snap = sfh->snapid; } + + if (ceph_vino_is_reserved(vino)) + return ERR_PTR(-ESTALE); + inode = ceph_find_inode(sb, vino); if (inode) return d_obtain_alias(inode); diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index 156f849f5385..179d2ef69a24 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -56,6 +56,9 @@ struct inode *ceph_get_inode(struct super_block *sb, struct ceph_vino vino) { struct inode *inode; + if (ceph_vino_is_reserved(vino)) + return ERR_PTR(-EREMOTEIO); + inode = iget5_locked(sb, (unsigned long)vino.ino, ceph_ino_compare, ceph_set_ino_cb, &vino); if (!inode) @@ -87,14 +90,15 @@ struct inode *ceph_get_snapdir(struct inode *parent) inode->i_mtime = parent->i_mtime; inode->i_ctime = parent->i_ctime; inode->i_atime = parent->i_atime; - inode->i_op = &ceph_snapdir_iops; - inode->i_fop = &ceph_snapdir_fops; - ci->i_snap_caps = CEPH_CAP_PIN; /* so we can open */ ci->i_rbytes = 0; ci->i_btime = ceph_inode(parent)->i_btime; - if (inode->i_state & I_NEW) + if (inode->i_state & I_NEW) { + inode->i_op = &ceph_snapdir_iops; + inode->i_fop = &ceph_snapdir_fops; + ci->i_snap_caps = CEPH_CAP_PIN; /* so we can open */ unlock_new_inode(inode); + } return inode; } @@ -1863,6 +1867,7 @@ static void ceph_do_invalidate_pages(struct inode *inode) orig_gen = ci->i_rdcache_gen; spin_unlock(&ci->i_ceph_lock); + ceph_fscache_invalidate(inode); if (invalidate_inode_pages2(inode->i_mapping) < 0) { pr_err("invalidate_pages %p fails\n", inode); } diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index d87bd852ed96..298cb0b3d28c 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -433,6 +433,13 @@ static int ceph_parse_deleg_inos(void **p, void *end, ceph_decode_64_safe(p, end, start, bad); ceph_decode_64_safe(p, end, len, bad); + + /* Don't accept a delegation of system inodes */ + if (start < CEPH_INO_SYSTEM_BASE) { + pr_warn_ratelimited("ceph: ignoring reserved inode range delegation (start=0x%llx len=0x%llx)\n", + start, len); + continue; + } while (len--) { int err = xa_insert(&s->s_delegated_inos, ino = start++, DELEGATED_INO_AVAILABLE, diff --git a/fs/ceph/super.h b/fs/ceph/super.h index c48bb30c8d70..1d2fe70439bd 100644 --- a/fs/ceph/super.h +++ b/fs/ceph/super.h @@ -529,10 +529,34 @@ static inline int ceph_ino_compare(struct inode *inode, void *data) ci->i_vino.snap == pvino->snap; } +/* + * The MDS reserves a set of inodes for its own usage. These should never + * be accessible by clients, and so the MDS has no reason to ever hand these + * out. The range is CEPH_MDS_INO_MDSDIR_OFFSET..CEPH_INO_SYSTEM_BASE. + * + * These come from src/mds/mdstypes.h in the ceph sources. + */ +#define CEPH_MAX_MDS 0x100 +#define CEPH_NUM_STRAY 10 +#define CEPH_MDS_INO_MDSDIR_OFFSET (1 * CEPH_MAX_MDS) +#define CEPH_INO_SYSTEM_BASE ((6*CEPH_MAX_MDS) + (CEPH_MAX_MDS * CEPH_NUM_STRAY)) + +static inline bool ceph_vino_is_reserved(const struct ceph_vino vino) +{ + if (vino.ino < CEPH_INO_SYSTEM_BASE && + vino.ino >= CEPH_MDS_INO_MDSDIR_OFFSET) { + WARN_RATELIMIT(1, "Attempt to access reserved inode number 0x%llx", vino.ino); + return true; + } + return false; +} static inline struct inode *ceph_find_inode(struct super_block *sb, struct ceph_vino vino) { + if (ceph_vino_is_reserved(vino)) + return NULL; + /* * NB: The hashval will be run through the fs/inode.c hash function * anyway, so there is no need to squash the inode number down to diff --git a/fs/cifs/cifsfs.c b/fs/cifs/cifsfs.c index 5ddd20b62484..fa896a1c8b07 100644 --- a/fs/cifs/cifsfs.c +++ b/fs/cifs/cifsfs.c @@ -834,7 +834,7 @@ cifs_smb3_do_mount(struct file_system_type *fs_type, goto out; } - rc = cifs_setup_volume_info(cifs_sb->ctx, NULL, old_ctx->UNC); + rc = cifs_setup_volume_info(cifs_sb->ctx, NULL, NULL); if (rc) { root = ERR_PTR(rc); goto out; diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c index 24668eb006c6..3d62d52d730b 100644 --- a/fs/cifs/connect.c +++ b/fs/cifs/connect.c @@ -488,6 +488,7 @@ server_unresponsive(struct TCP_Server_Info *server) */ if ((server->tcpStatus == CifsGood || server->tcpStatus == CifsNeedNegotiate) && + (!server->ops->can_echo || server->ops->can_echo(server)) && time_after(jiffies, server->lstrp + 3 * server->echo_interval)) { cifs_server_dbg(VFS, "has not responded in %lu seconds. Reconnecting...\n", (3 * server->echo_interval) / HZ); @@ -3175,17 +3176,29 @@ static int do_dfs_failover(const char *path, const char *full_path, struct cifs_ int cifs_setup_volume_info(struct smb3_fs_context *ctx, const char *mntopts, const char *devname) { - int rc = 0; + int rc; - smb3_parse_devname(devname, ctx); + if (devname) { + cifs_dbg(FYI, "%s: devname=%s\n", __func__, devname); + rc = smb3_parse_devname(devname, ctx); + if (rc) { + cifs_dbg(VFS, "%s: failed to parse %s: %d\n", __func__, devname, rc); + return rc; + } + } if (mntopts) { char *ip; - cifs_dbg(FYI, "%s: mntopts=%s\n", __func__, mntopts); rc = smb3_parse_opt(mntopts, "ip", &ip); - if (!rc && !cifs_convert_address((struct sockaddr *)&ctx->dstaddr, ip, - strlen(ip))) { + if (rc) { + cifs_dbg(VFS, "%s: failed to parse ip options: %d\n", __func__, rc); + return rc; + } + + rc = cifs_convert_address((struct sockaddr *)&ctx->dstaddr, ip, strlen(ip)); + kfree(ip); + if (!rc) { cifs_dbg(VFS, "%s: failed to convert ip address\n", __func__); return -EINVAL; } @@ -3205,7 +3218,7 @@ cifs_setup_volume_info(struct smb3_fs_context *ctx, const char *mntopts, const c return -EINVAL; } - return rc; + return 0; } static int diff --git a/fs/cifs/fs_context.c b/fs/cifs/fs_context.c index 78889024a7ed..a7253eb2e955 100644 --- a/fs/cifs/fs_context.c +++ b/fs/cifs/fs_context.c @@ -475,6 +475,7 @@ smb3_parse_devname(const char *devname, struct smb3_fs_context *ctx) /* move "pos" up to delimiter or NULL */ pos += len; + kfree(ctx->UNC); ctx->UNC = kstrndup(devname, pos - devname, GFP_KERNEL); if (!ctx->UNC) return -ENOMEM; @@ -485,6 +486,9 @@ smb3_parse_devname(const char *devname, struct smb3_fs_context *ctx) if (*pos == '/' || *pos == '\\') pos++; + kfree(ctx->prepath); + ctx->prepath = NULL; + /* If pos is NULL then no prepath */ if (!*pos) return 0; @@ -995,6 +999,9 @@ static int smb3_fs_context_parse_param(struct fs_context *fc, goto cifs_parse_mount_err; } ctx->max_channels = result.uint_32; + /* If more than one channel requested ... they want multichan */ + if (result.uint_32 > 1) + ctx->multichannel = true; break; case Opt_handletimeout: ctx->handle_timeout = result.uint_32; diff --git a/fs/cifs/sess.c b/fs/cifs/sess.c index 63d517b9f2ff..a92a1fb7cb52 100644 --- a/fs/cifs/sess.c +++ b/fs/cifs/sess.c @@ -97,6 +97,12 @@ int cifs_try_adding_channels(struct cifs_sb_info *cifs_sb, struct cifs_ses *ses) return 0; } + if (!(ses->server->capabilities & SMB2_GLOBAL_CAP_MULTI_CHANNEL)) { + cifs_dbg(VFS, "server %s does not support multichannel\n", ses->server->hostname); + ses->chan_max = 1; + return 0; + } + /* * Make a copy of the iface list at the time and use that * instead so as to not hold the iface spinlock for opening diff --git a/fs/cifs/smb2ops.c b/fs/cifs/smb2ops.c index f703204fb185..5df6daacc230 100644 --- a/fs/cifs/smb2ops.c +++ b/fs/cifs/smb2ops.c @@ -1763,18 +1763,14 @@ smb2_ioctl_query_info(const unsigned int xid, } iqinf_exit: - kfree(vars); - kfree(buffer); - SMB2_open_free(&rqst[0]); - if (qi.flags & PASSTHRU_FSCTL) - SMB2_ioctl_free(&rqst[1]); - else - SMB2_query_info_free(&rqst[1]); - - SMB2_close_free(&rqst[2]); + cifs_small_buf_release(rqst[0].rq_iov[0].iov_base); + cifs_small_buf_release(rqst[1].rq_iov[0].iov_base); + cifs_small_buf_release(rqst[2].rq_iov[0].iov_base); free_rsp_buf(resp_buftype[0], rsp_iov[0].iov_base); free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base); free_rsp_buf(resp_buftype[2], rsp_iov[2].iov_base); + kfree(vars); + kfree(buffer); return rc; e_fault: @@ -2232,7 +2228,7 @@ smb3_notify(const unsigned int xid, struct file *pfile, cifs_sb = CIFS_SB(inode->i_sb); - utf16_path = cifs_convert_path_to_utf16(path + 1, cifs_sb); + utf16_path = cifs_convert_path_to_utf16(path, cifs_sb); if (utf16_path == NULL) { rc = -ENOMEM; goto notify_exit; @@ -4178,7 +4174,7 @@ smb2_get_enc_key(struct TCP_Server_Info *server, __u64 ses_id, int enc, u8 *key) } spin_unlock(&cifs_tcp_ses_lock); - return 1; + return -EAGAIN; } /* * Encrypt or decrypt @rqst message. @rqst[0] has the following format: diff --git a/fs/cifs/smb2pdu.c b/fs/cifs/smb2pdu.c index 2199a9bfae8f..29272d99102c 100644 --- a/fs/cifs/smb2pdu.c +++ b/fs/cifs/smb2pdu.c @@ -841,6 +841,8 @@ SMB2_negotiate(const unsigned int xid, struct cifs_ses *ses) req->SecurityMode = 0; req->Capabilities = cpu_to_le32(server->vals->req_capabilities); + if (ses->chan_max > 1) + req->Capabilities |= cpu_to_le32(SMB2_GLOBAL_CAP_MULTI_CHANNEL); /* ClientGUID must be zero for SMB2.02 dialect */ if (server->vals->protocol_id == SMB20_PROT_ID) @@ -1032,6 +1034,9 @@ int smb3_validate_negotiate(const unsigned int xid, struct cifs_tcon *tcon) pneg_inbuf->Capabilities = cpu_to_le32(server->vals->req_capabilities); + if (tcon->ses->chan_max > 1) + pneg_inbuf->Capabilities |= cpu_to_le32(SMB2_GLOBAL_CAP_MULTI_CHANNEL); + memcpy(pneg_inbuf->Guid, server->client_guid, SMB2_CLIENT_GUID_SIZE); diff --git a/fs/dax.c b/fs/dax.c index b3d27fdc6775..df5485b4bddf 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -144,6 +144,16 @@ struct wait_exceptional_entry_queue { struct exceptional_entry_key key; }; +/** + * enum dax_wake_mode: waitqueue wakeup behaviour + * @WAKE_ALL: wake all waiters in the waitqueue + * @WAKE_NEXT: wake only the first waiter in the waitqueue + */ +enum dax_wake_mode { + WAKE_ALL, + WAKE_NEXT, +}; + static wait_queue_head_t *dax_entry_waitqueue(struct xa_state *xas, void *entry, struct exceptional_entry_key *key) { @@ -182,7 +192,8 @@ static int wake_exceptional_entry_func(wait_queue_entry_t *wait, * The important information it's conveying is whether the entry at * this index used to be a PMD entry. */ -static void dax_wake_entry(struct xa_state *xas, void *entry, bool wake_all) +static void dax_wake_entry(struct xa_state *xas, void *entry, + enum dax_wake_mode mode) { struct exceptional_entry_key key; wait_queue_head_t *wq; @@ -196,7 +207,7 @@ static void dax_wake_entry(struct xa_state *xas, void *entry, bool wake_all) * must be in the waitqueue and the following check will see them. */ if (waitqueue_active(wq)) - __wake_up(wq, TASK_NORMAL, wake_all ? 0 : 1, &key); + __wake_up(wq, TASK_NORMAL, mode == WAKE_ALL ? 0 : 1, &key); } /* @@ -264,11 +275,11 @@ static void wait_entry_unlocked(struct xa_state *xas, void *entry) finish_wait(wq, &ewait.wait); } -static void put_unlocked_entry(struct xa_state *xas, void *entry) +static void put_unlocked_entry(struct xa_state *xas, void *entry, + enum dax_wake_mode mode) { - /* If we were the only waiter woken, wake the next one */ if (entry && !dax_is_conflict(entry)) - dax_wake_entry(xas, entry, false); + dax_wake_entry(xas, entry, mode); } /* @@ -286,7 +297,7 @@ static void dax_unlock_entry(struct xa_state *xas, void *entry) old = xas_store(xas, entry); xas_unlock_irq(xas); BUG_ON(!dax_is_locked(old)); - dax_wake_entry(xas, entry, false); + dax_wake_entry(xas, entry, WAKE_NEXT); } /* @@ -524,7 +535,7 @@ static void *grab_mapping_entry(struct xa_state *xas, dax_disassociate_entry(entry, mapping, false); xas_store(xas, NULL); /* undo the PMD join */ - dax_wake_entry(xas, entry, true); + dax_wake_entry(xas, entry, WAKE_ALL); mapping->nrexceptional--; entry = NULL; xas_set(xas, index); @@ -622,7 +633,7 @@ struct page *dax_layout_busy_page_range(struct address_space *mapping, entry = get_unlocked_entry(&xas, 0); if (entry) page = dax_busy_page(entry); - put_unlocked_entry(&xas, entry); + put_unlocked_entry(&xas, entry, WAKE_NEXT); if (page) break; if (++scanned % XA_CHECK_SCHED) @@ -664,7 +675,7 @@ static int __dax_invalidate_entry(struct address_space *mapping, mapping->nrexceptional--; ret = 1; out: - put_unlocked_entry(&xas, entry); + put_unlocked_entry(&xas, entry, WAKE_ALL); xas_unlock_irq(&xas); return ret; } @@ -937,13 +948,13 @@ static int dax_writeback_one(struct xa_state *xas, struct dax_device *dax_dev, xas_lock_irq(xas); xas_store(xas, entry); xas_clear_mark(xas, PAGECACHE_TAG_DIRTY); - dax_wake_entry(xas, entry, false); + dax_wake_entry(xas, entry, WAKE_NEXT); trace_dax_writeback_one(mapping->host, index, count); return ret; put_unlocked: - put_unlocked_entry(xas, entry); + put_unlocked_entry(xas, entry, WAKE_NEXT); return ret; } @@ -1684,7 +1695,7 @@ dax_insert_pfn_mkwrite(struct vm_fault *vmf, pfn_t pfn, unsigned int order) /* Did we race with someone splitting entry or so? */ if (!entry || dax_is_conflict(entry) || (order == 0 && !dax_is_pte_entry(entry))) { - put_unlocked_entry(&xas, entry); + put_unlocked_entry(&xas, entry, WAKE_NEXT); xas_unlock_irq(&xas); trace_dax_insert_pfn_mkwrite_no_entry(mapping->host, vmf, VM_FAULT_NOPAGE); diff --git a/fs/debugfs/inode.c b/fs/debugfs/inode.c index 22e86ae4dd5a..1d252164d97b 100644 --- a/fs/debugfs/inode.c +++ b/fs/debugfs/inode.c @@ -35,7 +35,7 @@ static struct vfsmount *debugfs_mount; static int debugfs_mount_count; static bool debugfs_registered; -static unsigned int debugfs_allow = DEFAULT_DEBUGFS_ALLOW_BITS; +static unsigned int debugfs_allow __ro_after_init = DEFAULT_DEBUGFS_ALLOW_BITS; /* * Don't allow access attributes to be changed whilst the kernel is locked down diff --git a/fs/dlm/config.c b/fs/dlm/config.c index 49c5f9407098..88d95d96e36c 100644 --- a/fs/dlm/config.c +++ b/fs/dlm/config.c @@ -125,7 +125,7 @@ static ssize_t cluster_cluster_name_store(struct config_item *item, CONFIGFS_ATTR(cluster_, cluster_name); static ssize_t cluster_set(struct dlm_cluster *cl, unsigned int *cl_field, - int *info_field, bool (*check_cb)(unsigned int x), + int *info_field, int (*check_cb)(unsigned int x), const char *buf, size_t len) { unsigned int x; @@ -137,8 +137,11 @@ static ssize_t cluster_set(struct dlm_cluster *cl, unsigned int *cl_field, if (rc) return rc; - if (check_cb && check_cb(x)) - return -EINVAL; + if (check_cb) { + rc = check_cb(x); + if (rc) + return rc; + } *cl_field = x; *info_field = x; @@ -161,17 +164,53 @@ static ssize_t cluster_##name##_show(struct config_item *item, char *buf) \ } \ CONFIGFS_ATTR(cluster_, name); -static bool dlm_check_zero(unsigned int x) +static int dlm_check_protocol_and_dlm_running(unsigned int x) +{ + switch (x) { + case 0: + /* TCP */ + break; + case 1: + /* SCTP */ + break; + default: + return -EINVAL; + } + + if (dlm_allow_conn) + return -EBUSY; + + return 0; +} + +static int dlm_check_zero_and_dlm_running(unsigned int x) +{ + if (!x) + return -EINVAL; + + if (dlm_allow_conn) + return -EBUSY; + + return 0; +} + +static int dlm_check_zero(unsigned int x) { - return !x; + if (!x) + return -EINVAL; + + return 0; } -static bool dlm_check_buffer_size(unsigned int x) +static int dlm_check_buffer_size(unsigned int x) { - return (x < DEFAULT_BUFFER_SIZE); + if (x < DEFAULT_BUFFER_SIZE) + return -EINVAL; + + return 0; } -CLUSTER_ATTR(tcp_port, dlm_check_zero); +CLUSTER_ATTR(tcp_port, dlm_check_zero_and_dlm_running); CLUSTER_ATTR(buffer_size, dlm_check_buffer_size); CLUSTER_ATTR(rsbtbl_size, dlm_check_zero); CLUSTER_ATTR(recover_timer, dlm_check_zero); @@ -179,7 +218,7 @@ CLUSTER_ATTR(toss_secs, dlm_check_zero); CLUSTER_ATTR(scan_secs, dlm_check_zero); CLUSTER_ATTR(log_debug, NULL); CLUSTER_ATTR(log_info, NULL); -CLUSTER_ATTR(protocol, NULL); +CLUSTER_ATTR(protocol, dlm_check_protocol_and_dlm_running); CLUSTER_ATTR(mark, NULL); CLUSTER_ATTR(timewarn_cs, dlm_check_zero); CLUSTER_ATTR(waitwarn_us, NULL); @@ -688,6 +727,7 @@ static ssize_t comm_mark_show(struct config_item *item, char *buf) static ssize_t comm_mark_store(struct config_item *item, const char *buf, size_t len) { + struct dlm_comm *comm; unsigned int mark; int rc; @@ -695,7 +735,15 @@ static ssize_t comm_mark_store(struct config_item *item, const char *buf, if (rc) return rc; - config_item_to_comm(item)->mark = mark; + if (mark == 0) + mark = dlm_config.ci_mark; + + comm = config_item_to_comm(item); + rc = dlm_lowcomms_nodes_set_mark(comm->nodeid, mark); + if (rc) + return rc; + + comm->mark = mark; return len; } @@ -870,24 +918,6 @@ int dlm_comm_seq(int nodeid, uint32_t *seq) return 0; } -void dlm_comm_mark(int nodeid, unsigned int *mark) -{ - struct dlm_comm *cm; - - cm = get_comm(nodeid); - if (!cm) { - *mark = dlm_config.ci_mark; - return; - } - - if (cm->mark) - *mark = cm->mark; - else - *mark = dlm_config.ci_mark; - - put_comm(cm); -} - int dlm_our_nodeid(void) { return local_comm ? local_comm->nodeid : 0; diff --git a/fs/dlm/config.h b/fs/dlm/config.h index c210250a2581..d2cd4bd20313 100644 --- a/fs/dlm/config.h +++ b/fs/dlm/config.h @@ -48,7 +48,6 @@ void dlm_config_exit(void); int dlm_config_nodes(char *lsname, struct dlm_config_node **nodes_out, int *count_out); int dlm_comm_seq(int nodeid, uint32_t *seq); -void dlm_comm_mark(int nodeid, unsigned int *mark); int dlm_our_nodeid(void); int dlm_our_addr(struct sockaddr_storage *addr, int num); diff --git a/fs/dlm/debug_fs.c b/fs/dlm/debug_fs.c index d6bbccb0ed15..d5bd990bcab8 100644 --- a/fs/dlm/debug_fs.c +++ b/fs/dlm/debug_fs.c @@ -542,6 +542,7 @@ static void *table_seq_next(struct seq_file *seq, void *iter_ptr, loff_t *pos) if (bucket >= ls->ls_rsbtbl_size) { kfree(ri); + ++*pos; return NULL; } tree = toss ? &ls->ls_rsbtbl[bucket].toss : &ls->ls_rsbtbl[bucket].keep; diff --git a/fs/dlm/lockspace.c b/fs/dlm/lockspace.c index 561dcad08ad6..c14cf2b7faab 100644 --- a/fs/dlm/lockspace.c +++ b/fs/dlm/lockspace.c @@ -404,12 +404,6 @@ static int threads_start(void) return error; } -static void threads_stop(void) -{ - dlm_scand_stop(); - dlm_lowcomms_stop(); -} - static int new_lockspace(const char *name, const char *cluster, uint32_t flags, int lvblen, const struct dlm_lockspace_ops *ops, void *ops_arg, @@ -702,8 +696,11 @@ int dlm_new_lockspace(const char *name, const char *cluster, ls_count++; if (error > 0) error = 0; - if (!ls_count) - threads_stop(); + if (!ls_count) { + dlm_scand_stop(); + dlm_lowcomms_shutdown(); + dlm_lowcomms_stop(); + } out: mutex_unlock(&ls_lock); return error; @@ -788,6 +785,11 @@ static int release_lockspace(struct dlm_ls *ls, int force) dlm_recoverd_stop(ls); + if (ls_count == 1) { + dlm_scand_stop(); + dlm_lowcomms_shutdown(); + } + dlm_callback_stop(ls); remove_lockspace(ls); @@ -880,7 +882,7 @@ int dlm_release_lockspace(void *lockspace, int force) if (!error) ls_count--; if (!ls_count) - threads_stop(); + dlm_lowcomms_stop(); mutex_unlock(&ls_lock); return error; diff --git a/fs/dlm/lowcomms.c b/fs/dlm/lowcomms.c index 372c34ff8594..45c2fdaf34c4 100644 --- a/fs/dlm/lowcomms.c +++ b/fs/dlm/lowcomms.c @@ -116,6 +116,7 @@ struct writequeue_entry { struct dlm_node_addr { struct list_head list; int nodeid; + int mark; int addr_count; int curr_addr_index; struct sockaddr_storage *addr[DLM_MAX_ADDR_COUNT]; @@ -134,7 +135,7 @@ static DEFINE_SPINLOCK(dlm_node_addrs_spin); static struct listen_connection listen_con; static struct sockaddr_storage *dlm_local_addr[DLM_MAX_ADDR_COUNT]; static int dlm_local_count; -static int dlm_allow_conn; +int dlm_allow_conn; /* Work queues */ static struct workqueue_struct *recv_workqueue; @@ -303,7 +304,8 @@ static int addr_compare(const struct sockaddr_storage *x, } static int nodeid_to_addr(int nodeid, struct sockaddr_storage *sas_out, - struct sockaddr *sa_out, bool try_new_addr) + struct sockaddr *sa_out, bool try_new_addr, + unsigned int *mark) { struct sockaddr_storage sas; struct dlm_node_addr *na; @@ -331,6 +333,8 @@ static int nodeid_to_addr(int nodeid, struct sockaddr_storage *sas_out, if (!na->addr_count) return -ENOENT; + *mark = na->mark; + if (sas_out) memcpy(sas_out, &sas, sizeof(struct sockaddr_storage)); @@ -350,7 +354,8 @@ static int nodeid_to_addr(int nodeid, struct sockaddr_storage *sas_out, return 0; } -static int addr_to_nodeid(struct sockaddr_storage *addr, int *nodeid) +static int addr_to_nodeid(struct sockaddr_storage *addr, int *nodeid, + unsigned int *mark) { struct dlm_node_addr *na; int rv = -EEXIST; @@ -364,6 +369,7 @@ static int addr_to_nodeid(struct sockaddr_storage *addr, int *nodeid) for (addr_i = 0; addr_i < na->addr_count; addr_i++) { if (addr_compare(na->addr[addr_i], addr)) { *nodeid = na->nodeid; + *mark = na->mark; rv = 0; goto unlock; } @@ -412,6 +418,7 @@ int dlm_lowcomms_addr(int nodeid, struct sockaddr_storage *addr, int len) new_node->nodeid = nodeid; new_node->addr[0] = new_addr; new_node->addr_count = 1; + new_node->mark = dlm_config.ci_mark; list_add(&new_node->list, &dlm_node_addrs); spin_unlock(&dlm_node_addrs_spin); return 0; @@ -519,6 +526,23 @@ int dlm_lowcomms_connect_node(int nodeid) return 0; } +int dlm_lowcomms_nodes_set_mark(int nodeid, unsigned int mark) +{ + struct dlm_node_addr *na; + + spin_lock(&dlm_node_addrs_spin); + na = find_node_addr(nodeid); + if (!na) { + spin_unlock(&dlm_node_addrs_spin); + return -ENOENT; + } + + na->mark = mark; + spin_unlock(&dlm_node_addrs_spin); + + return 0; +} + static void lowcomms_error_report(struct sock *sk) { struct connection *con; @@ -685,10 +709,7 @@ static void shutdown_connection(struct connection *con) { int ret; - if (cancel_work_sync(&con->swork)) { - log_print("canceled swork for node %d", con->nodeid); - clear_bit(CF_WRITE_PENDING, &con->flags); - } + flush_work(&con->swork); mutex_lock(&con->sock_mutex); /* nothing to shutdown */ @@ -867,7 +888,7 @@ static int accept_from_sock(struct listen_connection *con) /* Get the new node's NODEID */ make_sockaddr(&peeraddr, 0, &len); - if (addr_to_nodeid(&peeraddr, &nodeid)) { + if (addr_to_nodeid(&peeraddr, &nodeid, &mark)) { unsigned char *b=(unsigned char *)&peeraddr; log_print("connect from non cluster node"); print_hex_dump_bytes("ss: ", DUMP_PREFIX_NONE, @@ -876,9 +897,6 @@ static int accept_from_sock(struct listen_connection *con) return -1; } - dlm_comm_mark(nodeid, &mark); - sock_set_mark(newsock->sk, mark); - log_print("got connection from %d", nodeid); /* Check to see if we already have a connection to this node. This @@ -892,6 +910,8 @@ static int accept_from_sock(struct listen_connection *con) goto accept_err; } + sock_set_mark(newsock->sk, mark); + mutex_lock(&newcon->sock_mutex); if (newcon->sock) { struct connection *othercon = newcon->othercon; @@ -908,6 +928,7 @@ static int accept_from_sock(struct listen_connection *con) result = dlm_con_init(othercon, nodeid); if (result < 0) { kfree(othercon); + mutex_unlock(&newcon->sock_mutex); goto accept_err; } @@ -1015,8 +1036,6 @@ static void sctp_connect_to_sock(struct connection *con) struct socket *sock; unsigned int mark; - dlm_comm_mark(con->nodeid, &mark); - mutex_lock(&con->sock_mutex); /* Some odd races can cause double-connects, ignore them */ @@ -1029,7 +1048,7 @@ static void sctp_connect_to_sock(struct connection *con) } memset(&daddr, 0, sizeof(daddr)); - result = nodeid_to_addr(con->nodeid, &daddr, NULL, true); + result = nodeid_to_addr(con->nodeid, &daddr, NULL, true, &mark); if (result < 0) { log_print("no address for nodeid %d", con->nodeid); goto out; @@ -1104,13 +1123,11 @@ static void sctp_connect_to_sock(struct connection *con) static void tcp_connect_to_sock(struct connection *con) { struct sockaddr_storage saddr, src_addr; + unsigned int mark; int addr_len; struct socket *sock = NULL; - unsigned int mark; int result; - dlm_comm_mark(con->nodeid, &mark); - mutex_lock(&con->sock_mutex); if (con->retries++ > MAX_CONNECT_RETRIES) goto out; @@ -1125,15 +1142,15 @@ static void tcp_connect_to_sock(struct connection *con) if (result < 0) goto out_err; - sock_set_mark(sock->sk, mark); - memset(&saddr, 0, sizeof(saddr)); - result = nodeid_to_addr(con->nodeid, &saddr, NULL, false); + result = nodeid_to_addr(con->nodeid, &saddr, NULL, false, &mark); if (result < 0) { log_print("no address for nodeid %d", con->nodeid); goto out_err; } + sock_set_mark(sock->sk, mark); + add_sock(sock, con); /* Bind to our cluster-known address connecting to avoid @@ -1355,9 +1372,11 @@ void *dlm_lowcomms_get_buffer(int nodeid, int len, gfp_t allocation, char **ppc) struct writequeue_entry *e; int offset = 0; - if (len > LOWCOMMS_MAX_TX_BUFFER_LEN) { - BUILD_BUG_ON(PAGE_SIZE < LOWCOMMS_MAX_TX_BUFFER_LEN); + if (len > DEFAULT_BUFFER_SIZE || + len < sizeof(struct dlm_header)) { + BUILD_BUG_ON(PAGE_SIZE < DEFAULT_BUFFER_SIZE); log_print("failed to allocate a buffer of size %d", len); + WARN_ON(1); return NULL; } @@ -1589,6 +1608,29 @@ static int work_start(void) return 0; } +static void shutdown_conn(struct connection *con) +{ + if (con->shutdown_action) + con->shutdown_action(con); +} + +void dlm_lowcomms_shutdown(void) +{ + /* Set all the flags to prevent any + * socket activity. + */ + dlm_allow_conn = 0; + + if (recv_workqueue) + flush_workqueue(recv_workqueue); + if (send_workqueue) + flush_workqueue(send_workqueue); + + dlm_close_sock(&listen_con.sock); + + foreach_conn(shutdown_conn); +} + static void _stop_conn(struct connection *con, bool and_other) { mutex_lock(&con->sock_mutex); @@ -1610,12 +1652,6 @@ static void stop_conn(struct connection *con) _stop_conn(con, true); } -static void shutdown_conn(struct connection *con) -{ - if (con->shutdown_action) - con->shutdown_action(con); -} - static void connection_release(struct rcu_head *rcu) { struct connection *con = container_of(rcu, struct connection, rcu); @@ -1672,19 +1708,6 @@ static void work_flush(void) void dlm_lowcomms_stop(void) { - /* Set all the flags to prevent any - socket activity. - */ - dlm_allow_conn = 0; - - if (recv_workqueue) - flush_workqueue(recv_workqueue); - if (send_workqueue) - flush_workqueue(send_workqueue); - - dlm_close_sock(&listen_con.sock); - - foreach_conn(shutdown_conn); work_flush(); foreach_conn(free_conn); work_stop(); diff --git a/fs/dlm/lowcomms.h b/fs/dlm/lowcomms.h index 0918f9376489..48bbc4e18761 100644 --- a/fs/dlm/lowcomms.h +++ b/fs/dlm/lowcomms.h @@ -14,13 +14,18 @@ #define LOWCOMMS_MAX_TX_BUFFER_LEN 4096 +/* switch to check if dlm is running */ +extern int dlm_allow_conn; + int dlm_lowcomms_start(void); +void dlm_lowcomms_shutdown(void); void dlm_lowcomms_stop(void); void dlm_lowcomms_exit(void); int dlm_lowcomms_close(int nodeid); void *dlm_lowcomms_get_buffer(int nodeid, int len, gfp_t allocation, char **ppc); void dlm_lowcomms_commit_buffer(void *mh); int dlm_lowcomms_connect_node(int nodeid); +int dlm_lowcomms_nodes_set_mark(int nodeid, unsigned int mark); int dlm_lowcomms_addr(int nodeid, struct sockaddr_storage *addr, int len); #endif /* __LOWCOMMS_DOT_H__ */ diff --git a/fs/dlm/midcomms.c b/fs/dlm/midcomms.c index fde3a6afe4be..0bedfa8606a2 100644 --- a/fs/dlm/midcomms.c +++ b/fs/dlm/midcomms.c @@ -49,9 +49,10 @@ int dlm_process_incoming_buffer(int nodeid, unsigned char *buf, int len) * cannot deliver this message to upper layers */ msglen = get_unaligned_le16(&hd->h_length); - if (msglen > DEFAULT_BUFFER_SIZE) { - log_print("received invalid length header: %u, will abort message parsing", - msglen); + if (msglen > DEFAULT_BUFFER_SIZE || + msglen < sizeof(struct dlm_header)) { + log_print("received invalid length header: %u from node %d, will abort message parsing", + msglen, nodeid); return -EBADMSG; } diff --git a/fs/ecryptfs/main.c b/fs/ecryptfs/main.c index cdf40a54a35d..cf772c72ab2b 100644 --- a/fs/ecryptfs/main.c +++ b/fs/ecryptfs/main.c @@ -492,6 +492,12 @@ static struct dentry *ecryptfs_mount(struct file_system_type *fs_type, int flags goto out; } + if (!dev_name) { + rc = -EINVAL; + err = "Device name cannot be null"; + goto out; + } + rc = ecryptfs_parse_options(sbi, raw_data, &check_ruid); if (rc) { err = "Error parsing options"; diff --git a/fs/erofs/erofs_fs.h b/fs/erofs/erofs_fs.h index 9ad1615f4474..e8d04d808fa6 100644 --- a/fs/erofs/erofs_fs.h +++ b/fs/erofs/erofs_fs.h @@ -75,6 +75,9 @@ static inline bool erofs_inode_is_data_compressed(unsigned int datamode) #define EROFS_I_VERSION_BIT 0 #define EROFS_I_DATALAYOUT_BIT 1 +#define EROFS_I_ALL \ + ((1 << (EROFS_I_DATALAYOUT_BIT + EROFS_I_DATALAYOUT_BITS)) - 1) + /* 32-byte reduced form of an ondisk inode */ struct erofs_inode_compact { __le16 i_format; /* inode format hints */ diff --git a/fs/erofs/inode.c b/fs/erofs/inode.c index 119fdce1b520..7ed2d7391692 100644 --- a/fs/erofs/inode.c +++ b/fs/erofs/inode.c @@ -44,6 +44,13 @@ static struct page *erofs_read_inode(struct inode *inode, dic = page_address(page) + *ofs; ifmt = le16_to_cpu(dic->i_format); + if (ifmt & ~EROFS_I_ALL) { + erofs_err(inode->i_sb, "unsupported i_format %u of nid %llu", + ifmt, vi->nid); + err = -EOPNOTSUPP; + goto err_out; + } + vi->datalayout = erofs_inode_datalayout(ifmt); if (vi->datalayout >= EROFS_INODE_DATALAYOUT_MAX) { erofs_err(inode->i_sb, "unsupported datalayout %u of nid %llu", diff --git a/fs/eventpoll.c b/fs/eventpoll.c index 3196474cbe24..e42477fcbfa0 100644 --- a/fs/eventpoll.c +++ b/fs/eventpoll.c @@ -657,6 +657,12 @@ static void ep_done_scan(struct eventpoll *ep, */ list_splice(txlist, &ep->rdllist); __pm_relax(ep->ws); + + if (!list_empty(&ep->rdllist)) { + if (waitqueue_active(&ep->wq)) + wake_up(&ep->wq); + } + write_unlock_irq(&ep->lock); } diff --git a/fs/exfat/balloc.c b/fs/exfat/balloc.c index 761c79c3a4ba..411fb0a8da10 100644 --- a/fs/exfat/balloc.c +++ b/fs/exfat/balloc.c @@ -141,10 +141,6 @@ void exfat_free_bitmap(struct exfat_sb_info *sbi) kfree(sbi->vol_amap); } -/* - * If the value of "clu" is 0, it means cluster 2 which is the first cluster of - * the cluster heap. - */ int exfat_set_bitmap(struct inode *inode, unsigned int clu) { int i, b; @@ -162,10 +158,6 @@ int exfat_set_bitmap(struct inode *inode, unsigned int clu) return 0; } -/* - * If the value of "clu" is 0, it means cluster 2 which is the first cluster of - * the cluster heap. - */ void exfat_clear_bitmap(struct inode *inode, unsigned int clu, bool sync) { int i, b; @@ -186,8 +178,7 @@ void exfat_clear_bitmap(struct inode *inode, unsigned int clu, bool sync) int ret_discard; ret_discard = sb_issue_discard(sb, - exfat_cluster_to_sector(sbi, clu + - EXFAT_RESERVED_CLUSTERS), + exfat_cluster_to_sector(sbi, clu), (1 << sbi->sect_per_clus_bits), GFP_NOFS, 0); if (ret_discard == -EOPNOTSUPP) { diff --git a/fs/ext4/fast_commit.c b/fs/ext4/fast_commit.c index 7541d0b5d706..eda14f630def 100644 --- a/fs/ext4/fast_commit.c +++ b/fs/ext4/fast_commit.c @@ -1088,8 +1088,10 @@ static int ext4_fc_perform_commit(journal_t *journal) head.fc_tid = cpu_to_le32( sbi->s_journal->j_running_transaction->t_tid); if (!ext4_fc_add_tlv(sb, EXT4_FC_TAG_HEAD, sizeof(head), - (u8 *)&head, &crc)) + (u8 *)&head, &crc)) { + ret = -ENOSPC; goto out; + } } spin_lock(&sbi->s_fc_lock); @@ -1734,7 +1736,7 @@ static int ext4_fc_replay_add_range(struct super_block *sb, } /* Range is mapped and needs a state change */ - jbd_debug(1, "Converting from %d to %d %lld", + jbd_debug(1, "Converting from %ld to %d %lld", map.m_flags & EXT4_MAP_UNWRITTEN, ext4_ext_is_unwritten(ex), map.m_pblk); ret = ext4_ext_replay_update_ex(inode, cur, map.m_len, diff --git a/fs/ext4/file.c b/fs/ext4/file.c index 194f5d00fa32..7924634ab0bf 100644 --- a/fs/ext4/file.c +++ b/fs/ext4/file.c @@ -371,15 +371,32 @@ static ssize_t ext4_handle_inode_extension(struct inode *inode, loff_t offset, static int ext4_dio_write_end_io(struct kiocb *iocb, ssize_t size, int error, unsigned int flags) { - loff_t offset = iocb->ki_pos; + loff_t pos = iocb->ki_pos; struct inode *inode = file_inode(iocb->ki_filp); if (error) return error; - if (size && flags & IOMAP_DIO_UNWRITTEN) - return ext4_convert_unwritten_extents(NULL, inode, - offset, size); + if (size && flags & IOMAP_DIO_UNWRITTEN) { + error = ext4_convert_unwritten_extents(NULL, inode, pos, size); + if (error < 0) + return error; + } + /* + * If we are extending the file, we have to update i_size here before + * page cache gets invalidated in iomap_dio_rw(). Otherwise racing + * buffered reads could zero out too much from page cache pages. Update + * of on-disk size will happen later in ext4_dio_write_iter() where + * we have enough information to also perform orphan list handling etc. + * Note that we perform all extending writes synchronously under + * i_rwsem held exclusively so i_size update is safe here in that case. + * If the write was not extending, we cannot see pos > i_size here + * because operations reducing i_size like truncate wait for all + * outstanding DIO before updating i_size. + */ + pos += size; + if (pos > i_size_read(inode)) + i_size_write(inode, pos); return 0; } diff --git a/fs/ext4/ialloc.c b/fs/ext4/ialloc.c index 633ae7becd61..71d321b3b984 100644 --- a/fs/ext4/ialloc.c +++ b/fs/ext4/ialloc.c @@ -1292,7 +1292,8 @@ struct inode *__ext4_new_inode(struct user_namespace *mnt_userns, ei->i_extra_isize = sbi->s_want_extra_isize; ei->i_inline_off = 0; - if (ext4_has_feature_inline_data(sb)) + if (ext4_has_feature_inline_data(sb) && + (!(ei->i_flags & EXT4_DAX_FL) || S_ISDIR(mode))) ext4_set_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); ret = inode; err = dquot_alloc_inode(inode); @@ -1513,6 +1514,7 @@ int ext4_init_inode_table(struct super_block *sb, ext4_group_t group, handle_t *handle; ext4_fsblk_t blk; int num, ret = 0, used_blks = 0; + unsigned long used_inos = 0; /* This should not happen, but just to be sure check this */ if (sb_rdonly(sb)) { @@ -1543,22 +1545,37 @@ int ext4_init_inode_table(struct super_block *sb, ext4_group_t group, * used inodes so we need to skip blocks with used inodes in * inode table. */ - if (!(gdp->bg_flags & cpu_to_le16(EXT4_BG_INODE_UNINIT))) - used_blks = DIV_ROUND_UP((EXT4_INODES_PER_GROUP(sb) - - ext4_itable_unused_count(sb, gdp)), - sbi->s_inodes_per_block); - - if ((used_blks < 0) || (used_blks > sbi->s_itb_per_group) || - ((group == 0) && ((EXT4_INODES_PER_GROUP(sb) - - ext4_itable_unused_count(sb, gdp)) < - EXT4_FIRST_INO(sb)))) { - ext4_error(sb, "Something is wrong with group %u: " - "used itable blocks: %d; " - "itable unused count: %u", - group, used_blks, - ext4_itable_unused_count(sb, gdp)); - ret = 1; - goto err_out; + if (!(gdp->bg_flags & cpu_to_le16(EXT4_BG_INODE_UNINIT))) { + used_inos = EXT4_INODES_PER_GROUP(sb) - + ext4_itable_unused_count(sb, gdp); + used_blks = DIV_ROUND_UP(used_inos, sbi->s_inodes_per_block); + + /* Bogus inode unused count? */ + if (used_blks < 0 || used_blks > sbi->s_itb_per_group) { + ext4_error(sb, "Something is wrong with group %u: " + "used itable blocks: %d; " + "itable unused count: %u", + group, used_blks, + ext4_itable_unused_count(sb, gdp)); + ret = 1; + goto err_out; + } + + used_inos += group * EXT4_INODES_PER_GROUP(sb); + /* + * Are there some uninitialized inodes in the inode table + * before the first normal inode? + */ + if ((used_blks != sbi->s_itb_per_group) && + (used_inos < EXT4_FIRST_INO(sb))) { + ext4_error(sb, "Something is wrong with group %u: " + "itable unused count: %u; " + "itables initialized count: %ld", + group, ext4_itable_unused_count(sb, gdp), + used_inos); + ret = 1; + goto err_out; + } } blk = ext4_inode_table(sb, gdp) + used_blks; diff --git a/fs/ext4/ioctl.c b/fs/ext4/ioctl.c index a2cf35066f46..0796bfa72829 100644 --- a/fs/ext4/ioctl.c +++ b/fs/ext4/ioctl.c @@ -315,6 +315,12 @@ static void ext4_dax_dontcache(struct inode *inode, unsigned int flags) static bool dax_compatible(struct inode *inode, unsigned int oldflags, unsigned int flags) { + /* Allow the DAX flag to be changed on inline directories */ + if (S_ISDIR(inode->i_mode)) { + flags &= ~EXT4_INLINE_DATA_FL; + oldflags &= ~EXT4_INLINE_DATA_FL; + } + if (flags & EXT4_DAX_FL) { if ((oldflags & EXT4_DAX_MUT_EXCL) || ext4_test_inode_state(inode, diff --git a/fs/ext4/mmp.c b/fs/ext4/mmp.c index 795c3ff2907c..68fbeedd627b 100644 --- a/fs/ext4/mmp.c +++ b/fs/ext4/mmp.c @@ -56,7 +56,7 @@ static int write_mmp_block(struct super_block *sb, struct buffer_head *bh) wait_on_buffer(bh); sb_end_write(sb); if (unlikely(!buffer_uptodate(bh))) - return 1; + return -EIO; return 0; } diff --git a/fs/ext4/super.c b/fs/ext4/super.c index b9693680463a..77c1cb258262 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -667,9 +667,6 @@ static void ext4_handle_error(struct super_block *sb, bool force_ro, int error, ext4_commit_super(sb); } - if (sb_rdonly(sb) || continue_fs) - return; - /* * We force ERRORS_RO behavior when system is rebooting. Otherwise we * could panic during 'reboot -f' as the underlying device got already @@ -679,6 +676,10 @@ static void ext4_handle_error(struct super_block *sb, bool force_ro, int error, panic("EXT4-fs (device %s): panic forced after error\n", sb->s_id); } + + if (sb_rdonly(sb) || continue_fs) + return; + ext4_msg(sb, KERN_CRIT, "Remounting filesystem read-only"); /* * Make sure updated value of ->s_mount_flags will be visible before @@ -3023,9 +3024,6 @@ static void ext4_orphan_cleanup(struct super_block *sb, sb->s_flags &= ~SB_RDONLY; } #ifdef CONFIG_QUOTA - /* Needed for iput() to work correctly and not trash data */ - sb->s_flags |= SB_ACTIVE; - /* * Turn on quotas which were not enabled for read-only mounts if * filesystem has quota feature, so that they are updated correctly. @@ -5561,8 +5559,10 @@ static int ext4_commit_super(struct super_block *sb) struct buffer_head *sbh = EXT4_SB(sb)->s_sbh; int error = 0; - if (!sbh || block_device_ejected(sb)) - return error; + if (!sbh) + return -EINVAL; + if (block_device_ejected(sb)) + return -ENODEV; ext4_update_super(sb); diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c index 77fa342de38f..582b11afb0d5 100644 --- a/fs/f2fs/compress.c +++ b/fs/f2fs/compress.c @@ -123,19 +123,6 @@ static void f2fs_unlock_rpages(struct compress_ctx *cc, int len) f2fs_drop_rpages(cc, len, true); } -static void f2fs_put_rpages_mapping(struct address_space *mapping, - pgoff_t start, int len) -{ - int i; - - for (i = 0; i < len; i++) { - struct page *page = find_get_page(mapping, start + i); - - put_page(page); - put_page(page); - } -} - static void f2fs_put_rpages_wbc(struct compress_ctx *cc, struct writeback_control *wbc, bool redirty, int unlock) { @@ -164,13 +151,14 @@ int f2fs_init_compress_ctx(struct compress_ctx *cc) return cc->rpages ? 0 : -ENOMEM; } -void f2fs_destroy_compress_ctx(struct compress_ctx *cc) +void f2fs_destroy_compress_ctx(struct compress_ctx *cc, bool reuse) { page_array_free(cc->inode, cc->rpages, cc->cluster_size); cc->rpages = NULL; cc->nr_rpages = 0; cc->nr_cpages = 0; - cc->cluster_idx = NULL_CLUSTER; + if (!reuse) + cc->cluster_idx = NULL_CLUSTER; } void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct page *page) @@ -351,8 +339,8 @@ static const struct f2fs_compress_ops f2fs_lz4_ops = { static int zstd_init_compress_ctx(struct compress_ctx *cc) { - ZSTD_parameters params; - ZSTD_CStream *stream; + zstd_parameters params; + zstd_cstream *stream; void *workspace; unsigned int workspace_size; unsigned char level = F2FS_I(cc->inode)->i_compress_flag >> @@ -361,17 +349,17 @@ static int zstd_init_compress_ctx(struct compress_ctx *cc) if (!level) level = F2FS_ZSTD_DEFAULT_CLEVEL; - params = ZSTD_getParams(level, cc->rlen, 0); - workspace_size = ZSTD_CStreamWorkspaceBound(params.cParams); + params = zstd_get_params(F2FS_ZSTD_DEFAULT_CLEVEL, cc->rlen); + workspace_size = zstd_cstream_workspace_bound(¶ms.cParams); workspace = f2fs_kvmalloc(F2FS_I_SB(cc->inode), workspace_size, GFP_NOFS); if (!workspace) return -ENOMEM; - stream = ZSTD_initCStream(params, 0, workspace, workspace_size); + stream = zstd_init_cstream(¶ms, 0, workspace, workspace_size); if (!stream) { - printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_initCStream failed\n", + printk_ratelimited("%sF2FS-fs (%s): %s zstd_init_cstream failed\n", KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, __func__); kvfree(workspace); @@ -394,9 +382,9 @@ static void zstd_destroy_compress_ctx(struct compress_ctx *cc) static int zstd_compress_pages(struct compress_ctx *cc) { - ZSTD_CStream *stream = cc->private2; - ZSTD_inBuffer inbuf; - ZSTD_outBuffer outbuf; + zstd_cstream *stream = cc->private2; + zstd_in_buffer inbuf; + zstd_out_buffer outbuf; int src_size = cc->rlen; int dst_size = src_size - PAGE_SIZE - COMPRESS_HEADER_SIZE; int ret; @@ -409,19 +397,19 @@ static int zstd_compress_pages(struct compress_ctx *cc) outbuf.dst = cc->cbuf->cdata; outbuf.size = dst_size; - ret = ZSTD_compressStream(stream, &outbuf, &inbuf); - if (ZSTD_isError(ret)) { - printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_compressStream failed, ret: %d\n", + ret = zstd_compress_stream(stream, &outbuf, &inbuf); + if (zstd_is_error(ret)) { + printk_ratelimited("%sF2FS-fs (%s): %s zstd_compress_stream failed, ret: %d\n", KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, - __func__, ZSTD_getErrorCode(ret)); + __func__, zstd_get_error_code(ret)); return -EIO; } - ret = ZSTD_endStream(stream, &outbuf); - if (ZSTD_isError(ret)) { - printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_endStream returned %d\n", + ret = zstd_end_stream(stream, &outbuf); + if (zstd_is_error(ret)) { + printk_ratelimited("%sF2FS-fs (%s): %s zstd_end_stream returned %d\n", KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, - __func__, ZSTD_getErrorCode(ret)); + __func__, zstd_get_error_code(ret)); return -EIO; } @@ -438,22 +426,22 @@ static int zstd_compress_pages(struct compress_ctx *cc) static int zstd_init_decompress_ctx(struct decompress_io_ctx *dic) { - ZSTD_DStream *stream; + zstd_dstream *stream; void *workspace; unsigned int workspace_size; unsigned int max_window_size = MAX_COMPRESS_WINDOW_SIZE(dic->log_cluster_size); - workspace_size = ZSTD_DStreamWorkspaceBound(max_window_size); + workspace_size = zstd_dstream_workspace_bound(max_window_size); workspace = f2fs_kvmalloc(F2FS_I_SB(dic->inode), workspace_size, GFP_NOFS); if (!workspace) return -ENOMEM; - stream = ZSTD_initDStream(max_window_size, workspace, workspace_size); + stream = zstd_init_dstream(max_window_size, workspace, workspace_size); if (!stream) { - printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_initDStream failed\n", + printk_ratelimited("%sF2FS-fs (%s): %s zstd_init_dstream failed\n", KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, __func__); kvfree(workspace); @@ -475,9 +463,9 @@ static void zstd_destroy_decompress_ctx(struct decompress_io_ctx *dic) static int zstd_decompress_pages(struct decompress_io_ctx *dic) { - ZSTD_DStream *stream = dic->private2; - ZSTD_inBuffer inbuf; - ZSTD_outBuffer outbuf; + zstd_dstream *stream = dic->private2; + zstd_in_buffer inbuf; + zstd_out_buffer outbuf; int ret; inbuf.pos = 0; @@ -488,11 +476,11 @@ static int zstd_decompress_pages(struct decompress_io_ctx *dic) outbuf.dst = dic->rbuf; outbuf.size = dic->rlen; - ret = ZSTD_decompressStream(stream, &outbuf, &inbuf); - if (ZSTD_isError(ret)) { - printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_compressStream failed, ret: %d\n", + ret = zstd_decompress_stream(stream, &outbuf, &inbuf); + if (zstd_is_error(ret)) { + printk_ratelimited("%sF2FS-fs (%s): %s zstd_decompress_stream failed, ret: %d\n", KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, - __func__, ZSTD_getErrorCode(ret)); + __func__, zstd_get_error_code(ret)); return -EIO; } @@ -1048,7 +1036,7 @@ static int prepare_compress_overwrite(struct compress_ctx *cc, } if (PageUptodate(page)) - unlock_page(page); + f2fs_put_page(page, 1); else f2fs_compress_ctx_add_page(cc, page); } @@ -1058,33 +1046,35 @@ static int prepare_compress_overwrite(struct compress_ctx *cc, ret = f2fs_read_multi_pages(cc, &bio, cc->cluster_size, &last_block_in_bio, false, true); - f2fs_destroy_compress_ctx(cc); + f2fs_put_rpages(cc); + f2fs_destroy_compress_ctx(cc, true); if (ret) - goto release_pages; + goto out; if (bio) f2fs_submit_bio(sbi, bio, DATA); ret = f2fs_init_compress_ctx(cc); if (ret) - goto release_pages; + goto out; } for (i = 0; i < cc->cluster_size; i++) { f2fs_bug_on(sbi, cc->rpages[i]); page = find_lock_page(mapping, start_idx + i); - f2fs_bug_on(sbi, !page); + if (!page) { + /* page can be truncated */ + goto release_and_retry; + } f2fs_wait_on_page_writeback(page, DATA, true, true); - f2fs_compress_ctx_add_page(cc, page); - f2fs_put_page(page, 0); if (!PageUptodate(page)) { +release_and_retry: + f2fs_put_rpages(cc); f2fs_unlock_rpages(cc, i + 1); - f2fs_put_rpages_mapping(mapping, start_idx, - cc->cluster_size); - f2fs_destroy_compress_ctx(cc); + f2fs_destroy_compress_ctx(cc, true); goto retry; } } @@ -1115,10 +1105,10 @@ static int prepare_compress_overwrite(struct compress_ctx *cc, } unlock_pages: + f2fs_put_rpages(cc); f2fs_unlock_rpages(cc, i); -release_pages: - f2fs_put_rpages_mapping(mapping, start_idx, i); - f2fs_destroy_compress_ctx(cc); + f2fs_destroy_compress_ctx(cc, true); +out: return ret; } @@ -1153,7 +1143,7 @@ bool f2fs_compress_write_end(struct inode *inode, void *fsdata, set_cluster_dirty(&cc); f2fs_put_rpages_wbc(&cc, NULL, false, 1); - f2fs_destroy_compress_ctx(&cc); + f2fs_destroy_compress_ctx(&cc, false); return first_index; } @@ -1372,7 +1362,7 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc, f2fs_put_rpages(cc); page_array_free(cc->inode, cc->cpages, cc->nr_cpages); cc->cpages = NULL; - f2fs_destroy_compress_ctx(cc); + f2fs_destroy_compress_ctx(cc, false); return 0; out_destroy_crypt: @@ -1383,7 +1373,8 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc, for (i = 0; i < cc->nr_cpages; i++) { if (!cc->cpages[i]) continue; - f2fs_put_page(cc->cpages[i], 1); + f2fs_compress_free_page(cc->cpages[i]); + cc->cpages[i] = NULL; } out_put_cic: kmem_cache_free(cic_entry_slab, cic); @@ -1533,7 +1524,7 @@ int f2fs_write_multi_pages(struct compress_ctx *cc, err = f2fs_write_raw_pages(cc, submitted, wbc, io_type); f2fs_put_rpages_wbc(cc, wbc, false, 0); destroy_out: - f2fs_destroy_compress_ctx(cc); + f2fs_destroy_compress_ctx(cc, false); return err; } diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 4e5257c763d0..8804a5d51380 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -2276,7 +2276,7 @@ static int f2fs_mpage_readpages(struct inode *inode, max_nr_pages, &last_block_in_bio, rac != NULL, false); - f2fs_destroy_compress_ctx(&cc); + f2fs_destroy_compress_ctx(&cc, false); if (ret) goto set_error_page; } @@ -2321,7 +2321,7 @@ static int f2fs_mpage_readpages(struct inode *inode, max_nr_pages, &last_block_in_bio, rac != NULL, false); - f2fs_destroy_compress_ctx(&cc); + f2fs_destroy_compress_ctx(&cc, false); } } #endif @@ -3022,7 +3022,7 @@ static int f2fs_write_cache_pages(struct address_space *mapping, } } if (f2fs_compressed_file(inode)) - f2fs_destroy_compress_ctx(&cc); + f2fs_destroy_compress_ctx(&cc, false); #endif if (retry) { index = 0; diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index e2d302ae3a46..f3fabb1edfe9 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -3376,6 +3376,7 @@ block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi); int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable); void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi); int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra); +bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno); void f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi); void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi); void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi); @@ -3383,7 +3384,7 @@ void f2fs_get_new_segment(struct f2fs_sb_info *sbi, unsigned int *newseg, bool new_sec, int dir); void f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type, unsigned int start, unsigned int end); -void f2fs_allocate_new_segment(struct f2fs_sb_info *sbi, int type); +void f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type); void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi); int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range); bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi, @@ -3547,7 +3548,7 @@ void f2fs_destroy_post_read_wq(struct f2fs_sb_info *sbi); int f2fs_start_gc_thread(struct f2fs_sb_info *sbi); void f2fs_stop_gc_thread(struct f2fs_sb_info *sbi); block_t f2fs_start_bidx_of_node(unsigned int node_ofs, struct inode *inode); -int f2fs_gc(struct f2fs_sb_info *sbi, bool sync, bool background, +int f2fs_gc(struct f2fs_sb_info *sbi, bool sync, bool background, bool force, unsigned int segno); void f2fs_build_gc_manager(struct f2fs_sb_info *sbi); int f2fs_resize_fs(struct f2fs_sb_info *sbi, __u64 block_count); @@ -3949,7 +3950,7 @@ struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc); void f2fs_decompress_end_io(struct decompress_io_ctx *dic, bool failed); void f2fs_put_page_dic(struct page *page); int f2fs_init_compress_ctx(struct compress_ctx *cc); -void f2fs_destroy_compress_ctx(struct compress_ctx *cc); +void f2fs_destroy_compress_ctx(struct compress_ctx *cc, bool reuse); void f2fs_init_compress_info(struct f2fs_sb_info *sbi); int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi); void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi); diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index d26ff2ae3f5e..dc79694e512c 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -1619,9 +1619,10 @@ static int expand_inode_data(struct inode *inode, loff_t offset, struct f2fs_map_blocks map = { .m_next_pgofs = NULL, .m_next_extent = NULL, .m_seg_type = NO_CHECK_TYPE, .m_may_create = true }; - pgoff_t pg_end; + pgoff_t pg_start, pg_end; loff_t new_size = i_size_read(inode); loff_t off_end; + block_t expanded = 0; int err; err = inode_newsize_ok(inode, (len + offset)); @@ -1634,11 +1635,12 @@ static int expand_inode_data(struct inode *inode, loff_t offset, f2fs_balance_fs(sbi, true); + pg_start = ((unsigned long long)offset) >> PAGE_SHIFT; pg_end = ((unsigned long long)offset + len) >> PAGE_SHIFT; off_end = (offset + len) & (PAGE_SIZE - 1); - map.m_lblk = ((unsigned long long)offset) >> PAGE_SHIFT; - map.m_len = pg_end - map.m_lblk; + map.m_lblk = pg_start; + map.m_len = pg_end - pg_start; if (off_end) map.m_len++; @@ -1646,19 +1648,15 @@ static int expand_inode_data(struct inode *inode, loff_t offset, return 0; if (f2fs_is_pinned_file(inode)) { - block_t len = (map.m_len >> sbi->log_blocks_per_seg) << - sbi->log_blocks_per_seg; - block_t done = 0; - - if (map.m_len % sbi->blocks_per_seg) - len += sbi->blocks_per_seg; + block_t sec_blks = BLKS_PER_SEC(sbi); + block_t sec_len = roundup(map.m_len, sec_blks); - map.m_len = sbi->blocks_per_seg; + map.m_len = sec_blks; next_alloc: if (has_not_enough_free_secs(sbi, 0, GET_SEC_FROM_SEG(sbi, overprovision_segments(sbi)))) { down_write(&sbi->gc_lock); - err = f2fs_gc(sbi, true, false, NULL_SEGNO); + err = f2fs_gc(sbi, true, false, false, NULL_SEGNO); if (err && err != -ENODATA && err != -EAGAIN) goto out_err; } @@ -1666,7 +1664,7 @@ static int expand_inode_data(struct inode *inode, loff_t offset, down_write(&sbi->pin_sem); f2fs_lock_op(sbi); - f2fs_allocate_new_segment(sbi, CURSEG_COLD_DATA_PINNED); + f2fs_allocate_new_section(sbi, CURSEG_COLD_DATA_PINNED); f2fs_unlock_op(sbi); map.m_seg_type = CURSEG_COLD_DATA_PINNED; @@ -1674,24 +1672,25 @@ static int expand_inode_data(struct inode *inode, loff_t offset, up_write(&sbi->pin_sem); - done += map.m_len; - len -= map.m_len; + expanded += map.m_len; + sec_len -= map.m_len; map.m_lblk += map.m_len; - if (!err && len) + if (!err && sec_len) goto next_alloc; - map.m_len = done; + map.m_len = expanded; } else { err = f2fs_map_blocks(inode, &map, 1, F2FS_GET_BLOCK_PRE_AIO); + expanded = map.m_len; } out_err: if (err) { pgoff_t last_off; - if (!map.m_len) + if (!expanded) return err; - last_off = map.m_lblk + map.m_len - 1; + last_off = pg_start + expanded - 1; /* update new size to the failed position */ new_size = (last_off == pg_end) ? offset + len : @@ -2489,7 +2488,7 @@ static int f2fs_ioc_gc(struct file *filp, unsigned long arg) down_write(&sbi->gc_lock); } - ret = f2fs_gc(sbi, sync, true, NULL_SEGNO); + ret = f2fs_gc(sbi, sync, true, false, NULL_SEGNO); out: mnt_drop_write_file(filp); return ret; @@ -2525,7 +2524,8 @@ static int __f2fs_ioc_gc_range(struct file *filp, struct f2fs_gc_range *range) down_write(&sbi->gc_lock); } - ret = f2fs_gc(sbi, range->sync, true, GET_SEGNO(sbi, range->start)); + ret = f2fs_gc(sbi, range->sync, true, false, + GET_SEGNO(sbi, range->start)); if (ret) { if (ret == -EBUSY) ret = -EAGAIN; @@ -2978,7 +2978,7 @@ static int f2fs_ioc_flush_device(struct file *filp, unsigned long arg) sm->last_victim[GC_CB] = end_segno + 1; sm->last_victim[GC_GREEDY] = end_segno + 1; sm->last_victim[ALLOC_NEXT] = end_segno + 1; - ret = f2fs_gc(sbi, true, true, start_segno); + ret = f2fs_gc(sbi, true, true, true, start_segno); if (ret == -EAGAIN) ret = 0; else if (ret < 0) diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c index 39330ad3c44e..a8567cb47621 100644 --- a/fs/f2fs/gc.c +++ b/fs/f2fs/gc.c @@ -112,7 +112,7 @@ static int gc_thread_func(void *data) sync_mode = F2FS_OPTION(sbi).bggc_mode == BGGC_MODE_SYNC; /* if return value is not zero, no victim was selected */ - if (f2fs_gc(sbi, sync_mode, true, NULL_SEGNO)) + if (f2fs_gc(sbi, sync_mode, true, false, NULL_SEGNO)) wait_ms = gc_th->no_gc_sleep_time; trace_f2fs_background_gc(sbi->sb, wait_ms, @@ -392,10 +392,6 @@ static void add_victim_entry(struct f2fs_sb_info *sbi, if (p->gc_mode == GC_AT && get_valid_blocks(sbi, segno, true) == 0) return; - - if (p->alloc_mode == AT_SSR && - get_seg_entry(sbi, segno)->ckpt_valid_blocks == 0) - return; } for (i = 0; i < sbi->segs_per_sec; i++) @@ -728,11 +724,27 @@ static int get_victim_by_default(struct f2fs_sb_info *sbi, if (sec_usage_check(sbi, secno)) goto next; + /* Don't touch checkpointed data */ - if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED) && - get_ckpt_valid_blocks(sbi, segno) && - p.alloc_mode == LFS)) - goto next; + if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { + if (p.alloc_mode == LFS) { + /* + * LFS is set to find source section during GC. + * The victim should have no checkpointed data. + */ + if (get_ckpt_valid_blocks(sbi, segno, true)) + goto next; + } else { + /* + * SSR | AT_SSR are set to find target segment + * for writes which can be full by checkpointed + * and newly written blocks. + */ + if (!f2fs_segment_has_free_slot(sbi, segno)) + goto next; + } + } + if (gc_type == BG_GC && test_bit(secno, dirty_i->victim_secmap)) goto next; @@ -1354,7 +1366,8 @@ static int move_data_page(struct inode *inode, block_t bidx, int gc_type, * the victim data block is ignored. */ static int gc_data_segment(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, - struct gc_inode_list *gc_list, unsigned int segno, int gc_type) + struct gc_inode_list *gc_list, unsigned int segno, int gc_type, + bool force_migrate) { struct super_block *sb = sbi->sb; struct f2fs_summary *entry; @@ -1383,8 +1396,8 @@ static int gc_data_segment(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, * race condition along with SSR block allocation. */ if ((gc_type == BG_GC && has_not_enough_free_secs(sbi, 0, 0)) || - get_valid_blocks(sbi, segno, true) == - BLKS_PER_SEC(sbi)) + (!force_migrate && get_valid_blocks(sbi, segno, true) == + BLKS_PER_SEC(sbi))) return submitted; if (check_valid_map(sbi, segno, off) == 0) @@ -1519,7 +1532,8 @@ static int __get_victim(struct f2fs_sb_info *sbi, unsigned int *victim, static int do_garbage_collect(struct f2fs_sb_info *sbi, unsigned int start_segno, - struct gc_inode_list *gc_list, int gc_type) + struct gc_inode_list *gc_list, int gc_type, + bool force_migrate) { struct page *sum_page; struct f2fs_summary_block *sum; @@ -1606,7 +1620,8 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, gc_type); else submitted += gc_data_segment(sbi, sum->entries, gc_list, - segno, gc_type); + segno, gc_type, + force_migrate); stat_inc_seg_count(sbi, type, gc_type); migrated++; @@ -1634,7 +1649,7 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, } int f2fs_gc(struct f2fs_sb_info *sbi, bool sync, - bool background, unsigned int segno) + bool background, bool force, unsigned int segno) { int gc_type = sync ? FG_GC : BG_GC; int sec_freed = 0, seg_freed = 0, total_freed = 0; @@ -1696,7 +1711,7 @@ int f2fs_gc(struct f2fs_sb_info *sbi, bool sync, if (ret) goto stop; - seg_freed = do_garbage_collect(sbi, segno, &gc_list, gc_type); + seg_freed = do_garbage_collect(sbi, segno, &gc_list, gc_type, force); if (gc_type == FG_GC && seg_freed == f2fs_usable_segs_in_sec(sbi, segno)) sec_freed++; @@ -1835,7 +1850,7 @@ static int free_segment_range(struct f2fs_sb_info *sbi, .iroot = RADIX_TREE_INIT(gc_list.iroot, GFP_NOFS), }; - do_garbage_collect(sbi, segno, &gc_list, FG_GC); + do_garbage_collect(sbi, segno, &gc_list, FG_GC, true); put_gc_inode(&gc_list); if (!gc_only && get_valid_blocks(sbi, segno, true)) { @@ -1974,7 +1989,20 @@ int f2fs_resize_fs(struct f2fs_sb_info *sbi, __u64 block_count) /* stop CP to protect MAIN_SEC in free_segment_range */ f2fs_lock_op(sbi); + + spin_lock(&sbi->stat_lock); + if (shrunk_blocks + valid_user_blocks(sbi) + + sbi->current_reserved_blocks + sbi->unusable_block_count + + F2FS_OPTION(sbi).root_reserved_blocks > sbi->user_block_count) + err = -ENOSPC; + spin_unlock(&sbi->stat_lock); + + if (err) + goto out_unlock; + err = free_segment_range(sbi, secs, true); + +out_unlock: f2fs_unlock_op(sbi); up_write(&sbi->gc_lock); if (err) diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c index 993caefcd2bb..92652ca7a7c8 100644 --- a/fs/f2fs/inline.c +++ b/fs/f2fs/inline.c @@ -219,7 +219,8 @@ int f2fs_convert_inline_inode(struct inode *inode) f2fs_put_page(page, 1); - f2fs_balance_fs(sbi, dn.node_changed); + if (!err) + f2fs_balance_fs(sbi, dn.node_changed); return err; } diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index 4b0e2e3c2c88..45c8cf1afe66 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -2785,6 +2785,9 @@ static void remove_nats_in_journal(struct f2fs_sb_info *sbi) struct f2fs_nat_entry raw_ne; nid_t nid = le32_to_cpu(nid_in_journal(journal, i)); + if (f2fs_check_nid_range(sbi, nid)) + continue; + raw_ne = nat_in_journal(journal, i); ne = __lookup_nat_cache(nm_i, nid); diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index c2866561263e..bb6d86255741 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -186,7 +186,10 @@ void f2fs_register_inmem_page(struct inode *inode, struct page *page) { struct inmem_pages *new; - f2fs_set_page_private(page, ATOMIC_WRITTEN_PAGE); + if (PagePrivate(page)) + set_page_private(page, (unsigned long)ATOMIC_WRITTEN_PAGE); + else + f2fs_set_page_private(page, ATOMIC_WRITTEN_PAGE); new = f2fs_kmem_cache_alloc(inmem_entry_slab, GFP_NOFS); @@ -324,23 +327,27 @@ void f2fs_drop_inmem_pages(struct inode *inode) struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct f2fs_inode_info *fi = F2FS_I(inode); - while (!list_empty(&fi->inmem_pages)) { + do { mutex_lock(&fi->inmem_lock); + if (list_empty(&fi->inmem_pages)) { + fi->i_gc_failures[GC_FAILURE_ATOMIC] = 0; + + spin_lock(&sbi->inode_lock[ATOMIC_FILE]); + if (!list_empty(&fi->inmem_ilist)) + list_del_init(&fi->inmem_ilist); + if (f2fs_is_atomic_file(inode)) { + clear_inode_flag(inode, FI_ATOMIC_FILE); + sbi->atomic_files--; + } + spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); + + mutex_unlock(&fi->inmem_lock); + break; + } __revoke_inmem_pages(inode, &fi->inmem_pages, true, false, true); mutex_unlock(&fi->inmem_lock); - } - - fi->i_gc_failures[GC_FAILURE_ATOMIC] = 0; - - spin_lock(&sbi->inode_lock[ATOMIC_FILE]); - if (!list_empty(&fi->inmem_ilist)) - list_del_init(&fi->inmem_ilist); - if (f2fs_is_atomic_file(inode)) { - clear_inode_flag(inode, FI_ATOMIC_FILE); - sbi->atomic_files--; - } - spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); + } while (1); } void f2fs_drop_inmem_page(struct inode *inode, struct page *page) @@ -504,7 +511,7 @@ void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need) */ if (has_not_enough_free_secs(sbi, 0, 0)) { down_write(&sbi->gc_lock); - f2fs_gc(sbi, false, false, NULL_SEGNO); + f2fs_gc(sbi, false, false, false, NULL_SEGNO); } } @@ -861,7 +868,7 @@ static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno) mutex_lock(&dirty_i->seglist_lock); valid_blocks = get_valid_blocks(sbi, segno, false); - ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno); + ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno, false); if (valid_blocks == 0 && (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) || ckpt_valid_blocks == usable_blocks)) { @@ -946,7 +953,7 @@ static unsigned int get_free_segment(struct f2fs_sb_info *sbi) for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) { if (get_valid_blocks(sbi, segno, false)) continue; - if (get_ckpt_valid_blocks(sbi, segno)) + if (get_ckpt_valid_blocks(sbi, segno, false)) continue; mutex_unlock(&dirty_i->seglist_lock); return segno; @@ -2636,6 +2643,23 @@ static void __refresh_next_blkoff(struct f2fs_sb_info *sbi, seg->next_blkoff++; } +bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno) +{ + struct seg_entry *se = get_seg_entry(sbi, segno); + int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long); + unsigned long *target_map = SIT_I(sbi)->tmp_map; + unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map; + unsigned long *cur_map = (unsigned long *)se->cur_valid_map; + int i, pos; + + for (i = 0; i < entries; i++) + target_map[i] = ckpt_map[i] | cur_map[i]; + + pos = __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, 0); + + return pos < sbi->blocks_per_seg; +} + /* * This function always allocates a used segment(from dirty seglist) by SSR * manner, so it should recover the existing segment information of valid blocks @@ -2893,7 +2917,8 @@ void f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type, up_read(&SM_I(sbi)->curseg_lock); } -static void __allocate_new_segment(struct f2fs_sb_info *sbi, int type) +static void __allocate_new_segment(struct f2fs_sb_info *sbi, int type, + bool new_sec) { struct curseg_info *curseg = CURSEG_I(sbi, type); unsigned int old_segno; @@ -2901,32 +2926,42 @@ static void __allocate_new_segment(struct f2fs_sb_info *sbi, int type) if (!curseg->inited) goto alloc; - if (!curseg->next_blkoff && - !get_valid_blocks(sbi, curseg->segno, false) && - !get_ckpt_valid_blocks(sbi, curseg->segno)) - return; + if (curseg->next_blkoff || + get_valid_blocks(sbi, curseg->segno, new_sec)) + goto alloc; + if (!get_ckpt_valid_blocks(sbi, curseg->segno, new_sec)) + return; alloc: old_segno = curseg->segno; SIT_I(sbi)->s_ops->allocate_segment(sbi, type, true); locate_dirty_segment(sbi, old_segno); } -void f2fs_allocate_new_segment(struct f2fs_sb_info *sbi, int type) +static void __allocate_new_section(struct f2fs_sb_info *sbi, int type) { + __allocate_new_segment(sbi, type, true); +} + +void f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type) +{ + down_read(&SM_I(sbi)->curseg_lock); down_write(&SIT_I(sbi)->sentry_lock); - __allocate_new_segment(sbi, type); + __allocate_new_section(sbi, type); up_write(&SIT_I(sbi)->sentry_lock); + up_read(&SM_I(sbi)->curseg_lock); } void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi) { int i; + down_read(&SM_I(sbi)->curseg_lock); down_write(&SIT_I(sbi)->sentry_lock); for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) - __allocate_new_segment(sbi, i); + __allocate_new_segment(sbi, i, false); up_write(&SIT_I(sbi)->sentry_lock); + up_read(&SM_I(sbi)->curseg_lock); } static const struct segment_allocation default_salloc_ops = { @@ -3365,12 +3400,12 @@ void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page, f2fs_inode_chksum_set(sbi, page); } - if (F2FS_IO_ALIGNED(sbi)) - fio->retry = false; - if (fio) { struct f2fs_bio_info *io; + if (F2FS_IO_ALIGNED(sbi)) + fio->retry = false; + INIT_LIST_HEAD(&fio->list); fio->in_list = true; io = sbi->write_io[fio->type] + fio->temp; diff --git a/fs/f2fs/segment.h b/fs/f2fs/segment.h index e9a7a637d688..afb175739de5 100644 --- a/fs/f2fs/segment.h +++ b/fs/f2fs/segment.h @@ -361,8 +361,20 @@ static inline unsigned int get_valid_blocks(struct f2fs_sb_info *sbi, } static inline unsigned int get_ckpt_valid_blocks(struct f2fs_sb_info *sbi, - unsigned int segno) + unsigned int segno, bool use_section) { + if (use_section && __is_large_section(sbi)) { + unsigned int start_segno = START_SEGNO(segno); + unsigned int blocks = 0; + int i; + + for (i = 0; i < sbi->segs_per_sec; i++, start_segno++) { + struct seg_entry *se = get_seg_entry(sbi, start_segno); + + blocks += se->ckpt_valid_blocks; + } + return blocks; + } return get_seg_entry(sbi, segno)->ckpt_valid_blocks; } diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 82592b19b4e0..3c8426709f34 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -525,7 +525,7 @@ static int f2fs_set_zstd_level(struct f2fs_sb_info *sbi, const char *str) if (kstrtouint(str + 1, 10, &level)) return -EINVAL; - if (!level || level > ZSTD_maxCLevel()) { + if (!level || level > zstd_max_clevel()) { f2fs_info(sbi, "invalid zstd compress level: %d", level); return -EINVAL; } @@ -1865,7 +1865,7 @@ static int f2fs_disable_checkpoint(struct f2fs_sb_info *sbi) while (!f2fs_time_over(sbi, DISABLE_TIME)) { down_write(&sbi->gc_lock); - err = f2fs_gc(sbi, true, false, NULL_SEGNO); + err = f2fs_gc(sbi, true, false, false, NULL_SEGNO); if (err == -ENODATA) { err = 0; break; @@ -3929,10 +3929,18 @@ static int f2fs_fill_super(struct super_block *sb, void *data, int silent) * previous checkpoint was not done by clean system shutdown. */ if (f2fs_hw_is_readonly(sbi)) { - if (!is_set_ckpt_flags(sbi, CP_UMOUNT_FLAG)) - f2fs_err(sbi, "Need to recover fsync data, but write access unavailable"); - else - f2fs_info(sbi, "write access unavailable, skipping recovery"); + if (!is_set_ckpt_flags(sbi, CP_UMOUNT_FLAG)) { + err = f2fs_recover_fsync_data(sbi, true); + if (err > 0) { + err = -EROFS; + f2fs_err(sbi, "Need to recover fsync data, but " + "write access unavailable, please try " + "mount w/ disable_roll_forward or norecovery"); + } + if (err < 0) + goto free_meta; + } + f2fs_info(sbi, "write access unavailable, skipping recovery"); goto reset_checkpoint; } diff --git a/fs/f2fs/verity.c b/fs/f2fs/verity.c index 054ec852b5ea..15ba36926fad 100644 --- a/fs/f2fs/verity.c +++ b/fs/f2fs/verity.c @@ -152,40 +152,73 @@ static int f2fs_end_enable_verity(struct file *filp, const void *desc, size_t desc_size, u64 merkle_tree_size) { struct inode *inode = file_inode(filp); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); u64 desc_pos = f2fs_verity_metadata_pos(inode) + merkle_tree_size; struct fsverity_descriptor_location dloc = { .version = cpu_to_le32(F2FS_VERIFY_VER), .size = cpu_to_le32(desc_size), .pos = cpu_to_le64(desc_pos), }; - int err = 0; + int err = 0, err2 = 0; - if (desc != NULL) { - /* Succeeded; write the verity descriptor. */ - err = pagecache_write(inode, desc, desc_size, desc_pos); + /* + * If an error already occurred (which fs/verity/ signals by passing + * desc == NULL), then only clean-up is needed. + */ + if (desc == NULL) + goto cleanup; - /* Write all pages before clearing FI_VERITY_IN_PROGRESS. */ - if (!err) - err = filemap_write_and_wait(inode->i_mapping); - } + /* Append the verity descriptor. */ + err = pagecache_write(inode, desc, desc_size, desc_pos); + if (err) + goto cleanup; + + /* + * Write all pages (both data and verity metadata). Note that this must + * happen before clearing FI_VERITY_IN_PROGRESS; otherwise pages beyond + * i_size won't be written properly. For crash consistency, this also + * must happen before the verity inode flag gets persisted. + */ + err = filemap_write_and_wait(inode->i_mapping); + if (err) + goto cleanup; + + /* Set the verity xattr. */ + err = f2fs_setxattr(inode, F2FS_XATTR_INDEX_VERITY, + F2FS_XATTR_NAME_VERITY, &dloc, sizeof(dloc), + NULL, XATTR_CREATE); + if (err) + goto cleanup; - /* If we failed, truncate anything we wrote past i_size. */ - if (desc == NULL || err) - f2fs_truncate(inode); + /* Finally, set the verity inode flag. */ + file_set_verity(inode); + f2fs_set_inode_flags(inode); + f2fs_mark_inode_dirty_sync(inode, true); clear_inode_flag(inode, FI_VERITY_IN_PROGRESS); + return 0; - if (desc != NULL && !err) { - err = f2fs_setxattr(inode, F2FS_XATTR_INDEX_VERITY, - F2FS_XATTR_NAME_VERITY, &dloc, sizeof(dloc), - NULL, XATTR_CREATE); - if (!err) { - file_set_verity(inode); - f2fs_set_inode_flags(inode); - f2fs_mark_inode_dirty_sync(inode, true); - } +cleanup: + /* + * Verity failed to be enabled, so clean up by truncating any verity + * metadata that was written beyond i_size (both from cache and from + * disk) and clearing FI_VERITY_IN_PROGRESS. + * + * Taking i_gc_rwsem[WRITE] is needed to stop f2fs garbage collection + * from re-instantiating cached pages we are truncating (since unlike + * normal file accesses, garbage collection isn't limited by i_size). + */ + down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); + truncate_inode_pages(inode->i_mapping, inode->i_size); + err2 = f2fs_truncate(inode); + if (err2) { + f2fs_err(sbi, "Truncating verity metadata failed (errno=%d)", + err2); + set_sbi_flag(sbi, SBI_NEED_FSCK); } - return err; + up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); + clear_inode_flag(inode, FI_VERITY_IN_PROGRESS); + return err ?: err2; } static int f2fs_get_verity_descriptor(struct inode *inode, void *buf, diff --git a/fs/fuse/cuse.c b/fs/fuse/cuse.c index 45082269e698..a37528b51798 100644 --- a/fs/fuse/cuse.c +++ b/fs/fuse/cuse.c @@ -627,6 +627,8 @@ static int __init cuse_init(void) cuse_channel_fops.owner = THIS_MODULE; cuse_channel_fops.open = cuse_channel_open; cuse_channel_fops.release = cuse_channel_release; + /* CUSE is not prepared for FUSE_DEV_IOC_CLONE */ + cuse_channel_fops.unlocked_ioctl = NULL; cuse_class = class_create(THIS_MODULE, "cuse"); if (IS_ERR(cuse_class)) diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index c0fee830a34e..a5ceccc5ef00 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -2233,11 +2233,8 @@ static long fuse_dev_ioctl(struct file *file, unsigned int cmd, int oldfd; struct fuse_dev *fud = NULL; - if (_IOC_TYPE(cmd) != FUSE_DEV_IOC_MAGIC) - return -ENOTTY; - - switch (_IOC_NR(cmd)) { - case _IOC_NR(FUSE_DEV_IOC_CLONE): + switch (cmd) { + case FUSE_DEV_IOC_CLONE: res = -EFAULT; if (!get_user(oldfd, (__u32 __user *)arg)) { struct file *old = fget(oldfd); diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 8cccecb55fb8..6e6d1e599869 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1099,6 +1099,7 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, struct fuse_file *ff = file->private_data; struct fuse_mount *fm = ff->fm; unsigned int offset, i; + bool short_write; int err; for (i = 0; i < ap->num_pages; i++) @@ -1113,32 +1114,38 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, if (!err && ia->write.out.size > count) err = -EIO; + short_write = ia->write.out.size < count; offset = ap->descs[0].offset; count = ia->write.out.size; for (i = 0; i < ap->num_pages; i++) { struct page *page = ap->pages[i]; - if (!err && !offset && count >= PAGE_SIZE) - SetPageUptodate(page); - - if (count > PAGE_SIZE - offset) - count -= PAGE_SIZE - offset; - else - count = 0; - offset = 0; - - unlock_page(page); + if (err) { + ClearPageUptodate(page); + } else { + if (count >= PAGE_SIZE - offset) + count -= PAGE_SIZE - offset; + else { + if (short_write) + ClearPageUptodate(page); + count = 0; + } + offset = 0; + } + if (ia->write.page_locked && (i == ap->num_pages - 1)) + unlock_page(page); put_page(page); } return err; } -static ssize_t fuse_fill_write_pages(struct fuse_args_pages *ap, +static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, struct address_space *mapping, struct iov_iter *ii, loff_t pos, unsigned int max_pages) { + struct fuse_args_pages *ap = &ia->ap; struct fuse_conn *fc = get_fuse_conn(mapping->host); unsigned offset = pos & (PAGE_SIZE - 1); size_t count = 0; @@ -1191,6 +1198,16 @@ static ssize_t fuse_fill_write_pages(struct fuse_args_pages *ap, if (offset == PAGE_SIZE) offset = 0; + /* If we copied full page, mark it uptodate */ + if (tmp == PAGE_SIZE) + SetPageUptodate(page); + + if (PageUptodate(page)) { + unlock_page(page); + } else { + ia->write.page_locked = true; + break; + } if (!fc->big_writes) break; } while (iov_iter_count(ii) && count < fc->max_write && @@ -1234,7 +1251,7 @@ static ssize_t fuse_perform_write(struct kiocb *iocb, break; } - count = fuse_fill_write_pages(ap, mapping, ii, pos, nr_pages); + count = fuse_fill_write_pages(&ia, mapping, ii, pos, nr_pages); if (count <= 0) { err = count; } else { @@ -1759,8 +1776,17 @@ static void fuse_writepage_end(struct fuse_mount *fm, struct fuse_args *args, container_of(args, typeof(*wpa), ia.ap.args); struct inode *inode = wpa->inode; struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_conn *fc = get_fuse_conn(inode); mapping_set_error(inode->i_mapping, error); + /* + * A writeback finished and this might have updated mtime/ctime on + * server making local mtime/ctime stale. Hence invalidate attrs. + * Do this only if writeback_cache is not enabled. If writeback_cache + * is enabled, we trust local ctime/mtime. + */ + if (!fc->writeback_cache) + fuse_invalidate_attr(inode); spin_lock(&fi->lock); rb_erase(&wpa->writepages_entry, &fi->writepages); while (wpa->next) { diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 63d97a15ffde..74d888c78fa4 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -912,6 +912,7 @@ struct fuse_io_args { struct { struct fuse_write_in in; struct fuse_write_out out; + bool page_locked; } write; }; struct fuse_args_pages ap; diff --git a/fs/fuse/virtio_fs.c b/fs/fuse/virtio_fs.c index 4ee6f734ba83..005209b1cd50 100644 --- a/fs/fuse/virtio_fs.c +++ b/fs/fuse/virtio_fs.c @@ -896,6 +896,7 @@ static int virtio_fs_probe(struct virtio_device *vdev) out_vqs: vdev->config->reset(vdev); virtio_fs_cleanup_vqs(vdev, fs); + kfree(fs->vqs); out: vdev->priv = NULL; @@ -1436,8 +1437,7 @@ static int virtio_fs_get_tree(struct fs_context *fsc) if (!fm) goto out_err; - fuse_conn_init(fc, fm, get_user_ns(current_user_ns()), - &virtio_fs_fiq_ops, fs); + fuse_conn_init(fc, fm, fsc->user_ns, &virtio_fs_fiq_ops, fs); fc->release = fuse_free_conn; fc->delete_stale = true; fc->auto_submounts = true; diff --git a/fs/hfsplus/extents.c b/fs/hfsplus/extents.c index a930ddd15681..7054a542689f 100644 --- a/fs/hfsplus/extents.c +++ b/fs/hfsplus/extents.c @@ -598,13 +598,15 @@ void hfsplus_file_truncate(struct inode *inode) res = __hfsplus_ext_cache_extent(&fd, inode, alloc_cnt); if (res) break; - hfs_brec_remove(&fd); - mutex_unlock(&fd.tree->tree_lock); start = hip->cached_start; + if (blk_cnt <= start) + hfs_brec_remove(&fd); + mutex_unlock(&fd.tree->tree_lock); hfsplus_free_extents(sb, hip->cached_extents, alloc_cnt - start, alloc_cnt - blk_cnt); hfsplus_dump_extent(hip->cached_extents); + mutex_lock(&fd.tree->tree_lock); if (blk_cnt > start) { hip->extent_state |= HFSPLUS_EXT_DIRTY; break; @@ -612,7 +614,6 @@ void hfsplus_file_truncate(struct inode *inode) alloc_cnt = start; hip->cached_start = hip->cached_blocks = 0; hip->extent_state &= ~(HFSPLUS_EXT_DIRTY | HFSPLUS_EXT_NEW); - mutex_lock(&fd.tree->tree_lock); } hfs_find_exit(&fd); diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 701c82c36138..99df69b84822 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -131,6 +131,7 @@ static void huge_pagevec_release(struct pagevec *pvec) static int hugetlbfs_file_mmap(struct file *file, struct vm_area_struct *vma) { struct inode *inode = file_inode(file); + struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode); loff_t len, vma_len; int ret; struct hstate *h = hstate_file(file); @@ -146,6 +147,10 @@ static int hugetlbfs_file_mmap(struct file *file, struct vm_area_struct *vma) vma->vm_flags |= VM_HUGETLB | VM_DONTEXPAND; vma->vm_ops = &hugetlb_vm_ops; + ret = seal_check_future_write(info->seals, vma); + if (ret) + return ret; + /* * page based offset in vm_pgoff could be sufficiently large to * overflow a loff_t when converted to byte offset. This can diff --git a/fs/io_uring.c b/fs/io_uring.c index dff34975d86b..144056b0cac9 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -238,7 +238,7 @@ struct fixed_rsrc_data { struct io_buffer { struct list_head list; __u64 addr; - __s32 len; + __u32 len; __u16 bid; }; @@ -614,7 +614,7 @@ struct io_splice { struct io_provide_buf { struct file *file; __u64 addr; - __s32 len; + __u32 len; __u32 bgid; __u16 nbufs; __u16 bid; @@ -1008,7 +1008,7 @@ static void io_uring_del_task_file(unsigned long index); static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, struct task_struct *task, struct files_struct *files); -static void io_uring_cancel_sqpoll(struct io_ring_ctx *ctx); +static void io_uring_cancel_sqpoll(struct io_sq_data *sqd); static void destroy_fixed_rsrc_ref_node(struct fixed_rsrc_ref_node *ref_node); static struct fixed_rsrc_ref_node *alloc_fixed_rsrc_ref_node( struct io_ring_ctx *ctx); @@ -3979,7 +3979,7 @@ static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags) static int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - unsigned long size; + unsigned long size, tmp_check; struct io_provide_buf *p = &req->pbuf; u64 tmp; @@ -3993,6 +3993,12 @@ static int io_provide_buffers_prep(struct io_kiocb *req, p->addr = READ_ONCE(sqe->addr); p->len = READ_ONCE(sqe->len); + if (check_mul_overflow((unsigned long)p->len, (unsigned long)p->nbufs, + &size)) + return -EOVERFLOW; + if (check_add_overflow((unsigned long)p->addr, size, &tmp_check)) + return -EOVERFLOW; + size = (unsigned long)p->len * p->nbufs; if (!access_ok(u64_to_user_ptr(p->addr), size)) return -EFAULT; @@ -4017,7 +4023,7 @@ static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head) break; buf->addr = addr; - buf->len = pbuf->len; + buf->len = min_t(__u32, pbuf->len, MAX_RW_COUNT); buf->bid = bid; addr += pbuf->len; bid++; @@ -6710,6 +6716,10 @@ static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries) if (!list_empty(&ctx->iopoll_list)) io_do_iopoll(ctx, &nr_events, 0); + /* + * Don't submit if refs are dying, good for io_uring_register(), + * but also it is relied upon by io_ring_exit_work() + */ if (to_submit && likely(!percpu_ref_is_dying(&ctx->refs)) && !(ctx->flags & IORING_SETUP_R_DISABLED)) ret = io_submit_sqes(ctx, to_submit); @@ -6832,15 +6842,14 @@ static int io_sq_thread(void *data) timeout = jiffies + sqd->sq_thread_idle; } - list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) - io_uring_cancel_sqpoll(ctx); + io_uring_cancel_sqpoll(sqd); sqd->thread = NULL; list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) io_ring_set_wakeup_flag(ctx); - mutex_unlock(&sqd->lock); - io_run_task_work(); io_run_task_work_head(&sqd->park_task_work); + mutex_unlock(&sqd->lock); + complete(&sqd->exited); do_exit(0); } @@ -7200,8 +7209,6 @@ static void io_sq_thread_finish(struct io_ring_ctx *ctx) io_put_sq_data(sqd); ctx->sq_data = NULL; - if (ctx->sq_creds) - put_cred(ctx->sq_creds); } } @@ -8469,6 +8476,8 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) mutex_unlock(&ctx->uring_lock); io_eventfd_unregister(ctx); io_destroy_buffers(ctx); + if (ctx->sq_creds) + put_cred(ctx->sq_creds); #if defined(CONFIG_UNIX) if (ctx->ring_sock) { @@ -8568,6 +8577,13 @@ static void io_tctx_exit_cb(struct callback_head *cb) complete(&work->completion); } +static bool io_cancel_ctx_cb(struct io_wq_work *work, void *data) +{ + struct io_kiocb *req = container_of(work, struct io_kiocb, work); + + return req->ctx == data; +} + static void io_ring_exit_work(struct work_struct *work) { struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work); @@ -8576,14 +8592,6 @@ static void io_ring_exit_work(struct work_struct *work) struct io_tctx_node *node; int ret; - /* prevent SQPOLL from submitting new requests */ - if (ctx->sq_data) { - io_sq_thread_park(ctx->sq_data); - list_del_init(&ctx->sqd_list); - io_sqd_update_thread_idle(ctx->sq_data); - io_sq_thread_unpark(ctx->sq_data); - } - /* * If we're doing polled IO and end up having requests being * submitted async (out-of-line), then completions can come in while @@ -8592,6 +8600,17 @@ static void io_ring_exit_work(struct work_struct *work) */ do { io_uring_try_cancel_requests(ctx, NULL, NULL); + if (ctx->sq_data) { + struct io_sq_data *sqd = ctx->sq_data; + struct task_struct *tsk; + + io_sq_thread_park(sqd); + tsk = sqd->thread; + if (tsk && tsk->io_uring && tsk->io_uring->io_wq) + io_wq_cancel_cb(tsk->io_uring->io_wq, + io_cancel_ctx_cb, ctx, true); + io_sq_thread_unpark(sqd); + } WARN_ON_ONCE(time_after(jiffies, timeout)); } while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20)); @@ -8736,13 +8755,6 @@ static bool io_cancel_defer_files(struct io_ring_ctx *ctx, return true; } -static bool io_cancel_ctx_cb(struct io_wq_work *work, void *data) -{ - struct io_kiocb *req = container_of(work, struct io_kiocb, work); - - return req->ctx == data; -} - static bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx) { struct io_tctx_node *node; @@ -8935,11 +8947,11 @@ static s64 tctx_inflight(struct io_uring_task *tctx) static void io_sqpoll_cancel_cb(struct callback_head *cb) { struct io_tctx_exit *work = container_of(cb, struct io_tctx_exit, task_work); - struct io_ring_ctx *ctx = work->ctx; - struct io_sq_data *sqd = ctx->sq_data; + struct io_sq_data *sqd = work->ctx->sq_data; if (sqd->thread) - io_uring_cancel_sqpoll(ctx); + io_uring_cancel_sqpoll(sqd); + list_del_init(&work->ctx->sqd_list); complete(&work->completion); } @@ -8950,7 +8962,6 @@ static void io_sqpoll_cancel_sync(struct io_ring_ctx *ctx) struct task_struct *task; io_sq_thread_park(sqd); - list_del_init(&ctx->sqd_list); io_sqd_update_thread_idle(sqd); task = sqd->thread; if (task) { @@ -8958,6 +8969,8 @@ static void io_sqpoll_cancel_sync(struct io_ring_ctx *ctx) init_task_work(&work.task_work, io_sqpoll_cancel_cb); io_task_work_add_head(&sqd->park_task_work, &work.task_work); wake_up_process(task); + } else { + list_del_init(&ctx->sqd_list); } io_sq_thread_unpark(sqd); @@ -8991,14 +9004,16 @@ void __io_uring_files_cancel(struct files_struct *files) } /* should only be called by SQPOLL task */ -static void io_uring_cancel_sqpoll(struct io_ring_ctx *ctx) +static void io_uring_cancel_sqpoll(struct io_sq_data *sqd) { - struct io_sq_data *sqd = ctx->sq_data; struct io_uring_task *tctx = current->io_uring; + struct io_ring_ctx *ctx; s64 inflight; DEFINE_WAIT(wait); - WARN_ON_ONCE(!sqd || ctx->sq_data->thread != current); + if (!current->io_uring) + return; + WARN_ON_ONCE(!sqd || sqd->thread != current); atomic_inc(&tctx->in_idle); do { @@ -9006,7 +9021,8 @@ static void io_uring_cancel_sqpoll(struct io_ring_ctx *ctx) inflight = tctx_inflight(tctx); if (!inflight) break; - io_uring_try_cancel_requests(ctx, current, NULL); + list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) + io_uring_try_cancel_requests(ctx, current, NULL); prepare_to_wait(&tctx->wait, &wait, TASK_UNINTERRUPTIBLE); /* diff --git a/fs/jbd2/recovery.c b/fs/jbd2/recovery.c index 69f18fe20923..d47a0d96bf30 100644 --- a/fs/jbd2/recovery.c +++ b/fs/jbd2/recovery.c @@ -245,15 +245,14 @@ static int fc_do_one_pass(journal_t *journal, return 0; while (next_fc_block <= journal->j_fc_last) { - jbd_debug(3, "Fast commit replay: next block %ld", + jbd_debug(3, "Fast commit replay: next block %ld\n", next_fc_block); err = jread(&bh, journal, next_fc_block); if (err) { - jbd_debug(3, "Fast commit replay: read error"); + jbd_debug(3, "Fast commit replay: read error\n"); break; } - jbd_debug(3, "Processing fast commit blk with seq %d"); err = journal->j_fc_replay_callback(journal, bh, pass, next_fc_block - journal->j_fc_first, expected_commit_id); diff --git a/fs/jbd2/transaction.c b/fs/jbd2/transaction.c index 9396666b7314..e8fc45fd751f 100644 --- a/fs/jbd2/transaction.c +++ b/fs/jbd2/transaction.c @@ -349,7 +349,12 @@ static int start_this_handle(journal_t *journal, handle_t *handle, } alloc_transaction: - if (!journal->j_running_transaction) { + /* + * This check is racy but it is just an optimization of allocating new + * transaction early if there are high chances we'll need it. If we + * guess wrong, we'll retry or free unused transaction. + */ + if (!data_race(journal->j_running_transaction)) { /* * If __GFP_FS is not present, then we may be being called from * inside the fs writeback layer, so we MUST NOT fail. @@ -1474,8 +1479,8 @@ int jbd2_journal_dirty_metadata(handle_t *handle, struct buffer_head *bh) * crucial to catch bugs so let's do a reliable check until the * lockless handling is fully proven. */ - if (jh->b_transaction != transaction && - jh->b_next_transaction != transaction) { + if (data_race(jh->b_transaction != transaction && + jh->b_next_transaction != transaction)) { spin_lock(&jh->b_state_lock); J_ASSERT_JH(jh, jh->b_transaction == transaction || jh->b_next_transaction == transaction); @@ -1483,8 +1488,8 @@ int jbd2_journal_dirty_metadata(handle_t *handle, struct buffer_head *bh) } if (jh->b_modified == 1) { /* If it's in our transaction it must be in BJ_Metadata list. */ - if (jh->b_transaction == transaction && - jh->b_jlist != BJ_Metadata) { + if (data_race(jh->b_transaction == transaction && + jh->b_jlist != BJ_Metadata)) { spin_lock(&jh->b_state_lock); if (jh->b_transaction == transaction && jh->b_jlist != BJ_Metadata) diff --git a/fs/jffs2/file.c b/fs/jffs2/file.c index f8fb89b10227..4fc8cd698d1a 100644 --- a/fs/jffs2/file.c +++ b/fs/jffs2/file.c @@ -57,6 +57,7 @@ const struct file_operations jffs2_file_operations = .mmap = generic_file_readonly_mmap, .fsync = jffs2_fsync, .splice_read = generic_file_splice_read, + .splice_write = iter_file_splice_write, }; /* jffs2_file_inode_operations */ diff --git a/fs/jffs2/scan.c b/fs/jffs2/scan.c index db72a9d2d0af..b676056826be 100644 --- a/fs/jffs2/scan.c +++ b/fs/jffs2/scan.c @@ -1079,7 +1079,7 @@ static int jffs2_scan_dirent_node(struct jffs2_sb_info *c, struct jffs2_eraseblo memcpy(&fd->name, rd->name, checkedlen); fd->name[checkedlen] = 0; - crc = crc32(0, fd->name, rd->nsize); + crc = crc32(0, fd->name, checkedlen); if (crc != je32_to_cpu(rd->name_crc)) { pr_notice("%s(): Name CRC failed on node at 0x%08x: Read 0x%08x, calculated 0x%08x\n", __func__, ofs, je32_to_cpu(rd->name_crc), crc); diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c index f7786e00a6a7..ed9d580826f5 100644 --- a/fs/nfs/callback_proc.c +++ b/fs/nfs/callback_proc.c @@ -137,12 +137,12 @@ static struct inode *nfs_layout_find_inode_by_stateid(struct nfs_client *clp, list_for_each_entry_rcu(lo, &server->layouts, plh_layouts) { if (!pnfs_layout_is_valid(lo)) continue; - if (stateid != NULL && - !nfs4_stateid_match_other(stateid, &lo->plh_stateid)) + if (!nfs4_stateid_match_other(stateid, &lo->plh_stateid)) continue; - if (!nfs_sb_active(server->super)) - continue; - inode = igrab(lo->plh_inode); + if (nfs_sb_active(server->super)) + inode = igrab(lo->plh_inode); + else + inode = ERR_PTR(-EAGAIN); rcu_read_unlock(); if (inode) return inode; @@ -176,9 +176,10 @@ static struct inode *nfs_layout_find_inode_by_fh(struct nfs_client *clp, continue; if (nfsi->layout != lo) continue; - if (!nfs_sb_active(server->super)) - continue; - inode = igrab(lo->plh_inode); + if (nfs_sb_active(server->super)) + inode = igrab(lo->plh_inode); + else + inode = ERR_PTR(-EAGAIN); rcu_read_unlock(); if (inode) return inode; diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index fc4f490f2d78..0cd7c59a6601 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -866,6 +866,8 @@ static int nfs_readdir_xdr_to_array(struct nfs_readdir_descriptor *desc, break; } + verf_arg = verf_res; + status = nfs_readdir_page_filler(desc, entry, pages, pglen, arrays, narrays); } while (!status && nfs_readdir_page_needs_filling(page)); @@ -927,7 +929,12 @@ static int find_and_lock_cache_page(struct nfs_readdir_descriptor *desc) } return res; } - memcpy(nfsi->cookieverf, verf, sizeof(nfsi->cookieverf)); + /* + * Set the cookie verifier if the page cache was empty + */ + if (desc->page_index == 0) + memcpy(nfsi->cookieverf, verf, + sizeof(nfsi->cookieverf)); } res = nfs_readdir_search_array(desc); if (res == 0) { @@ -974,10 +981,10 @@ static int readdir_search_pagecache(struct nfs_readdir_descriptor *desc) /* * Once we've found the start of the dirent within a page: fill 'er up... */ -static void nfs_do_filldir(struct nfs_readdir_descriptor *desc) +static void nfs_do_filldir(struct nfs_readdir_descriptor *desc, + const __be32 *verf) { struct file *file = desc->file; - struct nfs_inode *nfsi = NFS_I(file_inode(file)); struct nfs_cache_array *array; unsigned int i = 0; @@ -991,7 +998,7 @@ static void nfs_do_filldir(struct nfs_readdir_descriptor *desc) desc->eof = true; break; } - memcpy(desc->verf, nfsi->cookieverf, sizeof(desc->verf)); + memcpy(desc->verf, verf, sizeof(desc->verf)); if (i < (array->size-1)) desc->dir_cookie = array->array[i+1].cookie; else @@ -1048,7 +1055,7 @@ static int uncached_readdir(struct nfs_readdir_descriptor *desc) for (i = 0; !desc->eof && i < sz && arrays[i]; i++) { desc->page = arrays[i]; - nfs_do_filldir(desc); + nfs_do_filldir(desc, verf); } desc->page = NULL; @@ -1069,6 +1076,7 @@ static int nfs_readdir(struct file *file, struct dir_context *ctx) { struct dentry *dentry = file_dentry(file); struct inode *inode = d_inode(dentry); + struct nfs_inode *nfsi = NFS_I(inode); struct nfs_open_dir_context *dir_ctx = file->private_data; struct nfs_readdir_descriptor *desc; int res; @@ -1122,7 +1130,7 @@ static int nfs_readdir(struct file *file, struct dir_context *ctx) break; } if (res == -ETOOSMALL && desc->plus) { - clear_bit(NFS_INO_ADVISE_RDPLUS, &NFS_I(inode)->flags); + clear_bit(NFS_INO_ADVISE_RDPLUS, &nfsi->flags); nfs_zap_caches(inode); desc->page_index = 0; desc->plus = false; @@ -1132,7 +1140,7 @@ static int nfs_readdir(struct file *file, struct dir_context *ctx) if (res < 0) break; - nfs_do_filldir(desc); + nfs_do_filldir(desc, nfsi->cookieverf); nfs_readdir_page_unlock_and_put_cached(desc); } while (!desc->eof); diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index 872112bffcab..d383de00d486 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -106,7 +106,7 @@ static int decode_nfs_fh(struct xdr_stream *xdr, struct nfs_fh *fh) if (unlikely(!p)) return -ENOBUFS; fh->size = be32_to_cpup(p++); - if (fh->size > sizeof(struct nfs_fh)) { + if (fh->size > NFS_MAXFHSIZE) { printk(KERN_ERR "NFS flexfiles: Too big fh received %d\n", fh->size); return -EOVERFLOW; diff --git a/fs/nfs/fs_context.c b/fs/nfs/fs_context.c index 971a9251c1d9..902db1262d2b 100644 --- a/fs/nfs/fs_context.c +++ b/fs/nfs/fs_context.c @@ -973,6 +973,15 @@ static int nfs23_parse_monolithic(struct fs_context *fc, memset(mntfh->data + mntfh->size, 0, sizeof(mntfh->data) - mntfh->size); + /* + * for proto == XPRT_TRANSPORT_UDP, which is what uses + * to_exponential, implying shift: limit the shift value + * to BITS_PER_LONG (majortimeo is unsigned long) + */ + if (!(data->flags & NFS_MOUNT_TCP)) /* this will be UDP */ + if (data->retrans >= 64) /* shift value is too large */ + goto out_invalid_data; + /* * Translate to nfs_fs_context, which nfs_fill_super * can deal with. @@ -1073,6 +1082,9 @@ static int nfs23_parse_monolithic(struct fs_context *fc, out_invalid_fh: return nfs_invalf(fc, "NFS: invalid root filehandle"); + +out_invalid_data: + return nfs_invalf(fc, "NFS: invalid binary mount data"); } #if IS_ENABLED(CONFIG_NFS_V4) diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index a7fb076a5f44..ae8bc84e39fb 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -219,15 +219,16 @@ void nfs_set_cache_invalid(struct inode *inode, unsigned long flags) | NFS_INO_INVALID_SIZE | NFS_INO_REVAL_PAGECACHE | NFS_INO_INVALID_XATTR); - } + } else if (flags & NFS_INO_REVAL_PAGECACHE) + flags |= NFS_INO_INVALID_CHANGE | NFS_INO_INVALID_SIZE; if (!nfs_has_xattr_cache(nfsi)) flags &= ~NFS_INO_INVALID_XATTR; + if (flags & NFS_INO_INVALID_DATA) + nfs_fscache_invalidate(inode); if (inode->i_mapping->nrpages == 0) flags &= ~(NFS_INO_INVALID_DATA|NFS_INO_DATA_INVAL_DEFER); nfsi->cache_validity |= flags; - if (flags & NFS_INO_INVALID_DATA) - nfs_fscache_invalidate(inode); } EXPORT_SYMBOL_GPL(nfs_set_cache_invalid); @@ -1662,10 +1663,10 @@ EXPORT_SYMBOL_GPL(_nfs_display_fhandle); */ static int nfs_inode_attrs_need_update(const struct inode *inode, const struct nfs_fattr *fattr) { - const struct nfs_inode *nfsi = NFS_I(inode); + unsigned long attr_gencount = NFS_I(inode)->attr_gencount; - return ((long)fattr->gencount - (long)nfsi->attr_gencount) > 0 || - ((long)nfsi->attr_gencount - (long)nfs_read_attr_generation_counter() > 0); + return (long)(fattr->gencount - attr_gencount) > 0 || + (long)(attr_gencount - nfs_read_attr_generation_counter()) > 0; } static int nfs_refresh_inode_locked(struct inode *inode, struct nfs_fattr *fattr) @@ -2094,7 +2095,7 @@ static int nfs_update_inode(struct inode *inode, struct nfs_fattr *fattr) nfsi->attrtimeo_timestamp = now; } /* Set the barrier to be more recent than this fattr */ - if ((long)fattr->gencount - (long)nfsi->attr_gencount > 0) + if ((long)(fattr->gencount - nfsi->attr_gencount) > 0) nfsi->attr_gencount = fattr->gencount; } diff --git a/fs/nfs/nfs42proc.c b/fs/nfs/nfs42proc.c index 094024b0aca1..3875120ef3ef 100644 --- a/fs/nfs/nfs42proc.c +++ b/fs/nfs/nfs42proc.c @@ -46,11 +46,12 @@ static int _nfs42_proc_fallocate(struct rpc_message *msg, struct file *filep, { struct inode *inode = file_inode(filep); struct nfs_server *server = NFS_SERVER(inode); + u32 bitmask[3]; struct nfs42_falloc_args args = { .falloc_fh = NFS_FH(inode), .falloc_offset = offset, .falloc_length = len, - .falloc_bitmask = nfs4_fattr_bitmap, + .falloc_bitmask = bitmask, }; struct nfs42_falloc_res res = { .falloc_server = server, @@ -68,6 +69,10 @@ static int _nfs42_proc_fallocate(struct rpc_message *msg, struct file *filep, return status; } + memcpy(bitmask, server->cache_consistency_bitmask, sizeof(bitmask)); + if (server->attr_bitmask[1] & FATTR4_WORD1_SPACE_USED) + bitmask[1] |= FATTR4_WORD1_SPACE_USED; + res.falloc_fattr = nfs_alloc_fattr(); if (!res.falloc_fattr) return -ENOMEM; @@ -75,7 +80,8 @@ static int _nfs42_proc_fallocate(struct rpc_message *msg, struct file *filep, status = nfs4_call_sync(server->client, server, msg, &args.seq_args, &res.seq_res, 0); if (status == 0) - status = nfs_post_op_update_inode(inode, res.falloc_fattr); + status = nfs_post_op_update_inode_force_wcc(inode, + res.falloc_fattr); kfree(res.falloc_fattr); return status; @@ -84,7 +90,8 @@ static int _nfs42_proc_fallocate(struct rpc_message *msg, struct file *filep, static int nfs42_proc_fallocate(struct rpc_message *msg, struct file *filep, loff_t offset, loff_t len) { - struct nfs_server *server = NFS_SERVER(file_inode(filep)); + struct inode *inode = file_inode(filep); + struct nfs_server *server = NFS_SERVER(inode); struct nfs4_exception exception = { }; struct nfs_lock_context *lock; int err; @@ -93,9 +100,13 @@ static int nfs42_proc_fallocate(struct rpc_message *msg, struct file *filep, if (IS_ERR(lock)) return PTR_ERR(lock); - exception.inode = file_inode(filep); + exception.inode = inode; exception.state = lock->open_context->state; + err = nfs_sync_inode(inode); + if (err) + goto out; + do { err = _nfs42_proc_fallocate(msg, filep, lock, offset, len); if (err == -ENOTSUPP) { @@ -104,7 +115,7 @@ static int nfs42_proc_fallocate(struct rpc_message *msg, struct file *filep, } err = nfs4_handle_exception(server, err, &exception); } while (exception.retry); - +out: nfs_put_lock_context(lock); return err; } @@ -142,16 +153,13 @@ int nfs42_proc_deallocate(struct file *filep, loff_t offset, loff_t len) return -EOPNOTSUPP; inode_lock(inode); - err = nfs_sync_inode(inode); - if (err) - goto out_unlock; err = nfs42_proc_fallocate(&msg, filep, offset, len); if (err == 0) truncate_pagecache_range(inode, offset, (offset + len) -1); if (err == -EOPNOTSUPP) NFS_SERVER(inode)->caps &= ~NFS_CAP_DEALLOCATE; -out_unlock: + inode_unlock(inode); return err; } @@ -261,6 +269,33 @@ static int process_copy_commit(struct file *dst, loff_t pos_dst, return status; } +/** + * nfs42_copy_dest_done - perform inode cache updates after clone/copy offload + * @inode: pointer to destination inode + * @pos: destination offset + * @len: copy length + * + * Punch a hole in the inode page cache, so that the NFS client will + * know to retrieve new data. + * Update the file size if necessary, and then mark the inode as having + * invalid cached values for change attribute, ctime, mtime and space used. + */ +static void nfs42_copy_dest_done(struct inode *inode, loff_t pos, loff_t len) +{ + loff_t newsize = pos + len; + loff_t end = newsize - 1; + + truncate_pagecache_range(inode, pos, end); + spin_lock(&inode->i_lock); + if (newsize > i_size_read(inode)) + i_size_write(inode, newsize); + nfs_set_cache_invalid(inode, NFS_INO_INVALID_CHANGE | + NFS_INO_INVALID_CTIME | + NFS_INO_INVALID_MTIME | + NFS_INO_INVALID_BLOCKS); + spin_unlock(&inode->i_lock); +} + static ssize_t _nfs42_proc_copy(struct file *src, struct nfs_lock_context *src_lock, struct file *dst, @@ -354,14 +389,8 @@ static ssize_t _nfs42_proc_copy(struct file *src, goto out; } - truncate_pagecache_range(dst_inode, pos_dst, - pos_dst + res->write_res.count); - spin_lock(&dst_inode->i_lock); - nfs_set_cache_invalid( - dst_inode, NFS_INO_REVAL_PAGECACHE | NFS_INO_REVAL_FORCED | - NFS_INO_INVALID_SIZE | NFS_INO_INVALID_ATTR | - NFS_INO_INVALID_DATA); - spin_unlock(&dst_inode->i_lock); + nfs42_copy_dest_done(dst_inode, pos_dst, res->write_res.count); + spin_lock(&src_inode->i_lock); nfs_set_cache_invalid(src_inode, NFS_INO_REVAL_PAGECACHE | NFS_INO_REVAL_FORCED | @@ -659,7 +688,10 @@ static loff_t _nfs42_proc_llseek(struct file *filep, if (status) return status; - return vfs_setpos(filep, res.sr_offset, inode->i_sb->s_maxbytes); + if (whence == SEEK_DATA && res.sr_eof) + return -NFS4ERR_NXIO; + else + return vfs_setpos(filep, res.sr_offset, inode->i_sb->s_maxbytes); } loff_t nfs42_proc_llseek(struct file *filep, loff_t offset, int whence) @@ -1044,8 +1076,10 @@ static int _nfs42_proc_clone(struct rpc_message *msg, struct file *src_f, status = nfs4_call_sync(server->client, server, msg, &args.seq_args, &res.seq_res, 0); - if (status == 0) + if (status == 0) { + nfs42_copy_dest_done(dst_inode, dst_offset, count); status = nfs_post_op_update_inode(dst_inode, res.dst_fattr); + } kfree(res.dst_fattr); return status; diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index c65c4b41e2c1..820abae88cf0 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -108,9 +108,10 @@ static int nfs41_test_stateid(struct nfs_server *, nfs4_stateid *, static int nfs41_free_stateid(struct nfs_server *, const nfs4_stateid *, const struct cred *, bool); #endif -static void nfs4_bitmask_adjust(__u32 *bitmask, struct inode *inode, - struct nfs_server *server, - struct nfs4_label *label); +static void nfs4_bitmask_set(__u32 bitmask[NFS4_BITMASK_SZ], + const __u32 *src, struct inode *inode, + struct nfs_server *server, + struct nfs4_label *label); #ifdef CONFIG_NFS_V4_SECURITY_LABEL static inline struct nfs4_label * @@ -3591,6 +3592,7 @@ static void nfs4_close_prepare(struct rpc_task *task, void *data) struct nfs4_closedata *calldata = data; struct nfs4_state *state = calldata->state; struct inode *inode = calldata->inode; + struct nfs_server *server = NFS_SERVER(inode); struct pnfs_layout_hdr *lo; bool is_rdonly, is_wronly, is_rdwr; int call_close = 0; @@ -3647,8 +3649,10 @@ static void nfs4_close_prepare(struct rpc_task *task, void *data) if (calldata->arg.fmode == 0 || calldata->arg.fmode == FMODE_READ) { /* Close-to-open cache consistency revalidation */ if (!nfs4_have_delegation(inode, FMODE_READ)) { - calldata->arg.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask; - nfs4_bitmask_adjust(calldata->arg.bitmask, inode, NFS_SERVER(inode), NULL); + nfs4_bitmask_set(calldata->arg.bitmask_store, + server->cache_consistency_bitmask, + inode, server, NULL); + calldata->arg.bitmask = calldata->arg.bitmask_store; } else calldata->arg.bitmask = NULL; } @@ -5416,19 +5420,17 @@ bool nfs4_write_need_cache_consistency_data(struct nfs_pgio_header *hdr) return nfs4_have_delegation(hdr->inode, FMODE_READ) == 0; } -static void nfs4_bitmask_adjust(__u32 *bitmask, struct inode *inode, - struct nfs_server *server, - struct nfs4_label *label) +static void nfs4_bitmask_set(__u32 bitmask[NFS4_BITMASK_SZ], const __u32 *src, + struct inode *inode, struct nfs_server *server, + struct nfs4_label *label) { - unsigned long cache_validity = READ_ONCE(NFS_I(inode)->cache_validity); + unsigned int i; - if ((cache_validity & NFS_INO_INVALID_DATA) || - (cache_validity & NFS_INO_REVAL_PAGECACHE) || - (cache_validity & NFS_INO_REVAL_FORCED) || - (cache_validity & NFS_INO_INVALID_OTHER)) - nfs4_bitmap_copy_adjust(bitmask, nfs4_bitmask(server, label), inode); + memcpy(bitmask, src, sizeof(*bitmask) * NFS4_BITMASK_SZ); + if (cache_validity & (NFS_INO_INVALID_CHANGE | NFS_INO_REVAL_PAGECACHE)) + bitmask[0] |= FATTR4_WORD0_CHANGE; if (cache_validity & NFS_INO_INVALID_ATIME) bitmask[1] |= FATTR4_WORD1_TIME_ACCESS; if (cache_validity & NFS_INO_INVALID_OTHER) @@ -5437,16 +5439,22 @@ static void nfs4_bitmask_adjust(__u32 *bitmask, struct inode *inode, FATTR4_WORD1_NUMLINKS; if (label && label->len && cache_validity & NFS_INO_INVALID_LABEL) bitmask[2] |= FATTR4_WORD2_SECURITY_LABEL; - if (cache_validity & NFS_INO_INVALID_CHANGE) - bitmask[0] |= FATTR4_WORD0_CHANGE; if (cache_validity & NFS_INO_INVALID_CTIME) bitmask[1] |= FATTR4_WORD1_TIME_METADATA; if (cache_validity & NFS_INO_INVALID_MTIME) bitmask[1] |= FATTR4_WORD1_TIME_MODIFY; - if (cache_validity & NFS_INO_INVALID_SIZE) - bitmask[0] |= FATTR4_WORD0_SIZE; if (cache_validity & NFS_INO_INVALID_BLOCKS) bitmask[1] |= FATTR4_WORD1_SPACE_USED; + + if (nfs4_have_delegation(inode, FMODE_READ) && + !(cache_validity & NFS_INO_REVAL_FORCED)) + bitmask[0] &= ~FATTR4_WORD0_SIZE; + else if (cache_validity & + (NFS_INO_INVALID_SIZE | NFS_INO_REVAL_PAGECACHE)) + bitmask[0] |= FATTR4_WORD0_SIZE; + + for (i = 0; i < NFS4_BITMASK_SZ; i++) + bitmask[i] &= server->attr_bitmask[i]; } static void nfs4_proc_write_setup(struct nfs_pgio_header *hdr, @@ -5459,8 +5467,10 @@ static void nfs4_proc_write_setup(struct nfs_pgio_header *hdr, hdr->args.bitmask = NULL; hdr->res.fattr = NULL; } else { - hdr->args.bitmask = server->cache_consistency_bitmask; - nfs4_bitmask_adjust(hdr->args.bitmask, hdr->inode, server, NULL); + nfs4_bitmask_set(hdr->args.bitmask_store, + server->cache_consistency_bitmask, + hdr->inode, server, NULL); + hdr->args.bitmask = hdr->args.bitmask_store; } if (!hdr->pgio_done_cb) @@ -6502,8 +6512,10 @@ static int _nfs4_proc_delegreturn(struct inode *inode, const struct cred *cred, data->args.fhandle = &data->fh; data->args.stateid = &data->stateid; - data->args.bitmask = server->cache_consistency_bitmask; - nfs4_bitmask_adjust(data->args.bitmask, inode, server, NULL); + nfs4_bitmask_set(data->args.bitmask_store, + server->cache_consistency_bitmask, inode, server, + NULL); + data->args.bitmask = data->args.bitmask_store; nfs_copy_fh(&data->fh, NFS_FH(inode)); nfs4_stateid_copy(&data->stateid, stateid); data->res.fattr = &data->fattr; diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index 102b66e0bdef..f726f8b12b7e 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -1344,7 +1344,7 @@ _pnfs_return_layout(struct inode *ino) } valid_layout = pnfs_layout_is_valid(lo); pnfs_clear_layoutcommit(ino, &tmp_list); - pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL, 0); + pnfs_mark_matching_lsegs_return(lo, &tmp_list, NULL, 0); if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) { struct pnfs_layout_range range = { @@ -2468,6 +2468,9 @@ pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo, assert_spin_locked(&lo->plh_inode->i_lock); + if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) + tmp_list = &lo->plh_return_segs; + list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) if (pnfs_match_lseg_recall(lseg, return_range, seq)) { dprintk("%s: marking lseg %p iomode %d " @@ -2475,6 +2478,8 @@ pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo, lseg, lseg->pls_range.iomode, lseg->pls_range.offset, lseg->pls_range.length); + if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) + tmp_list = &lo->plh_return_segs; if (mark_lseg_invalid(lseg, tmp_list)) continue; remaining++; diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index dd9f38d072dd..e13c4c81fb89 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -1538,8 +1538,8 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, if (!nfs4_init_copy_state(nn, copy)) goto out_err; refcount_set(&async_copy->refcount, 1); - memcpy(©->cp_res.cb_stateid, ©->cp_stateid, - sizeof(copy->cp_stateid)); + memcpy(©->cp_res.cb_stateid, ©->cp_stateid.stid, + sizeof(copy->cp_res.cb_stateid)); dup_copy_fields(copy, async_copy); async_copy->copy_task = kthread_create(nfsd4_do_async_copy, async_copy, "%s", "copy thread"); diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 97447a64bad0..886e50ed07c2 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -4869,6 +4869,11 @@ static __be32 nfs4_get_vfs_file(struct svc_rqst *rqstp, struct nfs4_file *fp, if (nf) nfsd_file_put(nf); + status = nfserrno(nfsd_open_break_lease(cur_fh->fh_dentry->d_inode, + access)); + if (status) + goto out_put_access; + status = nfsd4_truncate(rqstp, cur_fh, open); if (status) goto out_put_access; @@ -6849,11 +6854,20 @@ nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, static __be32 nfsd_test_lock(struct svc_rqst *rqstp, struct svc_fh *fhp, struct file_lock *lock) { struct nfsd_file *nf; - __be32 err = nfsd_file_acquire(rqstp, fhp, NFSD_MAY_READ, &nf); - if (!err) { - err = nfserrno(vfs_test_lock(nf->nf_file, lock)); - nfsd_file_put(nf); - } + __be32 err; + + err = nfsd_file_acquire(rqstp, fhp, NFSD_MAY_READ, &nf); + if (err) + return err; + fh_lock(fhp); /* to block new leases till after test_lock: */ + err = nfserrno(nfsd_open_break_lease(fhp->fh_dentry->d_inode, + NFSD_MAY_READ)); + if (err) + goto out; + err = nfserrno(vfs_test_lock(nf->nf_file, lock)); +out: + fh_unlock(fhp); + nfsd_file_put(nf); return err; } diff --git a/fs/ntfs3/Kconfig b/fs/ntfs3/Kconfig new file mode 100644 index 000000000000..6e4cbc48ab8e --- /dev/null +++ b/fs/ntfs3/Kconfig @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: GPL-2.0-only +config NTFS3_FS + tristate "NTFS Read-Write file system support" + select NLS + help + Windows OS native file system (NTFS) support up to NTFS version 3.1. + + Y or M enables the NTFS3 driver with full features enabled (read, + write, journal replaying, sparse/compressed files support). + File system type to use on mount is "ntfs3". Module name (M option) + is also "ntfs3". + + Documentation: + +config NTFS3_64BIT_CLUSTER + bool "64 bits per NTFS clusters" + depends on NTFS3_FS && 64BIT + help + Windows implementation of ntfs.sys uses 32 bits per clusters. + If activated 64 bits per clusters you will be able to use 4k cluster + for 16T+ volumes. Windows will not be able to mount such volumes. + + It is recommended to say N here. + +config NTFS3_LZX_XPRESS + bool "activate support of external compressions lzx/xpress" + depends on NTFS3_FS + help + In Windows 10 one can use command "compact" to compress any files. + 4 possible variants of compression are: xpress4k, xpress8k, xpress16k and lzx. + If activated you will be able to read such files correctly. + + It is recommended to say Y here. + +config NTFS3_FS_POSIX_ACL + bool "NTFS POSIX Access Control Lists" + depends on NTFS3_FS + select FS_POSIX_ACL + help + POSIX Access Control Lists (ACLs) support additional access rights + for users and groups beyond the standard owner/group/world scheme, + and this option selects support for ACLs specifically for ntfs + filesystems. + NOTE: this is linux only feature. Windows will ignore these ACLs. + + If you don't know what Access Control Lists are, say N. diff --git a/fs/ntfs3/Makefile b/fs/ntfs3/Makefile new file mode 100644 index 000000000000..b06a06cc006f --- /dev/null +++ b/fs/ntfs3/Makefile @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the ntfs3 filesystem support. +# + +# to check robot warnings +ccflags-y += -Wunused-but-set-variable -Wold-style-declaration -Wint-to-pointer-cast + +obj-$(CONFIG_NTFS3_FS) += ntfs3.o + +ntfs3-y := attrib.o \ + attrlist.o \ + bitfunc.o \ + bitmap.o \ + dir.o \ + fsntfs.o \ + frecord.o \ + file.o \ + fslog.o \ + inode.o \ + index.o \ + lznt.o \ + namei.o \ + record.o \ + run.o \ + super.o \ + upcase.o \ + xattr.o + +ntfs3-$(CONFIG_NTFS3_LZX_XPRESS) += $(addprefix lib/,\ + decompress_common.o \ + lzx_decompress.o \ + xpress_decompress.o \ + ) \ No newline at end of file diff --git a/fs/ntfs3/attrib.c b/fs/ntfs3/attrib.c new file mode 100644 index 000000000000..bca85e7b6eaf --- /dev/null +++ b/fs/ntfs3/attrib.c @@ -0,0 +1,2082 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + * TODO: merge attr_set_size/attr_data_get_block/attr_allocate_frame? + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +/* + * You can set external NTFS_MIN_LOG2_OF_CLUMP/NTFS_MAX_LOG2_OF_CLUMP to manage + * preallocate algorithm + */ +#ifndef NTFS_MIN_LOG2_OF_CLUMP +#define NTFS_MIN_LOG2_OF_CLUMP 16 +#endif + +#ifndef NTFS_MAX_LOG2_OF_CLUMP +#define NTFS_MAX_LOG2_OF_CLUMP 26 +#endif + +// 16M +#define NTFS_CLUMP_MIN (1 << (NTFS_MIN_LOG2_OF_CLUMP + 8)) +// 16G +#define NTFS_CLUMP_MAX (1ull << (NTFS_MAX_LOG2_OF_CLUMP + 8)) + +/* + * get_pre_allocated + * + */ +static inline u64 get_pre_allocated(u64 size) +{ + u32 clump; + u8 align_shift; + u64 ret; + + if (size <= NTFS_CLUMP_MIN) { + clump = 1 << NTFS_MIN_LOG2_OF_CLUMP; + align_shift = NTFS_MIN_LOG2_OF_CLUMP; + } else if (size >= NTFS_CLUMP_MAX) { + clump = 1 << NTFS_MAX_LOG2_OF_CLUMP; + align_shift = NTFS_MAX_LOG2_OF_CLUMP; + } else { + align_shift = NTFS_MIN_LOG2_OF_CLUMP - 1 + + __ffs(size >> (8 + NTFS_MIN_LOG2_OF_CLUMP)); + clump = 1u << align_shift; + } + + ret = (((size + clump - 1) >> align_shift)) << align_shift; + + return ret; +} + +/* + * attr_must_be_resident + * + * returns true if attribute must be resident + */ +static inline bool attr_must_be_resident(struct ntfs_sb_info *sbi, + enum ATTR_TYPE type) +{ + const struct ATTR_DEF_ENTRY *de; + + switch (type) { + case ATTR_STD: + case ATTR_NAME: + case ATTR_ID: + case ATTR_LABEL: + case ATTR_VOL_INFO: + case ATTR_ROOT: + case ATTR_EA_INFO: + return true; + default: + de = ntfs_query_def(sbi, type); + if (de && (de->flags & NTFS_ATTR_MUST_BE_RESIDENT)) + return true; + return false; + } +} + +/* + * attr_load_runs + * + * load all runs stored in 'attr' + */ +int attr_load_runs(struct ATTRIB *attr, struct ntfs_inode *ni, + struct runs_tree *run, const CLST *vcn) +{ + int err; + CLST svcn = le64_to_cpu(attr->nres.svcn); + CLST evcn = le64_to_cpu(attr->nres.evcn); + u32 asize; + u16 run_off; + + if (svcn >= evcn + 1 || run_is_mapped_full(run, svcn, evcn)) + return 0; + + if (vcn && (evcn < *vcn || *vcn < svcn)) + return -EINVAL; + + asize = le32_to_cpu(attr->size); + run_off = le16_to_cpu(attr->nres.run_off); + err = run_unpack_ex(run, ni->mi.sbi, ni->mi.rno, svcn, evcn, + vcn ? *vcn : svcn, Add2Ptr(attr, run_off), + asize - run_off); + if (err < 0) + return err; + + return 0; +} + +/* + * int run_deallocate_ex + * + * Deallocate clusters + */ +static int run_deallocate_ex(struct ntfs_sb_info *sbi, struct runs_tree *run, + CLST vcn, CLST len, CLST *done, bool trim) +{ + int err = 0; + CLST vcn_next, vcn0 = vcn, lcn, clen, dn = 0; + size_t idx; + + if (!len) + goto out; + + if (!run_lookup_entry(run, vcn, &lcn, &clen, &idx)) { +failed: + run_truncate(run, vcn0); + err = -EINVAL; + goto out; + } + + for (;;) { + if (clen > len) + clen = len; + + if (!clen) { + err = -EINVAL; + goto out; + } + + if (lcn != SPARSE_LCN) { + mark_as_free_ex(sbi, lcn, clen, trim); + dn += clen; + } + + len -= clen; + if (!len) + break; + + vcn_next = vcn + clen; + if (!run_get_entry(run, ++idx, &vcn, &lcn, &clen) || + vcn != vcn_next) { + // save memory - don't load entire run + goto failed; + } + } + +out: + if (done) + *done += dn; + + return err; +} + +/* + * attr_allocate_clusters + * + * find free space, mark it as used and store in 'run' + */ +int attr_allocate_clusters(struct ntfs_sb_info *sbi, struct runs_tree *run, + CLST vcn, CLST lcn, CLST len, CLST *pre_alloc, + enum ALLOCATE_OPT opt, CLST *alen, const size_t fr, + CLST *new_lcn) +{ + int err; + CLST flen, vcn0 = vcn, pre = pre_alloc ? *pre_alloc : 0; + struct wnd_bitmap *wnd = &sbi->used.bitmap; + size_t cnt = run->count; + + for (;;) { + err = ntfs_look_for_free_space(sbi, lcn, len + pre, &lcn, &flen, + opt); + + if (err == -ENOSPC && pre) { + pre = 0; + if (*pre_alloc) + *pre_alloc = 0; + continue; + } + + if (err) + goto out; + + if (new_lcn && vcn == vcn0) + *new_lcn = lcn; + + /* Add new fragment into run storage */ + if (!run_add_entry(run, vcn, lcn, flen, opt == ALLOCATE_MFT)) { + down_write_nested(&wnd->rw_lock, BITMAP_MUTEX_CLUSTERS); + wnd_set_free(wnd, lcn, flen); + up_write(&wnd->rw_lock); + err = -ENOMEM; + goto out; + } + + vcn += flen; + + if (flen >= len || opt == ALLOCATE_MFT || + (fr && run->count - cnt >= fr)) { + *alen = vcn - vcn0; + return 0; + } + + len -= flen; + } + +out: + /* undo */ + run_deallocate_ex(sbi, run, vcn0, vcn - vcn0, NULL, false); + run_truncate(run, vcn0); + + return err; +} + +/* + * if page is not NULL - it is already contains resident data + * and locked (called from ni_write_frame) + */ +int attr_make_nonresident(struct ntfs_inode *ni, struct ATTRIB *attr, + struct ATTR_LIST_ENTRY *le, struct mft_inode *mi, + u64 new_size, struct runs_tree *run, + struct ATTRIB **ins_attr, struct page *page) +{ + struct ntfs_sb_info *sbi; + struct ATTRIB *attr_s; + struct MFT_REC *rec; + u32 used, asize, rsize, aoff, align; + bool is_data; + CLST len, alen; + char *next; + int err; + + if (attr->non_res) { + *ins_attr = attr; + return 0; + } + + sbi = mi->sbi; + rec = mi->mrec; + attr_s = NULL; + used = le32_to_cpu(rec->used); + asize = le32_to_cpu(attr->size); + next = Add2Ptr(attr, asize); + aoff = PtrOffset(rec, attr); + rsize = le32_to_cpu(attr->res.data_size); + is_data = attr->type == ATTR_DATA && !attr->name_len; + + align = sbi->cluster_size; + if (is_attr_compressed(attr)) + align <<= COMPRESSION_UNIT; + len = (rsize + align - 1) >> sbi->cluster_bits; + + run_init(run); + + /* make a copy of original attribute */ + attr_s = ntfs_memdup(attr, asize); + if (!attr_s) { + err = -ENOMEM; + goto out; + } + + if (!len) { + /* empty resident -> empty nonresident */ + alen = 0; + } else { + const char *data = resident_data(attr); + + err = attr_allocate_clusters(sbi, run, 0, 0, len, NULL, + ALLOCATE_DEF, &alen, 0, NULL); + if (err) + goto out1; + + if (!rsize) { + /* empty resident -> non empty nonresident */ + } else if (!is_data) { + err = ntfs_sb_write_run(sbi, run, 0, data, rsize); + if (err) + goto out2; + } else if (!page) { + char *kaddr; + + page = grab_cache_page(ni->vfs_inode.i_mapping, 0); + if (!page) { + err = -ENOMEM; + goto out2; + } + kaddr = kmap_atomic(page); + memcpy(kaddr, data, rsize); + memset(kaddr + rsize, 0, PAGE_SIZE - rsize); + kunmap_atomic(kaddr); + flush_dcache_page(page); + SetPageUptodate(page); + set_page_dirty(page); + unlock_page(page); + put_page(page); + } + } + + /* remove original attribute */ + used -= asize; + memmove(attr, Add2Ptr(attr, asize), used - aoff); + rec->used = cpu_to_le32(used); + mi->dirty = true; + if (le) + al_remove_le(ni, le); + + err = ni_insert_nonresident(ni, attr_s->type, attr_name(attr_s), + attr_s->name_len, run, 0, alen, + attr_s->flags, &attr, NULL); + if (err) + goto out3; + + ntfs_free(attr_s); + attr->nres.data_size = cpu_to_le64(rsize); + attr->nres.valid_size = attr->nres.data_size; + + *ins_attr = attr; + + if (is_data) + ni->ni_flags &= ~NI_FLAG_RESIDENT; + + /* Resident attribute becomes non resident */ + return 0; + +out3: + attr = Add2Ptr(rec, aoff); + memmove(next, attr, used - aoff); + memcpy(attr, attr_s, asize); + rec->used = cpu_to_le32(used + asize); + mi->dirty = true; +out2: + /* undo: do not trim new allocated clusters */ + run_deallocate(sbi, run, false); + run_close(run); +out1: + ntfs_free(attr_s); + /*reinsert le*/ +out: + return err; +} + +/* + * attr_set_size_res + * + * helper for attr_set_size + */ +static int attr_set_size_res(struct ntfs_inode *ni, struct ATTRIB *attr, + struct ATTR_LIST_ENTRY *le, struct mft_inode *mi, + u64 new_size, struct runs_tree *run, + struct ATTRIB **ins_attr) +{ + struct ntfs_sb_info *sbi = mi->sbi; + struct MFT_REC *rec = mi->mrec; + u32 used = le32_to_cpu(rec->used); + u32 asize = le32_to_cpu(attr->size); + u32 aoff = PtrOffset(rec, attr); + u32 rsize = le32_to_cpu(attr->res.data_size); + u32 tail = used - aoff - asize; + char *next = Add2Ptr(attr, asize); + s64 dsize = QuadAlign(new_size) - QuadAlign(rsize); + + if (dsize < 0) { + memmove(next + dsize, next, tail); + } else if (dsize > 0) { + if (used + dsize > sbi->max_bytes_per_attr) + return attr_make_nonresident(ni, attr, le, mi, new_size, + run, ins_attr, NULL); + + memmove(next + dsize, next, tail); + memset(next, 0, dsize); + } + + if (new_size > rsize) + memset(Add2Ptr(resident_data(attr), rsize), 0, + new_size - rsize); + + rec->used = cpu_to_le32(used + dsize); + attr->size = cpu_to_le32(asize + dsize); + attr->res.data_size = cpu_to_le32(new_size); + mi->dirty = true; + *ins_attr = attr; + + return 0; +} + +/* + * attr_set_size + * + * change the size of attribute + * Extend: + * - sparse/compressed: no allocated clusters + * - normal: append allocated and preallocated new clusters + * Shrink: + * - no deallocate if keep_prealloc is set + */ +int attr_set_size(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, struct runs_tree *run, + u64 new_size, const u64 *new_valid, bool keep_prealloc, + struct ATTRIB **ret) +{ + int err = 0; + struct ntfs_sb_info *sbi = ni->mi.sbi; + u8 cluster_bits = sbi->cluster_bits; + bool is_mft = + ni->mi.rno == MFT_REC_MFT && type == ATTR_DATA && !name_len; + u64 old_valid, old_size, old_alloc, new_alloc, new_alloc_tmp; + struct ATTRIB *attr = NULL, *attr_b; + struct ATTR_LIST_ENTRY *le, *le_b; + struct mft_inode *mi, *mi_b; + CLST alen, vcn, lcn, new_alen, old_alen, svcn, evcn; + CLST next_svcn, pre_alloc = -1, done = 0; + bool is_ext; + u32 align; + struct MFT_REC *rec; + +again: + le_b = NULL; + attr_b = ni_find_attr(ni, NULL, &le_b, type, name, name_len, NULL, + &mi_b); + if (!attr_b) { + err = -ENOENT; + goto out; + } + + if (!attr_b->non_res) { + err = attr_set_size_res(ni, attr_b, le_b, mi_b, new_size, run, + &attr_b); + if (err || !attr_b->non_res) + goto out; + + /* layout of records may be changed, so do a full search */ + goto again; + } + + is_ext = is_attr_ext(attr_b); + +again_1: + align = sbi->cluster_size; + + if (is_ext) { + align <<= attr_b->nres.c_unit; + if (is_attr_sparsed(attr_b)) + keep_prealloc = false; + } + + old_valid = le64_to_cpu(attr_b->nres.valid_size); + old_size = le64_to_cpu(attr_b->nres.data_size); + old_alloc = le64_to_cpu(attr_b->nres.alloc_size); + old_alen = old_alloc >> cluster_bits; + + new_alloc = (new_size + align - 1) & ~(u64)(align - 1); + new_alen = new_alloc >> cluster_bits; + + if (keep_prealloc && is_ext) + keep_prealloc = false; + + if (keep_prealloc && new_size < old_size) { + attr_b->nres.data_size = cpu_to_le64(new_size); + mi_b->dirty = true; + goto ok; + } + + vcn = old_alen - 1; + + svcn = le64_to_cpu(attr_b->nres.svcn); + evcn = le64_to_cpu(attr_b->nres.evcn); + + if (svcn <= vcn && vcn <= evcn) { + attr = attr_b; + le = le_b; + mi = mi_b; + } else if (!le_b) { + err = -EINVAL; + goto out; + } else { + le = le_b; + attr = ni_find_attr(ni, attr_b, &le, type, name, name_len, &vcn, + &mi); + if (!attr) { + err = -EINVAL; + goto out; + } + +next_le_1: + svcn = le64_to_cpu(attr->nres.svcn); + evcn = le64_to_cpu(attr->nres.evcn); + } + +next_le: + rec = mi->mrec; + + err = attr_load_runs(attr, ni, run, NULL); + if (err) + goto out; + + if (new_size > old_size) { + CLST to_allocate; + size_t free; + + if (new_alloc <= old_alloc) { + attr_b->nres.data_size = cpu_to_le64(new_size); + mi_b->dirty = true; + goto ok; + } + + to_allocate = new_alen - old_alen; +add_alloc_in_same_attr_seg: + lcn = 0; + if (is_mft) { + /* mft allocates clusters from mftzone */ + pre_alloc = 0; + } else if (is_ext) { + /* no preallocate for sparse/compress */ + pre_alloc = 0; + } else if (pre_alloc == -1) { + pre_alloc = 0; + if (type == ATTR_DATA && !name_len && + sbi->options.prealloc) { + CLST new_alen2 = bytes_to_cluster( + sbi, get_pre_allocated(new_size)); + pre_alloc = new_alen2 - new_alen; + } + + /* Get the last lcn to allocate from */ + if (old_alen && + !run_lookup_entry(run, vcn, &lcn, NULL, NULL)) { + lcn = SPARSE_LCN; + } + + if (lcn == SPARSE_LCN) + lcn = 0; + else if (lcn) + lcn += 1; + + free = wnd_zeroes(&sbi->used.bitmap); + if (to_allocate > free) { + err = -ENOSPC; + goto out; + } + + if (pre_alloc && to_allocate + pre_alloc > free) + pre_alloc = 0; + } + + vcn = old_alen; + + if (is_ext) { + if (!run_add_entry(run, vcn, SPARSE_LCN, to_allocate, + false)) { + err = -ENOMEM; + goto out; + } + alen = to_allocate; + } else { + /* ~3 bytes per fragment */ + err = attr_allocate_clusters( + sbi, run, vcn, lcn, to_allocate, &pre_alloc, + is_mft ? ALLOCATE_MFT : 0, &alen, + is_mft ? 0 + : (sbi->record_size - + le32_to_cpu(rec->used) + 8) / + 3 + + 1, + NULL); + if (err) + goto out; + } + + done += alen; + vcn += alen; + if (to_allocate > alen) + to_allocate -= alen; + else + to_allocate = 0; + +pack_runs: + err = mi_pack_runs(mi, attr, run, vcn - svcn); + if (err) + goto out; + + next_svcn = le64_to_cpu(attr->nres.evcn) + 1; + new_alloc_tmp = (u64)next_svcn << cluster_bits; + attr_b->nres.alloc_size = cpu_to_le64(new_alloc_tmp); + mi_b->dirty = true; + + if (next_svcn >= vcn && !to_allocate) { + /* Normal way. update attribute and exit */ + attr_b->nres.data_size = cpu_to_le64(new_size); + goto ok; + } + + /* at least two mft to avoid recursive loop*/ + if (is_mft && next_svcn == vcn && + ((u64)done << sbi->cluster_bits) >= 2 * sbi->record_size) { + new_size = new_alloc_tmp; + attr_b->nres.data_size = attr_b->nres.alloc_size; + goto ok; + } + + if (le32_to_cpu(rec->used) < sbi->record_size) { + old_alen = next_svcn; + evcn = old_alen - 1; + goto add_alloc_in_same_attr_seg; + } + + attr_b->nres.data_size = attr_b->nres.alloc_size; + if (new_alloc_tmp < old_valid) + attr_b->nres.valid_size = attr_b->nres.data_size; + + if (type == ATTR_LIST) { + err = ni_expand_list(ni); + if (err) + goto out; + if (next_svcn < vcn) + goto pack_runs; + + /* layout of records is changed */ + goto again; + } + + if (!ni->attr_list.size) { + err = ni_create_attr_list(ni); + if (err) + goto out; + /* layout of records is changed */ + } + + if (next_svcn >= vcn) { + /* this is mft data, repeat */ + goto again; + } + + /* insert new attribute segment */ + err = ni_insert_nonresident(ni, type, name, name_len, run, + next_svcn, vcn - next_svcn, + attr_b->flags, &attr, &mi); + if (err) + goto out; + + if (!is_mft) + run_truncate_head(run, evcn + 1); + + svcn = le64_to_cpu(attr->nres.svcn); + evcn = le64_to_cpu(attr->nres.evcn); + + le_b = NULL; + /* layout of records maybe changed */ + /* find base attribute to update*/ + attr_b = ni_find_attr(ni, NULL, &le_b, type, name, name_len, + NULL, &mi_b); + if (!attr_b) { + err = -ENOENT; + goto out; + } + + attr_b->nres.alloc_size = cpu_to_le64((u64)vcn << cluster_bits); + attr_b->nres.data_size = attr_b->nres.alloc_size; + attr_b->nres.valid_size = attr_b->nres.alloc_size; + mi_b->dirty = true; + goto again_1; + } + + if (new_size != old_size || + (new_alloc != old_alloc && !keep_prealloc)) { + vcn = max(svcn, new_alen); + new_alloc_tmp = (u64)vcn << cluster_bits; + + alen = 0; + err = run_deallocate_ex(sbi, run, vcn, evcn - vcn + 1, &alen, + true); + if (err) + goto out; + + run_truncate(run, vcn); + + if (vcn > svcn) { + err = mi_pack_runs(mi, attr, run, vcn - svcn); + if (err) + goto out; + } else if (le && le->vcn) { + u16 le_sz = le16_to_cpu(le->size); + + /* + * NOTE: list entries for one attribute are always + * the same size. We deal with last entry (vcn==0) + * and it is not first in entries array + * (list entry for std attribute always first) + * So it is safe to step back + */ + mi_remove_attr(mi, attr); + + if (!al_remove_le(ni, le)) { + err = -EINVAL; + goto out; + } + + le = (struct ATTR_LIST_ENTRY *)((u8 *)le - le_sz); + } else { + attr->nres.evcn = cpu_to_le64((u64)vcn - 1); + mi->dirty = true; + } + + attr_b->nres.alloc_size = cpu_to_le64(new_alloc_tmp); + + if (vcn == new_alen) { + attr_b->nres.data_size = cpu_to_le64(new_size); + if (new_size < old_valid) + attr_b->nres.valid_size = + attr_b->nres.data_size; + } else { + if (new_alloc_tmp <= + le64_to_cpu(attr_b->nres.data_size)) + attr_b->nres.data_size = + attr_b->nres.alloc_size; + if (new_alloc_tmp < + le64_to_cpu(attr_b->nres.valid_size)) + attr_b->nres.valid_size = + attr_b->nres.alloc_size; + } + + if (is_ext) + le64_sub_cpu(&attr_b->nres.total_size, + ((u64)alen << cluster_bits)); + + mi_b->dirty = true; + + if (new_alloc_tmp <= new_alloc) + goto ok; + + old_size = new_alloc_tmp; + vcn = svcn - 1; + + if (le == le_b) { + attr = attr_b; + mi = mi_b; + evcn = svcn - 1; + svcn = 0; + goto next_le; + } + + if (le->type != type || le->name_len != name_len || + memcmp(le_name(le), name, name_len * sizeof(short))) { + err = -EINVAL; + goto out; + } + + err = ni_load_mi(ni, le, &mi); + if (err) + goto out; + + attr = mi_find_attr(mi, NULL, type, name, name_len, &le->id); + if (!attr) { + err = -EINVAL; + goto out; + } + goto next_le_1; + } + +ok: + if (new_valid) { + __le64 valid = cpu_to_le64(min(*new_valid, new_size)); + + if (attr_b->nres.valid_size != valid) { + attr_b->nres.valid_size = valid; + mi_b->dirty = true; + } + } + +out: + if (!err && attr_b && ret) + *ret = attr_b; + + /* update inode_set_bytes*/ + if (!err && ((type == ATTR_DATA && !name_len) || + (type == ATTR_ALLOC && name == I30_NAME))) { + bool dirty = false; + + if (ni->vfs_inode.i_size != new_size) { + ni->vfs_inode.i_size = new_size; + dirty = true; + } + + if (attr_b && attr_b->non_res) { + new_alloc = le64_to_cpu(attr_b->nres.alloc_size); + if (inode_get_bytes(&ni->vfs_inode) != new_alloc) { + inode_set_bytes(&ni->vfs_inode, new_alloc); + dirty = true; + } + } + + if (dirty) { + ni->ni_flags |= NI_FLAG_UPDATE_PARENT; + mark_inode_dirty(&ni->vfs_inode); + } + } + + return err; +} + +int attr_data_get_block(struct ntfs_inode *ni, CLST vcn, CLST clen, CLST *lcn, + CLST *len, bool *new) +{ + int err = 0; + struct runs_tree *run = &ni->file.run; + struct ntfs_sb_info *sbi; + u8 cluster_bits; + struct ATTRIB *attr = NULL, *attr_b; + struct ATTR_LIST_ENTRY *le, *le_b; + struct mft_inode *mi, *mi_b; + CLST hint, svcn, to_alloc, evcn1, next_svcn, asize, end; + u64 total_size; + u32 clst_per_frame; + bool ok; + + if (new) + *new = false; + + down_read(&ni->file.run_lock); + ok = run_lookup_entry(run, vcn, lcn, len, NULL); + up_read(&ni->file.run_lock); + + if (ok && (*lcn != SPARSE_LCN || !new)) { + /* normal way */ + return 0; + } + + if (!clen) + clen = 1; + + if (ok && clen > *len) + clen = *len; + + sbi = ni->mi.sbi; + cluster_bits = sbi->cluster_bits; + + ni_lock(ni); + down_write(&ni->file.run_lock); + + le_b = NULL; + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, NULL, 0, NULL, &mi_b); + if (!attr_b) { + err = -ENOENT; + goto out; + } + + if (!attr_b->non_res) { + *lcn = RESIDENT_LCN; + *len = 1; + goto out; + } + + asize = le64_to_cpu(attr_b->nres.alloc_size) >> sbi->cluster_bits; + if (vcn >= asize) { + err = -EINVAL; + goto out; + } + + clst_per_frame = 1u << attr_b->nres.c_unit; + to_alloc = (clen + clst_per_frame - 1) & ~(clst_per_frame - 1); + + if (vcn + to_alloc > asize) + to_alloc = asize - vcn; + + svcn = le64_to_cpu(attr_b->nres.svcn); + evcn1 = le64_to_cpu(attr_b->nres.evcn) + 1; + + attr = attr_b; + le = le_b; + mi = mi_b; + + if (le_b && (vcn < svcn || evcn1 <= vcn)) { + attr = ni_find_attr(ni, attr_b, &le, ATTR_DATA, NULL, 0, &vcn, + &mi); + if (!attr) { + err = -EINVAL; + goto out; + } + svcn = le64_to_cpu(attr->nres.svcn); + evcn1 = le64_to_cpu(attr->nres.evcn) + 1; + } + + err = attr_load_runs(attr, ni, run, NULL); + if (err) + goto out; + + if (!ok) { + ok = run_lookup_entry(run, vcn, lcn, len, NULL); + if (ok && (*lcn != SPARSE_LCN || !new)) { + /* normal way */ + err = 0; + goto ok; + } + + if (!ok && !new) { + *len = 0; + err = 0; + goto ok; + } + + if (ok && clen > *len) { + clen = *len; + to_alloc = (clen + clst_per_frame - 1) & + ~(clst_per_frame - 1); + } + } + + if (!is_attr_ext(attr_b)) { + err = -EINVAL; + goto out; + } + + /* Get the last lcn to allocate from */ + hint = 0; + + if (vcn > evcn1) { + if (!run_add_entry(run, evcn1, SPARSE_LCN, vcn - evcn1, + false)) { + err = -ENOMEM; + goto out; + } + } else if (vcn && !run_lookup_entry(run, vcn - 1, &hint, NULL, NULL)) { + hint = -1; + } + + err = attr_allocate_clusters( + sbi, run, vcn, hint + 1, to_alloc, NULL, 0, len, + (sbi->record_size - le32_to_cpu(mi->mrec->used) + 8) / 3 + 1, + lcn); + if (err) + goto out; + *new = true; + + end = vcn + *len; + + total_size = le64_to_cpu(attr_b->nres.total_size) + + ((u64)*len << cluster_bits); + +repack: + err = mi_pack_runs(mi, attr, run, max(end, evcn1) - svcn); + if (err) + goto out; + + attr_b->nres.total_size = cpu_to_le64(total_size); + inode_set_bytes(&ni->vfs_inode, total_size); + ni->ni_flags |= NI_FLAG_UPDATE_PARENT; + + mi_b->dirty = true; + mark_inode_dirty(&ni->vfs_inode); + + /* stored [vcn : next_svcn) from [vcn : end) */ + next_svcn = le64_to_cpu(attr->nres.evcn) + 1; + + if (end <= evcn1) { + if (next_svcn == evcn1) { + /* Normal way. update attribute and exit */ + goto ok; + } + /* add new segment [next_svcn : evcn1 - next_svcn )*/ + if (!ni->attr_list.size) { + err = ni_create_attr_list(ni); + if (err) + goto out; + /* layout of records is changed */ + le_b = NULL; + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, NULL, + 0, NULL, &mi_b); + if (!attr_b) { + err = -ENOENT; + goto out; + } + + attr = attr_b; + le = le_b; + mi = mi_b; + goto repack; + } + } + + svcn = evcn1; + + /* Estimate next attribute */ + attr = ni_find_attr(ni, attr, &le, ATTR_DATA, NULL, 0, &svcn, &mi); + + if (attr) { + CLST alloc = bytes_to_cluster( + sbi, le64_to_cpu(attr_b->nres.alloc_size)); + CLST evcn = le64_to_cpu(attr->nres.evcn); + + if (end < next_svcn) + end = next_svcn; + while (end > evcn) { + /* remove segment [svcn : evcn)*/ + mi_remove_attr(mi, attr); + + if (!al_remove_le(ni, le)) { + err = -EINVAL; + goto out; + } + + if (evcn + 1 >= alloc) { + /* last attribute segment */ + evcn1 = evcn + 1; + goto ins_ext; + } + + if (ni_load_mi(ni, le, &mi)) { + attr = NULL; + goto out; + } + + attr = mi_find_attr(mi, NULL, ATTR_DATA, NULL, 0, + &le->id); + if (!attr) { + err = -EINVAL; + goto out; + } + svcn = le64_to_cpu(attr->nres.svcn); + evcn = le64_to_cpu(attr->nres.evcn); + } + + if (end < svcn) + end = svcn; + + err = attr_load_runs(attr, ni, run, &end); + if (err) + goto out; + + evcn1 = evcn + 1; + attr->nres.svcn = cpu_to_le64(next_svcn); + err = mi_pack_runs(mi, attr, run, evcn1 - next_svcn); + if (err) + goto out; + + le->vcn = cpu_to_le64(next_svcn); + ni->attr_list.dirty = true; + mi->dirty = true; + + next_svcn = le64_to_cpu(attr->nres.evcn) + 1; + } +ins_ext: + if (evcn1 > next_svcn) { + err = ni_insert_nonresident(ni, ATTR_DATA, NULL, 0, run, + next_svcn, evcn1 - next_svcn, + attr_b->flags, &attr, &mi); + if (err) + goto out; + } +ok: + run_truncate_around(run, vcn); +out: + up_write(&ni->file.run_lock); + ni_unlock(ni); + + return err; +} + +int attr_data_read_resident(struct ntfs_inode *ni, struct page *page) +{ + u64 vbo; + struct ATTRIB *attr; + u32 data_size; + + attr = ni_find_attr(ni, NULL, NULL, ATTR_DATA, NULL, 0, NULL, NULL); + if (!attr) + return -EINVAL; + + if (attr->non_res) + return E_NTFS_NONRESIDENT; + + vbo = page->index << PAGE_SHIFT; + data_size = le32_to_cpu(attr->res.data_size); + if (vbo < data_size) { + const char *data = resident_data(attr); + char *kaddr = kmap_atomic(page); + u32 use = data_size - vbo; + + if (use > PAGE_SIZE) + use = PAGE_SIZE; + + memcpy(kaddr, data + vbo, use); + memset(kaddr + use, 0, PAGE_SIZE - use); + kunmap_atomic(kaddr); + flush_dcache_page(page); + SetPageUptodate(page); + } else if (!PageUptodate(page)) { + zero_user_segment(page, 0, PAGE_SIZE); + SetPageUptodate(page); + } + + return 0; +} + +int attr_data_write_resident(struct ntfs_inode *ni, struct page *page) +{ + u64 vbo; + struct mft_inode *mi; + struct ATTRIB *attr; + u32 data_size; + + attr = ni_find_attr(ni, NULL, NULL, ATTR_DATA, NULL, 0, NULL, &mi); + if (!attr) + return -EINVAL; + + if (attr->non_res) { + /*return special error code to check this case*/ + return E_NTFS_NONRESIDENT; + } + + vbo = page->index << PAGE_SHIFT; + data_size = le32_to_cpu(attr->res.data_size); + if (vbo < data_size) { + char *data = resident_data(attr); + char *kaddr = kmap_atomic(page); + u32 use = data_size - vbo; + + if (use > PAGE_SIZE) + use = PAGE_SIZE; + memcpy(data + vbo, kaddr, use); + kunmap_atomic(kaddr); + mi->dirty = true; + } + ni->i_valid = data_size; + + return 0; +} + +/* + * attr_load_runs_vcn + * + * load runs with vcn + */ +int attr_load_runs_vcn(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, struct runs_tree *run, + CLST vcn) +{ + struct ATTRIB *attr; + int err; + CLST svcn, evcn; + u16 ro; + + attr = ni_find_attr(ni, NULL, NULL, type, name, name_len, &vcn, NULL); + if (!attr) + return -ENOENT; + + svcn = le64_to_cpu(attr->nres.svcn); + evcn = le64_to_cpu(attr->nres.evcn); + + if (evcn < vcn || vcn < svcn) + return -EINVAL; + + ro = le16_to_cpu(attr->nres.run_off); + err = run_unpack_ex(run, ni->mi.sbi, ni->mi.rno, svcn, evcn, svcn, + Add2Ptr(attr, ro), le32_to_cpu(attr->size) - ro); + if (err < 0) + return err; + return 0; +} + +/* + * load runs for given range [from to) + */ +int attr_load_runs_range(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, struct runs_tree *run, + u64 from, u64 to) +{ + struct ntfs_sb_info *sbi = ni->mi.sbi; + u8 cluster_bits = sbi->cluster_bits; + CLST vcn = from >> cluster_bits; + CLST vcn_last = (to - 1) >> cluster_bits; + CLST lcn, clen; + int err; + + for (vcn = from >> cluster_bits; vcn <= vcn_last; vcn += clen) { + if (!run_lookup_entry(run, vcn, &lcn, &clen, NULL)) { + err = attr_load_runs_vcn(ni, type, name, name_len, run, + vcn); + if (err) + return err; + clen = 0; /*next run_lookup_entry(vcn) must be success*/ + } + } + + return 0; +} + +#ifdef CONFIG_NTFS3_LZX_XPRESS +/* + * attr_wof_frame_info + * + * read header of xpress/lzx file to get info about frame + */ +int attr_wof_frame_info(struct ntfs_inode *ni, struct ATTRIB *attr, + struct runs_tree *run, u64 frame, u64 frames, + u8 frame_bits, u32 *ondisk_size, u64 *vbo_data) +{ + struct ntfs_sb_info *sbi = ni->mi.sbi; + u64 vbo[2], off[2], wof_size; + u32 voff; + u8 bytes_per_off; + char *addr; + struct page *page; + int i, err; + __le32 *off32; + __le64 *off64; + + if (ni->vfs_inode.i_size < 0x100000000ull) { + /* file starts with array of 32 bit offsets */ + bytes_per_off = sizeof(__le32); + vbo[1] = frame << 2; + *vbo_data = frames << 2; + } else { + /* file starts with array of 64 bit offsets */ + bytes_per_off = sizeof(__le64); + vbo[1] = frame << 3; + *vbo_data = frames << 3; + } + + /* + * read 4/8 bytes at [vbo - 4(8)] == offset where compressed frame starts + * read 4/8 bytes at [vbo] == offset where compressed frame ends + */ + if (!attr->non_res) { + if (vbo[1] + bytes_per_off > le32_to_cpu(attr->res.data_size)) { + ntfs_inode_err(&ni->vfs_inode, "is corrupted"); + return -EINVAL; + } + addr = resident_data(attr); + + if (bytes_per_off == sizeof(__le32)) { + off32 = Add2Ptr(addr, vbo[1]); + off[0] = vbo[1] ? le32_to_cpu(off32[-1]) : 0; + off[1] = le32_to_cpu(off32[0]); + } else { + off64 = Add2Ptr(addr, vbo[1]); + off[0] = vbo[1] ? le64_to_cpu(off64[-1]) : 0; + off[1] = le64_to_cpu(off64[0]); + } + + *vbo_data += off[0]; + *ondisk_size = off[1] - off[0]; + return 0; + } + + wof_size = le64_to_cpu(attr->nres.data_size); + down_write(&ni->file.run_lock); + page = ni->file.offs_page; + if (!page) { + page = alloc_page(GFP_KERNEL); + if (!page) { + err = -ENOMEM; + goto out; + } + page->index = -1; + ni->file.offs_page = page; + } + lock_page(page); + addr = page_address(page); + + if (vbo[1]) { + voff = vbo[1] & (PAGE_SIZE - 1); + vbo[0] = vbo[1] - bytes_per_off; + i = 0; + } else { + voff = 0; + vbo[0] = 0; + off[0] = 0; + i = 1; + } + + do { + pgoff_t index = vbo[i] >> PAGE_SHIFT; + + if (index != page->index) { + u64 from = vbo[i] & ~(u64)(PAGE_SIZE - 1); + u64 to = min(from + PAGE_SIZE, wof_size); + + err = attr_load_runs_range(ni, ATTR_DATA, WOF_NAME, + ARRAY_SIZE(WOF_NAME), run, + from, to); + if (err) + goto out1; + + err = ntfs_bio_pages(sbi, run, &page, 1, from, + to - from, REQ_OP_READ); + if (err) { + page->index = -1; + goto out1; + } + page->index = index; + } + + if (i) { + if (bytes_per_off == sizeof(__le32)) { + off32 = Add2Ptr(addr, voff); + off[1] = le32_to_cpu(*off32); + } else { + off64 = Add2Ptr(addr, voff); + off[1] = le64_to_cpu(*off64); + } + } else if (!voff) { + if (bytes_per_off == sizeof(__le32)) { + off32 = Add2Ptr(addr, PAGE_SIZE - sizeof(u32)); + off[0] = le32_to_cpu(*off32); + } else { + off64 = Add2Ptr(addr, PAGE_SIZE - sizeof(u64)); + off[0] = le64_to_cpu(*off64); + } + } else { + /* two values in one page*/ + if (bytes_per_off == sizeof(__le32)) { + off32 = Add2Ptr(addr, voff); + off[0] = le32_to_cpu(off32[-1]); + off[1] = le32_to_cpu(off32[0]); + } else { + off64 = Add2Ptr(addr, voff); + off[0] = le64_to_cpu(off64[-1]); + off[1] = le64_to_cpu(off64[0]); + } + break; + } + } while (++i < 2); + + *vbo_data += off[0]; + *ondisk_size = off[1] - off[0]; + +out1: + unlock_page(page); +out: + up_write(&ni->file.run_lock); + return err; +} +#endif + +/* + * attr_is_frame_compressed + * + * This function is used to detect compressed frame + */ +int attr_is_frame_compressed(struct ntfs_inode *ni, struct ATTRIB *attr, + CLST frame, CLST *clst_data) +{ + int err; + u32 clst_frame; + CLST clen, lcn, vcn, alen, slen, vcn_next; + size_t idx; + struct runs_tree *run; + + *clst_data = 0; + + if (!is_attr_compressed(attr)) + return 0; + + if (!attr->non_res) + return 0; + + clst_frame = 1u << attr->nres.c_unit; + vcn = frame * clst_frame; + run = &ni->file.run; + + if (!run_lookup_entry(run, vcn, &lcn, &clen, &idx)) { + err = attr_load_runs_vcn(ni, attr->type, attr_name(attr), + attr->name_len, run, vcn); + if (err) + return err; + + if (!run_lookup_entry(run, vcn, &lcn, &clen, &idx)) + return -EINVAL; + } + + if (lcn == SPARSE_LCN) { + /* sparsed frame */ + return 0; + } + + if (clen >= clst_frame) { + /* + * The frame is not compressed 'cause + * it does not contain any sparse clusters + */ + *clst_data = clst_frame; + return 0; + } + + alen = bytes_to_cluster(ni->mi.sbi, le64_to_cpu(attr->nres.alloc_size)); + slen = 0; + *clst_data = clen; + + /* + * The frame is compressed if *clst_data + slen >= clst_frame + * Check next fragments + */ + while ((vcn += clen) < alen) { + vcn_next = vcn; + + if (!run_get_entry(run, ++idx, &vcn, &lcn, &clen) || + vcn_next != vcn) { + err = attr_load_runs_vcn(ni, attr->type, + attr_name(attr), + attr->name_len, run, vcn_next); + if (err) + return err; + vcn = vcn_next; + + if (!run_lookup_entry(run, vcn, &lcn, &clen, &idx)) + return -EINVAL; + } + + if (lcn == SPARSE_LCN) { + slen += clen; + } else { + if (slen) { + /* + * data_clusters + sparse_clusters = + * not enough for frame + */ + return -EINVAL; + } + *clst_data += clen; + } + + if (*clst_data + slen >= clst_frame) { + if (!slen) { + /* + * There is no sparsed clusters in this frame + * So it is not compressed + */ + *clst_data = clst_frame; + } else { + /*frame is compressed*/ + } + break; + } + } + + return 0; +} + +/* + * attr_allocate_frame + * + * allocate/free clusters for 'frame' + * assumed: down_write(&ni->file.run_lock); + */ +int attr_allocate_frame(struct ntfs_inode *ni, CLST frame, size_t compr_size, + u64 new_valid) +{ + int err = 0; + struct runs_tree *run = &ni->file.run; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct ATTRIB *attr = NULL, *attr_b; + struct ATTR_LIST_ENTRY *le, *le_b; + struct mft_inode *mi, *mi_b; + CLST svcn, evcn1, next_svcn, lcn, len; + CLST vcn, end, clst_data; + u64 total_size, valid_size, data_size; + + le_b = NULL; + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, NULL, 0, NULL, &mi_b); + if (!attr_b) + return -ENOENT; + + if (!is_attr_ext(attr_b)) + return -EINVAL; + + vcn = frame << NTFS_LZNT_CUNIT; + total_size = le64_to_cpu(attr_b->nres.total_size); + + svcn = le64_to_cpu(attr_b->nres.svcn); + evcn1 = le64_to_cpu(attr_b->nres.evcn) + 1; + data_size = le64_to_cpu(attr_b->nres.data_size); + + if (svcn <= vcn && vcn < evcn1) { + attr = attr_b; + le = le_b; + mi = mi_b; + } else if (!le_b) { + err = -EINVAL; + goto out; + } else { + le = le_b; + attr = ni_find_attr(ni, attr_b, &le, ATTR_DATA, NULL, 0, &vcn, + &mi); + if (!attr) { + err = -EINVAL; + goto out; + } + svcn = le64_to_cpu(attr->nres.svcn); + evcn1 = le64_to_cpu(attr->nres.evcn) + 1; + } + + err = attr_load_runs(attr, ni, run, NULL); + if (err) + goto out; + + err = attr_is_frame_compressed(ni, attr_b, frame, &clst_data); + if (err) + goto out; + + total_size -= (u64)clst_data << sbi->cluster_bits; + + len = bytes_to_cluster(sbi, compr_size); + + if (len == clst_data) + goto out; + + if (len < clst_data) { + err = run_deallocate_ex(sbi, run, vcn + len, clst_data - len, + NULL, true); + if (err) + goto out; + + if (!run_add_entry(run, vcn + len, SPARSE_LCN, clst_data - len, + false)) { + err = -ENOMEM; + goto out; + } + end = vcn + clst_data; + /* run contains updated range [vcn + len : end) */ + } else { + CLST alen, hint = 0; + /* Get the last lcn to allocate from */ + if (vcn + clst_data && + !run_lookup_entry(run, vcn + clst_data - 1, &hint, NULL, + NULL)) { + hint = -1; + } + + err = attr_allocate_clusters(sbi, run, vcn + clst_data, + hint + 1, len - clst_data, NULL, 0, + &alen, 0, &lcn); + if (err) + goto out; + + end = vcn + len; + /* run contains updated range [vcn + clst_data : end) */ + } + + total_size += (u64)len << sbi->cluster_bits; + +repack: + err = mi_pack_runs(mi, attr, run, max(end, evcn1) - svcn); + if (err) + goto out; + + attr_b->nres.total_size = cpu_to_le64(total_size); + inode_set_bytes(&ni->vfs_inode, total_size); + + mi_b->dirty = true; + mark_inode_dirty(&ni->vfs_inode); + + /* stored [vcn : next_svcn) from [vcn : end) */ + next_svcn = le64_to_cpu(attr->nres.evcn) + 1; + + if (end <= evcn1) { + if (next_svcn == evcn1) { + /* Normal way. update attribute and exit */ + goto ok; + } + /* add new segment [next_svcn : evcn1 - next_svcn )*/ + if (!ni->attr_list.size) { + err = ni_create_attr_list(ni); + if (err) + goto out; + /* layout of records is changed */ + le_b = NULL; + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, NULL, + 0, NULL, &mi_b); + if (!attr_b) { + err = -ENOENT; + goto out; + } + + attr = attr_b; + le = le_b; + mi = mi_b; + goto repack; + } + } + + svcn = evcn1; + + /* Estimate next attribute */ + attr = ni_find_attr(ni, attr, &le, ATTR_DATA, NULL, 0, &svcn, &mi); + + if (attr) { + CLST alloc = bytes_to_cluster( + sbi, le64_to_cpu(attr_b->nres.alloc_size)); + CLST evcn = le64_to_cpu(attr->nres.evcn); + + if (end < next_svcn) + end = next_svcn; + while (end > evcn) { + /* remove segment [svcn : evcn)*/ + mi_remove_attr(mi, attr); + + if (!al_remove_le(ni, le)) { + err = -EINVAL; + goto out; + } + + if (evcn + 1 >= alloc) { + /* last attribute segment */ + evcn1 = evcn + 1; + goto ins_ext; + } + + if (ni_load_mi(ni, le, &mi)) { + attr = NULL; + goto out; + } + + attr = mi_find_attr(mi, NULL, ATTR_DATA, NULL, 0, + &le->id); + if (!attr) { + err = -EINVAL; + goto out; + } + svcn = le64_to_cpu(attr->nres.svcn); + evcn = le64_to_cpu(attr->nres.evcn); + } + + if (end < svcn) + end = svcn; + + err = attr_load_runs(attr, ni, run, &end); + if (err) + goto out; + + evcn1 = evcn + 1; + attr->nres.svcn = cpu_to_le64(next_svcn); + err = mi_pack_runs(mi, attr, run, evcn1 - next_svcn); + if (err) + goto out; + + le->vcn = cpu_to_le64(next_svcn); + ni->attr_list.dirty = true; + mi->dirty = true; + + next_svcn = le64_to_cpu(attr->nres.evcn) + 1; + } +ins_ext: + if (evcn1 > next_svcn) { + err = ni_insert_nonresident(ni, ATTR_DATA, NULL, 0, run, + next_svcn, evcn1 - next_svcn, + attr_b->flags, &attr, &mi); + if (err) + goto out; + } +ok: + run_truncate_around(run, vcn); +out: + if (new_valid > data_size) + new_valid = data_size; + + valid_size = le64_to_cpu(attr_b->nres.valid_size); + if (new_valid != valid_size) { + attr_b->nres.valid_size = cpu_to_le64(valid_size); + mi_b->dirty = true; + } + + return err; +} + +/* Collapse range in file */ +int attr_collapse_range(struct ntfs_inode *ni, u64 vbo, u64 bytes) +{ + int err = 0; + struct runs_tree *run = &ni->file.run; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct ATTRIB *attr = NULL, *attr_b; + struct ATTR_LIST_ENTRY *le, *le_b; + struct mft_inode *mi, *mi_b; + CLST svcn, evcn1, len, dealloc, alen; + CLST vcn, end; + u64 valid_size, data_size, alloc_size, total_size; + u32 mask; + __le16 a_flags; + + if (!bytes) + return 0; + + le_b = NULL; + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, NULL, 0, NULL, &mi_b); + if (!attr_b) + return -ENOENT; + + if (!attr_b->non_res) { + /* Attribute is resident. Nothing to do? */ + return 0; + } + + data_size = le64_to_cpu(attr_b->nres.data_size); + alloc_size = le64_to_cpu(attr_b->nres.alloc_size); + a_flags = attr_b->flags; + + if (is_attr_ext(attr_b)) { + total_size = le64_to_cpu(attr_b->nres.total_size); + mask = (sbi->cluster_size << attr_b->nres.c_unit) - 1; + } else { + total_size = alloc_size; + mask = sbi->cluster_mask; + } + + if ((vbo & mask) || (bytes & mask)) { + /* allow to collapse only cluster aligned ranges */ + return -EINVAL; + } + + if (vbo > data_size) + return -EINVAL; + + down_write(&ni->file.run_lock); + + if (vbo + bytes >= data_size) { + u64 new_valid = min(ni->i_valid, vbo); + + /* Simple truncate file at 'vbo' */ + truncate_setsize(&ni->vfs_inode, vbo); + err = attr_set_size(ni, ATTR_DATA, NULL, 0, &ni->file.run, vbo, + &new_valid, true, NULL); + + if (!err && new_valid < ni->i_valid) + ni->i_valid = new_valid; + + goto out; + } + + /* + * Enumerate all attribute segments and collapse + */ + alen = alloc_size >> sbi->cluster_bits; + vcn = vbo >> sbi->cluster_bits; + len = bytes >> sbi->cluster_bits; + end = vcn + len; + dealloc = 0; + + svcn = le64_to_cpu(attr_b->nres.svcn); + evcn1 = le64_to_cpu(attr_b->nres.evcn) + 1; + + if (svcn <= vcn && vcn < evcn1) { + attr = attr_b; + le = le_b; + mi = mi_b; + } else if (!le_b) { + err = -EINVAL; + goto out; + } else { + le = le_b; + attr = ni_find_attr(ni, attr_b, &le, ATTR_DATA, NULL, 0, &vcn, + &mi); + if (!attr) { + err = -EINVAL; + goto out; + } + + svcn = le64_to_cpu(attr->nres.svcn); + evcn1 = le64_to_cpu(attr->nres.evcn) + 1; + } + + for (;;) { + if (svcn >= end) { + /* shift vcn */ + attr->nres.svcn = cpu_to_le64(svcn - len); + attr->nres.evcn = cpu_to_le64(evcn1 - 1 - len); + if (le) { + le->vcn = attr->nres.svcn; + ni->attr_list.dirty = true; + } + mi->dirty = true; + } else if (svcn < vcn || end < evcn1) { + CLST vcn1, eat, next_svcn; + + /* collapse a part of this attribute segment */ + err = attr_load_runs(attr, ni, run, &svcn); + if (err) + goto out; + vcn1 = max(vcn, svcn); + eat = min(end, evcn1) - vcn1; + + err = run_deallocate_ex(sbi, run, vcn1, eat, &dealloc, + true); + if (err) + goto out; + + if (!run_collapse_range(run, vcn1, eat)) { + err = -ENOMEM; + goto out; + } + + if (svcn >= vcn) { + /* shift vcn */ + attr->nres.svcn = cpu_to_le64(vcn); + if (le) { + le->vcn = attr->nres.svcn; + ni->attr_list.dirty = true; + } + } + + err = mi_pack_runs(mi, attr, run, evcn1 - svcn - eat); + if (err) + goto out; + + next_svcn = le64_to_cpu(attr->nres.evcn) + 1; + if (next_svcn + eat < evcn1) { + err = ni_insert_nonresident( + ni, ATTR_DATA, NULL, 0, run, next_svcn, + evcn1 - eat - next_svcn, a_flags, &attr, + &mi); + if (err) + goto out; + + /* layout of records maybe changed */ + attr_b = NULL; + le = al_find_ex(ni, NULL, ATTR_DATA, NULL, 0, + &next_svcn); + if (!le) { + err = -EINVAL; + goto out; + } + } + + /* free all allocated memory */ + run_truncate(run, 0); + } else { + u16 le_sz; + u16 roff = le16_to_cpu(attr->nres.run_off); + + /*run==1 means unpack and deallocate*/ + run_unpack_ex(RUN_DEALLOCATE, sbi, ni->mi.rno, svcn, + evcn1 - 1, svcn, Add2Ptr(attr, roff), + le32_to_cpu(attr->size) - roff); + + /* delete this attribute segment */ + mi_remove_attr(mi, attr); + if (!le) + break; + + le_sz = le16_to_cpu(le->size); + if (!al_remove_le(ni, le)) { + err = -EINVAL; + goto out; + } + + if (evcn1 >= alen) + break; + + if (!svcn) { + /* Load next record that contains this attribute */ + if (ni_load_mi(ni, le, &mi)) { + err = -EINVAL; + goto out; + } + + /* Look for required attribute */ + attr = mi_find_attr(mi, NULL, ATTR_DATA, NULL, + 0, &le->id); + if (!attr) { + err = -EINVAL; + goto out; + } + goto next_attr; + } + le = (struct ATTR_LIST_ENTRY *)((u8 *)le - le_sz); + } + + if (evcn1 >= alen) + break; + + attr = ni_enum_attr_ex(ni, attr, &le, &mi); + if (!attr) { + err = -EINVAL; + goto out; + } + +next_attr: + svcn = le64_to_cpu(attr->nres.svcn); + evcn1 = le64_to_cpu(attr->nres.evcn) + 1; + } + + if (!attr_b) { + le_b = NULL; + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, NULL, 0, NULL, + &mi_b); + if (!attr_b) { + err = -ENOENT; + goto out; + } + } + + data_size -= bytes; + valid_size = ni->i_valid; + if (vbo + bytes <= valid_size) + valid_size -= bytes; + else if (vbo < valid_size) + valid_size = vbo; + + attr_b->nres.alloc_size = cpu_to_le64(alloc_size - bytes); + attr_b->nres.data_size = cpu_to_le64(data_size); + attr_b->nres.valid_size = cpu_to_le64(min(valid_size, data_size)); + total_size -= (u64)dealloc << sbi->cluster_bits; + if (is_attr_ext(attr_b)) + attr_b->nres.total_size = cpu_to_le64(total_size); + mi_b->dirty = true; + + /*update inode size*/ + ni->i_valid = valid_size; + ni->vfs_inode.i_size = data_size; + inode_set_bytes(&ni->vfs_inode, total_size); + ni->ni_flags |= NI_FLAG_UPDATE_PARENT; + mark_inode_dirty(&ni->vfs_inode); + +out: + up_write(&ni->file.run_lock); + if (err) + make_bad_inode(&ni->vfs_inode); + + return err; +} + +/* not for normal files */ +int attr_punch_hole(struct ntfs_inode *ni, u64 vbo, u64 bytes) +{ + int err = 0; + struct runs_tree *run = &ni->file.run; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct ATTRIB *attr = NULL, *attr_b; + struct ATTR_LIST_ENTRY *le, *le_b; + struct mft_inode *mi, *mi_b; + CLST svcn, evcn1, vcn, len, end, alen, dealloc; + u64 total_size, alloc_size; + + if (!bytes) + return 0; + + le_b = NULL; + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, NULL, 0, NULL, &mi_b); + if (!attr_b) + return -ENOENT; + + if (!attr_b->non_res) { + u32 data_size = le32_to_cpu(attr->res.data_size); + u32 from, to; + + if (vbo > data_size) + return 0; + + from = vbo; + to = (vbo + bytes) < data_size ? (vbo + bytes) : data_size; + memset(Add2Ptr(resident_data(attr_b), from), 0, to - from); + return 0; + } + + /* TODO: add support for normal files too */ + if (!is_attr_ext(attr_b)) + return -EOPNOTSUPP; + + alloc_size = le64_to_cpu(attr_b->nres.alloc_size); + total_size = le64_to_cpu(attr_b->nres.total_size); + + if (vbo >= alloc_size) { + // NOTE: it is allowed + return 0; + } + + if (vbo + bytes > alloc_size) + bytes = alloc_size - vbo; + + down_write(&ni->file.run_lock); + /* + * Enumerate all attribute segments and punch hole where necessary + */ + alen = alloc_size >> sbi->cluster_bits; + vcn = vbo >> sbi->cluster_bits; + len = bytes >> sbi->cluster_bits; + end = vcn + len; + dealloc = 0; + + svcn = le64_to_cpu(attr_b->nres.svcn); + evcn1 = le64_to_cpu(attr_b->nres.evcn) + 1; + + if (svcn <= vcn && vcn < evcn1) { + attr = attr_b; + le = le_b; + mi = mi_b; + } else if (!le_b) { + err = -EINVAL; + goto out; + } else { + le = le_b; + attr = ni_find_attr(ni, attr_b, &le, ATTR_DATA, NULL, 0, &vcn, + &mi); + if (!attr) { + err = -EINVAL; + goto out; + } + + svcn = le64_to_cpu(attr->nres.svcn); + evcn1 = le64_to_cpu(attr->nres.evcn) + 1; + } + + while (svcn < end) { + CLST vcn1, zero, dealloc2; + + err = attr_load_runs(attr, ni, run, &svcn); + if (err) + goto out; + vcn1 = max(vcn, svcn); + zero = min(end, evcn1) - vcn1; + + dealloc2 = dealloc; + err = run_deallocate_ex(sbi, run, vcn1, zero, &dealloc, true); + if (err) + goto out; + + if (dealloc2 == dealloc) { + /* looks like the required range is already sparsed */ + } else { + if (!run_add_entry(run, vcn1, SPARSE_LCN, zero, + false)) { + err = -ENOMEM; + goto out; + } + + err = mi_pack_runs(mi, attr, run, evcn1 - svcn); + if (err) + goto out; + } + /* free all allocated memory */ + run_truncate(run, 0); + + if (evcn1 >= alen) + break; + + attr = ni_enum_attr_ex(ni, attr, &le, &mi); + if (!attr) { + err = -EINVAL; + goto out; + } + + svcn = le64_to_cpu(attr->nres.svcn); + evcn1 = le64_to_cpu(attr->nres.evcn) + 1; + } + + total_size -= (u64)dealloc << sbi->cluster_bits; + attr_b->nres.total_size = cpu_to_le64(total_size); + mi_b->dirty = true; + + /*update inode size*/ + inode_set_bytes(&ni->vfs_inode, total_size); + ni->ni_flags |= NI_FLAG_UPDATE_PARENT; + mark_inode_dirty(&ni->vfs_inode); + +out: + up_write(&ni->file.run_lock); + if (err) + make_bad_inode(&ni->vfs_inode); + + return err; +} diff --git a/fs/ntfs3/attrlist.c b/fs/ntfs3/attrlist.c new file mode 100644 index 000000000000..ea561361b576 --- /dev/null +++ b/fs/ntfs3/attrlist.c @@ -0,0 +1,456 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + */ + +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +/* Returns true if le is valid */ +static inline bool al_is_valid_le(const struct ntfs_inode *ni, + struct ATTR_LIST_ENTRY *le) +{ + if (!le || !ni->attr_list.le || !ni->attr_list.size) + return false; + + return PtrOffset(ni->attr_list.le, le) + le16_to_cpu(le->size) <= + ni->attr_list.size; +} + +void al_destroy(struct ntfs_inode *ni) +{ + run_close(&ni->attr_list.run); + ntfs_free(ni->attr_list.le); + ni->attr_list.le = NULL; + ni->attr_list.size = 0; + ni->attr_list.dirty = false; +} + +/* + * ntfs_load_attr_list + * + * This method makes sure that the ATTRIB list, if present, + * has been properly set up. + */ +int ntfs_load_attr_list(struct ntfs_inode *ni, struct ATTRIB *attr) +{ + int err; + size_t lsize; + void *le = NULL; + + if (ni->attr_list.size) + return 0; + + if (!attr->non_res) { + lsize = le32_to_cpu(attr->res.data_size); + le = ntfs_malloc(al_aligned(lsize)); + if (!le) { + err = -ENOMEM; + goto out; + } + memcpy(le, resident_data(attr), lsize); + } else if (attr->nres.svcn) { + err = -EINVAL; + goto out; + } else { + u16 run_off = le16_to_cpu(attr->nres.run_off); + + lsize = le64_to_cpu(attr->nres.data_size); + + run_init(&ni->attr_list.run); + + err = run_unpack_ex(&ni->attr_list.run, ni->mi.sbi, ni->mi.rno, + 0, le64_to_cpu(attr->nres.evcn), 0, + Add2Ptr(attr, run_off), + le32_to_cpu(attr->size) - run_off); + if (err < 0) + goto out; + + le = ntfs_malloc(al_aligned(lsize)); + if (!le) { + err = -ENOMEM; + goto out; + } + + err = ntfs_read_run_nb(ni->mi.sbi, &ni->attr_list.run, 0, le, + lsize, NULL); + if (err) + goto out; + } + + ni->attr_list.size = lsize; + ni->attr_list.le = le; + + return 0; + +out: + ni->attr_list.le = le; + al_destroy(ni); + + return err; +} + +/* + * al_enumerate + * + * Returns the next list 'le' + * if 'le' is NULL then returns the first 'le' + */ +struct ATTR_LIST_ENTRY *al_enumerate(struct ntfs_inode *ni, + struct ATTR_LIST_ENTRY *le) +{ + size_t off; + u16 sz; + + if (!le) { + le = ni->attr_list.le; + } else { + sz = le16_to_cpu(le->size); + if (sz < sizeof(struct ATTR_LIST_ENTRY)) { + /* Impossible 'cause we should not return such 'le' */ + return NULL; + } + le = Add2Ptr(le, sz); + } + + /* Check boundary */ + off = PtrOffset(ni->attr_list.le, le); + if (off + sizeof(struct ATTR_LIST_ENTRY) > ni->attr_list.size) { + // The regular end of list + return NULL; + } + + sz = le16_to_cpu(le->size); + + /* Check 'le' for errors */ + if (sz < sizeof(struct ATTR_LIST_ENTRY) || + off + sz > ni->attr_list.size || + sz < le->name_off + le->name_len * sizeof(short)) { + return NULL; + } + + return le; +} + +/* + * al_find_le + * + * finds the first 'le' in the list which matches type, name and vcn + * Returns NULL if not found + */ +struct ATTR_LIST_ENTRY *al_find_le(struct ntfs_inode *ni, + struct ATTR_LIST_ENTRY *le, + const struct ATTRIB *attr) +{ + CLST svcn = attr_svcn(attr); + + return al_find_ex(ni, le, attr->type, attr_name(attr), attr->name_len, + &svcn); +} + +/* + * al_find_ex + * + * finds the first 'le' in the list which matches type, name and vcn + * Returns NULL if not found + */ +struct ATTR_LIST_ENTRY *al_find_ex(struct ntfs_inode *ni, + struct ATTR_LIST_ENTRY *le, + enum ATTR_TYPE type, const __le16 *name, + u8 name_len, const CLST *vcn) +{ + struct ATTR_LIST_ENTRY *ret = NULL; + u32 type_in = le32_to_cpu(type); + + while ((le = al_enumerate(ni, le))) { + u64 le_vcn; + int diff = le32_to_cpu(le->type) - type_in; + + /* List entries are sorted by type, name and vcn */ + if (diff < 0) + continue; + + if (diff > 0) + return ret; + + if (le->name_len != name_len) + continue; + + le_vcn = le64_to_cpu(le->vcn); + if (!le_vcn) { + /* + * compare entry names only for entry with vcn == 0 + */ + diff = ntfs_cmp_names(le_name(le), name_len, name, + name_len, ni->mi.sbi->upcase, + true); + if (diff < 0) + continue; + + if (diff > 0) + return ret; + } + + if (!vcn) + return le; + + if (*vcn == le_vcn) + return le; + + if (*vcn < le_vcn) + return ret; + + ret = le; + } + + return ret; +} + +/* + * al_find_le_to_insert + * + * finds the first list entry which matches type, name and vcn + */ +static struct ATTR_LIST_ENTRY *al_find_le_to_insert(struct ntfs_inode *ni, + enum ATTR_TYPE type, + const __le16 *name, + u8 name_len, CLST vcn) +{ + struct ATTR_LIST_ENTRY *le = NULL, *prev; + u32 type_in = le32_to_cpu(type); + + /* List entries are sorted by type, name, vcn */ + while ((le = al_enumerate(ni, prev = le))) { + int diff = le32_to_cpu(le->type) - type_in; + + if (diff < 0) + continue; + + if (diff > 0) + return le; + + if (!le->vcn) { + /* + * compare entry names only for entry with vcn == 0 + */ + diff = ntfs_cmp_names(le_name(le), le->name_len, name, + name_len, ni->mi.sbi->upcase, + true); + if (diff < 0) + continue; + + if (diff > 0) + return le; + } + + if (le64_to_cpu(le->vcn) >= vcn) + return le; + } + + return prev ? Add2Ptr(prev, le16_to_cpu(prev->size)) : ni->attr_list.le; +} + +/* + * al_add_le + * + * adds an "attribute list entry" to the list. + */ +int al_add_le(struct ntfs_inode *ni, enum ATTR_TYPE type, const __le16 *name, + u8 name_len, CLST svcn, __le16 id, const struct MFT_REF *ref, + struct ATTR_LIST_ENTRY **new_le) +{ + int err; + struct ATTRIB *attr; + struct ATTR_LIST_ENTRY *le; + size_t off; + u16 sz; + size_t asize, new_asize; + u64 new_size; + typeof(ni->attr_list) *al = &ni->attr_list; + + /* + * Compute the size of the new 'le' + */ + sz = le_size(name_len); + new_size = al->size + sz; + asize = al_aligned(al->size); + new_asize = al_aligned(new_size); + + /* Scan forward to the point at which the new 'le' should be inserted. */ + le = al_find_le_to_insert(ni, type, name, name_len, svcn); + off = PtrOffset(al->le, le); + + if (new_size > asize) { + void *ptr = ntfs_malloc(new_asize); + + if (!ptr) + return -ENOMEM; + + memcpy(ptr, al->le, off); + memcpy(Add2Ptr(ptr, off + sz), le, al->size - off); + le = Add2Ptr(ptr, off); + ntfs_free(al->le); + al->le = ptr; + } else { + memmove(Add2Ptr(le, sz), le, al->size - off); + } + + al->size = new_size; + + le->type = type; + le->size = cpu_to_le16(sz); + le->name_len = name_len; + le->name_off = offsetof(struct ATTR_LIST_ENTRY, name); + le->vcn = cpu_to_le64(svcn); + le->ref = *ref; + le->id = id; + memcpy(le->name, name, sizeof(short) * name_len); + + al->dirty = true; + + err = attr_set_size(ni, ATTR_LIST, NULL, 0, &al->run, new_size, + &new_size, true, &attr); + if (err) + return err; + + if (attr && attr->non_res) { + err = ntfs_sb_write_run(ni->mi.sbi, &al->run, 0, al->le, + al->size); + if (err) + return err; + } + + al->dirty = false; + *new_le = le; + + return 0; +} + +/* + * al_remove_le + * + * removes 'le' from attribute list + */ +bool al_remove_le(struct ntfs_inode *ni, struct ATTR_LIST_ENTRY *le) +{ + u16 size; + size_t off; + typeof(ni->attr_list) *al = &ni->attr_list; + + if (!al_is_valid_le(ni, le)) + return false; + + /* Save on stack the size of 'le' */ + size = le16_to_cpu(le->size); + off = PtrOffset(al->le, le); + + memmove(le, Add2Ptr(le, size), al->size - (off + size)); + + al->size -= size; + al->dirty = true; + + return true; +} + +/* + * al_delete_le + * + * deletes from the list the first 'le' which matches its parameters. + */ +bool al_delete_le(struct ntfs_inode *ni, enum ATTR_TYPE type, CLST vcn, + const __le16 *name, size_t name_len, + const struct MFT_REF *ref) +{ + u16 size; + struct ATTR_LIST_ENTRY *le; + size_t off; + typeof(ni->attr_list) *al = &ni->attr_list; + + /* Scan forward to the first 'le' that matches the input */ + le = al_find_ex(ni, NULL, type, name, name_len, &vcn); + if (!le) + return false; + + off = PtrOffset(al->le, le); + +next: + if (off >= al->size) + return false; + if (le->type != type) + return false; + if (le->name_len != name_len) + return false; + if (name_len && ntfs_cmp_names(le_name(le), name_len, name, name_len, + ni->mi.sbi->upcase, true)) + return false; + if (le64_to_cpu(le->vcn) != vcn) + return false; + + /* + * The caller specified a segment reference, so we have to + * scan through the matching entries until we find that segment + * reference or we run of matching entries. + */ + if (ref && memcmp(ref, &le->ref, sizeof(*ref))) { + off += le16_to_cpu(le->size); + le = Add2Ptr(al->le, off); + goto next; + } + + /* Save on stack the size of 'le' */ + size = le16_to_cpu(le->size); + /* Delete 'le'. */ + memmove(le, Add2Ptr(le, size), al->size - (off + size)); + + al->size -= size; + al->dirty = true; + + return true; +} + +/* + * al_update + */ +int al_update(struct ntfs_inode *ni) +{ + int err; + struct ATTRIB *attr; + typeof(ni->attr_list) *al = &ni->attr_list; + + if (!al->dirty || !al->size) + return 0; + + /* + * attribute list increased on demand in al_add_le + * attribute list decreased here + */ + err = attr_set_size(ni, ATTR_LIST, NULL, 0, &al->run, al->size, NULL, + false, &attr); + if (err) + goto out; + + if (!attr->non_res) { + memcpy(resident_data(attr), al->le, al->size); + } else { + err = ntfs_sb_write_run(ni->mi.sbi, &al->run, 0, al->le, + al->size); + if (err) + goto out; + + attr->nres.valid_size = attr->nres.data_size; + } + + ni->mi.dirty = true; + al->dirty = false; + +out: + return err; +} diff --git a/fs/ntfs3/bitfunc.c b/fs/ntfs3/bitfunc.c new file mode 100644 index 000000000000..2de5faef2721 --- /dev/null +++ b/fs/ntfs3/bitfunc.c @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + */ +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +#define BITS_IN_SIZE_T (sizeof(size_t) * 8) + +/* + * fill_mask[i] - first i bits are '1' , i = 0,1,2,3,4,5,6,7,8 + * fill_mask[i] = 0xFF >> (8-i) + */ +static const u8 fill_mask[] = { 0x00, 0x01, 0x03, 0x07, 0x0F, + 0x1F, 0x3F, 0x7F, 0xFF }; + +/* + * zero_mask[i] - first i bits are '0' , i = 0,1,2,3,4,5,6,7,8 + * zero_mask[i] = 0xFF << i + */ +static const u8 zero_mask[] = { 0xFF, 0xFE, 0xFC, 0xF8, 0xF0, + 0xE0, 0xC0, 0x80, 0x00 }; + +/* + * are_bits_clear + * + * Returns true if all bits [bit, bit+nbits) are zeros "0" + */ +bool are_bits_clear(const ulong *lmap, size_t bit, size_t nbits) +{ + size_t pos = bit & 7; + const u8 *map = (u8 *)lmap + (bit >> 3); + + if (pos) { + if (8 - pos >= nbits) + return !nbits || !(*map & fill_mask[pos + nbits] & + zero_mask[pos]); + + if (*map++ & zero_mask[pos]) + return false; + nbits -= 8 - pos; + } + + pos = ((size_t)map) & (sizeof(size_t) - 1); + if (pos) { + pos = sizeof(size_t) - pos; + if (nbits >= pos * 8) { + for (nbits -= pos * 8; pos; pos--, map++) { + if (*map) + return false; + } + } + } + + for (pos = nbits / BITS_IN_SIZE_T; pos; pos--, map += sizeof(size_t)) { + if (*((size_t *)map)) + return false; + } + + for (pos = (nbits % BITS_IN_SIZE_T) >> 3; pos; pos--, map++) { + if (*map) + return false; + } + + pos = nbits & 7; + if (pos && (*map & fill_mask[pos])) + return false; + + // All bits are zero + return true; +} + +/* + * are_bits_set + * + * Returns true if all bits [bit, bit+nbits) are ones "1" + */ +bool are_bits_set(const ulong *lmap, size_t bit, size_t nbits) +{ + u8 mask; + size_t pos = bit & 7; + const u8 *map = (u8 *)lmap + (bit >> 3); + + if (pos) { + if (8 - pos >= nbits) { + mask = fill_mask[pos + nbits] & zero_mask[pos]; + return !nbits || (*map & mask) == mask; + } + + mask = zero_mask[pos]; + if ((*map++ & mask) != mask) + return false; + nbits -= 8 - pos; + } + + pos = ((size_t)map) & (sizeof(size_t) - 1); + if (pos) { + pos = sizeof(size_t) - pos; + if (nbits >= pos * 8) { + for (nbits -= pos * 8; pos; pos--, map++) { + if (*map != 0xFF) + return false; + } + } + } + + for (pos = nbits / BITS_IN_SIZE_T; pos; pos--, map += sizeof(size_t)) { + if (*((size_t *)map) != MINUS_ONE_T) + return false; + } + + for (pos = (nbits % BITS_IN_SIZE_T) >> 3; pos; pos--, map++) { + if (*map != 0xFF) + return false; + } + + pos = nbits & 7; + if (pos) { + u8 mask = fill_mask[pos]; + + if ((*map & mask) != mask) + return false; + } + + // All bits are ones + return true; +} diff --git a/fs/ntfs3/bitmap.c b/fs/ntfs3/bitmap.c new file mode 100644 index 000000000000..32aab0031221 --- /dev/null +++ b/fs/ntfs3/bitmap.c @@ -0,0 +1,1519 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + * This code builds two trees of free clusters extents. + * Trees are sorted by start of extent and by length of extent. + * NTFS_MAX_WND_EXTENTS defines the maximum number of elements in trees. + * In extreme case code reads on-disk bitmap to find free clusters + * + */ + +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +/* + * Maximum number of extents in tree. + */ +#define NTFS_MAX_WND_EXTENTS (32u * 1024u) + +struct rb_node_key { + struct rb_node node; + size_t key; +}; + +/* + * Tree is sorted by start (key) + */ +struct e_node { + struct rb_node_key start; /* Tree sorted by start */ + struct rb_node_key count; /* Tree sorted by len*/ +}; + +static int wnd_rescan(struct wnd_bitmap *wnd); +static struct buffer_head *wnd_map(struct wnd_bitmap *wnd, size_t iw); +static bool wnd_is_free_hlp(struct wnd_bitmap *wnd, size_t bit, size_t bits); + +static struct kmem_cache *ntfs_enode_cachep; + +int __init ntfs3_init_bitmap(void) +{ + ntfs_enode_cachep = + kmem_cache_create("ntfs3_enode_cache", sizeof(struct e_node), 0, + SLAB_RECLAIM_ACCOUNT, NULL); + return ntfs_enode_cachep ? 0 : -ENOMEM; +} + +void ntfs3_exit_bitmap(void) +{ + kmem_cache_destroy(ntfs_enode_cachep); +} + +static inline u32 wnd_bits(const struct wnd_bitmap *wnd, size_t i) +{ + return i + 1 == wnd->nwnd ? wnd->bits_last : wnd->sb->s_blocksize * 8; +} + +/* + * b_pos + b_len - biggest fragment + * Scan range [wpos wbits) window 'buf' + * Returns -1 if not found + */ +static size_t wnd_scan(const ulong *buf, size_t wbit, u32 wpos, u32 wend, + size_t to_alloc, size_t *prev_tail, size_t *b_pos, + size_t *b_len) +{ + while (wpos < wend) { + size_t free_len; + u32 free_bits, end; + u32 used = find_next_zero_bit(buf, wend, wpos); + + if (used >= wend) { + if (*b_len < *prev_tail) { + *b_pos = wbit - *prev_tail; + *b_len = *prev_tail; + } + + *prev_tail = 0; + return -1; + } + + if (used > wpos) { + wpos = used; + if (*b_len < *prev_tail) { + *b_pos = wbit - *prev_tail; + *b_len = *prev_tail; + } + + *prev_tail = 0; + } + + /* + * Now we have a fragment [wpos, wend) staring with 0 + */ + end = wpos + to_alloc - *prev_tail; + free_bits = find_next_bit(buf, min(end, wend), wpos); + + free_len = *prev_tail + free_bits - wpos; + + if (*b_len < free_len) { + *b_pos = wbit + wpos - *prev_tail; + *b_len = free_len; + } + + if (free_len >= to_alloc) + return wbit + wpos - *prev_tail; + + if (free_bits >= wend) { + *prev_tail += free_bits - wpos; + return -1; + } + + wpos = free_bits + 1; + + *prev_tail = 0; + } + + return -1; +} + +/* + * wnd_close + * + * Frees all resources + */ +void wnd_close(struct wnd_bitmap *wnd) +{ + struct rb_node *node, *next; + + ntfs_free(wnd->free_bits); + run_close(&wnd->run); + + node = rb_first(&wnd->start_tree); + + while (node) { + next = rb_next(node); + rb_erase(node, &wnd->start_tree); + kmem_cache_free(ntfs_enode_cachep, + rb_entry(node, struct e_node, start.node)); + node = next; + } +} + +static struct rb_node *rb_lookup(struct rb_root *root, size_t v) +{ + struct rb_node **p = &root->rb_node; + struct rb_node *r = NULL; + + while (*p) { + struct rb_node_key *k; + + k = rb_entry(*p, struct rb_node_key, node); + if (v < k->key) { + p = &(*p)->rb_left; + } else if (v > k->key) { + r = &k->node; + p = &(*p)->rb_right; + } else { + return &k->node; + } + } + + return r; +} + +/* + * rb_insert_count + * + * Helper function to insert special kind of 'count' tree + */ +static inline bool rb_insert_count(struct rb_root *root, struct e_node *e) +{ + struct rb_node **p = &root->rb_node; + struct rb_node *parent = NULL; + size_t e_ckey = e->count.key; + size_t e_skey = e->start.key; + + while (*p) { + struct e_node *k = + rb_entry(parent = *p, struct e_node, count.node); + + if (e_ckey > k->count.key) { + p = &(*p)->rb_left; + } else if (e_ckey < k->count.key) { + p = &(*p)->rb_right; + } else if (e_skey < k->start.key) { + p = &(*p)->rb_left; + } else if (e_skey > k->start.key) { + p = &(*p)->rb_right; + } else { + WARN_ON(1); + return false; + } + } + + rb_link_node(&e->count.node, parent, p); + rb_insert_color(&e->count.node, root); + return true; +} + +/* + * inline bool rb_insert_start + * + * Helper function to insert special kind of 'start' tree + */ +static inline bool rb_insert_start(struct rb_root *root, struct e_node *e) +{ + struct rb_node **p = &root->rb_node; + struct rb_node *parent = NULL; + size_t e_skey = e->start.key; + + while (*p) { + struct e_node *k; + + parent = *p; + + k = rb_entry(parent, struct e_node, start.node); + if (e_skey < k->start.key) { + p = &(*p)->rb_left; + } else if (e_skey > k->start.key) { + p = &(*p)->rb_right; + } else { + WARN_ON(1); + return false; + } + } + + rb_link_node(&e->start.node, parent, p); + rb_insert_color(&e->start.node, root); + return true; +} + +/* + * wnd_add_free_ext + * + * adds a new extent of free space + * build = 1 when building tree + */ +static void wnd_add_free_ext(struct wnd_bitmap *wnd, size_t bit, size_t len, + bool build) +{ + struct e_node *e, *e0 = NULL; + size_t ib, end_in = bit + len; + struct rb_node *n; + + if (build) { + /* Use extent_min to filter too short extents */ + if (wnd->count >= NTFS_MAX_WND_EXTENTS && + len <= wnd->extent_min) { + wnd->uptodated = -1; + return; + } + } else { + /* Try to find extent before 'bit' */ + n = rb_lookup(&wnd->start_tree, bit); + + if (!n) { + n = rb_first(&wnd->start_tree); + } else { + e = rb_entry(n, struct e_node, start.node); + n = rb_next(n); + if (e->start.key + e->count.key == bit) { + /* Remove left */ + bit = e->start.key; + len += e->count.key; + rb_erase(&e->start.node, &wnd->start_tree); + rb_erase(&e->count.node, &wnd->count_tree); + wnd->count -= 1; + e0 = e; + } + } + + while (n) { + size_t next_end; + + e = rb_entry(n, struct e_node, start.node); + next_end = e->start.key + e->count.key; + if (e->start.key > end_in) + break; + + /* Remove right */ + n = rb_next(n); + len += next_end - end_in; + end_in = next_end; + rb_erase(&e->start.node, &wnd->start_tree); + rb_erase(&e->count.node, &wnd->count_tree); + wnd->count -= 1; + + if (!e0) + e0 = e; + else + kmem_cache_free(ntfs_enode_cachep, e); + } + + if (wnd->uptodated != 1) { + /* Check bits before 'bit' */ + ib = wnd->zone_bit == wnd->zone_end || + bit < wnd->zone_end + ? 0 + : wnd->zone_end; + + while (bit > ib && wnd_is_free_hlp(wnd, bit - 1, 1)) { + bit -= 1; + len += 1; + } + + /* Check bits after 'end_in' */ + ib = wnd->zone_bit == wnd->zone_end || + end_in > wnd->zone_bit + ? wnd->nbits + : wnd->zone_bit; + + while (end_in < ib && wnd_is_free_hlp(wnd, end_in, 1)) { + end_in += 1; + len += 1; + } + } + } + /* Insert new fragment */ + if (wnd->count >= NTFS_MAX_WND_EXTENTS) { + if (e0) + kmem_cache_free(ntfs_enode_cachep, e0); + + wnd->uptodated = -1; + + /* Compare with smallest fragment */ + n = rb_last(&wnd->count_tree); + e = rb_entry(n, struct e_node, count.node); + if (len <= e->count.key) + goto out; /* Do not insert small fragments */ + + if (build) { + struct e_node *e2; + + n = rb_prev(n); + e2 = rb_entry(n, struct e_node, count.node); + /* smallest fragment will be 'e2->count.key' */ + wnd->extent_min = e2->count.key; + } + + /* Replace smallest fragment by new one */ + rb_erase(&e->start.node, &wnd->start_tree); + rb_erase(&e->count.node, &wnd->count_tree); + wnd->count -= 1; + } else { + e = e0 ? e0 : kmem_cache_alloc(ntfs_enode_cachep, GFP_ATOMIC); + if (!e) { + wnd->uptodated = -1; + goto out; + } + + if (build && len <= wnd->extent_min) + wnd->extent_min = len; + } + e->start.key = bit; + e->count.key = len; + if (len > wnd->extent_max) + wnd->extent_max = len; + + rb_insert_start(&wnd->start_tree, e); + rb_insert_count(&wnd->count_tree, e); + wnd->count += 1; + +out:; +} + +/* + * wnd_remove_free_ext + * + * removes a run from the cached free space + */ +static void wnd_remove_free_ext(struct wnd_bitmap *wnd, size_t bit, size_t len) +{ + struct rb_node *n, *n3; + struct e_node *e, *e3; + size_t end_in = bit + len; + size_t end3, end, new_key, new_len, max_new_len; + + /* Try to find extent before 'bit' */ + n = rb_lookup(&wnd->start_tree, bit); + + if (!n) + return; + + e = rb_entry(n, struct e_node, start.node); + end = e->start.key + e->count.key; + + new_key = new_len = 0; + len = e->count.key; + + /* Range [bit,end_in) must be inside 'e' or outside 'e' and 'n' */ + if (e->start.key > bit) + ; + else if (end_in <= end) { + /* Range [bit,end_in) inside 'e' */ + new_key = end_in; + new_len = end - end_in; + len = bit - e->start.key; + } else if (bit > end) { + bool bmax = false; + + n3 = rb_next(n); + + while (n3) { + e3 = rb_entry(n3, struct e_node, start.node); + if (e3->start.key >= end_in) + break; + + if (e3->count.key == wnd->extent_max) + bmax = true; + + end3 = e3->start.key + e3->count.key; + if (end3 > end_in) { + e3->start.key = end_in; + rb_erase(&e3->count.node, &wnd->count_tree); + e3->count.key = end3 - end_in; + rb_insert_count(&wnd->count_tree, e3); + break; + } + + n3 = rb_next(n3); + rb_erase(&e3->start.node, &wnd->start_tree); + rb_erase(&e3->count.node, &wnd->count_tree); + wnd->count -= 1; + kmem_cache_free(ntfs_enode_cachep, e3); + } + if (!bmax) + return; + n3 = rb_first(&wnd->count_tree); + wnd->extent_max = + n3 ? rb_entry(n3, struct e_node, count.node)->count.key + : 0; + return; + } + + if (e->count.key != wnd->extent_max) { + ; + } else if (rb_prev(&e->count.node)) { + ; + } else { + n3 = rb_next(&e->count.node); + max_new_len = len > new_len ? len : new_len; + if (!n3) { + wnd->extent_max = max_new_len; + } else { + e3 = rb_entry(n3, struct e_node, count.node); + wnd->extent_max = max(e3->count.key, max_new_len); + } + } + + if (!len) { + if (new_len) { + e->start.key = new_key; + rb_erase(&e->count.node, &wnd->count_tree); + e->count.key = new_len; + rb_insert_count(&wnd->count_tree, e); + } else { + rb_erase(&e->start.node, &wnd->start_tree); + rb_erase(&e->count.node, &wnd->count_tree); + wnd->count -= 1; + kmem_cache_free(ntfs_enode_cachep, e); + } + goto out; + } + rb_erase(&e->count.node, &wnd->count_tree); + e->count.key = len; + rb_insert_count(&wnd->count_tree, e); + + if (!new_len) + goto out; + + if (wnd->count >= NTFS_MAX_WND_EXTENTS) { + wnd->uptodated = -1; + + /* Get minimal extent */ + e = rb_entry(rb_last(&wnd->count_tree), struct e_node, + count.node); + if (e->count.key > new_len) + goto out; + + /* Replace minimum */ + rb_erase(&e->start.node, &wnd->start_tree); + rb_erase(&e->count.node, &wnd->count_tree); + wnd->count -= 1; + } else { + e = kmem_cache_alloc(ntfs_enode_cachep, GFP_ATOMIC); + if (!e) + wnd->uptodated = -1; + } + + if (e) { + e->start.key = new_key; + e->count.key = new_len; + rb_insert_start(&wnd->start_tree, e); + rb_insert_count(&wnd->count_tree, e); + wnd->count += 1; + } + +out: + if (!wnd->count && 1 != wnd->uptodated) + wnd_rescan(wnd); +} + +/* + * wnd_rescan + * + * Scan all bitmap. used while initialization. + */ +static int wnd_rescan(struct wnd_bitmap *wnd) +{ + int err = 0; + size_t prev_tail = 0; + struct super_block *sb = wnd->sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + u64 lbo, len = 0; + u32 blocksize = sb->s_blocksize; + u8 cluster_bits = sbi->cluster_bits; + u32 wbits = 8 * sb->s_blocksize; + u32 used, frb; + const ulong *buf; + size_t wpos, wbit, iw, vbo; + struct buffer_head *bh = NULL; + CLST lcn, clen; + + wnd->uptodated = 0; + wnd->extent_max = 0; + wnd->extent_min = MINUS_ONE_T; + wnd->total_zeroes = 0; + + vbo = 0; + + for (iw = 0; iw < wnd->nwnd; iw++) { + if (iw + 1 == wnd->nwnd) + wbits = wnd->bits_last; + + if (wnd->inited) { + if (!wnd->free_bits[iw]) { + /* all ones */ + if (prev_tail) { + wnd_add_free_ext(wnd, + vbo * 8 - prev_tail, + prev_tail, true); + prev_tail = 0; + } + goto next_wnd; + } + if (wbits == wnd->free_bits[iw]) { + /* all zeroes */ + prev_tail += wbits; + wnd->total_zeroes += wbits; + goto next_wnd; + } + } + + if (!len) { + u32 off = vbo & sbi->cluster_mask; + + if (!run_lookup_entry(&wnd->run, vbo >> cluster_bits, + &lcn, &clen, NULL)) { + err = -ENOENT; + goto out; + } + + lbo = ((u64)lcn << cluster_bits) + off; + len = ((u64)clen << cluster_bits) - off; + } + + bh = ntfs_bread(sb, lbo >> sb->s_blocksize_bits); + if (!bh) { + err = -EIO; + goto out; + } + + buf = (ulong *)bh->b_data; + + used = __bitmap_weight(buf, wbits); + if (used < wbits) { + frb = wbits - used; + wnd->free_bits[iw] = frb; + wnd->total_zeroes += frb; + } + + wpos = 0; + wbit = vbo * 8; + + if (wbit + wbits > wnd->nbits) + wbits = wnd->nbits - wbit; + + do { + used = find_next_zero_bit(buf, wbits, wpos); + + if (used > wpos && prev_tail) { + wnd_add_free_ext(wnd, wbit + wpos - prev_tail, + prev_tail, true); + prev_tail = 0; + } + + wpos = used; + + if (wpos >= wbits) { + /* No free blocks */ + prev_tail = 0; + break; + } + + frb = find_next_bit(buf, wbits, wpos); + if (frb >= wbits) { + /* keep last free block */ + prev_tail += frb - wpos; + break; + } + + wnd_add_free_ext(wnd, wbit + wpos - prev_tail, + frb + prev_tail - wpos, true); + + /* Skip free block and first '1' */ + wpos = frb + 1; + /* Reset previous tail */ + prev_tail = 0; + } while (wpos < wbits); + +next_wnd: + + if (bh) + put_bh(bh); + bh = NULL; + + vbo += blocksize; + if (len) { + len -= blocksize; + lbo += blocksize; + } + } + + /* Add last block */ + if (prev_tail) + wnd_add_free_ext(wnd, wnd->nbits - prev_tail, prev_tail, true); + + /* + * Before init cycle wnd->uptodated was 0 + * If any errors or limits occurs while initialization then + * wnd->uptodated will be -1 + * If 'uptodated' is still 0 then Tree is really updated + */ + if (!wnd->uptodated) + wnd->uptodated = 1; + + if (wnd->zone_bit != wnd->zone_end) { + size_t zlen = wnd->zone_end - wnd->zone_bit; + + wnd->zone_end = wnd->zone_bit; + wnd_zone_set(wnd, wnd->zone_bit, zlen); + } + +out: + return err; +} + +/* + * wnd_init + */ +int wnd_init(struct wnd_bitmap *wnd, struct super_block *sb, size_t nbits) +{ + int err; + u32 blocksize = sb->s_blocksize; + u32 wbits = blocksize * 8; + + init_rwsem(&wnd->rw_lock); + + wnd->sb = sb; + wnd->nbits = nbits; + wnd->total_zeroes = nbits; + wnd->extent_max = MINUS_ONE_T; + wnd->zone_bit = wnd->zone_end = 0; + wnd->nwnd = bytes_to_block(sb, bitmap_size(nbits)); + wnd->bits_last = nbits & (wbits - 1); + if (!wnd->bits_last) + wnd->bits_last = wbits; + + wnd->free_bits = ntfs_zalloc(wnd->nwnd * sizeof(u16)); + if (!wnd->free_bits) + return -ENOMEM; + + err = wnd_rescan(wnd); + if (err) + return err; + + wnd->inited = true; + + return 0; +} + +/* + * wnd_map + * + * call sb_bread for requested window + */ +static struct buffer_head *wnd_map(struct wnd_bitmap *wnd, size_t iw) +{ + size_t vbo; + CLST lcn, clen; + struct super_block *sb = wnd->sb; + struct ntfs_sb_info *sbi; + struct buffer_head *bh; + u64 lbo; + + sbi = sb->s_fs_info; + vbo = (u64)iw << sb->s_blocksize_bits; + + if (!run_lookup_entry(&wnd->run, vbo >> sbi->cluster_bits, &lcn, &clen, + NULL)) { + return ERR_PTR(-ENOENT); + } + + lbo = ((u64)lcn << sbi->cluster_bits) + (vbo & sbi->cluster_mask); + + bh = ntfs_bread(wnd->sb, lbo >> sb->s_blocksize_bits); + if (!bh) + return ERR_PTR(-EIO); + + return bh; +} + +/* + * wnd_set_free + * + * Marks the bits range from bit to bit + bits as free + */ +int wnd_set_free(struct wnd_bitmap *wnd, size_t bit, size_t bits) +{ + int err = 0; + struct super_block *sb = wnd->sb; + size_t bits0 = bits; + u32 wbits = 8 * sb->s_blocksize; + size_t iw = bit >> (sb->s_blocksize_bits + 3); + u32 wbit = bit & (wbits - 1); + struct buffer_head *bh; + + while (iw < wnd->nwnd && bits) { + u32 tail, op; + ulong *buf; + + if (iw + 1 == wnd->nwnd) + wbits = wnd->bits_last; + + tail = wbits - wbit; + op = tail < bits ? tail : bits; + + bh = wnd_map(wnd, iw); + if (IS_ERR(bh)) { + err = PTR_ERR(bh); + break; + } + + buf = (ulong *)bh->b_data; + + lock_buffer(bh); + + __bitmap_clear(buf, wbit, op); + + wnd->free_bits[iw] += op; + + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); + unlock_buffer(bh); + put_bh(bh); + + wnd->total_zeroes += op; + bits -= op; + wbit = 0; + iw += 1; + } + + wnd_add_free_ext(wnd, bit, bits0, false); + + return err; +} + +/* + * wnd_set_used + * + * Marks the bits range from bit to bit + bits as used + */ +int wnd_set_used(struct wnd_bitmap *wnd, size_t bit, size_t bits) +{ + int err = 0; + struct super_block *sb = wnd->sb; + size_t bits0 = bits; + size_t iw = bit >> (sb->s_blocksize_bits + 3); + u32 wbits = 8 * sb->s_blocksize; + u32 wbit = bit & (wbits - 1); + struct buffer_head *bh; + + while (iw < wnd->nwnd && bits) { + u32 tail, op; + ulong *buf; + + if (unlikely(iw + 1 == wnd->nwnd)) + wbits = wnd->bits_last; + + tail = wbits - wbit; + op = tail < bits ? tail : bits; + + bh = wnd_map(wnd, iw); + if (IS_ERR(bh)) { + err = PTR_ERR(bh); + break; + } + buf = (ulong *)bh->b_data; + + lock_buffer(bh); + + __bitmap_set(buf, wbit, op); + wnd->free_bits[iw] -= op; + + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); + unlock_buffer(bh); + put_bh(bh); + + wnd->total_zeroes -= op; + bits -= op; + wbit = 0; + iw += 1; + } + + if (!RB_EMPTY_ROOT(&wnd->start_tree)) + wnd_remove_free_ext(wnd, bit, bits0); + + return err; +} + +/* + * wnd_is_free_hlp + * + * Returns true if all clusters [bit, bit+bits) are free (bitmap only) + */ +static bool wnd_is_free_hlp(struct wnd_bitmap *wnd, size_t bit, size_t bits) +{ + struct super_block *sb = wnd->sb; + size_t iw = bit >> (sb->s_blocksize_bits + 3); + u32 wbits = 8 * sb->s_blocksize; + u32 wbit = bit & (wbits - 1); + + while (iw < wnd->nwnd && bits) { + u32 tail, op; + + if (unlikely(iw + 1 == wnd->nwnd)) + wbits = wnd->bits_last; + + tail = wbits - wbit; + op = tail < bits ? tail : bits; + + if (wbits != wnd->free_bits[iw]) { + bool ret; + struct buffer_head *bh = wnd_map(wnd, iw); + + if (IS_ERR(bh)) + return false; + + ret = are_bits_clear((ulong *)bh->b_data, wbit, op); + + put_bh(bh); + if (!ret) + return false; + } + + bits -= op; + wbit = 0; + iw += 1; + } + + return true; +} + +/* + * wnd_is_free + * + * Returns true if all clusters [bit, bit+bits) are free + */ +bool wnd_is_free(struct wnd_bitmap *wnd, size_t bit, size_t bits) +{ + bool ret; + struct rb_node *n; + size_t end; + struct e_node *e; + + if (RB_EMPTY_ROOT(&wnd->start_tree)) + goto use_wnd; + + n = rb_lookup(&wnd->start_tree, bit); + if (!n) + goto use_wnd; + + e = rb_entry(n, struct e_node, start.node); + + end = e->start.key + e->count.key; + + if (bit < end && bit + bits <= end) + return true; + +use_wnd: + ret = wnd_is_free_hlp(wnd, bit, bits); + + return ret; +} + +/* + * wnd_is_used + * + * Returns true if all clusters [bit, bit+bits) are used + */ +bool wnd_is_used(struct wnd_bitmap *wnd, size_t bit, size_t bits) +{ + bool ret = false; + struct super_block *sb = wnd->sb; + size_t iw = bit >> (sb->s_blocksize_bits + 3); + u32 wbits = 8 * sb->s_blocksize; + u32 wbit = bit & (wbits - 1); + size_t end; + struct rb_node *n; + struct e_node *e; + + if (RB_EMPTY_ROOT(&wnd->start_tree)) + goto use_wnd; + + end = bit + bits; + n = rb_lookup(&wnd->start_tree, end - 1); + if (!n) + goto use_wnd; + + e = rb_entry(n, struct e_node, start.node); + if (e->start.key + e->count.key > bit) + return false; + +use_wnd: + while (iw < wnd->nwnd && bits) { + u32 tail, op; + + if (unlikely(iw + 1 == wnd->nwnd)) + wbits = wnd->bits_last; + + tail = wbits - wbit; + op = tail < bits ? tail : bits; + + if (wnd->free_bits[iw]) { + bool ret; + struct buffer_head *bh = wnd_map(wnd, iw); + + if (IS_ERR(bh)) + goto out; + + ret = are_bits_set((ulong *)bh->b_data, wbit, op); + put_bh(bh); + if (!ret) + goto out; + } + + bits -= op; + wbit = 0; + iw += 1; + } + ret = true; + +out: + return ret; +} + +/* + * wnd_find + * - flags - BITMAP_FIND_XXX flags + * + * looks for free space + * Returns 0 if not found + */ +size_t wnd_find(struct wnd_bitmap *wnd, size_t to_alloc, size_t hint, + size_t flags, size_t *allocated) +{ + struct super_block *sb; + u32 wbits, wpos, wzbit, wzend; + size_t fnd, max_alloc, b_len, b_pos; + size_t iw, prev_tail, nwnd, wbit, ebit, zbit, zend; + size_t to_alloc0 = to_alloc; + const ulong *buf; + const struct e_node *e; + const struct rb_node *pr, *cr; + u8 log2_bits; + bool fbits_valid; + struct buffer_head *bh; + + /* fast checking for available free space */ + if (flags & BITMAP_FIND_FULL) { + size_t zeroes = wnd_zeroes(wnd); + + zeroes -= wnd->zone_end - wnd->zone_bit; + if (zeroes < to_alloc0) + goto no_space; + + if (to_alloc0 > wnd->extent_max) + goto no_space; + } else { + if (to_alloc > wnd->extent_max) + to_alloc = wnd->extent_max; + } + + if (wnd->zone_bit <= hint && hint < wnd->zone_end) + hint = wnd->zone_end; + + max_alloc = wnd->nbits; + b_len = b_pos = 0; + + if (hint >= max_alloc) + hint = 0; + + if (RB_EMPTY_ROOT(&wnd->start_tree)) { + if (wnd->uptodated == 1) { + /* extents tree is updated -> no free space */ + goto no_space; + } + goto scan_bitmap; + } + + e = NULL; + if (!hint) + goto allocate_biggest; + + /* Use hint: enumerate extents by start >= hint */ + pr = NULL; + cr = wnd->start_tree.rb_node; + + for (;;) { + e = rb_entry(cr, struct e_node, start.node); + + if (e->start.key == hint) + break; + + if (e->start.key < hint) { + pr = cr; + cr = cr->rb_right; + if (!cr) + break; + continue; + } + + cr = cr->rb_left; + if (!cr) { + e = pr ? rb_entry(pr, struct e_node, start.node) : NULL; + break; + } + } + + if (!e) + goto allocate_biggest; + + if (e->start.key + e->count.key > hint) { + /* We have found extension with 'hint' inside */ + size_t len = e->start.key + e->count.key - hint; + + if (len >= to_alloc && hint + to_alloc <= max_alloc) { + fnd = hint; + goto found; + } + + if (!(flags & BITMAP_FIND_FULL)) { + if (len > to_alloc) + len = to_alloc; + + if (hint + len <= max_alloc) { + fnd = hint; + to_alloc = len; + goto found; + } + } + } + +allocate_biggest: + /* Allocate from biggest free extent */ + e = rb_entry(rb_first(&wnd->count_tree), struct e_node, count.node); + if (e->count.key != wnd->extent_max) + wnd->extent_max = e->count.key; + + if (e->count.key < max_alloc) { + if (e->count.key >= to_alloc) { + ; + } else if (flags & BITMAP_FIND_FULL) { + if (e->count.key < to_alloc0) { + /* Biggest free block is less then requested */ + goto no_space; + } + to_alloc = e->count.key; + } else if (-1 != wnd->uptodated) { + to_alloc = e->count.key; + } else { + /* Check if we can use more bits */ + size_t op, max_check; + struct rb_root start_tree; + + memcpy(&start_tree, &wnd->start_tree, + sizeof(struct rb_root)); + memset(&wnd->start_tree, 0, sizeof(struct rb_root)); + + max_check = e->start.key + to_alloc; + if (max_check > max_alloc) + max_check = max_alloc; + for (op = e->start.key + e->count.key; op < max_check; + op++) { + if (!wnd_is_free(wnd, op, 1)) + break; + } + memcpy(&wnd->start_tree, &start_tree, + sizeof(struct rb_root)); + to_alloc = op - e->start.key; + } + + /* Prepare to return */ + fnd = e->start.key; + if (e->start.key + to_alloc > max_alloc) + to_alloc = max_alloc - e->start.key; + goto found; + } + + if (wnd->uptodated == 1) { + /* extents tree is updated -> no free space */ + goto no_space; + } + + b_len = e->count.key; + b_pos = e->start.key; + +scan_bitmap: + sb = wnd->sb; + log2_bits = sb->s_blocksize_bits + 3; + + /* At most two ranges [hint, max_alloc) + [0, hint) */ +Again: + + /* TODO: optimize request for case nbits > wbits */ + iw = hint >> log2_bits; + wbits = sb->s_blocksize * 8; + wpos = hint & (wbits - 1); + prev_tail = 0; + fbits_valid = true; + + if (max_alloc == wnd->nbits) { + nwnd = wnd->nwnd; + } else { + size_t t = max_alloc + wbits - 1; + + nwnd = likely(t > max_alloc) ? (t >> log2_bits) : wnd->nwnd; + } + + /* Enumerate all windows */ + for (; iw < nwnd; iw++) { + wbit = iw << log2_bits; + + if (!wnd->free_bits[iw]) { + if (prev_tail > b_len) { + b_pos = wbit - prev_tail; + b_len = prev_tail; + } + + /* Skip full used window */ + prev_tail = 0; + wpos = 0; + continue; + } + + if (unlikely(iw + 1 == nwnd)) { + if (max_alloc == wnd->nbits) { + wbits = wnd->bits_last; + } else { + size_t t = max_alloc & (wbits - 1); + + if (t) { + wbits = t; + fbits_valid = false; + } + } + } + + if (wnd->zone_end > wnd->zone_bit) { + ebit = wbit + wbits; + zbit = max(wnd->zone_bit, wbit); + zend = min(wnd->zone_end, ebit); + + /* Here we have a window [wbit, ebit) and zone [zbit, zend) */ + if (zend <= zbit) { + /* Zone does not overlap window */ + } else { + wzbit = zbit - wbit; + wzend = zend - wbit; + + /* Zone overlaps window */ + if (wnd->free_bits[iw] == wzend - wzbit) { + prev_tail = 0; + wpos = 0; + continue; + } + + /* Scan two ranges window: [wbit, zbit) and [zend, ebit) */ + bh = wnd_map(wnd, iw); + + if (IS_ERR(bh)) { + /* TODO: error */ + prev_tail = 0; + wpos = 0; + continue; + } + + buf = (ulong *)bh->b_data; + + /* Scan range [wbit, zbit) */ + if (wpos < wzbit) { + /* Scan range [wpos, zbit) */ + fnd = wnd_scan(buf, wbit, wpos, wzbit, + to_alloc, &prev_tail, + &b_pos, &b_len); + if (fnd != MINUS_ONE_T) { + put_bh(bh); + goto found; + } + } + + prev_tail = 0; + + /* Scan range [zend, ebit) */ + if (wzend < wbits) { + fnd = wnd_scan(buf, wbit, + max(wzend, wpos), wbits, + to_alloc, &prev_tail, + &b_pos, &b_len); + if (fnd != MINUS_ONE_T) { + put_bh(bh); + goto found; + } + } + + wpos = 0; + put_bh(bh); + continue; + } + } + + /* Current window does not overlap zone */ + if (!wpos && fbits_valid && wnd->free_bits[iw] == wbits) { + /* window is empty */ + if (prev_tail + wbits >= to_alloc) { + fnd = wbit + wpos - prev_tail; + goto found; + } + + /* Increase 'prev_tail' and process next window */ + prev_tail += wbits; + wpos = 0; + continue; + } + + /* read window */ + bh = wnd_map(wnd, iw); + if (IS_ERR(bh)) { + // TODO: error + prev_tail = 0; + wpos = 0; + continue; + } + + buf = (ulong *)bh->b_data; + + /* Scan range [wpos, eBits) */ + fnd = wnd_scan(buf, wbit, wpos, wbits, to_alloc, &prev_tail, + &b_pos, &b_len); + put_bh(bh); + if (fnd != MINUS_ONE_T) + goto found; + } + + if (b_len < prev_tail) { + /* The last fragment */ + b_len = prev_tail; + b_pos = max_alloc - prev_tail; + } + + if (hint) { + /* + * We have scanned range [hint max_alloc) + * Prepare to scan range [0 hint + to_alloc) + */ + size_t nextmax = hint + to_alloc; + + if (likely(nextmax >= hint) && nextmax < max_alloc) + max_alloc = nextmax; + hint = 0; + goto Again; + } + + if (!b_len) + goto no_space; + + wnd->extent_max = b_len; + + if (flags & BITMAP_FIND_FULL) + goto no_space; + + fnd = b_pos; + to_alloc = b_len; + +found: + if (flags & BITMAP_FIND_MARK_AS_USED) { + /* TODO optimize remove extent (pass 'e'?) */ + if (wnd_set_used(wnd, fnd, to_alloc)) + goto no_space; + } else if (wnd->extent_max != MINUS_ONE_T && + to_alloc > wnd->extent_max) { + wnd->extent_max = to_alloc; + } + + *allocated = fnd; + return to_alloc; + +no_space: + return 0; +} + +/* + * wnd_extend + * + * Extend bitmap ($MFT bitmap) + */ +int wnd_extend(struct wnd_bitmap *wnd, size_t new_bits) +{ + int err; + struct super_block *sb = wnd->sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + u32 blocksize = sb->s_blocksize; + u32 wbits = blocksize * 8; + u32 b0, new_last; + size_t bits, iw, new_wnd; + size_t old_bits = wnd->nbits; + u16 *new_free; + + if (new_bits <= old_bits) + return -EINVAL; + + /* align to 8 byte boundary */ + new_wnd = bytes_to_block(sb, bitmap_size(new_bits)); + new_last = new_bits & (wbits - 1); + if (!new_last) + new_last = wbits; + + if (new_wnd != wnd->nwnd) { + new_free = ntfs_malloc(new_wnd * sizeof(u16)); + if (!new_free) + return -ENOMEM; + + if (new_free != wnd->free_bits) + memcpy(new_free, wnd->free_bits, + wnd->nwnd * sizeof(short)); + memset(new_free + wnd->nwnd, 0, + (new_wnd - wnd->nwnd) * sizeof(short)); + ntfs_free(wnd->free_bits); + wnd->free_bits = new_free; + } + + /* Zero bits [old_bits,new_bits) */ + bits = new_bits - old_bits; + b0 = old_bits & (wbits - 1); + + for (iw = old_bits >> (sb->s_blocksize_bits + 3); bits; iw += 1) { + u32 op; + size_t frb; + u64 vbo, lbo, bytes; + struct buffer_head *bh; + ulong *buf; + + if (iw + 1 == new_wnd) + wbits = new_last; + + op = b0 + bits > wbits ? wbits - b0 : bits; + vbo = (u64)iw * blocksize; + + err = ntfs_vbo_to_lbo(sbi, &wnd->run, vbo, &lbo, &bytes); + if (err) + break; + + bh = ntfs_bread(sb, lbo >> sb->s_blocksize_bits); + if (!bh) + return -EIO; + + lock_buffer(bh); + buf = (ulong *)bh->b_data; + + __bitmap_clear(buf, b0, blocksize * 8 - b0); + frb = wbits - __bitmap_weight(buf, wbits); + wnd->total_zeroes += frb - wnd->free_bits[iw]; + wnd->free_bits[iw] = frb; + + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); + unlock_buffer(bh); + /*err = sync_dirty_buffer(bh);*/ + + b0 = 0; + bits -= op; + } + + wnd->nbits = new_bits; + wnd->nwnd = new_wnd; + wnd->bits_last = new_last; + + wnd_add_free_ext(wnd, old_bits, new_bits - old_bits, false); + + return 0; +} + +/* + * wnd_zone_set + */ +void wnd_zone_set(struct wnd_bitmap *wnd, size_t lcn, size_t len) +{ + size_t zlen; + + zlen = wnd->zone_end - wnd->zone_bit; + if (zlen) + wnd_add_free_ext(wnd, wnd->zone_bit, zlen, false); + + if (!RB_EMPTY_ROOT(&wnd->start_tree) && len) + wnd_remove_free_ext(wnd, lcn, len); + + wnd->zone_bit = lcn; + wnd->zone_end = lcn + len; +} + +int ntfs_trim_fs(struct ntfs_sb_info *sbi, struct fstrim_range *range) +{ + int err = 0; + struct super_block *sb = sbi->sb; + struct wnd_bitmap *wnd = &sbi->used.bitmap; + u32 wbits = 8 * sb->s_blocksize; + CLST len = 0, lcn = 0, done = 0; + CLST minlen = bytes_to_cluster(sbi, range->minlen); + CLST lcn_from = bytes_to_cluster(sbi, range->start); + size_t iw = lcn_from >> (sb->s_blocksize_bits + 3); + u32 wbit = lcn_from & (wbits - 1); + const ulong *buf; + CLST lcn_to; + + if (!minlen) + minlen = 1; + + if (range->len == (u64)-1) + lcn_to = wnd->nbits; + else + lcn_to = bytes_to_cluster(sbi, range->start + range->len); + + down_read_nested(&wnd->rw_lock, BITMAP_MUTEX_CLUSTERS); + + for (; iw < wnd->nbits; iw++, wbit = 0) { + CLST lcn_wnd = iw * wbits; + struct buffer_head *bh; + + if (lcn_wnd > lcn_to) + break; + + if (!wnd->free_bits[iw]) + continue; + + if (iw + 1 == wnd->nwnd) + wbits = wnd->bits_last; + + if (lcn_wnd + wbits > lcn_to) + wbits = lcn_to - lcn_wnd; + + bh = wnd_map(wnd, iw); + if (IS_ERR(bh)) { + err = PTR_ERR(bh); + break; + } + + buf = (ulong *)bh->b_data; + + for (; wbit < wbits; wbit++) { + if (!test_bit(wbit, buf)) { + if (!len) + lcn = lcn_wnd + wbit; + len += 1; + continue; + } + if (len >= minlen) { + err = ntfs_discard(sbi, lcn, len); + if (err) + goto out; + done += len; + } + len = 0; + } + put_bh(bh); + } + + /* Process the last fragment */ + if (len >= minlen) { + err = ntfs_discard(sbi, lcn, len); + if (err) + goto out; + done += len; + } + +out: + range->len = (u64)done << sbi->cluster_bits; + + up_read(&wnd->rw_lock); + + return err; +} diff --git a/fs/ntfs3/debug.h b/fs/ntfs3/debug.h new file mode 100644 index 000000000000..dfaa4c79dc6d --- /dev/null +++ b/fs/ntfs3/debug.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + * useful functions for debuging + */ + +// clang-format off +#ifndef Add2Ptr +#define Add2Ptr(P, I) ((void *)((u8 *)(P) + (I))) +#define PtrOffset(B, O) ((size_t)((size_t)(O) - (size_t)(B))) +#endif + +#define QuadAlign(n) (((n) + 7u) & (~7u)) +#define IsQuadAligned(n) (!((size_t)(n)&7u)) +#define Quad2Align(n) (((n) + 15u) & (~15u)) +#define IsQuad2Aligned(n) (!((size_t)(n)&15u)) +#define Quad4Align(n) (((n) + 31u) & (~31u)) +#define IsSizeTAligned(n) (!((size_t)(n) & (sizeof(size_t) - 1))) +#define DwordAlign(n) (((n) + 3u) & (~3u)) +#define IsDwordAligned(n) (!((size_t)(n)&3u)) +#define WordAlign(n) (((n) + 1u) & (~1u)) +#define IsWordAligned(n) (!((size_t)(n)&1u)) + +#ifdef CONFIG_PRINTK +__printf(2, 3) +void ntfs_printk(const struct super_block *sb, const char *fmt, ...); +__printf(2, 3) +void ntfs_inode_printk(struct inode *inode, const char *fmt, ...); +#else +static inline __printf(2, 3) +void ntfs_printk(const struct super_block *sb, const char *fmt, ...) +{ +} + +static inline __printf(2, 3) +void ntfs_inode_printk(struct inode *inode, const char *fmt, ...) +{ +} +#endif + +/* + * Logging macros ( thanks Joe Perches for implementation ) + */ + +#define ntfs_err(sb, fmt, ...) ntfs_printk(sb, KERN_ERR fmt, ##__VA_ARGS__) +#define ntfs_warn(sb, fmt, ...) ntfs_printk(sb, KERN_WARNING fmt, ##__VA_ARGS__) +#define ntfs_info(sb, fmt, ...) ntfs_printk(sb, KERN_INFO fmt, ##__VA_ARGS__) +#define ntfs_notice(sb, fmt, ...) \ + ntfs_printk(sb, KERN_NOTICE fmt, ##__VA_ARGS__) + +#define ntfs_inode_err(inode, fmt, ...) \ + ntfs_inode_printk(inode, KERN_ERR fmt, ##__VA_ARGS__) +#define ntfs_inode_warn(inode, fmt, ...) \ + ntfs_inode_printk(inode, KERN_WARNING fmt, ##__VA_ARGS__) + +#define ntfs_malloc(s) kmalloc(s, GFP_NOFS) +#define ntfs_zalloc(s) kzalloc(s, GFP_NOFS) +#define ntfs_vmalloc(s) kvmalloc(s, GFP_KERNEL) +#define ntfs_free(p) kfree(p) +#define ntfs_vfree(p) kvfree(p) +#define ntfs_memdup(src, len) kmemdup(src, len, GFP_NOFS) +// clang-format on diff --git a/fs/ntfs3/dir.c b/fs/ntfs3/dir.c new file mode 100644 index 000000000000..9ec6012c405b --- /dev/null +++ b/fs/ntfs3/dir.c @@ -0,0 +1,594 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + * directory handling functions for ntfs-based filesystems + * + */ +#include +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +/* + * Convert little endian utf16 to nls string + */ +int ntfs_utf16_to_nls(struct ntfs_sb_info *sbi, const struct le_str *uni, + u8 *buf, int buf_len) +{ + int ret, uni_len, warn; + const __le16 *ip; + u8 *op; + struct nls_table *nls = sbi->options.nls; + + static_assert(sizeof(wchar_t) == sizeof(__le16)); + + if (!nls) { + /* utf16 -> utf8 */ + ret = utf16s_to_utf8s((wchar_t *)uni->name, uni->len, + UTF16_LITTLE_ENDIAN, buf, buf_len); + buf[ret] = '\0'; + return ret; + } + + ip = uni->name; + op = buf; + uni_len = uni->len; + warn = 0; + + while (uni_len--) { + u16 ec; + int charlen; + char dump[5]; + + if (buf_len < NLS_MAX_CHARSET_SIZE) { + ntfs_warn(sbi->sb, + "filename was truncated while converting."); + break; + } + + ec = le16_to_cpu(*ip++); + charlen = nls->uni2char(ec, op, buf_len); + + if (charlen > 0) { + op += charlen; + buf_len -= charlen; + continue; + } + + *op++ = '_'; + buf_len -= 1; + if (warn) + continue; + + warn = 1; + hex_byte_pack(&dump[0], ec >> 8); + hex_byte_pack(&dump[2], ec); + dump[4] = 0; + + ntfs_err(sbi->sb, "failed to convert \"%s\" to %s", dump, + nls->charset); + } + + *op = '\0'; + return op - buf; +} + +// clang-format off +#define PLANE_SIZE 0x00010000 + +#define SURROGATE_PAIR 0x0000d800 +#define SURROGATE_LOW 0x00000400 +#define SURROGATE_BITS 0x000003ff +// clang-format on + +/* + * modified version of put_utf16 from fs/nls/nls_base.c + * is sparse warnings free + */ +static inline void put_utf16(wchar_t *s, unsigned int c, + enum utf16_endian endian) +{ + static_assert(sizeof(wchar_t) == sizeof(__le16)); + static_assert(sizeof(wchar_t) == sizeof(__be16)); + + switch (endian) { + default: + *s = (wchar_t)c; + break; + case UTF16_LITTLE_ENDIAN: + *(__le16 *)s = __cpu_to_le16(c); + break; + case UTF16_BIG_ENDIAN: + *(__be16 *)s = __cpu_to_be16(c); + break; + } +} + +/* + * modified version of 'utf8s_to_utf16s' allows to + * detect -ENAMETOOLONG without writing out of expected maximum + */ +static int _utf8s_to_utf16s(const u8 *s, int inlen, enum utf16_endian endian, + wchar_t *pwcs, int maxout) +{ + u16 *op; + int size; + unicode_t u; + + op = pwcs; + while (inlen > 0 && *s) { + if (*s & 0x80) { + size = utf8_to_utf32(s, inlen, &u); + if (size < 0) + return -EINVAL; + s += size; + inlen -= size; + + if (u >= PLANE_SIZE) { + if (maxout < 2) + return -ENAMETOOLONG; + + u -= PLANE_SIZE; + put_utf16(op++, + SURROGATE_PAIR | + ((u >> 10) & SURROGATE_BITS), + endian); + put_utf16(op++, + SURROGATE_PAIR | SURROGATE_LOW | + (u & SURROGATE_BITS), + endian); + maxout -= 2; + } else { + if (maxout < 1) + return -ENAMETOOLONG; + + put_utf16(op++, u, endian); + maxout--; + } + } else { + if (maxout < 1) + return -ENAMETOOLONG; + + put_utf16(op++, *s++, endian); + inlen--; + maxout--; + } + } + return op - pwcs; +} + +/* + * Convert input string to utf16 + * + * name, name_len - input name + * uni, max_ulen - destination memory + * endian - endian of target utf16 string + * + * This function is called: + * - to create ntfs name + * - to create symlink + * + * returns utf16 string length or error (if negative) + */ +int ntfs_nls_to_utf16(struct ntfs_sb_info *sbi, const u8 *name, u32 name_len, + struct cpu_str *uni, u32 max_ulen, + enum utf16_endian endian) +{ + int ret, slen; + const u8 *end; + struct nls_table *nls = sbi->options.nls; + u16 *uname = uni->name; + + static_assert(sizeof(wchar_t) == sizeof(u16)); + + if (!nls) { + /* utf8 -> utf16 */ + ret = _utf8s_to_utf16s(name, name_len, endian, uname, max_ulen); + uni->len = ret; + return ret; + } + + for (ret = 0, end = name + name_len; name < end; ret++, name += slen) { + if (ret >= max_ulen) + return -ENAMETOOLONG; + + slen = nls->char2uni(name, end - name, uname + ret); + if (!slen) + return -EINVAL; + if (slen < 0) + return slen; + } + +#ifdef __BIG_ENDIAN + if (endian == UTF16_LITTLE_ENDIAN) { + int i = ret; + + while (i--) { + __cpu_to_le16s(uname); + uname++; + } + } +#else + if (endian == UTF16_BIG_ENDIAN) { + int i = ret; + + while (i--) { + __cpu_to_be16s(uname); + uname++; + } + } +#endif + + uni->len = ret; + return ret; +} + +/* helper function */ +struct inode *dir_search_u(struct inode *dir, const struct cpu_str *uni, + struct ntfs_fnd *fnd) +{ + int err = 0; + struct super_block *sb = dir->i_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + struct ntfs_inode *ni = ntfs_i(dir); + struct NTFS_DE *e; + int diff; + struct inode *inode = NULL; + struct ntfs_fnd *fnd_a = NULL; + + if (!fnd) { + fnd_a = fnd_get(); + if (!fnd_a) { + err = -ENOMEM; + goto out; + } + fnd = fnd_a; + } + + err = indx_find(&ni->dir, ni, NULL, uni, 0, sbi, &diff, &e, fnd); + + if (err) + goto out; + + if (diff) { + err = -ENOENT; + goto out; + } + + inode = ntfs_iget5(sb, &e->ref, uni); + if (!IS_ERR(inode) && is_bad_inode(inode)) { + iput(inode); + err = -EINVAL; + } +out: + fnd_put(fnd_a); + + return err == -ENOENT ? NULL : err ? ERR_PTR(err) : inode; +} + +static inline int ntfs_filldir(struct ntfs_sb_info *sbi, struct ntfs_inode *ni, + const struct NTFS_DE *e, u8 *name, + struct dir_context *ctx) +{ + const struct ATTR_FILE_NAME *fname; + unsigned long ino; + int name_len; + u32 dt_type; + + fname = Add2Ptr(e, sizeof(struct NTFS_DE)); + + if (fname->type == FILE_NAME_DOS) + return 0; + + if (!mi_is_ref(&ni->mi, &fname->home)) + return 0; + + ino = ino_get(&e->ref); + + if (ino == MFT_REC_ROOT) + return 0; + + /* Skip meta files ( unless option to show metafiles is set ) */ + if (!sbi->options.showmeta && ntfs_is_meta_file(sbi, ino)) + return 0; + + if (sbi->options.nohidden && (fname->dup.fa & FILE_ATTRIBUTE_HIDDEN)) + return 0; + + name_len = ntfs_utf16_to_nls(sbi, (struct le_str *)&fname->name_len, + name, PATH_MAX); + if (name_len <= 0) { + ntfs_warn(sbi->sb, "failed to convert name for inode %lx.", + ino); + return 0; + } + + dt_type = (fname->dup.fa & FILE_ATTRIBUTE_DIRECTORY) ? DT_DIR : DT_REG; + + return !dir_emit(ctx, (s8 *)name, name_len, ino, dt_type); +} + +/* + * ntfs_read_hdr + * + * helper function 'ntfs_readdir' + */ +static int ntfs_read_hdr(struct ntfs_sb_info *sbi, struct ntfs_inode *ni, + const struct INDEX_HDR *hdr, u64 vbo, u64 pos, + u8 *name, struct dir_context *ctx) +{ + int err; + const struct NTFS_DE *e; + u32 e_size; + u32 end = le32_to_cpu(hdr->used); + u32 off = le32_to_cpu(hdr->de_off); + + for (;; off += e_size) { + if (off + sizeof(struct NTFS_DE) > end) + return -1; + + e = Add2Ptr(hdr, off); + e_size = le16_to_cpu(e->size); + if (e_size < sizeof(struct NTFS_DE) || off + e_size > end) + return -1; + + if (de_is_last(e)) + return 0; + + /* Skip already enumerated*/ + if (vbo + off < pos) + continue; + + if (le16_to_cpu(e->key_size) < SIZEOF_ATTRIBUTE_FILENAME) + return -1; + + ctx->pos = vbo + off; + + /* Submit the name to the filldir callback. */ + err = ntfs_filldir(sbi, ni, e, name, ctx); + if (err) + return err; + } +} + +/* + * file_operations::iterate_shared + * + * Use non sorted enumeration. + * We have an example of broken volume where sorted enumeration + * counts each name twice + */ +static int ntfs_readdir(struct file *file, struct dir_context *ctx) +{ + const struct INDEX_ROOT *root; + u64 vbo; + size_t bit; + loff_t eod; + int err = 0; + struct inode *dir = file_inode(file); + struct ntfs_inode *ni = ntfs_i(dir); + struct super_block *sb = dir->i_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + loff_t i_size = i_size_read(dir); + u32 pos = ctx->pos; + u8 *name = NULL; + struct indx_node *node = NULL; + u8 index_bits = ni->dir.index_bits; + + /* name is a buffer of PATH_MAX length */ + static_assert(NTFS_NAME_LEN * 4 < PATH_MAX); + + eod = i_size + sbi->record_size; + + if (pos >= eod) + return 0; + + if (!dir_emit_dots(file, ctx)) + return 0; + + /* allocate PATH_MAX bytes */ + name = __getname(); + if (!name) + return -ENOMEM; + + if (!ni->mi_loaded && ni->attr_list.size) { + /* + * directory inode is locked for read + * load all subrecords to avoid 'write' access to 'ni' during + * directory reading + */ + ni_lock(ni); + if (!ni->mi_loaded && ni->attr_list.size) { + err = ni_load_all_mi(ni); + if (!err) + ni->mi_loaded = true; + } + ni_unlock(ni); + if (err) + goto out; + } + + root = indx_get_root(&ni->dir, ni, NULL, NULL); + if (!root) { + err = -EINVAL; + goto out; + } + + if (pos >= sbi->record_size) { + bit = (pos - sbi->record_size) >> index_bits; + } else { + err = ntfs_read_hdr(sbi, ni, &root->ihdr, 0, pos, name, ctx); + if (err) + goto out; + bit = 0; + } + + if (!i_size) { + ctx->pos = eod; + goto out; + } + + for (;;) { + vbo = (u64)bit << index_bits; + if (vbo >= i_size) { + ctx->pos = eod; + goto out; + } + + err = indx_used_bit(&ni->dir, ni, &bit); + if (err) + goto out; + + if (bit == MINUS_ONE_T) { + ctx->pos = eod; + goto out; + } + + vbo = (u64)bit << index_bits; + if (vbo >= i_size) { + ntfs_inode_err(dir, "Looks like your dir is corrupt"); + err = -EINVAL; + goto out; + } + + err = indx_read(&ni->dir, ni, bit << ni->dir.idx2vbn_bits, + &node); + if (err) + goto out; + + err = ntfs_read_hdr(sbi, ni, &node->index->ihdr, + vbo + sbi->record_size, pos, name, ctx); + if (err) + goto out; + + bit += 1; + } + +out: + + __putname(name); + put_indx_node(node); + + if (err == -ENOENT) { + err = 0; + ctx->pos = pos; + } + + return err; +} + +static int ntfs_dir_count(struct inode *dir, bool *is_empty, size_t *dirs, + size_t *files) +{ + int err = 0; + struct ntfs_inode *ni = ntfs_i(dir); + struct NTFS_DE *e = NULL; + struct INDEX_ROOT *root; + struct INDEX_HDR *hdr; + const struct ATTR_FILE_NAME *fname; + u32 e_size, off, end; + u64 vbo = 0; + size_t drs = 0, fles = 0, bit = 0; + loff_t i_size = ni->vfs_inode.i_size; + struct indx_node *node = NULL; + u8 index_bits = ni->dir.index_bits; + + if (is_empty) + *is_empty = true; + + root = indx_get_root(&ni->dir, ni, NULL, NULL); + if (!root) + return -EINVAL; + + hdr = &root->ihdr; + + for (;;) { + end = le32_to_cpu(hdr->used); + off = le32_to_cpu(hdr->de_off); + + for (; off + sizeof(struct NTFS_DE) <= end; off += e_size) { + e = Add2Ptr(hdr, off); + e_size = le16_to_cpu(e->size); + if (e_size < sizeof(struct NTFS_DE) || + off + e_size > end) + break; + + if (de_is_last(e)) + break; + + fname = de_get_fname(e); + if (!fname) + continue; + + if (fname->type == FILE_NAME_DOS) + continue; + + if (is_empty) { + *is_empty = false; + if (!dirs && !files) + goto out; + } + + if (fname->dup.fa & FILE_ATTRIBUTE_DIRECTORY) + drs += 1; + else + fles += 1; + } + + if (vbo >= i_size) + goto out; + + err = indx_used_bit(&ni->dir, ni, &bit); + if (err) + goto out; + + if (bit == MINUS_ONE_T) + goto out; + + vbo = (u64)bit << index_bits; + if (vbo >= i_size) + goto out; + + err = indx_read(&ni->dir, ni, bit << ni->dir.idx2vbn_bits, + &node); + if (err) + goto out; + + hdr = &node->index->ihdr; + bit += 1; + vbo = (u64)bit << ni->dir.idx2vbn_bits; + } + +out: + put_indx_node(node); + if (dirs) + *dirs = drs; + if (files) + *files = fles; + + return err; +} + +bool dir_is_empty(struct inode *dir) +{ + bool is_empty = false; + + ntfs_dir_count(dir, &is_empty, NULL, NULL); + + return is_empty; +} + +const struct file_operations ntfs_dir_operations = { + .llseek = generic_file_llseek, + .read = generic_read_dir, + .iterate_shared = ntfs_readdir, + .fsync = generic_file_fsync, + .open = ntfs_file_open, +}; diff --git a/fs/ntfs3/file.c b/fs/ntfs3/file.c new file mode 100644 index 000000000000..347baf674008 --- /dev/null +++ b/fs/ntfs3/file.c @@ -0,0 +1,1130 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + * regular file handling primitives for ntfs-based filesystems + */ +#include +#include +#include +#include +#include +#include /* FAT_IOCTL_XXX */ +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +static int ntfs_ioctl_fitrim(struct ntfs_sb_info *sbi, unsigned long arg) +{ + struct fstrim_range __user *user_range; + struct fstrim_range range; + struct request_queue *q = bdev_get_queue(sbi->sb->s_bdev); + int err; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (!blk_queue_discard(q)) + return -EOPNOTSUPP; + + user_range = (struct fstrim_range __user *)arg; + if (copy_from_user(&range, user_range, sizeof(range))) + return -EFAULT; + + range.minlen = max_t(u32, range.minlen, q->limits.discard_granularity); + + err = ntfs_trim_fs(sbi, &range); + if (err < 0) + return err; + + if (copy_to_user(user_range, &range, sizeof(range))) + return -EFAULT; + + return 0; +} + +static long ntfs_ioctl(struct file *filp, u32 cmd, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct ntfs_sb_info *sbi = inode->i_sb->s_fs_info; + u32 __user *user_attr = (u32 __user *)arg; + + switch (cmd) { + case FAT_IOCTL_GET_ATTRIBUTES: + return put_user(le32_to_cpu(ntfs_i(inode)->std_fa), user_attr); + + case FAT_IOCTL_GET_VOLUME_ID: + return put_user(sbi->volume.ser_num, user_attr); + + case FITRIM: + return ntfs_ioctl_fitrim(sbi, arg); + } + return -ENOTTY; /* Inappropriate ioctl for device */ +} + +#ifdef CONFIG_COMPAT +static long ntfs_compat_ioctl(struct file *filp, u32 cmd, unsigned long arg) + +{ + return ntfs_ioctl(filp, cmd, (unsigned long)compat_ptr(arg)); +} +#endif + +/* + * inode_operations::getattr + */ +int ntfs_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, u32 flags) +{ + struct inode *inode = d_inode(path->dentry); + struct ntfs_inode *ni = ntfs_i(inode); + + if (is_compressed(ni)) + stat->attributes |= STATX_ATTR_COMPRESSED; + + if (is_encrypted(ni)) + stat->attributes |= STATX_ATTR_ENCRYPTED; + + stat->attributes_mask |= STATX_ATTR_COMPRESSED | STATX_ATTR_ENCRYPTED; + + generic_fillattr(mnt_userns, inode, stat); + + stat->result_mask |= STATX_BTIME; + stat->btime = ni->i_crtime; + + return 0; +} + +static int ntfs_extend_initialized_size(struct file *file, + struct ntfs_inode *ni, + const loff_t valid, + const loff_t new_valid) +{ + struct inode *inode = &ni->vfs_inode; + struct address_space *mapping = inode->i_mapping; + struct ntfs_sb_info *sbi = inode->i_sb->s_fs_info; + loff_t pos = valid; + int err; + + if (is_resident(ni)) { + ni->i_valid = new_valid; + return 0; + } + + WARN_ON(is_compressed(ni)); + WARN_ON(valid >= new_valid); + + for (;;) { + u32 zerofrom, len; + struct page *page; + void *fsdata; + u8 bits; + CLST vcn, lcn, clen; + + if (is_sparsed(ni)) { + bits = sbi->cluster_bits; + vcn = pos >> bits; + + err = attr_data_get_block(ni, vcn, 0, &lcn, &clen, + NULL); + if (err) + goto out; + + if (lcn == SPARSE_LCN) { + loff_t vbo = (loff_t)vcn << bits; + loff_t to = vbo + ((loff_t)clen << bits); + + if (to <= new_valid) { + ni->i_valid = to; + pos = to; + goto next; + } + + if (vbo < pos) { + pos = vbo; + } else { + to = (new_valid >> bits) << bits; + if (pos < to) { + ni->i_valid = to; + pos = to; + goto next; + } + } + } + } + + zerofrom = pos & (PAGE_SIZE - 1); + len = PAGE_SIZE - zerofrom; + + if (pos + len > new_valid) + len = new_valid - pos; + + err = pagecache_write_begin(file, mapping, pos, len, 0, &page, + &fsdata); + if (err) + goto out; + + zero_user_segment(page, zerofrom, PAGE_SIZE); + + /* this function in any case puts page*/ + err = pagecache_write_end(file, mapping, pos, len, len, page, + fsdata); + if (err < 0) + goto out; + pos += len; + +next: + if (pos >= new_valid) + break; + + balance_dirty_pages_ratelimited(mapping); + cond_resched(); + } + + mark_inode_dirty(inode); + + return 0; + +out: + ni->i_valid = valid; + ntfs_inode_warn(inode, "failed to extend initialized size to %llx.", + new_valid); + return err; +} + +/* + * ntfs_sparse_cluster + * + * Helper function to zero a new allocated clusters + */ +void ntfs_sparse_cluster(struct inode *inode, struct page *page0, CLST vcn, + CLST len) +{ + struct address_space *mapping = inode->i_mapping; + struct ntfs_sb_info *sbi = inode->i_sb->s_fs_info; + u64 vbo = (u64)vcn << sbi->cluster_bits; + u64 bytes = (u64)len << sbi->cluster_bits; + u32 blocksize = 1 << inode->i_blkbits; + pgoff_t idx0 = page0 ? page0->index : -1; + loff_t vbo_clst = vbo & sbi->cluster_mask_inv; + loff_t end = ntfs_up_cluster(sbi, vbo + bytes); + pgoff_t idx = vbo_clst >> PAGE_SHIFT; + u32 from = vbo_clst & (PAGE_SIZE - 1); + pgoff_t idx_end = (end + PAGE_SIZE - 1) >> PAGE_SHIFT; + loff_t page_off; + u32 to; + bool partial; + struct page *page; + + for (; idx < idx_end; idx += 1, from = 0) { + page = idx == idx0 ? page0 : grab_cache_page(mapping, idx); + + if (!page) + continue; + + page_off = (loff_t)idx << PAGE_SHIFT; + to = (page_off + PAGE_SIZE) > end ? (end - page_off) + : PAGE_SIZE; + partial = false; + + if ((from || PAGE_SIZE != to) && + likely(!page_has_buffers(page))) { + create_empty_buffers(page, blocksize, 0); + if (!page_has_buffers(page)) { + ntfs_inode_err( + inode, + "failed to allocate page buffers."); + /*err = -ENOMEM;*/ + goto unlock_page; + } + } + + if (page_has_buffers(page)) { + struct buffer_head *head, *bh; + u32 bh_off = 0; + + bh = head = page_buffers(page); + do { + u32 bh_next = bh_off + blocksize; + + if (from <= bh_off && bh_next <= to) { + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); + } else if (!buffer_uptodate(bh)) { + partial = true; + } + bh_off = bh_next; + } while (head != (bh = bh->b_this_page)); + } + + zero_user_segment(page, from, to); + + if (!partial) { + if (!PageUptodate(page)) + SetPageUptodate(page); + set_page_dirty(page); + } + +unlock_page: + if (idx != idx0) { + unlock_page(page); + put_page(page); + } + cond_resched(); + } + mark_inode_dirty(inode); +} + +/* + * file_operations::mmap + */ +static int ntfs_file_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct address_space *mapping = file->f_mapping; + struct inode *inode = mapping->host; + struct ntfs_inode *ni = ntfs_i(inode); + u64 to, from = ((u64)vma->vm_pgoff << PAGE_SHIFT); + bool rw = vma->vm_flags & VM_WRITE; + int err; + + if (is_encrypted(ni)) { + ntfs_inode_warn(inode, + "mmap is not supported for encrypted files"); + err = -EOPNOTSUPP; + goto out; + } + + if (!rw) + goto do_map; + + if (is_compressed(ni)) { + ntfs_inode_warn( + inode, + "mmap(write) is not supported for compressed files"); + err = -EOPNOTSUPP; + goto out; + } + + to = min_t(loff_t, i_size_read(inode), + from + vma->vm_end - vma->vm_start); + + if (is_sparsed(ni)) { + /* allocate clusters for rw map */ + struct ntfs_sb_info *sbi = inode->i_sb->s_fs_info; + CLST vcn, lcn, len; + CLST end = bytes_to_cluster(sbi, to); + bool new; + + for (vcn = from >> sbi->cluster_bits; vcn < end; vcn += len) { + err = attr_data_get_block(ni, vcn, 1, &lcn, &len, &new); + if (err) + goto out; + if (!new) + continue; + ntfs_sparse_cluster(inode, NULL, vcn, 1); + } + } + + if (ni->i_valid < to) { + inode_lock(inode); + err = ntfs_extend_initialized_size(file, ni, ni->i_valid, to); + inode_unlock(inode); + if (err) + goto out; + } + +do_map: + err = generic_file_mmap(file, vma); +out: + return err; +} + +static int ntfs_extend(struct inode *inode, loff_t pos, size_t count, + struct file *file) +{ + struct ntfs_inode *ni = ntfs_i(inode); + struct address_space *mapping = inode->i_mapping; + loff_t end = pos + count; + bool extend_init = file && pos > ni->i_valid; + int err; + + if (end <= inode->i_size && !extend_init) + return 0; + + /*mark rw ntfs as dirty. it will be cleared at umount*/ + ntfs_set_state(ni->mi.sbi, NTFS_DIRTY_DIRTY); + + if (end > inode->i_size) { + err = ntfs_set_size(inode, end); + if (err) + goto out; + inode->i_size = end; + } + + if (extend_init && !is_compressed(ni)) { + err = ntfs_extend_initialized_size(file, ni, ni->i_valid, pos); + if (err) + goto out; + } else { + err = 0; + } + + inode->i_ctime = inode->i_mtime = current_time(inode); + mark_inode_dirty(inode); + + if (IS_SYNC(inode)) { + int err2; + + err = filemap_fdatawrite_range(mapping, pos, end - 1); + err2 = sync_mapping_buffers(mapping); + if (!err) + err = err2; + err2 = write_inode_now(inode, 1); + if (!err) + err = err2; + if (!err) + err = filemap_fdatawait_range(mapping, pos, end - 1); + } + +out: + return err; +} + +static int ntfs_truncate(struct inode *inode, loff_t new_size) +{ + struct super_block *sb = inode->i_sb; + struct ntfs_inode *ni = ntfs_i(inode); + int err, dirty = 0; + u64 new_valid; + + if (!S_ISREG(inode->i_mode)) + return 0; + + if (is_compressed(ni)) { + if (ni->i_valid > new_size) + ni->i_valid = new_size; + } else { + err = block_truncate_page(inode->i_mapping, new_size, + ntfs_get_block); + if (err) + return err; + } + + new_valid = ntfs_up_block(sb, min_t(u64, ni->i_valid, new_size)); + + ni_lock(ni); + + truncate_setsize(inode, new_size); + + down_write(&ni->file.run_lock); + err = attr_set_size(ni, ATTR_DATA, NULL, 0, &ni->file.run, new_size, + &new_valid, true, NULL); + up_write(&ni->file.run_lock); + + if (new_valid < ni->i_valid) + ni->i_valid = new_valid; + + ni_unlock(ni); + + ni->std_fa |= FILE_ATTRIBUTE_ARCHIVE; + inode->i_ctime = inode->i_mtime = current_time(inode); + if (!IS_DIRSYNC(inode)) { + dirty = 1; + } else { + err = ntfs_sync_inode(inode); + if (err) + return err; + } + + if (dirty) + mark_inode_dirty(inode); + + /*ntfs_flush_inodes(inode->i_sb, inode, NULL);*/ + + return 0; +} + +/* + * Preallocate space for a file. This implements ntfs's fallocate file + * operation, which gets called from sys_fallocate system call. User + * space requests 'len' bytes at 'vbo'. If FALLOC_FL_KEEP_SIZE is set + * we just allocate clusters without zeroing them out. Otherwise we + * allocate and zero out clusters via an expanding truncate. + */ +static long ntfs_fallocate(struct file *file, int mode, loff_t vbo, loff_t len) +{ + struct inode *inode = file->f_mapping->host; + struct super_block *sb = inode->i_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + struct ntfs_inode *ni = ntfs_i(inode); + loff_t end = vbo + len; + loff_t vbo_down = round_down(vbo, PAGE_SIZE); + loff_t i_size; + int err; + + /* No support for dir */ + if (!S_ISREG(inode->i_mode)) + return -EOPNOTSUPP; + + /* Return error if mode is not supported */ + if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE | + FALLOC_FL_COLLAPSE_RANGE)) + return -EOPNOTSUPP; + + ntfs_set_state(sbi, NTFS_DIRTY_DIRTY); + + inode_lock(inode); + i_size = inode->i_size; + + if (WARN_ON(ni->ni_flags & NI_FLAG_COMPRESSED_MASK)) { + /* should never be here, see ntfs_file_open*/ + err = -EOPNOTSUPP; + goto out; + } + + if (mode & FALLOC_FL_PUNCH_HOLE) { + if (!(mode & FALLOC_FL_KEEP_SIZE)) { + err = -EINVAL; + goto out; + } + + if (!is_sparsed(ni) && !is_compressed(ni)) { + ntfs_inode_warn( + inode, + "punch_hole only for sparsed/compressed files"); + err = -EOPNOTSUPP; + goto out; + } + + err = filemap_write_and_wait_range(inode->i_mapping, vbo, + end - 1); + if (err) + goto out; + + err = filemap_write_and_wait_range(inode->i_mapping, end, + LLONG_MAX); + if (err) + goto out; + + truncate_pagecache(inode, vbo_down); + + ni_lock(ni); + err = attr_punch_hole(ni, vbo, len); + ni_unlock(ni); + } else if (mode & FALLOC_FL_COLLAPSE_RANGE) { + if (mode & ~FALLOC_FL_COLLAPSE_RANGE) { + err = -EINVAL; + goto out; + } + + /* + * Write tail of the last page before removed range since + * it will get removed from the page cache below. + */ + err = filemap_write_and_wait_range(inode->i_mapping, vbo_down, + vbo); + if (err) + goto out; + + /* + * Write data that will be shifted to preserve them + * when discarding page cache below + */ + err = filemap_write_and_wait_range(inode->i_mapping, end, + LLONG_MAX); + if (err) + goto out; + + truncate_pagecache(inode, vbo_down); + + ni_lock(ni); + err = attr_collapse_range(ni, vbo, len); + ni_unlock(ni); + } else { + /* + * normal file: allocate clusters, do not change 'valid' size + */ + err = ntfs_set_size(inode, max(end, i_size)); + if (err) + goto out; + + if (is_sparsed(ni) || is_compressed(ni)) { + CLST vcn_v = ni->i_valid >> sbi->cluster_bits; + CLST vcn = vbo >> sbi->cluster_bits; + CLST cend = bytes_to_cluster(sbi, end); + CLST lcn, clen; + bool new; + + /* + * allocate but not zero new clusters (see below comments) + * this breaks security (one can read unused on-disk areas) + * zeroing these clusters may be too long + * may be we should check here for root rights? + */ + for (; vcn < cend; vcn += clen) { + err = attr_data_get_block(ni, vcn, cend - vcn, + &lcn, &clen, &new); + if (err) + goto out; + if (!new || vcn >= vcn_v) + continue; + + /* + * Unwritten area + * NTFS is not able to store several unwritten areas + * Activate 'ntfs_sparse_cluster' to zero new allocated clusters + * + * Dangerous in case: + * 1G of sparsed clusters + 1 cluster of data => + * valid_size == 1G + 1 cluster + * fallocate(1G) will zero 1G and this can be very long + * xfstest 016/086 will fail without 'ntfs_sparse_cluster' + */ + /*ntfs_sparse_cluster(inode, NULL, vcn, + * min(vcn_v - vcn, clen)); + */ + } + } + + if (mode & FALLOC_FL_KEEP_SIZE) { + ni_lock(ni); + /*true - keep preallocated*/ + err = attr_set_size(ni, ATTR_DATA, NULL, 0, + &ni->file.run, i_size, &ni->i_valid, + true, NULL); + ni_unlock(ni); + } + } + + if (!err) { + inode->i_ctime = inode->i_mtime = current_time(inode); + mark_inode_dirty(inode); + } +out: + if (err == -EFBIG) + err = -ENOSPC; + + inode_unlock(inode); + return err; +} + +/* + * inode_operations::setattr + */ +int ntfs3_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr) +{ + struct super_block *sb = dentry->d_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + struct inode *inode = d_inode(dentry); + struct ntfs_inode *ni = ntfs_i(inode); + u32 ia_valid = attr->ia_valid; + umode_t mode = inode->i_mode; + int err; + + if (sbi->options.no_acs_rules) { + /* "no access rules" - force any changes of time etc. */ + attr->ia_valid |= ATTR_FORCE; + /* and disable for editing some attributes */ + attr->ia_valid &= ~(ATTR_UID | ATTR_GID | ATTR_MODE); + ia_valid = attr->ia_valid; + } + + err = setattr_prepare(mnt_userns, dentry, attr); + if (err) + goto out; + + if (ia_valid & ATTR_SIZE) { + loff_t oldsize = inode->i_size; + + if (WARN_ON(ni->ni_flags & NI_FLAG_COMPRESSED_MASK)) { + /* should never be here, see ntfs_file_open*/ + err = -EOPNOTSUPP; + goto out; + } + inode_dio_wait(inode); + + if (attr->ia_size < oldsize) + err = ntfs_truncate(inode, attr->ia_size); + else if (attr->ia_size > oldsize) + err = ntfs_extend(inode, attr->ia_size, 0, NULL); + + if (err) + goto out; + + ni->ni_flags |= NI_FLAG_UPDATE_PARENT; + } + + setattr_copy(mnt_userns, inode, attr); + + if (mode != inode->i_mode) { + err = ntfs_acl_chmod(mnt_userns, inode); + if (err) + goto out; + + /* linux 'w' -> windows 'ro' */ + if (0222 & inode->i_mode) + ni->std_fa &= ~FILE_ATTRIBUTE_READONLY; + else + ni->std_fa |= FILE_ATTRIBUTE_READONLY; + } + + mark_inode_dirty(inode); +out: + return err; +} + +static ssize_t ntfs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) +{ + ssize_t err; + size_t count = iov_iter_count(iter); + struct file *file = iocb->ki_filp; + struct inode *inode = file->f_mapping->host; + struct ntfs_inode *ni = ntfs_i(inode); + + if (is_encrypted(ni)) { + ntfs_inode_warn(inode, "encrypted i/o not supported"); + return -EOPNOTSUPP; + } + + if (is_compressed(ni) && (iocb->ki_flags & IOCB_DIRECT)) { + ntfs_inode_warn(inode, "direct i/o + compressed not supported"); + return -EOPNOTSUPP; + } + +#ifndef CONFIG_NTFS3_LZX_XPRESS + if (ni->ni_flags & NI_FLAG_COMPRESSED_MASK) { + ntfs_inode_warn( + inode, + "activate CONFIG_NTFS3_LZX_XPRESS to read external compressed files"); + return -EOPNOTSUPP; + } +#endif + + if (is_dedup(ni)) { + ntfs_inode_warn(inode, "read deduplicated not supported"); + return -EOPNOTSUPP; + } + + err = count ? generic_file_read_iter(iocb, iter) : 0; + + return err; +} + +/* returns array of locked pages */ +static int ntfs_get_frame_pages(struct address_space *mapping, pgoff_t index, + struct page **pages, u32 pages_per_frame, + bool *frame_uptodate) +{ + gfp_t gfp_mask = mapping_gfp_mask(mapping); + u32 npages; + + *frame_uptodate = true; + + for (npages = 0; npages < pages_per_frame; npages++, index++) { + struct page *page; + + page = find_or_create_page(mapping, index, gfp_mask); + if (!page) { + while (npages--) { + page = pages[npages]; + unlock_page(page); + put_page(page); + } + + return -ENOMEM; + } + + if (!PageUptodate(page)) + *frame_uptodate = false; + + pages[npages] = page; + } + + return 0; +} + +/*helper for ntfs_file_write_iter (compressed files)*/ +static ssize_t ntfs_compress_write(struct kiocb *iocb, struct iov_iter *from) +{ + int err; + struct file *file = iocb->ki_filp; + size_t count = iov_iter_count(from); + loff_t pos = iocb->ki_pos; + struct inode *inode = file_inode(file); + loff_t i_size = inode->i_size; + struct address_space *mapping = inode->i_mapping; + struct ntfs_inode *ni = ntfs_i(inode); + u64 valid = ni->i_valid; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct page *page, **pages = NULL; + size_t written = 0; + u8 frame_bits = NTFS_LZNT_CUNIT + sbi->cluster_bits; + u32 frame_size = 1u << frame_bits; + u32 pages_per_frame = frame_size >> PAGE_SHIFT; + u32 ip, off; + CLST frame; + u64 frame_vbo; + pgoff_t index; + bool frame_uptodate; + + if (frame_size < PAGE_SIZE) { + /* + * frame_size == 8K if cluster 512 + * frame_size == 64K if cluster 4096 + */ + ntfs_inode_warn(inode, "page size is bigger than frame size"); + return -EOPNOTSUPP; + } + + pages = ntfs_malloc(pages_per_frame * sizeof(struct page *)); + if (!pages) + return -ENOMEM; + + current->backing_dev_info = inode_to_bdi(inode); + err = file_remove_privs(file); + if (err) + goto out; + + err = file_update_time(file); + if (err) + goto out; + + /* zero range [valid : pos) */ + while (valid < pos) { + CLST lcn, clen; + + frame = valid >> frame_bits; + frame_vbo = valid & ~(frame_size - 1); + off = valid & (frame_size - 1); + + err = attr_data_get_block(ni, frame << NTFS_LZNT_CUNIT, 0, &lcn, + &clen, NULL); + if (err) + goto out; + + if (lcn == SPARSE_LCN) { + ni->i_valid = valid = + frame_vbo + ((u64)clen << sbi->cluster_bits); + continue; + } + + /* Load full frame */ + err = ntfs_get_frame_pages(mapping, frame_vbo >> PAGE_SHIFT, + pages, pages_per_frame, + &frame_uptodate); + if (err) + goto out; + + if (!frame_uptodate && off) { + err = ni_read_frame(ni, frame_vbo, pages, + pages_per_frame); + if (err) { + for (ip = 0; ip < pages_per_frame; ip++) { + page = pages[ip]; + unlock_page(page); + put_page(page); + } + goto out; + } + } + + ip = off >> PAGE_SHIFT; + off = offset_in_page(valid); + for (; ip < pages_per_frame; ip++, off = 0) { + page = pages[ip]; + zero_user_segment(page, off, PAGE_SIZE); + flush_dcache_page(page); + SetPageUptodate(page); + } + + ni_lock(ni); + err = ni_write_frame(ni, pages, pages_per_frame); + ni_unlock(ni); + + for (ip = 0; ip < pages_per_frame; ip++) { + page = pages[ip]; + SetPageUptodate(page); + unlock_page(page); + put_page(page); + } + + if (err) + goto out; + + ni->i_valid = valid = frame_vbo + frame_size; + } + + /* copy user data [pos : pos + count) */ + while (count) { + size_t copied, bytes; + + off = pos & (frame_size - 1); + bytes = frame_size - off; + if (bytes > count) + bytes = count; + + frame = pos >> frame_bits; + frame_vbo = pos & ~(frame_size - 1); + index = frame_vbo >> PAGE_SHIFT; + + if (unlikely(iov_iter_fault_in_readable(from, bytes))) { + err = -EFAULT; + goto out; + } + + /* Load full frame */ + err = ntfs_get_frame_pages(mapping, index, pages, + pages_per_frame, &frame_uptodate); + if (err) + goto out; + + if (!frame_uptodate) { + loff_t to = pos + bytes; + + if (off || (to < i_size && (to & (frame_size - 1)))) { + err = ni_read_frame(ni, frame_vbo, pages, + pages_per_frame); + if (err) { + for (ip = 0; ip < pages_per_frame; + ip++) { + page = pages[ip]; + unlock_page(page); + put_page(page); + } + goto out; + } + } + } + + WARN_ON(!bytes); + copied = 0; + ip = off >> PAGE_SHIFT; + off = offset_in_page(pos); + + /* copy user data to pages */ + for (;;) { + size_t cp, tail = PAGE_SIZE - off; + + page = pages[ip]; + cp = iov_iter_copy_from_user_atomic(page, from, off, + min(tail, bytes)); + flush_dcache_page(page); + iov_iter_advance(from, cp); + copied += cp; + bytes -= cp; + if (!bytes || !cp) + break; + + if (cp < tail) { + off += cp; + } else { + ip++; + off = 0; + } + } + + ni_lock(ni); + err = ni_write_frame(ni, pages, pages_per_frame); + ni_unlock(ni); + + for (ip = 0; ip < pages_per_frame; ip++) { + page = pages[ip]; + ClearPageDirty(page); + SetPageUptodate(page); + unlock_page(page); + put_page(page); + } + + if (err) + goto out; + + /* + * We can loop for a long time in here. Be nice and allow + * us to schedule out to avoid softlocking if preempt + * is disabled. + */ + cond_resched(); + + pos += copied; + written += copied; + + count = iov_iter_count(from); + } + +out: + ntfs_free(pages); + + current->backing_dev_info = NULL; + + if (err < 0) + return err; + + iocb->ki_pos += written; + if (iocb->ki_pos > ni->i_valid) + ni->i_valid = iocb->ki_pos; + + return written; +} + +/* + * file_operations::write_iter + */ +static ssize_t ntfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + struct file *file = iocb->ki_filp; + struct address_space *mapping = file->f_mapping; + struct inode *inode = mapping->host; + ssize_t ret; + struct ntfs_inode *ni = ntfs_i(inode); + + if (is_encrypted(ni)) { + ntfs_inode_warn(inode, "encrypted i/o not supported"); + return -EOPNOTSUPP; + } + + if (is_compressed(ni) && (iocb->ki_flags & IOCB_DIRECT)) { + ntfs_inode_warn(inode, "direct i/o + compressed not supported"); + return -EOPNOTSUPP; + } + + if (is_dedup(ni)) { + ntfs_inode_warn(inode, "write into deduplicated not supported"); + return -EOPNOTSUPP; + } + + if (!inode_trylock(inode)) { + if (iocb->ki_flags & IOCB_NOWAIT) + return -EAGAIN; + inode_lock(inode); + } + + ret = generic_write_checks(iocb, from); + if (ret <= 0) + goto out; + + if (WARN_ON(ni->ni_flags & NI_FLAG_COMPRESSED_MASK)) { + /* should never be here, see ntfs_file_open*/ + ret = -EOPNOTSUPP; + goto out; + } + + ret = ntfs_extend(inode, iocb->ki_pos, ret, file); + if (ret) + goto out; + + ret = is_compressed(ni) ? ntfs_compress_write(iocb, from) + : __generic_file_write_iter(iocb, from); + +out: + inode_unlock(inode); + + if (ret > 0) + ret = generic_write_sync(iocb, ret); + + return ret; +} + +/* + * file_operations::open + */ +int ntfs_file_open(struct inode *inode, struct file *file) +{ + struct ntfs_inode *ni = ntfs_i(inode); + + if (unlikely((is_compressed(ni) || is_encrypted(ni)) && + (file->f_flags & O_DIRECT))) { + return -EOPNOTSUPP; + } + + /* Decompress "external compressed" file if opened for rw */ + if ((ni->ni_flags & NI_FLAG_COMPRESSED_MASK) && + (file->f_flags & (O_WRONLY | O_RDWR | O_TRUNC))) { +#ifdef CONFIG_NTFS3_LZX_XPRESS + int err = ni_decompress_file(ni); + + if (err) + return err; +#else + ntfs_inode_warn( + inode, + "activate CONFIG_NTFS3_LZX_XPRESS to write external compressed files"); + return -EOPNOTSUPP; +#endif + } + + return generic_file_open(inode, file); +} + +/* + * file_operations::release + */ +static int ntfs_file_release(struct inode *inode, struct file *file) +{ + struct ntfs_inode *ni = ntfs_i(inode); + struct ntfs_sb_info *sbi = ni->mi.sbi; + int err = 0; + + /* if we are the last writer on the inode, drop the block reservation */ + if (sbi->options.prealloc && ((file->f_mode & FMODE_WRITE) && + atomic_read(&inode->i_writecount) == 1)) { + ni_lock(ni); + down_write(&ni->file.run_lock); + + err = attr_set_size(ni, ATTR_DATA, NULL, 0, &ni->file.run, + inode->i_size, &ni->i_valid, false, NULL); + + up_write(&ni->file.run_lock); + ni_unlock(ni); + } + return err; +} + +/* file_operations::fiemap */ +int ntfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + __u64 start, __u64 len) +{ + int err; + struct ntfs_inode *ni = ntfs_i(inode); + + if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) + return -EOPNOTSUPP; + + ni_lock(ni); + + err = ni_fiemap(ni, fieinfo, start, len); + + ni_unlock(ni); + + return err; +} + +const struct inode_operations ntfs_file_inode_operations = { + .getattr = ntfs_getattr, + .setattr = ntfs3_setattr, + .listxattr = ntfs_listxattr, + .permission = ntfs_permission, + .get_acl = ntfs_get_acl, + .set_acl = ntfs_set_acl, + .fiemap = ntfs_fiemap, +}; + +const struct file_operations ntfs_file_operations = { + .llseek = generic_file_llseek, + .read_iter = ntfs_file_read_iter, + .write_iter = ntfs_file_write_iter, + .unlocked_ioctl = ntfs_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = ntfs_compat_ioctl, +#endif + .splice_read = generic_file_splice_read, + .mmap = ntfs_file_mmap, + .open = ntfs_file_open, + .fsync = generic_file_fsync, + .splice_write = iter_file_splice_write, + .fallocate = ntfs_fallocate, + .release = ntfs_file_release, +}; diff --git a/fs/ntfs3/frecord.c b/fs/ntfs3/frecord.c new file mode 100644 index 000000000000..c3121bf9c62f --- /dev/null +++ b/fs/ntfs3/frecord.c @@ -0,0 +1,3071 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + */ + +#include +#include +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" +#ifdef CONFIG_NTFS3_LZX_XPRESS +#include "lib/lib.h" +#endif + +static struct mft_inode *ni_ins_mi(struct ntfs_inode *ni, struct rb_root *tree, + CLST ino, struct rb_node *ins) +{ + struct rb_node **p = &tree->rb_node; + struct rb_node *pr = NULL; + + while (*p) { + struct mft_inode *mi; + + pr = *p; + mi = rb_entry(pr, struct mft_inode, node); + if (mi->rno > ino) + p = &pr->rb_left; + else if (mi->rno < ino) + p = &pr->rb_right; + else + return mi; + } + + if (!ins) + return NULL; + + rb_link_node(ins, pr, p); + rb_insert_color(ins, tree); + return rb_entry(ins, struct mft_inode, node); +} + +/* + * ni_find_mi + * + * finds mft_inode by record number + */ +static struct mft_inode *ni_find_mi(struct ntfs_inode *ni, CLST rno) +{ + return ni_ins_mi(ni, &ni->mi_tree, rno, NULL); +} + +/* + * ni_add_mi + * + * adds new mft_inode into ntfs_inode + */ +static void ni_add_mi(struct ntfs_inode *ni, struct mft_inode *mi) +{ + ni_ins_mi(ni, &ni->mi_tree, mi->rno, &mi->node); +} + +/* + * ni_remove_mi + * + * removes mft_inode from ntfs_inode + */ +void ni_remove_mi(struct ntfs_inode *ni, struct mft_inode *mi) +{ + rb_erase(&mi->node, &ni->mi_tree); +} + +/* + * ni_std + * + * returns pointer into std_info from primary record + */ +struct ATTR_STD_INFO *ni_std(struct ntfs_inode *ni) +{ + const struct ATTRIB *attr; + + attr = mi_find_attr(&ni->mi, NULL, ATTR_STD, NULL, 0, NULL); + return attr ? resident_data_ex(attr, sizeof(struct ATTR_STD_INFO)) + : NULL; +} + +/* + * ni_std5 + * + * returns pointer into std_info from primary record + */ +struct ATTR_STD_INFO5 *ni_std5(struct ntfs_inode *ni) +{ + const struct ATTRIB *attr; + + attr = mi_find_attr(&ni->mi, NULL, ATTR_STD, NULL, 0, NULL); + + return attr ? resident_data_ex(attr, sizeof(struct ATTR_STD_INFO5)) + : NULL; +} + +/* + * ni_clear + * + * clears resources allocated by ntfs_inode + */ +void ni_clear(struct ntfs_inode *ni) +{ + struct rb_node *node; + + if (!ni->vfs_inode.i_nlink && is_rec_inuse(ni->mi.mrec)) + ni_delete_all(ni); + + al_destroy(ni); + + for (node = rb_first(&ni->mi_tree); node;) { + struct rb_node *next = rb_next(node); + struct mft_inode *mi = rb_entry(node, struct mft_inode, node); + + rb_erase(node, &ni->mi_tree); + mi_put(mi); + node = next; + } + + /* bad inode always has mode == S_IFREG */ + if (ni->ni_flags & NI_FLAG_DIR) + indx_clear(&ni->dir); + else { + run_close(&ni->file.run); +#ifdef CONFIG_NTFS3_LZX_XPRESS + if (ni->file.offs_page) { + /* on-demand allocated page for offsets */ + put_page(ni->file.offs_page); + ni->file.offs_page = NULL; + } +#endif + } + + mi_clear(&ni->mi); +} + +/* + * ni_load_mi_ex + * + * finds mft_inode by record number. + */ +int ni_load_mi_ex(struct ntfs_inode *ni, CLST rno, struct mft_inode **mi) +{ + int err; + struct mft_inode *r; + + r = ni_find_mi(ni, rno); + if (r) + goto out; + + err = mi_get(ni->mi.sbi, rno, &r); + if (err) + return err; + + ni_add_mi(ni, r); + +out: + if (mi) + *mi = r; + return 0; +} + +/* + * ni_load_mi + * + * load mft_inode corresponded list_entry + */ +int ni_load_mi(struct ntfs_inode *ni, struct ATTR_LIST_ENTRY *le, + struct mft_inode **mi) +{ + CLST rno; + + if (!le) { + *mi = &ni->mi; + return 0; + } + + rno = ino_get(&le->ref); + if (rno == ni->mi.rno) { + *mi = &ni->mi; + return 0; + } + return ni_load_mi_ex(ni, rno, mi); +} + +/* + * ni_find_attr + * + * returns attribute and record this attribute belongs to + */ +struct ATTRIB *ni_find_attr(struct ntfs_inode *ni, struct ATTRIB *attr, + struct ATTR_LIST_ENTRY **le_o, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, const CLST *vcn, + struct mft_inode **mi) +{ + struct ATTR_LIST_ENTRY *le; + struct mft_inode *m; + + if (!ni->attr_list.size || + (!name_len && (type == ATTR_LIST || type == ATTR_STD))) { + if (le_o) + *le_o = NULL; + if (mi) + *mi = &ni->mi; + + /* Look for required attribute in primary record */ + return mi_find_attr(&ni->mi, attr, type, name, name_len, NULL); + } + + /* first look for list entry of required type */ + le = al_find_ex(ni, le_o ? *le_o : NULL, type, name, name_len, vcn); + if (!le) + return NULL; + + if (le_o) + *le_o = le; + + /* Load record that contains this attribute */ + if (ni_load_mi(ni, le, &m)) + return NULL; + + /* Look for required attribute */ + attr = mi_find_attr(m, NULL, type, name, name_len, &le->id); + + if (!attr) + goto out; + + if (!attr->non_res) { + if (vcn && *vcn) + goto out; + } else if (!vcn) { + if (attr->nres.svcn) + goto out; + } else if (le64_to_cpu(attr->nres.svcn) > *vcn || + *vcn > le64_to_cpu(attr->nres.evcn)) { + goto out; + } + + if (mi) + *mi = m; + return attr; + +out: + ntfs_set_state(ni->mi.sbi, NTFS_DIRTY_ERROR); + return NULL; +} + +/* + * ni_enum_attr_ex + * + * enumerates attributes in ntfs_inode + */ +struct ATTRIB *ni_enum_attr_ex(struct ntfs_inode *ni, struct ATTRIB *attr, + struct ATTR_LIST_ENTRY **le, + struct mft_inode **mi) +{ + struct mft_inode *mi2; + struct ATTR_LIST_ENTRY *le2; + + /* Do we have an attribute list? */ + if (!ni->attr_list.size) { + *le = NULL; + if (mi) + *mi = &ni->mi; + /* Enum attributes in primary record */ + return mi_enum_attr(&ni->mi, attr); + } + + /* get next list entry */ + le2 = *le = al_enumerate(ni, attr ? *le : NULL); + if (!le2) + return NULL; + + /* Load record that contains the required attribute */ + if (ni_load_mi(ni, le2, &mi2)) + return NULL; + + if (mi) + *mi = mi2; + + /* Find attribute in loaded record */ + return rec_find_attr_le(mi2, le2); +} + +/* + * ni_load_attr + * + * loads attribute that contains given vcn + */ +struct ATTRIB *ni_load_attr(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, CLST vcn, + struct mft_inode **pmi) +{ + struct ATTR_LIST_ENTRY *le; + struct ATTRIB *attr; + struct mft_inode *mi; + struct ATTR_LIST_ENTRY *next; + + if (!ni->attr_list.size) { + if (pmi) + *pmi = &ni->mi; + return mi_find_attr(&ni->mi, NULL, type, name, name_len, NULL); + } + + le = al_find_ex(ni, NULL, type, name, name_len, NULL); + if (!le) + return NULL; + + /* + * Unfortunately ATTR_LIST_ENTRY contains only start vcn + * So to find the ATTRIB segment that contains 'vcn' we should + * enumerate some entries + */ + if (vcn) { + for (;; le = next) { + next = al_find_ex(ni, le, type, name, name_len, NULL); + if (!next || le64_to_cpu(next->vcn) > vcn) + break; + } + } + + if (ni_load_mi(ni, le, &mi)) + return NULL; + + if (pmi) + *pmi = mi; + + attr = mi_find_attr(mi, NULL, type, name, name_len, &le->id); + if (!attr) + return NULL; + + if (!attr->non_res) + return attr; + + if (le64_to_cpu(attr->nres.svcn) <= vcn && + vcn <= le64_to_cpu(attr->nres.evcn)) + return attr; + + return NULL; +} + +/* + * ni_load_all_mi + * + * loads all subrecords + */ +int ni_load_all_mi(struct ntfs_inode *ni) +{ + int err; + struct ATTR_LIST_ENTRY *le; + + if (!ni->attr_list.size) + return 0; + + le = NULL; + + while ((le = al_enumerate(ni, le))) { + CLST rno = ino_get(&le->ref); + + if (rno == ni->mi.rno) + continue; + + err = ni_load_mi_ex(ni, rno, NULL); + if (err) + return err; + } + + return 0; +} + +/* + * ni_add_subrecord + * + * allocate + format + attach a new subrecord + */ +bool ni_add_subrecord(struct ntfs_inode *ni, CLST rno, struct mft_inode **mi) +{ + struct mft_inode *m; + + m = ntfs_zalloc(sizeof(struct mft_inode)); + if (!m) + return false; + + if (mi_format_new(m, ni->mi.sbi, rno, 0, ni->mi.rno == MFT_REC_MFT)) { + mi_put(m); + return false; + } + + mi_get_ref(&ni->mi, &m->mrec->parent_ref); + + ni_add_mi(ni, m); + *mi = m; + return true; +} + +/* + * ni_remove_attr + * + * removes all attributes for the given type/name/id + */ +int ni_remove_attr(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, size_t name_len, bool base_only, + const __le16 *id) +{ + int err; + struct ATTRIB *attr; + struct ATTR_LIST_ENTRY *le; + struct mft_inode *mi; + u32 type_in; + int diff; + + if (base_only || type == ATTR_LIST || !ni->attr_list.size) { + attr = mi_find_attr(&ni->mi, NULL, type, name, name_len, id); + if (!attr) + return -ENOENT; + + mi_remove_attr(&ni->mi, attr); + return 0; + } + + type_in = le32_to_cpu(type); + le = NULL; + + for (;;) { + le = al_enumerate(ni, le); + if (!le) + return 0; + +next_le2: + diff = le32_to_cpu(le->type) - type_in; + if (diff < 0) + continue; + + if (diff > 0) + return 0; + + if (le->name_len != name_len) + continue; + + if (name_len && + memcmp(le_name(le), name, name_len * sizeof(short))) + continue; + + if (id && le->id != *id) + continue; + err = ni_load_mi(ni, le, &mi); + if (err) + return err; + + al_remove_le(ni, le); + + attr = mi_find_attr(mi, NULL, type, name, name_len, id); + if (!attr) + return -ENOENT; + + mi_remove_attr(mi, attr); + + if (PtrOffset(ni->attr_list.le, le) >= ni->attr_list.size) + return 0; + goto next_le2; + } +} + +/* + * ni_ins_new_attr + * + * inserts the attribute into record + * Returns not full constructed attribute or NULL if not possible to create + */ +static struct ATTRIB *ni_ins_new_attr(struct ntfs_inode *ni, + struct mft_inode *mi, + struct ATTR_LIST_ENTRY *le, + enum ATTR_TYPE type, const __le16 *name, + u8 name_len, u32 asize, u16 name_off, + CLST svcn) +{ + int err; + struct ATTRIB *attr; + bool le_added = false; + struct MFT_REF ref; + + mi_get_ref(mi, &ref); + + if (type != ATTR_LIST && !le && ni->attr_list.size) { + err = al_add_le(ni, type, name, name_len, svcn, cpu_to_le16(-1), + &ref, &le); + if (err) { + /* no memory or no space */ + return NULL; + } + le_added = true; + + /* + * al_add_le -> attr_set_size (list) -> ni_expand_list + * which moves some attributes out of primary record + * this means that name may point into moved memory + * reinit 'name' from le + */ + name = le->name; + } + + attr = mi_insert_attr(mi, type, name, name_len, asize, name_off); + if (!attr) { + if (le_added) + al_remove_le(ni, le); + return NULL; + } + + if (type == ATTR_LIST) { + /*attr list is not in list entry array*/ + goto out; + } + + if (!le) + goto out; + + /* Update ATTRIB Id and record reference */ + le->id = attr->id; + ni->attr_list.dirty = true; + le->ref = ref; + +out: + return attr; +} + +/* + * random write access to sparsed or compressed file may result to + * not optimized packed runs. + * Here it is the place to optimize it + */ +static int ni_repack(struct ntfs_inode *ni) +{ + int err = 0; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct mft_inode *mi, *mi_p = NULL; + struct ATTRIB *attr = NULL, *attr_p; + struct ATTR_LIST_ENTRY *le = NULL, *le_p; + CLST alloc = 0; + u8 cluster_bits = sbi->cluster_bits; + CLST svcn, evcn = 0, svcn_p, evcn_p, next_svcn; + u32 roff, rs = sbi->record_size; + struct runs_tree run; + + run_init(&run); + + while ((attr = ni_enum_attr_ex(ni, attr, &le, &mi))) { + if (!attr->non_res) + continue; + + svcn = le64_to_cpu(attr->nres.svcn); + if (svcn != le64_to_cpu(le->vcn)) { + err = -EINVAL; + break; + } + + if (!svcn) { + alloc = le64_to_cpu(attr->nres.alloc_size) >> + cluster_bits; + mi_p = NULL; + } else if (svcn != evcn + 1) { + err = -EINVAL; + break; + } + + evcn = le64_to_cpu(attr->nres.evcn); + + if (svcn > evcn + 1) { + err = -EINVAL; + break; + } + + if (!mi_p) { + /* do not try if too little free space */ + if (le32_to_cpu(mi->mrec->used) + 8 >= rs) + continue; + + /* do not try if last attribute segment */ + if (evcn + 1 == alloc) + continue; + run_close(&run); + } + + roff = le16_to_cpu(attr->nres.run_off); + err = run_unpack(&run, sbi, ni->mi.rno, svcn, evcn, svcn, + Add2Ptr(attr, roff), + le32_to_cpu(attr->size) - roff); + if (err < 0) + break; + + if (!mi_p) { + mi_p = mi; + attr_p = attr; + svcn_p = svcn; + evcn_p = evcn; + le_p = le; + err = 0; + continue; + } + + /* + * run contains data from two records: mi_p and mi + * try to pack in one + */ + err = mi_pack_runs(mi_p, attr_p, &run, evcn + 1 - svcn_p); + if (err) + break; + + next_svcn = le64_to_cpu(attr_p->nres.evcn) + 1; + + if (next_svcn >= evcn + 1) { + /* we can remove this attribute segment */ + al_remove_le(ni, le); + mi_remove_attr(mi, attr); + le = le_p; + continue; + } + + attr->nres.svcn = le->vcn = cpu_to_le64(next_svcn); + mi->dirty = true; + ni->attr_list.dirty = true; + + if (evcn + 1 == alloc) { + err = mi_pack_runs(mi, attr, &run, + evcn + 1 - next_svcn); + if (err) + break; + mi_p = NULL; + } else { + mi_p = mi; + attr_p = attr; + svcn_p = next_svcn; + evcn_p = evcn; + le_p = le; + run_truncate_head(&run, next_svcn); + } + } + + if (err) { + ntfs_inode_warn(&ni->vfs_inode, "repack problem"); + ntfs_set_state(sbi, NTFS_DIRTY_ERROR); + + /* Pack loaded but not packed runs */ + if (mi_p) + mi_pack_runs(mi_p, attr_p, &run, evcn_p + 1 - svcn_p); + } + + run_close(&run); + return err; +} + +/* + * ni_try_remove_attr_list + * + * Can we remove attribute list? + * Check the case when primary record contains enough space for all attributes + */ +static int ni_try_remove_attr_list(struct ntfs_inode *ni) +{ + int err = 0; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct ATTRIB *attr, *attr_list, *attr_ins; + struct ATTR_LIST_ENTRY *le; + struct mft_inode *mi; + u32 asize, free; + struct MFT_REF ref; + __le16 id; + + if (!ni->attr_list.dirty) + return 0; + + err = ni_repack(ni); + if (err) + return err; + + attr_list = mi_find_attr(&ni->mi, NULL, ATTR_LIST, NULL, 0, NULL); + if (!attr_list) + return 0; + + asize = le32_to_cpu(attr_list->size); + + /* free space in primary record without attribute list */ + free = sbi->record_size - le32_to_cpu(ni->mi.mrec->used) + asize; + mi_get_ref(&ni->mi, &ref); + + le = NULL; + while ((le = al_enumerate(ni, le))) { + if (!memcmp(&le->ref, &ref, sizeof(ref))) + continue; + + if (le->vcn) + return 0; + + mi = ni_find_mi(ni, ino_get(&le->ref)); + if (!mi) + return 0; + + attr = mi_find_attr(mi, NULL, le->type, le_name(le), + le->name_len, &le->id); + if (!attr) + return 0; + + asize = le32_to_cpu(attr->size); + if (asize > free) + return 0; + + free -= asize; + } + + /* Is seems that attribute list can be removed from primary record */ + mi_remove_attr(&ni->mi, attr_list); + + /* + * Repeat the cycle above and move all attributes to primary record. + * It should be success! + */ + le = NULL; + while ((le = al_enumerate(ni, le))) { + if (!memcmp(&le->ref, &ref, sizeof(ref))) + continue; + + mi = ni_find_mi(ni, ino_get(&le->ref)); + + attr = mi_find_attr(mi, NULL, le->type, le_name(le), + le->name_len, &le->id); + asize = le32_to_cpu(attr->size); + + /* insert into primary record */ + attr_ins = mi_insert_attr(&ni->mi, le->type, le_name(le), + le->name_len, asize, + le16_to_cpu(attr->name_off)); + id = attr_ins->id; + + /* copy all except id */ + memcpy(attr_ins, attr, asize); + attr_ins->id = id; + + /* remove from original record */ + mi_remove_attr(mi, attr); + } + + run_deallocate(sbi, &ni->attr_list.run, true); + run_close(&ni->attr_list.run); + ni->attr_list.size = 0; + ntfs_free(ni->attr_list.le); + ni->attr_list.le = NULL; + ni->attr_list.dirty = false; + + return 0; +} + +/* + * ni_create_attr_list + * + * generates an attribute list for this primary record + */ +int ni_create_attr_list(struct ntfs_inode *ni) +{ + struct ntfs_sb_info *sbi = ni->mi.sbi; + int err; + u32 lsize; + struct ATTRIB *attr; + struct ATTRIB *arr_move[7]; + struct ATTR_LIST_ENTRY *le, *le_b[7]; + struct MFT_REC *rec; + bool is_mft; + CLST rno = 0; + struct mft_inode *mi; + u32 free_b, nb, to_free, rs; + u16 sz; + + is_mft = ni->mi.rno == MFT_REC_MFT; + rec = ni->mi.mrec; + rs = sbi->record_size; + + /* + * Skip estimating exact memory requirement + * Looks like one record_size is always enough + */ + le = ntfs_malloc(al_aligned(rs)); + if (!le) { + err = -ENOMEM; + goto out; + } + + mi_get_ref(&ni->mi, &le->ref); + ni->attr_list.le = le; + + attr = NULL; + nb = 0; + free_b = 0; + attr = NULL; + + for (; (attr = mi_enum_attr(&ni->mi, attr)); le = Add2Ptr(le, sz)) { + sz = le_size(attr->name_len); + le->type = attr->type; + le->size = cpu_to_le16(sz); + le->name_len = attr->name_len; + le->name_off = offsetof(struct ATTR_LIST_ENTRY, name); + le->vcn = 0; + if (le != ni->attr_list.le) + le->ref = ni->attr_list.le->ref; + le->id = attr->id; + + if (attr->name_len) + memcpy(le->name, attr_name(attr), + sizeof(short) * attr->name_len); + else if (attr->type == ATTR_STD) + continue; + else if (attr->type == ATTR_LIST) + continue; + else if (is_mft && attr->type == ATTR_DATA) + continue; + + if (!nb || nb < ARRAY_SIZE(arr_move)) { + le_b[nb] = le; + arr_move[nb++] = attr; + free_b += le32_to_cpu(attr->size); + } + } + + lsize = PtrOffset(ni->attr_list.le, le); + ni->attr_list.size = lsize; + + to_free = le32_to_cpu(rec->used) + lsize + SIZEOF_RESIDENT; + if (to_free <= rs) { + to_free = 0; + } else { + to_free -= rs; + + if (to_free > free_b) { + err = -EINVAL; + goto out1; + } + } + + /* Allocate child mft. */ + err = ntfs_look_free_mft(sbi, &rno, is_mft, ni, &mi); + if (err) + goto out1; + + /* Call 'mi_remove_attr' in reverse order to keep pointers 'arr_move' valid */ + while (to_free > 0) { + struct ATTRIB *b = arr_move[--nb]; + u32 asize = le32_to_cpu(b->size); + u16 name_off = le16_to_cpu(b->name_off); + + attr = mi_insert_attr(mi, b->type, Add2Ptr(b, name_off), + b->name_len, asize, name_off); + WARN_ON(!attr); + + mi_get_ref(mi, &le_b[nb]->ref); + le_b[nb]->id = attr->id; + + /* copy all except id */ + memcpy(attr, b, asize); + attr->id = le_b[nb]->id; + + WARN_ON(!mi_remove_attr(&ni->mi, b)); + + if (to_free <= asize) + break; + to_free -= asize; + WARN_ON(!nb); + } + + attr = mi_insert_attr(&ni->mi, ATTR_LIST, NULL, 0, + lsize + SIZEOF_RESIDENT, SIZEOF_RESIDENT); + WARN_ON(!attr); + + attr->non_res = 0; + attr->flags = 0; + attr->res.data_size = cpu_to_le32(lsize); + attr->res.data_off = SIZEOF_RESIDENT_LE; + attr->res.flags = 0; + attr->res.res = 0; + + memcpy(resident_data_ex(attr, lsize), ni->attr_list.le, lsize); + + ni->attr_list.dirty = false; + + mark_inode_dirty(&ni->vfs_inode); + goto out; + +out1: + ntfs_free(ni->attr_list.le); + ni->attr_list.le = NULL; + ni->attr_list.size = 0; + +out: + return err; +} + +/* + * ni_ins_attr_ext + * + * This method adds an external attribute to the ntfs_inode. + */ +static int ni_ins_attr_ext(struct ntfs_inode *ni, struct ATTR_LIST_ENTRY *le, + enum ATTR_TYPE type, const __le16 *name, u8 name_len, + u32 asize, CLST svcn, u16 name_off, bool force_ext, + struct ATTRIB **ins_attr, struct mft_inode **ins_mi) +{ + struct ATTRIB *attr; + struct mft_inode *mi; + CLST rno; + u64 vbo; + struct rb_node *node; + int err; + bool is_mft, is_mft_data; + struct ntfs_sb_info *sbi = ni->mi.sbi; + + is_mft = ni->mi.rno == MFT_REC_MFT; + is_mft_data = is_mft && type == ATTR_DATA && !name_len; + + if (asize > sbi->max_bytes_per_attr) { + err = -EINVAL; + goto out; + } + + /* + * standard information and attr_list cannot be made external. + * The Log File cannot have any external attributes + */ + if (type == ATTR_STD || type == ATTR_LIST || + ni->mi.rno == MFT_REC_LOG) { + err = -EINVAL; + goto out; + } + + /* Create attribute list if it is not already existed */ + if (!ni->attr_list.size) { + err = ni_create_attr_list(ni); + if (err) + goto out; + } + + vbo = is_mft_data ? ((u64)svcn << sbi->cluster_bits) : 0; + + if (force_ext) + goto insert_ext; + + /* Load all subrecords into memory. */ + err = ni_load_all_mi(ni); + if (err) + goto out; + + /* Check each of loaded subrecord */ + for (node = rb_first(&ni->mi_tree); node; node = rb_next(node)) { + mi = rb_entry(node, struct mft_inode, node); + + if (is_mft_data && + (mi_enum_attr(mi, NULL) || + vbo <= ((u64)mi->rno << sbi->record_bits))) { + /* We can't accept this record 'case MFT's bootstrapping */ + continue; + } + if (is_mft && + mi_find_attr(mi, NULL, ATTR_DATA, NULL, 0, NULL)) { + /* + * This child record already has a ATTR_DATA. + * So it can't accept any other records. + */ + continue; + } + + if ((type != ATTR_NAME || name_len) && + mi_find_attr(mi, NULL, type, name, name_len, NULL)) { + /* Only indexed attributes can share same record */ + continue; + } + + /* Try to insert attribute into this subrecord */ + attr = ni_ins_new_attr(ni, mi, le, type, name, name_len, asize, + name_off, svcn); + if (!attr) + continue; + + if (ins_attr) + *ins_attr = attr; + return 0; + } + +insert_ext: + /* We have to allocate a new child subrecord*/ + err = ntfs_look_free_mft(sbi, &rno, is_mft_data, ni, &mi); + if (err) + goto out; + + if (is_mft_data && vbo <= ((u64)rno << sbi->record_bits)) { + err = -EINVAL; + goto out1; + } + + attr = ni_ins_new_attr(ni, mi, le, type, name, name_len, asize, + name_off, svcn); + if (!attr) + goto out2; + + if (ins_attr) + *ins_attr = attr; + if (ins_mi) + *ins_mi = mi; + + return 0; + +out2: + ni_remove_mi(ni, mi); + mi_put(mi); + err = -EINVAL; + +out1: + ntfs_mark_rec_free(sbi, rno); + +out: + return err; +} + +/* + * ni_insert_attr + * + * inserts an attribute into the file. + * + * If the primary record has room, it will just insert the attribute. + * If not, it may make the attribute external. + * For $MFT::Data it may make room for the attribute by + * making other attributes external. + * + * NOTE: + * The ATTR_LIST and ATTR_STD cannot be made external. + * This function does not fill new attribute full + * It only fills 'size'/'type'/'id'/'name_len' fields + */ +static int ni_insert_attr(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, u32 asize, + u16 name_off, CLST svcn, struct ATTRIB **ins_attr, + struct mft_inode **ins_mi) +{ + struct ntfs_sb_info *sbi = ni->mi.sbi; + int err; + struct ATTRIB *attr, *eattr; + struct MFT_REC *rec; + bool is_mft; + struct ATTR_LIST_ENTRY *le; + u32 list_reserve, max_free, free, used, t32; + __le16 id; + u16 t16; + + is_mft = ni->mi.rno == MFT_REC_MFT; + rec = ni->mi.mrec; + + list_reserve = SIZEOF_NONRESIDENT + 3 * (1 + 2 * sizeof(u32)); + used = le32_to_cpu(rec->used); + free = sbi->record_size - used; + + if (is_mft && type != ATTR_LIST) { + /* Reserve space for the ATTRIB List. */ + if (free < list_reserve) + free = 0; + else + free -= list_reserve; + } + + if (asize <= free) { + attr = ni_ins_new_attr(ni, &ni->mi, NULL, type, name, name_len, + asize, name_off, svcn); + if (attr) { + if (ins_attr) + *ins_attr = attr; + if (ins_mi) + *ins_mi = &ni->mi; + err = 0; + goto out; + } + } + + if (!is_mft || type != ATTR_DATA || svcn) { + /* This ATTRIB will be external. */ + err = ni_ins_attr_ext(ni, NULL, type, name, name_len, asize, + svcn, name_off, false, ins_attr, ins_mi); + goto out; + } + + /* + * Here we have: "is_mft && type == ATTR_DATA && !svcn + * + * The first chunk of the $MFT::Data ATTRIB must be the base record. + * Evict as many other attributes as possible. + */ + max_free = free; + + /* Estimate the result of moving all possible attributes away.*/ + attr = NULL; + + while ((attr = mi_enum_attr(&ni->mi, attr))) { + if (attr->type == ATTR_STD) + continue; + if (attr->type == ATTR_LIST) + continue; + max_free += le32_to_cpu(attr->size); + } + + if (max_free < asize + list_reserve) { + /* Impossible to insert this attribute into primary record */ + err = -EINVAL; + goto out; + } + + /* Start real attribute moving */ + attr = NULL; + + for (;;) { + attr = mi_enum_attr(&ni->mi, attr); + if (!attr) { + /* We should never be here 'cause we have already check this case */ + err = -EINVAL; + goto out; + } + + /* Skip attributes that MUST be primary record */ + if (attr->type == ATTR_STD || attr->type == ATTR_LIST) + continue; + + le = NULL; + if (ni->attr_list.size) { + le = al_find_le(ni, NULL, attr); + if (!le) { + /* Really this is a serious bug */ + err = -EINVAL; + goto out; + } + } + + t32 = le32_to_cpu(attr->size); + t16 = le16_to_cpu(attr->name_off); + err = ni_ins_attr_ext(ni, le, attr->type, Add2Ptr(attr, t16), + attr->name_len, t32, attr_svcn(attr), t16, + false, &eattr, NULL); + if (err) + return err; + + id = eattr->id; + memcpy(eattr, attr, t32); + eattr->id = id; + + /* remove attrib from primary record */ + mi_remove_attr(&ni->mi, attr); + + /* attr now points to next attribute */ + if (attr->type == ATTR_END) + goto out; + } + while (asize + list_reserve > sbi->record_size - le32_to_cpu(rec->used)) + ; + + attr = ni_ins_new_attr(ni, &ni->mi, NULL, type, name, name_len, asize, + name_off, svcn); + if (!attr) { + err = -EINVAL; + goto out; + } + + if (ins_attr) + *ins_attr = attr; + if (ins_mi) + *ins_mi = &ni->mi; + +out: + return err; +} + +/* + * ni_expand_mft_list + * + * This method splits ATTR_DATA of $MFT + */ +static int ni_expand_mft_list(struct ntfs_inode *ni) +{ + int err = 0; + struct runs_tree *run = &ni->file.run; + u32 asize, run_size, done = 0; + struct ATTRIB *attr; + struct rb_node *node; + CLST mft_min, mft_new, svcn, evcn, plen; + struct mft_inode *mi, *mi_min, *mi_new; + struct ntfs_sb_info *sbi = ni->mi.sbi; + + /* Find the nearest Mft */ + mft_min = 0; + mft_new = 0; + mi_min = NULL; + + for (node = rb_first(&ni->mi_tree); node; node = rb_next(node)) { + mi = rb_entry(node, struct mft_inode, node); + + attr = mi_enum_attr(mi, NULL); + + if (!attr) { + mft_min = mi->rno; + mi_min = mi; + break; + } + } + + if (ntfs_look_free_mft(sbi, &mft_new, true, ni, &mi_new)) { + mft_new = 0; + // really this is not critical + } else if (mft_min > mft_new) { + mft_min = mft_new; + mi_min = mi_new; + } else { + ntfs_mark_rec_free(sbi, mft_new); + mft_new = 0; + ni_remove_mi(ni, mi_new); + } + + attr = mi_find_attr(&ni->mi, NULL, ATTR_DATA, NULL, 0, NULL); + if (!attr) { + err = -EINVAL; + goto out; + } + + asize = le32_to_cpu(attr->size); + + evcn = le64_to_cpu(attr->nres.evcn); + svcn = bytes_to_cluster(sbi, (u64)(mft_min + 1) << sbi->record_bits); + if (evcn + 1 >= svcn) { + err = -EINVAL; + goto out; + } + + /* + * split primary attribute [0 evcn] in two parts [0 svcn) + [svcn evcn] + * + * Update first part of ATTR_DATA in 'primary MFT + */ + err = run_pack(run, 0, svcn, Add2Ptr(attr, SIZEOF_NONRESIDENT), + asize - SIZEOF_NONRESIDENT, &plen); + if (err < 0) + goto out; + + run_size = QuadAlign(err); + err = 0; + + if (plen < svcn) { + err = -EINVAL; + goto out; + } + + attr->nres.evcn = cpu_to_le64(svcn - 1); + attr->size = cpu_to_le32(run_size + SIZEOF_NONRESIDENT); + /* 'done' - how many bytes of primary MFT becomes free */ + done = asize - run_size - SIZEOF_NONRESIDENT; + le32_sub_cpu(&ni->mi.mrec->used, done); + + /* Estimate the size of second part: run_buf=NULL */ + err = run_pack(run, svcn, evcn + 1 - svcn, NULL, sbi->record_size, + &plen); + if (err < 0) + goto out; + + run_size = QuadAlign(err); + err = 0; + + if (plen < evcn + 1 - svcn) { + err = -EINVAL; + goto out; + } + + /* + * This function may implicitly call expand attr_list + * Insert second part of ATTR_DATA in 'mi_min' + */ + attr = ni_ins_new_attr(ni, mi_min, NULL, ATTR_DATA, NULL, 0, + SIZEOF_NONRESIDENT + run_size, + SIZEOF_NONRESIDENT, svcn); + if (!attr) { + err = -EINVAL; + goto out; + } + + attr->non_res = 1; + attr->name_off = SIZEOF_NONRESIDENT_LE; + attr->flags = 0; + + run_pack(run, svcn, evcn + 1 - svcn, Add2Ptr(attr, SIZEOF_NONRESIDENT), + run_size, &plen); + + attr->nres.svcn = cpu_to_le64(svcn); + attr->nres.evcn = cpu_to_le64(evcn); + attr->nres.run_off = cpu_to_le16(SIZEOF_NONRESIDENT); + +out: + if (mft_new) { + ntfs_mark_rec_free(sbi, mft_new); + ni_remove_mi(ni, mi_new); + } + + return !err && !done ? -EOPNOTSUPP : err; +} + +/* + * ni_expand_list + * + * This method moves all possible attributes out of primary record + */ +int ni_expand_list(struct ntfs_inode *ni) +{ + int err = 0; + u32 asize, done = 0; + struct ATTRIB *attr, *ins_attr; + struct ATTR_LIST_ENTRY *le; + bool is_mft = ni->mi.rno == MFT_REC_MFT; + struct MFT_REF ref; + + mi_get_ref(&ni->mi, &ref); + le = NULL; + + while ((le = al_enumerate(ni, le))) { + if (le->type == ATTR_STD) + continue; + + if (memcmp(&ref, &le->ref, sizeof(struct MFT_REF))) + continue; + + if (is_mft && le->type == ATTR_DATA) + continue; + + /* Find attribute in primary record */ + attr = rec_find_attr_le(&ni->mi, le); + if (!attr) { + err = -EINVAL; + goto out; + } + + asize = le32_to_cpu(attr->size); + + /* Always insert into new record to avoid collisions (deep recursive) */ + err = ni_ins_attr_ext(ni, le, attr->type, attr_name(attr), + attr->name_len, asize, attr_svcn(attr), + le16_to_cpu(attr->name_off), true, + &ins_attr, NULL); + + if (err) + goto out; + + memcpy(ins_attr, attr, asize); + ins_attr->id = le->id; + mi_remove_attr(&ni->mi, attr); + + done += asize; + goto out; + } + + if (!is_mft) { + err = -EFBIG; /* attr list is too big(?) */ + goto out; + } + + /* split mft data as much as possible */ + err = ni_expand_mft_list(ni); + if (err) + goto out; + +out: + return !err && !done ? -EOPNOTSUPP : err; +} + +/* + * ni_insert_nonresident + * + * inserts new nonresident attribute + */ +int ni_insert_nonresident(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, + const struct runs_tree *run, CLST svcn, CLST len, + __le16 flags, struct ATTRIB **new_attr, + struct mft_inode **mi) +{ + int err; + CLST plen; + struct ATTRIB *attr; + bool is_ext = + (flags & (ATTR_FLAG_SPARSED | ATTR_FLAG_COMPRESSED)) && !svcn; + u32 name_size = QuadAlign(name_len * sizeof(short)); + u32 name_off = is_ext ? SIZEOF_NONRESIDENT_EX : SIZEOF_NONRESIDENT; + u32 run_off = name_off + name_size; + u32 run_size, asize; + struct ntfs_sb_info *sbi = ni->mi.sbi; + + err = run_pack(run, svcn, len, NULL, sbi->max_bytes_per_attr - run_off, + &plen); + if (err < 0) + goto out; + + run_size = QuadAlign(err); + + if (plen < len) { + err = -EINVAL; + goto out; + } + + asize = run_off + run_size; + + if (asize > sbi->max_bytes_per_attr) { + err = -EINVAL; + goto out; + } + + err = ni_insert_attr(ni, type, name, name_len, asize, name_off, svcn, + &attr, mi); + + if (err) + goto out; + + attr->non_res = 1; + attr->name_off = cpu_to_le16(name_off); + attr->flags = flags; + + run_pack(run, svcn, len, Add2Ptr(attr, run_off), run_size, &plen); + + attr->nres.svcn = cpu_to_le64(svcn); + attr->nres.evcn = cpu_to_le64((u64)svcn + len - 1); + + err = 0; + if (new_attr) + *new_attr = attr; + + *(__le64 *)&attr->nres.run_off = cpu_to_le64(run_off); + + attr->nres.alloc_size = + svcn ? 0 : cpu_to_le64((u64)len << ni->mi.sbi->cluster_bits); + attr->nres.data_size = attr->nres.alloc_size; + attr->nres.valid_size = attr->nres.alloc_size; + + if (is_ext) { + if (flags & ATTR_FLAG_COMPRESSED) + attr->nres.c_unit = COMPRESSION_UNIT; + attr->nres.total_size = attr->nres.alloc_size; + } + +out: + return err; +} + +/* + * ni_insert_resident + * + * inserts new resident attribute + */ +int ni_insert_resident(struct ntfs_inode *ni, u32 data_size, + enum ATTR_TYPE type, const __le16 *name, u8 name_len, + struct ATTRIB **new_attr, struct mft_inode **mi) +{ + int err; + u32 name_size = QuadAlign(name_len * sizeof(short)); + u32 asize = SIZEOF_RESIDENT + name_size + QuadAlign(data_size); + struct ATTRIB *attr; + + err = ni_insert_attr(ni, type, name, name_len, asize, SIZEOF_RESIDENT, + 0, &attr, mi); + if (err) + return err; + + attr->non_res = 0; + attr->flags = 0; + + attr->res.data_size = cpu_to_le32(data_size); + attr->res.data_off = cpu_to_le16(SIZEOF_RESIDENT + name_size); + if (type == ATTR_NAME) + attr->res.flags = RESIDENT_FLAG_INDEXED; + attr->res.res = 0; + + if (new_attr) + *new_attr = attr; + + return 0; +} + +/* + * ni_remove_attr_le + * + * removes attribute from record + */ +int ni_remove_attr_le(struct ntfs_inode *ni, struct ATTRIB *attr, + struct ATTR_LIST_ENTRY *le) +{ + int err; + struct mft_inode *mi; + + err = ni_load_mi(ni, le, &mi); + if (err) + return err; + + mi_remove_attr(mi, attr); + + if (le) + al_remove_le(ni, le); + + return 0; +} + +/* + * ni_delete_all + * + * removes all attributes and frees allocates space + * ntfs_evict_inode->ntfs_clear_inode->ni_delete_all (if no links) + */ +int ni_delete_all(struct ntfs_inode *ni) +{ + int err; + struct ATTR_LIST_ENTRY *le = NULL; + struct ATTRIB *attr = NULL; + struct rb_node *node; + u16 roff; + u32 asize; + CLST svcn, evcn; + struct ntfs_sb_info *sbi = ni->mi.sbi; + bool nt3 = is_ntfs3(sbi); + struct MFT_REF ref; + + while ((attr = ni_enum_attr_ex(ni, attr, &le, NULL))) { + if (!nt3 || attr->name_len) { + ; + } else if (attr->type == ATTR_REPARSE) { + mi_get_ref(&ni->mi, &ref); + ntfs_remove_reparse(sbi, 0, &ref); + } else if (attr->type == ATTR_ID && !attr->non_res && + le32_to_cpu(attr->res.data_size) >= + sizeof(struct GUID)) { + ntfs_objid_remove(sbi, resident_data(attr)); + } + + if (!attr->non_res) + continue; + + svcn = le64_to_cpu(attr->nres.svcn); + evcn = le64_to_cpu(attr->nres.evcn); + + if (evcn + 1 <= svcn) + continue; + + asize = le32_to_cpu(attr->size); + roff = le16_to_cpu(attr->nres.run_off); + + /*run==1 means unpack and deallocate*/ + run_unpack_ex(RUN_DEALLOCATE, sbi, ni->mi.rno, svcn, evcn, svcn, + Add2Ptr(attr, roff), asize - roff); + } + + if (ni->attr_list.size) { + run_deallocate(ni->mi.sbi, &ni->attr_list.run, true); + al_destroy(ni); + } + + /* Free all subrecords */ + for (node = rb_first(&ni->mi_tree); node;) { + struct rb_node *next = rb_next(node); + struct mft_inode *mi = rb_entry(node, struct mft_inode, node); + + clear_rec_inuse(mi->mrec); + mi->dirty = true; + mi_write(mi, 0); + + ntfs_mark_rec_free(sbi, mi->rno); + ni_remove_mi(ni, mi); + mi_put(mi); + node = next; + } + + // Free base record + clear_rec_inuse(ni->mi.mrec); + ni->mi.dirty = true; + err = mi_write(&ni->mi, 0); + + ntfs_mark_rec_free(sbi, ni->mi.rno); + + return err; +} + +/* + * ni_fname_name + * + * returns file name attribute by its value + */ +struct ATTR_FILE_NAME *ni_fname_name(struct ntfs_inode *ni, + const struct cpu_str *uni, + const struct MFT_REF *home_dir, + struct ATTR_LIST_ENTRY **le) +{ + struct ATTRIB *attr = NULL; + struct ATTR_FILE_NAME *fname; + + *le = NULL; + + /* Enumerate all names */ +next: + attr = ni_find_attr(ni, attr, le, ATTR_NAME, NULL, 0, NULL, NULL); + if (!attr) + return NULL; + + fname = resident_data_ex(attr, SIZEOF_ATTRIBUTE_FILENAME); + if (!fname) + goto next; + + if (home_dir && memcmp(home_dir, &fname->home, sizeof(*home_dir))) + goto next; + + if (!uni) + goto next; + + if (uni->len != fname->name_len) + goto next; + + if (ntfs_cmp_names_cpu(uni, (struct le_str *)&fname->name_len, NULL, + false)) + goto next; + + return fname; +} + +/* + * ni_fname_type + * + * returns file name attribute with given type + */ +struct ATTR_FILE_NAME *ni_fname_type(struct ntfs_inode *ni, u8 name_type, + struct ATTR_LIST_ENTRY **le) +{ + struct ATTRIB *attr = NULL; + struct ATTR_FILE_NAME *fname; + + *le = NULL; + + /* Enumerate all names */ + for (;;) { + attr = ni_find_attr(ni, attr, le, ATTR_NAME, NULL, 0, NULL, + NULL); + if (!attr) + return NULL; + + fname = resident_data_ex(attr, SIZEOF_ATTRIBUTE_FILENAME); + if (fname && name_type == fname->type) + return fname; + } +} + +/* + * Process compressed/sparsed in special way + * NOTE: you need to set ni->std_fa = new_fa + * after this function to keep internal structures in consistency + */ +int ni_new_attr_flags(struct ntfs_inode *ni, enum FILE_ATTRIBUTE new_fa) +{ + struct ATTRIB *attr; + struct mft_inode *mi; + __le16 new_aflags; + u32 new_asize; + + attr = ni_find_attr(ni, NULL, NULL, ATTR_DATA, NULL, 0, NULL, &mi); + if (!attr) + return -EINVAL; + + new_aflags = attr->flags; + + if (new_fa & FILE_ATTRIBUTE_SPARSE_FILE) + new_aflags |= ATTR_FLAG_SPARSED; + else + new_aflags &= ~ATTR_FLAG_SPARSED; + + if (new_fa & FILE_ATTRIBUTE_COMPRESSED) + new_aflags |= ATTR_FLAG_COMPRESSED; + else + new_aflags &= ~ATTR_FLAG_COMPRESSED; + + if (new_aflags == attr->flags) + return 0; + + if ((new_aflags & (ATTR_FLAG_COMPRESSED | ATTR_FLAG_SPARSED)) == + (ATTR_FLAG_COMPRESSED | ATTR_FLAG_SPARSED)) { + ntfs_inode_warn(&ni->vfs_inode, + "file can't be sparsed and compressed"); + return -EOPNOTSUPP; + } + + if (!attr->non_res) + goto out; + + if (attr->nres.data_size) { + ntfs_inode_warn( + &ni->vfs_inode, + "one can change sparsed/compressed only for empty files"); + return -EOPNOTSUPP; + } + + /* resize nonresident empty attribute in-place only*/ + new_asize = (new_aflags & (ATTR_FLAG_COMPRESSED | ATTR_FLAG_SPARSED)) + ? (SIZEOF_NONRESIDENT_EX + 8) + : (SIZEOF_NONRESIDENT + 8); + + if (!mi_resize_attr(mi, attr, new_asize - le32_to_cpu(attr->size))) + return -EOPNOTSUPP; + + if (new_aflags & ATTR_FLAG_SPARSED) { + attr->name_off = SIZEOF_NONRESIDENT_EX_LE; + /* windows uses 16 clusters per frame but supports one cluster per frame too*/ + attr->nres.c_unit = 0; + ni->vfs_inode.i_mapping->a_ops = &ntfs_aops; + } else if (new_aflags & ATTR_FLAG_COMPRESSED) { + attr->name_off = SIZEOF_NONRESIDENT_EX_LE; + /* the only allowed: 16 clusters per frame */ + attr->nres.c_unit = NTFS_LZNT_CUNIT; + ni->vfs_inode.i_mapping->a_ops = &ntfs_aops_cmpr; + } else { + attr->name_off = SIZEOF_NONRESIDENT_LE; + /* normal files */ + attr->nres.c_unit = 0; + ni->vfs_inode.i_mapping->a_ops = &ntfs_aops; + } + attr->nres.run_off = attr->name_off; +out: + attr->flags = new_aflags; + mi->dirty = true; + + return 0; +} + +/* + * ni_parse_reparse + * + * buffer is at least 24 bytes + */ +enum REPARSE_SIGN ni_parse_reparse(struct ntfs_inode *ni, struct ATTRIB *attr, + void *buffer) +{ + const struct REPARSE_DATA_BUFFER *rp = NULL; + u8 bits; + u16 len; + typeof(rp->CompressReparseBuffer) *cmpr; + + static_assert(sizeof(struct REPARSE_DATA_BUFFER) <= 24); + + /* Try to estimate reparse point */ + if (!attr->non_res) { + rp = resident_data_ex(attr, sizeof(struct REPARSE_DATA_BUFFER)); + } else if (le64_to_cpu(attr->nres.data_size) >= + sizeof(struct REPARSE_DATA_BUFFER)) { + struct runs_tree run; + + run_init(&run); + + if (!attr_load_runs_vcn(ni, ATTR_REPARSE, NULL, 0, &run, 0) && + !ntfs_read_run_nb(ni->mi.sbi, &run, 0, buffer, + sizeof(struct REPARSE_DATA_BUFFER), + NULL)) { + rp = buffer; + } + + run_close(&run); + } + + if (!rp) + return REPARSE_NONE; + + len = le16_to_cpu(rp->ReparseDataLength); + switch (rp->ReparseTag) { + case (IO_REPARSE_TAG_MICROSOFT | IO_REPARSE_TAG_SYMBOLIC_LINK): + break; /* Symbolic link */ + case IO_REPARSE_TAG_MOUNT_POINT: + break; /* Mount points and junctions */ + case IO_REPARSE_TAG_SYMLINK: + break; + case IO_REPARSE_TAG_COMPRESS: + /* + * WOF - Windows Overlay Filter - used to compress files with lzx/xpress + * Unlike native NTFS file compression, the Windows Overlay Filter supports + * only read operations. This means that it doesn’t need to sector-align each + * compressed chunk, so the compressed data can be packed more tightly together. + * If you open the file for writing, the Windows Overlay Filter just decompresses + * the entire file, turning it back into a plain file. + * + * ntfs3 driver decompresses the entire file only on write or change size requests + */ + + cmpr = &rp->CompressReparseBuffer; + if (len < sizeof(*cmpr) || + cmpr->WofVersion != WOF_CURRENT_VERSION || + cmpr->WofProvider != WOF_PROVIDER_SYSTEM || + cmpr->ProviderVer != WOF_PROVIDER_CURRENT_VERSION) { + return REPARSE_NONE; + } + + switch (cmpr->CompressionFormat) { + case WOF_COMPRESSION_XPRESS4K: + bits = 0xc; // 4k + break; + case WOF_COMPRESSION_XPRESS8K: + bits = 0xd; // 8k + break; + case WOF_COMPRESSION_XPRESS16K: + bits = 0xe; // 16k + break; + case WOF_COMPRESSION_LZX32K: + bits = 0xf; // 32k + break; + default: + bits = 0x10; // 64k + break; + } + ni_set_ext_compress_bits(ni, bits); + return REPARSE_COMPRESSED; + + case IO_REPARSE_TAG_DEDUP: + ni->ni_flags |= NI_FLAG_DEDUPLICATED; + return REPARSE_DEDUPLICATED; + + default: + if (rp->ReparseTag & IO_REPARSE_TAG_NAME_SURROGATE) + break; + + return REPARSE_NONE; + } + + /* Looks like normal symlink */ + return REPARSE_LINK; +} + +/* + * helper for file_fiemap + * assumed ni_lock + * TODO: less aggressive locks + */ +int ni_fiemap(struct ntfs_inode *ni, struct fiemap_extent_info *fieinfo, + __u64 vbo, __u64 len) +{ + int err = 0; + struct ntfs_sb_info *sbi = ni->mi.sbi; + u8 cluster_bits = sbi->cluster_bits; + struct runs_tree *run; + struct rw_semaphore *run_lock; + struct ATTRIB *attr; + CLST vcn = vbo >> cluster_bits; + CLST lcn, clen; + u64 valid = ni->i_valid; + u64 lbo, bytes; + u64 end, alloc_size; + size_t idx = -1; + u32 flags; + bool ok; + + if (S_ISDIR(ni->vfs_inode.i_mode)) { + run = &ni->dir.alloc_run; + attr = ni_find_attr(ni, NULL, NULL, ATTR_ALLOC, I30_NAME, + ARRAY_SIZE(I30_NAME), NULL, NULL); + run_lock = &ni->dir.run_lock; + } else { + run = &ni->file.run; + attr = ni_find_attr(ni, NULL, NULL, ATTR_DATA, NULL, 0, NULL, + NULL); + if (!attr) { + err = -EINVAL; + goto out; + } + if (is_attr_compressed(attr)) { + /*unfortunately cp -r incorrectly treats compressed clusters*/ + err = -EOPNOTSUPP; + ntfs_inode_warn( + &ni->vfs_inode, + "fiemap is not supported for compressed file (cp -r)"); + goto out; + } + run_lock = &ni->file.run_lock; + } + + if (!attr || !attr->non_res) { + err = fiemap_fill_next_extent( + fieinfo, 0, 0, + attr ? le32_to_cpu(attr->res.data_size) : 0, + FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_LAST | + FIEMAP_EXTENT_MERGED); + goto out; + } + + end = vbo + len; + alloc_size = le64_to_cpu(attr->nres.alloc_size); + if (end > alloc_size) + end = alloc_size; + + down_read(run_lock); + + while (vbo < end) { + if (idx == -1) { + ok = run_lookup_entry(run, vcn, &lcn, &clen, &idx); + } else { + CLST vcn_next = vcn; + + ok = run_get_entry(run, ++idx, &vcn, &lcn, &clen) && + vcn == vcn_next; + if (!ok) + vcn = vcn_next; + } + + if (!ok) { + up_read(run_lock); + down_write(run_lock); + + err = attr_load_runs_vcn(ni, attr->type, + attr_name(attr), + attr->name_len, run, vcn); + + up_write(run_lock); + down_read(run_lock); + + if (err) + break; + + ok = run_lookup_entry(run, vcn, &lcn, &clen, &idx); + + if (!ok) { + err = -EINVAL; + break; + } + } + + if (!clen) { + err = -EINVAL; // ? + break; + } + + if (lcn == SPARSE_LCN) { + vcn += clen; + vbo = (u64)vcn << cluster_bits; + continue; + } + + flags = FIEMAP_EXTENT_MERGED; + if (S_ISDIR(ni->vfs_inode.i_mode)) { + ; + } else if (is_attr_compressed(attr)) { + CLST clst_data; + + err = attr_is_frame_compressed( + ni, attr, vcn >> attr->nres.c_unit, &clst_data); + if (err) + break; + if (clst_data < NTFS_LZNT_CLUSTERS) + flags |= FIEMAP_EXTENT_ENCODED; + } else if (is_attr_encrypted(attr)) { + flags |= FIEMAP_EXTENT_DATA_ENCRYPTED; + } + + vbo = (u64)vcn << cluster_bits; + bytes = (u64)clen << cluster_bits; + lbo = (u64)lcn << cluster_bits; + + vcn += clen; + + if (vbo + bytes >= end) { + bytes = end - vbo; + flags |= FIEMAP_EXTENT_LAST; + } + + if (vbo + bytes <= valid) { + ; + } else if (vbo >= valid) { + flags |= FIEMAP_EXTENT_UNWRITTEN; + } else { + /* vbo < valid && valid < vbo + bytes */ + u64 dlen = valid - vbo; + + err = fiemap_fill_next_extent(fieinfo, vbo, lbo, dlen, + flags); + if (err < 0) + break; + if (err == 1) { + err = 0; + break; + } + + vbo = valid; + bytes -= dlen; + if (!bytes) + continue; + + lbo += dlen; + flags |= FIEMAP_EXTENT_UNWRITTEN; + } + + err = fiemap_fill_next_extent(fieinfo, vbo, lbo, bytes, flags); + if (err < 0) + break; + if (err == 1) { + err = 0; + break; + } + + vbo += bytes; + } + + up_read(run_lock); + +out: + return err; +} + +/* + * When decompressing, we typically obtain more than one page per reference. + * We inject the additional pages into the page cache. + */ +int ni_readpage_cmpr(struct ntfs_inode *ni, struct page *page) +{ + int err; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct address_space *mapping = page->mapping; + pgoff_t index = page->index; + u64 frame_vbo, vbo = (u64)index << PAGE_SHIFT; + struct page **pages = NULL; /*array of at most 16 pages. stack?*/ + u8 frame_bits; + CLST frame; + u32 i, idx, frame_size, pages_per_frame; + gfp_t gfp_mask; + struct page *pg; + + if (vbo >= ni->vfs_inode.i_size) { + SetPageUptodate(page); + err = 0; + goto out; + } + + if (ni->ni_flags & NI_FLAG_COMPRESSED_MASK) { + /* xpress or lzx */ + frame_bits = ni_ext_compress_bits(ni); + } else { + /* lznt compression*/ + frame_bits = NTFS_LZNT_CUNIT + sbi->cluster_bits; + } + frame_size = 1u << frame_bits; + frame = vbo >> frame_bits; + frame_vbo = (u64)frame << frame_bits; + idx = (vbo - frame_vbo) >> PAGE_SHIFT; + + pages_per_frame = frame_size >> PAGE_SHIFT; + pages = ntfs_zalloc(pages_per_frame * sizeof(struct page *)); + if (!pages) { + err = -ENOMEM; + goto out; + } + + pages[idx] = page; + index = frame_vbo >> PAGE_SHIFT; + gfp_mask = mapping_gfp_mask(mapping); + + for (i = 0; i < pages_per_frame; i++, index++) { + if (i == idx) + continue; + + pg = find_or_create_page(mapping, index, gfp_mask); + if (!pg) { + err = -ENOMEM; + goto out1; + } + pages[i] = pg; + } + + err = ni_read_frame(ni, frame_vbo, pages, pages_per_frame); + +out1: + if (err) + SetPageError(page); + + for (i = 0; i < pages_per_frame; i++) { + pg = pages[i]; + if (i == idx) + continue; + unlock_page(pg); + put_page(pg); + } + +out: + /* At this point, err contains 0 or -EIO depending on the "critical" page */ + ntfs_free(pages); + unlock_page(page); + + return err; +} + +#ifdef CONFIG_NTFS3_LZX_XPRESS +/* + * decompress lzx/xpress compressed file + * remove ATTR_DATA::WofCompressedData + * remove ATTR_REPARSE + */ +int ni_decompress_file(struct ntfs_inode *ni) +{ + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct inode *inode = &ni->vfs_inode; + loff_t i_size = inode->i_size; + struct address_space *mapping = inode->i_mapping; + gfp_t gfp_mask = mapping_gfp_mask(mapping); + struct page **pages = NULL; + struct ATTR_LIST_ENTRY *le; + struct ATTRIB *attr; + CLST vcn, cend, lcn, clen, end; + pgoff_t index; + u64 vbo; + u8 frame_bits; + u32 i, frame_size, pages_per_frame, bytes; + struct mft_inode *mi; + int err; + + /* clusters for decompressed data*/ + cend = bytes_to_cluster(sbi, i_size); + + if (!i_size) + goto remove_wof; + + /* check in advance */ + if (cend > wnd_zeroes(&sbi->used.bitmap)) { + err = -ENOSPC; + goto out; + } + + frame_bits = ni_ext_compress_bits(ni); + frame_size = 1u << frame_bits; + pages_per_frame = frame_size >> PAGE_SHIFT; + pages = ntfs_zalloc(pages_per_frame * sizeof(struct page *)); + if (!pages) { + err = -ENOMEM; + goto out; + } + + /* + * Step 1: decompress data and copy to new allocated clusters + */ + index = 0; + for (vbo = 0; vbo < i_size; vbo += bytes) { + u32 nr_pages; + bool new; + + if (vbo + frame_size > i_size) { + bytes = i_size - vbo; + nr_pages = (bytes + PAGE_SIZE - 1) >> PAGE_SHIFT; + } else { + nr_pages = pages_per_frame; + bytes = frame_size; + } + + end = bytes_to_cluster(sbi, vbo + bytes); + + for (vcn = vbo >> sbi->cluster_bits; vcn < end; vcn += clen) { + err = attr_data_get_block(ni, vcn, cend - vcn, &lcn, + &clen, &new); + if (err) + goto out; + } + + for (i = 0; i < pages_per_frame; i++, index++) { + struct page *pg; + + pg = find_or_create_page(mapping, index, gfp_mask); + if (!pg) { + while (i--) { + unlock_page(pages[i]); + put_page(pages[i]); + } + err = -ENOMEM; + goto out; + } + pages[i] = pg; + } + + err = ni_read_frame(ni, vbo, pages, pages_per_frame); + + if (!err) { + down_read(&ni->file.run_lock); + err = ntfs_bio_pages(sbi, &ni->file.run, pages, + nr_pages, vbo, bytes, + REQ_OP_WRITE); + up_read(&ni->file.run_lock); + } + + for (i = 0; i < pages_per_frame; i++) { + unlock_page(pages[i]); + put_page(pages[i]); + } + + if (err) + goto out; + + cond_resched(); + } + +remove_wof: + /* + * Step 2: deallocate attributes ATTR_DATA::WofCompressedData and ATTR_REPARSE + */ + attr = NULL; + le = NULL; + while ((attr = ni_enum_attr_ex(ni, attr, &le, NULL))) { + CLST svcn, evcn; + u32 asize, roff; + + if (attr->type == ATTR_REPARSE) { + struct MFT_REF ref; + + mi_get_ref(&ni->mi, &ref); + ntfs_remove_reparse(sbi, 0, &ref); + } + + if (!attr->non_res) + continue; + + if (attr->type != ATTR_REPARSE && + (attr->type != ATTR_DATA || + attr->name_len != ARRAY_SIZE(WOF_NAME) || + memcmp(attr_name(attr), WOF_NAME, sizeof(WOF_NAME)))) + continue; + + svcn = le64_to_cpu(attr->nres.svcn); + evcn = le64_to_cpu(attr->nres.evcn); + + if (evcn + 1 <= svcn) + continue; + + asize = le32_to_cpu(attr->size); + roff = le16_to_cpu(attr->nres.run_off); + + /*run==1 means unpack and deallocate*/ + run_unpack_ex(RUN_DEALLOCATE, sbi, ni->mi.rno, svcn, evcn, svcn, + Add2Ptr(attr, roff), asize - roff); + } + + /* + * Step 3: remove attribute ATTR_DATA::WofCompressedData + */ + err = ni_remove_attr(ni, ATTR_DATA, WOF_NAME, ARRAY_SIZE(WOF_NAME), + false, NULL); + if (err) + goto out; + + /* + * Step 4: remove ATTR_REPARSE + */ + err = ni_remove_attr(ni, ATTR_REPARSE, NULL, 0, false, NULL); + if (err) + goto out; + + /* + * Step 5: remove sparse flag from data attribute + */ + attr = ni_find_attr(ni, NULL, NULL, ATTR_DATA, NULL, 0, NULL, &mi); + if (!attr) { + err = -EINVAL; + goto out; + } + + if (attr->non_res && is_attr_sparsed(attr)) { + /* sparsed attribute header is 8 bytes bigger than normal*/ + struct MFT_REC *rec = mi->mrec; + u32 used = le32_to_cpu(rec->used); + u32 asize = le32_to_cpu(attr->size); + u16 roff = le16_to_cpu(attr->nres.run_off); + char *rbuf = Add2Ptr(attr, roff); + + memmove(rbuf - 8, rbuf, used - PtrOffset(rec, rbuf)); + attr->size = cpu_to_le32(asize - 8); + attr->flags &= ~ATTR_FLAG_SPARSED; + attr->nres.run_off = cpu_to_le16(roff - 8); + attr->nres.c_unit = 0; + rec->used = cpu_to_le32(used - 8); + mi->dirty = true; + ni->std_fa &= ~(FILE_ATTRIBUTE_SPARSE_FILE | + FILE_ATTRIBUTE_REPARSE_POINT); + + mark_inode_dirty(inode); + } + + /* clear cached flag */ + ni->ni_flags &= ~NI_FLAG_COMPRESSED_MASK; + if (ni->file.offs_page) { + put_page(ni->file.offs_page); + ni->file.offs_page = NULL; + } + mapping->a_ops = &ntfs_aops; + +out: + ntfs_free(pages); + if (err) { + make_bad_inode(inode); + ntfs_set_state(sbi, NTFS_DIRTY_ERROR); + } + + return err; +} + +/* external compression lzx/xpress */ +static int decompress_lzx_xpress(struct ntfs_sb_info *sbi, const char *cmpr, + size_t cmpr_size, void *unc, size_t unc_size, + u32 frame_size) +{ + int err; + void *ctx; + + if (cmpr_size == unc_size) { + /* frame not compressed */ + memcpy(unc, cmpr, unc_size); + return 0; + } + + err = 0; + if (frame_size == 0x8000) { + mutex_lock(&sbi->compress.mtx_lzx); + /* LZX: frame compressed */ + ctx = sbi->compress.lzx; + if (!ctx) { + /* Lazy initialize lzx decompress context */ + ctx = lzx_allocate_decompressor(); + if (!ctx) { + err = -ENOMEM; + goto out1; + } + + sbi->compress.lzx = ctx; + } + + if (lzx_decompress(ctx, cmpr, cmpr_size, unc, unc_size)) { + /* treat all errors as "invalid argument" */ + err = -EINVAL; + } +out1: + mutex_unlock(&sbi->compress.mtx_lzx); + } else { + /* XPRESS: frame compressed */ + mutex_lock(&sbi->compress.mtx_xpress); + ctx = sbi->compress.xpress; + if (!ctx) { + /* Lazy initialize xpress decompress context */ + ctx = xpress_allocate_decompressor(); + if (!ctx) { + err = -ENOMEM; + goto out2; + } + + sbi->compress.xpress = ctx; + } + + if (xpress_decompress(ctx, cmpr, cmpr_size, unc, unc_size)) { + /* treat all errors as "invalid argument" */ + err = -EINVAL; + } +out2: + mutex_unlock(&sbi->compress.mtx_xpress); + } + return err; +} +#endif + +/* + * ni_read_frame + * + * pages - array of locked pages + */ +int ni_read_frame(struct ntfs_inode *ni, u64 frame_vbo, struct page **pages, + u32 pages_per_frame) +{ + int err; + struct ntfs_sb_info *sbi = ni->mi.sbi; + u8 cluster_bits = sbi->cluster_bits; + char *frame_ondisk = NULL; + char *frame_mem = NULL; + struct page **pages_disk = NULL; + struct ATTR_LIST_ENTRY *le = NULL; + struct runs_tree *run = &ni->file.run; + u64 valid_size = ni->i_valid; + u64 vbo_disk; + size_t unc_size; + u32 frame_size, i, npages_disk, ondisk_size; + struct page *pg; + struct ATTRIB *attr; + CLST frame, clst_data; + + /* + * To simplify decompress algorithm do vmap for source and target pages + */ + for (i = 0; i < pages_per_frame; i++) + kmap(pages[i]); + + frame_size = pages_per_frame << PAGE_SHIFT; + frame_mem = vmap(pages, pages_per_frame, VM_MAP, PAGE_KERNEL); + if (!frame_mem) { + err = -ENOMEM; + goto out; + } + + attr = ni_find_attr(ni, NULL, &le, ATTR_DATA, NULL, 0, NULL, NULL); + if (!attr) { + err = -ENOENT; + goto out1; + } + + if (!attr->non_res) { + u32 data_size = le32_to_cpu(attr->res.data_size); + + memset(frame_mem, 0, frame_size); + if (frame_vbo < data_size) { + ondisk_size = data_size - frame_vbo; + memcpy(frame_mem, resident_data(attr) + frame_vbo, + min(ondisk_size, frame_size)); + } + err = 0; + goto out1; + } + + if (frame_vbo >= valid_size) { + memset(frame_mem, 0, frame_size); + err = 0; + goto out1; + } + + if (ni->ni_flags & NI_FLAG_COMPRESSED_MASK) { +#ifndef CONFIG_NTFS3_LZX_XPRESS + err = -EOPNOTSUPP; + goto out1; +#else + u32 frame_bits = ni_ext_compress_bits(ni); + u64 frame64 = frame_vbo >> frame_bits; + u64 frames, vbo_data; + + if (frame_size != (1u << frame_bits)) { + err = -EINVAL; + goto out1; + } + switch (frame_size) { + case 0x1000: + case 0x2000: + case 0x4000: + case 0x8000: + break; + default: + /* unknown compression */ + err = -EOPNOTSUPP; + goto out1; + } + + attr = ni_find_attr(ni, attr, &le, ATTR_DATA, WOF_NAME, + ARRAY_SIZE(WOF_NAME), NULL, NULL); + if (!attr) { + ntfs_inode_err( + &ni->vfs_inode, + "external compressed file should contains data attribute \"WofCompressedData\""); + err = -EINVAL; + goto out1; + } + + if (!attr->non_res) { + run = NULL; + } else { + run = run_alloc(); + if (!run) { + err = -ENOMEM; + goto out1; + } + } + + frames = (ni->vfs_inode.i_size - 1) >> frame_bits; + + err = attr_wof_frame_info(ni, attr, run, frame64, frames, + frame_bits, &ondisk_size, &vbo_data); + if (err) + goto out2; + + if (frame64 == frames) { + unc_size = 1 + ((ni->vfs_inode.i_size - 1) & + (frame_size - 1)); + ondisk_size = attr_size(attr) - vbo_data; + } else { + unc_size = frame_size; + } + + if (ondisk_size > frame_size) { + err = -EINVAL; + goto out2; + } + + if (!attr->non_res) { + if (vbo_data + ondisk_size > + le32_to_cpu(attr->res.data_size)) { + err = -EINVAL; + goto out1; + } + + err = decompress_lzx_xpress( + sbi, Add2Ptr(resident_data(attr), vbo_data), + ondisk_size, frame_mem, unc_size, frame_size); + goto out1; + } + vbo_disk = vbo_data; + /* load all runs to read [vbo_disk-vbo_to) */ + err = attr_load_runs_range(ni, ATTR_DATA, WOF_NAME, + ARRAY_SIZE(WOF_NAME), run, vbo_disk, + vbo_data + ondisk_size); + if (err) + goto out2; + npages_disk = (ondisk_size + (vbo_disk & (PAGE_SIZE - 1)) + + PAGE_SIZE - 1) >> + PAGE_SHIFT; +#endif + } else if (is_attr_compressed(attr)) { + /* lznt compression*/ + if (sbi->cluster_size > NTFS_LZNT_MAX_CLUSTER) { + err = -EOPNOTSUPP; + goto out1; + } + + if (attr->nres.c_unit != NTFS_LZNT_CUNIT) { + err = -EOPNOTSUPP; + goto out1; + } + + down_write(&ni->file.run_lock); + run_truncate_around(run, le64_to_cpu(attr->nres.svcn)); + frame = frame_vbo >> (cluster_bits + NTFS_LZNT_CUNIT); + err = attr_is_frame_compressed(ni, attr, frame, &clst_data); + up_write(&ni->file.run_lock); + if (err) + goto out1; + + if (!clst_data) { + memset(frame_mem, 0, frame_size); + goto out1; + } + + frame_size = sbi->cluster_size << NTFS_LZNT_CUNIT; + ondisk_size = clst_data << cluster_bits; + + if (clst_data >= NTFS_LZNT_CLUSTERS) { + /* frame is not compressed */ + down_read(&ni->file.run_lock); + err = ntfs_bio_pages(sbi, run, pages, pages_per_frame, + frame_vbo, ondisk_size, + REQ_OP_READ); + up_read(&ni->file.run_lock); + goto out1; + } + vbo_disk = frame_vbo; + npages_disk = (ondisk_size + PAGE_SIZE - 1) >> PAGE_SHIFT; + } else { + __builtin_unreachable(); + err = -EINVAL; + goto out1; + } + + pages_disk = ntfs_zalloc(npages_disk * sizeof(struct page *)); + if (!pages_disk) { + err = -ENOMEM; + goto out2; + } + + for (i = 0; i < npages_disk; i++) { + pg = alloc_page(GFP_KERNEL); + if (!pg) { + err = -ENOMEM; + goto out3; + } + pages_disk[i] = pg; + lock_page(pg); + kmap(pg); + } + + /* read 'ondisk_size' bytes from disk */ + down_read(&ni->file.run_lock); + err = ntfs_bio_pages(sbi, run, pages_disk, npages_disk, vbo_disk, + ondisk_size, REQ_OP_READ); + up_read(&ni->file.run_lock); + if (err) + goto out3; + + /* + * To simplify decompress algorithm do vmap for source and target pages + */ + frame_ondisk = vmap(pages_disk, npages_disk, VM_MAP, PAGE_KERNEL_RO); + if (!frame_ondisk) { + err = -ENOMEM; + goto out3; + } + + /* decompress: frame_ondisk -> frame_mem */ +#ifdef CONFIG_NTFS3_LZX_XPRESS + if (run != &ni->file.run) { + /* LZX or XPRESS */ + err = decompress_lzx_xpress( + sbi, frame_ondisk + (vbo_disk & (PAGE_SIZE - 1)), + ondisk_size, frame_mem, unc_size, frame_size); + } else +#endif + { + /* LZNT - native ntfs compression */ + unc_size = decompress_lznt(frame_ondisk, ondisk_size, frame_mem, + frame_size); + if ((ssize_t)unc_size < 0) + err = unc_size; + else if (!unc_size || unc_size > frame_size) + err = -EINVAL; + } + if (!err && valid_size < frame_vbo + frame_size) { + size_t ok = valid_size - frame_vbo; + + memset(frame_mem + ok, 0, frame_size - ok); + } + + vunmap(frame_ondisk); + +out3: + for (i = 0; i < npages_disk; i++) { + pg = pages_disk[i]; + if (pg) { + kunmap(pg); + unlock_page(pg); + put_page(pg); + } + } + ntfs_free(pages_disk); + +out2: +#ifdef CONFIG_NTFS3_LZX_XPRESS + if (run != &ni->file.run) + run_free(run); +#endif +out1: + vunmap(frame_mem); +out: + for (i = 0; i < pages_per_frame; i++) { + pg = pages[i]; + kunmap(pg); + ClearPageError(pg); + SetPageUptodate(pg); + } + + return err; +} + +/* + * ni_write_frame + * + * pages - array of locked pages + */ +int ni_write_frame(struct ntfs_inode *ni, struct page **pages, + u32 pages_per_frame) +{ + int err; + struct ntfs_sb_info *sbi = ni->mi.sbi; + u8 frame_bits = NTFS_LZNT_CUNIT + sbi->cluster_bits; + u32 frame_size = sbi->cluster_size << NTFS_LZNT_CUNIT; + u64 frame_vbo = (u64)pages[0]->index << PAGE_SHIFT; + CLST frame = frame_vbo >> frame_bits; + char *frame_ondisk = NULL; + struct page **pages_disk = NULL; + struct ATTR_LIST_ENTRY *le = NULL; + char *frame_mem; + struct ATTRIB *attr; + struct mft_inode *mi; + u32 i; + struct page *pg; + size_t compr_size, ondisk_size; + struct lznt *lznt; + + attr = ni_find_attr(ni, NULL, &le, ATTR_DATA, NULL, 0, NULL, &mi); + if (!attr) { + err = -ENOENT; + goto out; + } + + if (WARN_ON(!is_attr_compressed(attr))) { + err = -EINVAL; + goto out; + } + + if (sbi->cluster_size > NTFS_LZNT_MAX_CLUSTER) { + err = -EOPNOTSUPP; + goto out; + } + + if (!attr->non_res) { + down_write(&ni->file.run_lock); + err = attr_make_nonresident(ni, attr, le, mi, + le32_to_cpu(attr->res.data_size), + &ni->file.run, &attr, pages[0]); + up_write(&ni->file.run_lock); + if (err) + goto out; + } + + if (attr->nres.c_unit != NTFS_LZNT_CUNIT) { + err = -EOPNOTSUPP; + goto out; + } + + pages_disk = ntfs_zalloc(pages_per_frame * sizeof(struct page *)); + if (!pages_disk) { + err = -ENOMEM; + goto out; + } + + for (i = 0; i < pages_per_frame; i++) { + pg = alloc_page(GFP_KERNEL); + if (!pg) { + err = -ENOMEM; + goto out1; + } + pages_disk[i] = pg; + lock_page(pg); + kmap(pg); + } + + /* + * To simplify compress algorithm do vmap for source and target pages + */ + frame_ondisk = vmap(pages_disk, pages_per_frame, VM_MAP, PAGE_KERNEL); + if (!frame_ondisk) { + err = -ENOMEM; + goto out1; + } + + for (i = 0; i < pages_per_frame; i++) + kmap(pages[i]); + + /* map in-memory frame for read-only */ + frame_mem = vmap(pages, pages_per_frame, VM_MAP, PAGE_KERNEL_RO); + if (!frame_mem) { + err = -ENOMEM; + goto out2; + } + + mutex_lock(&sbi->compress.mtx_lznt); + lznt = NULL; + if (!sbi->compress.lznt) { + /* + * lznt implements two levels of compression: + * 0 - standard compression + * 1 - best compression, requires a lot of cpu + * use mount option? + */ + lznt = get_lznt_ctx(0); + if (!lznt) { + mutex_unlock(&sbi->compress.mtx_lznt); + err = -ENOMEM; + goto out3; + } + + sbi->compress.lznt = lznt; + lznt = NULL; + } + + /* compress: frame_mem -> frame_ondisk */ + compr_size = compress_lznt(frame_mem, frame_size, frame_ondisk, + frame_size, sbi->compress.lznt); + mutex_unlock(&sbi->compress.mtx_lznt); + ntfs_free(lznt); + + if (compr_size + sbi->cluster_size > frame_size) { + /* frame is not compressed */ + compr_size = frame_size; + ondisk_size = frame_size; + } else if (compr_size) { + /* frame is compressed */ + ondisk_size = ntfs_up_cluster(sbi, compr_size); + memset(frame_ondisk + compr_size, 0, ondisk_size - compr_size); + } else { + /* frame is sparsed */ + ondisk_size = 0; + } + + down_write(&ni->file.run_lock); + run_truncate_around(&ni->file.run, le64_to_cpu(attr->nres.svcn)); + err = attr_allocate_frame(ni, frame, compr_size, ni->i_valid); + up_write(&ni->file.run_lock); + if (err) + goto out2; + + if (!ondisk_size) + goto out2; + + down_read(&ni->file.run_lock); + err = ntfs_bio_pages(sbi, &ni->file.run, + ondisk_size < frame_size ? pages_disk : pages, + pages_per_frame, frame_vbo, ondisk_size, + REQ_OP_WRITE); + up_read(&ni->file.run_lock); + +out3: + vunmap(frame_mem); + +out2: + for (i = 0; i < pages_per_frame; i++) + kunmap(pages[i]); + + vunmap(frame_ondisk); +out1: + for (i = 0; i < pages_per_frame; i++) { + pg = pages_disk[i]; + if (pg) { + kunmap(pg); + unlock_page(pg); + put_page(pg); + } + } + ntfs_free(pages_disk); +out: + return err; +} + +/* + * update duplicate info of ATTR_FILE_NAME in MFT and in parent directories + */ +static bool ni_update_parent(struct ntfs_inode *ni, struct NTFS_DUP_INFO *dup, + int sync) +{ + struct ATTRIB *attr; + struct mft_inode *mi; + struct ATTR_LIST_ENTRY *le = NULL; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct super_block *sb = sbi->sb; + bool re_dirty = false; + bool active = sb->s_flags & SB_ACTIVE; + bool upd_parent = ni->ni_flags & NI_FLAG_UPDATE_PARENT; + + if (ni->mi.mrec->flags & RECORD_FLAG_DIR) { + dup->fa |= FILE_ATTRIBUTE_DIRECTORY; + attr = NULL; + dup->alloc_size = 0; + dup->data_size = 0; + } else { + dup->fa &= ~FILE_ATTRIBUTE_DIRECTORY; + + attr = ni_find_attr(ni, NULL, &le, ATTR_DATA, NULL, 0, NULL, + &mi); + if (!attr) { + dup->alloc_size = dup->data_size = 0; + } else if (!attr->non_res) { + u32 data_size = le32_to_cpu(attr->res.data_size); + + dup->alloc_size = cpu_to_le64(QuadAlign(data_size)); + dup->data_size = cpu_to_le64(data_size); + } else { + u64 new_valid = ni->i_valid; + u64 data_size = le64_to_cpu(attr->nres.data_size); + __le64 valid_le; + + dup->alloc_size = is_attr_ext(attr) + ? attr->nres.total_size + : attr->nres.alloc_size; + dup->data_size = attr->nres.data_size; + + if (new_valid > data_size) + new_valid = data_size; + + valid_le = cpu_to_le64(new_valid); + if (valid_le != attr->nres.valid_size) { + attr->nres.valid_size = valid_le; + mi->dirty = true; + } + } + } + + /* TODO: fill reparse info */ + dup->reparse = 0; + dup->ea_size = 0; + + if (ni->ni_flags & NI_FLAG_EA) { + attr = ni_find_attr(ni, attr, &le, ATTR_EA_INFO, NULL, 0, NULL, + NULL); + if (attr) { + const struct EA_INFO *info; + + info = resident_data_ex(attr, sizeof(struct EA_INFO)); + dup->ea_size = info->size_pack; + } + } + + attr = NULL; + le = NULL; + + while ((attr = ni_find_attr(ni, attr, &le, ATTR_NAME, NULL, 0, NULL, + &mi))) { + struct inode *dir; + struct ATTR_FILE_NAME *fname; + + fname = resident_data_ex(attr, SIZEOF_ATTRIBUTE_FILENAME); + if (!fname) + continue; + + if (memcmp(&fname->dup, dup, sizeof(fname->dup))) { + memcpy(&fname->dup, dup, sizeof(fname->dup)); + mi->dirty = true; + } else if (!upd_parent) { + continue; + } + + if (!active) + continue; /*avoid __wait_on_freeing_inode(inode); */ + + /*ntfs_iget5 may sleep*/ + dir = ntfs_iget5(sb, &fname->home, NULL); + if (IS_ERR(dir)) { + ntfs_inode_warn( + &ni->vfs_inode, + "failed to open parent directory r=%lx to update", + (long)ino_get(&fname->home)); + continue; + } + + if (!is_bad_inode(dir)) { + struct ntfs_inode *dir_ni = ntfs_i(dir); + + if (!ni_trylock(dir_ni)) { + re_dirty = true; + } else { + indx_update_dup(dir_ni, sbi, fname, dup, sync); + ni_unlock(dir_ni); + } + } + iput(dir); + } + + return re_dirty; +} + +/* + * ni_write_inode + * + * write mft base record and all subrecords to disk + */ +int ni_write_inode(struct inode *inode, int sync, const char *hint) +{ + int err = 0, err2; + struct ntfs_inode *ni = ntfs_i(inode); + struct super_block *sb = inode->i_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + bool re_dirty = false; + struct ATTR_STD_INFO *std; + struct rb_node *node, *next; + struct NTFS_DUP_INFO dup; + + if (is_bad_inode(inode) || sb_rdonly(sb)) + return 0; + + if (!ni_trylock(ni)) { + /* 'ni' is under modification, skip for now */ + mark_inode_dirty_sync(inode); + return 0; + } + + if (is_rec_inuse(ni->mi.mrec) && + !(sbi->flags & NTFS_FLAGS_LOG_REPLAYING) && inode->i_nlink) { + bool modified = false; + + /* update times in standard attribute */ + std = ni_std(ni); + if (!std) { + err = -EINVAL; + goto out; + } + + /* Update the access times if they have changed. */ + dup.m_time = kernel2nt(&inode->i_mtime); + if (std->m_time != dup.m_time) { + std->m_time = dup.m_time; + modified = true; + } + + dup.c_time = kernel2nt(&inode->i_ctime); + if (std->c_time != dup.c_time) { + std->c_time = dup.c_time; + modified = true; + } + + dup.a_time = kernel2nt(&inode->i_atime); + if (std->a_time != dup.a_time) { + std->a_time = dup.a_time; + modified = true; + } + + dup.fa = ni->std_fa; + if (std->fa != dup.fa) { + std->fa = dup.fa; + modified = true; + } + + if (modified) + ni->mi.dirty = true; + + if (!ntfs_is_meta_file(sbi, inode->i_ino) && + (modified || (ni->ni_flags & NI_FLAG_UPDATE_PARENT))) { + dup.cr_time = std->cr_time; + /* Not critical if this function fail */ + re_dirty = ni_update_parent(ni, &dup, sync); + + if (re_dirty) + ni->ni_flags |= NI_FLAG_UPDATE_PARENT; + else + ni->ni_flags &= ~NI_FLAG_UPDATE_PARENT; + } + + /* update attribute list */ + if (ni->attr_list.size && ni->attr_list.dirty) { + if (inode->i_ino != MFT_REC_MFT || sync) { + err = ni_try_remove_attr_list(ni); + if (err) + goto out; + } + + err = al_update(ni); + if (err) + goto out; + } + } + + for (node = rb_first(&ni->mi_tree); node; node = next) { + struct mft_inode *mi = rb_entry(node, struct mft_inode, node); + bool is_empty; + + next = rb_next(node); + + if (!mi->dirty) + continue; + + is_empty = !mi_enum_attr(mi, NULL); + + if (is_empty) + clear_rec_inuse(mi->mrec); + + err2 = mi_write(mi, sync); + if (!err && err2) + err = err2; + + if (is_empty) { + ntfs_mark_rec_free(sbi, mi->rno); + rb_erase(node, &ni->mi_tree); + mi_put(mi); + } + } + + if (ni->mi.dirty) { + err2 = mi_write(&ni->mi, sync); + if (!err && err2) + err = err2; + } +out: + ni_unlock(ni); + + if (err) { + ntfs_err(sb, "%s r=%lx failed, %d.", hint, inode->i_ino, err); + ntfs_set_state(sbi, NTFS_DIRTY_ERROR); + return err; + } + + if (re_dirty && (sb->s_flags & SB_ACTIVE)) + mark_inode_dirty_sync(inode); + + return 0; +} diff --git a/fs/ntfs3/fslog.c b/fs/ntfs3/fslog.c new file mode 100644 index 000000000000..53da12252408 --- /dev/null +++ b/fs/ntfs3/fslog.c @@ -0,0 +1,5181 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +/* + * LOG FILE structs + */ + +// clang-format off + +#define MaxLogFileSize 0x100000000ull +#define DefaultLogPageSize 4096 +#define MinLogRecordPages 0x30 + +struct RESTART_HDR { + struct NTFS_RECORD_HEADER rhdr; // 'RSTR' + __le32 sys_page_size; // 0x10: Page size of the system which initialized the log + __le32 page_size; // 0x14: Log page size used for this log file + __le16 ra_off; // 0x18: + __le16 minor_ver; // 0x1A: + __le16 major_ver; // 0x1C: + __le16 fixups[]; +}; + +#define LFS_NO_CLIENT 0xffff +#define LFS_NO_CLIENT_LE cpu_to_le16(0xffff) + +struct CLIENT_REC { + __le64 oldest_lsn; + __le64 restart_lsn; // 0x08: + __le16 prev_client; // 0x10: + __le16 next_client; // 0x12: + __le16 seq_num; // 0x14: + u8 align[6]; // 0x16 + __le32 name_bytes; // 0x1C: in bytes + __le16 name[32]; // 0x20: name of client +}; + +static_assert(sizeof(struct CLIENT_REC) == 0x60); + +/* Two copies of these will exist at the beginning of the log file */ +struct RESTART_AREA { + __le64 current_lsn; // 0x00: Current logical end of log file + __le16 log_clients; // 0x08: Maximum number of clients + __le16 client_idx[2]; // 0x0A: free/use index into the client record arrays + __le16 flags; // 0x0E: See RESTART_SINGLE_PAGE_IO + __le32 seq_num_bits; // 0x10: the number of bits in sequence number. + __le16 ra_len; // 0x14: + __le16 client_off; // 0x16: + __le64 l_size; // 0x18: Usable log file size. + __le32 last_lsn_data_len; // 0x20: + __le16 rec_hdr_len; // 0x24: log page data offset + __le16 data_off; // 0x26: log page data length + __le32 open_log_count; // 0x28: + __le32 align[5]; // 0x2C: + struct CLIENT_REC clients[]; // 0x40: +}; + +struct LOG_REC_HDR { + __le16 redo_op; // 0x00: NTFS_LOG_OPERATION + __le16 undo_op; // 0x02: NTFS_LOG_OPERATION + __le16 redo_off; // 0x04: Offset to Redo record + __le16 redo_len; // 0x06: Redo length + __le16 undo_off; // 0x08: Offset to Undo record + __le16 undo_len; // 0x0A: Undo length + __le16 target_attr; // 0x0C: + __le16 lcns_follow; // 0x0E: + __le16 record_off; // 0x10: + __le16 attr_off; // 0x12: + __le16 cluster_off; // 0x14: + __le16 reserved; // 0x16: + __le64 target_vcn; // 0x18: + __le64 page_lcns[]; // 0x20: +}; + +static_assert(sizeof(struct LOG_REC_HDR) == 0x20); + +#define RESTART_ENTRY_ALLOCATED 0xFFFFFFFF +#define RESTART_ENTRY_ALLOCATED_LE cpu_to_le32(0xFFFFFFFF) + +struct RESTART_TABLE { + __le16 size; // 0x00: In bytes + __le16 used; // 0x02: entries + __le16 total; // 0x04: entries + __le16 res[3]; // 0x06: + __le32 free_goal; // 0x0C: + __le32 first_free; // 0x10 + __le32 last_free; // 0x14 + +}; + +static_assert(sizeof(struct RESTART_TABLE) == 0x18); + +struct ATTR_NAME_ENTRY { + __le16 off; // offset in the Open attribute Table + __le16 name_bytes; + __le16 name[]; +}; + +struct OPEN_ATTR_ENRTY { + __le32 next; // 0x00: RESTART_ENTRY_ALLOCATED if allocated + __le32 bytes_per_index; // 0x04: + enum ATTR_TYPE type; // 0x08: + u8 is_dirty_pages; // 0x0C: + u8 is_attr_name; // 0x0B: Faked field to manage 'ptr' + u8 name_len; // 0x0C: Faked field to manage 'ptr' + u8 res; + struct MFT_REF ref; // 0x10: File Reference of file containing attribute + __le64 open_record_lsn; // 0x18: + void *ptr; // 0x20: +}; + +/* 32 bit version of 'struct OPEN_ATTR_ENRTY' */ +struct OPEN_ATTR_ENRTY_32 { + __le32 next; // 0x00: RESTART_ENTRY_ALLOCATED if allocated + __le32 ptr; // 0x04: + struct MFT_REF ref; // 0x08: + __le64 open_record_lsn; // 0x10: + u8 is_dirty_pages; // 0x18: + u8 is_attr_name; // 0x19 + u8 res1[2]; + enum ATTR_TYPE type; // 0x1C: + u8 name_len; // 0x20: in wchar + u8 res2[3]; + __le32 AttributeName; // 0x24: + __le32 bytes_per_index; // 0x28: +}; + +#define SIZEOF_OPENATTRIBUTEENTRY0 0x2c +// static_assert( 0x2C == sizeof(struct OPEN_ATTR_ENRTY_32) ); +static_assert(sizeof(struct OPEN_ATTR_ENRTY) < SIZEOF_OPENATTRIBUTEENTRY0); + +/* + * One entry exists in the Dirty Pages Table for each page which is dirty at the + * time the Restart Area is written + */ +struct DIR_PAGE_ENTRY { + __le32 next; // 0x00: RESTART_ENTRY_ALLOCATED if allocated + __le32 target_attr; // 0x04: Index into the Open attribute Table + __le32 transfer_len; // 0x08: + __le32 lcns_follow; // 0x0C: + __le64 vcn; // 0x10: Vcn of dirty page + __le64 oldest_lsn; // 0x18: + __le64 page_lcns[]; // 0x20: +}; + +static_assert(sizeof(struct DIR_PAGE_ENTRY) == 0x20); + +/* 32 bit version of 'struct DIR_PAGE_ENTRY' */ +struct DIR_PAGE_ENTRY_32 { + __le32 next; // 0x00: RESTART_ENTRY_ALLOCATED if allocated + __le32 target_attr; // 0x04: Index into the Open attribute Table + __le32 transfer_len; // 0x08: + __le32 lcns_follow; // 0x0C: + __le32 reserved; // 0x10: + __le32 vcn_low; // 0x14: Vcn of dirty page + __le32 vcn_hi; // 0x18: Vcn of dirty page + __le32 oldest_lsn_low; // 0x1C: + __le32 oldest_lsn_hi; // 0x1C: + __le32 page_lcns_low; // 0x24: + __le32 page_lcns_hi; // 0x24: +}; + +static_assert(offsetof(struct DIR_PAGE_ENTRY_32, vcn_low) == 0x14); +static_assert(sizeof(struct DIR_PAGE_ENTRY_32) == 0x2c); + +enum transact_state { + TransactionUninitialized = 0, + TransactionActive, + TransactionPrepared, + TransactionCommitted +}; + +struct TRANSACTION_ENTRY { + __le32 next; // 0x00: RESTART_ENTRY_ALLOCATED if allocated + u8 transact_state; // 0x04: + u8 reserved[3]; // 0x05: + __le64 first_lsn; // 0x08: + __le64 prev_lsn; // 0x10: + __le64 undo_next_lsn; // 0x18: + __le32 undo_records; // 0x20: Number of undo log records pending abort + __le32 undo_len; // 0x24: Total undo size +}; + +static_assert(sizeof(struct TRANSACTION_ENTRY) == 0x28); + +struct NTFS_RESTART { + __le32 major_ver; // 0x00: + __le32 minor_ver; // 0x04: + __le64 check_point_start; // 0x08: + __le64 open_attr_table_lsn; // 0x10: + __le64 attr_names_lsn; // 0x18: + __le64 dirty_pages_table_lsn; // 0x20: + __le64 transact_table_lsn; // 0x28: + __le32 open_attr_len; // 0x30: In bytes + __le32 attr_names_len; // 0x34: In bytes + __le32 dirty_pages_len; // 0x38: In bytes + __le32 transact_table_len; // 0x3C: In bytes +}; + +static_assert(sizeof(struct NTFS_RESTART) == 0x40); + +struct NEW_ATTRIBUTE_SIZES { + __le64 alloc_size; + __le64 valid_size; + __le64 data_size; + __le64 total_size; +}; + +struct BITMAP_RANGE { + __le32 bitmap_off; + __le32 bits; +}; + +struct LCN_RANGE { + __le64 lcn; + __le64 len; +}; + +/* The following type defines the different log record types */ +#define LfsClientRecord cpu_to_le32(1) +#define LfsClientRestart cpu_to_le32(2) + +/* This is used to uniquely identify a client for a particular log file */ +struct CLIENT_ID { + __le16 seq_num; + __le16 client_idx; +}; + +/* This is the header that begins every Log Record in the log file */ +struct LFS_RECORD_HDR { + __le64 this_lsn; // 0x00: + __le64 client_prev_lsn; // 0x08: + __le64 client_undo_next_lsn; // 0x10: + __le32 client_data_len; // 0x18: + struct CLIENT_ID client; // 0x1C: Owner of this log record + __le32 record_type; // 0x20: LfsClientRecord or LfsClientRestart + __le32 transact_id; // 0x24: + __le16 flags; // 0x28: LOG_RECORD_MULTI_PAGE + u8 align[6]; // 0x2A: +}; + +#define LOG_RECORD_MULTI_PAGE cpu_to_le16(1) + +static_assert(sizeof(struct LFS_RECORD_HDR) == 0x30); + +struct LFS_RECORD { + __le16 next_record_off; // 0x00: Offset of the free space in the page + u8 align[6]; // 0x02: + __le64 last_end_lsn; // 0x08: lsn for the last log record which ends on the page +}; + +static_assert(sizeof(struct LFS_RECORD) == 0x10); + +struct RECORD_PAGE_HDR { + struct NTFS_RECORD_HEADER rhdr; // 'RCRD' + __le32 rflags; // 0x10: See LOG_PAGE_LOG_RECORD_END + __le16 page_count; // 0x14: + __le16 page_pos; // 0x16: + struct LFS_RECORD record_hdr; // 0x18 + __le16 fixups[10]; // 0x28 + __le32 file_off; // 0x3c: used when major version >= 2 +}; + +// clang-format on + +// Page contains the end of a log record +#define LOG_PAGE_LOG_RECORD_END cpu_to_le32(0x00000001) + +static inline bool is_log_record_end(const struct RECORD_PAGE_HDR *hdr) +{ + return hdr->rflags & LOG_PAGE_LOG_RECORD_END; +} + +static_assert(offsetof(struct RECORD_PAGE_HDR, file_off) == 0x3c); + +/* + * END of NTFS LOG structures + */ + +/* Define some tuning parameters to keep the restart tables a reasonable size */ +#define INITIAL_NUMBER_TRANSACTIONS 5 + +enum NTFS_LOG_OPERATION { + + Noop = 0x00, + CompensationLogRecord = 0x01, + InitializeFileRecordSegment = 0x02, + DeallocateFileRecordSegment = 0x03, + WriteEndOfFileRecordSegment = 0x04, + CreateAttribute = 0x05, + DeleteAttribute = 0x06, + UpdateResidentValue = 0x07, + UpdateNonresidentValue = 0x08, + UpdateMappingPairs = 0x09, + DeleteDirtyClusters = 0x0A, + SetNewAttributeSizes = 0x0B, + AddIndexEntryRoot = 0x0C, + DeleteIndexEntryRoot = 0x0D, + AddIndexEntryAllocation = 0x0E, + DeleteIndexEntryAllocation = 0x0F, + WriteEndOfIndexBuffer = 0x10, + SetIndexEntryVcnRoot = 0x11, + SetIndexEntryVcnAllocation = 0x12, + UpdateFileNameRoot = 0x13, + UpdateFileNameAllocation = 0x14, + SetBitsInNonresidentBitMap = 0x15, + ClearBitsInNonresidentBitMap = 0x16, + HotFix = 0x17, + EndTopLevelAction = 0x18, + PrepareTransaction = 0x19, + CommitTransaction = 0x1A, + ForgetTransaction = 0x1B, + OpenNonresidentAttribute = 0x1C, + OpenAttributeTableDump = 0x1D, + AttributeNamesDump = 0x1E, + DirtyPageTableDump = 0x1F, + TransactionTableDump = 0x20, + UpdateRecordDataRoot = 0x21, + UpdateRecordDataAllocation = 0x22, + + UpdateRelativeDataInIndex = + 0x23, // NtOfsRestartUpdateRelativeDataInIndex + UpdateRelativeDataInIndex2 = 0x24, + ZeroEndOfFileRecord = 0x25, +}; + +/* + * Array for log records which require a target attribute + * A true indicates that the corresponding restart operation requires a target attribute + */ +static const u8 AttributeRequired[] = { + 0xFC, 0xFB, 0xFF, 0x10, 0x06, +}; + +static inline bool is_target_required(u16 op) +{ + bool ret = op <= UpdateRecordDataAllocation && + (AttributeRequired[op >> 3] >> (op & 7) & 1); + return ret; +} + +static inline bool can_skip_action(enum NTFS_LOG_OPERATION op) +{ + switch (op) { + case Noop: + case DeleteDirtyClusters: + case HotFix: + case EndTopLevelAction: + case PrepareTransaction: + case CommitTransaction: + case ForgetTransaction: + case CompensationLogRecord: + case OpenNonresidentAttribute: + case OpenAttributeTableDump: + case AttributeNamesDump: + case DirtyPageTableDump: + case TransactionTableDump: + return true; + default: + return false; + } +} + +enum { lcb_ctx_undo_next, lcb_ctx_prev, lcb_ctx_next }; + +/* bytes per restart table */ +static inline u32 bytes_per_rt(const struct RESTART_TABLE *rt) +{ + return le16_to_cpu(rt->used) * le16_to_cpu(rt->size) + + sizeof(struct RESTART_TABLE); +} + +/* log record length */ +static inline u32 lrh_length(const struct LOG_REC_HDR *lr) +{ + u16 t16 = le16_to_cpu(lr->lcns_follow); + + return struct_size(lr, page_lcns, max_t(u16, 1, t16)); +} + +struct lcb { + struct LFS_RECORD_HDR *lrh; // Log record header of the current lsn + struct LOG_REC_HDR *log_rec; + u32 ctx_mode; // lcb_ctx_undo_next/lcb_ctx_prev/lcb_ctx_next + struct CLIENT_ID client; + bool alloc; // if true the we should deallocate 'log_rec' +}; + +static void lcb_put(struct lcb *lcb) +{ + if (lcb->alloc) + ntfs_free(lcb->log_rec); + ntfs_free(lcb->lrh); + ntfs_free(lcb); +} + +/* + * oldest_client_lsn + * + * find the oldest lsn from active clients. + */ +static inline void oldest_client_lsn(const struct CLIENT_REC *ca, + __le16 next_client, u64 *oldest_lsn) +{ + while (next_client != LFS_NO_CLIENT_LE) { + const struct CLIENT_REC *cr = ca + le16_to_cpu(next_client); + u64 lsn = le64_to_cpu(cr->oldest_lsn); + + /* ignore this block if it's oldest lsn is 0 */ + if (lsn && lsn < *oldest_lsn) + *oldest_lsn = lsn; + + next_client = cr->next_client; + } +} + +static inline bool is_rst_page_hdr_valid(u32 file_off, + const struct RESTART_HDR *rhdr) +{ + u32 sys_page = le32_to_cpu(rhdr->sys_page_size); + u32 page_size = le32_to_cpu(rhdr->page_size); + u32 end_usa; + u16 ro; + + if (sys_page < SECTOR_SIZE || page_size < SECTOR_SIZE || + sys_page & (sys_page - 1) || page_size & (page_size - 1)) { + return false; + } + + /* Check that if the file offset isn't 0, it is the system page size */ + if (file_off && file_off != sys_page) + return false; + + /* Check support version 1.1+ */ + if (le16_to_cpu(rhdr->major_ver) <= 1 && !rhdr->minor_ver) + return false; + + if (le16_to_cpu(rhdr->major_ver) > 2) + return false; + + ro = le16_to_cpu(rhdr->ra_off); + if (!IsQuadAligned(ro) || ro > sys_page) + return false; + + end_usa = ((sys_page >> SECTOR_SHIFT) + 1) * sizeof(short); + end_usa += le16_to_cpu(rhdr->rhdr.fix_off); + + if (ro < end_usa) + return false; + + return true; +} + +static inline bool is_rst_area_valid(const struct RESTART_HDR *rhdr) +{ + const struct RESTART_AREA *ra; + u16 cl, fl, ul; + u32 off, l_size, file_dat_bits, file_size_round; + u16 ro = le16_to_cpu(rhdr->ra_off); + u32 sys_page = le32_to_cpu(rhdr->sys_page_size); + + if (ro + offsetof(struct RESTART_AREA, l_size) > + SECTOR_SIZE - sizeof(short)) + return false; + + ra = Add2Ptr(rhdr, ro); + cl = le16_to_cpu(ra->log_clients); + + if (cl > 1) + return false; + + off = le16_to_cpu(ra->client_off); + + if (!IsQuadAligned(off) || ro + off > SECTOR_SIZE - sizeof(short)) + return false; + + off += cl * sizeof(struct CLIENT_REC); + + if (off > sys_page) + return false; + + /* + * Check the restart length field and whether the entire + * restart area is contained that length + */ + if (le16_to_cpu(rhdr->ra_off) + le16_to_cpu(ra->ra_len) > sys_page || + off > le16_to_cpu(ra->ra_len)) { + return false; + } + + /* + * As a final check make sure that the use list and the free list + * are either empty or point to a valid client + */ + fl = le16_to_cpu(ra->client_idx[0]); + ul = le16_to_cpu(ra->client_idx[1]); + if ((fl != LFS_NO_CLIENT && fl >= cl) || + (ul != LFS_NO_CLIENT && ul >= cl)) + return false; + + /* Make sure the sequence number bits match the log file size */ + l_size = le64_to_cpu(ra->l_size); + + file_dat_bits = sizeof(u64) * 8 - le32_to_cpu(ra->seq_num_bits); + file_size_round = 1u << (file_dat_bits + 3); + if (file_size_round != l_size && + (file_size_round < l_size || (file_size_round / 2) > l_size)) { + return false; + } + + /* The log page data offset and record header length must be quad-aligned */ + if (!IsQuadAligned(le16_to_cpu(ra->data_off)) || + !IsQuadAligned(le16_to_cpu(ra->rec_hdr_len))) + return false; + + return true; +} + +static inline bool is_client_area_valid(const struct RESTART_HDR *rhdr, + bool usa_error) +{ + u16 ro = le16_to_cpu(rhdr->ra_off); + const struct RESTART_AREA *ra = Add2Ptr(rhdr, ro); + u16 ra_len = le16_to_cpu(ra->ra_len); + const struct CLIENT_REC *ca; + u32 i; + + if (usa_error && ra_len + ro > SECTOR_SIZE - sizeof(short)) + return false; + + /* Find the start of the client array */ + ca = Add2Ptr(ra, le16_to_cpu(ra->client_off)); + + /* + * Start with the free list + * Check that all the clients are valid and that there isn't a cycle + * Do the in-use list on the second pass + */ + for (i = 0; i < 2; i++) { + u16 client_idx = le16_to_cpu(ra->client_idx[i]); + bool first_client = true; + u16 clients = le16_to_cpu(ra->log_clients); + + while (client_idx != LFS_NO_CLIENT) { + const struct CLIENT_REC *cr; + + if (!clients || + client_idx >= le16_to_cpu(ra->log_clients)) + return false; + + clients -= 1; + cr = ca + client_idx; + + client_idx = le16_to_cpu(cr->next_client); + + if (first_client) { + first_client = false; + if (cr->prev_client != LFS_NO_CLIENT_LE) + return false; + } + } + } + + return true; +} + +/* + * remove_client + * + * remove a client record from a client record list an restart area + */ +static inline void remove_client(struct CLIENT_REC *ca, + const struct CLIENT_REC *cr, __le16 *head) +{ + if (cr->prev_client == LFS_NO_CLIENT_LE) + *head = cr->next_client; + else + ca[le16_to_cpu(cr->prev_client)].next_client = cr->next_client; + + if (cr->next_client != LFS_NO_CLIENT_LE) + ca[le16_to_cpu(cr->next_client)].prev_client = cr->prev_client; +} + +/* + * add_client + * + * add a client record to the start of a list + */ +static inline void add_client(struct CLIENT_REC *ca, u16 index, __le16 *head) +{ + struct CLIENT_REC *cr = ca + index; + + cr->prev_client = LFS_NO_CLIENT_LE; + cr->next_client = *head; + + if (*head != LFS_NO_CLIENT_LE) + ca[le16_to_cpu(*head)].prev_client = cpu_to_le16(index); + + *head = cpu_to_le16(index); +} + +/* + * enum_rstbl + * + */ +static inline void *enum_rstbl(struct RESTART_TABLE *t, void *c) +{ + __le32 *e; + u32 bprt; + u16 rsize = t ? le16_to_cpu(t->size) : 0; + + if (!c) { + if (!t || !t->total) + return NULL; + e = Add2Ptr(t, sizeof(struct RESTART_TABLE)); + } else { + e = Add2Ptr(c, rsize); + } + + /* Loop until we hit the first one allocated, or the end of the list */ + for (bprt = bytes_per_rt(t); PtrOffset(t, e) < bprt; + e = Add2Ptr(e, rsize)) { + if (*e == RESTART_ENTRY_ALLOCATED_LE) + return e; + } + return NULL; +} + +/* + * find_dp + * + * searches for a 'vcn' in Dirty Page Table, + */ +static inline struct DIR_PAGE_ENTRY *find_dp(struct RESTART_TABLE *dptbl, + u32 target_attr, u64 vcn) +{ + __le32 ta = cpu_to_le32(target_attr); + struct DIR_PAGE_ENTRY *dp = NULL; + + while ((dp = enum_rstbl(dptbl, dp))) { + u64 dp_vcn = le64_to_cpu(dp->vcn); + + if (dp->target_attr == ta && vcn >= dp_vcn && + vcn < dp_vcn + le32_to_cpu(dp->lcns_follow)) { + return dp; + } + } + return NULL; +} + +static inline u32 norm_file_page(u32 page_size, u32 *l_size, bool use_default) +{ + if (use_default) + page_size = DefaultLogPageSize; + + /* Round the file size down to a system page boundary */ + *l_size &= ~(page_size - 1); + + /* File should contain at least 2 restart pages and MinLogRecordPages pages */ + if (*l_size < (MinLogRecordPages + 2) * page_size) + return 0; + + return page_size; +} + +static bool check_log_rec(const struct LOG_REC_HDR *lr, u32 bytes, u32 tr, + u32 bytes_per_attr_entry) +{ + u16 t16; + + if (bytes < sizeof(struct LOG_REC_HDR)) + return false; + if (!tr) + return false; + + if ((tr - sizeof(struct RESTART_TABLE)) % + sizeof(struct TRANSACTION_ENTRY)) + return false; + + if (le16_to_cpu(lr->redo_off) & 7) + return false; + + if (le16_to_cpu(lr->undo_off) & 7) + return false; + + if (lr->target_attr) + goto check_lcns; + + if (is_target_required(le16_to_cpu(lr->redo_op))) + return false; + + if (is_target_required(le16_to_cpu(lr->undo_op))) + return false; + +check_lcns: + if (!lr->lcns_follow) + goto check_length; + + t16 = le16_to_cpu(lr->target_attr); + if ((t16 - sizeof(struct RESTART_TABLE)) % bytes_per_attr_entry) + return false; + +check_length: + if (bytes < lrh_length(lr)) + return false; + + return true; +} + +static bool check_rstbl(const struct RESTART_TABLE *rt, size_t bytes) +{ + u32 ts; + u32 i, off; + u16 rsize = le16_to_cpu(rt->size); + u16 ne = le16_to_cpu(rt->used); + u32 ff = le32_to_cpu(rt->first_free); + u32 lf = le32_to_cpu(rt->last_free); + + ts = rsize * ne + sizeof(struct RESTART_TABLE); + + if (!rsize || rsize > bytes || + rsize + sizeof(struct RESTART_TABLE) > bytes || bytes < ts || + le16_to_cpu(rt->total) > ne || ff > ts || lf > ts || + (ff && ff < sizeof(struct RESTART_TABLE)) || + (lf && lf < sizeof(struct RESTART_TABLE))) { + return false; + } + + /* Verify each entry is either allocated or points + * to a valid offset the table + */ + for (i = 0; i < ne; i++) { + off = le32_to_cpu(*(__le32 *)Add2Ptr( + rt, i * rsize + sizeof(struct RESTART_TABLE))); + + if (off != RESTART_ENTRY_ALLOCATED && off && + (off < sizeof(struct RESTART_TABLE) || + ((off - sizeof(struct RESTART_TABLE)) % rsize))) { + return false; + } + } + + /* Walk through the list headed by the first entry to make + * sure none of the entries are currently being used + */ + for (off = ff; off;) { + if (off == RESTART_ENTRY_ALLOCATED) + return false; + + off = le32_to_cpu(*(__le32 *)Add2Ptr(rt, off)); + } + + return true; +} + +/* + * free_rsttbl_idx + * + * frees a previously allocated index a Restart Table. + */ +static inline void free_rsttbl_idx(struct RESTART_TABLE *rt, u32 off) +{ + __le32 *e; + u32 lf = le32_to_cpu(rt->last_free); + __le32 off_le = cpu_to_le32(off); + + e = Add2Ptr(rt, off); + + if (off < le32_to_cpu(rt->free_goal)) { + *e = rt->first_free; + rt->first_free = off_le; + if (!lf) + rt->last_free = off_le; + } else { + if (lf) + *(__le32 *)Add2Ptr(rt, lf) = off_le; + else + rt->first_free = off_le; + + rt->last_free = off_le; + *e = 0; + } + + le16_sub_cpu(&rt->total, 1); +} + +static inline struct RESTART_TABLE *init_rsttbl(u16 esize, u16 used) +{ + __le32 *e, *last_free; + u32 off; + u32 bytes = esize * used + sizeof(struct RESTART_TABLE); + u32 lf = sizeof(struct RESTART_TABLE) + (used - 1) * esize; + struct RESTART_TABLE *t = ntfs_zalloc(bytes); + + t->size = cpu_to_le16(esize); + t->used = cpu_to_le16(used); + t->free_goal = cpu_to_le32(~0u); + t->first_free = cpu_to_le32(sizeof(struct RESTART_TABLE)); + t->last_free = cpu_to_le32(lf); + + e = (__le32 *)(t + 1); + last_free = Add2Ptr(t, lf); + + for (off = sizeof(struct RESTART_TABLE) + esize; e < last_free; + e = Add2Ptr(e, esize), off += esize) { + *e = cpu_to_le32(off); + } + return t; +} + +static inline struct RESTART_TABLE *extend_rsttbl(struct RESTART_TABLE *tbl, + u32 add, u32 free_goal) +{ + u16 esize = le16_to_cpu(tbl->size); + __le32 osize = cpu_to_le32(bytes_per_rt(tbl)); + u32 used = le16_to_cpu(tbl->used); + struct RESTART_TABLE *rt = init_rsttbl(esize, used + add); + + memcpy(rt + 1, tbl + 1, esize * used); + + rt->free_goal = free_goal == ~0u + ? cpu_to_le32(~0u) + : cpu_to_le32(sizeof(struct RESTART_TABLE) + + free_goal * esize); + + if (tbl->first_free) { + rt->first_free = tbl->first_free; + *(__le32 *)Add2Ptr(rt, le32_to_cpu(tbl->last_free)) = osize; + } else { + rt->first_free = osize; + } + + rt->total = tbl->total; + + ntfs_free(tbl); + return rt; +} + +/* + * alloc_rsttbl_idx + * + * allocates an index from within a previously initialized Restart Table + */ +static inline void *alloc_rsttbl_idx(struct RESTART_TABLE **tbl) +{ + u32 off; + __le32 *e; + struct RESTART_TABLE *t = *tbl; + + if (!t->first_free) + *tbl = t = extend_rsttbl(t, 16, ~0u); + + off = le32_to_cpu(t->first_free); + + /* Dequeue this entry and zero it. */ + e = Add2Ptr(t, off); + + t->first_free = *e; + + memset(e, 0, le16_to_cpu(t->size)); + + *e = RESTART_ENTRY_ALLOCATED_LE; + + /* If list is going empty, then we fix the last_free as well. */ + if (!t->first_free) + t->last_free = 0; + + le16_add_cpu(&t->total, 1); + + return Add2Ptr(t, off); +} + +/* + * alloc_rsttbl_from_idx + * + * allocates a specific index from within a previously initialized Restart Table + */ +static inline void *alloc_rsttbl_from_idx(struct RESTART_TABLE **tbl, u32 vbo) +{ + u32 off; + __le32 *e; + struct RESTART_TABLE *rt = *tbl; + u32 bytes = bytes_per_rt(rt); + u16 esize = le16_to_cpu(rt->size); + + /* If the entry is not the table, we will have to extend the table */ + if (vbo >= bytes) { + /* + * extend the size by computing the number of entries between + * the existing size and the desired index and adding + * 1 to that + */ + u32 bytes2idx = vbo - bytes; + + /* There should always be an integral number of entries being added */ + /* Now extend the table */ + *tbl = rt = extend_rsttbl(rt, bytes2idx / esize + 1, bytes); + if (!rt) + return NULL; + } + + /* see if the entry is already allocated, and just return if it is. */ + e = Add2Ptr(rt, vbo); + + if (*e == RESTART_ENTRY_ALLOCATED_LE) + return e; + + /* + * Walk through the table, looking for the entry we're + * interested and the previous entry + */ + off = le32_to_cpu(rt->first_free); + e = Add2Ptr(rt, off); + + if (off == vbo) { + /* this is a match */ + rt->first_free = *e; + goto skip_looking; + } + + /* + * need to walk through the list looking for the predecessor of our entry + */ + for (;;) { + /* Remember the entry just found */ + u32 last_off = off; + __le32 *last_e = e; + + /* should never run of entries. */ + + /* Lookup up the next entry the list */ + off = le32_to_cpu(*last_e); + e = Add2Ptr(rt, off); + + /* If this is our match we are done */ + if (off == vbo) { + *last_e = *e; + + /* If this was the last entry, we update that the table as well */ + if (le32_to_cpu(rt->last_free) == off) + rt->last_free = cpu_to_le32(last_off); + break; + } + } + +skip_looking: + /* If the list is now empty, we fix the last_free as well */ + if (!rt->first_free) + rt->last_free = 0; + + /* Zero this entry */ + memset(e, 0, esize); + *e = RESTART_ENTRY_ALLOCATED_LE; + + le16_add_cpu(&rt->total, 1); + + return e; +} + +#define RESTART_SINGLE_PAGE_IO cpu_to_le16(0x0001) + +#define NTFSLOG_WRAPPED 0x00000001 +#define NTFSLOG_MULTIPLE_PAGE_IO 0x00000002 +#define NTFSLOG_NO_LAST_LSN 0x00000004 +#define NTFSLOG_REUSE_TAIL 0x00000010 +#define NTFSLOG_NO_OLDEST_LSN 0x00000020 + +/* + * Helper struct to work with NTFS LogFile + */ +struct ntfs_log { + struct ntfs_inode *ni; + + u32 l_size; + u32 sys_page_size; + u32 sys_page_mask; + u32 page_size; + u32 page_mask; // page_size - 1 + u8 page_bits; + struct RECORD_PAGE_HDR *one_page_buf; + + struct RESTART_TABLE *open_attr_tbl; + u32 transaction_id; + u32 clst_per_page; + + u32 first_page; + u32 next_page; + u32 ra_off; + u32 data_off; + u32 restart_size; + u32 data_size; + u16 record_header_len; + u64 seq_num; + u32 seq_num_bits; + u32 file_data_bits; + u32 seq_num_mask; /* (1 << file_data_bits) - 1 */ + + struct RESTART_AREA *ra; /* in-memory image of the next restart area */ + u32 ra_size; /* the usable size of the restart area */ + + /* + * If true, then the in-memory restart area is to be written + * to the first position on the disk + */ + bool init_ra; + bool set_dirty; /* true if we need to set dirty flag */ + + u64 oldest_lsn; + + u32 oldest_lsn_off; + u64 last_lsn; + + u32 total_avail; + u32 total_avail_pages; + u32 total_undo_commit; + u32 max_current_avail; + u32 current_avail; + u32 reserved; + + short major_ver; + short minor_ver; + + u32 l_flags; /* See NTFSLOG_XXX */ + u32 current_openlog_count; /* On-disk value for open_log_count */ + + struct CLIENT_ID client_id; + u32 client_undo_commit; +}; + +static inline u32 lsn_to_vbo(struct ntfs_log *log, const u64 lsn) +{ + u32 vbo = (lsn << log->seq_num_bits) >> (log->seq_num_bits - 3); + + return vbo; +} + +/* compute the offset in the log file of the next log page */ +static inline u32 next_page_off(struct ntfs_log *log, u32 off) +{ + off = (off & ~log->sys_page_mask) + log->page_size; + return off >= log->l_size ? log->first_page : off; +} + +static inline u32 lsn_to_page_off(struct ntfs_log *log, u64 lsn) +{ + return (((u32)lsn) << 3) & log->page_mask; +} + +static inline u64 vbo_to_lsn(struct ntfs_log *log, u32 off, u64 Seq) +{ + return (off >> 3) + (Seq << log->file_data_bits); +} + +static inline bool is_lsn_in_file(struct ntfs_log *log, u64 lsn) +{ + return lsn >= log->oldest_lsn && + lsn <= le64_to_cpu(log->ra->current_lsn); +} + +static inline u32 hdr_file_off(struct ntfs_log *log, + struct RECORD_PAGE_HDR *hdr) +{ + if (log->major_ver < 2) + return le64_to_cpu(hdr->rhdr.lsn); + + return le32_to_cpu(hdr->file_off); +} + +static inline u64 base_lsn(struct ntfs_log *log, + const struct RECORD_PAGE_HDR *hdr, u64 lsn) +{ + u64 h_lsn = le64_to_cpu(hdr->rhdr.lsn); + u64 ret = (((h_lsn >> log->file_data_bits) + + (lsn < (lsn_to_vbo(log, h_lsn) & ~log->page_mask) ? 1 : 0)) + << log->file_data_bits) + + ((((is_log_record_end(hdr) && + h_lsn <= le64_to_cpu(hdr->record_hdr.last_end_lsn)) + ? le16_to_cpu(hdr->record_hdr.next_record_off) + : log->page_size) + + lsn) >> + 3); + + return ret; +} + +static inline bool verify_client_lsn(struct ntfs_log *log, + const struct CLIENT_REC *client, u64 lsn) +{ + return lsn >= le64_to_cpu(client->oldest_lsn) && + lsn <= le64_to_cpu(log->ra->current_lsn) && lsn; +} + +struct restart_info { + u64 last_lsn; + struct RESTART_HDR *r_page; + u32 vbo; + bool chkdsk_was_run; + bool valid_page; + bool initialized; + bool restart; +}; + +static int read_log_page(struct ntfs_log *log, u32 vbo, + struct RECORD_PAGE_HDR **buffer, bool *usa_error) +{ + int err = 0; + u32 page_idx = vbo >> log->page_bits; + u32 page_off = vbo & log->page_mask; + u32 bytes = log->page_size - page_off; + void *to_free = NULL; + u32 page_vbo = page_idx << log->page_bits; + struct RECORD_PAGE_HDR *page_buf; + struct ntfs_inode *ni = log->ni; + bool bBAAD; + + if (vbo >= log->l_size) + return -EINVAL; + + if (!*buffer) { + to_free = ntfs_malloc(bytes); + if (!to_free) + return -ENOMEM; + *buffer = to_free; + } + + page_buf = page_off ? log->one_page_buf : *buffer; + + err = ntfs_read_run_nb(ni->mi.sbi, &ni->file.run, page_vbo, page_buf, + log->page_size, NULL); + if (err) + goto out; + + if (page_buf->rhdr.sign != NTFS_FFFF_SIGNATURE) + ntfs_fix_post_read(&page_buf->rhdr, PAGE_SIZE, false); + + if (page_buf != *buffer) + memcpy(*buffer, Add2Ptr(page_buf, page_off), bytes); + + bBAAD = page_buf->rhdr.sign == NTFS_BAAD_SIGNATURE; + + if (usa_error) + *usa_error = bBAAD; + /* Check that the update sequence array for this page is valid */ + /* If we don't allow errors, raise an error status */ + else if (bBAAD) + err = -EINVAL; + +out: + if (err && to_free) { + ntfs_free(to_free); + *buffer = NULL; + } + + return err; +} + +/* + * log_read_rst + * + * it walks through 512 blocks of the file looking for a valid restart page header + * It will stop the first time we find a valid page header + */ +static int log_read_rst(struct ntfs_log *log, u32 l_size, bool first, + struct restart_info *info) +{ + u32 skip, vbo; + struct RESTART_HDR *r_page = ntfs_malloc(DefaultLogPageSize); + + if (!r_page) + return -ENOMEM; + + memset(info, 0, sizeof(struct restart_info)); + + /* Determine which restart area we are looking for */ + if (first) { + vbo = 0; + skip = 512; + } else { + vbo = 512; + skip = 0; + } + + /* loop continuously until we succeed */ + for (; vbo < l_size; vbo = 2 * vbo + skip, skip = 0) { + bool usa_error; + u32 sys_page_size; + bool brst, bchk; + struct RESTART_AREA *ra; + + /* Read a page header at the current offset */ + if (read_log_page(log, vbo, (struct RECORD_PAGE_HDR **)&r_page, + &usa_error)) { + /* ignore any errors */ + continue; + } + + /* exit if the signature is a log record page */ + if (r_page->rhdr.sign == NTFS_RCRD_SIGNATURE) { + info->initialized = true; + break; + } + + brst = r_page->rhdr.sign == NTFS_RSTR_SIGNATURE; + bchk = r_page->rhdr.sign == NTFS_CHKD_SIGNATURE; + + if (!bchk && !brst) { + if (r_page->rhdr.sign != NTFS_FFFF_SIGNATURE) { + /* + * Remember if the signature does not + * indicate uninitialized file + */ + info->initialized = true; + } + continue; + } + + ra = NULL; + info->valid_page = false; + info->initialized = true; + info->vbo = vbo; + + /* Let's check the restart area if this is a valid page */ + if (!is_rst_page_hdr_valid(vbo, r_page)) + goto check_result; + ra = Add2Ptr(r_page, le16_to_cpu(r_page->ra_off)); + + if (!is_rst_area_valid(r_page)) + goto check_result; + + /* + * We have a valid restart page header and restart area. + * If chkdsk was run or we have no clients then we have + * no more checking to do + */ + if (bchk || ra->client_idx[1] == LFS_NO_CLIENT_LE) { + info->valid_page = true; + goto check_result; + } + + /* Read the entire restart area */ + sys_page_size = le32_to_cpu(r_page->sys_page_size); + if (DefaultLogPageSize != sys_page_size) { + ntfs_free(r_page); + r_page = ntfs_zalloc(sys_page_size); + if (!r_page) + return -ENOMEM; + + if (read_log_page(log, vbo, + (struct RECORD_PAGE_HDR **)&r_page, + &usa_error)) { + /* ignore any errors */ + ntfs_free(r_page); + r_page = NULL; + continue; + } + } + + if (is_client_area_valid(r_page, usa_error)) { + info->valid_page = true; + ra = Add2Ptr(r_page, le16_to_cpu(r_page->ra_off)); + } + +check_result: + /* If chkdsk was run then update the caller's values and return */ + if (r_page->rhdr.sign == NTFS_CHKD_SIGNATURE) { + info->chkdsk_was_run = true; + info->last_lsn = le64_to_cpu(r_page->rhdr.lsn); + info->restart = true; + info->r_page = r_page; + return 0; + } + + /* If we have a valid page then copy the values we need from it */ + if (info->valid_page) { + info->last_lsn = le64_to_cpu(ra->current_lsn); + info->restart = true; + info->r_page = r_page; + return 0; + } + } + + ntfs_free(r_page); + + return 0; +} + +/* + * log_init_pg_hdr + * + * init "log' from restart page header + */ +static void log_init_pg_hdr(struct ntfs_log *log, u32 sys_page_size, + u32 page_size, u16 major_ver, u16 minor_ver) +{ + log->sys_page_size = sys_page_size; + log->sys_page_mask = sys_page_size - 1; + log->page_size = page_size; + log->page_mask = page_size - 1; + log->page_bits = blksize_bits(page_size); + + log->clst_per_page = log->page_size >> log->ni->mi.sbi->cluster_bits; + if (!log->clst_per_page) + log->clst_per_page = 1; + + log->first_page = major_ver >= 2 + ? 0x22 * page_size + : ((sys_page_size << 1) + (page_size << 1)); + log->major_ver = major_ver; + log->minor_ver = minor_ver; +} + +/* + * log_create + * + * init "log" in cases when we don't have a restart area to use + */ +static void log_create(struct ntfs_log *log, u32 l_size, const u64 last_lsn, + u32 open_log_count, bool wrapped, bool use_multi_page) +{ + log->l_size = l_size; + /* All file offsets must be quadword aligned */ + log->file_data_bits = blksize_bits(l_size) - 3; + log->seq_num_mask = (8 << log->file_data_bits) - 1; + log->seq_num_bits = sizeof(u64) * 8 - log->file_data_bits; + log->seq_num = (last_lsn >> log->file_data_bits) + 2; + log->next_page = log->first_page; + log->oldest_lsn = log->seq_num << log->file_data_bits; + log->oldest_lsn_off = 0; + log->last_lsn = log->oldest_lsn; + + log->l_flags |= NTFSLOG_NO_LAST_LSN | NTFSLOG_NO_OLDEST_LSN; + + /* Set the correct flags for the I/O and indicate if we have wrapped */ + if (wrapped) + log->l_flags |= NTFSLOG_WRAPPED; + + if (use_multi_page) + log->l_flags |= NTFSLOG_MULTIPLE_PAGE_IO; + + /* Compute the log page values */ + log->data_off = QuadAlign( + offsetof(struct RECORD_PAGE_HDR, fixups) + + sizeof(short) * ((log->page_size >> SECTOR_SHIFT) + 1)); + log->data_size = log->page_size - log->data_off; + log->record_header_len = sizeof(struct LFS_RECORD_HDR); + + /* Remember the different page sizes for reservation */ + log->reserved = log->data_size - log->record_header_len; + + /* Compute the restart page values. */ + log->ra_off = QuadAlign( + offsetof(struct RESTART_HDR, fixups) + + sizeof(short) * ((log->sys_page_size >> SECTOR_SHIFT) + 1)); + log->restart_size = log->sys_page_size - log->ra_off; + log->ra_size = struct_size(log->ra, clients, 1); + log->current_openlog_count = open_log_count; + + /* + * The total available log file space is the number of + * log file pages times the space available on each page + */ + log->total_avail_pages = log->l_size - log->first_page; + log->total_avail = log->total_avail_pages >> log->page_bits; + + /* + * We assume that we can't use the end of the page less than + * the file record size + * Then we won't need to reserve more than the caller asks for + */ + log->max_current_avail = log->total_avail * log->reserved; + log->total_avail = log->total_avail * log->data_size; + log->current_avail = log->max_current_avail; +} + +/* + * log_create_ra + * + * This routine is called to fill a restart area from the values stored in 'log' + */ +static struct RESTART_AREA *log_create_ra(struct ntfs_log *log) +{ + struct CLIENT_REC *cr; + struct RESTART_AREA *ra = ntfs_zalloc(log->restart_size); + + if (!ra) + return NULL; + + ra->current_lsn = cpu_to_le64(log->last_lsn); + ra->log_clients = cpu_to_le16(1); + ra->client_idx[1] = LFS_NO_CLIENT_LE; + if (log->l_flags & NTFSLOG_MULTIPLE_PAGE_IO) + ra->flags = RESTART_SINGLE_PAGE_IO; + ra->seq_num_bits = cpu_to_le32(log->seq_num_bits); + ra->ra_len = cpu_to_le16(log->ra_size); + ra->client_off = cpu_to_le16(offsetof(struct RESTART_AREA, clients)); + ra->l_size = cpu_to_le64(log->l_size); + ra->rec_hdr_len = cpu_to_le16(log->record_header_len); + ra->data_off = cpu_to_le16(log->data_off); + ra->open_log_count = cpu_to_le32(log->current_openlog_count + 1); + + cr = ra->clients; + + cr->prev_client = LFS_NO_CLIENT_LE; + cr->next_client = LFS_NO_CLIENT_LE; + + return ra; +} + +static u32 final_log_off(struct ntfs_log *log, u64 lsn, u32 data_len) +{ + u32 base_vbo = lsn << 3; + u32 final_log_off = (base_vbo & log->seq_num_mask) & ~log->page_mask; + u32 page_off = base_vbo & log->page_mask; + u32 tail = log->page_size - page_off; + + page_off -= 1; + + /* Add the length of the header */ + data_len += log->record_header_len; + + /* + * If this lsn is contained this log page we are done + * Otherwise we need to walk through several log pages + */ + if (data_len > tail) { + data_len -= tail; + tail = log->data_size; + page_off = log->data_off - 1; + + for (;;) { + final_log_off = next_page_off(log, final_log_off); + + /* We are done if the remaining bytes fit on this page */ + if (data_len <= tail) + break; + data_len -= tail; + } + } + + /* + * We add the remaining bytes to our starting position on this page + * and then add that value to the file offset of this log page + */ + return final_log_off + data_len + page_off; +} + +static int next_log_lsn(struct ntfs_log *log, const struct LFS_RECORD_HDR *rh, + u64 *lsn) +{ + int err; + u64 this_lsn = le64_to_cpu(rh->this_lsn); + u32 vbo = lsn_to_vbo(log, this_lsn); + u32 end = + final_log_off(log, this_lsn, le32_to_cpu(rh->client_data_len)); + u32 hdr_off = end & ~log->sys_page_mask; + u64 seq = this_lsn >> log->file_data_bits; + struct RECORD_PAGE_HDR *page = NULL; + + /* Remember if we wrapped */ + if (end <= vbo) + seq += 1; + + /* log page header for this page */ + err = read_log_page(log, hdr_off, &page, NULL); + if (err) + return err; + + /* + * If the lsn we were given was not the last lsn on this page, + * then the starting offset for the next lsn is on a quad word + * boundary following the last file offset for the current lsn + * Otherwise the file offset is the start of the data on the next page + */ + if (this_lsn == le64_to_cpu(page->rhdr.lsn)) { + /* If we wrapped, we need to increment the sequence number */ + hdr_off = next_page_off(log, hdr_off); + if (hdr_off == log->first_page) + seq += 1; + + vbo = hdr_off + log->data_off; + } else { + vbo = QuadAlign(end); + } + + /* Compute the lsn based on the file offset and the sequence count */ + *lsn = vbo_to_lsn(log, vbo, seq); + + /* + * If this lsn is within the legal range for the file, we return true + * Otherwise false indicates that there are no more lsn's + */ + if (!is_lsn_in_file(log, *lsn)) + *lsn = 0; + + ntfs_free(page); + + return 0; +} + +/* + * current_log_avail + * + * calculate the number of bytes available for log records + */ +static u32 current_log_avail(struct ntfs_log *log) +{ + u32 oldest_off, next_free_off, free_bytes; + + if (log->l_flags & NTFSLOG_NO_LAST_LSN) { + /* The entire file is available */ + return log->max_current_avail; + } + + /* + * If there is a last lsn the restart area then we know that we will + * have to compute the free range + * If there is no oldest lsn then start at the first page of the file + */ + oldest_off = (log->l_flags & NTFSLOG_NO_OLDEST_LSN) + ? log->first_page + : (log->oldest_lsn_off & ~log->sys_page_mask); + + /* + * We will use the next log page offset to compute the next free page\ + * If we are going to reuse this page go to the next page + * If we are at the first page then use the end of the file + */ + next_free_off = (log->l_flags & NTFSLOG_REUSE_TAIL) + ? log->next_page + log->page_size + : log->next_page == log->first_page ? log->l_size + : log->next_page; + + /* If the two offsets are the same then there is no available space */ + if (oldest_off == next_free_off) + return 0; + /* + * If the free offset follows the oldest offset then subtract + * this range from the total available pages + */ + free_bytes = + oldest_off < next_free_off + ? log->total_avail_pages - (next_free_off - oldest_off) + : oldest_off - next_free_off; + + free_bytes >>= log->page_bits; + return free_bytes * log->reserved; +} + +static bool check_subseq_log_page(struct ntfs_log *log, + const struct RECORD_PAGE_HDR *rp, u32 vbo, + u64 seq) +{ + u64 lsn_seq; + const struct NTFS_RECORD_HEADER *rhdr = &rp->rhdr; + u64 lsn = le64_to_cpu(rhdr->lsn); + + if (rhdr->sign == NTFS_FFFF_SIGNATURE || !rhdr->sign) + return false; + + /* + * If the last lsn on the page occurs was written after the page + * that caused the original error then we have a fatal error + */ + lsn_seq = lsn >> log->file_data_bits; + + /* + * If the sequence number for the lsn the page is equal or greater + * than lsn we expect, then this is a subsequent write + */ + return lsn_seq >= seq || + (lsn_seq == seq - 1 && log->first_page == vbo && + vbo != (lsn_to_vbo(log, lsn) & ~log->page_mask)); +} + +/* + * last_log_lsn + * + * This routine walks through the log pages for a file, searching for the + * last log page written to the file + */ +static int last_log_lsn(struct ntfs_log *log) +{ + int err; + bool usa_error = false; + bool replace_page = false; + bool reuse_page = log->l_flags & NTFSLOG_REUSE_TAIL; + bool wrapped_file, wrapped; + + u32 page_cnt = 1, page_pos = 1; + u32 page_off = 0, page_off1 = 0, saved_off = 0; + u32 final_off, second_off, final_off_prev = 0, second_off_prev = 0; + u32 first_file_off = 0, second_file_off = 0; + u32 part_io_count = 0; + u32 tails = 0; + u32 this_off, curpage_off, nextpage_off, remain_pages; + + u64 expected_seq, seq_base = 0, lsn_base = 0; + u64 best_lsn, best_lsn1, best_lsn2; + u64 lsn_cur, lsn1, lsn2; + u64 last_ok_lsn = reuse_page ? log->last_lsn : 0; + + u16 cur_pos, best_page_pos; + + struct RECORD_PAGE_HDR *page = NULL; + struct RECORD_PAGE_HDR *tst_page = NULL; + struct RECORD_PAGE_HDR *first_tail = NULL; + struct RECORD_PAGE_HDR *second_tail = NULL; + struct RECORD_PAGE_HDR *tail_page = NULL; + struct RECORD_PAGE_HDR *second_tail_prev = NULL; + struct RECORD_PAGE_HDR *first_tail_prev = NULL; + struct RECORD_PAGE_HDR *page_bufs = NULL; + struct RECORD_PAGE_HDR *best_page; + + if (log->major_ver >= 2) { + final_off = 0x02 * log->page_size; + second_off = 0x12 * log->page_size; + + // 0x10 == 0x12 - 0x2 + page_bufs = ntfs_malloc(log->page_size * 0x10); + if (!page_bufs) + return -ENOMEM; + } else { + second_off = log->first_page - log->page_size; + final_off = second_off - log->page_size; + } + +next_tail: + /* Read second tail page (at pos 3/0x12000) */ + if (read_log_page(log, second_off, &second_tail, &usa_error) || + usa_error || second_tail->rhdr.sign != NTFS_RCRD_SIGNATURE) { + ntfs_free(second_tail); + second_tail = NULL; + second_file_off = 0; + lsn2 = 0; + } else { + second_file_off = hdr_file_off(log, second_tail); + lsn2 = le64_to_cpu(second_tail->record_hdr.last_end_lsn); + } + + /* Read first tail page (at pos 2/0x2000 ) */ + if (read_log_page(log, final_off, &first_tail, &usa_error) || + usa_error || first_tail->rhdr.sign != NTFS_RCRD_SIGNATURE) { + ntfs_free(first_tail); + first_tail = NULL; + first_file_off = 0; + lsn1 = 0; + } else { + first_file_off = hdr_file_off(log, first_tail); + lsn1 = le64_to_cpu(first_tail->record_hdr.last_end_lsn); + } + + if (log->major_ver < 2) { + int best_page; + + first_tail_prev = first_tail; + final_off_prev = first_file_off; + second_tail_prev = second_tail; + second_off_prev = second_file_off; + tails = 1; + + if (!first_tail && !second_tail) + goto tail_read; + + if (first_tail && second_tail) + best_page = lsn1 < lsn2 ? 1 : 0; + else if (first_tail) + best_page = 0; + else + best_page = 1; + + page_off = best_page ? second_file_off : first_file_off; + seq_base = (best_page ? lsn2 : lsn1) >> log->file_data_bits; + goto tail_read; + } + + best_lsn1 = first_tail ? base_lsn(log, first_tail, first_file_off) : 0; + best_lsn2 = + second_tail ? base_lsn(log, second_tail, second_file_off) : 0; + + if (first_tail && second_tail) { + if (best_lsn1 > best_lsn2) { + best_lsn = best_lsn1; + best_page = first_tail; + this_off = first_file_off; + } else { + best_lsn = best_lsn2; + best_page = second_tail; + this_off = second_file_off; + } + } else if (first_tail) { + best_lsn = best_lsn1; + best_page = first_tail; + this_off = first_file_off; + } else if (second_tail) { + best_lsn = best_lsn2; + best_page = second_tail; + this_off = second_file_off; + } else { + goto tail_read; + } + + best_page_pos = le16_to_cpu(best_page->page_pos); + + if (!tails) { + if (best_page_pos == page_pos) { + seq_base = best_lsn >> log->file_data_bits; + saved_off = page_off = le32_to_cpu(best_page->file_off); + lsn_base = best_lsn; + + memmove(page_bufs, best_page, log->page_size); + + page_cnt = le16_to_cpu(best_page->page_count); + if (page_cnt > 1) + page_pos += 1; + + tails = 1; + } + } else if (seq_base == (best_lsn >> log->file_data_bits) && + saved_off + log->page_size == this_off && + lsn_base < best_lsn && + (page_pos != page_cnt || best_page_pos == page_pos || + best_page_pos == 1) && + (page_pos >= page_cnt || best_page_pos == page_pos)) { + u16 bppc = le16_to_cpu(best_page->page_count); + + saved_off += log->page_size; + lsn_base = best_lsn; + + memmove(Add2Ptr(page_bufs, tails * log->page_size), best_page, + log->page_size); + + tails += 1; + + if (best_page_pos != bppc) { + page_cnt = bppc; + page_pos = best_page_pos; + + if (page_cnt > 1) + page_pos += 1; + } else { + page_pos = page_cnt = 1; + } + } else { + ntfs_free(first_tail); + ntfs_free(second_tail); + goto tail_read; + } + + ntfs_free(first_tail_prev); + first_tail_prev = first_tail; + final_off_prev = first_file_off; + first_tail = NULL; + + ntfs_free(second_tail_prev); + second_tail_prev = second_tail; + second_off_prev = second_file_off; + second_tail = NULL; + + final_off += log->page_size; + second_off += log->page_size; + + if (tails < 0x10) + goto next_tail; +tail_read: + first_tail = first_tail_prev; + final_off = final_off_prev; + + second_tail = second_tail_prev; + second_off = second_off_prev; + + page_cnt = page_pos = 1; + + curpage_off = seq_base == log->seq_num ? min(log->next_page, page_off) + : log->next_page; + + wrapped_file = + curpage_off == log->first_page && + !(log->l_flags & (NTFSLOG_NO_LAST_LSN | NTFSLOG_REUSE_TAIL)); + + expected_seq = wrapped_file ? (log->seq_num + 1) : log->seq_num; + + nextpage_off = curpage_off; + +next_page: + tail_page = NULL; + /* Read the next log page */ + err = read_log_page(log, curpage_off, &page, &usa_error); + + /* Compute the next log page offset the file */ + nextpage_off = next_page_off(log, curpage_off); + wrapped = nextpage_off == log->first_page; + + if (tails > 1) { + struct RECORD_PAGE_HDR *cur_page = + Add2Ptr(page_bufs, curpage_off - page_off); + + if (curpage_off == saved_off) { + tail_page = cur_page; + goto use_tail_page; + } + + if (page_off > curpage_off || curpage_off >= saved_off) + goto use_tail_page; + + if (page_off1) + goto use_cur_page; + + if (!err && !usa_error && + page->rhdr.sign == NTFS_RCRD_SIGNATURE && + cur_page->rhdr.lsn == page->rhdr.lsn && + cur_page->record_hdr.next_record_off == + page->record_hdr.next_record_off && + ((page_pos == page_cnt && + le16_to_cpu(page->page_pos) == 1) || + (page_pos != page_cnt && + le16_to_cpu(page->page_pos) == page_pos + 1 && + le16_to_cpu(page->page_count) == page_cnt))) { + cur_page = NULL; + goto use_tail_page; + } + + page_off1 = page_off; + +use_cur_page: + + lsn_cur = le64_to_cpu(cur_page->rhdr.lsn); + + if (last_ok_lsn != + le64_to_cpu(cur_page->record_hdr.last_end_lsn) && + ((lsn_cur >> log->file_data_bits) + + ((curpage_off < + (lsn_to_vbo(log, lsn_cur) & ~log->page_mask)) + ? 1 + : 0)) != expected_seq) { + goto check_tail; + } + + if (!is_log_record_end(cur_page)) { + tail_page = NULL; + last_ok_lsn = lsn_cur; + goto next_page_1; + } + + log->seq_num = expected_seq; + log->l_flags &= ~NTFSLOG_NO_LAST_LSN; + log->last_lsn = le64_to_cpu(cur_page->record_hdr.last_end_lsn); + log->ra->current_lsn = cur_page->record_hdr.last_end_lsn; + + if (log->record_header_len <= + log->page_size - + le16_to_cpu(cur_page->record_hdr.next_record_off)) { + log->l_flags |= NTFSLOG_REUSE_TAIL; + log->next_page = curpage_off; + } else { + log->l_flags &= ~NTFSLOG_REUSE_TAIL; + log->next_page = nextpage_off; + } + + if (wrapped_file) + log->l_flags |= NTFSLOG_WRAPPED; + + last_ok_lsn = le64_to_cpu(cur_page->record_hdr.last_end_lsn); + goto next_page_1; + } + + /* + * If we are at the expected first page of a transfer check to see + * if either tail copy is at this offset + * If this page is the last page of a transfer, check if we wrote + * a subsequent tail copy + */ + if (page_cnt == page_pos || page_cnt == page_pos + 1) { + /* + * Check if the offset matches either the first or second + * tail copy. It is possible it will match both + */ + if (curpage_off == final_off) + tail_page = first_tail; + + /* + * If we already matched on the first page then + * check the ending lsn's. + */ + if (curpage_off == second_off) { + if (!tail_page || + (second_tail && + le64_to_cpu(second_tail->record_hdr.last_end_lsn) > + le64_to_cpu(first_tail->record_hdr + .last_end_lsn))) { + tail_page = second_tail; + } + } + } + +use_tail_page: + if (tail_page) { + /* we have a candidate for a tail copy */ + lsn_cur = le64_to_cpu(tail_page->record_hdr.last_end_lsn); + + if (last_ok_lsn < lsn_cur) { + /* + * If the sequence number is not expected, + * then don't use the tail copy + */ + if (expected_seq != (lsn_cur >> log->file_data_bits)) + tail_page = NULL; + } else if (last_ok_lsn > lsn_cur) { + /* + * If the last lsn is greater than the one on + * this page then forget this tail + */ + tail_page = NULL; + } + } + + /* If we have an error on the current page, we will break of this loop */ + if (err || usa_error) + goto check_tail; + + /* + * Done if the last lsn on this page doesn't match the previous known + * last lsn or the sequence number is not expected + */ + lsn_cur = le64_to_cpu(page->rhdr.lsn); + if (last_ok_lsn != lsn_cur && + expected_seq != (lsn_cur >> log->file_data_bits)) { + goto check_tail; + } + + /* + * Check that the page position and page count values are correct + * If this is the first page of a transfer the position must be 1 + * and the count will be unknown + */ + if (page_cnt == page_pos) { + if (page->page_pos != cpu_to_le16(1) && + (!reuse_page || page->page_pos != page->page_count)) { + /* + * If the current page is the first page we are + * looking at and we are reusing this page then + * it can be either the first or last page of a + * transfer. Otherwise it can only be the first. + */ + goto check_tail; + } + } else if (le16_to_cpu(page->page_count) != page_cnt || + le16_to_cpu(page->page_pos) != page_pos + 1) { + /* + * The page position better be 1 more than the last page + * position and the page count better match + */ + goto check_tail; + } + + /* + * We have a valid page the file and may have a valid page + * the tail copy area + * If the tail page was written after the page the file then + * break of the loop + */ + if (tail_page && + le64_to_cpu(tail_page->record_hdr.last_end_lsn) > lsn_cur) { + /* Remember if we will replace the page */ + replace_page = true; + goto check_tail; + } + + tail_page = NULL; + + if (is_log_record_end(page)) { + /* + * Since we have read this page we know the sequence number + * is the same as our expected value + */ + log->seq_num = expected_seq; + log->last_lsn = le64_to_cpu(page->record_hdr.last_end_lsn); + log->ra->current_lsn = page->record_hdr.last_end_lsn; + log->l_flags &= ~NTFSLOG_NO_LAST_LSN; + + /* + * If there is room on this page for another header then + * remember we want to reuse the page + */ + if (log->record_header_len <= + log->page_size - + le16_to_cpu(page->record_hdr.next_record_off)) { + log->l_flags |= NTFSLOG_REUSE_TAIL; + log->next_page = curpage_off; + } else { + log->l_flags &= ~NTFSLOG_REUSE_TAIL; + log->next_page = nextpage_off; + } + + /* Remember if we wrapped the log file */ + if (wrapped_file) + log->l_flags |= NTFSLOG_WRAPPED; + } + + /* + * Remember the last page count and position. + * Also remember the last known lsn + */ + page_cnt = le16_to_cpu(page->page_count); + page_pos = le16_to_cpu(page->page_pos); + last_ok_lsn = le64_to_cpu(page->rhdr.lsn); + +next_page_1: + + if (wrapped) { + expected_seq += 1; + wrapped_file = 1; + } + + curpage_off = nextpage_off; + ntfs_free(page); + page = NULL; + reuse_page = 0; + goto next_page; + +check_tail: + if (tail_page) { + log->seq_num = expected_seq; + log->last_lsn = le64_to_cpu(tail_page->record_hdr.last_end_lsn); + log->ra->current_lsn = tail_page->record_hdr.last_end_lsn; + log->l_flags &= ~NTFSLOG_NO_LAST_LSN; + + if (log->page_size - + le16_to_cpu( + tail_page->record_hdr.next_record_off) >= + log->record_header_len) { + log->l_flags |= NTFSLOG_REUSE_TAIL; + log->next_page = curpage_off; + } else { + log->l_flags &= ~NTFSLOG_REUSE_TAIL; + log->next_page = nextpage_off; + } + + if (wrapped) + log->l_flags |= NTFSLOG_WRAPPED; + } + + /* Remember that the partial IO will start at the next page */ + second_off = nextpage_off; + + /* + * If the next page is the first page of the file then update + * the sequence number for log records which begon the next page + */ + if (wrapped) + expected_seq += 1; + + /* + * If we have a tail copy or are performing single page I/O we can + * immediately look at the next page + */ + if (replace_page || (log->ra->flags & RESTART_SINGLE_PAGE_IO)) { + page_cnt = 2; + page_pos = 1; + goto check_valid; + } + + if (page_pos != page_cnt) + goto check_valid; + /* + * If the next page causes us to wrap to the beginning of the log + * file then we know which page to check next. + */ + if (wrapped) { + page_cnt = 2; + page_pos = 1; + goto check_valid; + } + + cur_pos = 2; + +next_test_page: + ntfs_free(tst_page); + tst_page = NULL; + + /* Walk through the file, reading log pages */ + err = read_log_page(log, nextpage_off, &tst_page, &usa_error); + + /* + * If we get a USA error then assume that we correctly found + * the end of the original transfer + */ + if (usa_error) + goto file_is_valid; + + /* + * If we were able to read the page, we examine it to see if it + * is the same or different Io block + */ + if (err) + goto next_test_page_1; + + if (le16_to_cpu(tst_page->page_pos) == cur_pos && + check_subseq_log_page(log, tst_page, nextpage_off, expected_seq)) { + page_cnt = le16_to_cpu(tst_page->page_count) + 1; + page_pos = le16_to_cpu(tst_page->page_pos); + goto check_valid; + } else { + goto file_is_valid; + } + +next_test_page_1: + + nextpage_off = next_page_off(log, curpage_off); + wrapped = nextpage_off == log->first_page; + + if (wrapped) { + expected_seq += 1; + page_cnt = 2; + page_pos = 1; + } + + cur_pos += 1; + part_io_count += 1; + if (!wrapped) + goto next_test_page; + +check_valid: + /* Skip over the remaining pages this transfer */ + remain_pages = page_cnt - page_pos - 1; + part_io_count += remain_pages; + + while (remain_pages--) { + nextpage_off = next_page_off(log, curpage_off); + wrapped = nextpage_off == log->first_page; + + if (wrapped) + expected_seq += 1; + } + + /* Call our routine to check this log page */ + ntfs_free(tst_page); + tst_page = NULL; + + err = read_log_page(log, nextpage_off, &tst_page, &usa_error); + if (!err && !usa_error && + check_subseq_log_page(log, tst_page, nextpage_off, expected_seq)) { + err = -EINVAL; + goto out; + } + +file_is_valid: + + /* We have a valid file */ + if (page_off1 || tail_page) { + struct RECORD_PAGE_HDR *tmp_page; + + if (sb_rdonly(log->ni->mi.sbi->sb)) { + err = -EROFS; + goto out; + } + + if (page_off1) { + tmp_page = Add2Ptr(page_bufs, page_off1 - page_off); + tails -= (page_off1 - page_off) / log->page_size; + if (!tail_page) + tails -= 1; + } else { + tmp_page = tail_page; + tails = 1; + } + + while (tails--) { + u64 off = hdr_file_off(log, tmp_page); + + if (!page) { + page = ntfs_malloc(log->page_size); + if (!page) + return -ENOMEM; + } + + /* + * Correct page and copy the data from this page + * into it and flush it to disk + */ + memcpy(page, tmp_page, log->page_size); + + /* Fill last flushed lsn value flush the page */ + if (log->major_ver < 2) + page->rhdr.lsn = page->record_hdr.last_end_lsn; + else + page->file_off = 0; + + page->page_pos = page->page_count = cpu_to_le16(1); + + ntfs_fix_pre_write(&page->rhdr, log->page_size); + + err = ntfs_sb_write_run(log->ni->mi.sbi, + &log->ni->file.run, off, page, + log->page_size); + + if (err) + goto out; + + if (part_io_count && second_off == off) { + second_off += log->page_size; + part_io_count -= 1; + } + + tmp_page = Add2Ptr(tmp_page, log->page_size); + } + } + + if (part_io_count) { + if (sb_rdonly(log->ni->mi.sbi->sb)) { + err = -EROFS; + goto out; + } + } + +out: + ntfs_free(second_tail); + ntfs_free(first_tail); + ntfs_free(page); + ntfs_free(tst_page); + ntfs_free(page_bufs); + + return err; +} + +/* + * read_log_rec_buf + * + * copies a log record from the file to a buffer + * The log record may span several log pages and may even wrap the file + */ +static int read_log_rec_buf(struct ntfs_log *log, + const struct LFS_RECORD_HDR *rh, void *buffer) +{ + int err; + struct RECORD_PAGE_HDR *ph = NULL; + u64 lsn = le64_to_cpu(rh->this_lsn); + u32 vbo = lsn_to_vbo(log, lsn) & ~log->page_mask; + u32 off = lsn_to_page_off(log, lsn) + log->record_header_len; + u32 data_len = le32_to_cpu(rh->client_data_len); + + /* + * While there are more bytes to transfer, + * we continue to attempt to perform the read + */ + for (;;) { + bool usa_error; + u32 tail = log->page_size - off; + + if (tail >= data_len) + tail = data_len; + + data_len -= tail; + + err = read_log_page(log, vbo, &ph, &usa_error); + if (err) + goto out; + + /* + * The last lsn on this page better be greater or equal + * to the lsn we are copying + */ + if (lsn > le64_to_cpu(ph->rhdr.lsn)) { + err = -EINVAL; + goto out; + } + + memcpy(buffer, Add2Ptr(ph, off), tail); + + /* If there are no more bytes to transfer, we exit the loop */ + if (!data_len) { + if (!is_log_record_end(ph) || + lsn > le64_to_cpu(ph->record_hdr.last_end_lsn)) { + err = -EINVAL; + goto out; + } + break; + } + + if (ph->rhdr.lsn == ph->record_hdr.last_end_lsn || + lsn > le64_to_cpu(ph->rhdr.lsn)) { + err = -EINVAL; + goto out; + } + + vbo = next_page_off(log, vbo); + off = log->data_off; + + /* + * adjust our pointer the user's buffer to transfer + * the next block to + */ + buffer = Add2Ptr(buffer, tail); + } + +out: + ntfs_free(ph); + return err; +} + +static int read_rst_area(struct ntfs_log *log, struct NTFS_RESTART **rst_, + u64 *lsn) +{ + int err; + struct LFS_RECORD_HDR *rh = NULL; + const struct CLIENT_REC *cr = + Add2Ptr(log->ra, le16_to_cpu(log->ra->client_off)); + u64 lsnr, lsnc = le64_to_cpu(cr->restart_lsn); + u32 len; + struct NTFS_RESTART *rst; + + *lsn = 0; + *rst_ = NULL; + + /* If the client doesn't have a restart area, go ahead and exit now */ + if (!lsnc) + return 0; + + err = read_log_page(log, lsn_to_vbo(log, lsnc), + (struct RECORD_PAGE_HDR **)&rh, NULL); + if (err) + return err; + + rst = NULL; + lsnr = le64_to_cpu(rh->this_lsn); + + if (lsnc != lsnr) { + /* If the lsn values don't match, then the disk is corrupt */ + err = -EINVAL; + goto out; + } + + *lsn = lsnr; + len = le32_to_cpu(rh->client_data_len); + + if (!len) { + err = 0; + goto out; + } + + if (len < sizeof(struct NTFS_RESTART)) { + err = -EINVAL; + goto out; + } + + rst = ntfs_malloc(len); + if (!rst) { + err = -ENOMEM; + goto out; + } + + /* Copy the data into the 'rst' buffer */ + err = read_log_rec_buf(log, rh, rst); + if (err) + goto out; + + *rst_ = rst; + rst = NULL; + +out: + ntfs_free(rh); + ntfs_free(rst); + + return err; +} + +static int find_log_rec(struct ntfs_log *log, u64 lsn, struct lcb *lcb) +{ + int err; + struct LFS_RECORD_HDR *rh = lcb->lrh; + u32 rec_len, len; + + /* Read the record header for this lsn */ + if (!rh) { + err = read_log_page(log, lsn_to_vbo(log, lsn), + (struct RECORD_PAGE_HDR **)&rh, NULL); + + lcb->lrh = rh; + if (err) + return err; + } + + /* + * If the lsn the log record doesn't match the desired + * lsn then the disk is corrupt + */ + if (lsn != le64_to_cpu(rh->this_lsn)) + return -EINVAL; + + len = le32_to_cpu(rh->client_data_len); + + /* + * check that the length field isn't greater than the total + * available space the log file + */ + rec_len = len + log->record_header_len; + if (rec_len >= log->total_avail) + return -EINVAL; + + /* + * If the entire log record is on this log page, + * put a pointer to the log record the context block + */ + if (rh->flags & LOG_RECORD_MULTI_PAGE) { + void *lr = ntfs_malloc(len); + + if (!lr) + return -ENOMEM; + + lcb->log_rec = lr; + lcb->alloc = true; + + /* Copy the data into the buffer returned */ + err = read_log_rec_buf(log, rh, lr); + if (err) + return err; + } else { + /* If beyond the end of the current page -> an error */ + u32 page_off = lsn_to_page_off(log, lsn); + + if (page_off + len + log->record_header_len > log->page_size) + return -EINVAL; + + lcb->log_rec = Add2Ptr(rh, sizeof(struct LFS_RECORD_HDR)); + lcb->alloc = false; + } + + return 0; +} + +/* + * read_log_rec_lcb + * + * initiates the query operation. + */ +static int read_log_rec_lcb(struct ntfs_log *log, u64 lsn, u32 ctx_mode, + struct lcb **lcb_) +{ + int err; + const struct CLIENT_REC *cr; + struct lcb *lcb; + + switch (ctx_mode) { + case lcb_ctx_undo_next: + case lcb_ctx_prev: + case lcb_ctx_next: + break; + default: + return -EINVAL; + } + + /* check that the given lsn is the legal range for this client */ + cr = Add2Ptr(log->ra, le16_to_cpu(log->ra->client_off)); + + if (!verify_client_lsn(log, cr, lsn)) + return -EINVAL; + + lcb = ntfs_zalloc(sizeof(struct lcb)); + if (!lcb) + return -ENOMEM; + lcb->client = log->client_id; + lcb->ctx_mode = ctx_mode; + + /* Find the log record indicated by the given lsn */ + err = find_log_rec(log, lsn, lcb); + if (err) + goto out; + + *lcb_ = lcb; + return 0; + +out: + lcb_put(lcb); + *lcb_ = NULL; + return err; +} + +/* + * find_client_next_lsn + * + * attempt to find the next lsn to return to a client based on the context mode. + */ +static int find_client_next_lsn(struct ntfs_log *log, struct lcb *lcb, u64 *lsn) +{ + int err; + u64 next_lsn; + struct LFS_RECORD_HDR *hdr; + + hdr = lcb->lrh; + *lsn = 0; + + if (lcb_ctx_next != lcb->ctx_mode) + goto check_undo_next; + + /* Loop as long as another lsn can be found */ + for (;;) { + u64 current_lsn; + + err = next_log_lsn(log, hdr, ¤t_lsn); + if (err) + goto out; + + if (!current_lsn) + break; + + if (hdr != lcb->lrh) + ntfs_free(hdr); + + hdr = NULL; + err = read_log_page(log, lsn_to_vbo(log, current_lsn), + (struct RECORD_PAGE_HDR **)&hdr, NULL); + if (err) + goto out; + + if (memcmp(&hdr->client, &lcb->client, + sizeof(struct CLIENT_ID))) { + /*err = -EINVAL; */ + } else if (LfsClientRecord == hdr->record_type) { + ntfs_free(lcb->lrh); + lcb->lrh = hdr; + *lsn = current_lsn; + return 0; + } + } + +out: + if (hdr != lcb->lrh) + ntfs_free(hdr); + return err; + +check_undo_next: + if (lcb_ctx_undo_next == lcb->ctx_mode) + next_lsn = le64_to_cpu(hdr->client_undo_next_lsn); + else if (lcb_ctx_prev == lcb->ctx_mode) + next_lsn = le64_to_cpu(hdr->client_prev_lsn); + else + return 0; + + if (!next_lsn) + return 0; + + if (!verify_client_lsn( + log, Add2Ptr(log->ra, le16_to_cpu(log->ra->client_off)), + next_lsn)) + return 0; + + hdr = NULL; + err = read_log_page(log, lsn_to_vbo(log, next_lsn), + (struct RECORD_PAGE_HDR **)&hdr, NULL); + if (err) + return err; + ntfs_free(lcb->lrh); + lcb->lrh = hdr; + + *lsn = next_lsn; + + return 0; +} + +static int read_next_log_rec(struct ntfs_log *log, struct lcb *lcb, u64 *lsn) +{ + int err; + + err = find_client_next_lsn(log, lcb, lsn); + if (err) + return err; + + if (!*lsn) + return 0; + + if (lcb->alloc) + ntfs_free(lcb->log_rec); + + lcb->log_rec = NULL; + lcb->alloc = false; + ntfs_free(lcb->lrh); + lcb->lrh = NULL; + + return find_log_rec(log, *lsn, lcb); +} + +static inline bool check_index_header(const struct INDEX_HDR *hdr, size_t bytes) +{ + __le16 mask; + u32 min_de, de_off, used, total; + const struct NTFS_DE *e; + + if (hdr_has_subnode(hdr)) { + min_de = sizeof(struct NTFS_DE) + sizeof(u64); + mask = NTFS_IE_HAS_SUBNODES; + } else { + min_de = sizeof(struct NTFS_DE); + mask = 0; + } + + de_off = le32_to_cpu(hdr->de_off); + used = le32_to_cpu(hdr->used); + total = le32_to_cpu(hdr->total); + + if (de_off > bytes - min_de || used > bytes || total > bytes || + de_off + min_de > used || used > total) { + return false; + } + + e = Add2Ptr(hdr, de_off); + for (;;) { + u16 esize = le16_to_cpu(e->size); + struct NTFS_DE *next = Add2Ptr(e, esize); + + if (esize < min_de || PtrOffset(hdr, next) > used || + (e->flags & NTFS_IE_HAS_SUBNODES) != mask) { + return false; + } + + if (de_is_last(e)) + break; + + e = next; + } + + return true; +} + +static inline bool check_index_buffer(const struct INDEX_BUFFER *ib, u32 bytes) +{ + u16 fo; + const struct NTFS_RECORD_HEADER *r = &ib->rhdr; + + if (r->sign != NTFS_INDX_SIGNATURE) + return false; + + fo = (SECTOR_SIZE - ((bytes >> SECTOR_SHIFT) + 1) * sizeof(short)); + + if (le16_to_cpu(r->fix_off) > fo) + return false; + + if ((le16_to_cpu(r->fix_num) - 1) * SECTOR_SIZE != bytes) + return false; + + return check_index_header(&ib->ihdr, + bytes - offsetof(struct INDEX_BUFFER, ihdr)); +} + +static inline bool check_index_root(const struct ATTRIB *attr, + struct ntfs_sb_info *sbi) +{ + bool ret; + const struct INDEX_ROOT *root = resident_data(attr); + u8 index_bits = le32_to_cpu(root->index_block_size) >= sbi->cluster_size + ? sbi->cluster_bits + : SECTOR_SHIFT; + u8 block_clst = root->index_block_clst; + + if (le32_to_cpu(attr->res.data_size) < sizeof(struct INDEX_ROOT) || + (root->type != ATTR_NAME && root->type != ATTR_ZERO) || + (root->type == ATTR_NAME && + root->rule != NTFS_COLLATION_TYPE_FILENAME) || + (le32_to_cpu(root->index_block_size) != + (block_clst << index_bits)) || + (block_clst != 1 && block_clst != 2 && block_clst != 4 && + block_clst != 8 && block_clst != 0x10 && block_clst != 0x20 && + block_clst != 0x40 && block_clst != 0x80)) { + return false; + } + + ret = check_index_header(&root->ihdr, + le32_to_cpu(attr->res.data_size) - + offsetof(struct INDEX_ROOT, ihdr)); + return ret; +} + +static inline bool check_attr(const struct MFT_REC *rec, + const struct ATTRIB *attr, + struct ntfs_sb_info *sbi) +{ + u32 asize = le32_to_cpu(attr->size); + u32 rsize = 0; + u64 dsize, svcn, evcn; + u16 run_off; + + /* Check the fixed part of the attribute record header */ + if (asize >= sbi->record_size || + asize + PtrOffset(rec, attr) >= sbi->record_size || + (attr->name_len && + le16_to_cpu(attr->name_off) + attr->name_len * sizeof(short) > + asize)) { + return false; + } + + /* Check the attribute fields */ + switch (attr->non_res) { + case 0: + rsize = le32_to_cpu(attr->res.data_size); + if (rsize >= asize || + le16_to_cpu(attr->res.data_off) + rsize > asize) { + return false; + } + break; + + case 1: + dsize = le64_to_cpu(attr->nres.data_size); + svcn = le64_to_cpu(attr->nres.svcn); + evcn = le64_to_cpu(attr->nres.evcn); + run_off = le16_to_cpu(attr->nres.run_off); + + if (svcn > evcn + 1 || run_off >= asize || + le64_to_cpu(attr->nres.valid_size) > dsize || + dsize > le64_to_cpu(attr->nres.alloc_size)) { + return false; + } + + if (run_unpack(NULL, sbi, 0, svcn, evcn, svcn, + Add2Ptr(attr, run_off), asize - run_off) < 0) { + return false; + } + + return true; + + default: + return false; + } + + switch (attr->type) { + case ATTR_NAME: + if (fname_full_size(Add2Ptr( + attr, le16_to_cpu(attr->res.data_off))) > asize) { + return false; + } + break; + + case ATTR_ROOT: + return check_index_root(attr, sbi); + + case ATTR_STD: + if (rsize < sizeof(struct ATTR_STD_INFO5) && + rsize != sizeof(struct ATTR_STD_INFO)) { + return false; + } + break; + + case ATTR_LIST: + case ATTR_ID: + case ATTR_SECURE: + case ATTR_LABEL: + case ATTR_VOL_INFO: + case ATTR_DATA: + case ATTR_ALLOC: + case ATTR_BITMAP: + case ATTR_REPARSE: + case ATTR_EA_INFO: + case ATTR_EA: + case ATTR_PROPERTYSET: + case ATTR_LOGGED_UTILITY_STREAM: + break; + + default: + return false; + } + + return true; +} + +static inline bool check_file_record(const struct MFT_REC *rec, + const struct MFT_REC *rec2, + struct ntfs_sb_info *sbi) +{ + const struct ATTRIB *attr; + u16 fo = le16_to_cpu(rec->rhdr.fix_off); + u16 fn = le16_to_cpu(rec->rhdr.fix_num); + u16 ao = le16_to_cpu(rec->attr_off); + u32 rs = sbi->record_size; + + /* check the file record header for consistency */ + if (rec->rhdr.sign != NTFS_FILE_SIGNATURE || + fo > (SECTOR_SIZE - ((rs >> SECTOR_SHIFT) + 1) * sizeof(short)) || + (fn - 1) * SECTOR_SIZE != rs || ao < MFTRECORD_FIXUP_OFFSET_1 || + ao > sbi->record_size - SIZEOF_RESIDENT || !is_rec_inuse(rec) || + le32_to_cpu(rec->total) != rs) { + return false; + } + + /* Loop to check all of the attributes */ + for (attr = Add2Ptr(rec, ao); attr->type != ATTR_END; + attr = Add2Ptr(attr, le32_to_cpu(attr->size))) { + if (check_attr(rec, attr, sbi)) + continue; + return false; + } + + return true; +} + +static inline int check_lsn(const struct NTFS_RECORD_HEADER *hdr, + const u64 *rlsn) +{ + u64 lsn; + + if (!rlsn) + return true; + + lsn = le64_to_cpu(hdr->lsn); + + if (hdr->sign == NTFS_HOLE_SIGNATURE) + return false; + + if (*rlsn > lsn) + return true; + + return false; +} + +static inline bool check_if_attr(const struct MFT_REC *rec, + const struct LOG_REC_HDR *lrh) +{ + u16 ro = le16_to_cpu(lrh->record_off); + u16 o = le16_to_cpu(rec->attr_off); + const struct ATTRIB *attr = Add2Ptr(rec, o); + + while (o < ro) { + u32 asize; + + if (attr->type == ATTR_END) + break; + + asize = le32_to_cpu(attr->size); + if (!asize) + break; + + o += asize; + attr = Add2Ptr(attr, asize); + } + + return o == ro; +} + +static inline bool check_if_index_root(const struct MFT_REC *rec, + const struct LOG_REC_HDR *lrh) +{ + u16 ro = le16_to_cpu(lrh->record_off); + u16 o = le16_to_cpu(rec->attr_off); + const struct ATTRIB *attr = Add2Ptr(rec, o); + + while (o < ro) { + u32 asize; + + if (attr->type == ATTR_END) + break; + + asize = le32_to_cpu(attr->size); + if (!asize) + break; + + o += asize; + attr = Add2Ptr(attr, asize); + } + + return o == ro && attr->type == ATTR_ROOT; +} + +static inline bool check_if_root_index(const struct ATTRIB *attr, + const struct INDEX_HDR *hdr, + const struct LOG_REC_HDR *lrh) +{ + u16 ao = le16_to_cpu(lrh->attr_off); + u32 de_off = le32_to_cpu(hdr->de_off); + u32 o = PtrOffset(attr, hdr) + de_off; + const struct NTFS_DE *e = Add2Ptr(hdr, de_off); + u32 asize = le32_to_cpu(attr->size); + + while (o < ao) { + u16 esize; + + if (o >= asize) + break; + + esize = le16_to_cpu(e->size); + if (!esize) + break; + + o += esize; + e = Add2Ptr(e, esize); + } + + return o == ao; +} + +static inline bool check_if_alloc_index(const struct INDEX_HDR *hdr, + u32 attr_off) +{ + u32 de_off = le32_to_cpu(hdr->de_off); + u32 o = offsetof(struct INDEX_BUFFER, ihdr) + de_off; + const struct NTFS_DE *e = Add2Ptr(hdr, de_off); + u32 used = le32_to_cpu(hdr->used); + + while (o < attr_off) { + u16 esize; + + if (de_off >= used) + break; + + esize = le16_to_cpu(e->size); + if (!esize) + break; + + o += esize; + de_off += esize; + e = Add2Ptr(e, esize); + } + + return o == attr_off; +} + +static inline void change_attr_size(struct MFT_REC *rec, struct ATTRIB *attr, + u32 nsize) +{ + u32 asize = le32_to_cpu(attr->size); + int dsize = nsize - asize; + u8 *next = Add2Ptr(attr, asize); + u32 used = le32_to_cpu(rec->used); + + memmove(Add2Ptr(attr, nsize), next, used - PtrOffset(rec, next)); + + rec->used = cpu_to_le32(used + dsize); + attr->size = cpu_to_le32(nsize); +} + +struct OpenAttr { + struct ATTRIB *attr; + struct runs_tree *run1; + struct runs_tree run0; + struct ntfs_inode *ni; + // CLST rno; +}; + +/* Returns 0 if 'attr' has the same type and name */ +static inline int cmp_type_and_name(const struct ATTRIB *a1, + const struct ATTRIB *a2) +{ + return a1->type != a2->type || a1->name_len != a2->name_len || + (a1->name_len && memcmp(attr_name(a1), attr_name(a2), + a1->name_len * sizeof(short))); +} + +static struct OpenAttr *find_loaded_attr(struct ntfs_log *log, + const struct ATTRIB *attr, CLST rno) +{ + struct OPEN_ATTR_ENRTY *oe = NULL; + + while ((oe = enum_rstbl(log->open_attr_tbl, oe))) { + struct OpenAttr *op_attr; + + if (ino_get(&oe->ref) != rno) + continue; + + op_attr = (struct OpenAttr *)oe->ptr; + if (!cmp_type_and_name(op_attr->attr, attr)) + return op_attr; + } + return NULL; +} + +static struct ATTRIB *attr_create_nonres_log(struct ntfs_sb_info *sbi, + enum ATTR_TYPE type, u64 size, + const u16 *name, size_t name_len, + __le16 flags) +{ + struct ATTRIB *attr; + u32 name_size = QuadAlign(name_len * sizeof(short)); + bool is_ext = flags & (ATTR_FLAG_COMPRESSED | ATTR_FLAG_SPARSED); + u32 asize = name_size + + (is_ext ? SIZEOF_NONRESIDENT_EX : SIZEOF_NONRESIDENT); + + attr = ntfs_zalloc(asize); + if (!attr) + return NULL; + + attr->type = type; + attr->size = cpu_to_le32(asize); + attr->flags = flags; + attr->non_res = 1; + attr->name_len = name_len; + + attr->nres.evcn = cpu_to_le64((u64)bytes_to_cluster(sbi, size) - 1); + attr->nres.alloc_size = cpu_to_le64(ntfs_up_cluster(sbi, size)); + attr->nres.data_size = cpu_to_le64(size); + attr->nres.valid_size = attr->nres.data_size; + if (is_ext) { + attr->name_off = SIZEOF_NONRESIDENT_EX_LE; + if (is_attr_compressed(attr)) + attr->nres.c_unit = COMPRESSION_UNIT; + + attr->nres.run_off = + cpu_to_le16(SIZEOF_NONRESIDENT_EX + name_size); + memcpy(Add2Ptr(attr, SIZEOF_NONRESIDENT_EX), name, + name_len * sizeof(short)); + } else { + attr->name_off = SIZEOF_NONRESIDENT_LE; + attr->nres.run_off = + cpu_to_le16(SIZEOF_NONRESIDENT + name_size); + memcpy(Add2Ptr(attr, SIZEOF_NONRESIDENT), name, + name_len * sizeof(short)); + } + + return attr; +} + +/* + * do_action + * + * common routine for the Redo and Undo Passes + * If rlsn is NULL then undo + */ +static int do_action(struct ntfs_log *log, struct OPEN_ATTR_ENRTY *oe, + const struct LOG_REC_HDR *lrh, u32 op, void *data, + u32 dlen, u32 rec_len, const u64 *rlsn) +{ + int err = 0; + struct ntfs_sb_info *sbi = log->ni->mi.sbi; + struct inode *inode = NULL, *inode_parent; + struct mft_inode *mi = NULL, *mi2_child = NULL; + CLST rno = 0, rno_base = 0; + struct INDEX_BUFFER *ib = NULL; + struct MFT_REC *rec = NULL; + struct ATTRIB *attr = NULL, *attr2; + struct INDEX_HDR *hdr; + struct INDEX_ROOT *root; + struct NTFS_DE *e, *e1, *e2; + struct NEW_ATTRIBUTE_SIZES *new_sz; + struct ATTR_FILE_NAME *fname; + struct OpenAttr *oa, *oa2; + u32 nsize, t32, asize, used, esize, bmp_off, bmp_bits; + u16 id, id2; + u32 record_size = sbi->record_size; + u64 t64; + u16 roff = le16_to_cpu(lrh->record_off); + u16 aoff = le16_to_cpu(lrh->attr_off); + u64 lco = 0; + u64 cbo = (u64)le16_to_cpu(lrh->cluster_off) << SECTOR_SHIFT; + u64 tvo = le64_to_cpu(lrh->target_vcn) << sbi->cluster_bits; + u64 vbo = cbo + tvo; + void *buffer_le = NULL; + u32 bytes = 0; + bool a_dirty = false; + u16 data_off; + + oa = oe->ptr; + + /* Big switch to prepare */ + switch (op) { + /* ============================================================ + * Process MFT records, as described by the current log record + * ============================================================ + */ + case InitializeFileRecordSegment: + case DeallocateFileRecordSegment: + case WriteEndOfFileRecordSegment: + case CreateAttribute: + case DeleteAttribute: + case UpdateResidentValue: + case UpdateMappingPairs: + case SetNewAttributeSizes: + case AddIndexEntryRoot: + case DeleteIndexEntryRoot: + case SetIndexEntryVcnRoot: + case UpdateFileNameRoot: + case UpdateRecordDataRoot: + case ZeroEndOfFileRecord: + rno = vbo >> sbi->record_bits; + inode = ilookup(sbi->sb, rno); + if (inode) { + mi = &ntfs_i(inode)->mi; + } else if (op == InitializeFileRecordSegment) { + mi = ntfs_zalloc(sizeof(struct mft_inode)); + if (!mi) + return -ENOMEM; + err = mi_format_new(mi, sbi, rno, 0, false); + if (err) + goto out; + } else { + /* read from disk */ + err = mi_get(sbi, rno, &mi); + if (err) + return err; + } + rec = mi->mrec; + + if (op == DeallocateFileRecordSegment) + goto skip_load_parent; + + if (InitializeFileRecordSegment != op) { + if (rec->rhdr.sign == NTFS_BAAD_SIGNATURE) + goto dirty_vol; + if (!check_lsn(&rec->rhdr, rlsn)) + goto out; + if (!check_file_record(rec, NULL, sbi)) + goto dirty_vol; + attr = Add2Ptr(rec, roff); + } + + if (is_rec_base(rec) || InitializeFileRecordSegment == op) { + rno_base = rno; + goto skip_load_parent; + } + + rno_base = ino_get(&rec->parent_ref); + inode_parent = ntfs_iget5(sbi->sb, &rec->parent_ref, NULL); + if (IS_ERR(inode_parent)) + goto skip_load_parent; + + if (is_bad_inode(inode_parent)) { + iput(inode_parent); + goto skip_load_parent; + } + + if (ni_load_mi_ex(ntfs_i(inode_parent), rno, &mi2_child)) { + iput(inode_parent); + } else { + if (mi2_child->mrec != mi->mrec) + memcpy(mi2_child->mrec, mi->mrec, + sbi->record_size); + + if (inode) + iput(inode); + else if (mi) + mi_put(mi); + + inode = inode_parent; + mi = mi2_child; + rec = mi2_child->mrec; + attr = Add2Ptr(rec, roff); + } + +skip_load_parent: + inode_parent = NULL; + break; + + /* ============================================================ + * Process attributes, as described by the current log record + * ============================================================ + */ + case UpdateNonresidentValue: + case AddIndexEntryAllocation: + case DeleteIndexEntryAllocation: + case WriteEndOfIndexBuffer: + case SetIndexEntryVcnAllocation: + case UpdateFileNameAllocation: + case SetBitsInNonresidentBitMap: + case ClearBitsInNonresidentBitMap: + case UpdateRecordDataAllocation: + attr = oa->attr; + bytes = UpdateNonresidentValue == op ? dlen : 0; + lco = (u64)le16_to_cpu(lrh->lcns_follow) << sbi->cluster_bits; + + if (attr->type == ATTR_ALLOC) { + t32 = le32_to_cpu(oe->bytes_per_index); + if (bytes < t32) + bytes = t32; + } + + if (!bytes) + bytes = lco - cbo; + + bytes += roff; + if (attr->type == ATTR_ALLOC) + bytes = (bytes + 511) & ~511; // align + + buffer_le = ntfs_malloc(bytes); + if (!buffer_le) + return -ENOMEM; + + err = ntfs_read_run_nb(sbi, oa->run1, vbo, buffer_le, bytes, + NULL); + if (err) + goto out; + + if (attr->type == ATTR_ALLOC && *(int *)buffer_le) + ntfs_fix_post_read(buffer_le, bytes, false); + break; + + default: + WARN_ON(1); + } + + /* Big switch to do operation */ + switch (op) { + case InitializeFileRecordSegment: + if (roff + dlen > record_size) + goto dirty_vol; + + memcpy(Add2Ptr(rec, roff), data, dlen); + mi->dirty = true; + break; + + case DeallocateFileRecordSegment: + clear_rec_inuse(rec); + le16_add_cpu(&rec->seq, 1); + mi->dirty = true; + break; + + case WriteEndOfFileRecordSegment: + attr2 = (struct ATTRIB *)data; + if (!check_if_attr(rec, lrh) || roff + dlen > record_size) + goto dirty_vol; + + memmove(attr, attr2, dlen); + rec->used = cpu_to_le32(QuadAlign(roff + dlen)); + + mi->dirty = true; + break; + + case CreateAttribute: + attr2 = (struct ATTRIB *)data; + asize = le32_to_cpu(attr2->size); + used = le32_to_cpu(rec->used); + + if (!check_if_attr(rec, lrh) || dlen < SIZEOF_RESIDENT || + !IsQuadAligned(asize) || + Add2Ptr(attr2, asize) > Add2Ptr(lrh, rec_len) || + dlen > record_size - used) { + goto dirty_vol; + } + + memmove(Add2Ptr(attr, asize), attr, used - roff); + memcpy(attr, attr2, asize); + + rec->used = cpu_to_le32(used + asize); + id = le16_to_cpu(rec->next_attr_id); + id2 = le16_to_cpu(attr2->id); + if (id <= id2) + rec->next_attr_id = cpu_to_le16(id2 + 1); + if (is_attr_indexed(attr)) + le16_add_cpu(&rec->hard_links, 1); + + oa2 = find_loaded_attr(log, attr, rno_base); + if (oa2) { + void *p2 = ntfs_memdup(attr, le32_to_cpu(attr->size)); + + if (p2) { + // run_close(oa2->run1); + ntfs_free(oa2->attr); + oa2->attr = p2; + } + } + + mi->dirty = true; + break; + + case DeleteAttribute: + asize = le32_to_cpu(attr->size); + used = le32_to_cpu(rec->used); + + if (!check_if_attr(rec, lrh)) + goto dirty_vol; + + rec->used = cpu_to_le32(used - asize); + if (is_attr_indexed(attr)) + le16_add_cpu(&rec->hard_links, -1); + + memmove(attr, Add2Ptr(attr, asize), used - asize - roff); + + mi->dirty = true; + break; + + case UpdateResidentValue: + nsize = aoff + dlen; + + if (!check_if_attr(rec, lrh)) + goto dirty_vol; + + asize = le32_to_cpu(attr->size); + used = le32_to_cpu(rec->used); + + if (lrh->redo_len == lrh->undo_len) { + if (nsize > asize) + goto dirty_vol; + goto move_data; + } + + if (nsize > asize && nsize - asize > record_size - used) + goto dirty_vol; + + nsize = QuadAlign(nsize); + data_off = le16_to_cpu(attr->res.data_off); + + if (nsize < asize) { + memmove(Add2Ptr(attr, aoff), data, dlen); + data = NULL; // To skip below memmove + } + + memmove(Add2Ptr(attr, nsize), Add2Ptr(attr, asize), + used - le16_to_cpu(lrh->record_off) - asize); + + rec->used = cpu_to_le32(used + nsize - asize); + attr->size = cpu_to_le32(nsize); + attr->res.data_size = cpu_to_le32(aoff + dlen - data_off); + +move_data: + if (data) + memmove(Add2Ptr(attr, aoff), data, dlen); + + oa2 = find_loaded_attr(log, attr, rno_base); + if (oa2) { + void *p2 = ntfs_memdup(attr, le32_to_cpu(attr->size)); + + if (p2) { + // run_close(&oa2->run0); + oa2->run1 = &oa2->run0; + ntfs_free(oa2->attr); + oa2->attr = p2; + } + } + + mi->dirty = true; + break; + + case UpdateMappingPairs: + nsize = aoff + dlen; + asize = le32_to_cpu(attr->size); + used = le32_to_cpu(rec->used); + + if (!check_if_attr(rec, lrh) || !attr->non_res || + aoff < le16_to_cpu(attr->nres.run_off) || aoff > asize || + (nsize > asize && nsize - asize > record_size - used)) { + goto dirty_vol; + } + + nsize = QuadAlign(nsize); + + memmove(Add2Ptr(attr, nsize), Add2Ptr(attr, asize), + used - le16_to_cpu(lrh->record_off) - asize); + rec->used = cpu_to_le32(used + nsize - asize); + attr->size = cpu_to_le32(nsize); + memmove(Add2Ptr(attr, aoff), data, dlen); + + if (run_get_highest_vcn(le64_to_cpu(attr->nres.svcn), + attr_run(attr), &t64)) { + goto dirty_vol; + } + + attr->nres.evcn = cpu_to_le64(t64); + oa2 = find_loaded_attr(log, attr, rno_base); + if (oa2 && oa2->attr->non_res) + oa2->attr->nres.evcn = attr->nres.evcn; + + mi->dirty = true; + break; + + case SetNewAttributeSizes: + new_sz = data; + if (!check_if_attr(rec, lrh) || !attr->non_res) + goto dirty_vol; + + attr->nres.alloc_size = new_sz->alloc_size; + attr->nres.data_size = new_sz->data_size; + attr->nres.valid_size = new_sz->valid_size; + + if (dlen >= sizeof(struct NEW_ATTRIBUTE_SIZES)) + attr->nres.total_size = new_sz->total_size; + + oa2 = find_loaded_attr(log, attr, rno_base); + if (oa2) { + void *p2 = ntfs_memdup(attr, le32_to_cpu(attr->size)); + + if (p2) { + ntfs_free(oa2->attr); + oa2->attr = p2; + } + } + mi->dirty = true; + break; + + case AddIndexEntryRoot: + e = (struct NTFS_DE *)data; + esize = le16_to_cpu(e->size); + root = resident_data(attr); + hdr = &root->ihdr; + used = le32_to_cpu(hdr->used); + + if (!check_if_index_root(rec, lrh) || + !check_if_root_index(attr, hdr, lrh) || + Add2Ptr(data, esize) > Add2Ptr(lrh, rec_len) || + esize > le32_to_cpu(rec->total) - le32_to_cpu(rec->used)) { + goto dirty_vol; + } + + e1 = Add2Ptr(attr, le16_to_cpu(lrh->attr_off)); + + change_attr_size(rec, attr, le32_to_cpu(attr->size) + esize); + + memmove(Add2Ptr(e1, esize), e1, + PtrOffset(e1, Add2Ptr(hdr, used))); + memmove(e1, e, esize); + + le32_add_cpu(&attr->res.data_size, esize); + hdr->used = cpu_to_le32(used + esize); + le32_add_cpu(&hdr->total, esize); + + mi->dirty = true; + break; + + case DeleteIndexEntryRoot: + root = resident_data(attr); + hdr = &root->ihdr; + used = le32_to_cpu(hdr->used); + + if (!check_if_index_root(rec, lrh) || + !check_if_root_index(attr, hdr, lrh)) { + goto dirty_vol; + } + + e1 = Add2Ptr(attr, le16_to_cpu(lrh->attr_off)); + esize = le16_to_cpu(e1->size); + e2 = Add2Ptr(e1, esize); + + memmove(e1, e2, PtrOffset(e2, Add2Ptr(hdr, used))); + + le32_sub_cpu(&attr->res.data_size, esize); + hdr->used = cpu_to_le32(used - esize); + le32_sub_cpu(&hdr->total, esize); + + change_attr_size(rec, attr, le32_to_cpu(attr->size) - esize); + + mi->dirty = true; + break; + + case SetIndexEntryVcnRoot: + root = resident_data(attr); + hdr = &root->ihdr; + + if (!check_if_index_root(rec, lrh) || + !check_if_root_index(attr, hdr, lrh)) { + goto dirty_vol; + } + + e = Add2Ptr(attr, le16_to_cpu(lrh->attr_off)); + + de_set_vbn_le(e, *(__le64 *)data); + mi->dirty = true; + break; + + case UpdateFileNameRoot: + root = resident_data(attr); + hdr = &root->ihdr; + + if (!check_if_index_root(rec, lrh) || + !check_if_root_index(attr, hdr, lrh)) { + goto dirty_vol; + } + + e = Add2Ptr(attr, le16_to_cpu(lrh->attr_off)); + fname = (struct ATTR_FILE_NAME *)(e + 1); + memmove(&fname->dup, data, sizeof(fname->dup)); // + mi->dirty = true; + break; + + case UpdateRecordDataRoot: + root = resident_data(attr); + hdr = &root->ihdr; + + if (!check_if_index_root(rec, lrh) || + !check_if_root_index(attr, hdr, lrh)) { + goto dirty_vol; + } + + e = Add2Ptr(attr, le16_to_cpu(lrh->attr_off)); + + memmove(Add2Ptr(e, le16_to_cpu(e->view.data_off)), data, dlen); + + mi->dirty = true; + break; + + case ZeroEndOfFileRecord: + if (roff + dlen > record_size) + goto dirty_vol; + + memset(attr, 0, dlen); + mi->dirty = true; + break; + + case UpdateNonresidentValue: + if (lco < cbo + roff + dlen) + goto dirty_vol; + + memcpy(Add2Ptr(buffer_le, roff), data, dlen); + + a_dirty = true; + if (attr->type == ATTR_ALLOC) + ntfs_fix_pre_write(buffer_le, bytes); + break; + + case AddIndexEntryAllocation: + ib = Add2Ptr(buffer_le, roff); + hdr = &ib->ihdr; + e = data; + esize = le16_to_cpu(e->size); + e1 = Add2Ptr(ib, aoff); + + if (is_baad(&ib->rhdr)) + goto dirty_vol; + if (!check_lsn(&ib->rhdr, rlsn)) + goto out; + + used = le32_to_cpu(hdr->used); + + if (!check_index_buffer(ib, bytes) || + !check_if_alloc_index(hdr, aoff) || + Add2Ptr(e, esize) > Add2Ptr(lrh, rec_len) || + used + esize > le32_to_cpu(hdr->total)) { + goto dirty_vol; + } + + memmove(Add2Ptr(e1, esize), e1, + PtrOffset(e1, Add2Ptr(hdr, used))); + memcpy(e1, e, esize); + + hdr->used = cpu_to_le32(used + esize); + + a_dirty = true; + + ntfs_fix_pre_write(&ib->rhdr, bytes); + break; + + case DeleteIndexEntryAllocation: + ib = Add2Ptr(buffer_le, roff); + hdr = &ib->ihdr; + e = Add2Ptr(ib, aoff); + esize = le16_to_cpu(e->size); + + if (is_baad(&ib->rhdr)) + goto dirty_vol; + if (!check_lsn(&ib->rhdr, rlsn)) + goto out; + + if (!check_index_buffer(ib, bytes) || + !check_if_alloc_index(hdr, aoff)) { + goto dirty_vol; + } + + e1 = Add2Ptr(e, esize); + nsize = esize; + used = le32_to_cpu(hdr->used); + + memmove(e, e1, PtrOffset(e1, Add2Ptr(hdr, used))); + + hdr->used = cpu_to_le32(used - nsize); + + a_dirty = true; + + ntfs_fix_pre_write(&ib->rhdr, bytes); + break; + + case WriteEndOfIndexBuffer: + ib = Add2Ptr(buffer_le, roff); + hdr = &ib->ihdr; + e = Add2Ptr(ib, aoff); + + if (is_baad(&ib->rhdr)) + goto dirty_vol; + if (!check_lsn(&ib->rhdr, rlsn)) + goto out; + if (!check_index_buffer(ib, bytes) || + !check_if_alloc_index(hdr, aoff) || + aoff + dlen > offsetof(struct INDEX_BUFFER, ihdr) + + le32_to_cpu(hdr->total)) { + goto dirty_vol; + } + + hdr->used = cpu_to_le32(dlen + PtrOffset(hdr, e)); + memmove(e, data, dlen); + + a_dirty = true; + ntfs_fix_pre_write(&ib->rhdr, bytes); + break; + + case SetIndexEntryVcnAllocation: + ib = Add2Ptr(buffer_le, roff); + hdr = &ib->ihdr; + e = Add2Ptr(ib, aoff); + + if (is_baad(&ib->rhdr)) + goto dirty_vol; + + if (!check_lsn(&ib->rhdr, rlsn)) + goto out; + if (!check_index_buffer(ib, bytes) || + !check_if_alloc_index(hdr, aoff)) { + goto dirty_vol; + } + + de_set_vbn_le(e, *(__le64 *)data); + + a_dirty = true; + ntfs_fix_pre_write(&ib->rhdr, bytes); + break; + + case UpdateFileNameAllocation: + ib = Add2Ptr(buffer_le, roff); + hdr = &ib->ihdr; + e = Add2Ptr(ib, aoff); + + if (is_baad(&ib->rhdr)) + goto dirty_vol; + + if (!check_lsn(&ib->rhdr, rlsn)) + goto out; + if (!check_index_buffer(ib, bytes) || + !check_if_alloc_index(hdr, aoff)) { + goto dirty_vol; + } + + fname = (struct ATTR_FILE_NAME *)(e + 1); + memmove(&fname->dup, data, sizeof(fname->dup)); + + a_dirty = true; + ntfs_fix_pre_write(&ib->rhdr, bytes); + break; + + case SetBitsInNonresidentBitMap: + bmp_off = + le32_to_cpu(((struct BITMAP_RANGE *)data)->bitmap_off); + bmp_bits = le32_to_cpu(((struct BITMAP_RANGE *)data)->bits); + + if (cbo + (bmp_off + 7) / 8 > lco || + cbo + ((bmp_off + bmp_bits + 7) / 8) > lco) { + goto dirty_vol; + } + + __bitmap_set(Add2Ptr(buffer_le, roff), bmp_off, bmp_bits); + a_dirty = true; + break; + + case ClearBitsInNonresidentBitMap: + bmp_off = + le32_to_cpu(((struct BITMAP_RANGE *)data)->bitmap_off); + bmp_bits = le32_to_cpu(((struct BITMAP_RANGE *)data)->bits); + + if (cbo + (bmp_off + 7) / 8 > lco || + cbo + ((bmp_off + bmp_bits + 7) / 8) > lco) { + goto dirty_vol; + } + + __bitmap_clear(Add2Ptr(buffer_le, roff), bmp_off, bmp_bits); + a_dirty = true; + break; + + case UpdateRecordDataAllocation: + ib = Add2Ptr(buffer_le, roff); + hdr = &ib->ihdr; + e = Add2Ptr(ib, aoff); + + if (is_baad(&ib->rhdr)) + goto dirty_vol; + + if (!check_lsn(&ib->rhdr, rlsn)) + goto out; + if (!check_index_buffer(ib, bytes) || + !check_if_alloc_index(hdr, aoff)) { + goto dirty_vol; + } + + memmove(Add2Ptr(e, le16_to_cpu(e->view.data_off)), data, dlen); + + a_dirty = true; + ntfs_fix_pre_write(&ib->rhdr, bytes); + break; + + default: + WARN_ON(1); + } + + if (rlsn) { + __le64 t64 = cpu_to_le64(*rlsn); + + if (rec) + rec->rhdr.lsn = t64; + if (ib) + ib->rhdr.lsn = t64; + } + + if (mi && mi->dirty) { + err = mi_write(mi, 0); + if (err) + goto out; + } + + if (a_dirty) { + attr = oa->attr; + err = ntfs_sb_write_run(sbi, oa->run1, vbo, buffer_le, bytes); + if (err) + goto out; + } + +out: + + if (inode) + iput(inode); + else if (mi != mi2_child) + mi_put(mi); + + ntfs_free(buffer_le); + + return err; + +dirty_vol: + log->set_dirty = true; + goto out; +} + +/* + * log_replay + * + * this function is called during mount operation + * it replays log and empties it + * initialized is set false if logfile contains '-1' + */ +int log_replay(struct ntfs_inode *ni, bool *initialized) +{ + int err; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct ntfs_log *log; + + struct restart_info rst_info, rst_info2; + u64 rec_lsn, ra_lsn, checkpt_lsn = 0, rlsn = 0; + struct ATTR_NAME_ENTRY *attr_names = NULL; + struct ATTR_NAME_ENTRY *ane; + struct RESTART_TABLE *dptbl = NULL; + struct RESTART_TABLE *trtbl = NULL; + const struct RESTART_TABLE *rt; + struct RESTART_TABLE *oatbl = NULL; + struct inode *inode; + struct OpenAttr *oa; + struct ntfs_inode *ni_oe; + struct ATTRIB *attr = NULL; + u64 size, vcn, undo_next_lsn; + CLST rno, lcn, lcn0, len0, clen; + void *data; + struct NTFS_RESTART *rst = NULL; + struct lcb *lcb = NULL; + struct OPEN_ATTR_ENRTY *oe; + struct TRANSACTION_ENTRY *tr; + struct DIR_PAGE_ENTRY *dp; + u32 i, bytes_per_attr_entry; + u32 l_size = ni->vfs_inode.i_size; + u32 orig_file_size = l_size; + u32 page_size, vbo, tail, off, dlen; + u32 saved_len, rec_len, transact_id; + bool use_second_page; + struct RESTART_AREA *ra2, *ra = NULL; + struct CLIENT_REC *ca, *cr; + __le16 client; + struct RESTART_HDR *rh; + const struct LFS_RECORD_HDR *frh; + const struct LOG_REC_HDR *lrh; + bool is_mapped; + bool is_ro = sb_rdonly(sbi->sb); + u64 t64; + u16 t16; + u32 t32; + + /* Get the size of page. NOTE: To replay we can use default page */ +#if PAGE_SIZE >= DefaultLogPageSize && PAGE_SIZE <= DefaultLogPageSize * 2 + page_size = norm_file_page(PAGE_SIZE, &l_size, true); +#else + page_size = norm_file_page(PAGE_SIZE, &l_size, false); +#endif + if (!page_size) + return -EINVAL; + + log = ntfs_zalloc(sizeof(struct ntfs_log)); + if (!log) + return -ENOMEM; + + log->ni = ni; + log->l_size = l_size; + log->one_page_buf = ntfs_malloc(page_size); + + if (!log->one_page_buf) { + err = -ENOMEM; + goto out; + } + + log->page_size = page_size; + log->page_mask = page_size - 1; + log->page_bits = blksize_bits(page_size); + + /* Look for a restart area on the disk */ + err = log_read_rst(log, l_size, true, &rst_info); + if (err) + goto out; + + /* remember 'initialized' */ + *initialized = rst_info.initialized; + + if (!rst_info.restart) { + if (rst_info.initialized) { + /* no restart area but the file is not initialized */ + err = -EINVAL; + goto out; + } + + log_init_pg_hdr(log, page_size, page_size, 1, 1); + log_create(log, l_size, 0, get_random_int(), false, false); + + log->ra = ra; + + ra = log_create_ra(log); + if (!ra) { + err = -ENOMEM; + goto out; + } + log->ra = ra; + log->init_ra = true; + + goto process_log; + } + + /* + * If the restart offset above wasn't zero then we won't + * look for a second restart + */ + if (rst_info.vbo) + goto check_restart_area; + + err = log_read_rst(log, l_size, false, &rst_info2); + + /* Determine which restart area to use */ + if (!rst_info2.restart || rst_info2.last_lsn <= rst_info.last_lsn) + goto use_first_page; + + use_second_page = true; + + if (rst_info.chkdsk_was_run && page_size != rst_info.vbo) { + struct RECORD_PAGE_HDR *sp = NULL; + bool usa_error; + + if (!read_log_page(log, page_size, &sp, &usa_error) && + sp->rhdr.sign == NTFS_CHKD_SIGNATURE) { + use_second_page = false; + } + ntfs_free(sp); + } + + if (use_second_page) { + ntfs_free(rst_info.r_page); + memcpy(&rst_info, &rst_info2, sizeof(struct restart_info)); + rst_info2.r_page = NULL; + } + +use_first_page: + ntfs_free(rst_info2.r_page); + +check_restart_area: + /* If the restart area is at offset 0, we want to write the second restart area first */ + log->init_ra = !!rst_info.vbo; + + /* If we have a valid page then grab a pointer to the restart area */ + ra2 = rst_info.valid_page + ? Add2Ptr(rst_info.r_page, + le16_to_cpu(rst_info.r_page->ra_off)) + : NULL; + + if (rst_info.chkdsk_was_run || + (ra2 && ra2->client_idx[1] == LFS_NO_CLIENT_LE)) { + bool wrapped = false; + bool use_multi_page = false; + u32 open_log_count; + + /* Do some checks based on whether we have a valid log page */ + if (!rst_info.valid_page) { + open_log_count = get_random_int(); + goto init_log_instance; + } + open_log_count = le32_to_cpu(ra2->open_log_count); + + /* + * If the restart page size isn't changing then we want to + * check how much work we need to do + */ + if (page_size != le32_to_cpu(rst_info.r_page->sys_page_size)) + goto init_log_instance; + +init_log_instance: + log_init_pg_hdr(log, page_size, page_size, 1, 1); + + log_create(log, l_size, rst_info.last_lsn, open_log_count, + wrapped, use_multi_page); + + ra = log_create_ra(log); + if (!ra) { + err = -ENOMEM; + goto out; + } + log->ra = ra; + + /* Put the restart areas and initialize the log file as required */ + goto process_log; + } + + if (!ra2) { + err = -EINVAL; + goto out; + } + + /* + * If the log page or the system page sizes have changed, we can't use the log file + * We must use the system page size instead of the default size + * if there is not a clean shutdown + */ + t32 = le32_to_cpu(rst_info.r_page->sys_page_size); + if (page_size != t32) { + l_size = orig_file_size; + page_size = + norm_file_page(t32, &l_size, t32 == DefaultLogPageSize); + } + + if (page_size != t32 || + page_size != le32_to_cpu(rst_info.r_page->page_size)) { + err = -EINVAL; + goto out; + } + + /* If the file size has shrunk then we won't mount it */ + if (l_size < le64_to_cpu(ra2->l_size)) { + err = -EINVAL; + goto out; + } + + log_init_pg_hdr(log, page_size, page_size, + le16_to_cpu(rst_info.r_page->major_ver), + le16_to_cpu(rst_info.r_page->minor_ver)); + + log->l_size = le64_to_cpu(ra2->l_size); + log->seq_num_bits = le32_to_cpu(ra2->seq_num_bits); + log->file_data_bits = sizeof(u64) * 8 - log->seq_num_bits; + log->seq_num_mask = (8 << log->file_data_bits) - 1; + log->last_lsn = le64_to_cpu(ra2->current_lsn); + log->seq_num = log->last_lsn >> log->file_data_bits; + log->ra_off = le16_to_cpu(rst_info.r_page->ra_off); + log->restart_size = log->sys_page_size - log->ra_off; + log->record_header_len = le16_to_cpu(ra2->rec_hdr_len); + log->ra_size = le16_to_cpu(ra2->ra_len); + log->data_off = le16_to_cpu(ra2->data_off); + log->data_size = log->page_size - log->data_off; + log->reserved = log->data_size - log->record_header_len; + + vbo = lsn_to_vbo(log, log->last_lsn); + + if (vbo < log->first_page) { + /* This is a pseudo lsn */ + log->l_flags |= NTFSLOG_NO_LAST_LSN; + log->next_page = log->first_page; + goto find_oldest; + } + + /* Find the end of this log record */ + off = final_log_off(log, log->last_lsn, + le32_to_cpu(ra2->last_lsn_data_len)); + + /* If we wrapped the file then increment the sequence number */ + if (off <= vbo) { + log->seq_num += 1; + log->l_flags |= NTFSLOG_WRAPPED; + } + + /* Now compute the next log page to use */ + vbo &= ~log->sys_page_mask; + tail = log->page_size - (off & log->page_mask) - 1; + + /* If we can fit another log record on the page, move back a page the log file */ + if (tail >= log->record_header_len) { + log->l_flags |= NTFSLOG_REUSE_TAIL; + log->next_page = vbo; + } else { + log->next_page = next_page_off(log, vbo); + } + +find_oldest: + /* Find the oldest client lsn. Use the last flushed lsn as a starting point */ + log->oldest_lsn = log->last_lsn; + oldest_client_lsn(Add2Ptr(ra2, le16_to_cpu(ra2->client_off)), + ra2->client_idx[1], &log->oldest_lsn); + log->oldest_lsn_off = lsn_to_vbo(log, log->oldest_lsn); + + if (log->oldest_lsn_off < log->first_page) + log->l_flags |= NTFSLOG_NO_OLDEST_LSN; + + if (!(ra2->flags & RESTART_SINGLE_PAGE_IO)) + log->l_flags |= NTFSLOG_WRAPPED | NTFSLOG_MULTIPLE_PAGE_IO; + + log->current_openlog_count = le32_to_cpu(ra2->open_log_count); + log->total_avail_pages = log->l_size - log->first_page; + log->total_avail = log->total_avail_pages >> log->page_bits; + log->max_current_avail = log->total_avail * log->reserved; + log->total_avail = log->total_avail * log->data_size; + + log->current_avail = current_log_avail(log); + + ra = ntfs_zalloc(log->restart_size); + if (!ra) { + err = -ENOMEM; + goto out; + } + log->ra = ra; + + t16 = le16_to_cpu(ra2->client_off); + if (t16 == offsetof(struct RESTART_AREA, clients)) { + memcpy(ra, ra2, log->ra_size); + } else { + memcpy(ra, ra2, offsetof(struct RESTART_AREA, clients)); + memcpy(ra->clients, Add2Ptr(ra2, t16), + le16_to_cpu(ra2->ra_len) - t16); + + log->current_openlog_count = get_random_int(); + ra->open_log_count = cpu_to_le32(log->current_openlog_count); + log->ra_size = offsetof(struct RESTART_AREA, clients) + + sizeof(struct CLIENT_REC); + ra->client_off = + cpu_to_le16(offsetof(struct RESTART_AREA, clients)); + ra->ra_len = cpu_to_le16(log->ra_size); + } + + le32_add_cpu(&ra->open_log_count, 1); + + /* Now we need to walk through looking for the last lsn */ + err = last_log_lsn(log); + if (err) + goto out; + + log->current_avail = current_log_avail(log); + + /* Remember which restart area to write first */ + log->init_ra = rst_info.vbo; + +process_log: + /* 1.0, 1.1, 2.0 log->major_ver/minor_ver - short values */ + switch ((log->major_ver << 16) + log->minor_ver) { + case 0x10000: + case 0x10001: + case 0x20000: + break; + default: + ntfs_warn(sbi->sb, "\x24LogFile version %d.%d is not supported", + log->major_ver, log->minor_ver); + err = -EOPNOTSUPP; + log->set_dirty = true; + goto out; + } + + /* One client "NTFS" per logfile */ + ca = Add2Ptr(ra, le16_to_cpu(ra->client_off)); + + for (client = ra->client_idx[1];; client = cr->next_client) { + if (client == LFS_NO_CLIENT_LE) { + /* Insert "NTFS" client LogFile */ + client = ra->client_idx[0]; + if (client == LFS_NO_CLIENT_LE) + return -EINVAL; + + t16 = le16_to_cpu(client); + cr = ca + t16; + + remove_client(ca, cr, &ra->client_idx[0]); + + cr->restart_lsn = 0; + cr->oldest_lsn = cpu_to_le64(log->oldest_lsn); + cr->name_bytes = cpu_to_le32(8); + cr->name[0] = cpu_to_le16('N'); + cr->name[1] = cpu_to_le16('T'); + cr->name[2] = cpu_to_le16('F'); + cr->name[3] = cpu_to_le16('S'); + + add_client(ca, t16, &ra->client_idx[1]); + break; + } + + cr = ca + le16_to_cpu(client); + + if (cpu_to_le32(8) == cr->name_bytes && + cpu_to_le16('N') == cr->name[0] && + cpu_to_le16('T') == cr->name[1] && + cpu_to_le16('F') == cr->name[2] && + cpu_to_le16('S') == cr->name[3]) + break; + } + + /* Update the client handle with the client block information */ + log->client_id.seq_num = cr->seq_num; + log->client_id.client_idx = client; + + err = read_rst_area(log, &rst, &ra_lsn); + if (err) + goto out; + + if (!rst) + goto out; + + bytes_per_attr_entry = !rst->major_ver ? 0x2C : 0x28; + + checkpt_lsn = le64_to_cpu(rst->check_point_start); + if (!checkpt_lsn) + checkpt_lsn = ra_lsn; + + /* Allocate and Read the Transaction Table */ + if (!rst->transact_table_len) + goto check_dirty_page_table; + + t64 = le64_to_cpu(rst->transact_table_lsn); + err = read_log_rec_lcb(log, t64, lcb_ctx_prev, &lcb); + if (err) + goto out; + + lrh = lcb->log_rec; + frh = lcb->lrh; + rec_len = le32_to_cpu(frh->client_data_len); + + if (!check_log_rec(lrh, rec_len, le32_to_cpu(frh->transact_id), + bytes_per_attr_entry)) { + err = -EINVAL; + goto out; + } + + t16 = le16_to_cpu(lrh->redo_off); + + rt = Add2Ptr(lrh, t16); + t32 = rec_len - t16; + + /* Now check that this is a valid restart table */ + if (!check_rstbl(rt, t32)) { + err = -EINVAL; + goto out; + } + + trtbl = ntfs_memdup(rt, t32); + if (!trtbl) { + err = -ENOMEM; + goto out; + } + + lcb_put(lcb); + lcb = NULL; + +check_dirty_page_table: + /* The next record back should be the Dirty Pages Table */ + if (!rst->dirty_pages_len) + goto check_attribute_names; + + t64 = le64_to_cpu(rst->dirty_pages_table_lsn); + err = read_log_rec_lcb(log, t64, lcb_ctx_prev, &lcb); + if (err) + goto out; + + lrh = lcb->log_rec; + frh = lcb->lrh; + rec_len = le32_to_cpu(frh->client_data_len); + + if (!check_log_rec(lrh, rec_len, le32_to_cpu(frh->transact_id), + bytes_per_attr_entry)) { + err = -EINVAL; + goto out; + } + + t16 = le16_to_cpu(lrh->redo_off); + + rt = Add2Ptr(lrh, t16); + t32 = rec_len - t16; + + /* Now check that this is a valid restart table */ + if (!check_rstbl(rt, t32)) { + err = -EINVAL; + goto out; + } + + dptbl = ntfs_memdup(rt, t32); + if (!dptbl) { + err = -ENOMEM; + goto out; + } + + /* Convert Ra version '0' into version '1' */ + if (rst->major_ver) + goto end_conv_1; + + dp = NULL; + while ((dp = enum_rstbl(dptbl, dp))) { + struct DIR_PAGE_ENTRY_32 *dp0 = (struct DIR_PAGE_ENTRY_32 *)dp; + // NOTE: Danger. Check for of boundary + memmove(&dp->vcn, &dp0->vcn_low, + 2 * sizeof(u64) + + le32_to_cpu(dp->lcns_follow) * sizeof(u64)); + } + +end_conv_1: + lcb_put(lcb); + lcb = NULL; + + /* Go through the table and remove the duplicates, remembering the oldest lsn values */ + if (sbi->cluster_size <= log->page_size) + goto trace_dp_table; + + dp = NULL; + while ((dp = enum_rstbl(dptbl, dp))) { + struct DIR_PAGE_ENTRY *next = dp; + + while ((next = enum_rstbl(dptbl, next))) { + if (next->target_attr == dp->target_attr && + next->vcn == dp->vcn) { + if (le64_to_cpu(next->oldest_lsn) < + le64_to_cpu(dp->oldest_lsn)) { + dp->oldest_lsn = next->oldest_lsn; + } + + free_rsttbl_idx(dptbl, PtrOffset(dptbl, next)); + } + } + } +trace_dp_table: +check_attribute_names: + /* The next record should be the Attribute Names */ + if (!rst->attr_names_len) + goto check_attr_table; + + t64 = le64_to_cpu(rst->attr_names_lsn); + err = read_log_rec_lcb(log, t64, lcb_ctx_prev, &lcb); + if (err) + goto out; + + lrh = lcb->log_rec; + frh = lcb->lrh; + rec_len = le32_to_cpu(frh->client_data_len); + + if (!check_log_rec(lrh, rec_len, le32_to_cpu(frh->transact_id), + bytes_per_attr_entry)) { + err = -EINVAL; + goto out; + } + + t32 = lrh_length(lrh); + rec_len -= t32; + + attr_names = ntfs_memdup(Add2Ptr(lrh, t32), rec_len); + + lcb_put(lcb); + lcb = NULL; + +check_attr_table: + /* The next record should be the attribute Table */ + if (!rst->open_attr_len) + goto check_attribute_names2; + + t64 = le64_to_cpu(rst->open_attr_table_lsn); + err = read_log_rec_lcb(log, t64, lcb_ctx_prev, &lcb); + if (err) + goto out; + + lrh = lcb->log_rec; + frh = lcb->lrh; + rec_len = le32_to_cpu(frh->client_data_len); + + if (!check_log_rec(lrh, rec_len, le32_to_cpu(frh->transact_id), + bytes_per_attr_entry)) { + err = -EINVAL; + goto out; + } + + t16 = le16_to_cpu(lrh->redo_off); + + rt = Add2Ptr(lrh, t16); + t32 = rec_len - t16; + + if (!check_rstbl(rt, t32)) { + err = -EINVAL; + goto out; + } + + oatbl = ntfs_memdup(rt, t32); + if (!oatbl) { + err = -ENOMEM; + goto out; + } + + log->open_attr_tbl = oatbl; + + /* Clear all of the Attr pointers */ + oe = NULL; + while ((oe = enum_rstbl(oatbl, oe))) { + if (!rst->major_ver) { + struct OPEN_ATTR_ENRTY_32 oe0; + + /* Really 'oe' points to OPEN_ATTR_ENRTY_32 */ + memcpy(&oe0, oe, SIZEOF_OPENATTRIBUTEENTRY0); + + oe->bytes_per_index = oe0.bytes_per_index; + oe->type = oe0.type; + oe->is_dirty_pages = oe0.is_dirty_pages; + oe->name_len = 0; + oe->ref = oe0.ref; + oe->open_record_lsn = oe0.open_record_lsn; + } + + oe->is_attr_name = 0; + oe->ptr = NULL; + } + + lcb_put(lcb); + lcb = NULL; + +check_attribute_names2: + if (!rst->attr_names_len) + goto trace_attribute_table; + + ane = attr_names; + if (!oatbl) + goto trace_attribute_table; + while (ane->off) { + /* TODO: Clear table on exit! */ + oe = Add2Ptr(oatbl, le16_to_cpu(ane->off)); + t16 = le16_to_cpu(ane->name_bytes); + oe->name_len = t16 / sizeof(short); + oe->ptr = ane->name; + oe->is_attr_name = 2; + ane = Add2Ptr(ane, sizeof(struct ATTR_NAME_ENTRY) + t16); + } + +trace_attribute_table: + /* + * If the checkpt_lsn is zero, then this is a freshly + * formatted disk and we have no work to do + */ + if (!checkpt_lsn) { + err = 0; + goto out; + } + + if (!oatbl) { + oatbl = init_rsttbl(bytes_per_attr_entry, 8); + if (!oatbl) { + err = -ENOMEM; + goto out; + } + } + + log->open_attr_tbl = oatbl; + + /* Start the analysis pass from the Checkpoint lsn. */ + rec_lsn = checkpt_lsn; + + /* Read the first lsn */ + err = read_log_rec_lcb(log, checkpt_lsn, lcb_ctx_next, &lcb); + if (err) + goto out; + + /* Loop to read all subsequent records to the end of the log file */ +next_log_record_analyze: + err = read_next_log_rec(log, lcb, &rec_lsn); + if (err) + goto out; + + if (!rec_lsn) + goto end_log_records_enumerate; + + frh = lcb->lrh; + transact_id = le32_to_cpu(frh->transact_id); + rec_len = le32_to_cpu(frh->client_data_len); + lrh = lcb->log_rec; + + if (!check_log_rec(lrh, rec_len, transact_id, bytes_per_attr_entry)) { + err = -EINVAL; + goto out; + } + + /* + * The first lsn after the previous lsn remembered + * the checkpoint is the first candidate for the rlsn + */ + if (!rlsn) + rlsn = rec_lsn; + + if (LfsClientRecord != frh->record_type) + goto next_log_record_analyze; + + /* + * Now update the Transaction Table for this transaction + * If there is no entry present or it is unallocated we allocate the entry + */ + if (!trtbl) { + trtbl = init_rsttbl(sizeof(struct TRANSACTION_ENTRY), + INITIAL_NUMBER_TRANSACTIONS); + if (!trtbl) { + err = -ENOMEM; + goto out; + } + } + + tr = Add2Ptr(trtbl, transact_id); + + if (transact_id >= bytes_per_rt(trtbl) || + tr->next != RESTART_ENTRY_ALLOCATED_LE) { + tr = alloc_rsttbl_from_idx(&trtbl, transact_id); + if (!tr) { + err = -ENOMEM; + goto out; + } + tr->transact_state = TransactionActive; + tr->first_lsn = cpu_to_le64(rec_lsn); + } + + tr->prev_lsn = tr->undo_next_lsn = cpu_to_le64(rec_lsn); + + /* + * If this is a compensation log record, then change + * the undo_next_lsn to be the undo_next_lsn of this record + */ + if (lrh->undo_op == cpu_to_le16(CompensationLogRecord)) + tr->undo_next_lsn = frh->client_undo_next_lsn; + + /* Dispatch to handle log record depending on type */ + switch (le16_to_cpu(lrh->redo_op)) { + case InitializeFileRecordSegment: + case DeallocateFileRecordSegment: + case WriteEndOfFileRecordSegment: + case CreateAttribute: + case DeleteAttribute: + case UpdateResidentValue: + case UpdateNonresidentValue: + case UpdateMappingPairs: + case SetNewAttributeSizes: + case AddIndexEntryRoot: + case DeleteIndexEntryRoot: + case AddIndexEntryAllocation: + case DeleteIndexEntryAllocation: + case WriteEndOfIndexBuffer: + case SetIndexEntryVcnRoot: + case SetIndexEntryVcnAllocation: + case UpdateFileNameRoot: + case UpdateFileNameAllocation: + case SetBitsInNonresidentBitMap: + case ClearBitsInNonresidentBitMap: + case UpdateRecordDataRoot: + case UpdateRecordDataAllocation: + case ZeroEndOfFileRecord: + t16 = le16_to_cpu(lrh->target_attr); + t64 = le64_to_cpu(lrh->target_vcn); + dp = find_dp(dptbl, t16, t64); + + if (dp) + goto copy_lcns; + + /* + * Calculate the number of clusters per page the system + * which wrote the checkpoint, possibly creating the table + */ + if (dptbl) { + t32 = (le16_to_cpu(dptbl->size) - + sizeof(struct DIR_PAGE_ENTRY)) / + sizeof(u64); + } else { + t32 = log->clst_per_page; + ntfs_free(dptbl); + dptbl = init_rsttbl(struct_size(dp, page_lcns, t32), + 32); + if (!dptbl) { + err = -ENOMEM; + goto out; + } + } + + dp = alloc_rsttbl_idx(&dptbl); + dp->target_attr = cpu_to_le32(t16); + dp->transfer_len = cpu_to_le32(t32 << sbi->cluster_bits); + dp->lcns_follow = cpu_to_le32(t32); + dp->vcn = cpu_to_le64(t64 & ~((u64)t32 - 1)); + dp->oldest_lsn = cpu_to_le64(rec_lsn); + +copy_lcns: + /* + * Copy the Lcns from the log record into the Dirty Page Entry + * TODO: for different page size support, must somehow make + * whole routine a loop, case Lcns do not fit below + */ + t16 = le16_to_cpu(lrh->lcns_follow); + for (i = 0; i < t16; i++) { + size_t j = (size_t)(le64_to_cpu(lrh->target_vcn) - + le64_to_cpu(dp->vcn)); + dp->page_lcns[j + i] = lrh->page_lcns[i]; + } + + goto next_log_record_analyze; + + case DeleteDirtyClusters: { + u32 range_count = + le16_to_cpu(lrh->redo_len) / sizeof(struct LCN_RANGE); + const struct LCN_RANGE *r = + Add2Ptr(lrh, le16_to_cpu(lrh->redo_off)); + + /* Loop through all of the Lcn ranges this log record */ + for (i = 0; i < range_count; i++, r++) { + u64 lcn0 = le64_to_cpu(r->lcn); + u64 lcn_e = lcn0 + le64_to_cpu(r->len) - 1; + + dp = NULL; + while ((dp = enum_rstbl(dptbl, dp))) { + u32 j; + + t32 = le32_to_cpu(dp->lcns_follow); + for (j = 0; j < t32; j++) { + t64 = le64_to_cpu(dp->page_lcns[j]); + if (t64 >= lcn0 && t64 <= lcn_e) + dp->page_lcns[j] = 0; + } + } + } + goto next_log_record_analyze; + ; + } + + case OpenNonresidentAttribute: + t16 = le16_to_cpu(lrh->target_attr); + if (t16 >= bytes_per_rt(oatbl)) { + /* + * Compute how big the table needs to be. + * Add 10 extra entries for some cushion + */ + u32 new_e = t16 / le16_to_cpu(oatbl->size); + + new_e += 10 - le16_to_cpu(oatbl->used); + + oatbl = extend_rsttbl(oatbl, new_e, ~0u); + log->open_attr_tbl = oatbl; + if (!oatbl) { + err = -ENOMEM; + goto out; + } + } + + /* Point to the entry being opened */ + oe = alloc_rsttbl_from_idx(&oatbl, t16); + log->open_attr_tbl = oatbl; + if (!oe) { + err = -ENOMEM; + goto out; + } + + /* Initialize this entry from the log record */ + t16 = le16_to_cpu(lrh->redo_off); + if (!rst->major_ver) { + /* Convert version '0' into version '1' */ + struct OPEN_ATTR_ENRTY_32 *oe0 = Add2Ptr(lrh, t16); + + oe->bytes_per_index = oe0->bytes_per_index; + oe->type = oe0->type; + oe->is_dirty_pages = oe0->is_dirty_pages; + oe->name_len = 0; //oe0.name_len; + oe->ref = oe0->ref; + oe->open_record_lsn = oe0->open_record_lsn; + } else { + memcpy(oe, Add2Ptr(lrh, t16), bytes_per_attr_entry); + } + + t16 = le16_to_cpu(lrh->undo_len); + if (t16) { + oe->ptr = ntfs_malloc(t16); + if (!oe->ptr) { + err = -ENOMEM; + goto out; + } + oe->name_len = t16 / sizeof(short); + memcpy(oe->ptr, + Add2Ptr(lrh, le16_to_cpu(lrh->undo_off)), t16); + oe->is_attr_name = 1; + } else { + oe->ptr = NULL; + oe->is_attr_name = 0; + } + + goto next_log_record_analyze; + + case HotFix: + t16 = le16_to_cpu(lrh->target_attr); + t64 = le64_to_cpu(lrh->target_vcn); + dp = find_dp(dptbl, t16, t64); + if (dp) { + size_t j = le64_to_cpu(lrh->target_vcn) - + le64_to_cpu(dp->vcn); + if (dp->page_lcns[j]) + dp->page_lcns[j] = lrh->page_lcns[0]; + } + goto next_log_record_analyze; + + case EndTopLevelAction: + tr = Add2Ptr(trtbl, transact_id); + tr->prev_lsn = cpu_to_le64(rec_lsn); + tr->undo_next_lsn = frh->client_undo_next_lsn; + goto next_log_record_analyze; + + case PrepareTransaction: + tr = Add2Ptr(trtbl, transact_id); + tr->transact_state = TransactionPrepared; + goto next_log_record_analyze; + + case CommitTransaction: + tr = Add2Ptr(trtbl, transact_id); + tr->transact_state = TransactionCommitted; + goto next_log_record_analyze; + + case ForgetTransaction: + free_rsttbl_idx(trtbl, transact_id); + goto next_log_record_analyze; + + case Noop: + case OpenAttributeTableDump: + case AttributeNamesDump: + case DirtyPageTableDump: + case TransactionTableDump: + /* The following cases require no action the Analysis Pass */ + goto next_log_record_analyze; + + default: + /* + * All codes will be explicitly handled. + * If we see a code we do not expect, then we are trouble + */ + goto next_log_record_analyze; + } + +end_log_records_enumerate: + lcb_put(lcb); + lcb = NULL; + + /* + * Scan the Dirty Page Table and Transaction Table for + * the lowest lsn, and return it as the Redo lsn + */ + dp = NULL; + while ((dp = enum_rstbl(dptbl, dp))) { + t64 = le64_to_cpu(dp->oldest_lsn); + if (t64 && t64 < rlsn) + rlsn = t64; + } + + tr = NULL; + while ((tr = enum_rstbl(trtbl, tr))) { + t64 = le64_to_cpu(tr->first_lsn); + if (t64 && t64 < rlsn) + rlsn = t64; + } + + /* Only proceed if the Dirty Page Table or Transaction table are not empty */ + if ((!dptbl || !dptbl->total) && (!trtbl || !trtbl->total)) + goto end_reply; + + sbi->flags |= NTFS_FLAGS_NEED_REPLAY; + if (is_ro) + goto out; + + /* Reopen all of the attributes with dirty pages */ + oe = NULL; +next_open_attribute: + + oe = enum_rstbl(oatbl, oe); + if (!oe) { + err = 0; + dp = NULL; + goto next_dirty_page; + } + + oa = ntfs_zalloc(sizeof(struct OpenAttr)); + if (!oa) { + err = -ENOMEM; + goto out; + } + + inode = ntfs_iget5(sbi->sb, &oe->ref, NULL); + if (IS_ERR(inode)) + goto fake_attr; + + if (is_bad_inode(inode)) { + iput(inode); +fake_attr: + if (oa->ni) { + iput(&oa->ni->vfs_inode); + oa->ni = NULL; + } + + attr = attr_create_nonres_log(sbi, oe->type, 0, oe->ptr, + oe->name_len, 0); + if (!attr) { + ntfs_free(oa); + err = -ENOMEM; + goto out; + } + oa->attr = attr; + oa->run1 = &oa->run0; + goto final_oe; + } + + ni_oe = ntfs_i(inode); + oa->ni = ni_oe; + + attr = ni_find_attr(ni_oe, NULL, NULL, oe->type, oe->ptr, oe->name_len, + NULL, NULL); + + if (!attr) + goto fake_attr; + + t32 = le32_to_cpu(attr->size); + oa->attr = ntfs_memdup(attr, t32); + if (!oa->attr) + goto fake_attr; + + if (!S_ISDIR(inode->i_mode)) { + if (attr->type == ATTR_DATA && !attr->name_len) { + oa->run1 = &ni_oe->file.run; + goto final_oe; + } + } else { + if (attr->type == ATTR_ALLOC && + attr->name_len == ARRAY_SIZE(I30_NAME) && + !memcmp(attr_name(attr), I30_NAME, sizeof(I30_NAME))) { + oa->run1 = &ni_oe->dir.alloc_run; + goto final_oe; + } + } + + if (attr->non_res) { + u16 roff = le16_to_cpu(attr->nres.run_off); + CLST svcn = le64_to_cpu(attr->nres.svcn); + + err = run_unpack(&oa->run0, sbi, inode->i_ino, svcn, + le64_to_cpu(attr->nres.evcn), svcn, + Add2Ptr(attr, roff), t32 - roff); + if (err < 0) { + ntfs_free(oa->attr); + oa->attr = NULL; + goto fake_attr; + } + err = 0; + } + oa->run1 = &oa->run0; + attr = oa->attr; + +final_oe: + if (oe->is_attr_name == 1) + ntfs_free(oe->ptr); + oe->is_attr_name = 0; + oe->ptr = oa; + oe->name_len = attr->name_len; + + goto next_open_attribute; + + /* + * Now loop through the dirty page table to extract all of the Vcn/Lcn + * Mapping that we have, and insert it into the appropriate run + */ +next_dirty_page: + dp = enum_rstbl(dptbl, dp); + if (!dp) + goto do_redo_1; + + oe = Add2Ptr(oatbl, le32_to_cpu(dp->target_attr)); + + if (oe->next != RESTART_ENTRY_ALLOCATED_LE) + goto next_dirty_page; + + oa = oe->ptr; + if (!oa) + goto next_dirty_page; + + i = -1; +next_dirty_page_vcn: + i += 1; + if (i >= le32_to_cpu(dp->lcns_follow)) + goto next_dirty_page; + + vcn = le64_to_cpu(dp->vcn) + i; + size = (vcn + 1) << sbi->cluster_bits; + + if (!dp->page_lcns[i]) + goto next_dirty_page_vcn; + + rno = ino_get(&oe->ref); + if (rno <= MFT_REC_MIRR && + size < (MFT_REC_VOL + 1) * sbi->record_size && + oe->type == ATTR_DATA) { + goto next_dirty_page_vcn; + } + + lcn = le64_to_cpu(dp->page_lcns[i]); + + if ((!run_lookup_entry(oa->run1, vcn, &lcn0, &len0, NULL) || + lcn0 != lcn) && + !run_add_entry(oa->run1, vcn, lcn, 1, false)) { + err = -ENOMEM; + goto out; + } + attr = oa->attr; + t64 = le64_to_cpu(attr->nres.alloc_size); + if (size > t64) { + attr->nres.valid_size = attr->nres.data_size = + attr->nres.alloc_size = cpu_to_le64(size); + } + goto next_dirty_page_vcn; + +do_redo_1: + /* + * Perform the Redo Pass, to restore all of the dirty pages to the same + * contents that they had immediately before the crash + * If the dirty page table is empty, then we can skip the entire Redo Pass + */ + if (!dptbl || !dptbl->total) + goto do_undo_action; + + rec_lsn = rlsn; + + /* + * Read the record at the Redo lsn, before falling + * into common code to handle each record + */ + err = read_log_rec_lcb(log, rlsn, lcb_ctx_next, &lcb); + if (err) + goto out; + + /* + * Now loop to read all of our log records forwards, + * until we hit the end of the file, cleaning up at the end + */ +do_action_next: + frh = lcb->lrh; + + if (LfsClientRecord != frh->record_type) + goto read_next_log_do_action; + + transact_id = le32_to_cpu(frh->transact_id); + rec_len = le32_to_cpu(frh->client_data_len); + lrh = lcb->log_rec; + + if (!check_log_rec(lrh, rec_len, transact_id, bytes_per_attr_entry)) { + err = -EINVAL; + goto out; + } + + /* Ignore log records that do not update pages */ + if (lrh->lcns_follow) + goto find_dirty_page; + + goto read_next_log_do_action; + +find_dirty_page: + t16 = le16_to_cpu(lrh->target_attr); + t64 = le64_to_cpu(lrh->target_vcn); + dp = find_dp(dptbl, t16, t64); + + if (!dp) + goto read_next_log_do_action; + + if (rec_lsn < le64_to_cpu(dp->oldest_lsn)) + goto read_next_log_do_action; + + t16 = le16_to_cpu(lrh->target_attr); + if (t16 >= bytes_per_rt(oatbl)) { + err = -EINVAL; + goto out; + } + + oe = Add2Ptr(oatbl, t16); + + if (oe->next != RESTART_ENTRY_ALLOCATED_LE) { + err = -EINVAL; + goto out; + } + + oa = oe->ptr; + + if (!oa) { + err = -EINVAL; + goto out; + } + attr = oa->attr; + + vcn = le64_to_cpu(lrh->target_vcn); + + if (!run_lookup_entry(oa->run1, vcn, &lcn, NULL, NULL) || + lcn == SPARSE_LCN) { + goto read_next_log_do_action; + } + + /* Point to the Redo data and get its length */ + data = Add2Ptr(lrh, le16_to_cpu(lrh->redo_off)); + dlen = le16_to_cpu(lrh->redo_len); + + /* Shorten length by any Lcns which were deleted */ + saved_len = dlen; + + for (i = le16_to_cpu(lrh->lcns_follow); i; i--) { + size_t j; + u32 alen, voff; + + voff = le16_to_cpu(lrh->record_off) + + le16_to_cpu(lrh->attr_off); + voff += le16_to_cpu(lrh->cluster_off) << SECTOR_SHIFT; + + /* If the Vcn question is allocated, we can just get out.*/ + j = le64_to_cpu(lrh->target_vcn) - le64_to_cpu(dp->vcn); + if (dp->page_lcns[j + i - 1]) + break; + + if (!saved_len) + saved_len = 1; + + /* + * Calculate the allocated space left relative to the + * log record Vcn, after removing this unallocated Vcn + */ + alen = (i - 1) << sbi->cluster_bits; + + /* + * If the update described this log record goes beyond + * the allocated space, then we will have to reduce the length + */ + if (voff >= alen) + dlen = 0; + else if (voff + dlen > alen) + dlen = alen - voff; + } + + /* If the resulting dlen from above is now zero, we can skip this log record */ + if (!dlen && saved_len) + goto read_next_log_do_action; + + t16 = le16_to_cpu(lrh->redo_op); + if (can_skip_action(t16)) + goto read_next_log_do_action; + + /* Apply the Redo operation a common routine */ + err = do_action(log, oe, lrh, t16, data, dlen, rec_len, &rec_lsn); + if (err) + goto out; + + /* Keep reading and looping back until end of file */ +read_next_log_do_action: + err = read_next_log_rec(log, lcb, &rec_lsn); + if (!err && rec_lsn) + goto do_action_next; + + lcb_put(lcb); + lcb = NULL; + +do_undo_action: + /* Scan Transaction Table */ + tr = NULL; +transaction_table_next: + tr = enum_rstbl(trtbl, tr); + if (!tr) + goto undo_action_done; + + if (TransactionActive != tr->transact_state || !tr->undo_next_lsn) { + free_rsttbl_idx(trtbl, PtrOffset(trtbl, tr)); + goto transaction_table_next; + } + + log->transaction_id = PtrOffset(trtbl, tr); + undo_next_lsn = le64_to_cpu(tr->undo_next_lsn); + + /* + * We only have to do anything if the transaction has + * something its undo_next_lsn field + */ + if (!undo_next_lsn) + goto commit_undo; + + /* Read the first record to be undone by this transaction */ + err = read_log_rec_lcb(log, undo_next_lsn, lcb_ctx_undo_next, &lcb); + if (err) + goto out; + + /* + * Now loop to read all of our log records forwards, + * until we hit the end of the file, cleaning up at the end + */ +undo_action_next: + + lrh = lcb->log_rec; + frh = lcb->lrh; + transact_id = le32_to_cpu(frh->transact_id); + rec_len = le32_to_cpu(frh->client_data_len); + + if (!check_log_rec(lrh, rec_len, transact_id, bytes_per_attr_entry)) { + err = -EINVAL; + goto out; + } + + if (lrh->undo_op == cpu_to_le16(Noop)) + goto read_next_log_undo_action; + + oe = Add2Ptr(oatbl, le16_to_cpu(lrh->target_attr)); + oa = oe->ptr; + + t16 = le16_to_cpu(lrh->lcns_follow); + if (!t16) + goto add_allocated_vcns; + + is_mapped = run_lookup_entry(oa->run1, le64_to_cpu(lrh->target_vcn), + &lcn, &clen, NULL); + + /* + * If the mapping isn't already the table or the mapping + * corresponds to a hole the mapping, we need to make sure + * there is no partial page already memory + */ + if (is_mapped && lcn != SPARSE_LCN && clen >= t16) + goto add_allocated_vcns; + + vcn = le64_to_cpu(lrh->target_vcn); + vcn &= ~(log->clst_per_page - 1); + +add_allocated_vcns: + for (i = 0, vcn = le64_to_cpu(lrh->target_vcn), + size = (vcn + 1) << sbi->cluster_bits; + i < t16; i++, vcn += 1, size += sbi->cluster_size) { + attr = oa->attr; + if (!attr->non_res) { + if (size > le32_to_cpu(attr->res.data_size)) + attr->res.data_size = cpu_to_le32(size); + } else { + if (size > le64_to_cpu(attr->nres.data_size)) + attr->nres.valid_size = attr->nres.data_size = + attr->nres.alloc_size = + cpu_to_le64(size); + } + } + + t16 = le16_to_cpu(lrh->undo_op); + if (can_skip_action(t16)) + goto read_next_log_undo_action; + + /* Point to the Redo data and get its length */ + data = Add2Ptr(lrh, le16_to_cpu(lrh->undo_off)); + dlen = le16_to_cpu(lrh->undo_len); + + /* it is time to apply the undo action */ + err = do_action(log, oe, lrh, t16, data, dlen, rec_len, NULL); + +read_next_log_undo_action: + /* + * Keep reading and looping back until we have read the + * last record for this transaction + */ + err = read_next_log_rec(log, lcb, &rec_lsn); + if (err) + goto out; + + if (rec_lsn) + goto undo_action_next; + + lcb_put(lcb); + lcb = NULL; + +commit_undo: + free_rsttbl_idx(trtbl, log->transaction_id); + + log->transaction_id = 0; + + goto transaction_table_next; + +undo_action_done: + + ntfs_update_mftmirr(sbi, 0); + + sbi->flags &= ~NTFS_FLAGS_NEED_REPLAY; + +end_reply: + + err = 0; + if (is_ro) + goto out; + + rh = ntfs_zalloc(log->page_size); + if (!rh) { + err = -ENOMEM; + goto out; + } + + rh->rhdr.sign = NTFS_RSTR_SIGNATURE; + rh->rhdr.fix_off = cpu_to_le16(offsetof(struct RESTART_HDR, fixups)); + t16 = (log->page_size >> SECTOR_SHIFT) + 1; + rh->rhdr.fix_num = cpu_to_le16(t16); + rh->sys_page_size = cpu_to_le32(log->page_size); + rh->page_size = cpu_to_le32(log->page_size); + + t16 = QuadAlign(offsetof(struct RESTART_HDR, fixups) + + sizeof(short) * t16); + rh->ra_off = cpu_to_le16(t16); + rh->minor_ver = cpu_to_le16(1); // 0x1A: + rh->major_ver = cpu_to_le16(1); // 0x1C: + + ra2 = Add2Ptr(rh, t16); + memcpy(ra2, ra, sizeof(struct RESTART_AREA)); + + ra2->client_idx[0] = 0; + ra2->client_idx[1] = LFS_NO_CLIENT_LE; + ra2->flags = cpu_to_le16(2); + + le32_add_cpu(&ra2->open_log_count, 1); + + ntfs_fix_pre_write(&rh->rhdr, log->page_size); + + err = ntfs_sb_write_run(sbi, &ni->file.run, 0, rh, log->page_size); + if (!err) + err = ntfs_sb_write_run(sbi, &log->ni->file.run, log->page_size, + rh, log->page_size); + + ntfs_free(rh); + if (err) + goto out; + +out: + ntfs_free(rst); + if (lcb) + lcb_put(lcb); + + /* Scan the Open Attribute Table to close all of the open attributes */ + oe = NULL; + while ((oe = enum_rstbl(oatbl, oe))) { + rno = ino_get(&oe->ref); + + if (oe->is_attr_name == 1) { + ntfs_free(oe->ptr); + oe->ptr = NULL; + continue; + } + + if (oe->is_attr_name) + continue; + + oa = oe->ptr; + if (!oa) + continue; + + run_close(&oa->run0); + ntfs_free(oa->attr); + if (oa->ni) + iput(&oa->ni->vfs_inode); + ntfs_free(oa); + } + + ntfs_free(trtbl); + ntfs_free(oatbl); + ntfs_free(dptbl); + ntfs_free(attr_names); + ntfs_free(rst_info.r_page); + + ntfs_free(ra); + ntfs_free(log->one_page_buf); + + if (err) + sbi->flags |= NTFS_FLAGS_NEED_REPLAY; + + if (err == -EROFS) + err = 0; + else if (log->set_dirty) + ntfs_set_state(sbi, NTFS_DIRTY_ERROR); + + ntfs_free(log); + + return err; +} diff --git a/fs/ntfs3/fsntfs.c b/fs/ntfs3/fsntfs.c new file mode 100644 index 000000000000..327356b08187 --- /dev/null +++ b/fs/ntfs3/fsntfs.c @@ -0,0 +1,2542 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + */ + +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +// clang-format off +const struct cpu_str NAME_MFT = { + 4, 0, { '$', 'M', 'F', 'T' }, +}; +const struct cpu_str NAME_MIRROR = { + 8, 0, { '$', 'M', 'F', 'T', 'M', 'i', 'r', 'r' }, +}; +const struct cpu_str NAME_LOGFILE = { + 8, 0, { '$', 'L', 'o', 'g', 'F', 'i', 'l', 'e' }, +}; +const struct cpu_str NAME_VOLUME = { + 7, 0, { '$', 'V', 'o', 'l', 'u', 'm', 'e' }, +}; +const struct cpu_str NAME_ATTRDEF = { + 8, 0, { '$', 'A', 't', 't', 'r', 'D', 'e', 'f' }, +}; +const struct cpu_str NAME_ROOT = { + 1, 0, { '.' }, +}; +const struct cpu_str NAME_BITMAP = { + 7, 0, { '$', 'B', 'i', 't', 'm', 'a', 'p' }, +}; +const struct cpu_str NAME_BOOT = { + 5, 0, { '$', 'B', 'o', 'o', 't' }, +}; +const struct cpu_str NAME_BADCLUS = { + 8, 0, { '$', 'B', 'a', 'd', 'C', 'l', 'u', 's' }, +}; +const struct cpu_str NAME_QUOTA = { + 6, 0, { '$', 'Q', 'u', 'o', 't', 'a' }, +}; +const struct cpu_str NAME_SECURE = { + 7, 0, { '$', 'S', 'e', 'c', 'u', 'r', 'e' }, +}; +const struct cpu_str NAME_UPCASE = { + 7, 0, { '$', 'U', 'p', 'C', 'a', 's', 'e' }, +}; +const struct cpu_str NAME_EXTEND = { + 7, 0, { '$', 'E', 'x', 't', 'e', 'n', 'd' }, +}; +const struct cpu_str NAME_OBJID = { + 6, 0, { '$', 'O', 'b', 'j', 'I', 'd' }, +}; +const struct cpu_str NAME_REPARSE = { + 8, 0, { '$', 'R', 'e', 'p', 'a', 'r', 's', 'e' }, +}; +const struct cpu_str NAME_USNJRNL = { + 8, 0, { '$', 'U', 's', 'n', 'J', 'r', 'n', 'l' }, +}; +const __le16 BAD_NAME[4] = { + cpu_to_le16('$'), cpu_to_le16('B'), cpu_to_le16('a'), cpu_to_le16('d'), +}; +const __le16 I30_NAME[4] = { + cpu_to_le16('$'), cpu_to_le16('I'), cpu_to_le16('3'), cpu_to_le16('0'), +}; +const __le16 SII_NAME[4] = { + cpu_to_le16('$'), cpu_to_le16('S'), cpu_to_le16('I'), cpu_to_le16('I'), +}; +const __le16 SDH_NAME[4] = { + cpu_to_le16('$'), cpu_to_le16('S'), cpu_to_le16('D'), cpu_to_le16('H'), +}; +const __le16 SDS_NAME[4] = { + cpu_to_le16('$'), cpu_to_le16('S'), cpu_to_le16('D'), cpu_to_le16('S'), +}; +const __le16 SO_NAME[2] = { + cpu_to_le16('$'), cpu_to_le16('O'), +}; +const __le16 SQ_NAME[2] = { + cpu_to_le16('$'), cpu_to_le16('Q'), +}; +const __le16 SR_NAME[2] = { + cpu_to_le16('$'), cpu_to_le16('R'), +}; + +#ifdef CONFIG_NTFS3_LZX_XPRESS +const __le16 WOF_NAME[17] = { + cpu_to_le16('W'), cpu_to_le16('o'), cpu_to_le16('f'), cpu_to_le16('C'), + cpu_to_le16('o'), cpu_to_le16('m'), cpu_to_le16('p'), cpu_to_le16('r'), + cpu_to_le16('e'), cpu_to_le16('s'), cpu_to_le16('s'), cpu_to_le16('e'), + cpu_to_le16('d'), cpu_to_le16('D'), cpu_to_le16('a'), cpu_to_le16('t'), + cpu_to_le16('a'), +}; +#endif + +// clang-format on + +/* + * ntfs_fix_pre_write + * + * inserts fixups into 'rhdr' before writing to disk + */ +bool ntfs_fix_pre_write(struct NTFS_RECORD_HEADER *rhdr, size_t bytes) +{ + u16 *fixup, *ptr; + u16 sample; + u16 fo = le16_to_cpu(rhdr->fix_off); + u16 fn = le16_to_cpu(rhdr->fix_num); + + if ((fo & 1) || fo + fn * sizeof(short) > SECTOR_SIZE || !fn-- || + fn * SECTOR_SIZE > bytes) { + return false; + } + + /* Get fixup pointer */ + fixup = Add2Ptr(rhdr, fo); + + if (*fixup >= 0x7FFF) + *fixup = 1; + else + *fixup += 1; + + sample = *fixup; + + ptr = Add2Ptr(rhdr, SECTOR_SIZE - sizeof(short)); + + while (fn--) { + *++fixup = *ptr; + *ptr = sample; + ptr += SECTOR_SIZE / sizeof(short); + } + return true; +} + +/* + * ntfs_fix_post_read + * + * remove fixups after reading from disk + * Returns < 0 if error, 0 if ok, 1 if need to update fixups + */ +int ntfs_fix_post_read(struct NTFS_RECORD_HEADER *rhdr, size_t bytes, + bool simple) +{ + int ret; + u16 *fixup, *ptr; + u16 sample, fo, fn; + + fo = le16_to_cpu(rhdr->fix_off); + fn = simple ? ((bytes >> SECTOR_SHIFT) + 1) + : le16_to_cpu(rhdr->fix_num); + + /* Check errors */ + if ((fo & 1) || fo + fn * sizeof(short) > SECTOR_SIZE || !fn-- || + fn * SECTOR_SIZE > bytes) { + return -EINVAL; /* native chkntfs returns ok! */ + } + + /* Get fixup pointer */ + fixup = Add2Ptr(rhdr, fo); + sample = *fixup; + ptr = Add2Ptr(rhdr, SECTOR_SIZE - sizeof(short)); + ret = 0; + + while (fn--) { + /* Test current word */ + if (*ptr != sample) { + /* Fixup does not match! Is it serious error? */ + ret = -E_NTFS_FIXUP; + } + + /* Replace fixup */ + *ptr = *++fixup; + ptr += SECTOR_SIZE / sizeof(short); + } + + return ret; +} + +/* + * ntfs_extend_init + * + * loads $Extend file + */ +int ntfs_extend_init(struct ntfs_sb_info *sbi) +{ + int err; + struct super_block *sb = sbi->sb; + struct inode *inode, *inode2; + struct MFT_REF ref; + + if (sbi->volume.major_ver < 3) { + ntfs_notice(sb, "Skip $Extend 'cause NTFS version"); + return 0; + } + + ref.low = cpu_to_le32(MFT_REC_EXTEND); + ref.high = 0; + ref.seq = cpu_to_le16(MFT_REC_EXTEND); + inode = ntfs_iget5(sb, &ref, &NAME_EXTEND); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + ntfs_err(sb, "Failed to load $Extend."); + inode = NULL; + goto out; + } + + /* if ntfs_iget5 reads from disk it never returns bad inode */ + if (!S_ISDIR(inode->i_mode)) { + err = -EINVAL; + goto out; + } + + /* Try to find $ObjId */ + inode2 = dir_search_u(inode, &NAME_OBJID, NULL); + if (inode2 && !IS_ERR(inode2)) { + if (is_bad_inode(inode2)) { + iput(inode2); + } else { + sbi->objid.ni = ntfs_i(inode2); + sbi->objid_no = inode2->i_ino; + } + } + + /* Try to find $Quota */ + inode2 = dir_search_u(inode, &NAME_QUOTA, NULL); + if (inode2 && !IS_ERR(inode2)) { + sbi->quota_no = inode2->i_ino; + iput(inode2); + } + + /* Try to find $Reparse */ + inode2 = dir_search_u(inode, &NAME_REPARSE, NULL); + if (inode2 && !IS_ERR(inode2)) { + sbi->reparse.ni = ntfs_i(inode2); + sbi->reparse_no = inode2->i_ino; + } + + /* Try to find $UsnJrnl */ + inode2 = dir_search_u(inode, &NAME_USNJRNL, NULL); + if (inode2 && !IS_ERR(inode2)) { + sbi->usn_jrnl_no = inode2->i_ino; + iput(inode2); + } + + err = 0; +out: + iput(inode); + return err; +} + +int ntfs_loadlog_and_replay(struct ntfs_inode *ni, struct ntfs_sb_info *sbi) +{ + int err = 0; + struct super_block *sb = sbi->sb; + bool initialized = false; + struct MFT_REF ref; + struct inode *inode; + + /* Check for 4GB */ + if (ni->vfs_inode.i_size >= 0x100000000ull) { + ntfs_err(sb, "\x24LogFile is too big"); + err = -EINVAL; + goto out; + } + + sbi->flags |= NTFS_FLAGS_LOG_REPLAYING; + + ref.low = cpu_to_le32(MFT_REC_MFT); + ref.high = 0; + ref.seq = cpu_to_le16(1); + + inode = ntfs_iget5(sb, &ref, NULL); + + if (IS_ERR(inode)) + inode = NULL; + + if (!inode) { + /* Try to use mft copy */ + u64 t64 = sbi->mft.lbo; + + sbi->mft.lbo = sbi->mft.lbo2; + inode = ntfs_iget5(sb, &ref, NULL); + sbi->mft.lbo = t64; + if (IS_ERR(inode)) + inode = NULL; + } + + if (!inode) { + err = -EINVAL; + ntfs_err(sb, "Failed to load $MFT."); + goto out; + } + + sbi->mft.ni = ntfs_i(inode); + + /* LogFile should not contains attribute list */ + err = ni_load_all_mi(sbi->mft.ni); + if (!err) + err = log_replay(ni, &initialized); + + iput(inode); + sbi->mft.ni = NULL; + + sync_blockdev(sb->s_bdev); + invalidate_bdev(sb->s_bdev); + + if (sbi->flags & NTFS_FLAGS_NEED_REPLAY) { + err = 0; + goto out; + } + + if (sb_rdonly(sb) || !initialized) + goto out; + + /* fill LogFile by '-1' if it is initialized */ + err = ntfs_bio_fill_1(sbi, &ni->file.run); + +out: + sbi->flags &= ~NTFS_FLAGS_LOG_REPLAYING; + + return err; +} + +/* + * ntfs_query_def + * + * returns current ATTR_DEF_ENTRY for given attribute type + */ +const struct ATTR_DEF_ENTRY *ntfs_query_def(struct ntfs_sb_info *sbi, + enum ATTR_TYPE type) +{ + int type_in = le32_to_cpu(type); + size_t min_idx = 0; + size_t max_idx = sbi->def_entries - 1; + + while (min_idx <= max_idx) { + size_t i = min_idx + ((max_idx - min_idx) >> 1); + const struct ATTR_DEF_ENTRY *entry = sbi->def_table + i; + int diff = le32_to_cpu(entry->type) - type_in; + + if (!diff) + return entry; + if (diff < 0) + min_idx = i + 1; + else if (i) + max_idx = i - 1; + else + return NULL; + } + return NULL; +} + +/* + * ntfs_look_for_free_space + * + * looks for a free space in bitmap + */ +int ntfs_look_for_free_space(struct ntfs_sb_info *sbi, CLST lcn, CLST len, + CLST *new_lcn, CLST *new_len, + enum ALLOCATE_OPT opt) +{ + int err; + struct super_block *sb = sbi->sb; + size_t a_lcn, zlen, zeroes, zlcn, zlen2, ztrim, new_zlen; + struct wnd_bitmap *wnd = &sbi->used.bitmap; + + down_write_nested(&wnd->rw_lock, BITMAP_MUTEX_CLUSTERS); + if (opt & ALLOCATE_MFT) { + CLST alen; + + zlen = wnd_zone_len(wnd); + + if (!zlen) { + err = ntfs_refresh_zone(sbi); + if (err) + goto out; + + zlen = wnd_zone_len(wnd); + + if (!zlen) { + ntfs_err(sbi->sb, + "no free space to extend mft"); + err = -ENOSPC; + goto out; + } + } + + lcn = wnd_zone_bit(wnd); + alen = zlen > len ? len : zlen; + + wnd_zone_set(wnd, lcn + alen, zlen - alen); + + err = wnd_set_used(wnd, lcn, alen); + if (err) + goto out; + + *new_lcn = lcn; + *new_len = alen; + goto ok; + } + + /* + * 'Cause cluster 0 is always used this value means that we should use + * cached value of 'next_free_lcn' to improve performance + */ + if (!lcn) + lcn = sbi->used.next_free_lcn; + + if (lcn >= wnd->nbits) + lcn = 0; + + *new_len = wnd_find(wnd, len, lcn, BITMAP_FIND_MARK_AS_USED, &a_lcn); + if (*new_len) { + *new_lcn = a_lcn; + goto ok; + } + + /* Try to use clusters from MftZone */ + zlen = wnd_zone_len(wnd); + zeroes = wnd_zeroes(wnd); + + /* Check too big request */ + if (len > zeroes + zlen) + goto no_space; + + if (zlen <= NTFS_MIN_MFT_ZONE) + goto no_space; + + /* How many clusters to cat from zone */ + zlcn = wnd_zone_bit(wnd); + zlen2 = zlen >> 1; + ztrim = len > zlen ? zlen : (len > zlen2 ? len : zlen2); + new_zlen = zlen - ztrim; + + if (new_zlen < NTFS_MIN_MFT_ZONE) { + new_zlen = NTFS_MIN_MFT_ZONE; + if (new_zlen > zlen) + new_zlen = zlen; + } + + wnd_zone_set(wnd, zlcn, new_zlen); + + /* allocate continues clusters */ + *new_len = + wnd_find(wnd, len, 0, + BITMAP_FIND_MARK_AS_USED | BITMAP_FIND_FULL, &a_lcn); + if (*new_len) { + *new_lcn = a_lcn; + goto ok; + } + +no_space: + up_write(&wnd->rw_lock); + + return -ENOSPC; + +ok: + err = 0; + + ntfs_unmap_meta(sb, *new_lcn, *new_len); + + if (opt & ALLOCATE_MFT) + goto out; + + /* Set hint for next requests */ + sbi->used.next_free_lcn = *new_lcn + *new_len; + +out: + up_write(&wnd->rw_lock); + return err; +} + +/* + * ntfs_extend_mft + * + * allocates additional MFT records + * sbi->mft.bitmap is locked for write + * + * NOTE: recursive: + * ntfs_look_free_mft -> + * ntfs_extend_mft -> + * attr_set_size -> + * ni_insert_nonresident -> + * ni_insert_attr -> + * ni_ins_attr_ext -> + * ntfs_look_free_mft -> + * ntfs_extend_mft + * To avoid recursive always allocate space for two new mft records + * see attrib.c: "at least two mft to avoid recursive loop" + */ +static int ntfs_extend_mft(struct ntfs_sb_info *sbi) +{ + int err; + struct ntfs_inode *ni = sbi->mft.ni; + size_t new_mft_total; + u64 new_mft_bytes, new_bitmap_bytes; + struct ATTRIB *attr; + struct wnd_bitmap *wnd = &sbi->mft.bitmap; + + new_mft_total = (wnd->nbits + MFT_INCREASE_CHUNK + 127) & (CLST)~127; + new_mft_bytes = (u64)new_mft_total << sbi->record_bits; + + /* Step 1: Resize $MFT::DATA */ + down_write(&ni->file.run_lock); + err = attr_set_size(ni, ATTR_DATA, NULL, 0, &ni->file.run, + new_mft_bytes, NULL, false, &attr); + + if (err) { + up_write(&ni->file.run_lock); + goto out; + } + + attr->nres.valid_size = attr->nres.data_size; + new_mft_total = le64_to_cpu(attr->nres.alloc_size) >> sbi->record_bits; + ni->mi.dirty = true; + + /* Step 2: Resize $MFT::BITMAP */ + new_bitmap_bytes = bitmap_size(new_mft_total); + + err = attr_set_size(ni, ATTR_BITMAP, NULL, 0, &sbi->mft.bitmap.run, + new_bitmap_bytes, &new_bitmap_bytes, true, NULL); + + /* Refresh Mft Zone if necessary */ + down_write_nested(&sbi->used.bitmap.rw_lock, BITMAP_MUTEX_CLUSTERS); + + ntfs_refresh_zone(sbi); + + up_write(&sbi->used.bitmap.rw_lock); + up_write(&ni->file.run_lock); + + if (err) + goto out; + + err = wnd_extend(wnd, new_mft_total); + + if (err) + goto out; + + ntfs_clear_mft_tail(sbi, sbi->mft.used, new_mft_total); + + err = _ni_write_inode(&ni->vfs_inode, 0); +out: + return err; +} + +/* + * ntfs_look_free_mft + * + * looks for a free MFT record + */ +int ntfs_look_free_mft(struct ntfs_sb_info *sbi, CLST *rno, bool mft, + struct ntfs_inode *ni, struct mft_inode **mi) +{ + int err = 0; + size_t zbit, zlen, from, to, fr; + size_t mft_total; + struct MFT_REF ref; + struct super_block *sb = sbi->sb; + struct wnd_bitmap *wnd = &sbi->mft.bitmap; + u32 ir; + + static_assert(sizeof(sbi->mft.reserved_bitmap) * 8 >= + MFT_REC_FREE - MFT_REC_RESERVED); + + if (!mft) + down_write_nested(&wnd->rw_lock, BITMAP_MUTEX_MFT); + + zlen = wnd_zone_len(wnd); + + /* Always reserve space for MFT */ + if (zlen) { + if (mft) { + zbit = wnd_zone_bit(wnd); + *rno = zbit; + wnd_zone_set(wnd, zbit + 1, zlen - 1); + } + goto found; + } + + /* No MFT zone. find the nearest to '0' free MFT */ + if (!wnd_find(wnd, 1, MFT_REC_FREE, 0, &zbit)) { + /* Resize MFT */ + mft_total = wnd->nbits; + + err = ntfs_extend_mft(sbi); + if (!err) { + zbit = mft_total; + goto reserve_mft; + } + + if (!mft || MFT_REC_FREE == sbi->mft.next_reserved) + goto out; + + err = 0; + + /* + * Look for free record reserved area [11-16) == + * [MFT_REC_RESERVED, MFT_REC_FREE ) MFT bitmap always + * marks it as used + */ + if (!sbi->mft.reserved_bitmap) { + /* Once per session create internal bitmap for 5 bits */ + sbi->mft.reserved_bitmap = 0xFF; + + ref.high = 0; + for (ir = MFT_REC_RESERVED; ir < MFT_REC_FREE; ir++) { + struct inode *i; + struct ntfs_inode *ni; + struct MFT_REC *mrec; + + ref.low = cpu_to_le32(ir); + ref.seq = cpu_to_le16(ir); + + i = ntfs_iget5(sb, &ref, NULL); + if (IS_ERR(i)) { +next: + ntfs_notice( + sb, + "Invalid reserved record %x", + ref.low); + continue; + } + if (is_bad_inode(i)) { + iput(i); + goto next; + } + + ni = ntfs_i(i); + + mrec = ni->mi.mrec; + + if (!is_rec_base(mrec)) + goto next; + + if (mrec->hard_links) + goto next; + + if (!ni_std(ni)) + goto next; + + if (ni_find_attr(ni, NULL, NULL, ATTR_NAME, + NULL, 0, NULL, NULL)) + goto next; + + __clear_bit(ir - MFT_REC_RESERVED, + &sbi->mft.reserved_bitmap); + } + } + + /* Scan 5 bits for zero. Bit 0 == MFT_REC_RESERVED */ + zbit = find_next_zero_bit(&sbi->mft.reserved_bitmap, + MFT_REC_FREE, MFT_REC_RESERVED); + if (zbit >= MFT_REC_FREE) { + sbi->mft.next_reserved = MFT_REC_FREE; + goto out; + } + + zlen = 1; + sbi->mft.next_reserved = zbit; + } else { +reserve_mft: + zlen = zbit == MFT_REC_FREE ? (MFT_REC_USER - MFT_REC_FREE) : 4; + if (zbit + zlen > wnd->nbits) + zlen = wnd->nbits - zbit; + + while (zlen > 1 && !wnd_is_free(wnd, zbit, zlen)) + zlen -= 1; + + /* [zbit, zbit + zlen) will be used for Mft itself */ + from = sbi->mft.used; + if (from < zbit) + from = zbit; + to = zbit + zlen; + if (from < to) { + ntfs_clear_mft_tail(sbi, from, to); + sbi->mft.used = to; + } + } + + if (mft) { + *rno = zbit; + zbit += 1; + zlen -= 1; + } + + wnd_zone_set(wnd, zbit, zlen); + +found: + if (!mft) { + /* The request to get record for general purpose */ + if (sbi->mft.next_free < MFT_REC_USER) + sbi->mft.next_free = MFT_REC_USER; + + for (;;) { + if (sbi->mft.next_free >= sbi->mft.bitmap.nbits) { + } else if (!wnd_find(wnd, 1, MFT_REC_USER, 0, &fr)) { + sbi->mft.next_free = sbi->mft.bitmap.nbits; + } else { + *rno = fr; + sbi->mft.next_free = *rno + 1; + break; + } + + err = ntfs_extend_mft(sbi); + if (err) + goto out; + } + } + + if (ni && !ni_add_subrecord(ni, *rno, mi)) { + err = -ENOMEM; + goto out; + } + + /* We have found a record that are not reserved for next MFT */ + if (*rno >= MFT_REC_FREE) + wnd_set_used(wnd, *rno, 1); + else if (*rno >= MFT_REC_RESERVED && sbi->mft.reserved_bitmap_inited) + __set_bit(*rno - MFT_REC_RESERVED, &sbi->mft.reserved_bitmap); + +out: + if (!mft) + up_write(&wnd->rw_lock); + + return err; +} + +/* + * ntfs_mark_rec_free + * + * marks record as free + */ +void ntfs_mark_rec_free(struct ntfs_sb_info *sbi, CLST rno) +{ + struct wnd_bitmap *wnd = &sbi->mft.bitmap; + + down_write_nested(&wnd->rw_lock, BITMAP_MUTEX_MFT); + if (rno >= wnd->nbits) + goto out; + + if (rno >= MFT_REC_FREE) { + if (!wnd_is_used(wnd, rno, 1)) + ntfs_set_state(sbi, NTFS_DIRTY_ERROR); + else + wnd_set_free(wnd, rno, 1); + } else if (rno >= MFT_REC_RESERVED && sbi->mft.reserved_bitmap_inited) { + __clear_bit(rno - MFT_REC_RESERVED, &sbi->mft.reserved_bitmap); + } + + if (rno < wnd_zone_bit(wnd)) + wnd_zone_set(wnd, rno, 1); + else if (rno < sbi->mft.next_free && rno >= MFT_REC_USER) + sbi->mft.next_free = rno; + +out: + up_write(&wnd->rw_lock); +} + +/* + * ntfs_clear_mft_tail + * + * formats empty records [from, to) + * sbi->mft.bitmap is locked for write + */ +int ntfs_clear_mft_tail(struct ntfs_sb_info *sbi, size_t from, size_t to) +{ + int err; + u32 rs; + u64 vbo; + struct runs_tree *run; + struct ntfs_inode *ni; + + if (from >= to) + return 0; + + rs = sbi->record_size; + ni = sbi->mft.ni; + run = &ni->file.run; + + down_read(&ni->file.run_lock); + vbo = (u64)from * rs; + for (; from < to; from++, vbo += rs) { + struct ntfs_buffers nb; + + err = ntfs_get_bh(sbi, run, vbo, rs, &nb); + if (err) + goto out; + + err = ntfs_write_bh(sbi, &sbi->new_rec->rhdr, &nb, 0); + nb_put(&nb); + if (err) + goto out; + } + +out: + sbi->mft.used = from; + up_read(&ni->file.run_lock); + return err; +} + +/* + * ntfs_refresh_zone + * + * refreshes Mft zone + * sbi->used.bitmap is locked for rw + * sbi->mft.bitmap is locked for write + * sbi->mft.ni->file.run_lock for write + */ +int ntfs_refresh_zone(struct ntfs_sb_info *sbi) +{ + CLST zone_limit, zone_max, lcn, vcn, len; + size_t lcn_s, zlen; + struct wnd_bitmap *wnd = &sbi->used.bitmap; + struct ntfs_inode *ni = sbi->mft.ni; + + /* Do not change anything unless we have non empty Mft zone */ + if (wnd_zone_len(wnd)) + return 0; + + /* + * Compute the mft zone at two steps + * It would be nice if we are able to allocate + * 1/8 of total clusters for MFT but not more then 512 MB + */ + zone_limit = (512 * 1024 * 1024) >> sbi->cluster_bits; + zone_max = wnd->nbits >> 3; + if (zone_max > zone_limit) + zone_max = zone_limit; + + vcn = bytes_to_cluster(sbi, + (u64)sbi->mft.bitmap.nbits << sbi->record_bits); + + if (!run_lookup_entry(&ni->file.run, vcn - 1, &lcn, &len, NULL)) + lcn = SPARSE_LCN; + + /* We should always find Last Lcn for MFT */ + if (lcn == SPARSE_LCN) + return -EINVAL; + + lcn_s = lcn + 1; + + /* Try to allocate clusters after last MFT run */ + zlen = wnd_find(wnd, zone_max, lcn_s, 0, &lcn_s); + if (!zlen) { + ntfs_notice(sbi->sb, "MftZone: unavailable"); + return 0; + } + + /* Truncate too large zone */ + wnd_zone_set(wnd, lcn_s, zlen); + + return 0; +} + +/* + * ntfs_update_mftmirr + * + * updates $MFTMirr data + */ +int ntfs_update_mftmirr(struct ntfs_sb_info *sbi, int wait) +{ + int err; + struct super_block *sb = sbi->sb; + u32 blocksize = sb->s_blocksize; + sector_t block1, block2; + u32 bytes; + + if (!(sbi->flags & NTFS_FLAGS_MFTMIRR)) + return 0; + + err = 0; + bytes = sbi->mft.recs_mirr << sbi->record_bits; + block1 = sbi->mft.lbo >> sb->s_blocksize_bits; + block2 = sbi->mft.lbo2 >> sb->s_blocksize_bits; + + for (; bytes >= blocksize; bytes -= blocksize) { + struct buffer_head *bh1, *bh2; + + bh1 = sb_bread(sb, block1++); + if (!bh1) { + err = -EIO; + goto out; + } + + bh2 = sb_getblk(sb, block2++); + if (!bh2) { + put_bh(bh1); + err = -EIO; + goto out; + } + + if (buffer_locked(bh2)) + __wait_on_buffer(bh2); + + lock_buffer(bh2); + memcpy(bh2->b_data, bh1->b_data, blocksize); + set_buffer_uptodate(bh2); + mark_buffer_dirty(bh2); + unlock_buffer(bh2); + + put_bh(bh1); + bh1 = NULL; + + if (wait) + err = sync_dirty_buffer(bh2); + + put_bh(bh2); + if (err) + goto out; + } + + sbi->flags &= ~NTFS_FLAGS_MFTMIRR; + +out: + return err; +} + +/* + * ntfs_set_state + * + * mount: ntfs_set_state(NTFS_DIRTY_DIRTY) + * umount: ntfs_set_state(NTFS_DIRTY_CLEAR) + * ntfs error: ntfs_set_state(NTFS_DIRTY_ERROR) + */ +int ntfs_set_state(struct ntfs_sb_info *sbi, enum NTFS_DIRTY_FLAGS dirty) +{ + int err; + struct ATTRIB *attr; + struct VOLUME_INFO *info; + struct mft_inode *mi; + struct ntfs_inode *ni; + + /* + * do not change state if fs was real_dirty + * do not change state if fs already dirty(clear) + * do not change any thing if mounted read only + */ + if (sbi->volume.real_dirty || sb_rdonly(sbi->sb)) + return 0; + + /* Check cached value */ + if ((dirty == NTFS_DIRTY_CLEAR ? 0 : VOLUME_FLAG_DIRTY) == + (sbi->volume.flags & VOLUME_FLAG_DIRTY)) + return 0; + + ni = sbi->volume.ni; + if (!ni) + return -EINVAL; + + mutex_lock_nested(&ni->ni_lock, NTFS_INODE_MUTEX_DIRTY); + + attr = ni_find_attr(ni, NULL, NULL, ATTR_VOL_INFO, NULL, 0, NULL, &mi); + if (!attr) { + err = -EINVAL; + goto out; + } + + info = resident_data_ex(attr, SIZEOF_ATTRIBUTE_VOLUME_INFO); + if (!info) { + err = -EINVAL; + goto out; + } + + switch (dirty) { + case NTFS_DIRTY_ERROR: + ntfs_notice(sbi->sb, "Mark volume as dirty due to NTFS errors"); + sbi->volume.real_dirty = true; + fallthrough; + case NTFS_DIRTY_DIRTY: + info->flags |= VOLUME_FLAG_DIRTY; + break; + case NTFS_DIRTY_CLEAR: + info->flags &= ~VOLUME_FLAG_DIRTY; + break; + } + /* cache current volume flags*/ + sbi->volume.flags = info->flags; + mi->dirty = true; + err = 0; + +out: + ni_unlock(ni); + if (err) + return err; + + mark_inode_dirty(&ni->vfs_inode); + /*verify(!ntfs_update_mftmirr()); */ + err = sync_inode_metadata(&ni->vfs_inode, 1); + + return err; +} + +/* + * security_hash + * + * calculates a hash of security descriptor + */ +static inline __le32 security_hash(const void *sd, size_t bytes) +{ + u32 hash = 0; + const __le32 *ptr = sd; + + bytes >>= 2; + while (bytes--) + hash = ((hash >> 0x1D) | (hash << 3)) + le32_to_cpu(*ptr++); + return cpu_to_le32(hash); +} + +int ntfs_sb_read(struct super_block *sb, u64 lbo, size_t bytes, void *buffer) +{ + struct block_device *bdev = sb->s_bdev; + u32 blocksize = sb->s_blocksize; + u64 block = lbo >> sb->s_blocksize_bits; + u32 off = lbo & (blocksize - 1); + u32 op = blocksize - off; + + for (; bytes; block += 1, off = 0, op = blocksize) { + struct buffer_head *bh = __bread(bdev, block, blocksize); + + if (!bh) + return -EIO; + + if (op > bytes) + op = bytes; + + memcpy(buffer, bh->b_data + off, op); + + put_bh(bh); + + bytes -= op; + buffer = Add2Ptr(buffer, op); + } + + return 0; +} + +int ntfs_sb_write(struct super_block *sb, u64 lbo, size_t bytes, + const void *buf, int wait) +{ + u32 blocksize = sb->s_blocksize; + struct block_device *bdev = sb->s_bdev; + sector_t block = lbo >> sb->s_blocksize_bits; + u32 off = lbo & (blocksize - 1); + u32 op = blocksize - off; + struct buffer_head *bh; + + if (!wait && (sb->s_flags & SB_SYNCHRONOUS)) + wait = 1; + + for (; bytes; block += 1, off = 0, op = blocksize) { + if (op > bytes) + op = bytes; + + if (op < blocksize) { + bh = __bread(bdev, block, blocksize); + if (!bh) { + ntfs_err(sb, "failed to read block %llx", + (u64)block); + return -EIO; + } + } else { + bh = __getblk(bdev, block, blocksize); + if (!bh) + return -ENOMEM; + } + + if (buffer_locked(bh)) + __wait_on_buffer(bh); + + lock_buffer(bh); + if (buf) { + memcpy(bh->b_data + off, buf, op); + buf = Add2Ptr(buf, op); + } else { + memset(bh->b_data + off, -1, op); + } + + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); + unlock_buffer(bh); + + if (wait) { + int err = sync_dirty_buffer(bh); + + if (err) { + ntfs_err( + sb, + "failed to sync buffer at block %llx, error %d", + (u64)block, err); + put_bh(bh); + return err; + } + } + + put_bh(bh); + + bytes -= op; + } + return 0; +} + +int ntfs_sb_write_run(struct ntfs_sb_info *sbi, const struct runs_tree *run, + u64 vbo, const void *buf, size_t bytes) +{ + struct super_block *sb = sbi->sb; + u8 cluster_bits = sbi->cluster_bits; + u32 off = vbo & sbi->cluster_mask; + CLST lcn, clen, vcn = vbo >> cluster_bits, vcn_next; + u64 lbo, len; + size_t idx; + + if (!run_lookup_entry(run, vcn, &lcn, &clen, &idx)) + return -ENOENT; + + if (lcn == SPARSE_LCN) + return -EINVAL; + + lbo = ((u64)lcn << cluster_bits) + off; + len = ((u64)clen << cluster_bits) - off; + + for (;;) { + u32 op = len < bytes ? len : bytes; + int err = ntfs_sb_write(sb, lbo, op, buf, 0); + + if (err) + return err; + + bytes -= op; + if (!bytes) + break; + + vcn_next = vcn + clen; + if (!run_get_entry(run, ++idx, &vcn, &lcn, &clen) || + vcn != vcn_next) + return -ENOENT; + + if (lcn == SPARSE_LCN) + return -EINVAL; + + if (buf) + buf = Add2Ptr(buf, op); + + lbo = ((u64)lcn << cluster_bits); + len = ((u64)clen << cluster_bits); + } + + return 0; +} + +struct buffer_head *ntfs_bread_run(struct ntfs_sb_info *sbi, + const struct runs_tree *run, u64 vbo) +{ + struct super_block *sb = sbi->sb; + u8 cluster_bits = sbi->cluster_bits; + CLST lcn; + u64 lbo; + + if (!run_lookup_entry(run, vbo >> cluster_bits, &lcn, NULL, NULL)) + return ERR_PTR(-ENOENT); + + lbo = ((u64)lcn << cluster_bits) + (vbo & sbi->cluster_mask); + + return ntfs_bread(sb, lbo >> sb->s_blocksize_bits); +} + +int ntfs_read_run_nb(struct ntfs_sb_info *sbi, const struct runs_tree *run, + u64 vbo, void *buf, u32 bytes, struct ntfs_buffers *nb) +{ + int err; + struct super_block *sb = sbi->sb; + u32 blocksize = sb->s_blocksize; + u8 cluster_bits = sbi->cluster_bits; + u32 off = vbo & sbi->cluster_mask; + u32 nbh = 0; + CLST vcn_next, vcn = vbo >> cluster_bits; + CLST lcn, clen; + u64 lbo, len; + size_t idx; + struct buffer_head *bh; + + if (!run) { + /* first reading of $Volume + $MFTMirr + LogFile goes here*/ + if (vbo > MFT_REC_VOL * sbi->record_size) { + err = -ENOENT; + goto out; + } + + /* use absolute boot's 'MFTCluster' to read record */ + lbo = vbo + sbi->mft.lbo; + len = sbi->record_size; + } else if (!run_lookup_entry(run, vcn, &lcn, &clen, &idx)) { + err = -ENOENT; + goto out; + } else { + if (lcn == SPARSE_LCN) { + err = -EINVAL; + goto out; + } + + lbo = ((u64)lcn << cluster_bits) + off; + len = ((u64)clen << cluster_bits) - off; + } + + off = lbo & (blocksize - 1); + if (nb) { + nb->off = off; + nb->bytes = bytes; + } + + for (;;) { + u32 len32 = len >= bytes ? bytes : len; + sector_t block = lbo >> sb->s_blocksize_bits; + + do { + u32 op = blocksize - off; + + if (op > len32) + op = len32; + + bh = ntfs_bread(sb, block); + if (!bh) { + err = -EIO; + goto out; + } + + if (buf) { + memcpy(buf, bh->b_data + off, op); + buf = Add2Ptr(buf, op); + } + + if (!nb) { + put_bh(bh); + } else if (nbh >= ARRAY_SIZE(nb->bh)) { + err = -EINVAL; + goto out; + } else { + nb->bh[nbh++] = bh; + nb->nbufs = nbh; + } + + bytes -= op; + if (!bytes) + return 0; + len32 -= op; + block += 1; + off = 0; + + } while (len32); + + vcn_next = vcn + clen; + if (!run_get_entry(run, ++idx, &vcn, &lcn, &clen) || + vcn != vcn_next) { + err = -ENOENT; + goto out; + } + + if (lcn == SPARSE_LCN) { + err = -EINVAL; + goto out; + } + + lbo = ((u64)lcn << cluster_bits); + len = ((u64)clen << cluster_bits); + } + +out: + if (!nbh) + return err; + + while (nbh) { + put_bh(nb->bh[--nbh]); + nb->bh[nbh] = NULL; + } + + nb->nbufs = 0; + return err; +} + +/* Returns < 0 if error, 0 if ok, '-E_NTFS_FIXUP' if need to update fixups */ +int ntfs_read_bh(struct ntfs_sb_info *sbi, const struct runs_tree *run, u64 vbo, + struct NTFS_RECORD_HEADER *rhdr, u32 bytes, + struct ntfs_buffers *nb) +{ + int err = ntfs_read_run_nb(sbi, run, vbo, rhdr, bytes, nb); + + if (err) + return err; + return ntfs_fix_post_read(rhdr, nb->bytes, true); +} + +int ntfs_get_bh(struct ntfs_sb_info *sbi, const struct runs_tree *run, u64 vbo, + u32 bytes, struct ntfs_buffers *nb) +{ + int err = 0; + struct super_block *sb = sbi->sb; + u32 blocksize = sb->s_blocksize; + u8 cluster_bits = sbi->cluster_bits; + CLST vcn_next, vcn = vbo >> cluster_bits; + u32 off; + u32 nbh = 0; + CLST lcn, clen; + u64 lbo, len; + size_t idx; + + nb->bytes = bytes; + + if (!run_lookup_entry(run, vcn, &lcn, &clen, &idx)) { + err = -ENOENT; + goto out; + } + + off = vbo & sbi->cluster_mask; + lbo = ((u64)lcn << cluster_bits) + off; + len = ((u64)clen << cluster_bits) - off; + + nb->off = off = lbo & (blocksize - 1); + + for (;;) { + u32 len32 = len < bytes ? len : bytes; + sector_t block = lbo >> sb->s_blocksize_bits; + + do { + u32 op; + struct buffer_head *bh; + + if (nbh >= ARRAY_SIZE(nb->bh)) { + err = -EINVAL; + goto out; + } + + op = blocksize - off; + if (op > len32) + op = len32; + + if (op == blocksize) { + bh = sb_getblk(sb, block); + if (!bh) { + err = -ENOMEM; + goto out; + } + if (buffer_locked(bh)) + __wait_on_buffer(bh); + set_buffer_uptodate(bh); + } else { + bh = ntfs_bread(sb, block); + if (!bh) { + err = -EIO; + goto out; + } + } + + nb->bh[nbh++] = bh; + bytes -= op; + if (!bytes) { + nb->nbufs = nbh; + return 0; + } + + block += 1; + len32 -= op; + off = 0; + } while (len32); + + vcn_next = vcn + clen; + if (!run_get_entry(run, ++idx, &vcn, &lcn, &clen) || + vcn != vcn_next) { + err = -ENOENT; + goto out; + } + + lbo = ((u64)lcn << cluster_bits); + len = ((u64)clen << cluster_bits); + } + +out: + while (nbh) { + put_bh(nb->bh[--nbh]); + nb->bh[nbh] = NULL; + } + + nb->nbufs = 0; + + return err; +} + +int ntfs_write_bh(struct ntfs_sb_info *sbi, struct NTFS_RECORD_HEADER *rhdr, + struct ntfs_buffers *nb, int sync) +{ + int err = 0; + struct super_block *sb = sbi->sb; + u32 block_size = sb->s_blocksize; + u32 bytes = nb->bytes; + u32 off = nb->off; + u16 fo = le16_to_cpu(rhdr->fix_off); + u16 fn = le16_to_cpu(rhdr->fix_num); + u32 idx; + __le16 *fixup; + __le16 sample; + + if ((fo & 1) || fo + fn * sizeof(short) > SECTOR_SIZE || !fn-- || + fn * SECTOR_SIZE > bytes) { + return -EINVAL; + } + + for (idx = 0; bytes && idx < nb->nbufs; idx += 1, off = 0) { + u32 op = block_size - off; + char *bh_data; + struct buffer_head *bh = nb->bh[idx]; + __le16 *ptr, *end_data; + + if (op > bytes) + op = bytes; + + if (buffer_locked(bh)) + __wait_on_buffer(bh); + + lock_buffer(nb->bh[idx]); + + bh_data = bh->b_data + off; + end_data = Add2Ptr(bh_data, op); + memcpy(bh_data, rhdr, op); + + if (!idx) { + u16 t16; + + fixup = Add2Ptr(bh_data, fo); + sample = *fixup; + t16 = le16_to_cpu(sample); + if (t16 >= 0x7FFF) { + sample = *fixup = cpu_to_le16(1); + } else { + sample = cpu_to_le16(t16 + 1); + *fixup = sample; + } + + *(__le16 *)Add2Ptr(rhdr, fo) = sample; + } + + ptr = Add2Ptr(bh_data, SECTOR_SIZE - sizeof(short)); + + do { + *++fixup = *ptr; + *ptr = sample; + ptr += SECTOR_SIZE / sizeof(short); + } while (ptr < end_data); + + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); + unlock_buffer(bh); + + if (sync) { + int err2 = sync_dirty_buffer(bh); + + if (!err && err2) + err = err2; + } + + bytes -= op; + rhdr = Add2Ptr(rhdr, op); + } + + return err; +} + +static inline struct bio *ntfs_alloc_bio(u32 nr_vecs) +{ + struct bio *bio = bio_alloc(GFP_NOFS | __GFP_HIGH, nr_vecs); + + if (!bio && (current->flags & PF_MEMALLOC)) { + while (!bio && (nr_vecs /= 2)) + bio = bio_alloc(GFP_NOFS | __GFP_HIGH, nr_vecs); + } + return bio; +} + +/* read/write pages from/to disk*/ +int ntfs_bio_pages(struct ntfs_sb_info *sbi, const struct runs_tree *run, + struct page **pages, u32 nr_pages, u64 vbo, u32 bytes, + u32 op) +{ + int err = 0; + struct bio *new, *bio = NULL; + struct super_block *sb = sbi->sb; + struct block_device *bdev = sb->s_bdev; + struct page *page; + u8 cluster_bits = sbi->cluster_bits; + CLST lcn, clen, vcn, vcn_next; + u32 add, off, page_idx; + u64 lbo, len; + size_t run_idx; + struct blk_plug plug; + + if (!bytes) + return 0; + + blk_start_plug(&plug); + + /* align vbo and bytes to be 512 bytes aligned */ + lbo = (vbo + bytes + 511) & ~511ull; + vbo = vbo & ~511ull; + bytes = lbo - vbo; + + vcn = vbo >> cluster_bits; + if (!run_lookup_entry(run, vcn, &lcn, &clen, &run_idx)) { + err = -ENOENT; + goto out; + } + off = vbo & sbi->cluster_mask; + page_idx = 0; + page = pages[0]; + + for (;;) { + lbo = ((u64)lcn << cluster_bits) + off; + len = ((u64)clen << cluster_bits) - off; +new_bio: + new = ntfs_alloc_bio(nr_pages - page_idx); + if (!new) { + err = -ENOMEM; + goto out; + } + if (bio) { + bio_chain(bio, new); + submit_bio(bio); + } + bio = new; + bio_set_dev(bio, bdev); + bio->bi_iter.bi_sector = lbo >> 9; + bio->bi_opf = op; + + while (len) { + off = vbo & (PAGE_SIZE - 1); + add = off + len > PAGE_SIZE ? (PAGE_SIZE - off) : len; + + if (bio_add_page(bio, page, add, off) < add) + goto new_bio; + + if (bytes <= add) + goto out; + bytes -= add; + vbo += add; + + if (add + off == PAGE_SIZE) { + page_idx += 1; + if (WARN_ON(page_idx >= nr_pages)) { + err = -EINVAL; + goto out; + } + page = pages[page_idx]; + } + + if (len <= add) + break; + len -= add; + lbo += add; + } + + vcn_next = vcn + clen; + if (!run_get_entry(run, ++run_idx, &vcn, &lcn, &clen) || + vcn != vcn_next) { + err = -ENOENT; + goto out; + } + off = 0; + } +out: + if (bio) { + if (!err) + err = submit_bio_wait(bio); + bio_put(bio); + } + blk_finish_plug(&plug); + + return err; +} + +/* + * Helper for ntfs_loadlog_and_replay + * fill on-disk logfile range by (-1) + * this means empty logfile + */ +int ntfs_bio_fill_1(struct ntfs_sb_info *sbi, const struct runs_tree *run) +{ + int err = 0; + struct super_block *sb = sbi->sb; + struct block_device *bdev = sb->s_bdev; + u8 cluster_bits = sbi->cluster_bits; + struct bio *new, *bio = NULL; + CLST lcn, clen; + u64 lbo, len; + size_t run_idx; + struct page *fill; + void *kaddr; + struct blk_plug plug; + + fill = alloc_page(GFP_KERNEL); + if (!fill) + return -ENOMEM; + + kaddr = kmap_atomic(fill); + memset(kaddr, -1, PAGE_SIZE); + kunmap_atomic(kaddr); + flush_dcache_page(fill); + lock_page(fill); + + if (!run_lookup_entry(run, 0, &lcn, &clen, &run_idx)) { + err = -ENOENT; + goto out; + } + + /* + * TODO: try blkdev_issue_write_same + */ + blk_start_plug(&plug); + do { + lbo = (u64)lcn << cluster_bits; + len = (u64)clen << cluster_bits; +new_bio: + new = ntfs_alloc_bio(BIO_MAX_VECS); + if (!new) { + err = -ENOMEM; + break; + } + if (bio) { + bio_chain(bio, new); + submit_bio(bio); + } + bio = new; + bio_set_dev(bio, bdev); + bio->bi_opf = REQ_OP_WRITE; + bio->bi_iter.bi_sector = lbo >> 9; + + for (;;) { + u32 add = len > PAGE_SIZE ? PAGE_SIZE : len; + + if (bio_add_page(bio, fill, add, 0) < add) + goto new_bio; + + lbo += add; + if (len <= add) + break; + len -= add; + } + } while (run_get_entry(run, ++run_idx, NULL, &lcn, &clen)); + + if (bio) { + if (!err) + err = submit_bio_wait(bio); + bio_put(bio); + } + blk_finish_plug(&plug); +out: + unlock_page(fill); + put_page(fill); + + return err; +} + +int ntfs_vbo_to_lbo(struct ntfs_sb_info *sbi, const struct runs_tree *run, + u64 vbo, u64 *lbo, u64 *bytes) +{ + u32 off; + CLST lcn, len; + u8 cluster_bits = sbi->cluster_bits; + + if (!run_lookup_entry(run, vbo >> cluster_bits, &lcn, &len, NULL)) + return -ENOENT; + + off = vbo & sbi->cluster_mask; + *lbo = lcn == SPARSE_LCN ? -1 : (((u64)lcn << cluster_bits) + off); + *bytes = ((u64)len << cluster_bits) - off; + + return 0; +} + +struct ntfs_inode *ntfs_new_inode(struct ntfs_sb_info *sbi, CLST rno, bool dir) +{ + int err = 0; + struct super_block *sb = sbi->sb; + struct inode *inode = new_inode(sb); + struct ntfs_inode *ni; + + if (!inode) + return ERR_PTR(-ENOMEM); + + ni = ntfs_i(inode); + + err = mi_format_new(&ni->mi, sbi, rno, dir ? RECORD_FLAG_DIR : 0, + false); + if (err) + goto out; + + inode->i_ino = rno; + if (insert_inode_locked(inode) < 0) { + err = -EIO; + goto out; + } + +out: + if (err) { + iput(inode); + ni = ERR_PTR(err); + } + return ni; +} + +/* + * O:BAG:BAD:(A;OICI;FA;;;WD) + * owner S-1-5-32-544 (Administrators) + * group S-1-5-32-544 (Administrators) + * ACE: allow S-1-1-0 (Everyone) with FILE_ALL_ACCESS + */ +const u8 s_default_security[] __aligned(8) = { + 0x01, 0x00, 0x04, 0x80, 0x30, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x02, 0x00, 0x1C, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x03, 0x14, 0x00, 0xFF, 0x01, 0x1F, 0x00, + 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x20, 0x00, 0x00, 0x00, + 0x20, 0x02, 0x00, 0x00, 0x01, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, + 0x20, 0x00, 0x00, 0x00, 0x20, 0x02, 0x00, 0x00, +}; + +static_assert(sizeof(s_default_security) == 0x50); + +static inline u32 sid_length(const struct SID *sid) +{ + return struct_size(sid, SubAuthority, sid->SubAuthorityCount); +} + +/* + * Thanks Mark Harmstone for idea + */ +static bool is_acl_valid(const struct ACL *acl, u32 len) +{ + const struct ACE_HEADER *ace; + u32 i; + u16 ace_count, ace_size; + + if (acl->AclRevision != ACL_REVISION && + acl->AclRevision != ACL_REVISION_DS) { + /* + * This value should be ACL_REVISION, unless the ACL contains an + * object-specific ACE, in which case this value must be ACL_REVISION_DS. + * All ACEs in an ACL must be at the same revision level. + */ + return false; + } + + if (acl->Sbz1) + return false; + + if (le16_to_cpu(acl->AclSize) > len) + return false; + + if (acl->Sbz2) + return false; + + len -= sizeof(struct ACL); + ace = (struct ACE_HEADER *)&acl[1]; + ace_count = le16_to_cpu(acl->AceCount); + + for (i = 0; i < ace_count; i++) { + if (len < sizeof(struct ACE_HEADER)) + return false; + + ace_size = le16_to_cpu(ace->AceSize); + if (len < ace_size) + return false; + + len -= ace_size; + ace = Add2Ptr(ace, ace_size); + } + + return true; +} + +bool is_sd_valid(const struct SECURITY_DESCRIPTOR_RELATIVE *sd, u32 len) +{ + u32 sd_owner, sd_group, sd_sacl, sd_dacl; + + if (len < sizeof(struct SECURITY_DESCRIPTOR_RELATIVE)) + return false; + + if (sd->Revision != 1) + return false; + + if (sd->Sbz1) + return false; + + if (!(sd->Control & SE_SELF_RELATIVE)) + return false; + + sd_owner = le32_to_cpu(sd->Owner); + if (sd_owner) { + const struct SID *owner = Add2Ptr(sd, sd_owner); + + if (sd_owner + offsetof(struct SID, SubAuthority) > len) + return false; + + if (owner->Revision != 1) + return false; + + if (sd_owner + sid_length(owner) > len) + return false; + } + + sd_group = le32_to_cpu(sd->Group); + if (sd_group) { + const struct SID *group = Add2Ptr(sd, sd_group); + + if (sd_group + offsetof(struct SID, SubAuthority) > len) + return false; + + if (group->Revision != 1) + return false; + + if (sd_group + sid_length(group) > len) + return false; + } + + sd_sacl = le32_to_cpu(sd->Sacl); + if (sd_sacl) { + const struct ACL *sacl = Add2Ptr(sd, sd_sacl); + + if (sd_sacl + sizeof(struct ACL) > len) + return false; + + if (!is_acl_valid(sacl, len - sd_sacl)) + return false; + } + + sd_dacl = le32_to_cpu(sd->Dacl); + if (sd_dacl) { + const struct ACL *dacl = Add2Ptr(sd, sd_dacl); + + if (sd_dacl + sizeof(struct ACL) > len) + return false; + + if (!is_acl_valid(dacl, len - sd_dacl)) + return false; + } + + return true; +} + +/* + * ntfs_security_init + * + * loads and parse $Secure + */ +int ntfs_security_init(struct ntfs_sb_info *sbi) +{ + int err; + struct super_block *sb = sbi->sb; + struct inode *inode; + struct ntfs_inode *ni; + struct MFT_REF ref; + struct ATTRIB *attr; + struct ATTR_LIST_ENTRY *le; + u64 sds_size; + size_t cnt, off; + struct NTFS_DE *ne; + struct NTFS_DE_SII *sii_e; + struct ntfs_fnd *fnd_sii = NULL; + const struct INDEX_ROOT *root_sii; + const struct INDEX_ROOT *root_sdh; + struct ntfs_index *indx_sdh = &sbi->security.index_sdh; + struct ntfs_index *indx_sii = &sbi->security.index_sii; + + ref.low = cpu_to_le32(MFT_REC_SECURE); + ref.high = 0; + ref.seq = cpu_to_le16(MFT_REC_SECURE); + + inode = ntfs_iget5(sb, &ref, &NAME_SECURE); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + ntfs_err(sb, "Failed to load $Secure."); + inode = NULL; + goto out; + } + + ni = ntfs_i(inode); + + le = NULL; + + attr = ni_find_attr(ni, NULL, &le, ATTR_ROOT, SDH_NAME, + ARRAY_SIZE(SDH_NAME), NULL, NULL); + if (!attr) { + err = -EINVAL; + goto out; + } + + root_sdh = resident_data(attr); + if (root_sdh->type != ATTR_ZERO || + root_sdh->rule != NTFS_COLLATION_TYPE_SECURITY_HASH) { + err = -EINVAL; + goto out; + } + + err = indx_init(indx_sdh, sbi, attr, INDEX_MUTEX_SDH); + if (err) + goto out; + + attr = ni_find_attr(ni, attr, &le, ATTR_ROOT, SII_NAME, + ARRAY_SIZE(SII_NAME), NULL, NULL); + if (!attr) { + err = -EINVAL; + goto out; + } + + root_sii = resident_data(attr); + if (root_sii->type != ATTR_ZERO || + root_sii->rule != NTFS_COLLATION_TYPE_UINT) { + err = -EINVAL; + goto out; + } + + err = indx_init(indx_sii, sbi, attr, INDEX_MUTEX_SII); + if (err) + goto out; + + fnd_sii = fnd_get(); + if (!fnd_sii) { + err = -ENOMEM; + goto out; + } + + sds_size = inode->i_size; + + /* Find the last valid Id */ + sbi->security.next_id = SECURITY_ID_FIRST; + /* Always write new security at the end of bucket */ + sbi->security.next_off = + Quad2Align(sds_size - SecurityDescriptorsBlockSize); + + cnt = 0; + off = 0; + ne = NULL; + + for (;;) { + u32 next_id; + + err = indx_find_raw(indx_sii, ni, root_sii, &ne, &off, fnd_sii); + if (err || !ne) + break; + + sii_e = (struct NTFS_DE_SII *)ne; + if (le16_to_cpu(ne->view.data_size) < SIZEOF_SECURITY_HDR) + continue; + + next_id = le32_to_cpu(sii_e->sec_id) + 1; + if (next_id >= sbi->security.next_id) + sbi->security.next_id = next_id; + + cnt += 1; + } + + sbi->security.ni = ni; + inode = NULL; +out: + iput(inode); + fnd_put(fnd_sii); + + return err; +} + +/* + * ntfs_get_security_by_id + * + * reads security descriptor by id + */ +int ntfs_get_security_by_id(struct ntfs_sb_info *sbi, __le32 security_id, + struct SECURITY_DESCRIPTOR_RELATIVE **sd, + size_t *size) +{ + int err; + int diff; + struct ntfs_inode *ni = sbi->security.ni; + struct ntfs_index *indx = &sbi->security.index_sii; + void *p = NULL; + struct NTFS_DE_SII *sii_e; + struct ntfs_fnd *fnd_sii; + struct SECURITY_HDR d_security; + const struct INDEX_ROOT *root_sii; + u32 t32; + + *sd = NULL; + + mutex_lock_nested(&ni->ni_lock, NTFS_INODE_MUTEX_SECURITY); + + fnd_sii = fnd_get(); + if (!fnd_sii) { + err = -ENOMEM; + goto out; + } + + root_sii = indx_get_root(indx, ni, NULL, NULL); + if (!root_sii) { + err = -EINVAL; + goto out; + } + + /* Try to find this SECURITY descriptor in SII indexes */ + err = indx_find(indx, ni, root_sii, &security_id, sizeof(security_id), + NULL, &diff, (struct NTFS_DE **)&sii_e, fnd_sii); + if (err) + goto out; + + if (diff) + goto out; + + t32 = le32_to_cpu(sii_e->sec_hdr.size); + if (t32 < SIZEOF_SECURITY_HDR) { + err = -EINVAL; + goto out; + } + + if (t32 > SIZEOF_SECURITY_HDR + 0x10000) { + /* + * looks like too big security. 0x10000 - is arbitrary big number + */ + err = -EFBIG; + goto out; + } + + *size = t32 - SIZEOF_SECURITY_HDR; + + p = ntfs_malloc(*size); + if (!p) { + err = -ENOMEM; + goto out; + } + + err = ntfs_read_run_nb(sbi, &ni->file.run, + le64_to_cpu(sii_e->sec_hdr.off), &d_security, + sizeof(d_security), NULL); + if (err) + goto out; + + if (memcmp(&d_security, &sii_e->sec_hdr, SIZEOF_SECURITY_HDR)) { + err = -EINVAL; + goto out; + } + + err = ntfs_read_run_nb(sbi, &ni->file.run, + le64_to_cpu(sii_e->sec_hdr.off) + + SIZEOF_SECURITY_HDR, + p, *size, NULL); + if (err) + goto out; + + *sd = p; + p = NULL; + +out: + ntfs_free(p); + fnd_put(fnd_sii); + ni_unlock(ni); + + return err; +} + +/* + * ntfs_insert_security + * + * inserts security descriptor into $Secure::SDS + * + * SECURITY Descriptor Stream data is organized into chunks of 256K bytes + * and it contains a mirror copy of each security descriptor. When writing + * to a security descriptor at location X, another copy will be written at + * location (X+256K). + * When writing a security descriptor that will cross the 256K boundary, + * the pointer will be advanced by 256K to skip + * over the mirror portion. + */ +int ntfs_insert_security(struct ntfs_sb_info *sbi, + const struct SECURITY_DESCRIPTOR_RELATIVE *sd, + u32 size_sd, __le32 *security_id, bool *inserted) +{ + int err, diff; + struct ntfs_inode *ni = sbi->security.ni; + struct ntfs_index *indx_sdh = &sbi->security.index_sdh; + struct ntfs_index *indx_sii = &sbi->security.index_sii; + struct NTFS_DE_SDH *e; + struct NTFS_DE_SDH sdh_e; + struct NTFS_DE_SII sii_e; + struct SECURITY_HDR *d_security; + u32 new_sec_size = size_sd + SIZEOF_SECURITY_HDR; + u32 aligned_sec_size = Quad2Align(new_sec_size); + struct SECURITY_KEY hash_key; + struct ntfs_fnd *fnd_sdh = NULL; + const struct INDEX_ROOT *root_sdh; + const struct INDEX_ROOT *root_sii; + u64 mirr_off, new_sds_size; + u32 next, left; + + static_assert((1 << Log2OfSecurityDescriptorsBlockSize) == + SecurityDescriptorsBlockSize); + + hash_key.hash = security_hash(sd, size_sd); + hash_key.sec_id = SECURITY_ID_INVALID; + + if (inserted) + *inserted = false; + *security_id = SECURITY_ID_INVALID; + + /* Allocate a temporal buffer*/ + d_security = ntfs_zalloc(aligned_sec_size); + if (!d_security) + return -ENOMEM; + + mutex_lock_nested(&ni->ni_lock, NTFS_INODE_MUTEX_SECURITY); + + fnd_sdh = fnd_get(); + if (!fnd_sdh) { + err = -ENOMEM; + goto out; + } + + root_sdh = indx_get_root(indx_sdh, ni, NULL, NULL); + if (!root_sdh) { + err = -EINVAL; + goto out; + } + + root_sii = indx_get_root(indx_sii, ni, NULL, NULL); + if (!root_sii) { + err = -EINVAL; + goto out; + } + + /* + * Check if such security already exists + * use "SDH" and hash -> to get the offset in "SDS" + */ + err = indx_find(indx_sdh, ni, root_sdh, &hash_key, sizeof(hash_key), + &d_security->key.sec_id, &diff, (struct NTFS_DE **)&e, + fnd_sdh); + if (err) + goto out; + + while (e) { + if (le32_to_cpu(e->sec_hdr.size) == new_sec_size) { + err = ntfs_read_run_nb(sbi, &ni->file.run, + le64_to_cpu(e->sec_hdr.off), + d_security, new_sec_size, NULL); + if (err) + goto out; + + if (le32_to_cpu(d_security->size) == new_sec_size && + d_security->key.hash == hash_key.hash && + !memcmp(d_security + 1, sd, size_sd)) { + *security_id = d_security->key.sec_id; + /*such security already exists*/ + err = 0; + goto out; + } + } + + err = indx_find_sort(indx_sdh, ni, root_sdh, + (struct NTFS_DE **)&e, fnd_sdh); + if (err) + goto out; + + if (!e || e->key.hash != hash_key.hash) + break; + } + + /* Zero unused space */ + next = sbi->security.next_off & (SecurityDescriptorsBlockSize - 1); + left = SecurityDescriptorsBlockSize - next; + + /* Zero gap until SecurityDescriptorsBlockSize */ + if (left < new_sec_size) { + /* zero "left" bytes from sbi->security.next_off */ + sbi->security.next_off += SecurityDescriptorsBlockSize + left; + } + + /* Zero tail of previous security */ + //used = ni->vfs_inode.i_size & (SecurityDescriptorsBlockSize - 1); + + /* + * Example: + * 0x40438 == ni->vfs_inode.i_size + * 0x00440 == sbi->security.next_off + * need to zero [0x438-0x440) + * if (next > used) { + * u32 tozero = next - used; + * zero "tozero" bytes from sbi->security.next_off - tozero + */ + + /* format new security descriptor */ + d_security->key.hash = hash_key.hash; + d_security->key.sec_id = cpu_to_le32(sbi->security.next_id); + d_security->off = cpu_to_le64(sbi->security.next_off); + d_security->size = cpu_to_le32(new_sec_size); + memcpy(d_security + 1, sd, size_sd); + + /* Write main SDS bucket */ + err = ntfs_sb_write_run(sbi, &ni->file.run, sbi->security.next_off, + d_security, aligned_sec_size); + + if (err) + goto out; + + mirr_off = sbi->security.next_off + SecurityDescriptorsBlockSize; + new_sds_size = mirr_off + aligned_sec_size; + + if (new_sds_size > ni->vfs_inode.i_size) { + err = attr_set_size(ni, ATTR_DATA, SDS_NAME, + ARRAY_SIZE(SDS_NAME), &ni->file.run, + new_sds_size, &new_sds_size, false, NULL); + if (err) + goto out; + } + + /* Write copy SDS bucket */ + err = ntfs_sb_write_run(sbi, &ni->file.run, mirr_off, d_security, + aligned_sec_size); + if (err) + goto out; + + /* Fill SII entry */ + sii_e.de.view.data_off = + cpu_to_le16(offsetof(struct NTFS_DE_SII, sec_hdr)); + sii_e.de.view.data_size = cpu_to_le16(SIZEOF_SECURITY_HDR); + sii_e.de.view.res = 0; + sii_e.de.size = cpu_to_le16(SIZEOF_SII_DIRENTRY); + sii_e.de.key_size = cpu_to_le16(sizeof(d_security->key.sec_id)); + sii_e.de.flags = 0; + sii_e.de.res = 0; + sii_e.sec_id = d_security->key.sec_id; + memcpy(&sii_e.sec_hdr, d_security, SIZEOF_SECURITY_HDR); + + err = indx_insert_entry(indx_sii, ni, &sii_e.de, NULL, NULL); + if (err) + goto out; + + /* Fill SDH entry */ + sdh_e.de.view.data_off = + cpu_to_le16(offsetof(struct NTFS_DE_SDH, sec_hdr)); + sdh_e.de.view.data_size = cpu_to_le16(SIZEOF_SECURITY_HDR); + sdh_e.de.view.res = 0; + sdh_e.de.size = cpu_to_le16(SIZEOF_SDH_DIRENTRY); + sdh_e.de.key_size = cpu_to_le16(sizeof(sdh_e.key)); + sdh_e.de.flags = 0; + sdh_e.de.res = 0; + sdh_e.key.hash = d_security->key.hash; + sdh_e.key.sec_id = d_security->key.sec_id; + memcpy(&sdh_e.sec_hdr, d_security, SIZEOF_SECURITY_HDR); + sdh_e.magic[0] = cpu_to_le16('I'); + sdh_e.magic[1] = cpu_to_le16('I'); + + fnd_clear(fnd_sdh); + err = indx_insert_entry(indx_sdh, ni, &sdh_e.de, (void *)(size_t)1, + fnd_sdh); + if (err) + goto out; + + *security_id = d_security->key.sec_id; + if (inserted) + *inserted = true; + + /* Update Id and offset for next descriptor */ + sbi->security.next_id += 1; + sbi->security.next_off += aligned_sec_size; + +out: + fnd_put(fnd_sdh); + mark_inode_dirty(&ni->vfs_inode); + ni_unlock(ni); + ntfs_free(d_security); + + return err; +} + +/* + * ntfs_reparse_init + * + * loads and parse $Extend/$Reparse + */ +int ntfs_reparse_init(struct ntfs_sb_info *sbi) +{ + int err; + struct ntfs_inode *ni = sbi->reparse.ni; + struct ntfs_index *indx = &sbi->reparse.index_r; + struct ATTRIB *attr; + struct ATTR_LIST_ENTRY *le; + const struct INDEX_ROOT *root_r; + + if (!ni) + return 0; + + le = NULL; + attr = ni_find_attr(ni, NULL, &le, ATTR_ROOT, SR_NAME, + ARRAY_SIZE(SR_NAME), NULL, NULL); + if (!attr) { + err = -EINVAL; + goto out; + } + + root_r = resident_data(attr); + if (root_r->type != ATTR_ZERO || + root_r->rule != NTFS_COLLATION_TYPE_UINTS) { + err = -EINVAL; + goto out; + } + + err = indx_init(indx, sbi, attr, INDEX_MUTEX_SR); + if (err) + goto out; + +out: + return err; +} + +/* + * ntfs_objid_init + * + * loads and parse $Extend/$ObjId + */ +int ntfs_objid_init(struct ntfs_sb_info *sbi) +{ + int err; + struct ntfs_inode *ni = sbi->objid.ni; + struct ntfs_index *indx = &sbi->objid.index_o; + struct ATTRIB *attr; + struct ATTR_LIST_ENTRY *le; + const struct INDEX_ROOT *root; + + if (!ni) + return 0; + + le = NULL; + attr = ni_find_attr(ni, NULL, &le, ATTR_ROOT, SO_NAME, + ARRAY_SIZE(SO_NAME), NULL, NULL); + if (!attr) { + err = -EINVAL; + goto out; + } + + root = resident_data(attr); + if (root->type != ATTR_ZERO || + root->rule != NTFS_COLLATION_TYPE_UINTS) { + err = -EINVAL; + goto out; + } + + err = indx_init(indx, sbi, attr, INDEX_MUTEX_SO); + if (err) + goto out; + +out: + return err; +} + +int ntfs_objid_remove(struct ntfs_sb_info *sbi, struct GUID *guid) +{ + int err; + struct ntfs_inode *ni = sbi->objid.ni; + struct ntfs_index *indx = &sbi->objid.index_o; + + if (!ni) + return -EINVAL; + + mutex_lock_nested(&ni->ni_lock, NTFS_INODE_MUTEX_OBJID); + + err = indx_delete_entry(indx, ni, guid, sizeof(*guid), NULL); + + mark_inode_dirty(&ni->vfs_inode); + ni_unlock(ni); + + return err; +} + +int ntfs_insert_reparse(struct ntfs_sb_info *sbi, __le32 rtag, + const struct MFT_REF *ref) +{ + int err; + struct ntfs_inode *ni = sbi->reparse.ni; + struct ntfs_index *indx = &sbi->reparse.index_r; + struct NTFS_DE_R re; + + if (!ni) + return -EINVAL; + + memset(&re, 0, sizeof(re)); + + re.de.view.data_off = cpu_to_le16(offsetof(struct NTFS_DE_R, zero)); + re.de.size = cpu_to_le16(sizeof(struct NTFS_DE_R)); + re.de.key_size = cpu_to_le16(sizeof(re.key)); + + re.key.ReparseTag = rtag; + memcpy(&re.key.ref, ref, sizeof(*ref)); + + mutex_lock_nested(&ni->ni_lock, NTFS_INODE_MUTEX_REPARSE); + + err = indx_insert_entry(indx, ni, &re.de, NULL, NULL); + + mark_inode_dirty(&ni->vfs_inode); + ni_unlock(ni); + + return err; +} + +int ntfs_remove_reparse(struct ntfs_sb_info *sbi, __le32 rtag, + const struct MFT_REF *ref) +{ + int err, diff; + struct ntfs_inode *ni = sbi->reparse.ni; + struct ntfs_index *indx = &sbi->reparse.index_r; + struct ntfs_fnd *fnd = NULL; + struct REPARSE_KEY rkey; + struct NTFS_DE_R *re; + struct INDEX_ROOT *root_r; + + if (!ni) + return -EINVAL; + + rkey.ReparseTag = rtag; + rkey.ref = *ref; + + mutex_lock_nested(&ni->ni_lock, NTFS_INODE_MUTEX_REPARSE); + + if (rtag) { + err = indx_delete_entry(indx, ni, &rkey, sizeof(rkey), NULL); + goto out1; + } + + fnd = fnd_get(); + if (!fnd) { + err = -ENOMEM; + goto out1; + } + + root_r = indx_get_root(indx, ni, NULL, NULL); + if (!root_r) { + err = -EINVAL; + goto out; + } + + /* 1 - forces to ignore rkey.ReparseTag when comparing keys */ + err = indx_find(indx, ni, root_r, &rkey, sizeof(rkey), (void *)1, &diff, + (struct NTFS_DE **)&re, fnd); + if (err) + goto out; + + if (memcmp(&re->key.ref, ref, sizeof(*ref))) { + /* Impossible. Looks like volume corrupt?*/ + goto out; + } + + memcpy(&rkey, &re->key, sizeof(rkey)); + + fnd_put(fnd); + fnd = NULL; + + err = indx_delete_entry(indx, ni, &rkey, sizeof(rkey), NULL); + if (err) + goto out; + +out: + fnd_put(fnd); + +out1: + mark_inode_dirty(&ni->vfs_inode); + ni_unlock(ni); + + return err; +} + +static inline void ntfs_unmap_and_discard(struct ntfs_sb_info *sbi, CLST lcn, + CLST len) +{ + ntfs_unmap_meta(sbi->sb, lcn, len); + ntfs_discard(sbi, lcn, len); +} + +void mark_as_free_ex(struct ntfs_sb_info *sbi, CLST lcn, CLST len, bool trim) +{ + CLST end, i; + struct wnd_bitmap *wnd = &sbi->used.bitmap; + + down_write_nested(&wnd->rw_lock, BITMAP_MUTEX_CLUSTERS); + if (!wnd_is_used(wnd, lcn, len)) { + ntfs_set_state(sbi, NTFS_DIRTY_ERROR); + + end = lcn + len; + len = 0; + for (i = lcn; i < end; i++) { + if (wnd_is_used(wnd, i, 1)) { + if (!len) + lcn = i; + len += 1; + continue; + } + + if (!len) + continue; + + if (trim) + ntfs_unmap_and_discard(sbi, lcn, len); + + wnd_set_free(wnd, lcn, len); + len = 0; + } + + if (!len) + goto out; + } + + if (trim) + ntfs_unmap_and_discard(sbi, lcn, len); + wnd_set_free(wnd, lcn, len); + +out: + up_write(&wnd->rw_lock); +} + +/* + * run_deallocate + * + * deallocate clusters + */ +int run_deallocate(struct ntfs_sb_info *sbi, struct runs_tree *run, bool trim) +{ + CLST lcn, len; + size_t idx = 0; + + while (run_get_entry(run, idx++, NULL, &lcn, &len)) { + if (lcn == SPARSE_LCN) + continue; + + mark_as_free_ex(sbi, lcn, len, trim); + } + + return 0; +} diff --git a/fs/ntfs3/index.c b/fs/ntfs3/index.c new file mode 100644 index 000000000000..931a7241ef00 --- /dev/null +++ b/fs/ntfs3/index.c @@ -0,0 +1,2641 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + */ + +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +static const struct INDEX_NAMES { + const __le16 *name; + u8 name_len; +} s_index_names[INDEX_MUTEX_TOTAL] = { + { I30_NAME, ARRAY_SIZE(I30_NAME) }, { SII_NAME, ARRAY_SIZE(SII_NAME) }, + { SDH_NAME, ARRAY_SIZE(SDH_NAME) }, { SO_NAME, ARRAY_SIZE(SO_NAME) }, + { SQ_NAME, ARRAY_SIZE(SQ_NAME) }, { SR_NAME, ARRAY_SIZE(SR_NAME) }, +}; + +/* + * compare two names in index + * if l1 != 0 + * both names are little endian on-disk ATTR_FILE_NAME structs + * else + * key1 - cpu_str, key2 - ATTR_FILE_NAME + */ +static int cmp_fnames(const void *key1, size_t l1, const void *key2, size_t l2, + const void *data) +{ + const struct ATTR_FILE_NAME *f2 = key2; + const struct ntfs_sb_info *sbi = data; + const struct ATTR_FILE_NAME *f1; + u16 fsize2; + bool both_case; + + if (l2 <= offsetof(struct ATTR_FILE_NAME, name)) + return -1; + + fsize2 = fname_full_size(f2); + if (l2 < fsize2) + return -1; + + both_case = f2->type != FILE_NAME_DOS /*&& !sbi->options.nocase*/; + if (!l1) { + const struct le_str *s2 = (struct le_str *)&f2->name_len; + + /* + * If names are equal (case insensitive) + * try to compare it case sensitive + */ + return ntfs_cmp_names_cpu(key1, s2, sbi->upcase, both_case); + } + + f1 = key1; + return ntfs_cmp_names(f1->name, f1->name_len, f2->name, f2->name_len, + sbi->upcase, both_case); +} + +/* $SII of $Secure and $Q of Quota */ +static int cmp_uint(const void *key1, size_t l1, const void *key2, size_t l2, + const void *data) +{ + const u32 *k1 = key1; + const u32 *k2 = key2; + + if (l2 < sizeof(u32)) + return -1; + + if (*k1 < *k2) + return -1; + if (*k1 > *k2) + return 1; + return 0; +} + +/* $SDH of $Secure */ +static int cmp_sdh(const void *key1, size_t l1, const void *key2, size_t l2, + const void *data) +{ + const struct SECURITY_KEY *k1 = key1; + const struct SECURITY_KEY *k2 = key2; + u32 t1, t2; + + if (l2 < sizeof(struct SECURITY_KEY)) + return -1; + + t1 = le32_to_cpu(k1->hash); + t2 = le32_to_cpu(k2->hash); + + /* First value is a hash value itself */ + if (t1 < t2) + return -1; + if (t1 > t2) + return 1; + + /* Second value is security Id */ + if (data) { + t1 = le32_to_cpu(k1->sec_id); + t2 = le32_to_cpu(k2->sec_id); + if (t1 < t2) + return -1; + if (t1 > t2) + return 1; + } + + return 0; +} + +/* $O of ObjId and "$R" for Reparse */ +static int cmp_uints(const void *key1, size_t l1, const void *key2, size_t l2, + const void *data) +{ + const __le32 *k1 = key1; + const __le32 *k2 = key2; + size_t count; + + if ((size_t)data == 1) { + /* + * ni_delete_all -> ntfs_remove_reparse -> delete all with this reference + * k1, k2 - pointers to REPARSE_KEY + */ + + k1 += 1; // skip REPARSE_KEY.ReparseTag + k2 += 1; // skip REPARSE_KEY.ReparseTag + if (l2 <= sizeof(int)) + return -1; + l2 -= sizeof(int); + if (l1 <= sizeof(int)) + return 1; + l1 -= sizeof(int); + } + + if (l2 < sizeof(int)) + return -1; + + for (count = min(l1, l2) >> 2; count > 0; --count, ++k1, ++k2) { + u32 t1 = le32_to_cpu(*k1); + u32 t2 = le32_to_cpu(*k2); + + if (t1 > t2) + return 1; + if (t1 < t2) + return -1; + } + + if (l1 > l2) + return 1; + if (l1 < l2) + return -1; + + return 0; +} + +static inline NTFS_CMP_FUNC get_cmp_func(const struct INDEX_ROOT *root) +{ + switch (root->type) { + case ATTR_NAME: + if (root->rule == NTFS_COLLATION_TYPE_FILENAME) + return &cmp_fnames; + break; + case ATTR_ZERO: + switch (root->rule) { + case NTFS_COLLATION_TYPE_UINT: + return &cmp_uint; + case NTFS_COLLATION_TYPE_SECURITY_HASH: + return &cmp_sdh; + case NTFS_COLLATION_TYPE_UINTS: + return &cmp_uints; + default: + break; + } + default: + break; + } + + return NULL; +} + +struct bmp_buf { + struct ATTRIB *b; + struct mft_inode *mi; + struct buffer_head *bh; + ulong *buf; + size_t bit; + u32 nbits; + u64 new_valid; +}; + +static int bmp_buf_get(struct ntfs_index *indx, struct ntfs_inode *ni, + size_t bit, struct bmp_buf *bbuf) +{ + struct ATTRIB *b; + size_t data_size, valid_size, vbo, off = bit >> 3; + struct ntfs_sb_info *sbi = ni->mi.sbi; + CLST vcn = off >> sbi->cluster_bits; + struct ATTR_LIST_ENTRY *le = NULL; + struct buffer_head *bh; + struct super_block *sb; + u32 blocksize; + const struct INDEX_NAMES *in = &s_index_names[indx->type]; + + bbuf->bh = NULL; + + b = ni_find_attr(ni, NULL, &le, ATTR_BITMAP, in->name, in->name_len, + &vcn, &bbuf->mi); + bbuf->b = b; + if (!b) + return -EINVAL; + + if (!b->non_res) { + data_size = le32_to_cpu(b->res.data_size); + + if (off >= data_size) + return -EINVAL; + + bbuf->buf = (ulong *)resident_data(b); + bbuf->bit = 0; + bbuf->nbits = data_size * 8; + + return 0; + } + + data_size = le64_to_cpu(b->nres.data_size); + if (WARN_ON(off >= data_size)) { + /* looks like filesystem error */ + return -EINVAL; + } + + valid_size = le64_to_cpu(b->nres.valid_size); + + bh = ntfs_bread_run(sbi, &indx->bitmap_run, off); + if (!bh) + return -EIO; + + if (IS_ERR(bh)) + return PTR_ERR(bh); + + bbuf->bh = bh; + + if (buffer_locked(bh)) + __wait_on_buffer(bh); + + lock_buffer(bh); + + sb = sbi->sb; + blocksize = sb->s_blocksize; + + vbo = off & ~(size_t)sbi->block_mask; + + bbuf->new_valid = vbo + blocksize; + if (bbuf->new_valid <= valid_size) + bbuf->new_valid = 0; + else if (bbuf->new_valid > data_size) + bbuf->new_valid = data_size; + + if (vbo >= valid_size) { + memset(bh->b_data, 0, blocksize); + } else if (vbo + blocksize > valid_size) { + u32 voff = valid_size & sbi->block_mask; + + memset(bh->b_data + voff, 0, blocksize - voff); + } + + bbuf->buf = (ulong *)bh->b_data; + bbuf->bit = 8 * (off & ~(size_t)sbi->block_mask); + bbuf->nbits = 8 * blocksize; + + return 0; +} + +static void bmp_buf_put(struct bmp_buf *bbuf, bool dirty) +{ + struct buffer_head *bh = bbuf->bh; + struct ATTRIB *b = bbuf->b; + + if (!bh) { + if (b && !b->non_res && dirty) + bbuf->mi->dirty = true; + return; + } + + if (!dirty) + goto out; + + if (bbuf->new_valid) { + b->nres.valid_size = cpu_to_le64(bbuf->new_valid); + bbuf->mi->dirty = true; + } + + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); + +out: + unlock_buffer(bh); + put_bh(bh); +} + +/* + * indx_mark_used + * + * marks the bit 'bit' as used + */ +static int indx_mark_used(struct ntfs_index *indx, struct ntfs_inode *ni, + size_t bit) +{ + int err; + struct bmp_buf bbuf; + + err = bmp_buf_get(indx, ni, bit, &bbuf); + if (err) + return err; + + __set_bit(bit - bbuf.bit, bbuf.buf); + + bmp_buf_put(&bbuf, true); + + return 0; +} + +/* + * indx_mark_free + * + * the bit 'bit' as free + */ +static int indx_mark_free(struct ntfs_index *indx, struct ntfs_inode *ni, + size_t bit) +{ + int err; + struct bmp_buf bbuf; + + err = bmp_buf_get(indx, ni, bit, &bbuf); + if (err) + return err; + + __clear_bit(bit - bbuf.bit, bbuf.buf); + + bmp_buf_put(&bbuf, true); + + return 0; +} + +/* + * if ntfs_readdir calls this function (indx_used_bit -> scan_nres_bitmap), + * inode is shared locked and no ni_lock + * use rw_semaphore for read/write access to bitmap_run + */ +static int scan_nres_bitmap(struct ntfs_inode *ni, struct ATTRIB *bitmap, + struct ntfs_index *indx, size_t from, + bool (*fn)(const ulong *buf, u32 bit, u32 bits, + size_t *ret), + size_t *ret) +{ + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct super_block *sb = sbi->sb; + struct runs_tree *run = &indx->bitmap_run; + struct rw_semaphore *lock = &indx->run_lock; + u32 nbits = sb->s_blocksize * 8; + u32 blocksize = sb->s_blocksize; + u64 valid_size = le64_to_cpu(bitmap->nres.valid_size); + u64 data_size = le64_to_cpu(bitmap->nres.data_size); + sector_t eblock = bytes_to_block(sb, data_size); + size_t vbo = from >> 3; + sector_t blk = (vbo & sbi->cluster_mask) >> sb->s_blocksize_bits; + sector_t vblock = vbo >> sb->s_blocksize_bits; + sector_t blen, block; + CLST lcn, clen, vcn, vcn_next; + size_t idx; + struct buffer_head *bh; + bool ok; + + *ret = MINUS_ONE_T; + + if (vblock >= eblock) + return 0; + + from &= nbits - 1; + vcn = vbo >> sbi->cluster_bits; + + down_read(lock); + ok = run_lookup_entry(run, vcn, &lcn, &clen, &idx); + up_read(lock); + +next_run: + if (!ok) { + int err; + const struct INDEX_NAMES *name = &s_index_names[indx->type]; + + down_write(lock); + err = attr_load_runs_vcn(ni, ATTR_BITMAP, name->name, + name->name_len, run, vcn); + up_write(lock); + if (err) + return err; + down_read(lock); + ok = run_lookup_entry(run, vcn, &lcn, &clen, &idx); + up_read(lock); + if (!ok) + return -EINVAL; + } + + blen = (sector_t)clen * sbi->blocks_per_cluster; + block = (sector_t)lcn * sbi->blocks_per_cluster; + + for (; blk < blen; blk++, from = 0) { + bh = ntfs_bread(sb, block + blk); + if (!bh) + return -EIO; + + vbo = (u64)vblock << sb->s_blocksize_bits; + if (vbo >= valid_size) { + memset(bh->b_data, 0, blocksize); + } else if (vbo + blocksize > valid_size) { + u32 voff = valid_size & sbi->block_mask; + + memset(bh->b_data + voff, 0, blocksize - voff); + } + + if (vbo + blocksize > data_size) + nbits = 8 * (data_size - vbo); + + ok = nbits > from ? (*fn)((ulong *)bh->b_data, from, nbits, ret) + : false; + put_bh(bh); + + if (ok) { + *ret += 8 * vbo; + return 0; + } + + if (++vblock >= eblock) { + *ret = MINUS_ONE_T; + return 0; + } + } + blk = 0; + vcn_next = vcn + clen; + down_read(lock); + ok = run_get_entry(run, ++idx, &vcn, &lcn, &clen) && vcn == vcn_next; + if (!ok) + vcn = vcn_next; + up_read(lock); + goto next_run; +} + +static bool scan_for_free(const ulong *buf, u32 bit, u32 bits, size_t *ret) +{ + size_t pos = find_next_zero_bit(buf, bits, bit); + + if (pos >= bits) + return false; + *ret = pos; + return true; +} + +/* + * indx_find_free + * + * looks for free bit + * returns -1 if no free bits + */ +static int indx_find_free(struct ntfs_index *indx, struct ntfs_inode *ni, + size_t *bit, struct ATTRIB **bitmap) +{ + struct ATTRIB *b; + struct ATTR_LIST_ENTRY *le = NULL; + const struct INDEX_NAMES *in = &s_index_names[indx->type]; + int err; + + b = ni_find_attr(ni, NULL, &le, ATTR_BITMAP, in->name, in->name_len, + NULL, NULL); + + if (!b) + return -ENOENT; + + *bitmap = b; + *bit = MINUS_ONE_T; + + if (!b->non_res) { + u32 nbits = 8 * le32_to_cpu(b->res.data_size); + size_t pos = find_next_zero_bit(resident_data(b), nbits, 0); + + if (pos < nbits) + *bit = pos; + } else { + err = scan_nres_bitmap(ni, b, indx, 0, &scan_for_free, bit); + + if (err) + return err; + } + + return 0; +} + +static bool scan_for_used(const ulong *buf, u32 bit, u32 bits, size_t *ret) +{ + size_t pos = find_next_bit(buf, bits, bit); + + if (pos >= bits) + return false; + *ret = pos; + return true; +} + +/* + * indx_used_bit + * + * looks for used bit + * returns MINUS_ONE_T if no used bits + */ +int indx_used_bit(struct ntfs_index *indx, struct ntfs_inode *ni, size_t *bit) +{ + struct ATTRIB *b; + struct ATTR_LIST_ENTRY *le = NULL; + size_t from = *bit; + const struct INDEX_NAMES *in = &s_index_names[indx->type]; + int err; + + b = ni_find_attr(ni, NULL, &le, ATTR_BITMAP, in->name, in->name_len, + NULL, NULL); + + if (!b) + return -ENOENT; + + *bit = MINUS_ONE_T; + + if (!b->non_res) { + u32 nbits = le32_to_cpu(b->res.data_size) * 8; + size_t pos = find_next_bit(resident_data(b), nbits, from); + + if (pos < nbits) + *bit = pos; + } else { + err = scan_nres_bitmap(ni, b, indx, from, &scan_for_used, bit); + if (err) + return err; + } + + return 0; +} + +/* + * hdr_find_split + * + * finds a point at which the index allocation buffer would like to + * be split. + * NOTE: This function should never return 'END' entry NULL returns on error + */ +static const struct NTFS_DE *hdr_find_split(const struct INDEX_HDR *hdr) +{ + size_t o; + const struct NTFS_DE *e = hdr_first_de(hdr); + u32 used_2 = le32_to_cpu(hdr->used) >> 1; + u16 esize = le16_to_cpu(e->size); + + if (!e || de_is_last(e)) + return NULL; + + for (o = le32_to_cpu(hdr->de_off) + esize; o < used_2; o += esize) { + const struct NTFS_DE *p = e; + + e = Add2Ptr(hdr, o); + + /* We must not return END entry */ + if (de_is_last(e)) + return p; + + esize = le16_to_cpu(e->size); + } + + return e; +} + +/* + * hdr_insert_head + * + * inserts some entries at the beginning of the buffer. + * It is used to insert entries into a newly-created buffer. + */ +static const struct NTFS_DE *hdr_insert_head(struct INDEX_HDR *hdr, + const void *ins, u32 ins_bytes) +{ + u32 to_move; + struct NTFS_DE *e = hdr_first_de(hdr); + u32 used = le32_to_cpu(hdr->used); + + if (!e) + return NULL; + + /* Now we just make room for the inserted entries and jam it in. */ + to_move = used - le32_to_cpu(hdr->de_off); + memmove(Add2Ptr(e, ins_bytes), e, to_move); + memcpy(e, ins, ins_bytes); + hdr->used = cpu_to_le32(used + ins_bytes); + + return e; +} + +void fnd_clear(struct ntfs_fnd *fnd) +{ + int i; + + for (i = 0; i < fnd->level; i++) { + struct indx_node *n = fnd->nodes[i]; + + if (!n) + continue; + + put_indx_node(n); + fnd->nodes[i] = NULL; + } + fnd->level = 0; + fnd->root_de = NULL; +} + +static int fnd_push(struct ntfs_fnd *fnd, struct indx_node *n, + struct NTFS_DE *e) +{ + int i; + + i = fnd->level; + if (i < 0 || i >= ARRAY_SIZE(fnd->nodes)) + return -EINVAL; + fnd->nodes[i] = n; + fnd->de[i] = e; + fnd->level += 1; + return 0; +} + +static struct indx_node *fnd_pop(struct ntfs_fnd *fnd) +{ + struct indx_node *n; + int i = fnd->level; + + i -= 1; + n = fnd->nodes[i]; + fnd->nodes[i] = NULL; + fnd->level = i; + + return n; +} + +static bool fnd_is_empty(struct ntfs_fnd *fnd) +{ + if (!fnd->level) + return !fnd->root_de; + + return !fnd->de[fnd->level - 1]; +} + +/* + * hdr_find_e + * + * locates an entry the index buffer. + * If no matching entry is found, it returns the first entry which is greater + * than the desired entry If the search key is greater than all the entries the + * buffer, it returns the 'end' entry. This function does a binary search of the + * current index buffer, for the first entry that is <= to the search value + * Returns NULL if error + */ +static struct NTFS_DE *hdr_find_e(const struct ntfs_index *indx, + const struct INDEX_HDR *hdr, const void *key, + size_t key_len, const void *ctx, int *diff) +{ + struct NTFS_DE *e; + NTFS_CMP_FUNC cmp = indx->cmp; + u32 e_size, e_key_len; + u32 end = le32_to_cpu(hdr->used); + u32 off = le32_to_cpu(hdr->de_off); + +#ifdef NTFS3_INDEX_BINARY_SEARCH + int max_idx = 0, fnd, min_idx; + int nslots = 64; + u16 *offs; + + if (end > 0x10000) + goto next; + + offs = ntfs_malloc(sizeof(u16) * nslots); + if (!offs) + goto next; + + /* use binary search algorithm */ +next1: + if (off + sizeof(struct NTFS_DE) > end) { + e = NULL; + goto out1; + } + e = Add2Ptr(hdr, off); + e_size = le16_to_cpu(e->size); + + if (e_size < sizeof(struct NTFS_DE) || off + e_size > end) { + e = NULL; + goto out1; + } + + if (max_idx >= nslots) { + u16 *ptr; + int new_slots = QuadAlign(2 * nslots); + + ptr = ntfs_malloc(sizeof(u16) * new_slots); + if (ptr) + memcpy(ptr, offs, sizeof(u16) * max_idx); + ntfs_free(offs); + offs = ptr; + nslots = new_slots; + if (!ptr) + goto next; + } + + /* Store entry table */ + offs[max_idx] = off; + + if (!de_is_last(e)) { + off += e_size; + max_idx += 1; + goto next1; + } + + /* + * Table of pointers is created + * Use binary search to find entry that is <= to the search value + */ + fnd = -1; + min_idx = 0; + + while (min_idx <= max_idx) { + int mid_idx = min_idx + ((max_idx - min_idx) >> 1); + int diff2; + + e = Add2Ptr(hdr, offs[mid_idx]); + + e_key_len = le16_to_cpu(e->key_size); + + diff2 = (*cmp)(key, key_len, e + 1, e_key_len, ctx); + + if (!diff2) { + *diff = 0; + goto out1; + } + + if (diff2 < 0) { + max_idx = mid_idx - 1; + fnd = mid_idx; + if (!fnd) + break; + } else { + min_idx = mid_idx + 1; + } + } + + if (fnd == -1) { + e = NULL; + goto out1; + } + + *diff = -1; + e = Add2Ptr(hdr, offs[fnd]); + +out1: + ntfs_free(offs); + + return e; +#endif + +next: + /* + * Entries index are sorted + * Enumerate all entries until we find entry that is <= to the search value + */ + if (off + sizeof(struct NTFS_DE) > end) + return NULL; + + e = Add2Ptr(hdr, off); + e_size = le16_to_cpu(e->size); + + if (e_size < sizeof(struct NTFS_DE) || off + e_size > end) + return NULL; + + off += e_size; + + e_key_len = le16_to_cpu(e->key_size); + + *diff = (*cmp)(key, key_len, e + 1, e_key_len, ctx); + if (!*diff) + return e; + + if (*diff <= 0) + return e; + + if (de_is_last(e)) { + *diff = 1; + return e; + } + goto next; +} + +/* + * hdr_insert_de + * + * inserts an index entry into the buffer. + * 'before' should be a pointer previously returned from hdr_find_e + */ +static struct NTFS_DE *hdr_insert_de(const struct ntfs_index *indx, + struct INDEX_HDR *hdr, + const struct NTFS_DE *de, + struct NTFS_DE *before, const void *ctx) +{ + int diff; + size_t off = PtrOffset(hdr, before); + u32 used = le32_to_cpu(hdr->used); + u32 total = le32_to_cpu(hdr->total); + u16 de_size = le16_to_cpu(de->size); + + /* First, check to see if there's enough room */ + if (used + de_size > total) + return NULL; + + /* We know there's enough space, so we know we'll succeed. */ + if (before) { + /* Check that before is inside Index */ + if (off >= used || off < le32_to_cpu(hdr->de_off) || + off + le16_to_cpu(before->size) > total) { + return NULL; + } + goto ok; + } + /* No insert point is applied. Get it manually */ + before = hdr_find_e(indx, hdr, de + 1, le16_to_cpu(de->key_size), ctx, + &diff); + if (!before) + return NULL; + off = PtrOffset(hdr, before); + +ok: + /* Now we just make room for the entry and jam it in. */ + memmove(Add2Ptr(before, de_size), before, used - off); + + hdr->used = cpu_to_le32(used + de_size); + memcpy(before, de, de_size); + + return before; +} + +/* + * hdr_delete_de + * + * removes an entry from the index buffer + */ +static inline struct NTFS_DE *hdr_delete_de(struct INDEX_HDR *hdr, + struct NTFS_DE *re) +{ + u32 used = le32_to_cpu(hdr->used); + u16 esize = le16_to_cpu(re->size); + u32 off = PtrOffset(hdr, re); + int bytes = used - (off + esize); + + if (off >= used || esize < sizeof(struct NTFS_DE) || + bytes < sizeof(struct NTFS_DE)) + return NULL; + + hdr->used = cpu_to_le32(used - esize); + memmove(re, Add2Ptr(re, esize), bytes); + + return re; +} + +void indx_clear(struct ntfs_index *indx) +{ + run_close(&indx->alloc_run); + run_close(&indx->bitmap_run); +} + +int indx_init(struct ntfs_index *indx, struct ntfs_sb_info *sbi, + const struct ATTRIB *attr, enum index_mutex_classed type) +{ + u32 t32; + const struct INDEX_ROOT *root = resident_data(attr); + + /* Check root fields */ + if (!root->index_block_clst) + return -EINVAL; + + indx->type = type; + indx->idx2vbn_bits = __ffs(root->index_block_clst); + + t32 = le32_to_cpu(root->index_block_size); + indx->index_bits = blksize_bits(t32); + + /* Check index record size */ + if (t32 < sbi->cluster_size) { + /* index record is smaller than a cluster, use 512 blocks */ + if (t32 != root->index_block_clst * SECTOR_SIZE) + return -EINVAL; + + /* Check alignment to a cluster */ + if ((sbi->cluster_size >> SECTOR_SHIFT) & + (root->index_block_clst - 1)) { + return -EINVAL; + } + + indx->vbn2vbo_bits = SECTOR_SHIFT; + } else { + /* index record must be a multiple of cluster size */ + if (t32 != root->index_block_clst << sbi->cluster_bits) + return -EINVAL; + + indx->vbn2vbo_bits = sbi->cluster_bits; + } + + init_rwsem(&indx->run_lock); + + indx->cmp = get_cmp_func(root); + return indx->cmp ? 0 : -EINVAL; +} + +static struct indx_node *indx_new(struct ntfs_index *indx, + struct ntfs_inode *ni, CLST vbn, + const __le64 *sub_vbn) +{ + int err; + struct NTFS_DE *e; + struct indx_node *r; + struct INDEX_HDR *hdr; + struct INDEX_BUFFER *index; + u64 vbo = (u64)vbn << indx->vbn2vbo_bits; + u32 bytes = 1u << indx->index_bits; + u16 fn; + u32 eo; + + r = ntfs_zalloc(sizeof(struct indx_node)); + if (!r) + return ERR_PTR(-ENOMEM); + + index = ntfs_zalloc(bytes); + if (!index) { + ntfs_free(r); + return ERR_PTR(-ENOMEM); + } + + err = ntfs_get_bh(ni->mi.sbi, &indx->alloc_run, vbo, bytes, &r->nb); + + if (err) { + ntfs_free(index); + ntfs_free(r); + return ERR_PTR(err); + } + + /* Create header */ + index->rhdr.sign = NTFS_INDX_SIGNATURE; + index->rhdr.fix_off = cpu_to_le16(sizeof(struct INDEX_BUFFER)); // 0x28 + fn = (bytes >> SECTOR_SHIFT) + 1; // 9 + index->rhdr.fix_num = cpu_to_le16(fn); + index->vbn = cpu_to_le64(vbn); + hdr = &index->ihdr; + eo = QuadAlign(sizeof(struct INDEX_BUFFER) + fn * sizeof(short)); + hdr->de_off = cpu_to_le32(eo); + + e = Add2Ptr(hdr, eo); + + if (sub_vbn) { + e->flags = NTFS_IE_LAST | NTFS_IE_HAS_SUBNODES; + e->size = cpu_to_le16(sizeof(struct NTFS_DE) + sizeof(u64)); + hdr->used = + cpu_to_le32(eo + sizeof(struct NTFS_DE) + sizeof(u64)); + de_set_vbn_le(e, *sub_vbn); + hdr->flags = 1; + } else { + e->size = cpu_to_le16(sizeof(struct NTFS_DE)); + hdr->used = cpu_to_le32(eo + sizeof(struct NTFS_DE)); + e->flags = NTFS_IE_LAST; + } + + hdr->total = cpu_to_le32(bytes - offsetof(struct INDEX_BUFFER, ihdr)); + + r->index = index; + return r; +} + +struct INDEX_ROOT *indx_get_root(struct ntfs_index *indx, struct ntfs_inode *ni, + struct ATTRIB **attr, struct mft_inode **mi) +{ + struct ATTR_LIST_ENTRY *le = NULL; + struct ATTRIB *a; + const struct INDEX_NAMES *in = &s_index_names[indx->type]; + + a = ni_find_attr(ni, NULL, &le, ATTR_ROOT, in->name, in->name_len, NULL, + mi); + if (!a) + return NULL; + + if (attr) + *attr = a; + + return resident_data_ex(a, sizeof(struct INDEX_ROOT)); +} + +static int indx_write(struct ntfs_index *indx, struct ntfs_inode *ni, + struct indx_node *node, int sync) +{ + struct INDEX_BUFFER *ib = node->index; + + return ntfs_write_bh(ni->mi.sbi, &ib->rhdr, &node->nb, sync); +} + +/* + * if ntfs_readdir calls this function + * inode is shared locked and no ni_lock + * use rw_semaphore for read/write access to alloc_run + */ +int indx_read(struct ntfs_index *indx, struct ntfs_inode *ni, CLST vbn, + struct indx_node **node) +{ + int err; + struct INDEX_BUFFER *ib; + struct runs_tree *run = &indx->alloc_run; + struct rw_semaphore *lock = &indx->run_lock; + u64 vbo = (u64)vbn << indx->vbn2vbo_bits; + u32 bytes = 1u << indx->index_bits; + struct indx_node *in = *node; + const struct INDEX_NAMES *name; + + if (!in) { + in = ntfs_zalloc(sizeof(struct indx_node)); + if (!in) + return -ENOMEM; + } else { + nb_put(&in->nb); + } + + ib = in->index; + if (!ib) { + ib = ntfs_malloc(bytes); + if (!ib) { + err = -ENOMEM; + goto out; + } + } + + down_read(lock); + err = ntfs_read_bh(ni->mi.sbi, run, vbo, &ib->rhdr, bytes, &in->nb); + up_read(lock); + if (!err) + goto ok; + + if (err == -E_NTFS_FIXUP) + goto ok; + + if (err != -ENOENT) + goto out; + + name = &s_index_names[indx->type]; + down_write(lock); + err = attr_load_runs_range(ni, ATTR_ALLOC, name->name, name->name_len, + run, vbo, vbo + bytes); + up_write(lock); + if (err) + goto out; + + down_read(lock); + err = ntfs_read_bh(ni->mi.sbi, run, vbo, &ib->rhdr, bytes, &in->nb); + up_read(lock); + if (err == -E_NTFS_FIXUP) + goto ok; + + if (err) + goto out; + +ok: + if (err == -E_NTFS_FIXUP) { + ntfs_write_bh(ni->mi.sbi, &ib->rhdr, &in->nb, 0); + err = 0; + } + + in->index = ib; + *node = in; + +out: + if (ib != in->index) + ntfs_free(ib); + + if (*node != in) { + nb_put(&in->nb); + ntfs_free(in); + } + + return err; +} + +/* + * indx_find + * + * scans NTFS directory for given entry + */ +int indx_find(struct ntfs_index *indx, struct ntfs_inode *ni, + const struct INDEX_ROOT *root, const void *key, size_t key_len, + const void *ctx, int *diff, struct NTFS_DE **entry, + struct ntfs_fnd *fnd) +{ + int err; + struct NTFS_DE *e; + const struct INDEX_HDR *hdr; + struct indx_node *node; + + if (!root) + root = indx_get_root(&ni->dir, ni, NULL, NULL); + + if (!root) { + err = -EINVAL; + goto out; + } + + hdr = &root->ihdr; + + /* Check cache */ + e = fnd->level ? fnd->de[fnd->level - 1] : fnd->root_de; + if (e && !de_is_last(e) && + !(*indx->cmp)(key, key_len, e + 1, le16_to_cpu(e->key_size), ctx)) { + *entry = e; + *diff = 0; + return 0; + } + + /* Soft finder reset */ + fnd_clear(fnd); + + /* Lookup entry that is <= to the search value */ + e = hdr_find_e(indx, hdr, key, key_len, ctx, diff); + if (!e) + return -EINVAL; + + if (fnd) + fnd->root_de = e; + + err = 0; + + for (;;) { + node = NULL; + if (*diff >= 0 || !de_has_vcn_ex(e)) { + *entry = e; + goto out; + } + + /* Read next level. */ + err = indx_read(indx, ni, de_get_vbn(e), &node); + if (err) + goto out; + + /* Lookup entry that is <= to the search value */ + e = hdr_find_e(indx, &node->index->ihdr, key, key_len, ctx, + diff); + if (!e) { + err = -EINVAL; + put_indx_node(node); + goto out; + } + + fnd_push(fnd, node, e); + } + +out: + return err; +} + +int indx_find_sort(struct ntfs_index *indx, struct ntfs_inode *ni, + const struct INDEX_ROOT *root, struct NTFS_DE **entry, + struct ntfs_fnd *fnd) +{ + int err; + struct indx_node *n = NULL; + struct NTFS_DE *e; + size_t iter = 0; + int level = fnd->level; + + if (!*entry) { + /* Start find */ + e = hdr_first_de(&root->ihdr); + if (!e) + return 0; + fnd_clear(fnd); + fnd->root_de = e; + } else if (!level) { + if (de_is_last(fnd->root_de)) { + *entry = NULL; + return 0; + } + + e = hdr_next_de(&root->ihdr, fnd->root_de); + if (!e) + return -EINVAL; + fnd->root_de = e; + } else { + n = fnd->nodes[level - 1]; + e = fnd->de[level - 1]; + + if (de_is_last(e)) + goto pop_level; + + e = hdr_next_de(&n->index->ihdr, e); + if (!e) + return -EINVAL; + + fnd->de[level - 1] = e; + } + + /* Just to avoid tree cycle */ +next_iter: + if (iter++ >= 1000) + return -EINVAL; + + while (de_has_vcn_ex(e)) { + if (le16_to_cpu(e->size) < + sizeof(struct NTFS_DE) + sizeof(u64)) { + if (n) { + fnd_pop(fnd); + ntfs_free(n); + } + return -EINVAL; + } + + /* Read next level */ + err = indx_read(indx, ni, de_get_vbn(e), &n); + if (err) + return err; + + /* Try next level */ + e = hdr_first_de(&n->index->ihdr); + if (!e) { + ntfs_free(n); + return -EINVAL; + } + + fnd_push(fnd, n, e); + } + + if (le16_to_cpu(e->size) > sizeof(struct NTFS_DE)) { + *entry = e; + return 0; + } + +pop_level: + for (;;) { + if (!de_is_last(e)) + goto next_iter; + + /* Pop one level */ + if (n) { + fnd_pop(fnd); + ntfs_free(n); + } + + level = fnd->level; + + if (level) { + n = fnd->nodes[level - 1]; + e = fnd->de[level - 1]; + } else if (fnd->root_de) { + n = NULL; + e = fnd->root_de; + fnd->root_de = NULL; + } else { + *entry = NULL; + return 0; + } + + if (le16_to_cpu(e->size) > sizeof(struct NTFS_DE)) { + *entry = e; + if (!fnd->root_de) + fnd->root_de = e; + return 0; + } + } +} + +int indx_find_raw(struct ntfs_index *indx, struct ntfs_inode *ni, + const struct INDEX_ROOT *root, struct NTFS_DE **entry, + size_t *off, struct ntfs_fnd *fnd) +{ + int err; + struct indx_node *n = NULL; + struct NTFS_DE *e = NULL; + struct NTFS_DE *e2; + size_t bit; + CLST next_used_vbn; + CLST next_vbn; + u32 record_size = ni->mi.sbi->record_size; + + /* Use non sorted algorithm */ + if (!*entry) { + /* This is the first call */ + e = hdr_first_de(&root->ihdr); + if (!e) + return 0; + fnd_clear(fnd); + fnd->root_de = e; + + /* The first call with setup of initial element */ + if (*off >= record_size) { + next_vbn = (((*off - record_size) >> indx->index_bits)) + << indx->idx2vbn_bits; + /* jump inside cycle 'for'*/ + goto next; + } + + /* Start enumeration from root */ + *off = 0; + } else if (!fnd->root_de) + return -EINVAL; + + for (;;) { + /* Check if current entry can be used */ + if (e && le16_to_cpu(e->size) > sizeof(struct NTFS_DE)) + goto ok; + + if (!fnd->level) { + /* Continue to enumerate root */ + if (!de_is_last(fnd->root_de)) { + e = hdr_next_de(&root->ihdr, fnd->root_de); + if (!e) + return -EINVAL; + fnd->root_de = e; + continue; + } + + /* Start to enumerate indexes from 0 */ + next_vbn = 0; + } else { + /* Continue to enumerate indexes */ + e2 = fnd->de[fnd->level - 1]; + + n = fnd->nodes[fnd->level - 1]; + + if (!de_is_last(e2)) { + e = hdr_next_de(&n->index->ihdr, e2); + if (!e) + return -EINVAL; + fnd->de[fnd->level - 1] = e; + continue; + } + + /* Continue with next index */ + next_vbn = le64_to_cpu(n->index->vbn) + + root->index_block_clst; + } + +next: + /* Release current index */ + if (n) { + fnd_pop(fnd); + put_indx_node(n); + n = NULL; + } + + /* Skip all free indexes */ + bit = next_vbn >> indx->idx2vbn_bits; + err = indx_used_bit(indx, ni, &bit); + if (err == -ENOENT || bit == MINUS_ONE_T) { + /* No used indexes */ + *entry = NULL; + return 0; + } + + next_used_vbn = bit << indx->idx2vbn_bits; + + /* Read buffer into memory */ + err = indx_read(indx, ni, next_used_vbn, &n); + if (err) + return err; + + e = hdr_first_de(&n->index->ihdr); + fnd_push(fnd, n, e); + if (!e) + return -EINVAL; + } + +ok: + /* return offset to restore enumerator if necessary */ + if (!n) { + /* 'e' points in root */ + *off = PtrOffset(&root->ihdr, e); + } else { + /* 'e' points in index */ + *off = (le64_to_cpu(n->index->vbn) << indx->vbn2vbo_bits) + + record_size + PtrOffset(&n->index->ihdr, e); + } + + *entry = e; + return 0; +} + +/* + * indx_create_allocate + * + * create "Allocation + Bitmap" attributes + */ +static int indx_create_allocate(struct ntfs_index *indx, struct ntfs_inode *ni, + CLST *vbn) +{ + int err = -ENOMEM; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct ATTRIB *bitmap; + struct ATTRIB *alloc; + u32 data_size = 1u << indx->index_bits; + u32 alloc_size = ntfs_up_cluster(sbi, data_size); + CLST len = alloc_size >> sbi->cluster_bits; + const struct INDEX_NAMES *in = &s_index_names[indx->type]; + CLST alen; + struct runs_tree run; + + run_init(&run); + + err = attr_allocate_clusters(sbi, &run, 0, 0, len, NULL, 0, &alen, 0, + NULL); + if (err) + goto out; + + err = ni_insert_nonresident(ni, ATTR_ALLOC, in->name, in->name_len, + &run, 0, len, 0, &alloc, NULL); + if (err) + goto out1; + + alloc->nres.valid_size = alloc->nres.data_size = cpu_to_le64(data_size); + + err = ni_insert_resident(ni, bitmap_size(1), ATTR_BITMAP, in->name, + in->name_len, &bitmap, NULL); + if (err) + goto out2; + + if (in->name == I30_NAME) { + ni->vfs_inode.i_size = data_size; + inode_set_bytes(&ni->vfs_inode, alloc_size); + } + + memcpy(&indx->alloc_run, &run, sizeof(run)); + + *vbn = 0; + + return 0; + +out2: + mi_remove_attr(&ni->mi, alloc); + +out1: + run_deallocate(sbi, &run, false); + +out: + return err; +} + +/* + * indx_add_allocate + * + * add clusters to index + */ +static int indx_add_allocate(struct ntfs_index *indx, struct ntfs_inode *ni, + CLST *vbn) +{ + int err; + size_t bit; + u64 data_size; + u64 bmp_size, bmp_size_v; + struct ATTRIB *bmp, *alloc; + struct mft_inode *mi; + const struct INDEX_NAMES *in = &s_index_names[indx->type]; + + err = indx_find_free(indx, ni, &bit, &bmp); + if (err) + goto out1; + + if (bit != MINUS_ONE_T) { + bmp = NULL; + } else { + if (bmp->non_res) { + bmp_size = le64_to_cpu(bmp->nres.data_size); + bmp_size_v = le64_to_cpu(bmp->nres.valid_size); + } else { + bmp_size = bmp_size_v = le32_to_cpu(bmp->res.data_size); + } + + bit = bmp_size << 3; + } + + data_size = (u64)(bit + 1) << indx->index_bits; + + if (bmp) { + /* Increase bitmap */ + err = attr_set_size(ni, ATTR_BITMAP, in->name, in->name_len, + &indx->bitmap_run, bitmap_size(bit + 1), + NULL, true, NULL); + if (err) + goto out1; + } + + alloc = ni_find_attr(ni, NULL, NULL, ATTR_ALLOC, in->name, in->name_len, + NULL, &mi); + if (!alloc) { + if (bmp) + goto out2; + goto out1; + } + + /* Increase allocation */ + err = attr_set_size(ni, ATTR_ALLOC, in->name, in->name_len, + &indx->alloc_run, data_size, &data_size, true, + NULL); + if (err) { + if (bmp) + goto out2; + goto out1; + } + + *vbn = bit << indx->idx2vbn_bits; + + return 0; + +out2: + /* Ops (no space?) */ + attr_set_size(ni, ATTR_BITMAP, in->name, in->name_len, + &indx->bitmap_run, bmp_size, &bmp_size_v, false, NULL); + +out1: + return err; +} + +/* + * indx_insert_into_root + * + * attempts to insert an entry into the index root + * If necessary, it will twiddle the index b-tree. + */ +static int indx_insert_into_root(struct ntfs_index *indx, struct ntfs_inode *ni, + const struct NTFS_DE *new_de, + struct NTFS_DE *root_de, const void *ctx, + struct ntfs_fnd *fnd) +{ + int err = 0; + struct NTFS_DE *e, *e0, *re; + struct mft_inode *mi; + struct ATTRIB *attr; + struct MFT_REC *rec; + struct INDEX_HDR *hdr; + struct indx_node *n; + CLST new_vbn; + __le64 *sub_vbn, t_vbn; + u16 new_de_size; + u32 hdr_used, hdr_total, asize, used, to_move; + u32 root_size, new_root_size; + struct ntfs_sb_info *sbi; + int ds_root; + struct INDEX_ROOT *root, *a_root = NULL; + + /* Get the record this root placed in */ + root = indx_get_root(indx, ni, &attr, &mi); + if (!root) + goto out; + + /* + * Try easy case: + * hdr_insert_de will succeed if there's room the root for the new entry. + */ + hdr = &root->ihdr; + sbi = ni->mi.sbi; + rec = mi->mrec; + used = le32_to_cpu(rec->used); + new_de_size = le16_to_cpu(new_de->size); + hdr_used = le32_to_cpu(hdr->used); + hdr_total = le32_to_cpu(hdr->total); + asize = le32_to_cpu(attr->size); + root_size = le32_to_cpu(attr->res.data_size); + + ds_root = new_de_size + hdr_used - hdr_total; + + if (used + ds_root < sbi->max_bytes_per_attr) { + /* make a room for new elements */ + mi_resize_attr(mi, attr, ds_root); + hdr->total = cpu_to_le32(hdr_total + ds_root); + e = hdr_insert_de(indx, hdr, new_de, root_de, ctx); + WARN_ON(!e); + fnd_clear(fnd); + fnd->root_de = e; + + return 0; + } + + /* Make a copy of root attribute to restore if error */ + a_root = ntfs_memdup(attr, asize); + if (!a_root) { + err = -ENOMEM; + goto out; + } + + /* copy all the non-end entries from the index root to the new buffer.*/ + to_move = 0; + e0 = hdr_first_de(hdr); + + /* Calculate the size to copy */ + for (e = e0;; e = hdr_next_de(hdr, e)) { + if (!e) { + err = -EINVAL; + goto out; + } + + if (de_is_last(e)) + break; + to_move += le16_to_cpu(e->size); + } + + n = NULL; + if (!to_move) { + re = NULL; + } else { + re = ntfs_memdup(e0, to_move); + if (!re) { + err = -ENOMEM; + goto out; + } + } + + sub_vbn = NULL; + if (de_has_vcn(e)) { + t_vbn = de_get_vbn_le(e); + sub_vbn = &t_vbn; + } + + new_root_size = sizeof(struct INDEX_ROOT) + sizeof(struct NTFS_DE) + + sizeof(u64); + ds_root = new_root_size - root_size; + + if (ds_root > 0 && used + ds_root > sbi->max_bytes_per_attr) { + /* make root external */ + err = -EOPNOTSUPP; + goto out; + } + + if (ds_root) + mi_resize_attr(mi, attr, ds_root); + + /* Fill first entry (vcn will be set later) */ + e = (struct NTFS_DE *)(root + 1); + memset(e, 0, sizeof(struct NTFS_DE)); + e->size = cpu_to_le16(sizeof(struct NTFS_DE) + sizeof(u64)); + e->flags = NTFS_IE_HAS_SUBNODES | NTFS_IE_LAST; + + hdr->flags = 1; + hdr->used = hdr->total = + cpu_to_le32(new_root_size - offsetof(struct INDEX_ROOT, ihdr)); + + fnd->root_de = hdr_first_de(hdr); + mi->dirty = true; + + /* Create alloc and bitmap attributes (if not) */ + err = run_is_empty(&indx->alloc_run) + ? indx_create_allocate(indx, ni, &new_vbn) + : indx_add_allocate(indx, ni, &new_vbn); + + /* layout of record may be changed, so rescan root */ + root = indx_get_root(indx, ni, &attr, &mi); + if (!root) { + /* bug? */ + ntfs_set_state(sbi, NTFS_DIRTY_ERROR); + err = -EINVAL; + goto out1; + } + + if (err) { + /* restore root */ + if (mi_resize_attr(mi, attr, -ds_root)) + memcpy(attr, a_root, asize); + else { + /* bug? */ + ntfs_set_state(sbi, NTFS_DIRTY_ERROR); + } + goto out1; + } + + e = (struct NTFS_DE *)(root + 1); + *(__le64 *)(e + 1) = cpu_to_le64(new_vbn); + mi->dirty = true; + + /* now we can create/format the new buffer and copy the entries into */ + n = indx_new(indx, ni, new_vbn, sub_vbn); + if (IS_ERR(n)) { + err = PTR_ERR(n); + goto out1; + } + + hdr = &n->index->ihdr; + hdr_used = le32_to_cpu(hdr->used); + hdr_total = le32_to_cpu(hdr->total); + + /* Copy root entries into new buffer */ + hdr_insert_head(hdr, re, to_move); + + /* Update bitmap attribute */ + indx_mark_used(indx, ni, new_vbn >> indx->idx2vbn_bits); + + /* Check if we can insert new entry new index buffer */ + if (hdr_used + new_de_size > hdr_total) { + /* + * This occurs if mft record is the same or bigger than index + * buffer. Move all root new index and have no space to add + * new entry classic case when mft record is 1K and index + * buffer 4K the problem should not occurs + */ + ntfs_free(re); + indx_write(indx, ni, n, 0); + + put_indx_node(n); + fnd_clear(fnd); + err = indx_insert_entry(indx, ni, new_de, ctx, fnd); + goto out; + } + + /* + * Now root is a parent for new index buffer + * Insert NewEntry a new buffer + */ + e = hdr_insert_de(indx, hdr, new_de, NULL, ctx); + if (!e) { + err = -EINVAL; + goto out1; + } + fnd_push(fnd, n, e); + + /* Just write updates index into disk */ + indx_write(indx, ni, n, 0); + + n = NULL; + +out1: + ntfs_free(re); + if (n) + put_indx_node(n); + +out: + ntfs_free(a_root); + return err; +} + +/* + * indx_insert_into_buffer + * + * attempts to insert an entry into an Index Allocation Buffer. + * If necessary, it will split the buffer. + */ +static int +indx_insert_into_buffer(struct ntfs_index *indx, struct ntfs_inode *ni, + struct INDEX_ROOT *root, const struct NTFS_DE *new_de, + const void *ctx, int level, struct ntfs_fnd *fnd) +{ + int err; + const struct NTFS_DE *sp; + struct NTFS_DE *e, *de_t, *up_e = NULL; + struct indx_node *n2 = NULL; + struct indx_node *n1 = fnd->nodes[level]; + struct INDEX_HDR *hdr1 = &n1->index->ihdr; + struct INDEX_HDR *hdr2; + u32 to_copy, used; + CLST new_vbn; + __le64 t_vbn, *sub_vbn; + u16 sp_size; + + /* Try the most easy case */ + e = fnd->level - 1 == level ? fnd->de[level] : NULL; + e = hdr_insert_de(indx, hdr1, new_de, e, ctx); + fnd->de[level] = e; + if (e) { + /* Just write updated index into disk */ + indx_write(indx, ni, n1, 0); + return 0; + } + + /* + * No space to insert into buffer. Split it. + * To split we: + * - Save split point ('cause index buffers will be changed) + * - Allocate NewBuffer and copy all entries <= sp into new buffer + * - Remove all entries (sp including) from TargetBuffer + * - Insert NewEntry into left or right buffer (depending on sp <=> + * NewEntry) + * - Insert sp into parent buffer (or root) + * - Make sp a parent for new buffer + */ + sp = hdr_find_split(hdr1); + if (!sp) + return -EINVAL; + + sp_size = le16_to_cpu(sp->size); + up_e = ntfs_malloc(sp_size + sizeof(u64)); + if (!up_e) + return -ENOMEM; + memcpy(up_e, sp, sp_size); + + if (!hdr1->flags) { + up_e->flags |= NTFS_IE_HAS_SUBNODES; + up_e->size = cpu_to_le16(sp_size + sizeof(u64)); + sub_vbn = NULL; + } else { + t_vbn = de_get_vbn_le(up_e); + sub_vbn = &t_vbn; + } + + /* Allocate on disk a new index allocation buffer. */ + err = indx_add_allocate(indx, ni, &new_vbn); + if (err) + goto out; + + /* Allocate and format memory a new index buffer */ + n2 = indx_new(indx, ni, new_vbn, sub_vbn); + if (IS_ERR(n2)) { + err = PTR_ERR(n2); + goto out; + } + + hdr2 = &n2->index->ihdr; + + /* Make sp a parent for new buffer */ + de_set_vbn(up_e, new_vbn); + + /* copy all the entries <= sp into the new buffer. */ + de_t = hdr_first_de(hdr1); + to_copy = PtrOffset(de_t, sp); + hdr_insert_head(hdr2, de_t, to_copy); + + /* remove all entries (sp including) from hdr1 */ + used = le32_to_cpu(hdr1->used) - to_copy - sp_size; + memmove(de_t, Add2Ptr(sp, sp_size), used - le32_to_cpu(hdr1->de_off)); + hdr1->used = cpu_to_le32(used); + + /* Insert new entry into left or right buffer (depending on sp <=> new_de) */ + hdr_insert_de(indx, + (*indx->cmp)(new_de + 1, le16_to_cpu(new_de->key_size), + up_e + 1, le16_to_cpu(up_e->key_size), + ctx) < 0 + ? hdr2 + : hdr1, + new_de, NULL, ctx); + + indx_mark_used(indx, ni, new_vbn >> indx->idx2vbn_bits); + + indx_write(indx, ni, n1, 0); + indx_write(indx, ni, n2, 0); + + put_indx_node(n2); + + /* + * we've finished splitting everybody, so we are ready to + * insert the promoted entry into the parent. + */ + if (!level) { + /* Insert in root */ + err = indx_insert_into_root(indx, ni, up_e, NULL, ctx, fnd); + if (err) + goto out; + } else { + /* + * The target buffer's parent is another index buffer + * TODO: Remove recursion + */ + err = indx_insert_into_buffer(indx, ni, root, up_e, ctx, + level - 1, fnd); + if (err) + goto out; + } + +out: + ntfs_free(up_e); + + return err; +} + +/* + * indx_insert_entry + * + * inserts new entry into index + */ +int indx_insert_entry(struct ntfs_index *indx, struct ntfs_inode *ni, + const struct NTFS_DE *new_de, const void *ctx, + struct ntfs_fnd *fnd) +{ + int err; + int diff; + struct NTFS_DE *e; + struct ntfs_fnd *fnd_a = NULL; + struct INDEX_ROOT *root; + + if (!fnd) { + fnd_a = fnd_get(); + if (!fnd_a) { + err = -ENOMEM; + goto out1; + } + fnd = fnd_a; + } + + root = indx_get_root(indx, ni, NULL, NULL); + if (!root) { + err = -EINVAL; + goto out; + } + + if (fnd_is_empty(fnd)) { + /* Find the spot the tree where we want to insert the new entry. */ + err = indx_find(indx, ni, root, new_de + 1, + le16_to_cpu(new_de->key_size), ctx, &diff, &e, + fnd); + if (err) + goto out; + + if (!diff) { + err = -EEXIST; + goto out; + } + } + + if (!fnd->level) { + /* The root is also a leaf, so we'll insert the new entry into it. */ + err = indx_insert_into_root(indx, ni, new_de, fnd->root_de, ctx, + fnd); + if (err) + goto out; + } else { + /* found a leaf buffer, so we'll insert the new entry into it.*/ + err = indx_insert_into_buffer(indx, ni, root, new_de, ctx, + fnd->level - 1, fnd); + if (err) + goto out; + } + +out: + fnd_put(fnd_a); +out1: + return err; +} + +/* + * indx_find_buffer + * + * locates a buffer the tree. + */ +static struct indx_node *indx_find_buffer(struct ntfs_index *indx, + struct ntfs_inode *ni, + const struct INDEX_ROOT *root, + __le64 vbn, struct indx_node *n) +{ + int err; + const struct NTFS_DE *e; + struct indx_node *r; + const struct INDEX_HDR *hdr = n ? &n->index->ihdr : &root->ihdr; + + /* Step 1: Scan one level */ + for (e = hdr_first_de(hdr);; e = hdr_next_de(hdr, e)) { + if (!e) + return ERR_PTR(-EINVAL); + + if (de_has_vcn(e) && vbn == de_get_vbn_le(e)) + return n; + + if (de_is_last(e)) + break; + } + + /* Step2: Do recursion */ + e = Add2Ptr(hdr, le32_to_cpu(hdr->de_off)); + for (;;) { + if (de_has_vcn_ex(e)) { + err = indx_read(indx, ni, de_get_vbn(e), &n); + if (err) + return ERR_PTR(err); + + r = indx_find_buffer(indx, ni, root, vbn, n); + if (r) + return r; + } + + if (de_is_last(e)) + break; + + e = Add2Ptr(e, le16_to_cpu(e->size)); + } + + return NULL; +} + +/* + * indx_shrink + * + * deallocates unused tail indexes + */ +static int indx_shrink(struct ntfs_index *indx, struct ntfs_inode *ni, + size_t bit) +{ + int err = 0; + u64 bpb, new_data; + size_t nbits; + struct ATTRIB *b; + struct ATTR_LIST_ENTRY *le = NULL; + const struct INDEX_NAMES *in = &s_index_names[indx->type]; + + b = ni_find_attr(ni, NULL, &le, ATTR_BITMAP, in->name, in->name_len, + NULL, NULL); + + if (!b) + return -ENOENT; + + if (!b->non_res) { + unsigned long pos; + const unsigned long *bm = resident_data(b); + + nbits = le32_to_cpu(b->res.data_size) * 8; + + if (bit >= nbits) + return 0; + + pos = find_next_bit(bm, nbits, bit); + if (pos < nbits) + return 0; + } else { + size_t used = MINUS_ONE_T; + + nbits = le64_to_cpu(b->nres.data_size) * 8; + + if (bit >= nbits) + return 0; + + err = scan_nres_bitmap(ni, b, indx, bit, &scan_for_used, &used); + if (err) + return err; + + if (used != MINUS_ONE_T) + return 0; + } + + new_data = (u64)bit << indx->index_bits; + + err = attr_set_size(ni, ATTR_ALLOC, in->name, in->name_len, + &indx->alloc_run, new_data, &new_data, false, NULL); + if (err) + return err; + + bpb = bitmap_size(bit); + if (bpb * 8 == nbits) + return 0; + + err = attr_set_size(ni, ATTR_BITMAP, in->name, in->name_len, + &indx->bitmap_run, bpb, &bpb, false, NULL); + + return err; +} + +static int indx_free_children(struct ntfs_index *indx, struct ntfs_inode *ni, + const struct NTFS_DE *e, bool trim) +{ + int err; + struct indx_node *n; + struct INDEX_HDR *hdr; + CLST vbn = de_get_vbn(e); + size_t i; + + err = indx_read(indx, ni, vbn, &n); + if (err) + return err; + + hdr = &n->index->ihdr; + /* First, recurse into the children, if any.*/ + if (hdr_has_subnode(hdr)) { + for (e = hdr_first_de(hdr); e; e = hdr_next_de(hdr, e)) { + indx_free_children(indx, ni, e, false); + if (de_is_last(e)) + break; + } + } + + put_indx_node(n); + + i = vbn >> indx->idx2vbn_bits; + /* We've gotten rid of the children; add this buffer to the free list. */ + indx_mark_free(indx, ni, i); + + if (!trim) + return 0; + + /* + * If there are no used indexes after current free index + * then we can truncate allocation and bitmap + * Use bitmap to estimate the case + */ + indx_shrink(indx, ni, i + 1); + return 0; +} + +/* + * indx_get_entry_to_replace + * + * finds a replacement entry for a deleted entry + * always returns a node entry: + * NTFS_IE_HAS_SUBNODES is set the flags and the size includes the sub_vcn + */ +static int indx_get_entry_to_replace(struct ntfs_index *indx, + struct ntfs_inode *ni, + const struct NTFS_DE *de_next, + struct NTFS_DE **de_to_replace, + struct ntfs_fnd *fnd) +{ + int err; + int level = -1; + CLST vbn; + struct NTFS_DE *e, *te, *re; + struct indx_node *n; + struct INDEX_BUFFER *ib; + + *de_to_replace = NULL; + + /* Find first leaf entry down from de_next */ + vbn = de_get_vbn(de_next); + for (;;) { + n = NULL; + err = indx_read(indx, ni, vbn, &n); + if (err) + goto out; + + e = hdr_first_de(&n->index->ihdr); + fnd_push(fnd, n, e); + + if (!de_is_last(e)) { + /* + * This buffer is non-empty, so its first entry could be used as the + * replacement entry. + */ + level = fnd->level - 1; + } + + if (!de_has_vcn(e)) + break; + + /* This buffer is a node. Continue to go down */ + vbn = de_get_vbn(e); + } + + if (level == -1) + goto out; + + n = fnd->nodes[level]; + te = hdr_first_de(&n->index->ihdr); + /* Copy the candidate entry into the replacement entry buffer. */ + re = ntfs_malloc(le16_to_cpu(te->size) + sizeof(u64)); + if (!re) { + err = -ENOMEM; + goto out; + } + + *de_to_replace = re; + memcpy(re, te, le16_to_cpu(te->size)); + + if (!de_has_vcn(re)) { + /* + * The replacement entry we found doesn't have a sub_vcn. increase its size + * to hold one. + */ + le16_add_cpu(&re->size, sizeof(u64)); + re->flags |= NTFS_IE_HAS_SUBNODES; + } else { + /* + * The replacement entry we found was a node entry, which means that all + * its child buffers are empty. Return them to the free pool. + */ + indx_free_children(indx, ni, te, true); + } + + /* + * Expunge the replacement entry from its former location, + * and then write that buffer. + */ + ib = n->index; + e = hdr_delete_de(&ib->ihdr, te); + + fnd->de[level] = e; + indx_write(indx, ni, n, 0); + + /* Check to see if this action created an empty leaf. */ + if (ib_is_leaf(ib) && ib_is_empty(ib)) + return 0; + +out: + fnd_clear(fnd); + return err; +} + +/* + * indx_delete_entry + * + * deletes an entry from the index. + */ +int indx_delete_entry(struct ntfs_index *indx, struct ntfs_inode *ni, + const void *key, u32 key_len, const void *ctx) +{ + int err, diff; + struct INDEX_ROOT *root; + struct INDEX_HDR *hdr; + struct ntfs_fnd *fnd, *fnd2; + struct INDEX_BUFFER *ib; + struct NTFS_DE *e, *re, *next, *prev, *me; + struct indx_node *n, *n2d = NULL; + __le64 sub_vbn; + int level, level2; + struct ATTRIB *attr; + struct mft_inode *mi; + u32 e_size, root_size, new_root_size; + size_t trim_bit; + const struct INDEX_NAMES *in; + + fnd = fnd_get(); + if (!fnd) { + err = -ENOMEM; + goto out2; + } + + fnd2 = fnd_get(); + if (!fnd2) { + err = -ENOMEM; + goto out1; + } + + root = indx_get_root(indx, ni, &attr, &mi); + if (!root) { + err = -EINVAL; + goto out; + } + + /* Locate the entry to remove. */ + err = indx_find(indx, ni, root, key, key_len, ctx, &diff, &e, fnd); + if (err) + goto out; + + if (!e || diff) { + err = -ENOENT; + goto out; + } + + level = fnd->level; + + if (level) { + n = fnd->nodes[level - 1]; + e = fnd->de[level - 1]; + ib = n->index; + hdr = &ib->ihdr; + } else { + hdr = &root->ihdr; + e = fnd->root_de; + n = NULL; + } + + e_size = le16_to_cpu(e->size); + + if (!de_has_vcn_ex(e)) { + /* The entry to delete is a leaf, so we can just rip it out */ + hdr_delete_de(hdr, e); + + if (!level) { + hdr->total = hdr->used; + + /* Shrink resident root attribute */ + mi_resize_attr(mi, attr, 0 - e_size); + goto out; + } + + indx_write(indx, ni, n, 0); + + /* + * Check to see if removing that entry made + * the leaf empty. + */ + if (ib_is_leaf(ib) && ib_is_empty(ib)) { + fnd_pop(fnd); + fnd_push(fnd2, n, e); + } + } else { + /* + * The entry we wish to delete is a node buffer, so we + * have to find a replacement for it. + */ + next = de_get_next(e); + + err = indx_get_entry_to_replace(indx, ni, next, &re, fnd2); + if (err) + goto out; + + if (re) { + de_set_vbn_le(re, de_get_vbn_le(e)); + hdr_delete_de(hdr, e); + + err = level ? indx_insert_into_buffer(indx, ni, root, + re, ctx, + fnd->level - 1, + fnd) + : indx_insert_into_root(indx, ni, re, e, + ctx, fnd); + ntfs_free(re); + + if (err) + goto out; + } else { + /* + * There is no replacement for the current entry. + * This means that the subtree rooted at its node is empty, + * and can be deleted, which turn means that the node can + * just inherit the deleted entry sub_vcn + */ + indx_free_children(indx, ni, next, true); + + de_set_vbn_le(next, de_get_vbn_le(e)); + hdr_delete_de(hdr, e); + if (level) { + indx_write(indx, ni, n, 0); + } else { + hdr->total = hdr->used; + + /* Shrink resident root attribute */ + mi_resize_attr(mi, attr, 0 - e_size); + } + } + } + + /* Delete a branch of tree */ + if (!fnd2 || !fnd2->level) + goto out; + + /* Reinit root 'cause it can be changed */ + root = indx_get_root(indx, ni, &attr, &mi); + if (!root) { + err = -EINVAL; + goto out; + } + + n2d = NULL; + sub_vbn = fnd2->nodes[0]->index->vbn; + level2 = 0; + level = fnd->level; + + hdr = level ? &fnd->nodes[level - 1]->index->ihdr : &root->ihdr; + + /* Scan current level */ + for (e = hdr_first_de(hdr);; e = hdr_next_de(hdr, e)) { + if (!e) { + err = -EINVAL; + goto out; + } + + if (de_has_vcn(e) && sub_vbn == de_get_vbn_le(e)) + break; + + if (de_is_last(e)) { + e = NULL; + break; + } + } + + if (!e) { + /* Do slow search from root */ + struct indx_node *in; + + fnd_clear(fnd); + + in = indx_find_buffer(indx, ni, root, sub_vbn, NULL); + if (IS_ERR(in)) { + err = PTR_ERR(in); + goto out; + } + + if (in) + fnd_push(fnd, in, NULL); + } + + /* Merge fnd2 -> fnd */ + for (level = 0; level < fnd2->level; level++) { + fnd_push(fnd, fnd2->nodes[level], fnd2->de[level]); + fnd2->nodes[level] = NULL; + } + fnd2->level = 0; + + hdr = NULL; + for (level = fnd->level; level; level--) { + struct indx_node *in = fnd->nodes[level - 1]; + + ib = in->index; + if (ib_is_empty(ib)) { + sub_vbn = ib->vbn; + } else { + hdr = &ib->ihdr; + n2d = in; + level2 = level; + break; + } + } + + if (!hdr) + hdr = &root->ihdr; + + e = hdr_first_de(hdr); + if (!e) { + err = -EINVAL; + goto out; + } + + if (hdr != &root->ihdr || !de_is_last(e)) { + prev = NULL; + while (!de_is_last(e)) { + if (de_has_vcn(e) && sub_vbn == de_get_vbn_le(e)) + break; + prev = e; + e = hdr_next_de(hdr, e); + if (!e) { + err = -EINVAL; + goto out; + } + } + + if (sub_vbn != de_get_vbn_le(e)) { + /* + * Didn't find the parent entry, although this buffer is the parent trail. + * Something is corrupt. + */ + err = -EINVAL; + goto out; + } + + if (de_is_last(e)) { + /* + * Since we can't remove the end entry, we'll remove its + * predecessor instead. This means we have to transfer the + * predecessor's sub_vcn to the end entry. + * Note: that this index block is not empty, so the + * predecessor must exist + */ + if (!prev) { + err = -EINVAL; + goto out; + } + + if (de_has_vcn(prev)) { + de_set_vbn_le(e, de_get_vbn_le(prev)); + } else if (de_has_vcn(e)) { + le16_sub_cpu(&e->size, sizeof(u64)); + e->flags &= ~NTFS_IE_HAS_SUBNODES; + le32_sub_cpu(&hdr->used, sizeof(u64)); + } + e = prev; + } + + /* + * Copy the current entry into a temporary buffer (stripping off its + * down-pointer, if any) and delete it from the current buffer or root, + * as appropriate. + */ + e_size = le16_to_cpu(e->size); + me = ntfs_memdup(e, e_size); + if (!me) { + err = -ENOMEM; + goto out; + } + + if (de_has_vcn(me)) { + me->flags &= ~NTFS_IE_HAS_SUBNODES; + le16_sub_cpu(&me->size, sizeof(u64)); + } + + hdr_delete_de(hdr, e); + + if (hdr == &root->ihdr) { + level = 0; + hdr->total = hdr->used; + + /* Shrink resident root attribute */ + mi_resize_attr(mi, attr, 0 - e_size); + } else { + indx_write(indx, ni, n2d, 0); + level = level2; + } + + /* Mark unused buffers as free */ + trim_bit = -1; + for (; level < fnd->level; level++) { + ib = fnd->nodes[level]->index; + if (ib_is_empty(ib)) { + size_t k = le64_to_cpu(ib->vbn) >> + indx->idx2vbn_bits; + + indx_mark_free(indx, ni, k); + if (k < trim_bit) + trim_bit = k; + } + } + + fnd_clear(fnd); + /*fnd->root_de = NULL;*/ + + /* + * Re-insert the entry into the tree. + * Find the spot the tree where we want to insert the new entry. + */ + err = indx_insert_entry(indx, ni, me, ctx, fnd); + ntfs_free(me); + if (err) + goto out; + + if (trim_bit != -1) + indx_shrink(indx, ni, trim_bit); + } else { + /* + * This tree needs to be collapsed down to an empty root. + * Recreate the index root as an empty leaf and free all the bits the + * index allocation bitmap. + */ + fnd_clear(fnd); + fnd_clear(fnd2); + + in = &s_index_names[indx->type]; + + err = attr_set_size(ni, ATTR_ALLOC, in->name, in->name_len, + &indx->alloc_run, 0, NULL, false, NULL); + err = ni_remove_attr(ni, ATTR_ALLOC, in->name, in->name_len, + false, NULL); + run_close(&indx->alloc_run); + + err = attr_set_size(ni, ATTR_BITMAP, in->name, in->name_len, + &indx->bitmap_run, 0, NULL, false, NULL); + err = ni_remove_attr(ni, ATTR_BITMAP, in->name, in->name_len, + false, NULL); + run_close(&indx->bitmap_run); + + root = indx_get_root(indx, ni, &attr, &mi); + if (!root) { + err = -EINVAL; + goto out; + } + + root_size = le32_to_cpu(attr->res.data_size); + new_root_size = + sizeof(struct INDEX_ROOT) + sizeof(struct NTFS_DE); + + if (new_root_size != root_size && + !mi_resize_attr(mi, attr, new_root_size - root_size)) { + err = -EINVAL; + goto out; + } + + /* Fill first entry */ + e = (struct NTFS_DE *)(root + 1); + e->ref.low = 0; + e->ref.high = 0; + e->ref.seq = 0; + e->size = cpu_to_le16(sizeof(struct NTFS_DE)); + e->flags = NTFS_IE_LAST; // 0x02 + e->key_size = 0; + e->res = 0; + + hdr = &root->ihdr; + hdr->flags = 0; + hdr->used = hdr->total = cpu_to_le32( + new_root_size - offsetof(struct INDEX_ROOT, ihdr)); + mi->dirty = true; + } + +out: + fnd_put(fnd2); +out1: + fnd_put(fnd); +out2: + return err; +} + +int indx_update_dup(struct ntfs_inode *ni, struct ntfs_sb_info *sbi, + const struct ATTR_FILE_NAME *fname, + const struct NTFS_DUP_INFO *dup, int sync) +{ + int err, diff; + struct NTFS_DE *e = NULL; + struct ATTR_FILE_NAME *e_fname; + struct ntfs_fnd *fnd; + struct INDEX_ROOT *root; + struct mft_inode *mi; + struct ntfs_index *indx = &ni->dir; + + fnd = fnd_get(); + if (!fnd) { + err = -ENOMEM; + goto out1; + } + + root = indx_get_root(indx, ni, NULL, &mi); + if (!root) { + err = -EINVAL; + goto out; + } + + /* Find entries tree and on disk */ + err = indx_find(indx, ni, root, fname, fname_full_size(fname), sbi, + &diff, &e, fnd); + if (err) + goto out; + + if (!e) { + err = -EINVAL; + goto out; + } + + if (diff) { + err = -EINVAL; + goto out; + } + + e_fname = (struct ATTR_FILE_NAME *)(e + 1); + + if (!memcmp(&e_fname->dup, dup, sizeof(*dup))) { + /* nothing to update in index! Try to avoid this call */ + goto out; + } + + memcpy(&e_fname->dup, dup, sizeof(*dup)); + + if (fnd->level) { + err = indx_write(indx, ni, fnd->nodes[fnd->level - 1], sync); + } else if (sync) { + mi->dirty = true; + err = mi_write(mi, 1); + } else { + mi->dirty = true; + mark_inode_dirty(&ni->vfs_inode); + } + +out: + fnd_put(fnd); + +out1: + return err; +} diff --git a/fs/ntfs3/inode.c b/fs/ntfs3/inode.c new file mode 100644 index 000000000000..9e836c192ddf --- /dev/null +++ b/fs/ntfs3/inode.c @@ -0,0 +1,2033 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +/* + * ntfs_read_mft + * + * reads record and parses MFT + */ +static struct inode *ntfs_read_mft(struct inode *inode, + const struct cpu_str *name, + const struct MFT_REF *ref) +{ + int err = 0; + struct ntfs_inode *ni = ntfs_i(inode); + struct super_block *sb = inode->i_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + mode_t mode = 0; + struct ATTR_STD_INFO5 *std5 = NULL; + struct ATTR_LIST_ENTRY *le; + struct ATTRIB *attr; + bool is_match = false; + bool is_root = false; + bool is_dir; + unsigned long ino = inode->i_ino; + u32 rp_fa = 0, asize, t32; + u16 roff, rsize, names = 0; + const struct ATTR_FILE_NAME *fname = NULL; + const struct INDEX_ROOT *root; + struct REPARSE_DATA_BUFFER rp; // 0x18 bytes + u64 t64; + struct MFT_REC *rec; + struct runs_tree *run; + + inode->i_op = NULL; + + err = mi_init(&ni->mi, sbi, ino); + if (err) + goto out; + + if (!sbi->mft.ni && ino == MFT_REC_MFT && !sb->s_root) { + t64 = sbi->mft.lbo >> sbi->cluster_bits; + t32 = bytes_to_cluster(sbi, MFT_REC_VOL * sbi->record_size); + sbi->mft.ni = ni; + init_rwsem(&ni->file.run_lock); + + if (!run_add_entry(&ni->file.run, 0, t64, t32, true)) { + err = -ENOMEM; + goto out; + } + } + + err = mi_read(&ni->mi, ino == MFT_REC_MFT); + + if (err) + goto out; + + rec = ni->mi.mrec; + + if (sbi->flags & NTFS_FLAGS_LOG_REPLAYING) { + ; + } else if (ref->seq != rec->seq) { + err = -EINVAL; + ntfs_err(sb, "MFT: r=%lx, expect seq=%x instead of %x!", ino, + le16_to_cpu(ref->seq), le16_to_cpu(rec->seq)); + goto out; + } else if (!is_rec_inuse(rec)) { + err = -EINVAL; + ntfs_err(sb, "Inode r=%x is not in use!", (u32)ino); + goto out; + } + + if (le32_to_cpu(rec->total) != sbi->record_size) { + // bad inode? + err = -EINVAL; + goto out; + } + + if (!is_rec_base(rec)) + goto Ok; + + /* record should contain $I30 root */ + is_dir = rec->flags & RECORD_FLAG_DIR; + + inode->i_generation = le16_to_cpu(rec->seq); + + /* Enumerate all struct Attributes MFT */ + le = NULL; + attr = NULL; + + /* + * to reduce tab pressure use goto instead of + * while( (attr = ni_enum_attr_ex(ni, attr, &le, NULL) )) + */ +next_attr: + run = NULL; + err = -EINVAL; + attr = ni_enum_attr_ex(ni, attr, &le, NULL); + if (!attr) + goto end_enum; + + if (le && le->vcn) { + /* This is non primary attribute segment. Ignore if not MFT */ + if (ino != MFT_REC_MFT || attr->type != ATTR_DATA) + goto next_attr; + + run = &ni->file.run; + asize = le32_to_cpu(attr->size); + goto attr_unpack_run; + } + + roff = attr->non_res ? 0 : le16_to_cpu(attr->res.data_off); + rsize = attr->non_res ? 0 : le32_to_cpu(attr->res.data_size); + asize = le32_to_cpu(attr->size); + + switch (attr->type) { + case ATTR_STD: + if (attr->non_res || + asize < sizeof(struct ATTR_STD_INFO) + roff || + rsize < sizeof(struct ATTR_STD_INFO)) + goto out; + + if (std5) + goto next_attr; + + std5 = Add2Ptr(attr, roff); + +#ifdef STATX_BTIME + nt2kernel(std5->cr_time, &ni->i_crtime); +#endif + nt2kernel(std5->a_time, &inode->i_atime); + nt2kernel(std5->c_time, &inode->i_ctime); + nt2kernel(std5->m_time, &inode->i_mtime); + + ni->std_fa = std5->fa; + + if (asize >= sizeof(struct ATTR_STD_INFO5) + roff && + rsize >= sizeof(struct ATTR_STD_INFO5)) + ni->std_security_id = std5->security_id; + goto next_attr; + + case ATTR_LIST: + if (attr->name_len || le || ino == MFT_REC_LOG) + goto out; + + err = ntfs_load_attr_list(ni, attr); + if (err) + goto out; + + le = NULL; + attr = NULL; + goto next_attr; + + case ATTR_NAME: + if (attr->non_res || asize < SIZEOF_ATTRIBUTE_FILENAME + roff || + rsize < SIZEOF_ATTRIBUTE_FILENAME) + goto out; + + fname = Add2Ptr(attr, roff); + if (fname->type == FILE_NAME_DOS) + goto next_attr; + + names += 1; + if (name && name->len == fname->name_len && + !ntfs_cmp_names_cpu(name, (struct le_str *)&fname->name_len, + NULL, false)) + is_match = true; + + goto next_attr; + + case ATTR_DATA: + if (is_dir) { + /* ignore data attribute in dir record */ + goto next_attr; + } + + if (ino == MFT_REC_BADCLUST && !attr->non_res) + goto next_attr; + + if (attr->name_len && + ((ino != MFT_REC_BADCLUST || !attr->non_res || + attr->name_len != ARRAY_SIZE(BAD_NAME) || + memcmp(attr_name(attr), BAD_NAME, sizeof(BAD_NAME))) && + (ino != MFT_REC_SECURE || !attr->non_res || + attr->name_len != ARRAY_SIZE(SDS_NAME) || + memcmp(attr_name(attr), SDS_NAME, sizeof(SDS_NAME))))) { + /* file contains stream attribute. ignore it */ + goto next_attr; + } + + if (is_attr_sparsed(attr)) + ni->std_fa |= FILE_ATTRIBUTE_SPARSE_FILE; + else + ni->std_fa &= ~FILE_ATTRIBUTE_SPARSE_FILE; + + if (is_attr_compressed(attr)) + ni->std_fa |= FILE_ATTRIBUTE_COMPRESSED; + else + ni->std_fa &= ~FILE_ATTRIBUTE_COMPRESSED; + + if (is_attr_encrypted(attr)) + ni->std_fa |= FILE_ATTRIBUTE_ENCRYPTED; + else + ni->std_fa &= ~FILE_ATTRIBUTE_ENCRYPTED; + + if (!attr->non_res) { + ni->i_valid = inode->i_size = rsize; + inode_set_bytes(inode, rsize); + t32 = asize; + } else { + t32 = le16_to_cpu(attr->nres.run_off); + } + + mode = S_IFREG | (0777 & sbi->options.fs_fmask_inv); + + if (!attr->non_res) { + ni->ni_flags |= NI_FLAG_RESIDENT; + goto next_attr; + } + + inode_set_bytes(inode, attr_ondisk_size(attr)); + + ni->i_valid = le64_to_cpu(attr->nres.valid_size); + inode->i_size = le64_to_cpu(attr->nres.data_size); + if (!attr->nres.alloc_size) + goto next_attr; + + run = ino == MFT_REC_BITMAP ? &sbi->used.bitmap.run + : &ni->file.run; + break; + + case ATTR_ROOT: + if (attr->non_res) + goto out; + + root = Add2Ptr(attr, roff); + is_root = true; + + if (attr->name_len != ARRAY_SIZE(I30_NAME) || + memcmp(attr_name(attr), I30_NAME, sizeof(I30_NAME))) + goto next_attr; + + if (root->type != ATTR_NAME || + root->rule != NTFS_COLLATION_TYPE_FILENAME) + goto out; + + if (!is_dir) + goto next_attr; + + ni->ni_flags |= NI_FLAG_DIR; + + err = indx_init(&ni->dir, sbi, attr, INDEX_MUTEX_I30); + if (err) + goto out; + + mode = sb->s_root + ? (S_IFDIR | (0777 & sbi->options.fs_dmask_inv)) + : (S_IFDIR | 0777); + goto next_attr; + + case ATTR_ALLOC: + if (!is_root || attr->name_len != ARRAY_SIZE(I30_NAME) || + memcmp(attr_name(attr), I30_NAME, sizeof(I30_NAME))) + goto next_attr; + + inode->i_size = le64_to_cpu(attr->nres.data_size); + ni->i_valid = le64_to_cpu(attr->nres.valid_size); + inode_set_bytes(inode, le64_to_cpu(attr->nres.alloc_size)); + + run = &ni->dir.alloc_run; + break; + + case ATTR_BITMAP: + if (ino == MFT_REC_MFT) { + if (!attr->non_res) + goto out; +#ifndef CONFIG_NTFS3_64BIT_CLUSTER + /* 0x20000000 = 2^32 / 8 */ + if (le64_to_cpu(attr->nres.alloc_size) >= 0x20000000) + goto out; +#endif + run = &sbi->mft.bitmap.run; + break; + } else if (is_dir && attr->name_len == ARRAY_SIZE(I30_NAME) && + !memcmp(attr_name(attr), I30_NAME, + sizeof(I30_NAME)) && + attr->non_res) { + run = &ni->dir.bitmap_run; + break; + } + goto next_attr; + + case ATTR_REPARSE: + if (attr->name_len) + goto next_attr; + + rp_fa = ni_parse_reparse(ni, attr, &rp); + switch (rp_fa) { + case REPARSE_LINK: + if (!attr->non_res) { + inode->i_size = rsize; + inode_set_bytes(inode, rsize); + t32 = asize; + } else { + inode->i_size = + le64_to_cpu(attr->nres.data_size); + t32 = le16_to_cpu(attr->nres.run_off); + } + + /* Looks like normal symlink */ + ni->i_valid = inode->i_size; + + /* Clear directory bit */ + if (ni->ni_flags & NI_FLAG_DIR) { + indx_clear(&ni->dir); + memset(&ni->dir, 0, sizeof(ni->dir)); + ni->ni_flags &= ~NI_FLAG_DIR; + } else { + run_close(&ni->file.run); + } + mode = S_IFLNK | 0777; + is_dir = false; + if (attr->non_res) { + run = &ni->file.run; + goto attr_unpack_run; // double break + } + break; + + case REPARSE_COMPRESSED: + break; + + case REPARSE_DEDUPLICATED: + break; + } + goto next_attr; + + case ATTR_EA_INFO: + if (!attr->name_len && + resident_data_ex(attr, sizeof(struct EA_INFO))) + ni->ni_flags |= NI_FLAG_EA; + goto next_attr; + + default: + goto next_attr; + } + +attr_unpack_run: + roff = le16_to_cpu(attr->nres.run_off); + + t64 = le64_to_cpu(attr->nres.svcn); + err = run_unpack_ex(run, sbi, ino, t64, le64_to_cpu(attr->nres.evcn), + t64, Add2Ptr(attr, roff), asize - roff); + if (err < 0) + goto out; + err = 0; + goto next_attr; + +end_enum: + + if (!std5) + goto out; + + if (!is_match && name) { + /* reuse rec as buffer for ascii name */ + err = -ENOENT; + goto out; + } + + if (std5->fa & FILE_ATTRIBUTE_READONLY) + mode &= ~0222; + + /* Setup 'uid' and 'gid' */ + inode->i_uid = sbi->options.fs_uid; + inode->i_gid = sbi->options.fs_gid; + + if (!names) { + err = -EINVAL; + goto out; + } + + if (S_ISDIR(mode)) { + ni->std_fa |= FILE_ATTRIBUTE_DIRECTORY; + + /* + * dot and dot-dot should be included in count but was not + * included in enumeration. + * Usually a hard links to directories are disabled + */ + set_nlink(inode, 1); + inode->i_op = &ntfs_dir_inode_operations; + inode->i_fop = &ntfs_dir_operations; + ni->i_valid = 0; + } else if (S_ISLNK(mode)) { + ni->std_fa &= ~FILE_ATTRIBUTE_DIRECTORY; + inode->i_op = &ntfs_link_inode_operations; + inode->i_fop = NULL; + inode_nohighmem(inode); // ?? + set_nlink(inode, names); + } else if (S_ISREG(mode)) { + ni->std_fa &= ~FILE_ATTRIBUTE_DIRECTORY; + + set_nlink(inode, names); + + inode->i_op = &ntfs_file_inode_operations; + inode->i_fop = &ntfs_file_operations; + inode->i_mapping->a_ops = + is_compressed(ni) ? &ntfs_aops_cmpr : &ntfs_aops; + + if (ino != MFT_REC_MFT) + init_rwsem(&ni->file.run_lock); + } else if (fname && fname->home.low == cpu_to_le32(MFT_REC_EXTEND) && + fname->home.seq == cpu_to_le16(MFT_REC_EXTEND)) { + /* Records in $Extend are not a files or general directories */ + } else { + err = -EINVAL; + goto out; + } + + if ((sbi->options.sys_immutable && + (std5->fa & FILE_ATTRIBUTE_SYSTEM)) && + !S_ISFIFO(mode) && !S_ISSOCK(mode) && !S_ISLNK(mode)) { + inode->i_flags |= S_IMMUTABLE; + } else { + inode->i_flags &= ~S_IMMUTABLE; + } + + inode->i_mode = mode; + if (!(ni->ni_flags & NI_FLAG_EA)) { + /* if no xattr then no security (stored in xattr) */ + inode->i_flags |= S_NOSEC; + } + +Ok: + if (ino == MFT_REC_MFT && !sb->s_root) + sbi->mft.ni = NULL; + + unlock_new_inode(inode); + + return inode; + +out: + if (ino == MFT_REC_MFT && !sb->s_root) + sbi->mft.ni = NULL; + + iget_failed(inode); + return ERR_PTR(err); +} + +/* returns 1 if match */ +static int ntfs_test_inode(struct inode *inode, void *data) +{ + struct MFT_REF *ref = data; + + return ino_get(ref) == inode->i_ino; +} + +static int ntfs_set_inode(struct inode *inode, void *data) +{ + const struct MFT_REF *ref = data; + + inode->i_ino = ino_get(ref); + return 0; +} + +struct inode *ntfs_iget5(struct super_block *sb, const struct MFT_REF *ref, + const struct cpu_str *name) +{ + struct inode *inode; + + inode = iget5_locked(sb, ino_get(ref), ntfs_test_inode, ntfs_set_inode, + (void *)ref); + if (unlikely(!inode)) + return ERR_PTR(-ENOMEM); + + /* If this is a freshly allocated inode, need to read it now. */ + if (inode->i_state & I_NEW) + inode = ntfs_read_mft(inode, name, ref); + else if (ref->seq != ntfs_i(inode)->mi.mrec->seq) { + /* inode overlaps? */ + make_bad_inode(inode); + } + + return inode; +} + +enum get_block_ctx { + GET_BLOCK_GENERAL = 0, + GET_BLOCK_WRITE_BEGIN = 1, + GET_BLOCK_DIRECT_IO_R = 2, + GET_BLOCK_DIRECT_IO_W = 3, + GET_BLOCK_BMAP = 4, +}; + +static noinline int ntfs_get_block_vbo(struct inode *inode, u64 vbo, + struct buffer_head *bh, int create, + enum get_block_ctx ctx) +{ + struct super_block *sb = inode->i_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + struct ntfs_inode *ni = ntfs_i(inode); + struct page *page = bh->b_page; + u8 cluster_bits = sbi->cluster_bits; + u32 block_size = sb->s_blocksize; + u64 bytes, lbo, valid; + u32 off; + int err; + CLST vcn, lcn, len; + bool new; + + /*clear previous state*/ + clear_buffer_new(bh); + clear_buffer_uptodate(bh); + + /* direct write uses 'create=0'*/ + if (!create && vbo >= ni->i_valid) { + /* out of valid */ + return 0; + } + + if (vbo >= inode->i_size) { + /* out of size */ + return 0; + } + + if (is_resident(ni)) { + ni_lock(ni); + err = attr_data_read_resident(ni, page); + ni_unlock(ni); + + if (!err) + set_buffer_uptodate(bh); + bh->b_size = block_size; + return err; + } + + vcn = vbo >> cluster_bits; + off = vbo & sbi->cluster_mask; + new = false; + + err = attr_data_get_block(ni, vcn, 1, &lcn, &len, create ? &new : NULL); + if (err) + goto out; + + if (!len) + return 0; + + bytes = ((u64)len << cluster_bits) - off; + + if (lcn == SPARSE_LCN) { + if (!create) { + if (bh->b_size > bytes) + bh->b_size = bytes; + + return 0; + } + WARN_ON(1); + } + + if (new) { + set_buffer_new(bh); + if ((len << cluster_bits) > block_size) + ntfs_sparse_cluster(inode, page, vcn, len); + } + + lbo = ((u64)lcn << cluster_bits) + off; + + set_buffer_mapped(bh); + bh->b_bdev = sb->s_bdev; + bh->b_blocknr = lbo >> sb->s_blocksize_bits; + + valid = ni->i_valid; + + if (ctx == GET_BLOCK_DIRECT_IO_W) { + /*ntfs_direct_IO will update ni->i_valid */ + if (vbo >= valid) + set_buffer_new(bh); + } else if (create) { + /*normal write*/ + if (vbo >= valid) { + set_buffer_new(bh); + if (bytes > bh->b_size) + bytes = bh->b_size; + ni->i_valid = vbo + bytes; + mark_inode_dirty(inode); + } + } else if (valid >= inode->i_size) { + /* normal read of normal file*/ + } else if (vbo >= valid) { + /* read out of valid data*/ + /* should never be here 'cause already checked */ + clear_buffer_mapped(bh); + } else if (vbo + bytes <= valid) { + /* normal read */ + } else if (vbo + block_size <= valid) { + /* normal short read */ + bytes = block_size; + } else { + /* + * read across valid size: vbo < valid && valid < vbo + block_size + */ + u32 voff = valid - vbo; + + bh->b_size = bytes = block_size; + off = vbo & (PAGE_SIZE - 1); + set_bh_page(bh, page, off); + ll_rw_block(REQ_OP_READ, 0, 1, &bh); + wait_on_buffer(bh); + /* Uhhuh. Read error. Complain and punt. */ + if (!buffer_uptodate(bh)) { + err = -EIO; + goto out; + } + zero_user_segment(page, off + voff, off + block_size); + } + + if (bh->b_size > bytes) + bh->b_size = bytes; + +#ifndef __LP64__ + if (ctx == GET_BLOCK_DIRECT_IO_W || ctx == GET_BLOCK_DIRECT_IO_R) { + static_assert(sizeof(size_t) < sizeof(loff_t)); + if (bytes > 0x40000000u) + bh->b_size = 0x40000000u; + } +#endif + + return 0; + +out: + return err; +} + +int ntfs_get_block(struct inode *inode, sector_t vbn, + struct buffer_head *bh_result, int create) +{ + return ntfs_get_block_vbo(inode, (u64)vbn << inode->i_blkbits, + bh_result, create, GET_BLOCK_GENERAL); +} + +static int ntfs_get_block_bmap(struct inode *inode, sector_t vsn, + struct buffer_head *bh_result, int create) +{ + return ntfs_get_block_vbo(inode, + (u64)vsn << inode->i_sb->s_blocksize_bits, + bh_result, create, GET_BLOCK_BMAP); +} + +static sector_t ntfs_bmap(struct address_space *mapping, sector_t block) +{ + return generic_block_bmap(mapping, block, ntfs_get_block_bmap); +} + +static int ntfs_readpage(struct file *file, struct page *page) +{ + int err; + struct address_space *mapping = page->mapping; + struct inode *inode = mapping->host; + struct ntfs_inode *ni = ntfs_i(inode); + + if (is_resident(ni)) { + ni_lock(ni); + err = attr_data_read_resident(ni, page); + ni_unlock(ni); + if (err != E_NTFS_NONRESIDENT) { + unlock_page(page); + return err; + } + } + + if (is_compressed(ni)) { + ni_lock(ni); + err = ni_readpage_cmpr(ni, page); + ni_unlock(ni); + return err; + } + + /* normal + sparse files */ + return mpage_readpage(page, ntfs_get_block); +} + +static void ntfs_readahead(struct readahead_control *rac) +{ + struct address_space *mapping = rac->mapping; + struct inode *inode = mapping->host; + struct ntfs_inode *ni = ntfs_i(inode); + u64 valid; + loff_t pos; + + if (is_resident(ni)) { + /* no readahead for resident */ + return; + } + + if (is_compressed(ni)) { + /* no readahead for compressed */ + return; + } + + valid = ni->i_valid; + pos = readahead_pos(rac); + + if (valid < i_size_read(inode) && pos <= valid && + valid < pos + readahead_length(rac)) { + /* range cross 'valid'. read it page by page */ + return; + } + + mpage_readahead(rac, ntfs_get_block); +} + +static int ntfs_get_block_direct_IO_R(struct inode *inode, sector_t iblock, + struct buffer_head *bh_result, int create) +{ + return ntfs_get_block_vbo(inode, (u64)iblock << inode->i_blkbits, + bh_result, create, GET_BLOCK_DIRECT_IO_R); +} + +static int ntfs_get_block_direct_IO_W(struct inode *inode, sector_t iblock, + struct buffer_head *bh_result, int create) +{ + return ntfs_get_block_vbo(inode, (u64)iblock << inode->i_blkbits, + bh_result, create, GET_BLOCK_DIRECT_IO_W); +} + +static ssize_t ntfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) +{ + struct file *file = iocb->ki_filp; + struct address_space *mapping = file->f_mapping; + struct inode *inode = mapping->host; + struct ntfs_inode *ni = ntfs_i(inode); + size_t count = iov_iter_count(iter); + loff_t vbo = iocb->ki_pos; + loff_t end = vbo + count; + int wr = iov_iter_rw(iter) & WRITE; + const struct iovec *iov = iter->iov; + unsigned long nr_segs = iter->nr_segs; + loff_t valid; + ssize_t ret; + + if (is_resident(ni)) { + /*switch to buffered write*/ + ret = 0; + goto out; + } + + ret = blockdev_direct_IO(iocb, inode, iter, + wr ? ntfs_get_block_direct_IO_W + : ntfs_get_block_direct_IO_R); + valid = ni->i_valid; + if (wr) { + if (ret <= 0) + goto out; + + vbo += ret; + if (vbo > valid && !S_ISBLK(inode->i_mode)) { + ni->i_valid = vbo; + mark_inode_dirty(inode); + } + } else if (vbo < valid && valid < end) { + /* fix page */ + unsigned long uaddr = ~0ul; + struct page *page; + long i, npages; + size_t dvbo = valid - vbo; + size_t off = 0; + + /*Find user address*/ + for (i = 0; i < nr_segs; i++) { + if (off <= dvbo && dvbo < off + iov[i].iov_len) { + uaddr = (unsigned long)iov[i].iov_base + dvbo - + off; + break; + } + off += iov[i].iov_len; + } + + if (uaddr == ~0ul) + goto fix_error; + + npages = get_user_pages_unlocked(uaddr, 1, &page, FOLL_WRITE); + + if (npages <= 0) + goto fix_error; + + zero_user_segment(page, valid & (PAGE_SIZE - 1), PAGE_SIZE); + put_page(page); + } + +out: + return ret; +fix_error: + ntfs_inode_warn(inode, "file garbage at 0x%llx", valid); + goto out; +} + +int ntfs_set_size(struct inode *inode, u64 new_size) +{ + struct super_block *sb = inode->i_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + struct ntfs_inode *ni = ntfs_i(inode); + int err; + + /* Check for maximum file size */ + if (is_sparsed(ni) || is_compressed(ni)) { + if (new_size > sbi->maxbytes_sparse) { + err = -EFBIG; + goto out; + } + } else if (new_size > sbi->maxbytes) { + err = -EFBIG; + goto out; + } + + ni_lock(ni); + down_write(&ni->file.run_lock); + + err = attr_set_size(ni, ATTR_DATA, NULL, 0, &ni->file.run, new_size, + &ni->i_valid, true, NULL); + + up_write(&ni->file.run_lock); + ni_unlock(ni); + + mark_inode_dirty(inode); + +out: + return err; +} + +static int ntfs_writepage(struct page *page, struct writeback_control *wbc) +{ + struct address_space *mapping = page->mapping; + struct inode *inode = mapping->host; + struct ntfs_inode *ni = ntfs_i(inode); + int err; + + if (is_resident(ni)) { + ni_lock(ni); + err = attr_data_write_resident(ni, page); + ni_unlock(ni); + if (err != E_NTFS_NONRESIDENT) { + unlock_page(page); + return err; + } + } + + return block_write_full_page(page, ntfs_get_block, wbc); +} + +static int ntfs_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct inode *inode = mapping->host; + struct ntfs_inode *ni = ntfs_i(inode); + /* redirect call to 'ntfs_writepage' for resident files*/ + get_block_t *get_block = is_resident(ni) ? NULL : &ntfs_get_block; + + return mpage_writepages(mapping, wbc, get_block); +} + +static int ntfs_get_block_write_begin(struct inode *inode, sector_t vbn, + struct buffer_head *bh_result, int create) +{ + return ntfs_get_block_vbo(inode, (u64)vbn << inode->i_blkbits, + bh_result, create, GET_BLOCK_WRITE_BEGIN); +} + +static int ntfs_write_begin(struct file *file, struct address_space *mapping, + loff_t pos, u32 len, u32 flags, struct page **pagep, + void **fsdata) +{ + int err; + struct inode *inode = mapping->host; + struct ntfs_inode *ni = ntfs_i(inode); + + *pagep = NULL; + if (is_resident(ni)) { + struct page *page = grab_cache_page_write_begin( + mapping, pos >> PAGE_SHIFT, flags); + + if (!page) { + err = -ENOMEM; + goto out; + } + + ni_lock(ni); + err = attr_data_read_resident(ni, page); + ni_unlock(ni); + + if (!err) { + *pagep = page; + goto out; + } + unlock_page(page); + put_page(page); + + if (err != E_NTFS_NONRESIDENT) + goto out; + } + + err = block_write_begin(mapping, pos, len, flags, pagep, + ntfs_get_block_write_begin); + +out: + return err; +} + +/* address_space_operations::write_end */ +static int ntfs_write_end(struct file *file, struct address_space *mapping, + loff_t pos, u32 len, u32 copied, struct page *page, + void *fsdata) + +{ + struct inode *inode = mapping->host; + struct ntfs_inode *ni = ntfs_i(inode); + u64 valid = ni->i_valid; + bool dirty = false; + int err; + + if (is_resident(ni)) { + ni_lock(ni); + err = attr_data_write_resident(ni, page); + ni_unlock(ni); + if (!err) { + dirty = true; + /* clear any buffers in page*/ + if (page_has_buffers(page)) { + struct buffer_head *head, *bh; + + bh = head = page_buffers(page); + do { + clear_buffer_dirty(bh); + clear_buffer_mapped(bh); + set_buffer_uptodate(bh); + } while (head != (bh = bh->b_this_page)); + } + SetPageUptodate(page); + err = copied; + } + unlock_page(page); + put_page(page); + } else { + err = generic_write_end(file, mapping, pos, len, copied, page, + fsdata); + } + + if (err >= 0) { + if (!(ni->std_fa & FILE_ATTRIBUTE_ARCHIVE)) { + inode->i_ctime = inode->i_mtime = current_time(inode); + ni->std_fa |= FILE_ATTRIBUTE_ARCHIVE; + dirty = true; + } + + if (valid != ni->i_valid) { + /* ni->i_valid is changed in ntfs_get_block_vbo */ + dirty = true; + } + + if (dirty) + mark_inode_dirty(inode); + } + + return err; +} + +int reset_log_file(struct inode *inode) +{ + int err; + loff_t pos = 0; + u32 log_size = inode->i_size; + struct address_space *mapping = inode->i_mapping; + + for (;;) { + u32 len; + void *kaddr; + struct page *page; + + len = pos + PAGE_SIZE > log_size ? (log_size - pos) : PAGE_SIZE; + + err = block_write_begin(mapping, pos, len, 0, &page, + ntfs_get_block_write_begin); + if (err) + goto out; + + kaddr = kmap_atomic(page); + memset(kaddr, -1, len); + kunmap_atomic(kaddr); + flush_dcache_page(page); + + err = block_write_end(NULL, mapping, pos, len, len, page, NULL); + if (err < 0) + goto out; + pos += len; + + if (pos >= log_size) + break; + balance_dirty_pages_ratelimited(mapping); + } +out: + mark_inode_dirty_sync(inode); + + return err; +} + +int ntfs3_write_inode(struct inode *inode, struct writeback_control *wbc) +{ + return _ni_write_inode(inode, wbc->sync_mode == WB_SYNC_ALL); +} + +int ntfs_sync_inode(struct inode *inode) +{ + return _ni_write_inode(inode, 1); +} + +/* + * helper function for ntfs_flush_inodes. This writes both the inode + * and the file data blocks, waiting for in flight data blocks before + * the start of the call. It does not wait for any io started + * during the call + */ +static int writeback_inode(struct inode *inode) +{ + int ret = sync_inode_metadata(inode, 0); + + if (!ret) + ret = filemap_fdatawrite(inode->i_mapping); + return ret; +} + +/* + * write data and metadata corresponding to i1 and i2. The io is + * started but we do not wait for any of it to finish. + * + * filemap_flush is used for the block device, so if there is a dirty + * page for a block already in flight, we will not wait and start the + * io over again + */ +int ntfs_flush_inodes(struct super_block *sb, struct inode *i1, + struct inode *i2) +{ + int ret = 0; + + if (i1) + ret = writeback_inode(i1); + if (!ret && i2) + ret = writeback_inode(i2); + if (!ret) + ret = filemap_flush(sb->s_bdev->bd_inode->i_mapping); + return ret; +} + +int inode_write_data(struct inode *inode, const void *data, size_t bytes) +{ + pgoff_t idx; + + /* Write non resident data */ + for (idx = 0; bytes; idx++) { + size_t op = bytes > PAGE_SIZE ? PAGE_SIZE : bytes; + struct page *page = ntfs_map_page(inode->i_mapping, idx); + + if (IS_ERR(page)) + return PTR_ERR(page); + + lock_page(page); + WARN_ON(!PageUptodate(page)); + ClearPageUptodate(page); + + memcpy(page_address(page), data, op); + + flush_dcache_page(page); + SetPageUptodate(page); + unlock_page(page); + + ntfs_unmap_page(page); + + bytes -= op; + data = Add2Ptr(data, PAGE_SIZE); + } + return 0; +} + +/* + * number of bytes to for REPARSE_DATA_BUFFER(IO_REPARSE_TAG_SYMLINK) + * for unicode string of 'uni_len' length + */ +static inline u32 ntfs_reparse_bytes(u32 uni_len) +{ + /* header + unicode string + decorated unicode string */ + return sizeof(short) * (2 * uni_len + 4) + + offsetof(struct REPARSE_DATA_BUFFER, + SymbolicLinkReparseBuffer.PathBuffer); +} + +static struct REPARSE_DATA_BUFFER * +ntfs_create_reparse_buffer(struct ntfs_sb_info *sbi, const char *symname, + u32 size, u16 *nsize) +{ + int i, err; + struct REPARSE_DATA_BUFFER *rp; + __le16 *rp_name; + typeof(rp->SymbolicLinkReparseBuffer) *rs; + + rp = ntfs_zalloc(ntfs_reparse_bytes(2 * size + 2)); + if (!rp) + return ERR_PTR(-ENOMEM); + + rs = &rp->SymbolicLinkReparseBuffer; + rp_name = rs->PathBuffer; + + /* Convert link name to utf16 */ + err = ntfs_nls_to_utf16(sbi, symname, size, + (struct cpu_str *)(rp_name - 1), 2 * size, + UTF16_LITTLE_ENDIAN); + if (err < 0) + goto out; + + /* err = the length of unicode name of symlink */ + *nsize = ntfs_reparse_bytes(err); + + if (*nsize > sbi->reparse.max_size) { + err = -EFBIG; + goto out; + } + + /* translate linux '/' into windows '\' */ + for (i = 0; i < err; i++) { + if (rp_name[i] == cpu_to_le16('/')) + rp_name[i] = cpu_to_le16('\\'); + } + + rp->ReparseTag = IO_REPARSE_TAG_SYMLINK; + rp->ReparseDataLength = + cpu_to_le16(*nsize - offsetof(struct REPARSE_DATA_BUFFER, + SymbolicLinkReparseBuffer)); + + /* PrintName + SubstituteName */ + rs->SubstituteNameOffset = cpu_to_le16(sizeof(short) * err); + rs->SubstituteNameLength = cpu_to_le16(sizeof(short) * err + 8); + rs->PrintNameLength = rs->SubstituteNameOffset; + + /* + * TODO: use relative path if possible to allow windows to parse this path + * 0-absolute path 1- relative path (SYMLINK_FLAG_RELATIVE) + */ + rs->Flags = 0; + + memmove(rp_name + err + 4, rp_name, sizeof(short) * err); + + /* decorate SubstituteName */ + rp_name += err; + rp_name[0] = cpu_to_le16('\\'); + rp_name[1] = cpu_to_le16('?'); + rp_name[2] = cpu_to_le16('?'); + rp_name[3] = cpu_to_le16('\\'); + + return rp; +out: + ntfs_free(rp); + return ERR_PTR(err); +} + +struct inode *ntfs_create_inode(struct user_namespace *mnt_userns, + struct inode *dir, struct dentry *dentry, + const struct cpu_str *uni, umode_t mode, + dev_t dev, const char *symname, u32 size, + int excl, struct ntfs_fnd *fnd) +{ + int err; + struct super_block *sb = dir->i_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + const struct qstr *name = &dentry->d_name; + CLST ino = 0; + struct ntfs_inode *dir_ni = ntfs_i(dir); + struct ntfs_inode *ni = NULL; + struct inode *inode = NULL; + struct ATTRIB *attr; + struct ATTR_STD_INFO5 *std5; + struct ATTR_FILE_NAME *fname; + struct MFT_REC *rec; + u32 asize, dsize, sd_size; + enum FILE_ATTRIBUTE fa; + __le32 security_id = SECURITY_ID_INVALID; + CLST vcn; + const void *sd; + u16 t16, nsize = 0, aid = 0; + struct INDEX_ROOT *root, *dir_root; + struct NTFS_DE *e, *new_de = NULL; + struct REPARSE_DATA_BUFFER *rp = NULL; + bool is_dir = S_ISDIR(mode); + bool is_link = S_ISLNK(mode); + bool rp_inserted = false; + bool is_sp = S_ISCHR(mode) || S_ISBLK(mode) || S_ISFIFO(mode) || + S_ISSOCK(mode); + + if (is_sp) + return ERR_PTR(-EOPNOTSUPP); + + dir_root = indx_get_root(&dir_ni->dir, dir_ni, NULL, NULL); + if (!dir_root) + return ERR_PTR(-EINVAL); + + if (is_dir) { + /* use parent's directory attributes */ + fa = dir_ni->std_fa | FILE_ATTRIBUTE_DIRECTORY | + FILE_ATTRIBUTE_ARCHIVE; + /* + * By default child directory inherits parent attributes + * root directory is hidden + system + * Make an exception for children in root + */ + if (dir->i_ino == MFT_REC_ROOT) + fa &= ~(FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_SYSTEM); + } else if (is_link) { + /* It is good idea that link should be the same type (file/dir) as target */ + fa = FILE_ATTRIBUTE_REPARSE_POINT; + + /* + * linux: there are dir/file/symlink and so on + * NTFS: symlinks are "dir + reparse" or "file + reparse" + * It is good idea to create: + * dir + reparse if 'symname' points to directory + * or + * file + reparse if 'symname' points to file + * Unfortunately kern_path hangs if symname contains 'dir' + */ + + /* + * struct path path; + * + * if (!kern_path(symname, LOOKUP_FOLLOW, &path)){ + * struct inode *target = d_inode(path.dentry); + * + * if (S_ISDIR(target->i_mode)) + * fa |= FILE_ATTRIBUTE_DIRECTORY; + * // if ( target->i_sb == sb ){ + * // use relative path? + * // } + * path_put(&path); + * } + */ + } else if (sbi->options.sparse) { + /* sparsed regular file, cause option 'sparse' */ + fa = FILE_ATTRIBUTE_SPARSE_FILE | FILE_ATTRIBUTE_ARCHIVE; + } else if (dir_ni->std_fa & FILE_ATTRIBUTE_COMPRESSED) { + /* compressed regular file, if parent is compressed */ + fa = FILE_ATTRIBUTE_COMPRESSED | FILE_ATTRIBUTE_ARCHIVE; + } else { + /* regular file, default attributes */ + fa = FILE_ATTRIBUTE_ARCHIVE; + } + + if (!(mode & 0222)) + fa |= FILE_ATTRIBUTE_READONLY; + + /* allocate PATH_MAX bytes */ + new_de = __getname(); + if (!new_de) { + err = -ENOMEM; + goto out1; + } + + /*mark rw ntfs as dirty. it will be cleared at umount*/ + ntfs_set_state(sbi, NTFS_DIRTY_DIRTY); + + /* Step 1: allocate and fill new mft record */ + err = ntfs_look_free_mft(sbi, &ino, false, NULL, NULL); + if (err) + goto out2; + + ni = ntfs_new_inode(sbi, ino, fa & FILE_ATTRIBUTE_DIRECTORY); + if (IS_ERR(ni)) { + err = PTR_ERR(ni); + ni = NULL; + goto out3; + } + inode = &ni->vfs_inode; + + inode->i_atime = inode->i_mtime = inode->i_ctime = ni->i_crtime = + current_time(inode); + + rec = ni->mi.mrec; + rec->hard_links = cpu_to_le16(1); + attr = Add2Ptr(rec, le16_to_cpu(rec->attr_off)); + + /* Get default security id */ + sd = s_default_security; + sd_size = sizeof(s_default_security); + + if (is_ntfs3(sbi)) { + security_id = dir_ni->std_security_id; + if (le32_to_cpu(security_id) < SECURITY_ID_FIRST) { + security_id = sbi->security.def_security_id; + + if (security_id == SECURITY_ID_INVALID && + !ntfs_insert_security(sbi, sd, sd_size, + &security_id, NULL)) + sbi->security.def_security_id = security_id; + } + } + + /* Insert standard info */ + std5 = Add2Ptr(attr, SIZEOF_RESIDENT); + + if (security_id == SECURITY_ID_INVALID) { + dsize = sizeof(struct ATTR_STD_INFO); + } else { + dsize = sizeof(struct ATTR_STD_INFO5); + std5->security_id = security_id; + ni->std_security_id = security_id; + } + asize = SIZEOF_RESIDENT + dsize; + + attr->type = ATTR_STD; + attr->size = cpu_to_le32(asize); + attr->id = cpu_to_le16(aid++); + attr->res.data_off = SIZEOF_RESIDENT_LE; + attr->res.data_size = cpu_to_le32(dsize); + + std5->cr_time = std5->m_time = std5->c_time = std5->a_time = + kernel2nt(&inode->i_atime); + + ni->std_fa = fa; + std5->fa = fa; + + attr = Add2Ptr(attr, asize); + + /* Insert file name */ + err = fill_name_de(sbi, new_de, name, uni); + if (err) + goto out4; + + mi_get_ref(&ni->mi, &new_de->ref); + + fname = (struct ATTR_FILE_NAME *)(new_de + 1); + mi_get_ref(&dir_ni->mi, &fname->home); + fname->dup.cr_time = fname->dup.m_time = fname->dup.c_time = + fname->dup.a_time = std5->cr_time; + fname->dup.alloc_size = fname->dup.data_size = 0; + fname->dup.fa = std5->fa; + fname->dup.ea_size = fname->dup.reparse = 0; + + dsize = le16_to_cpu(new_de->key_size); + asize = QuadAlign(SIZEOF_RESIDENT + dsize); + + attr->type = ATTR_NAME; + attr->size = cpu_to_le32(asize); + attr->res.data_off = SIZEOF_RESIDENT_LE; + attr->res.flags = RESIDENT_FLAG_INDEXED; + attr->id = cpu_to_le16(aid++); + attr->res.data_size = cpu_to_le32(dsize); + memcpy(Add2Ptr(attr, SIZEOF_RESIDENT), fname, dsize); + + attr = Add2Ptr(attr, asize); + + if (security_id == SECURITY_ID_INVALID) { + /* Insert security attribute */ + asize = SIZEOF_RESIDENT + QuadAlign(sd_size); + + attr->type = ATTR_SECURE; + attr->size = cpu_to_le32(asize); + attr->id = cpu_to_le16(aid++); + attr->res.data_off = SIZEOF_RESIDENT_LE; + attr->res.data_size = cpu_to_le32(sd_size); + memcpy(Add2Ptr(attr, SIZEOF_RESIDENT), sd, sd_size); + + attr = Add2Ptr(attr, asize); + } + + if (fa & FILE_ATTRIBUTE_DIRECTORY) { + /* + * regular directory or symlink to directory + * Create root attribute + */ + dsize = sizeof(struct INDEX_ROOT) + sizeof(struct NTFS_DE); + asize = sizeof(I30_NAME) + SIZEOF_RESIDENT + dsize; + + attr->type = ATTR_ROOT; + attr->size = cpu_to_le32(asize); + attr->id = cpu_to_le16(aid++); + + attr->name_len = ARRAY_SIZE(I30_NAME); + attr->name_off = SIZEOF_RESIDENT_LE; + attr->res.data_off = + cpu_to_le16(sizeof(I30_NAME) + SIZEOF_RESIDENT); + attr->res.data_size = cpu_to_le32(dsize); + memcpy(Add2Ptr(attr, SIZEOF_RESIDENT), I30_NAME, + sizeof(I30_NAME)); + + root = Add2Ptr(attr, sizeof(I30_NAME) + SIZEOF_RESIDENT); + memcpy(root, dir_root, offsetof(struct INDEX_ROOT, ihdr)); + root->ihdr.de_off = + cpu_to_le32(sizeof(struct INDEX_HDR)); // 0x10 + root->ihdr.used = cpu_to_le32(sizeof(struct INDEX_HDR) + + sizeof(struct NTFS_DE)); + root->ihdr.total = root->ihdr.used; + + e = Add2Ptr(root, sizeof(struct INDEX_ROOT)); + e->size = cpu_to_le16(sizeof(struct NTFS_DE)); + e->flags = NTFS_IE_LAST; + } else if (is_link) { + /* + * symlink to file + * Create empty resident data attribute + */ + asize = SIZEOF_RESIDENT; + + /* insert empty ATTR_DATA */ + attr->type = ATTR_DATA; + attr->size = cpu_to_le32(SIZEOF_RESIDENT); + attr->id = cpu_to_le16(aid++); + attr->name_off = SIZEOF_RESIDENT_LE; + attr->res.data_off = SIZEOF_RESIDENT_LE; + } else { + /* + * regular file + */ + attr->type = ATTR_DATA; + attr->id = cpu_to_le16(aid++); + /* Create empty non resident data attribute */ + attr->non_res = 1; + attr->nres.evcn = cpu_to_le64(-1ll); + if (fa & FILE_ATTRIBUTE_SPARSE_FILE) { + attr->size = cpu_to_le32(SIZEOF_NONRESIDENT_EX + 8); + attr->name_off = SIZEOF_NONRESIDENT_EX_LE; + attr->flags = ATTR_FLAG_SPARSED; + asize = SIZEOF_NONRESIDENT_EX + 8; + } else if (fa & FILE_ATTRIBUTE_COMPRESSED) { + attr->size = cpu_to_le32(SIZEOF_NONRESIDENT_EX + 8); + attr->name_off = SIZEOF_NONRESIDENT_EX_LE; + attr->flags = ATTR_FLAG_COMPRESSED; + attr->nres.c_unit = COMPRESSION_UNIT; + asize = SIZEOF_NONRESIDENT_EX + 8; + } else { + attr->size = cpu_to_le32(SIZEOF_NONRESIDENT + 8); + attr->name_off = SIZEOF_NONRESIDENT_LE; + asize = SIZEOF_NONRESIDENT + 8; + } + attr->nres.run_off = attr->name_off; + } + + if (is_dir) { + ni->ni_flags |= NI_FLAG_DIR; + err = indx_init(&ni->dir, sbi, attr, INDEX_MUTEX_I30); + if (err) + goto out4; + } else if (is_link) { + rp = ntfs_create_reparse_buffer(sbi, symname, size, &nsize); + + if (IS_ERR(rp)) { + err = PTR_ERR(rp); + rp = NULL; + goto out4; + } + + /* + * Insert ATTR_REPARSE + */ + attr = Add2Ptr(attr, asize); + attr->type = ATTR_REPARSE; + attr->id = cpu_to_le16(aid++); + + /* resident or non resident? */ + asize = QuadAlign(SIZEOF_RESIDENT + nsize); + t16 = PtrOffset(rec, attr); + + if (asize + t16 + 8 > sbi->record_size) { + CLST alen; + CLST clst = bytes_to_cluster(sbi, nsize); + + /* bytes per runs */ + t16 = sbi->record_size - t16 - SIZEOF_NONRESIDENT; + + attr->non_res = 1; + attr->nres.evcn = cpu_to_le64(clst - 1); + attr->name_off = SIZEOF_NONRESIDENT_LE; + attr->nres.run_off = attr->name_off; + attr->nres.data_size = cpu_to_le64(nsize); + attr->nres.valid_size = attr->nres.data_size; + attr->nres.alloc_size = + cpu_to_le64(ntfs_up_cluster(sbi, nsize)); + + err = attr_allocate_clusters(sbi, &ni->file.run, 0, 0, + clst, NULL, 0, &alen, 0, + NULL); + if (err) + goto out5; + + err = run_pack(&ni->file.run, 0, clst, + Add2Ptr(attr, SIZEOF_NONRESIDENT), t16, + &vcn); + if (err < 0) + goto out5; + + if (vcn != clst) { + err = -EINVAL; + goto out5; + } + + asize = SIZEOF_NONRESIDENT + QuadAlign(err); + inode->i_size = nsize; + } else { + attr->res.data_off = SIZEOF_RESIDENT_LE; + attr->res.data_size = cpu_to_le32(nsize); + memcpy(Add2Ptr(attr, SIZEOF_RESIDENT), rp, nsize); + inode->i_size = nsize; + nsize = 0; + } + + attr->size = cpu_to_le32(asize); + + err = ntfs_insert_reparse(sbi, IO_REPARSE_TAG_SYMLINK, + &new_de->ref); + if (err) + goto out5; + + rp_inserted = true; + } + + attr = Add2Ptr(attr, asize); + attr->type = ATTR_END; + + rec->used = cpu_to_le32(PtrOffset(rec, attr) + 8); + rec->next_attr_id = cpu_to_le16(aid); + + /* Step 2: Add new name in index */ + err = indx_insert_entry(&dir_ni->dir, dir_ni, new_de, sbi, fnd); + if (err) + goto out6; + + /* Update current directory record */ + mark_inode_dirty(dir); + + /* Fill vfs inode fields */ + inode->i_uid = sbi->options.uid ? sbi->options.fs_uid : current_fsuid(); + inode->i_gid = sbi->options.gid ? sbi->options.fs_gid + : (dir->i_mode & S_ISGID) ? dir->i_gid + : current_fsgid(); + inode->i_generation = le16_to_cpu(rec->seq); + + dir->i_mtime = dir->i_ctime = inode->i_atime; + + if (is_dir) { + if (dir->i_mode & S_ISGID) + mode |= S_ISGID; + inode->i_op = &ntfs_dir_inode_operations; + inode->i_fop = &ntfs_dir_operations; + } else if (is_link) { + inode->i_op = &ntfs_link_inode_operations; + inode->i_fop = NULL; + inode->i_mapping->a_ops = &ntfs_aops; + } else { + inode->i_op = &ntfs_file_inode_operations; + inode->i_fop = &ntfs_file_operations; + inode->i_mapping->a_ops = + is_compressed(ni) ? &ntfs_aops_cmpr : &ntfs_aops; + init_rwsem(&ni->file.run_lock); + } + + inode->i_mode = mode; + +#ifdef CONFIG_NTFS3_FS_POSIX_ACL + if (!is_link && (sb->s_flags & SB_POSIXACL)) { + err = ntfs_init_acl(mnt_userns, inode, dir); + if (err) + goto out6; + } else +#endif + { + inode->i_flags |= S_NOSEC; + } + + /* Write non resident data */ + if (nsize) { + err = ntfs_sb_write_run(sbi, &ni->file.run, 0, rp, nsize); + if (err) + goto out7; + } + + /* call 'd_instantiate' after inode->i_op is set but before finish_open */ + d_instantiate(dentry, inode); + + mark_inode_dirty(inode); + mark_inode_dirty(dir); + + /* normal exit */ + goto out2; + +out7: + + /* undo 'indx_insert_entry' */ + indx_delete_entry(&dir_ni->dir, dir_ni, new_de + 1, + le16_to_cpu(new_de->key_size), sbi); +out6: + if (rp_inserted) + ntfs_remove_reparse(sbi, IO_REPARSE_TAG_SYMLINK, &new_de->ref); + +out5: + if (is_dir || run_is_empty(&ni->file.run)) + goto out4; + + run_deallocate(sbi, &ni->file.run, false); + +out4: + clear_rec_inuse(rec); + clear_nlink(inode); + ni->mi.dirty = false; + discard_new_inode(inode); +out3: + ntfs_mark_rec_free(sbi, ino); + +out2: + __putname(new_de); + ntfs_free(rp); + +out1: + if (err) + return ERR_PTR(err); + + unlock_new_inode(inode); + + return inode; +} + +int ntfs_link_inode(struct inode *inode, struct dentry *dentry) +{ + int err; + struct inode *dir = d_inode(dentry->d_parent); + struct ntfs_inode *dir_ni = ntfs_i(dir); + struct ntfs_inode *ni = ntfs_i(inode); + struct super_block *sb = inode->i_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + const struct qstr *name = &dentry->d_name; + struct NTFS_DE *new_de = NULL; + struct ATTR_FILE_NAME *fname; + struct ATTRIB *attr; + u16 key_size; + struct INDEX_ROOT *dir_root; + + dir_root = indx_get_root(&dir_ni->dir, dir_ni, NULL, NULL); + if (!dir_root) + return -EINVAL; + + /* allocate PATH_MAX bytes */ + new_de = __getname(); + if (!new_de) + return -ENOMEM; + + /*mark rw ntfs as dirty. it will be cleared at umount*/ + ntfs_set_state(ni->mi.sbi, NTFS_DIRTY_DIRTY); + + // Insert file name + err = fill_name_de(sbi, new_de, name, NULL); + if (err) + goto out; + + key_size = le16_to_cpu(new_de->key_size); + err = ni_insert_resident(ni, key_size, ATTR_NAME, NULL, 0, &attr, NULL); + if (err) + goto out; + + mi_get_ref(&ni->mi, &new_de->ref); + + fname = (struct ATTR_FILE_NAME *)(new_de + 1); + mi_get_ref(&dir_ni->mi, &fname->home); + fname->dup.cr_time = fname->dup.m_time = fname->dup.c_time = + fname->dup.a_time = kernel2nt(&inode->i_ctime); + fname->dup.alloc_size = fname->dup.data_size = 0; + fname->dup.fa = ni->std_fa; + fname->dup.ea_size = fname->dup.reparse = 0; + + memcpy(Add2Ptr(attr, SIZEOF_RESIDENT), fname, key_size); + + err = indx_insert_entry(&dir_ni->dir, dir_ni, new_de, sbi, NULL); + if (err) + goto out; + + le16_add_cpu(&ni->mi.mrec->hard_links, 1); + ni->mi.dirty = true; + +out: + __putname(new_de); + return err; +} + +/* + * ntfs_unlink_inode + * + * inode_operations::unlink + * inode_operations::rmdir + */ +int ntfs_unlink_inode(struct inode *dir, const struct dentry *dentry) +{ + int err; + struct super_block *sb = dir->i_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + struct inode *inode = d_inode(dentry); + struct ntfs_inode *ni = ntfs_i(inode); + const struct qstr *name = &dentry->d_name; + struct ntfs_inode *dir_ni = ntfs_i(dir); + struct ntfs_index *indx = &dir_ni->dir; + struct cpu_str *uni = NULL; + struct ATTR_FILE_NAME *fname; + u8 name_type; + struct ATTR_LIST_ENTRY *le; + struct MFT_REF ref; + bool is_dir = S_ISDIR(inode->i_mode); + struct INDEX_ROOT *dir_root; + + dir_root = indx_get_root(indx, dir_ni, NULL, NULL); + if (!dir_root) + return -EINVAL; + + ni_lock(ni); + + if (is_dir && !dir_is_empty(inode)) { + err = -ENOTEMPTY; + goto out1; + } + + if (ntfs_is_meta_file(sbi, inode->i_ino)) { + err = -EINVAL; + goto out1; + } + + /* allocate PATH_MAX bytes */ + uni = __getname(); + if (!uni) { + err = -ENOMEM; + goto out1; + } + + /* Convert input string to unicode */ + err = ntfs_nls_to_utf16(sbi, name->name, name->len, uni, NTFS_NAME_LEN, + UTF16_HOST_ENDIAN); + if (err < 0) + goto out2; + + /*mark rw ntfs as dirty. it will be cleared at umount*/ + ntfs_set_state(sbi, NTFS_DIRTY_DIRTY); + + /* find name in record */ + mi_get_ref(&dir_ni->mi, &ref); + + le = NULL; + fname = ni_fname_name(ni, uni, &ref, &le); + if (!fname) { + err = -ENOENT; + goto out3; + } + + name_type = paired_name(fname->type); + + err = indx_delete_entry(indx, dir_ni, fname, fname_full_size(fname), + sbi); + if (err) + goto out3; + + /* Then remove name from mft */ + ni_remove_attr_le(ni, attr_from_name(fname), le); + + le16_add_cpu(&ni->mi.mrec->hard_links, -1); + ni->mi.dirty = true; + + if (name_type != FILE_NAME_POSIX) { + /* Now we should delete name by type */ + fname = ni_fname_type(ni, name_type, &le); + if (fname) { + err = indx_delete_entry(indx, dir_ni, fname, + fname_full_size(fname), sbi); + if (err) + goto out3; + + ni_remove_attr_le(ni, attr_from_name(fname), le); + + le16_add_cpu(&ni->mi.mrec->hard_links, -1); + } + } +out3: + switch (err) { + case 0: + drop_nlink(inode); + case -ENOTEMPTY: + case -ENOSPC: + case -EROFS: + break; + default: + make_bad_inode(inode); + } + + dir->i_mtime = dir->i_ctime = current_time(dir); + mark_inode_dirty(dir); + inode->i_ctime = dir->i_ctime; + if (inode->i_nlink) + mark_inode_dirty(inode); + +out2: + __putname(uni); +out1: + ni_unlock(ni); + return err; +} + +void ntfs_evict_inode(struct inode *inode) +{ + truncate_inode_pages_final(&inode->i_data); + + if (inode->i_nlink) + _ni_write_inode(inode, inode_needs_sync(inode)); + + invalidate_inode_buffers(inode); + clear_inode(inode); + + ni_clear(ntfs_i(inode)); +} + +static noinline int ntfs_readlink_hlp(struct inode *inode, char *buffer, + int buflen) +{ + int i, err = 0; + struct ntfs_inode *ni = ntfs_i(inode); + struct super_block *sb = inode->i_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + u64 i_size = inode->i_size; + u16 nlen = 0; + void *to_free = NULL; + struct REPARSE_DATA_BUFFER *rp; + struct le_str *uni; + struct ATTRIB *attr; + + /* Reparse data present. Try to parse it */ + static_assert(!offsetof(struct REPARSE_DATA_BUFFER, ReparseTag)); + static_assert(sizeof(u32) == sizeof(rp->ReparseTag)); + + *buffer = 0; + + /* Read into temporal buffer */ + if (i_size > sbi->reparse.max_size || i_size <= sizeof(u32)) { + err = -EINVAL; + goto out; + } + + attr = ni_find_attr(ni, NULL, NULL, ATTR_REPARSE, NULL, 0, NULL, NULL); + if (!attr) { + err = -EINVAL; + goto out; + } + + if (!attr->non_res) { + rp = resident_data_ex(attr, i_size); + if (!rp) { + err = -EINVAL; + goto out; + } + } else { + rp = ntfs_malloc(i_size); + if (!rp) { + err = -ENOMEM; + goto out; + } + to_free = rp; + err = ntfs_read_run_nb(sbi, &ni->file.run, 0, rp, i_size, NULL); + if (err) + goto out; + } + + err = -EINVAL; + + /* Microsoft Tag */ + switch (rp->ReparseTag) { + case IO_REPARSE_TAG_MOUNT_POINT: + /* Mount points and junctions */ + /* Can we use 'Rp->MountPointReparseBuffer.PrintNameLength'? */ + if (i_size <= offsetof(struct REPARSE_DATA_BUFFER, + MountPointReparseBuffer.PathBuffer)) + goto out; + uni = Add2Ptr(rp, + offsetof(struct REPARSE_DATA_BUFFER, + MountPointReparseBuffer.PathBuffer) + + le16_to_cpu(rp->MountPointReparseBuffer + .PrintNameOffset) - + 2); + nlen = le16_to_cpu(rp->MountPointReparseBuffer.PrintNameLength); + break; + + case IO_REPARSE_TAG_SYMLINK: + /* FolderSymbolicLink */ + /* Can we use 'Rp->SymbolicLinkReparseBuffer.PrintNameLength'? */ + if (i_size <= offsetof(struct REPARSE_DATA_BUFFER, + SymbolicLinkReparseBuffer.PathBuffer)) + goto out; + uni = Add2Ptr(rp, + offsetof(struct REPARSE_DATA_BUFFER, + SymbolicLinkReparseBuffer.PathBuffer) + + le16_to_cpu(rp->SymbolicLinkReparseBuffer + .PrintNameOffset) - + 2); + nlen = le16_to_cpu( + rp->SymbolicLinkReparseBuffer.PrintNameLength); + break; + + case IO_REPARSE_TAG_CLOUD: + case IO_REPARSE_TAG_CLOUD_1: + case IO_REPARSE_TAG_CLOUD_2: + case IO_REPARSE_TAG_CLOUD_3: + case IO_REPARSE_TAG_CLOUD_4: + case IO_REPARSE_TAG_CLOUD_5: + case IO_REPARSE_TAG_CLOUD_6: + case IO_REPARSE_TAG_CLOUD_7: + case IO_REPARSE_TAG_CLOUD_8: + case IO_REPARSE_TAG_CLOUD_9: + case IO_REPARSE_TAG_CLOUD_A: + case IO_REPARSE_TAG_CLOUD_B: + case IO_REPARSE_TAG_CLOUD_C: + case IO_REPARSE_TAG_CLOUD_D: + case IO_REPARSE_TAG_CLOUD_E: + case IO_REPARSE_TAG_CLOUD_F: + err = sizeof("OneDrive") - 1; + if (err > buflen) + err = buflen; + memcpy(buffer, "OneDrive", err); + goto out; + + default: + if (IsReparseTagMicrosoft(rp->ReparseTag)) { + /* unknown Microsoft Tag */ + goto out; + } + if (!IsReparseTagNameSurrogate(rp->ReparseTag) || + i_size <= sizeof(struct REPARSE_POINT)) { + goto out; + } + + /* Users tag */ + uni = Add2Ptr(rp, sizeof(struct REPARSE_POINT) - 2); + nlen = le16_to_cpu(rp->ReparseDataLength) - + sizeof(struct REPARSE_POINT); + } + + /* Convert nlen from bytes to UNICODE chars */ + nlen >>= 1; + + /* Check that name is available */ + if (!nlen || &uni->name[nlen] > (__le16 *)Add2Ptr(rp, i_size)) + goto out; + + /* If name is already zero terminated then truncate it now */ + if (!uni->name[nlen - 1]) + nlen -= 1; + uni->len = nlen; + + err = ntfs_utf16_to_nls(sbi, uni, buffer, buflen); + + if (err < 0) + goto out; + + /* translate windows '\' into linux '/' */ + for (i = 0; i < err; i++) { + if (buffer[i] == '\\') + buffer[i] = '/'; + } + + /* Always set last zero */ + buffer[err] = 0; +out: + ntfs_free(to_free); + return err; +} + +static const char *ntfs_get_link(struct dentry *de, struct inode *inode, + struct delayed_call *done) +{ + int err; + char *ret; + + if (!de) + return ERR_PTR(-ECHILD); + + ret = kmalloc(PAGE_SIZE, GFP_NOFS); + if (!ret) + return ERR_PTR(-ENOMEM); + + err = ntfs_readlink_hlp(inode, ret, PAGE_SIZE); + if (err < 0) { + kfree(ret); + return ERR_PTR(err); + } + + set_delayed_call(done, kfree_link, ret); + + return ret; +} + +const struct inode_operations ntfs_link_inode_operations = { + .get_link = ntfs_get_link, + .setattr = ntfs3_setattr, + .listxattr = ntfs_listxattr, + .permission = ntfs_permission, + .get_acl = ntfs_get_acl, + .set_acl = ntfs_set_acl, +}; + +const struct address_space_operations ntfs_aops = { + .readpage = ntfs_readpage, + .readahead = ntfs_readahead, + .writepage = ntfs_writepage, + .writepages = ntfs_writepages, + .write_begin = ntfs_write_begin, + .write_end = ntfs_write_end, + .direct_IO = ntfs_direct_IO, + .bmap = ntfs_bmap, +}; + +const struct address_space_operations ntfs_aops_cmpr = { + .readpage = ntfs_readpage, + .readahead = ntfs_readahead, +}; diff --git a/fs/ntfs3/lib/decompress_common.c b/fs/ntfs3/lib/decompress_common.c new file mode 100644 index 000000000000..83c9e93aea77 --- /dev/null +++ b/fs/ntfs3/lib/decompress_common.c @@ -0,0 +1,332 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * decompress_common.c - Code shared by the XPRESS and LZX decompressors + * + * Copyright (C) 2015 Eric Biggers + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ + +#include "decompress_common.h" + +/* + * make_huffman_decode_table() - + * + * Build a decoding table for a canonical prefix code, or "Huffman code". + * + * This is an internal function, not part of the library API! + * + * This takes as input the length of the codeword for each symbol in the + * alphabet and produces as output a table that can be used for fast + * decoding of prefix-encoded symbols using read_huffsym(). + * + * Strictly speaking, a canonical prefix code might not be a Huffman + * code. But this algorithm will work either way; and in fact, since + * Huffman codes are defined in terms of symbol frequencies, there is no + * way for the decompressor to know whether the code is a true Huffman + * code or not until all symbols have been decoded. + * + * Because the prefix code is assumed to be "canonical", it can be + * reconstructed directly from the codeword lengths. A prefix code is + * canonical if and only if a longer codeword never lexicographically + * precedes a shorter codeword, and the lexicographic ordering of + * codewords of the same length is the same as the lexicographic ordering + * of the corresponding symbols. Consequently, we can sort the symbols + * primarily by codeword length and secondarily by symbol value, then + * reconstruct the prefix code by generating codewords lexicographically + * in that order. + * + * This function does not, however, generate the prefix code explicitly. + * Instead, it directly builds a table for decoding symbols using the + * code. The basic idea is this: given the next 'max_codeword_len' bits + * in the input, we can look up the decoded symbol by indexing a table + * containing 2**max_codeword_len entries. A codeword with length + * 'max_codeword_len' will have exactly one entry in this table, whereas + * a codeword shorter than 'max_codeword_len' will have multiple entries + * in this table. Precisely, a codeword of length n will be represented + * by 2**(max_codeword_len - n) entries in this table. The 0-based index + * of each such entry will contain the corresponding codeword as a prefix + * when zero-padded on the left to 'max_codeword_len' binary digits. + * + * That's the basic idea, but we implement two optimizations regarding + * the format of the decode table itself: + * + * - For many compression formats, the maximum codeword length is too + * long for it to be efficient to build the full decoding table + * whenever a new prefix code is used. Instead, we can build the table + * using only 2**table_bits entries, where 'table_bits' is some number + * less than or equal to 'max_codeword_len'. Then, only codewords of + * length 'table_bits' and shorter can be directly looked up. For + * longer codewords, the direct lookup instead produces the root of a + * binary tree. Using this tree, the decoder can do traditional + * bit-by-bit decoding of the remainder of the codeword. Child nodes + * are allocated in extra entries at the end of the table; leaf nodes + * contain symbols. Note that the long-codeword case is, in general, + * not performance critical, since in Huffman codes the most frequently + * used symbols are assigned the shortest codeword lengths. + * + * - When we decode a symbol using a direct lookup of the table, we still + * need to know its length so that the bitstream can be advanced by the + * appropriate number of bits. The simple solution is to simply retain + * the 'lens' array and use the decoded symbol as an index into it. + * However, this requires two separate array accesses in the fast path. + * The optimization is to store the length directly in the decode + * table. We use the bottom 11 bits for the symbol and the top 5 bits + * for the length. In addition, to combine this optimization with the + * previous one, we introduce a special case where the top 2 bits of + * the length are both set if the entry is actually the root of a + * binary tree. + * + * @decode_table: + * The array in which to create the decoding table. This must have + * a length of at least ((2**table_bits) + 2 * num_syms) entries. + * + * @num_syms: + * The number of symbols in the alphabet; also, the length of the + * 'lens' array. Must be less than or equal to 2048. + * + * @table_bits: + * The order of the decode table size, as explained above. Must be + * less than or equal to 13. + * + * @lens: + * An array of length @num_syms, indexable by symbol, that gives the + * length of the codeword, in bits, for that symbol. The length can + * be 0, which means that the symbol does not have a codeword + * assigned. + * + * @max_codeword_len: + * The longest codeword length allowed in the compression format. + * All entries in 'lens' must be less than or equal to this value. + * This must be less than or equal to 23. + * + * @working_space + * A temporary array of length '2 * (max_codeword_len + 1) + + * num_syms'. + * + * Returns 0 on success, or -1 if the lengths do not form a valid prefix + * code. + */ +int make_huffman_decode_table(u16 decode_table[], const u32 num_syms, + const u32 table_bits, const u8 lens[], + const u32 max_codeword_len, + u16 working_space[]) +{ + const u32 table_num_entries = 1 << table_bits; + u16 * const len_counts = &working_space[0]; + u16 * const offsets = &working_space[1 * (max_codeword_len + 1)]; + u16 * const sorted_syms = &working_space[2 * (max_codeword_len + 1)]; + int left; + void *decode_table_ptr; + u32 sym_idx; + u32 codeword_len; + u32 stores_per_loop; + u32 decode_table_pos; + u32 len; + u32 sym; + + /* Count how many symbols have each possible codeword length. + * Note that a length of 0 indicates the corresponding symbol is not + * used in the code and therefore does not have a codeword. + */ + for (len = 0; len <= max_codeword_len; len++) + len_counts[len] = 0; + for (sym = 0; sym < num_syms; sym++) + len_counts[lens[sym]]++; + + /* We can assume all lengths are <= max_codeword_len, but we + * cannot assume they form a valid prefix code. A codeword of + * length n should require a proportion of the codespace equaling + * (1/2)^n. The code is valid if and only if the codespace is + * exactly filled by the lengths, by this measure. + */ + left = 1; + for (len = 1; len <= max_codeword_len; len++) { + left <<= 1; + left -= len_counts[len]; + if (left < 0) { + /* The lengths overflow the codespace; that is, the code + * is over-subscribed. + */ + return -1; + } + } + + if (left) { + /* The lengths do not fill the codespace; that is, they form an + * incomplete set. + */ + if (left == (1 << max_codeword_len)) { + /* The code is completely empty. This is arguably + * invalid, but in fact it is valid in LZX and XPRESS, + * so we must allow it. By definition, no symbols can + * be decoded with an empty code. Consequently, we + * technically don't even need to fill in the decode + * table. However, to avoid accessing uninitialized + * memory if the algorithm nevertheless attempts to + * decode symbols using such a code, we zero out the + * decode table. + */ + memset(decode_table, 0, + table_num_entries * sizeof(decode_table[0])); + return 0; + } + return -1; + } + + /* Sort the symbols primarily by length and secondarily by symbol order. + */ + + /* Initialize 'offsets' so that offsets[len] for 1 <= len <= + * max_codeword_len is the number of codewords shorter than 'len' bits. + */ + offsets[1] = 0; + for (len = 1; len < max_codeword_len; len++) + offsets[len + 1] = offsets[len] + len_counts[len]; + + /* Use the 'offsets' array to sort the symbols. Note that we do not + * include symbols that are not used in the code. Consequently, fewer + * than 'num_syms' entries in 'sorted_syms' may be filled. + */ + for (sym = 0; sym < num_syms; sym++) + if (lens[sym]) + sorted_syms[offsets[lens[sym]]++] = sym; + + /* Fill entries for codewords with length <= table_bits + * --- that is, those short enough for a direct mapping. + * + * The table will start with entries for the shortest codeword(s), which + * have the most entries. From there, the number of entries per + * codeword will decrease. + */ + decode_table_ptr = decode_table; + sym_idx = 0; + codeword_len = 1; + stores_per_loop = (1 << (table_bits - codeword_len)); + for (; stores_per_loop != 0; codeword_len++, stores_per_loop >>= 1) { + u32 end_sym_idx = sym_idx + len_counts[codeword_len]; + + for (; sym_idx < end_sym_idx; sym_idx++) { + u16 entry; + u16 *p; + u32 n; + + entry = ((u32)codeword_len << 11) | sorted_syms[sym_idx]; + p = (u16 *)decode_table_ptr; + n = stores_per_loop; + + do { + *p++ = entry; + } while (--n); + + decode_table_ptr = p; + } + } + + /* If we've filled in the entire table, we are done. Otherwise, + * there are codewords longer than table_bits for which we must + * generate binary trees. + */ + decode_table_pos = (u16 *)decode_table_ptr - decode_table; + if (decode_table_pos != table_num_entries) { + u32 j; + u32 next_free_tree_slot; + u32 cur_codeword; + + /* First, zero out the remaining entries. This is + * necessary so that these entries appear as + * "unallocated" in the next part. Each of these entries + * will eventually be filled with the representation of + * the root node of a binary tree. + */ + j = decode_table_pos; + do { + decode_table[j] = 0; + } while (++j != table_num_entries); + + /* We allocate child nodes starting at the end of the + * direct lookup table. Note that there should be + * 2*num_syms extra entries for this purpose, although + * fewer than this may actually be needed. + */ + next_free_tree_slot = table_num_entries; + + /* Iterate through each codeword with length greater than + * 'table_bits', primarily in order of codeword length + * and secondarily in order of symbol. + */ + for (cur_codeword = decode_table_pos << 1; + codeword_len <= max_codeword_len; + codeword_len++, cur_codeword <<= 1) { + u32 end_sym_idx = sym_idx + len_counts[codeword_len]; + + for (; sym_idx < end_sym_idx; sym_idx++, cur_codeword++) { + /* 'sorted_sym' is the symbol represented by the + * codeword. + */ + u32 sorted_sym = sorted_syms[sym_idx]; + u32 extra_bits = codeword_len - table_bits; + u32 node_idx = cur_codeword >> extra_bits; + + /* Go through each bit of the current codeword + * beyond the prefix of length @table_bits and + * walk the appropriate binary tree, allocating + * any slots that have not yet been allocated. + * + * Note that the 'pointer' entry to the binary + * tree, which is stored in the direct lookup + * portion of the table, is represented + * identically to other internal (non-leaf) + * nodes of the binary tree; it can be thought + * of as simply the root of the tree. The + * representation of these internal nodes is + * simply the index of the left child combined + * with the special bits 0xC000 to distingush + * the entry from direct mapping and leaf node + * entries. + */ + do { + /* At least one bit remains in the + * codeword, but the current node is an + * unallocated leaf. Change it to an + * internal node. + */ + if (decode_table[node_idx] == 0) { + decode_table[node_idx] = + next_free_tree_slot | 0xC000; + decode_table[next_free_tree_slot++] = 0; + decode_table[next_free_tree_slot++] = 0; + } + + /* Go to the left child if the next bit + * in the codeword is 0; otherwise go to + * the right child. + */ + node_idx = decode_table[node_idx] & 0x3FFF; + --extra_bits; + node_idx += (cur_codeword >> extra_bits) & 1; + } while (extra_bits != 0); + + /* We've traversed the tree using the entire + * codeword, and we're now at the entry where + * the actual symbol will be stored. This is + * distinguished from internal nodes by not + * having its high two bits set. + */ + decode_table[node_idx] = sorted_sym; + } + } + } + return 0; +} diff --git a/fs/ntfs3/lib/decompress_common.h b/fs/ntfs3/lib/decompress_common.h new file mode 100644 index 000000000000..66297f398403 --- /dev/null +++ b/fs/ntfs3/lib/decompress_common.h @@ -0,0 +1,352 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +/* + * decompress_common.h - Code shared by the XPRESS and LZX decompressors + * + * Copyright (C) 2015 Eric Biggers + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ + +#include +#include +#include +#include +#include + + +/* "Force inline" macro (not required, but helpful for performance) */ +#define forceinline __always_inline + +/* Enable whole-word match copying on selected architectures */ +#if defined(__i386__) || defined(__x86_64__) || defined(__ARM_FEATURE_UNALIGNED) +# define FAST_UNALIGNED_ACCESS +#endif + +/* Size of a machine word */ +#define WORDBYTES (sizeof(size_t)) + +static forceinline void +copy_unaligned_word(const void *src, void *dst) +{ + put_unaligned(get_unaligned((const size_t *)src), (size_t *)dst); +} + + +/* Generate a "word" with platform-dependent size whose bytes all contain the + * value 'b'. + */ +static forceinline size_t repeat_byte(u8 b) +{ + size_t v; + + v = b; + v |= v << 8; + v |= v << 16; + v |= v << ((WORDBYTES == 8) ? 32 : 0); + return v; +} + +/* Structure that encapsulates a block of in-memory data being interpreted as a + * stream of bits, optionally with interwoven literal bytes. Bits are assumed + * to be stored in little endian 16-bit coding units, with the bits ordered high + * to low. + */ +struct input_bitstream { + + /* Bits that have been read from the input buffer. The bits are + * left-justified; the next bit is always bit 31. + */ + u32 bitbuf; + + /* Number of bits currently held in @bitbuf. */ + u32 bitsleft; + + /* Pointer to the next byte to be retrieved from the input buffer. */ + const u8 *next; + + /* Pointer to just past the end of the input buffer. */ + const u8 *end; +}; + +/* Initialize a bitstream to read from the specified input buffer. */ +static forceinline void init_input_bitstream(struct input_bitstream *is, + const void *buffer, u32 size) +{ + is->bitbuf = 0; + is->bitsleft = 0; + is->next = buffer; + is->end = is->next + size; +} + +/* Ensure the bit buffer variable for the bitstream contains at least @num_bits + * bits. Following this, bitstream_peek_bits() and/or bitstream_remove_bits() + * may be called on the bitstream to peek or remove up to @num_bits bits. Note + * that @num_bits must be <= 16. + */ +static forceinline void bitstream_ensure_bits(struct input_bitstream *is, + u32 num_bits) +{ + if (is->bitsleft < num_bits) { + if (is->end - is->next >= 2) { + is->bitbuf |= (u32)get_unaligned_le16(is->next) + << (16 - is->bitsleft); + is->next += 2; + } + is->bitsleft += 16; + } +} + +/* Return the next @num_bits bits from the bitstream, without removing them. + * There must be at least @num_bits remaining in the buffer variable, from a + * previous call to bitstream_ensure_bits(). + */ +static forceinline u32 +bitstream_peek_bits(const struct input_bitstream *is, const u32 num_bits) +{ + return (is->bitbuf >> 1) >> (sizeof(is->bitbuf) * 8 - num_bits - 1); +} + +/* Remove @num_bits from the bitstream. There must be at least @num_bits + * remaining in the buffer variable, from a previous call to + * bitstream_ensure_bits(). + */ +static forceinline void +bitstream_remove_bits(struct input_bitstream *is, u32 num_bits) +{ + is->bitbuf <<= num_bits; + is->bitsleft -= num_bits; +} + +/* Remove and return @num_bits bits from the bitstream. There must be at least + * @num_bits remaining in the buffer variable, from a previous call to + * bitstream_ensure_bits(). + */ +static forceinline u32 +bitstream_pop_bits(struct input_bitstream *is, u32 num_bits) +{ + u32 bits = bitstream_peek_bits(is, num_bits); + + bitstream_remove_bits(is, num_bits); + return bits; +} + +/* Read and return the next @num_bits bits from the bitstream. */ +static forceinline u32 +bitstream_read_bits(struct input_bitstream *is, u32 num_bits) +{ + bitstream_ensure_bits(is, num_bits); + return bitstream_pop_bits(is, num_bits); +} + +/* Read and return the next literal byte embedded in the bitstream. */ +static forceinline u8 +bitstream_read_byte(struct input_bitstream *is) +{ + if (unlikely(is->end == is->next)) + return 0; + return *is->next++; +} + +/* Read and return the next 16-bit integer embedded in the bitstream. */ +static forceinline u16 +bitstream_read_u16(struct input_bitstream *is) +{ + u16 v; + + if (unlikely(is->end - is->next < 2)) + return 0; + v = get_unaligned_le16(is->next); + is->next += 2; + return v; +} + +/* Read and return the next 32-bit integer embedded in the bitstream. */ +static forceinline u32 +bitstream_read_u32(struct input_bitstream *is) +{ + u32 v; + + if (unlikely(is->end - is->next < 4)) + return 0; + v = get_unaligned_le32(is->next); + is->next += 4; + return v; +} + +/* Read into @dst_buffer an array of literal bytes embedded in the bitstream. + * Return either a pointer to the byte past the last written, or NULL if the + * read overflows the input buffer. + */ +static forceinline void *bitstream_read_bytes(struct input_bitstream *is, + void *dst_buffer, size_t count) +{ + if ((size_t)(is->end - is->next) < count) + return NULL; + memcpy(dst_buffer, is->next, count); + is->next += count; + return (u8 *)dst_buffer + count; +} + +/* Align the input bitstream on a coding-unit boundary. */ +static forceinline void bitstream_align(struct input_bitstream *is) +{ + is->bitsleft = 0; + is->bitbuf = 0; +} + +extern int make_huffman_decode_table(u16 decode_table[], const u32 num_syms, + const u32 num_bits, const u8 lens[], + const u32 max_codeword_len, + u16 working_space[]); + + +/* Reads and returns the next Huffman-encoded symbol from a bitstream. If the + * input data is exhausted, the Huffman symbol is decoded as if the missing bits + * are all zeroes. + */ +static forceinline u32 read_huffsym(struct input_bitstream *istream, + const u16 decode_table[], + u32 table_bits, + u32 max_codeword_len) +{ + u32 entry; + u32 key_bits; + + bitstream_ensure_bits(istream, max_codeword_len); + + /* Index the decode table by the next table_bits bits of the input. */ + key_bits = bitstream_peek_bits(istream, table_bits); + entry = decode_table[key_bits]; + if (entry < 0xC000) { + /* Fast case: The decode table directly provided the + * symbol and codeword length. The low 11 bits are the + * symbol, and the high 5 bits are the codeword length. + */ + bitstream_remove_bits(istream, entry >> 11); + return entry & 0x7FF; + } + /* Slow case: The codeword for the symbol is longer than + * table_bits, so the symbol does not have an entry + * directly in the first (1 << table_bits) entries of the + * decode table. Traverse the appropriate binary tree + * bit-by-bit to decode the symbol. + */ + bitstream_remove_bits(istream, table_bits); + do { + key_bits = (entry & 0x3FFF) + bitstream_pop_bits(istream, 1); + } while ((entry = decode_table[key_bits]) >= 0xC000); + return entry; +} + +/* + * Copy an LZ77 match at (dst - offset) to dst. + * + * The length and offset must be already validated --- that is, (dst - offset) + * can't underrun the output buffer, and (dst + length) can't overrun the output + * buffer. Also, the length cannot be 0. + * + * @bufend points to the byte past the end of the output buffer. This function + * won't write any data beyond this position. + * + * Returns dst + length. + */ +static forceinline u8 *lz_copy(u8 *dst, u32 length, u32 offset, const u8 *bufend, + u32 min_length) +{ + const u8 *src = dst - offset; + + /* + * Try to copy one machine word at a time. On i386 and x86_64 this is + * faster than copying one byte at a time, unless the data is + * near-random and all the matches have very short lengths. Note that + * since this requires unaligned memory accesses, it won't necessarily + * be faster on every architecture. + * + * Also note that we might copy more than the length of the match. For + * example, if a word is 8 bytes and the match is of length 5, then + * we'll simply copy 8 bytes. This is okay as long as we don't write + * beyond the end of the output buffer, hence the check for (bufend - + * end >= WORDBYTES - 1). + */ +#ifdef FAST_UNALIGNED_ACCESS + u8 * const end = dst + length; + + if (bufend - end >= (ptrdiff_t)(WORDBYTES - 1)) { + + if (offset >= WORDBYTES) { + /* The source and destination words don't overlap. */ + + /* To improve branch prediction, one iteration of this + * loop is unrolled. Most matches are short and will + * fail the first check. But if that check passes, then + * it becomes increasing likely that the match is long + * and we'll need to continue copying. + */ + + copy_unaligned_word(src, dst); + src += WORDBYTES; + dst += WORDBYTES; + + if (dst < end) { + do { + copy_unaligned_word(src, dst); + src += WORDBYTES; + dst += WORDBYTES; + } while (dst < end); + } + return end; + } else if (offset == 1) { + + /* Offset 1 matches are equivalent to run-length + * encoding of the previous byte. This case is common + * if the data contains many repeated bytes. + */ + size_t v = repeat_byte(*(dst - 1)); + + do { + put_unaligned(v, (size_t *)dst); + src += WORDBYTES; + dst += WORDBYTES; + } while (dst < end); + return end; + } + /* + * We don't bother with special cases for other 'offset < + * WORDBYTES', which are usually rarer than 'offset == 1'. Extra + * checks will just slow things down. Actually, it's possible + * to handle all the 'offset < WORDBYTES' cases using the same + * code, but it still becomes more complicated doesn't seem any + * faster overall; it definitely slows down the more common + * 'offset == 1' case. + */ + } +#endif /* FAST_UNALIGNED_ACCESS */ + + /* Fall back to a bytewise copy. */ + + if (min_length >= 2) { + *dst++ = *src++; + length--; + } + if (min_length >= 3) { + *dst++ = *src++; + length--; + } + do { + *dst++ = *src++; + } while (--length); + + return dst; +} diff --git a/fs/ntfs3/lib/lib.h b/fs/ntfs3/lib/lib.h new file mode 100644 index 000000000000..f508fbad2e71 --- /dev/null +++ b/fs/ntfs3/lib/lib.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Adapted for linux kernel by Alexander Mamaev: + * - remove implementations of get_unaligned_ + * - assume GCC is always defined + * - ISO C90 + * - linux kernel code style + */ + + +/* globals from xpress_decompress.c */ +struct xpress_decompressor *xpress_allocate_decompressor(void); +void xpress_free_decompressor(struct xpress_decompressor *d); +int xpress_decompress(struct xpress_decompressor *__restrict d, + const void *__restrict compressed_data, + size_t compressed_size, + void *__restrict uncompressed_data, + size_t uncompressed_size); + +/* globals from lzx_decompress.c */ +struct lzx_decompressor *lzx_allocate_decompressor(void); +void lzx_free_decompressor(struct lzx_decompressor *d); +int lzx_decompress(struct lzx_decompressor *__restrict d, + const void *__restrict compressed_data, + size_t compressed_size, void *__restrict uncompressed_data, + size_t uncompressed_size); diff --git a/fs/ntfs3/lib/lzx_decompress.c b/fs/ntfs3/lib/lzx_decompress.c new file mode 100644 index 000000000000..77a381a693d1 --- /dev/null +++ b/fs/ntfs3/lib/lzx_decompress.c @@ -0,0 +1,683 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * lzx_decompress.c - A decompressor for the LZX compression format, which can + * be used in "System Compressed" files. This is based on the code from wimlib. + * This code only supports a window size (dictionary size) of 32768 bytes, since + * this is the only size used in System Compression. + * + * Copyright (C) 2015 Eric Biggers + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ + +#include "decompress_common.h" +#include "lib.h" + +/* Number of literal byte values */ +#define LZX_NUM_CHARS 256 + +/* The smallest and largest allowed match lengths */ +#define LZX_MIN_MATCH_LEN 2 +#define LZX_MAX_MATCH_LEN 257 + +/* Number of distinct match lengths that can be represented */ +#define LZX_NUM_LENS (LZX_MAX_MATCH_LEN - LZX_MIN_MATCH_LEN + 1) + +/* Number of match lengths for which no length symbol is required */ +#define LZX_NUM_PRIMARY_LENS 7 +#define LZX_NUM_LEN_HEADERS (LZX_NUM_PRIMARY_LENS + 1) + +/* Valid values of the 3-bit block type field */ +#define LZX_BLOCKTYPE_VERBATIM 1 +#define LZX_BLOCKTYPE_ALIGNED 2 +#define LZX_BLOCKTYPE_UNCOMPRESSED 3 + +/* Number of offset slots for a window size of 32768 */ +#define LZX_NUM_OFFSET_SLOTS 30 + +/* Number of symbols in the main code for a window size of 32768 */ +#define LZX_MAINCODE_NUM_SYMBOLS \ + (LZX_NUM_CHARS + (LZX_NUM_OFFSET_SLOTS * LZX_NUM_LEN_HEADERS)) + +/* Number of symbols in the length code */ +#define LZX_LENCODE_NUM_SYMBOLS (LZX_NUM_LENS - LZX_NUM_PRIMARY_LENS) + +/* Number of symbols in the precode */ +#define LZX_PRECODE_NUM_SYMBOLS 20 + +/* Number of bits in which each precode codeword length is represented */ +#define LZX_PRECODE_ELEMENT_SIZE 4 + +/* Number of low-order bits of each match offset that are entropy-encoded in + * aligned offset blocks + */ +#define LZX_NUM_ALIGNED_OFFSET_BITS 3 + +/* Number of symbols in the aligned offset code */ +#define LZX_ALIGNEDCODE_NUM_SYMBOLS (1 << LZX_NUM_ALIGNED_OFFSET_BITS) + +/* Mask for the match offset bits that are entropy-encoded in aligned offset + * blocks + */ +#define LZX_ALIGNED_OFFSET_BITMASK ((1 << LZX_NUM_ALIGNED_OFFSET_BITS) - 1) + +/* Number of bits in which each aligned offset codeword length is represented */ +#define LZX_ALIGNEDCODE_ELEMENT_SIZE 3 + +/* Maximum lengths (in bits) of the codewords in each Huffman code */ +#define LZX_MAX_MAIN_CODEWORD_LEN 16 +#define LZX_MAX_LEN_CODEWORD_LEN 16 +#define LZX_MAX_PRE_CODEWORD_LEN ((1 << LZX_PRECODE_ELEMENT_SIZE) - 1) +#define LZX_MAX_ALIGNED_CODEWORD_LEN ((1 << LZX_ALIGNEDCODE_ELEMENT_SIZE) - 1) + +/* The default "filesize" value used in pre/post-processing. In the LZX format + * used in cabinet files this value must be given to the decompressor, whereas + * in the LZX format used in WIM files and system-compressed files this value is + * fixed at 12000000. + */ +#define LZX_DEFAULT_FILESIZE 12000000 + +/* Assumed block size when the encoded block size begins with a 0 bit. */ +#define LZX_DEFAULT_BLOCK_SIZE 32768 + +/* Number of offsets in the recent (or "repeat") offsets queue. */ +#define LZX_NUM_RECENT_OFFSETS 3 + +/* These values are chosen for fast decompression. */ +#define LZX_MAINCODE_TABLEBITS 11 +#define LZX_LENCODE_TABLEBITS 10 +#define LZX_PRECODE_TABLEBITS 6 +#define LZX_ALIGNEDCODE_TABLEBITS 7 + +#define LZX_READ_LENS_MAX_OVERRUN 50 + +/* Mapping: offset slot => first match offset that uses that offset slot. + */ +static const u32 lzx_offset_slot_base[LZX_NUM_OFFSET_SLOTS + 1] = { + 0, 1, 2, 3, 4, /* 0 --- 4 */ + 6, 8, 12, 16, 24, /* 5 --- 9 */ + 32, 48, 64, 96, 128, /* 10 --- 14 */ + 192, 256, 384, 512, 768, /* 15 --- 19 */ + 1024, 1536, 2048, 3072, 4096, /* 20 --- 24 */ + 6144, 8192, 12288, 16384, 24576, /* 25 --- 29 */ + 32768, /* extra */ +}; + +/* Mapping: offset slot => how many extra bits must be read and added to the + * corresponding offset slot base to decode the match offset. + */ +static const u8 lzx_extra_offset_bits[LZX_NUM_OFFSET_SLOTS] = { + 0, 0, 0, 0, 1, + 1, 2, 2, 3, 3, + 4, 4, 5, 5, 6, + 6, 7, 7, 8, 8, + 9, 9, 10, 10, 11, + 11, 12, 12, 13, 13, +}; + +/* Reusable heap-allocated memory for LZX decompression */ +struct lzx_decompressor { + + /* Huffman decoding tables, and arrays that map symbols to codeword + * lengths + */ + + u16 maincode_decode_table[(1 << LZX_MAINCODE_TABLEBITS) + + (LZX_MAINCODE_NUM_SYMBOLS * 2)]; + u8 maincode_lens[LZX_MAINCODE_NUM_SYMBOLS + LZX_READ_LENS_MAX_OVERRUN]; + + + u16 lencode_decode_table[(1 << LZX_LENCODE_TABLEBITS) + + (LZX_LENCODE_NUM_SYMBOLS * 2)]; + u8 lencode_lens[LZX_LENCODE_NUM_SYMBOLS + LZX_READ_LENS_MAX_OVERRUN]; + + + u16 alignedcode_decode_table[(1 << LZX_ALIGNEDCODE_TABLEBITS) + + (LZX_ALIGNEDCODE_NUM_SYMBOLS * 2)]; + u8 alignedcode_lens[LZX_ALIGNEDCODE_NUM_SYMBOLS]; + + u16 precode_decode_table[(1 << LZX_PRECODE_TABLEBITS) + + (LZX_PRECODE_NUM_SYMBOLS * 2)]; + u8 precode_lens[LZX_PRECODE_NUM_SYMBOLS]; + + /* Temporary space for make_huffman_decode_table() */ + u16 working_space[2 * (1 + LZX_MAX_MAIN_CODEWORD_LEN) + + LZX_MAINCODE_NUM_SYMBOLS]; +}; + +static void undo_e8_translation(void *target, s32 input_pos) +{ + s32 abs_offset, rel_offset; + + abs_offset = get_unaligned_le32(target); + if (abs_offset >= 0) { + if (abs_offset < LZX_DEFAULT_FILESIZE) { + /* "good translation" */ + rel_offset = abs_offset - input_pos; + put_unaligned_le32(rel_offset, target); + } + } else { + if (abs_offset >= -input_pos) { + /* "compensating translation" */ + rel_offset = abs_offset + LZX_DEFAULT_FILESIZE; + put_unaligned_le32(rel_offset, target); + } + } +} + +/* + * Undo the 'E8' preprocessing used in LZX. Before compression, the + * uncompressed data was preprocessed by changing the targets of suspected x86 + * CALL instructions from relative offsets to absolute offsets. After + * match/literal decoding, the decompressor must undo the translation. + */ +static void lzx_postprocess(u8 *data, u32 size) +{ + /* + * A worthwhile optimization is to push the end-of-buffer check into the + * relatively rare E8 case. This is possible if we replace the last six + * bytes of data with E8 bytes; then we are guaranteed to hit an E8 byte + * before reaching end-of-buffer. In addition, this scheme guarantees + * that no translation can begin following an E8 byte in the last 10 + * bytes because a 4-byte offset containing E8 as its high byte is a + * large negative number that is not valid for translation. That is + * exactly what we need. + */ + u8 *tail; + u8 saved_bytes[6]; + u8 *p; + + if (size <= 10) + return; + + tail = &data[size - 6]; + memcpy(saved_bytes, tail, 6); + memset(tail, 0xE8, 6); + p = data; + for (;;) { + while (*p != 0xE8) + p++; + if (p >= tail) + break; + undo_e8_translation(p + 1, p - data); + p += 5; + } + memcpy(tail, saved_bytes, 6); +} + +/* Read a Huffman-encoded symbol using the precode. */ +static forceinline u32 read_presym(const struct lzx_decompressor *d, + struct input_bitstream *is) +{ + return read_huffsym(is, d->precode_decode_table, + LZX_PRECODE_TABLEBITS, LZX_MAX_PRE_CODEWORD_LEN); +} + +/* Read a Huffman-encoded symbol using the main code. */ +static forceinline u32 read_mainsym(const struct lzx_decompressor *d, + struct input_bitstream *is) +{ + return read_huffsym(is, d->maincode_decode_table, + LZX_MAINCODE_TABLEBITS, LZX_MAX_MAIN_CODEWORD_LEN); +} + +/* Read a Huffman-encoded symbol using the length code. */ +static forceinline u32 read_lensym(const struct lzx_decompressor *d, + struct input_bitstream *is) +{ + return read_huffsym(is, d->lencode_decode_table, + LZX_LENCODE_TABLEBITS, LZX_MAX_LEN_CODEWORD_LEN); +} + +/* Read a Huffman-encoded symbol using the aligned offset code. */ +static forceinline u32 read_alignedsym(const struct lzx_decompressor *d, + struct input_bitstream *is) +{ + return read_huffsym(is, d->alignedcode_decode_table, + LZX_ALIGNEDCODE_TABLEBITS, + LZX_MAX_ALIGNED_CODEWORD_LEN); +} + +/* + * Read the precode from the compressed input bitstream, then use it to decode + * @num_lens codeword length values. + * + * @is: The input bitstream. + * + * @lens: An array that contains the length values from the previous time + * the codeword lengths for this Huffman code were read, or all 0's + * if this is the first time. This array must have at least + * (@num_lens + LZX_READ_LENS_MAX_OVERRUN) entries. + * + * @num_lens: Number of length values to decode. + * + * Returns 0 on success, or -1 if the data was invalid. + */ +static int lzx_read_codeword_lens(struct lzx_decompressor *d, + struct input_bitstream *is, + u8 *lens, u32 num_lens) +{ + u8 *len_ptr = lens; + u8 *lens_end = lens + num_lens; + int i; + + /* Read the lengths of the precode codewords. These are given + * explicitly. + */ + for (i = 0; i < LZX_PRECODE_NUM_SYMBOLS; i++) { + d->precode_lens[i] = + bitstream_read_bits(is, LZX_PRECODE_ELEMENT_SIZE); + } + + /* Make the decoding table for the precode. */ + if (make_huffman_decode_table(d->precode_decode_table, + LZX_PRECODE_NUM_SYMBOLS, + LZX_PRECODE_TABLEBITS, + d->precode_lens, + LZX_MAX_PRE_CODEWORD_LEN, + d->working_space)) + return -1; + + /* Decode the codeword lengths. */ + do { + u32 presym; + u8 len; + + /* Read the next precode symbol. */ + presym = read_presym(d, is); + if (presym < 17) { + /* Difference from old length */ + len = *len_ptr - presym; + if ((s8)len < 0) + len += 17; + *len_ptr++ = len; + } else { + /* Special RLE values */ + + u32 run_len; + + if (presym == 17) { + /* Run of 0's */ + run_len = 4 + bitstream_read_bits(is, 4); + len = 0; + } else if (presym == 18) { + /* Longer run of 0's */ + run_len = 20 + bitstream_read_bits(is, 5); + len = 0; + } else { + /* Run of identical lengths */ + run_len = 4 + bitstream_read_bits(is, 1); + presym = read_presym(d, is); + if (presym > 17) + return -1; + len = *len_ptr - presym; + if ((s8)len < 0) + len += 17; + } + + do { + *len_ptr++ = len; + } while (--run_len); + /* Worst case overrun is when presym == 18, + * run_len == 20 + 31, and only 1 length was remaining. + * So LZX_READ_LENS_MAX_OVERRUN == 50. + * + * Overrun while reading the first half of maincode_lens + * can corrupt the previous values in the second half. + * This doesn't really matter because the resulting + * lengths will still be in range, and data that + * generates overruns is invalid anyway. + */ + } + } while (len_ptr < lens_end); + + return 0; +} + +/* + * Read the header of an LZX block and save the block type and (uncompressed) + * size in *block_type_ret and *block_size_ret, respectively. + * + * If the block is compressed, also update the Huffman decode @tables with the + * new Huffman codes. If the block is uncompressed, also update the match + * offset @queue with the new match offsets. + * + * Return 0 on success, or -1 if the data was invalid. + */ +static int lzx_read_block_header(struct lzx_decompressor *d, + struct input_bitstream *is, + int *block_type_ret, + u32 *block_size_ret, + u32 recent_offsets[]) +{ + int block_type; + u32 block_size; + int i; + + bitstream_ensure_bits(is, 4); + + /* The first three bits tell us what kind of block it is, and should be + * one of the LZX_BLOCKTYPE_* values. + */ + block_type = bitstream_pop_bits(is, 3); + + /* Read the block size. */ + if (bitstream_pop_bits(is, 1)) { + block_size = LZX_DEFAULT_BLOCK_SIZE; + } else { + block_size = 0; + block_size |= bitstream_read_bits(is, 8); + block_size <<= 8; + block_size |= bitstream_read_bits(is, 8); + } + + switch (block_type) { + + case LZX_BLOCKTYPE_ALIGNED: + + /* Read the aligned offset code and prepare its decode table. + */ + + for (i = 0; i < LZX_ALIGNEDCODE_NUM_SYMBOLS; i++) { + d->alignedcode_lens[i] = + bitstream_read_bits(is, + LZX_ALIGNEDCODE_ELEMENT_SIZE); + } + + if (make_huffman_decode_table(d->alignedcode_decode_table, + LZX_ALIGNEDCODE_NUM_SYMBOLS, + LZX_ALIGNEDCODE_TABLEBITS, + d->alignedcode_lens, + LZX_MAX_ALIGNED_CODEWORD_LEN, + d->working_space)) + return -1; + + /* Fall though, since the rest of the header for aligned offset + * blocks is the same as that for verbatim blocks. + */ + fallthrough; + + case LZX_BLOCKTYPE_VERBATIM: + + /* Read the main code and prepare its decode table. + * + * Note that the codeword lengths in the main code are encoded + * in two parts: one part for literal symbols, and one part for + * match symbols. + */ + + if (lzx_read_codeword_lens(d, is, d->maincode_lens, + LZX_NUM_CHARS)) + return -1; + + if (lzx_read_codeword_lens(d, is, + d->maincode_lens + LZX_NUM_CHARS, + LZX_MAINCODE_NUM_SYMBOLS - LZX_NUM_CHARS)) + return -1; + + if (make_huffman_decode_table(d->maincode_decode_table, + LZX_MAINCODE_NUM_SYMBOLS, + LZX_MAINCODE_TABLEBITS, + d->maincode_lens, + LZX_MAX_MAIN_CODEWORD_LEN, + d->working_space)) + return -1; + + /* Read the length code and prepare its decode table. */ + + if (lzx_read_codeword_lens(d, is, d->lencode_lens, + LZX_LENCODE_NUM_SYMBOLS)) + return -1; + + if (make_huffman_decode_table(d->lencode_decode_table, + LZX_LENCODE_NUM_SYMBOLS, + LZX_LENCODE_TABLEBITS, + d->lencode_lens, + LZX_MAX_LEN_CODEWORD_LEN, + d->working_space)) + return -1; + + break; + + case LZX_BLOCKTYPE_UNCOMPRESSED: + + /* Before reading the three recent offsets from the uncompressed + * block header, the stream must be aligned on a 16-bit + * boundary. But if the stream is *already* aligned, then the + * next 16 bits must be discarded. + */ + bitstream_ensure_bits(is, 1); + bitstream_align(is); + + recent_offsets[0] = bitstream_read_u32(is); + recent_offsets[1] = bitstream_read_u32(is); + recent_offsets[2] = bitstream_read_u32(is); + + /* Offsets of 0 are invalid. */ + if (recent_offsets[0] == 0 || recent_offsets[1] == 0 || + recent_offsets[2] == 0) + return -1; + break; + + default: + /* Unrecognized block type. */ + return -1; + } + + *block_type_ret = block_type; + *block_size_ret = block_size; + return 0; +} + +/* Decompress a block of LZX-compressed data. */ +static int lzx_decompress_block(const struct lzx_decompressor *d, + struct input_bitstream *is, + int block_type, u32 block_size, + u8 * const out_begin, u8 *out_next, + u32 recent_offsets[]) +{ + u8 * const block_end = out_next + block_size; + u32 ones_if_aligned = 0U - (block_type == LZX_BLOCKTYPE_ALIGNED); + + do { + u32 mainsym; + u32 match_len; + u32 match_offset; + u32 offset_slot; + u32 num_extra_bits; + + mainsym = read_mainsym(d, is); + if (mainsym < LZX_NUM_CHARS) { + /* Literal */ + *out_next++ = mainsym; + continue; + } + + /* Match */ + + /* Decode the length header and offset slot. */ + mainsym -= LZX_NUM_CHARS; + match_len = mainsym % LZX_NUM_LEN_HEADERS; + offset_slot = mainsym / LZX_NUM_LEN_HEADERS; + + /* If needed, read a length symbol to decode the full length. */ + if (match_len == LZX_NUM_PRIMARY_LENS) + match_len += read_lensym(d, is); + match_len += LZX_MIN_MATCH_LEN; + + if (offset_slot < LZX_NUM_RECENT_OFFSETS) { + /* Repeat offset */ + + /* Note: This isn't a real LRU queue, since using the R2 + * offset doesn't bump the R1 offset down to R2. This + * quirk allows all 3 recent offsets to be handled by + * the same code. (For R0, the swap is a no-op.) + */ + match_offset = recent_offsets[offset_slot]; + recent_offsets[offset_slot] = recent_offsets[0]; + recent_offsets[0] = match_offset; + } else { + /* Explicit offset */ + + /* Look up the number of extra bits that need to be read + * to decode offsets with this offset slot. + */ + num_extra_bits = lzx_extra_offset_bits[offset_slot]; + + /* Start with the offset slot base value. */ + match_offset = lzx_offset_slot_base[offset_slot]; + + /* In aligned offset blocks, the low-order 3 bits of + * each offset are encoded using the aligned offset + * code. Otherwise, all the extra bits are literal. + */ + + if ((num_extra_bits & ones_if_aligned) >= LZX_NUM_ALIGNED_OFFSET_BITS) { + match_offset += + bitstream_read_bits(is, num_extra_bits - + LZX_NUM_ALIGNED_OFFSET_BITS) + << LZX_NUM_ALIGNED_OFFSET_BITS; + match_offset += read_alignedsym(d, is); + } else { + match_offset += bitstream_read_bits(is, num_extra_bits); + } + + /* Adjust the offset. */ + match_offset -= (LZX_NUM_RECENT_OFFSETS - 1); + + /* Update the recent offsets. */ + recent_offsets[2] = recent_offsets[1]; + recent_offsets[1] = recent_offsets[0]; + recent_offsets[0] = match_offset; + } + + /* Validate the match, then copy it to the current position. */ + + if (match_len > (size_t)(block_end - out_next)) + return -1; + + if (match_offset > (size_t)(out_next - out_begin)) + return -1; + + out_next = lz_copy(out_next, match_len, match_offset, + block_end, LZX_MIN_MATCH_LEN); + + } while (out_next != block_end); + + return 0; +} + +/* + * lzx_allocate_decompressor - Allocate an LZX decompressor + * + * Return the pointer to the decompressor on success, or return NULL and set + * errno on failure. + */ +struct lzx_decompressor *lzx_allocate_decompressor(void) +{ + return kmalloc(sizeof(struct lzx_decompressor), GFP_NOFS); +} + +/* + * lzx_decompress - Decompress a buffer of LZX-compressed data + * + * @decompressor: A decompressor allocated with lzx_allocate_decompressor() + * @compressed_data: The buffer of data to decompress + * @compressed_size: Number of bytes of compressed data + * @uncompressed_data: The buffer in which to store the decompressed data + * @uncompressed_size: The number of bytes the data decompresses into + * + * Return 0 on success, or return -1 and set errno on failure. + */ +int lzx_decompress(struct lzx_decompressor *decompressor, + const void *compressed_data, size_t compressed_size, + void *uncompressed_data, size_t uncompressed_size) +{ + struct lzx_decompressor *d = decompressor; + u8 * const out_begin = uncompressed_data; + u8 *out_next = out_begin; + u8 * const out_end = out_begin + uncompressed_size; + struct input_bitstream is; + u32 recent_offsets[LZX_NUM_RECENT_OFFSETS] = {1, 1, 1}; + int e8_status = 0; + + init_input_bitstream(&is, compressed_data, compressed_size); + + /* Codeword lengths begin as all 0's for delta encoding purposes. */ + memset(d->maincode_lens, 0, LZX_MAINCODE_NUM_SYMBOLS); + memset(d->lencode_lens, 0, LZX_LENCODE_NUM_SYMBOLS); + + /* Decompress blocks until we have all the uncompressed data. */ + + while (out_next != out_end) { + int block_type; + u32 block_size; + + if (lzx_read_block_header(d, &is, &block_type, &block_size, + recent_offsets)) + goto invalid; + + if (block_size < 1 || block_size > (size_t)(out_end - out_next)) + goto invalid; + + if (block_type != LZX_BLOCKTYPE_UNCOMPRESSED) { + + /* Compressed block */ + + if (lzx_decompress_block(d, + &is, + block_type, + block_size, + out_begin, + out_next, + recent_offsets)) + goto invalid; + + e8_status |= d->maincode_lens[0xe8]; + out_next += block_size; + } else { + /* Uncompressed block */ + + out_next = bitstream_read_bytes(&is, out_next, + block_size); + if (!out_next) + goto invalid; + + if (block_size & 1) + bitstream_read_byte(&is); + + e8_status = 1; + } + } + + /* Postprocess the data unless it cannot possibly contain 0xe8 bytes. */ + if (e8_status) + lzx_postprocess(uncompressed_data, uncompressed_size); + + return 0; + +invalid: + return -1; +} + +/* + * lzx_free_decompressor - Free an LZX decompressor + * + * @decompressor: A decompressor that was allocated with + * lzx_allocate_decompressor(), or NULL. + */ +void lzx_free_decompressor(struct lzx_decompressor *decompressor) +{ + kfree(decompressor); +} diff --git a/fs/ntfs3/lib/xpress_decompress.c b/fs/ntfs3/lib/xpress_decompress.c new file mode 100644 index 000000000000..3d98f36a981e --- /dev/null +++ b/fs/ntfs3/lib/xpress_decompress.c @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * xpress_decompress.c - A decompressor for the XPRESS compression format + * (Huffman variant), which can be used in "System Compressed" files. This is + * based on the code from wimlib. + * + * Copyright (C) 2015 Eric Biggers + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ + +#include "decompress_common.h" +#include "lib.h" + +#define XPRESS_NUM_SYMBOLS 512 +#define XPRESS_MAX_CODEWORD_LEN 15 +#define XPRESS_MIN_MATCH_LEN 3 + +/* This value is chosen for fast decompression. */ +#define XPRESS_TABLEBITS 12 + +/* Reusable heap-allocated memory for XPRESS decompression */ +struct xpress_decompressor { + + /* The Huffman decoding table */ + u16 decode_table[(1 << XPRESS_TABLEBITS) + 2 * XPRESS_NUM_SYMBOLS]; + + /* An array that maps symbols to codeword lengths */ + u8 lens[XPRESS_NUM_SYMBOLS]; + + /* Temporary space for make_huffman_decode_table() */ + u16 working_space[2 * (1 + XPRESS_MAX_CODEWORD_LEN) + + XPRESS_NUM_SYMBOLS]; +}; + +/* + * xpress_allocate_decompressor - Allocate an XPRESS decompressor + * + * Return the pointer to the decompressor on success, or return NULL and set + * errno on failure. + */ +struct xpress_decompressor *xpress_allocate_decompressor(void) +{ + return kmalloc(sizeof(struct xpress_decompressor), GFP_NOFS); +} + +/* + * xpress_decompress - Decompress a buffer of XPRESS-compressed data + * + * @decompressor: A decompressor that was allocated with + * xpress_allocate_decompressor() + * @compressed_data: The buffer of data to decompress + * @compressed_size: Number of bytes of compressed data + * @uncompressed_data: The buffer in which to store the decompressed data + * @uncompressed_size: The number of bytes the data decompresses into + * + * Return 0 on success, or return -1 and set errno on failure. + */ +int xpress_decompress(struct xpress_decompressor *decompressor, + const void *compressed_data, size_t compressed_size, + void *uncompressed_data, size_t uncompressed_size) +{ + struct xpress_decompressor *d = decompressor; + const u8 * const in_begin = compressed_data; + u8 * const out_begin = uncompressed_data; + u8 *out_next = out_begin; + u8 * const out_end = out_begin + uncompressed_size; + struct input_bitstream is; + u32 i; + + /* Read the Huffman codeword lengths. */ + if (compressed_size < XPRESS_NUM_SYMBOLS / 2) + goto invalid; + for (i = 0; i < XPRESS_NUM_SYMBOLS / 2; i++) { + d->lens[i*2 + 0] = in_begin[i] & 0xF; + d->lens[i*2 + 1] = in_begin[i] >> 4; + } + + /* Build a decoding table for the Huffman code. */ + if (make_huffman_decode_table(d->decode_table, XPRESS_NUM_SYMBOLS, + XPRESS_TABLEBITS, d->lens, + XPRESS_MAX_CODEWORD_LEN, + d->working_space)) + goto invalid; + + /* Decode the matches and literals. */ + + init_input_bitstream(&is, in_begin + XPRESS_NUM_SYMBOLS / 2, + compressed_size - XPRESS_NUM_SYMBOLS / 2); + + while (out_next != out_end) { + u32 sym; + u32 log2_offset; + u32 length; + u32 offset; + + sym = read_huffsym(&is, d->decode_table, + XPRESS_TABLEBITS, XPRESS_MAX_CODEWORD_LEN); + if (sym < 256) { + /* Literal */ + *out_next++ = sym; + } else { + /* Match */ + length = sym & 0xf; + log2_offset = (sym >> 4) & 0xf; + + bitstream_ensure_bits(&is, 16); + + offset = ((u32)1 << log2_offset) | + bitstream_pop_bits(&is, log2_offset); + + if (length == 0xf) { + length += bitstream_read_byte(&is); + if (length == 0xf + 0xff) + length = bitstream_read_u16(&is); + } + length += XPRESS_MIN_MATCH_LEN; + + if (offset > (size_t)(out_next - out_begin)) + goto invalid; + + if (length > (size_t)(out_end - out_next)) + goto invalid; + + out_next = lz_copy(out_next, length, offset, out_end, + XPRESS_MIN_MATCH_LEN); + } + } + return 0; + +invalid: + return -1; +} + +/* + * xpress_free_decompressor - Free an XPRESS decompressor + * + * @decompressor: A decompressor that was allocated with + * xpress_allocate_decompressor(), or NULL. + */ +void xpress_free_decompressor(struct xpress_decompressor *decompressor) +{ + kfree(decompressor); +} diff --git a/fs/ntfs3/lznt.c b/fs/ntfs3/lznt.c new file mode 100644 index 000000000000..ead9ab7d69b3 --- /dev/null +++ b/fs/ntfs3/lznt.c @@ -0,0 +1,452 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + */ +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +// clang-format off +/* src buffer is zero */ +#define LZNT_ERROR_ALL_ZEROS 1 +#define LZNT_CHUNK_SIZE 0x1000 +// clang-format on + +struct lznt_hash { + const u8 *p1; + const u8 *p2; +}; + +struct lznt { + const u8 *unc; + const u8 *unc_end; + const u8 *best_match; + size_t max_len; + bool std; + + struct lznt_hash hash[LZNT_CHUNK_SIZE]; +}; + +static inline size_t get_match_len(const u8 *ptr, const u8 *end, const u8 *prev, + size_t max_len) +{ + size_t len = 0; + + while (ptr + len < end && ptr[len] == prev[len] && ++len < max_len) + ; + return len; +} + +static size_t longest_match_std(const u8 *src, struct lznt *ctx) +{ + size_t hash_index; + size_t len1 = 0, len2 = 0; + const u8 **hash; + + hash_index = + ((40543U * ((((src[0] << 4) ^ src[1]) << 4) ^ src[2])) >> 4) & + (LZNT_CHUNK_SIZE - 1); + + hash = &(ctx->hash[hash_index].p1); + + if (hash[0] >= ctx->unc && hash[0] < src && hash[0][0] == src[0] && + hash[0][1] == src[1] && hash[0][2] == src[2]) { + len1 = 3; + if (ctx->max_len > 3) + len1 += get_match_len(src + 3, ctx->unc_end, + hash[0] + 3, ctx->max_len - 3); + } + + if (hash[1] >= ctx->unc && hash[1] < src && hash[1][0] == src[0] && + hash[1][1] == src[1] && hash[1][2] == src[2]) { + len2 = 3; + if (ctx->max_len > 3) + len2 += get_match_len(src + 3, ctx->unc_end, + hash[1] + 3, ctx->max_len - 3); + } + + /* Compare two matches and select the best one */ + if (len1 < len2) { + ctx->best_match = hash[1]; + len1 = len2; + } else { + ctx->best_match = hash[0]; + } + + hash[1] = hash[0]; + hash[0] = src; + return len1; +} + +static size_t longest_match_best(const u8 *src, struct lznt *ctx) +{ + size_t max_len; + const u8 *ptr; + + if (ctx->unc >= src || !ctx->max_len) + return 0; + + max_len = 0; + for (ptr = ctx->unc; ptr < src; ++ptr) { + size_t len = + get_match_len(src, ctx->unc_end, ptr, ctx->max_len); + if (len >= max_len) { + max_len = len; + ctx->best_match = ptr; + } + } + + return max_len >= 3 ? max_len : 0; +} + +static const size_t s_max_len[] = { + 0x1002, 0x802, 0x402, 0x202, 0x102, 0x82, 0x42, 0x22, 0x12, +}; + +static const size_t s_max_off[] = { + 0x10, 0x20, 0x40, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, +}; + +static inline u16 make_pair(size_t offset, size_t len, size_t index) +{ + return ((offset - 1) << (12 - index)) | + ((len - 3) & (((1 << (12 - index)) - 1))); +} + +static inline size_t parse_pair(u16 pair, size_t *offset, size_t index) +{ + *offset = 1 + (pair >> (12 - index)); + return 3 + (pair & ((1 << (12 - index)) - 1)); +} + +/* + * compress_chunk + * + * returns one of the three values: + * 0 - ok, 'cmpr' contains 'cmpr_chunk_size' bytes of compressed data + * 1 - input buffer is full zero + * -2 - the compressed buffer is too small to hold the compressed data + */ +static inline int compress_chunk(size_t (*match)(const u8 *, struct lznt *), + const u8 *unc, const u8 *unc_end, u8 *cmpr, + u8 *cmpr_end, size_t *cmpr_chunk_size, + struct lznt *ctx) +{ + size_t cnt = 0; + size_t idx = 0; + const u8 *up = unc; + u8 *cp = cmpr + 3; + u8 *cp2 = cmpr + 2; + u8 not_zero = 0; + /* Control byte of 8-bit values: ( 0 - means byte as is, 1 - short pair ) */ + u8 ohdr = 0; + u8 *last; + u16 t16; + + if (unc + LZNT_CHUNK_SIZE < unc_end) + unc_end = unc + LZNT_CHUNK_SIZE; + + last = min(cmpr + LZNT_CHUNK_SIZE + sizeof(short), cmpr_end); + + ctx->unc = unc; + ctx->unc_end = unc_end; + ctx->max_len = s_max_len[0]; + + while (up < unc_end) { + size_t max_len; + + while (unc + s_max_off[idx] < up) + ctx->max_len = s_max_len[++idx]; + + // Find match + max_len = up + 3 <= unc_end ? (*match)(up, ctx) : 0; + + if (!max_len) { + if (cp >= last) + goto NotCompressed; + not_zero |= *cp++ = *up++; + } else if (cp + 1 >= last) { + goto NotCompressed; + } else { + t16 = make_pair(up - ctx->best_match, max_len, idx); + *cp++ = t16; + *cp++ = t16 >> 8; + + ohdr |= 1 << cnt; + up += max_len; + } + + cnt = (cnt + 1) & 7; + if (!cnt) { + *cp2 = ohdr; + ohdr = 0; + cp2 = cp; + cp += 1; + } + } + + if (cp2 < last) + *cp2 = ohdr; + else + cp -= 1; + + *cmpr_chunk_size = cp - cmpr; + + t16 = (*cmpr_chunk_size - 3) | 0xB000; + cmpr[0] = t16; + cmpr[1] = t16 >> 8; + + return not_zero ? 0 : LZNT_ERROR_ALL_ZEROS; + +NotCompressed: + + if ((cmpr + LZNT_CHUNK_SIZE + sizeof(short)) > last) + return -2; + + /* + * Copy non cmpr data + * 0x3FFF == ((LZNT_CHUNK_SIZE + 2 - 3) | 0x3000) + */ + cmpr[0] = 0xff; + cmpr[1] = 0x3f; + + memcpy(cmpr + sizeof(short), unc, LZNT_CHUNK_SIZE); + *cmpr_chunk_size = LZNT_CHUNK_SIZE + sizeof(short); + + return 0; +} + +static inline ssize_t decompress_chunk(u8 *unc, u8 *unc_end, const u8 *cmpr, + const u8 *cmpr_end) +{ + u8 *up = unc; + u8 ch = *cmpr++; + size_t bit = 0; + size_t index = 0; + u16 pair; + size_t offset, length; + + /* Do decompression until pointers are inside range */ + while (up < unc_end && cmpr < cmpr_end) { + /* Correct index */ + while (unc + s_max_off[index] < up) + index += 1; + + /* Check the current flag for zero */ + if (!(ch & (1 << bit))) { + /* Just copy byte */ + *up++ = *cmpr++; + goto next; + } + + /* Check for boundary */ + if (cmpr + 1 >= cmpr_end) + return -EINVAL; + + /* Read a short from little endian stream */ + pair = cmpr[1]; + pair <<= 8; + pair |= cmpr[0]; + + cmpr += 2; + + /* Translate packed information into offset and length */ + length = parse_pair(pair, &offset, index); + + /* Check offset for boundary */ + if (unc + offset > up) + return -EINVAL; + + /* Truncate the length if necessary */ + if (up + length >= unc_end) + length = unc_end - up; + + /* Now we copy bytes. This is the heart of LZ algorithm. */ + for (; length > 0; length--, up++) + *up = *(up - offset); + +next: + /* Advance flag bit value */ + bit = (bit + 1) & 7; + + if (!bit) { + if (cmpr >= cmpr_end) + break; + + ch = *cmpr++; + } + } + + /* return the size of uncompressed data */ + return up - unc; +} + +/* + * 0 - standard compression + * !0 - best compression, requires a lot of cpu + */ +struct lznt *get_lznt_ctx(int level) +{ + struct lznt *r = ntfs_zalloc(level ? offsetof(struct lznt, hash) + : sizeof(struct lznt)); + + if (r) + r->std = !level; + return r; +} + +/* + * compress_lznt + * + * Compresses "unc" into "cmpr" + * +x - ok, 'cmpr' contains 'final_compressed_size' bytes of compressed data + * 0 - input buffer is full zero + */ +size_t compress_lznt(const void *unc, size_t unc_size, void *cmpr, + size_t cmpr_size, struct lznt *ctx) +{ + int err; + size_t (*match)(const u8 *src, struct lznt *ctx); + u8 *p = cmpr; + u8 *end = p + cmpr_size; + const u8 *unc_chunk = unc; + const u8 *unc_end = unc_chunk + unc_size; + bool is_zero = true; + + if (ctx->std) { + match = &longest_match_std; + memset(ctx->hash, 0, sizeof(ctx->hash)); + } else { + match = &longest_match_best; + } + + /* compression cycle */ + for (; unc_chunk < unc_end; unc_chunk += LZNT_CHUNK_SIZE) { + cmpr_size = 0; + err = compress_chunk(match, unc_chunk, unc_end, p, end, + &cmpr_size, ctx); + if (err < 0) + return unc_size; + + if (is_zero && err != LZNT_ERROR_ALL_ZEROS) + is_zero = false; + + p += cmpr_size; + } + + if (p <= end - 2) + p[0] = p[1] = 0; + + return is_zero ? 0 : PtrOffset(cmpr, p); +} + +/* + * decompress_lznt + * + * decompresses "cmpr" into "unc" + */ +ssize_t decompress_lznt(const void *cmpr, size_t cmpr_size, void *unc, + size_t unc_size) +{ + const u8 *cmpr_chunk = cmpr; + const u8 *cmpr_end = cmpr_chunk + cmpr_size; + u8 *unc_chunk = unc; + u8 *unc_end = unc_chunk + unc_size; + u16 chunk_hdr; + + if (cmpr_size < sizeof(short)) + return -EINVAL; + + /* read chunk header */ + chunk_hdr = cmpr_chunk[1]; + chunk_hdr <<= 8; + chunk_hdr |= cmpr_chunk[0]; + + /* loop through decompressing chunks */ + for (;;) { + size_t chunk_size_saved; + size_t unc_use; + size_t cmpr_use = 3 + (chunk_hdr & (LZNT_CHUNK_SIZE - 1)); + + /* Check that the chunk actually fits the supplied buffer */ + if (cmpr_chunk + cmpr_use > cmpr_end) + return -EINVAL; + + /* First make sure the chunk contains compressed data */ + if (chunk_hdr & 0x8000) { + /* Decompress a chunk and return if we get an error */ + ssize_t err = + decompress_chunk(unc_chunk, unc_end, + cmpr_chunk + sizeof(chunk_hdr), + cmpr_chunk + cmpr_use); + if (err < 0) + return err; + unc_use = err; + } else { + /* This chunk does not contain compressed data */ + unc_use = unc_chunk + LZNT_CHUNK_SIZE > unc_end + ? unc_end - unc_chunk + : LZNT_CHUNK_SIZE; + + if (cmpr_chunk + sizeof(chunk_hdr) + unc_use > + cmpr_end) { + return -EINVAL; + } + + memcpy(unc_chunk, cmpr_chunk + sizeof(chunk_hdr), + unc_use); + } + + /* Advance pointers */ + cmpr_chunk += cmpr_use; + unc_chunk += unc_use; + + /* Check for the end of unc buffer */ + if (unc_chunk >= unc_end) + break; + + /* Proceed the next chunk */ + if (cmpr_chunk > cmpr_end - 2) + break; + + chunk_size_saved = LZNT_CHUNK_SIZE; + + /* read chunk header */ + chunk_hdr = cmpr_chunk[1]; + chunk_hdr <<= 8; + chunk_hdr |= cmpr_chunk[0]; + + if (!chunk_hdr) + break; + + /* Check the size of unc buffer */ + if (unc_use < chunk_size_saved) { + size_t t1 = chunk_size_saved - unc_use; + u8 *t2 = unc_chunk + t1; + + /* 'Zero' memory */ + if (t2 >= unc_end) + break; + + memset(unc_chunk, 0, t1); + unc_chunk = t2; + } + } + + /* Check compression boundary */ + if (cmpr_chunk > cmpr_end) + return -EINVAL; + + /* + * The unc size is just a difference between current + * pointer and original one + */ + return PtrOffset(unc, unc_chunk); +} diff --git a/fs/ntfs3/namei.c b/fs/ntfs3/namei.c new file mode 100644 index 000000000000..f5db12cd3b20 --- /dev/null +++ b/fs/ntfs3/namei.c @@ -0,0 +1,578 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + */ + +#include +#include +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +/* + * fill_name_de + * + * formats NTFS_DE in 'buf' + */ +int fill_name_de(struct ntfs_sb_info *sbi, void *buf, const struct qstr *name, + const struct cpu_str *uni) +{ + int err; + struct NTFS_DE *e = buf; + u16 data_size; + struct ATTR_FILE_NAME *fname = (struct ATTR_FILE_NAME *)(e + 1); + +#ifndef CONFIG_NTFS3_64BIT_CLUSTER + e->ref.high = fname->home.high = 0; +#endif + if (uni) { +#ifdef __BIG_ENDIAN + int ulen = uni->len; + __le16 *uname = fname->name; + const u16 *name_cpu = uni->name; + + while (ulen--) + *uname++ = cpu_to_le16(*name_cpu++); +#else + memcpy(fname->name, uni->name, uni->len * sizeof(u16)); +#endif + fname->name_len = uni->len; + + } else { + /* Convert input string to unicode */ + err = ntfs_nls_to_utf16(sbi, name->name, name->len, + (struct cpu_str *)&fname->name_len, + NTFS_NAME_LEN, UTF16_LITTLE_ENDIAN); + if (err < 0) + return err; + } + + fname->type = FILE_NAME_POSIX; + data_size = fname_full_size(fname); + + e->size = cpu_to_le16(QuadAlign(data_size) + sizeof(struct NTFS_DE)); + e->key_size = cpu_to_le16(data_size); + e->flags = 0; + e->res = 0; + + return 0; +} + +/* + * ntfs_lookup + * + * inode_operations::lookup + */ +static struct dentry *ntfs_lookup(struct inode *dir, struct dentry *dentry, + u32 flags) +{ + struct ntfs_inode *ni = ntfs_i(dir); + struct cpu_str *uni = __getname(); + struct inode *inode; + int err; + + if (!uni) + inode = ERR_PTR(-ENOMEM); + else { + err = ntfs_nls_to_utf16(ni->mi.sbi, dentry->d_name.name, + dentry->d_name.len, uni, NTFS_NAME_LEN, + UTF16_HOST_ENDIAN); + if (err < 0) + inode = ERR_PTR(err); + else { + ni_lock(ni); + inode = dir_search_u(dir, uni, NULL); + ni_unlock(ni); + } + __putname(uni); + } + + return d_splice_alias(inode, dentry); +} + +/* + * ntfs_create + * + * inode_operations::create + */ +static int ntfs_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) +{ + struct ntfs_inode *ni = ntfs_i(dir); + struct inode *inode; + + ni_lock_dir(ni); + + inode = ntfs_create_inode(mnt_userns, dir, dentry, NULL, S_IFREG | mode, + 0, NULL, 0, excl, NULL); + + ni_unlock(ni); + + return IS_ERR(inode) ? PTR_ERR(inode) : 0; +} + +/* + * ntfs_link + * + * inode_operations::link + */ +static int ntfs_link(struct dentry *ode, struct inode *dir, struct dentry *de) +{ + int err; + struct inode *inode = d_inode(ode); + struct ntfs_inode *ni = ntfs_i(inode); + + if (S_ISDIR(inode->i_mode)) + return -EPERM; + + if (inode->i_nlink >= NTFS_LINK_MAX) + return -EMLINK; + + ni_lock_dir(ntfs_i(dir)); + if (inode != dir) + ni_lock(ni); + + dir->i_ctime = dir->i_mtime = inode->i_ctime = current_time(inode); + inc_nlink(inode); + ihold(inode); + + err = ntfs_link_inode(inode, de); + if (!err) { + mark_inode_dirty(inode); + mark_inode_dirty(dir); + d_instantiate(de, inode); + } else { + drop_nlink(inode); + iput(inode); + } + + if (inode != dir) + ni_unlock(ni); + ni_unlock(ntfs_i(dir)); + + return err; +} + +/* + * ntfs_unlink + * + * inode_operations::unlink + */ +static int ntfs_unlink(struct inode *dir, struct dentry *dentry) +{ + struct ntfs_inode *ni = ntfs_i(dir); + int err; + + ni_lock_dir(ni); + + err = ntfs_unlink_inode(dir, dentry); + + ni_unlock(ni); + + return err; +} + +/* + * ntfs_symlink + * + * inode_operations::symlink + */ +static int ntfs_symlink(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, const char *symname) +{ + u32 size = strlen(symname); + struct inode *inode; + struct ntfs_inode *ni = ntfs_i(dir); + + ni_lock_dir(ni); + + inode = ntfs_create_inode(mnt_userns, dir, dentry, NULL, S_IFLNK | 0777, + 0, symname, size, 0, NULL); + + ni_unlock(ni); + + return IS_ERR(inode) ? PTR_ERR(inode) : 0; +} + +/* + * ntfs_mkdir + * + * inode_operations::mkdir + */ +static int ntfs_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct inode *inode; + struct ntfs_inode *ni = ntfs_i(dir); + + ni_lock_dir(ni); + + inode = ntfs_create_inode(mnt_userns, dir, dentry, NULL, S_IFDIR | mode, + 0, NULL, -1, 0, NULL); + + ni_unlock(ni); + + return IS_ERR(inode) ? PTR_ERR(inode) : 0; +} + +/* + * ntfs_rmdir + * + * inode_operations::rm_dir + */ +static int ntfs_rmdir(struct inode *dir, struct dentry *dentry) +{ + struct ntfs_inode *ni = ntfs_i(dir); + int err; + + ni_lock_dir(ni); + + err = ntfs_unlink_inode(dir, dentry); + + ni_unlock(ni); + + return err; +} + +/* + * ntfs_rename + * + * inode_operations::rename + */ +static int ntfs_rename(struct user_namespace *mnt_userns, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, u32 flags) +{ + int err; + struct super_block *sb = old_dir->i_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + struct ntfs_inode *old_dir_ni = ntfs_i(old_dir); + struct ntfs_inode *new_dir_ni = ntfs_i(new_dir); + struct ntfs_inode *old_ni; + struct ATTR_FILE_NAME *old_name, *new_name, *fname; + u8 name_type; + bool is_same; + struct inode *old_inode, *new_inode; + struct NTFS_DE *old_de, *new_de; + struct ATTRIB *attr; + struct ATTR_LIST_ENTRY *le; + u16 new_de_key_size; + + static_assert(SIZEOF_ATTRIBUTE_FILENAME_MAX + SIZEOF_RESIDENT < 1024); + static_assert(SIZEOF_ATTRIBUTE_FILENAME_MAX + sizeof(struct NTFS_DE) < + 1024); + static_assert(PATH_MAX >= 4 * 1024); + + if (flags & ~RENAME_NOREPLACE) + return -EINVAL; + + old_inode = d_inode(old_dentry); + new_inode = d_inode(new_dentry); + + old_ni = ntfs_i(old_inode); + + is_same = old_dentry->d_name.len == new_dentry->d_name.len && + !memcmp(old_dentry->d_name.name, new_dentry->d_name.name, + old_dentry->d_name.len); + + if (is_same && old_dir == new_dir) { + /* Nothing to do */ + err = 0; + goto out; + } + + if (ntfs_is_meta_file(sbi, old_inode->i_ino)) { + err = -EINVAL; + goto out; + } + + if (new_inode) { + /*target name exists. unlink it*/ + dget(new_dentry); + ni_lock_dir(new_dir_ni); + err = ntfs_unlink_inode(new_dir, new_dentry); + ni_unlock(new_dir_ni); + dput(new_dentry); + if (err) + goto out; + } + + /* allocate PATH_MAX bytes */ + old_de = __getname(); + if (!old_de) { + err = -ENOMEM; + goto out; + } + + err = fill_name_de(sbi, old_de, &old_dentry->d_name, NULL); + if (err < 0) + goto out1; + + old_name = (struct ATTR_FILE_NAME *)(old_de + 1); + + if (is_same) { + new_de = old_de; + } else { + new_de = Add2Ptr(old_de, 1024); + err = fill_name_de(sbi, new_de, &new_dentry->d_name, NULL); + if (err < 0) + goto out1; + } + + ni_lock_dir(old_dir_ni); + ni_lock(old_ni); + + mi_get_ref(&old_dir_ni->mi, &old_name->home); + + /*get pointer to file_name in mft*/ + fname = ni_fname_name(old_ni, (struct cpu_str *)&old_name->name_len, + &old_name->home, &le); + if (!fname) { + err = -EINVAL; + goto out2; + } + + /* Copy fname info from record into new fname */ + new_name = (struct ATTR_FILE_NAME *)(new_de + 1); + memcpy(&new_name->dup, &fname->dup, sizeof(fname->dup)); + + name_type = paired_name(fname->type); + + /* remove first name from directory */ + err = indx_delete_entry(&old_dir_ni->dir, old_dir_ni, old_de + 1, + le16_to_cpu(old_de->key_size), sbi); + if (err) + goto out3; + + /* remove first name from mft */ + err = ni_remove_attr_le(old_ni, attr_from_name(fname), le); + if (err) + goto out4; + + le16_add_cpu(&old_ni->mi.mrec->hard_links, -1); + old_ni->mi.dirty = true; + + if (name_type != FILE_NAME_POSIX) { + /* get paired name */ + fname = ni_fname_type(old_ni, name_type, &le); + if (fname) { + /* remove second name from directory */ + err = indx_delete_entry(&old_dir_ni->dir, old_dir_ni, + fname, fname_full_size(fname), + sbi); + if (err) + goto out5; + + /* remove second name from mft */ + err = ni_remove_attr_le(old_ni, attr_from_name(fname), + le); + if (err) + goto out6; + + le16_add_cpu(&old_ni->mi.mrec->hard_links, -1); + old_ni->mi.dirty = true; + } + } + + /* Add new name */ + mi_get_ref(&old_ni->mi, &new_de->ref); + mi_get_ref(&ntfs_i(new_dir)->mi, &new_name->home); + + new_de_key_size = le16_to_cpu(new_de->key_size); + + /* insert new name in mft */ + err = ni_insert_resident(old_ni, new_de_key_size, ATTR_NAME, NULL, 0, + &attr, NULL); + if (err) + goto out7; + + attr->res.flags = RESIDENT_FLAG_INDEXED; + + memcpy(Add2Ptr(attr, SIZEOF_RESIDENT), new_name, new_de_key_size); + + le16_add_cpu(&old_ni->mi.mrec->hard_links, 1); + old_ni->mi.dirty = true; + + /* insert new name in directory */ + err = indx_insert_entry(&new_dir_ni->dir, new_dir_ni, new_de, sbi, + NULL); + if (err) + goto out8; + + if (IS_DIRSYNC(new_dir)) + err = ntfs_sync_inode(old_inode); + else + mark_inode_dirty(old_inode); + + old_dir->i_ctime = old_dir->i_mtime = current_time(old_dir); + if (IS_DIRSYNC(old_dir)) + (void)ntfs_sync_inode(old_dir); + else + mark_inode_dirty(old_dir); + + if (old_dir != new_dir) { + new_dir->i_mtime = new_dir->i_ctime = old_dir->i_ctime; + mark_inode_dirty(new_dir); + } + + if (old_inode) { + old_inode->i_ctime = old_dir->i_ctime; + mark_inode_dirty(old_inode); + } + + err = 0; + /* normal way */ + goto out2; + +out8: + /* undo + * ni_insert_resident(old_ni, new_de_key_size, ATTR_NAME, NULL, 0, + * &attr, NULL); + */ + mi_remove_attr(&old_ni->mi, attr); +out7: + /* undo + * ni_remove_attr_le(old_ni, attr_from_name(fname), le); + */ +out6: + /* undo + * indx_delete_entry(&old_dir_ni->dir, old_dir_ni, + * fname, fname_full_size(fname), + * sbi); + */ +out5: + /* undo + * ni_remove_attr_le(old_ni, attr_from_name(fname), le); + */ +out4: + /* undo: + * indx_delete_entry(&old_dir_ni->dir, old_dir_ni, old_de + 1, + * old_de->key_size, NULL); + */ +out3: +out2: + ni_unlock(old_ni); + ni_unlock(old_dir_ni); +out1: + __putname(old_de); +out: + return err; +} + +/* + * ntfs_atomic_open + * + * inode_operations::atomic_open + */ +static int ntfs_atomic_open(struct inode *dir, struct dentry *dentry, + struct file *file, u32 flags, umode_t mode) +{ + int err; + bool excl = !!(flags & O_EXCL); + struct inode *inode; + struct ntfs_fnd *fnd = NULL; + struct ntfs_inode *ni = ntfs_i(dir); + struct dentry *d = NULL; + struct cpu_str *uni = __getname(); + + if (!uni) + return -ENOMEM; + + err = ntfs_nls_to_utf16(ni->mi.sbi, dentry->d_name.name, + dentry->d_name.len, uni, NTFS_NAME_LEN, + UTF16_HOST_ENDIAN); + if (err < 0) + goto out; + + ni_lock_dir(ni); + + if (d_in_lookup(dentry)) { + fnd = fnd_get(); + if (!fnd) { + err = -ENOMEM; + goto out1; + } + + d = d_splice_alias(dir_search_u(dir, uni, fnd), dentry); + if (IS_ERR(d)) { + err = PTR_ERR(d); + d = NULL; + goto out2; + } + + if (d) + dentry = d; + } + + if (!(flags & O_CREAT) || d_really_is_positive(dentry)) { + err = finish_no_open(file, d); + goto out2; + } + + file->f_mode |= FMODE_CREATED; + + /*fnd contains tree's path to insert to*/ + /* TODO: init_user_ns? */ + inode = ntfs_create_inode(&init_user_ns, dir, dentry, uni, mode, 0, + NULL, 0, excl, fnd); + err = IS_ERR(inode) ? PTR_ERR(inode) + : finish_open(file, dentry, ntfs_file_open); + dput(d); + +out2: + fnd_put(fnd); +out1: + ni_unlock(ni); +out: + __putname(uni); + + return err; +} + +struct dentry *ntfs3_get_parent(struct dentry *child) +{ + struct inode *inode = d_inode(child); + struct ntfs_inode *ni = ntfs_i(inode); + + struct ATTR_LIST_ENTRY *le = NULL; + struct ATTRIB *attr = NULL; + struct ATTR_FILE_NAME *fname; + + while ((attr = ni_find_attr(ni, attr, &le, ATTR_NAME, NULL, 0, NULL, + NULL))) { + fname = resident_data_ex(attr, SIZEOF_ATTRIBUTE_FILENAME); + if (!fname) + continue; + + return d_obtain_alias( + ntfs_iget5(inode->i_sb, &fname->home, NULL)); + } + + return ERR_PTR(-ENOENT); +} + +const struct inode_operations ntfs_dir_inode_operations = { + .lookup = ntfs_lookup, + .create = ntfs_create, + .link = ntfs_link, + .unlink = ntfs_unlink, + .symlink = ntfs_symlink, + .mkdir = ntfs_mkdir, + .rmdir = ntfs_rmdir, + .rename = ntfs_rename, + .permission = ntfs_permission, + .get_acl = ntfs_get_acl, + .set_acl = ntfs_set_acl, + .setattr = ntfs3_setattr, + .getattr = ntfs_getattr, + .listxattr = ntfs_listxattr, + .atomic_open = ntfs_atomic_open, + .fiemap = ntfs_fiemap, +}; diff --git a/fs/ntfs3/ntfs.h b/fs/ntfs3/ntfs.h new file mode 100644 index 000000000000..40398e6c39c9 --- /dev/null +++ b/fs/ntfs3/ntfs.h @@ -0,0 +1,1238 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + * on-disk ntfs structs + */ + +// clang-format off + +/* TODO: + * - Check 4K mft record and 512 bytes cluster + */ + +/* + * Activate this define to use binary search in indexes + */ +#define NTFS3_INDEX_BINARY_SEARCH + +/* + * Check each run for marked clusters + */ +#define NTFS3_CHECK_FREE_CLST + +#define NTFS_NAME_LEN 255 + +/* + * ntfs.sys used 500 maximum links + * on-disk struct allows up to 0xffff + */ +#define NTFS_LINK_MAX 0x400 +//#define NTFS_LINK_MAX 0xffff + +/* + * Activate to use 64 bit clusters instead of 32 bits in ntfs.sys + * Logical and virtual cluster number + * If needed, may be redefined to use 64 bit value + */ +//#define CONFIG_NTFS3_64BIT_CLUSTER + +#define NTFS_LZNT_MAX_CLUSTER 4096 +#define NTFS_LZNT_CUNIT 4 +#define NTFS_LZNT_CLUSTERS (1u<low) | ((u64)le16_to_cpu(ref->high) << 32); +#else + return le32_to_cpu(ref->low); +#endif +} + +struct NTFS_BOOT { + u8 jump_code[3]; // 0x00: Jump to boot code + u8 system_id[8]; // 0x03: System ID, equals "NTFS " + + // NOTE: this member is not aligned(!) + // bytes_per_sector[0] must be 0 + // bytes_per_sector[1] must be multiplied by 256 + u8 bytes_per_sector[2]; // 0x0B: Bytes per sector + + u8 sectors_per_clusters;// 0x0D: Sectors per cluster + u8 unused1[7]; + u8 media_type; // 0x15: Media type (0xF8 - harddisk) + u8 unused2[2]; + __le16 sct_per_track; // 0x18: number of sectors per track + __le16 heads; // 0x1A: number of heads per cylinder + __le32 hidden_sectors; // 0x1C: number of 'hidden' sectors + u8 unused3[4]; + u8 bios_drive_num; // 0x24: BIOS drive number =0x80 + u8 unused4; + u8 signature_ex; // 0x26: Extended BOOT signature =0x80 + u8 unused5; + __le64 sectors_per_volume;// 0x28: size of volume in sectors + __le64 mft_clst; // 0x30: first cluster of $MFT + __le64 mft2_clst; // 0x38: first cluster of $MFTMirr + s8 record_size; // 0x40: size of MFT record in clusters(sectors) + u8 unused6[3]; + s8 index_size; // 0x44: size of INDX record in clusters(sectors) + u8 unused7[3]; + __le64 serial_num; // 0x48: Volume serial number + __le32 check_sum; // 0x50: Simple additive checksum of all + // of the u32's which precede the 'check_sum' + + u8 boot_code[0x200 - 0x50 - 2 - 4]; // 0x54: + u8 boot_magic[2]; // 0x1FE: Boot signature =0x55 + 0xAA +}; + +static_assert(sizeof(struct NTFS_BOOT) == 0x200); + +enum NTFS_SIGNATURE { + NTFS_FILE_SIGNATURE = cpu_to_le32(0x454C4946), // 'FILE' + NTFS_INDX_SIGNATURE = cpu_to_le32(0x58444E49), // 'INDX' + NTFS_CHKD_SIGNATURE = cpu_to_le32(0x444B4843), // 'CHKD' + NTFS_RSTR_SIGNATURE = cpu_to_le32(0x52545352), // 'RSTR' + NTFS_RCRD_SIGNATURE = cpu_to_le32(0x44524352), // 'RCRD' + NTFS_BAAD_SIGNATURE = cpu_to_le32(0x44414142), // 'BAAD' + NTFS_HOLE_SIGNATURE = cpu_to_le32(0x454C4F48), // 'HOLE' + NTFS_FFFF_SIGNATURE = cpu_to_le32(0xffffffff), +}; + +static_assert(sizeof(enum NTFS_SIGNATURE) == 4); + +/* MFT Record header structure */ +struct NTFS_RECORD_HEADER { + /* Record magic number, equals 'FILE'/'INDX'/'RSTR'/'RCRD' */ + enum NTFS_SIGNATURE sign; // 0x00: + __le16 fix_off; // 0x04: + __le16 fix_num; // 0x06: + __le64 lsn; // 0x08: Log file sequence number +}; + +static_assert(sizeof(struct NTFS_RECORD_HEADER) == 0x10); + +static inline int is_baad(const struct NTFS_RECORD_HEADER *hdr) +{ + return hdr->sign == NTFS_BAAD_SIGNATURE; +} + +/* Possible bits in struct MFT_REC.flags */ +enum RECORD_FLAG { + RECORD_FLAG_IN_USE = cpu_to_le16(0x0001), + RECORD_FLAG_DIR = cpu_to_le16(0x0002), + RECORD_FLAG_SYSTEM = cpu_to_le16(0x0004), + RECORD_FLAG_UNKNOWN = cpu_to_le16(0x0008), +}; + +/* MFT Record structure */ +struct MFT_REC { + struct NTFS_RECORD_HEADER rhdr; // 'FILE' + + __le16 seq; // 0x10: Sequence number for this record + __le16 hard_links; // 0x12: The number of hard links to record + __le16 attr_off; // 0x14: Offset to attributes + __le16 flags; // 0x16: See RECORD_FLAG + __le32 used; // 0x18: The size of used part + __le32 total; // 0x1C: Total record size + + struct MFT_REF parent_ref; // 0x20: Parent MFT record + __le16 next_attr_id; // 0x28: The next attribute Id + + __le16 res; // 0x2A: High part of mft record? + __le32 mft_record; // 0x2C: Current mft record number + __le16 fixups[]; // 0x30: +}; + +#define MFTRECORD_FIXUP_OFFSET_1 offsetof(struct MFT_REC, res) +#define MFTRECORD_FIXUP_OFFSET_3 offsetof(struct MFT_REC, fixups) + +static_assert(MFTRECORD_FIXUP_OFFSET_1 == 0x2A); +static_assert(MFTRECORD_FIXUP_OFFSET_3 == 0x30); + +static inline bool is_rec_base(const struct MFT_REC *rec) +{ + const struct MFT_REF *r = &rec->parent_ref; + + return !r->low && !r->high && !r->seq; +} + +static inline bool is_mft_rec5(const struct MFT_REC *rec) +{ + return le16_to_cpu(rec->rhdr.fix_off) >= + offsetof(struct MFT_REC, fixups); +} + +static inline bool is_rec_inuse(const struct MFT_REC *rec) +{ + return rec->flags & RECORD_FLAG_IN_USE; +} + +static inline bool clear_rec_inuse(struct MFT_REC *rec) +{ + return rec->flags &= ~RECORD_FLAG_IN_USE; +} + +/* Possible values of ATTR_RESIDENT.flags */ +#define RESIDENT_FLAG_INDEXED 0x01 + +struct ATTR_RESIDENT { + __le32 data_size; // 0x10: The size of data + __le16 data_off; // 0x14: Offset to data + u8 flags; // 0x16: resident flags ( 1 - indexed ) + u8 res; // 0x17: +}; // sizeof() = 0x18 + +struct ATTR_NONRESIDENT { + __le64 svcn; // 0x10: Starting VCN of this segment + __le64 evcn; // 0x18: End VCN of this segment + __le16 run_off; // 0x20: Offset to packed runs + // Unit of Compression size for this stream, expressed + // as a log of the cluster size. + // + // 0 means file is not compressed + // 1, 2, 3, and 4 are potentially legal values if the + // stream is compressed, however the implementation + // may only choose to use 4, or possibly 3. Note + // that 4 means cluster size time 16. If convenient + // the implementation may wish to accept a + // reasonable range of legal values here (1-5?), + // even if the implementation only generates + // a smaller set of values itself. + u8 c_unit; // 0x22 + u8 res1[5]; // 0x23: + __le64 alloc_size; // 0x28: The allocated size of attribute in bytes + // (multiple of cluster size) + __le64 data_size; // 0x30: The size of attribute in bytes <= alloc_size + __le64 valid_size; // 0x38: The size of valid part in bytes <= data_size + __le64 total_size; // 0x40: The sum of the allocated clusters for a file + // (present only for the first segment (0 == vcn) + // of compressed attribute) + +}; // sizeof()=0x40 or 0x48 (if compressed) + +/* Possible values of ATTRIB.flags: */ +#define ATTR_FLAG_COMPRESSED cpu_to_le16(0x0001) +#define ATTR_FLAG_COMPRESSED_MASK cpu_to_le16(0x00FF) +#define ATTR_FLAG_ENCRYPTED cpu_to_le16(0x4000) +#define ATTR_FLAG_SPARSED cpu_to_le16(0x8000) + +struct ATTRIB { + enum ATTR_TYPE type; // 0x00: The type of this attribute + __le32 size; // 0x04: The size of this attribute + u8 non_res; // 0x08: Is this attribute non-resident ? + u8 name_len; // 0x09: This attribute name length + __le16 name_off; // 0x0A: Offset to the attribute name + __le16 flags; // 0x0C: See ATTR_FLAG_XXX + __le16 id; // 0x0E: unique id (per record) + + union { + struct ATTR_RESIDENT res; // 0x10 + struct ATTR_NONRESIDENT nres; // 0x10 + }; +}; + +/* Define attribute sizes */ +#define SIZEOF_RESIDENT 0x18 +#define SIZEOF_NONRESIDENT_EX 0x48 +#define SIZEOF_NONRESIDENT 0x40 + +#define SIZEOF_RESIDENT_LE cpu_to_le16(0x18) +#define SIZEOF_NONRESIDENT_EX_LE cpu_to_le16(0x48) +#define SIZEOF_NONRESIDENT_LE cpu_to_le16(0x40) + +static inline u64 attr_ondisk_size(const struct ATTRIB *attr) +{ + return attr->non_res ? ((attr->flags & + (ATTR_FLAG_COMPRESSED | ATTR_FLAG_SPARSED)) ? + le64_to_cpu(attr->nres.total_size) : + le64_to_cpu(attr->nres.alloc_size)) : + QuadAlign(le32_to_cpu(attr->res.data_size)); +} + +static inline u64 attr_size(const struct ATTRIB *attr) +{ + return attr->non_res ? le64_to_cpu(attr->nres.data_size) : + le32_to_cpu(attr->res.data_size); +} + +static inline bool is_attr_encrypted(const struct ATTRIB *attr) +{ + return attr->flags & ATTR_FLAG_ENCRYPTED; +} + +static inline bool is_attr_sparsed(const struct ATTRIB *attr) +{ + return attr->flags & ATTR_FLAG_SPARSED; +} + +static inline bool is_attr_compressed(const struct ATTRIB *attr) +{ + return attr->flags & ATTR_FLAG_COMPRESSED; +} + +static inline bool is_attr_ext(const struct ATTRIB *attr) +{ + return attr->flags & (ATTR_FLAG_SPARSED | ATTR_FLAG_COMPRESSED); +} + +static inline bool is_attr_indexed(const struct ATTRIB *attr) +{ + return !attr->non_res && (attr->res.flags & RESIDENT_FLAG_INDEXED); +} + +static inline __le16 const *attr_name(const struct ATTRIB *attr) +{ + return Add2Ptr(attr, le16_to_cpu(attr->name_off)); +} + +static inline u64 attr_svcn(const struct ATTRIB *attr) +{ + return attr->non_res ? le64_to_cpu(attr->nres.svcn) : 0; +} + +/* the size of resident attribute by its resident size */ +#define BYTES_PER_RESIDENT(b) (0x18 + (b)) + +static_assert(sizeof(struct ATTRIB) == 0x48); +static_assert(sizeof(((struct ATTRIB *)NULL)->res) == 0x08); +static_assert(sizeof(((struct ATTRIB *)NULL)->nres) == 0x38); + +static inline void *resident_data_ex(const struct ATTRIB *attr, u32 datasize) +{ + u32 asize, rsize; + u16 off; + + if (attr->non_res) + return NULL; + + asize = le32_to_cpu(attr->size); + off = le16_to_cpu(attr->res.data_off); + + if (asize < datasize + off) + return NULL; + + rsize = le32_to_cpu(attr->res.data_size); + if (rsize < datasize) + return NULL; + + return Add2Ptr(attr, off); +} + +static inline void *resident_data(const struct ATTRIB *attr) +{ + return Add2Ptr(attr, le16_to_cpu(attr->res.data_off)); +} + +static inline void *attr_run(const struct ATTRIB *attr) +{ + return Add2Ptr(attr, le16_to_cpu(attr->nres.run_off)); +} + +/* Standard information attribute (0x10) */ +struct ATTR_STD_INFO { + __le64 cr_time; // 0x00: File creation file + __le64 m_time; // 0x08: File modification time + __le64 c_time; // 0x10: Last time any attribute was modified + __le64 a_time; // 0x18: File last access time + enum FILE_ATTRIBUTE fa; // 0x20: Standard DOS attributes & more + __le32 max_ver_num; // 0x24: Maximum Number of Versions + __le32 ver_num; // 0x28: Version Number + __le32 class_id; // 0x2C: Class Id from bidirectional Class Id index +}; + +static_assert(sizeof(struct ATTR_STD_INFO) == 0x30); + +#define SECURITY_ID_INVALID 0x00000000 +#define SECURITY_ID_FIRST 0x00000100 + +struct ATTR_STD_INFO5 { + __le64 cr_time; // 0x00: File creation file + __le64 m_time; // 0x08: File modification time + __le64 c_time; // 0x10: Last time any attribute was modified + __le64 a_time; // 0x18: File last access time + enum FILE_ATTRIBUTE fa; // 0x20: Standard DOS attributes & more + __le32 max_ver_num; // 0x24: Maximum Number of Versions + __le32 ver_num; // 0x28: Version Number + __le32 class_id; // 0x2C: Class Id from bidirectional Class Id index + + __le32 owner_id; // 0x30: Owner Id of the user owning the file. + __le32 security_id; // 0x34: The Security Id is a key in the $SII Index and $SDS + __le64 quota_charge; // 0x38: + __le64 usn; // 0x40: Last Update Sequence Number of the file. This is a direct + // index into the file $UsnJrnl. If zero, the USN Journal is + // disabled. +}; + +static_assert(sizeof(struct ATTR_STD_INFO5) == 0x48); + +/* attribute list entry structure (0x20) */ +struct ATTR_LIST_ENTRY { + enum ATTR_TYPE type; // 0x00: The type of attribute + __le16 size; // 0x04: The size of this record + u8 name_len; // 0x06: The length of attribute name + u8 name_off; // 0x07: The offset to attribute name + __le64 vcn; // 0x08: Starting VCN of this attribute + struct MFT_REF ref; // 0x10: MFT record number with attribute + __le16 id; // 0x18: struct ATTRIB ID + __le16 name[3]; // 0x1A: Just to align. To get real name can use bNameOffset + +}; // sizeof(0x20) + +static_assert(sizeof(struct ATTR_LIST_ENTRY) == 0x20); + +static inline u32 le_size(u8 name_len) +{ + return QuadAlign(offsetof(struct ATTR_LIST_ENTRY, name) + + name_len * sizeof(short)); +} + +/* returns 0 if 'attr' has the same type and name */ +static inline int le_cmp(const struct ATTR_LIST_ENTRY *le, + const struct ATTRIB *attr) +{ + return le->type != attr->type || le->name_len != attr->name_len || + (!le->name_len && + memcmp(Add2Ptr(le, le->name_off), + Add2Ptr(attr, le16_to_cpu(attr->name_off)), + le->name_len * sizeof(short))); +} + +static inline __le16 const *le_name(const struct ATTR_LIST_ENTRY *le) +{ + return Add2Ptr(le, le->name_off); +} + +/* File name types (the field type in struct ATTR_FILE_NAME ) */ +#define FILE_NAME_POSIX 0 +#define FILE_NAME_UNICODE 1 +#define FILE_NAME_DOS 2 +#define FILE_NAME_UNICODE_AND_DOS (FILE_NAME_DOS | FILE_NAME_UNICODE) + +/* Filename attribute structure (0x30) */ +struct NTFS_DUP_INFO { + __le64 cr_time; // 0x00: File creation file + __le64 m_time; // 0x08: File modification time + __le64 c_time; // 0x10: Last time any attribute was modified + __le64 a_time; // 0x18: File last access time + __le64 alloc_size; // 0x20: Data attribute allocated size, multiple of cluster size + __le64 data_size; // 0x28: Data attribute size <= Dataalloc_size + enum FILE_ATTRIBUTE fa; // 0x30: Standard DOS attributes & more + __le16 ea_size; // 0x34: Packed EAs + __le16 reparse; // 0x36: Used by Reparse + +}; // 0x38 + +struct ATTR_FILE_NAME { + struct MFT_REF home; // 0x00: MFT record for directory + struct NTFS_DUP_INFO dup;// 0x08 + u8 name_len; // 0x40: File name length in words + u8 type; // 0x41: File name type + __le16 name[]; // 0x42: File name +}; + +static_assert(sizeof(((struct ATTR_FILE_NAME *)NULL)->dup) == 0x38); +static_assert(offsetof(struct ATTR_FILE_NAME, name) == 0x42); +#define SIZEOF_ATTRIBUTE_FILENAME 0x44 +#define SIZEOF_ATTRIBUTE_FILENAME_MAX (0x42 + 255 * 2) + +static inline struct ATTRIB *attr_from_name(struct ATTR_FILE_NAME *fname) +{ + return (struct ATTRIB *)((char *)fname - SIZEOF_RESIDENT); +} + +static inline u16 fname_full_size(const struct ATTR_FILE_NAME *fname) +{ + // don't return struct_size(fname, name, fname->name_len); + return offsetof(struct ATTR_FILE_NAME, name) + + fname->name_len * sizeof(short); +} + +static inline u8 paired_name(u8 type) +{ + if (type == FILE_NAME_UNICODE) + return FILE_NAME_DOS; + if (type == FILE_NAME_DOS) + return FILE_NAME_UNICODE; + return FILE_NAME_POSIX; +} + +/* Index entry defines ( the field flags in NtfsDirEntry ) */ +#define NTFS_IE_HAS_SUBNODES cpu_to_le16(1) +#define NTFS_IE_LAST cpu_to_le16(2) + +/* Directory entry structure */ +struct NTFS_DE { + union { + struct MFT_REF ref; // 0x00: MFT record number with this file + struct { + __le16 data_off; // 0x00: + __le16 data_size; // 0x02: + __le32 res; // 0x04: must be 0 + } view; + }; + __le16 size; // 0x08: The size of this entry + __le16 key_size; // 0x0A: The size of File name length in bytes + 0x42 + __le16 flags; // 0x0C: Entry flags: NTFS_IE_XXX + __le16 res; // 0x0E: + + // Here any indexed attribute can be placed + // One of them is: + // struct ATTR_FILE_NAME AttrFileName; + // + + // The last 8 bytes of this structure contains + // the VBN of subnode + // !!! Note !!! + // This field is presented only if (flags & NTFS_IE_HAS_SUBNODES) + // __le64 vbn; +}; + +static_assert(sizeof(struct NTFS_DE) == 0x10); + +static inline void de_set_vbn_le(struct NTFS_DE *e, __le64 vcn) +{ + __le64 *v = Add2Ptr(e, le16_to_cpu(e->size) - sizeof(__le64)); + + *v = vcn; +} + +static inline void de_set_vbn(struct NTFS_DE *e, CLST vcn) +{ + __le64 *v = Add2Ptr(e, le16_to_cpu(e->size) - sizeof(__le64)); + + *v = cpu_to_le64(vcn); +} + +static inline __le64 de_get_vbn_le(const struct NTFS_DE *e) +{ + return *(__le64 *)Add2Ptr(e, le16_to_cpu(e->size) - sizeof(__le64)); +} + +static inline CLST de_get_vbn(const struct NTFS_DE *e) +{ + __le64 *v = Add2Ptr(e, le16_to_cpu(e->size) - sizeof(__le64)); + + return le64_to_cpu(*v); +} + +static inline struct NTFS_DE *de_get_next(const struct NTFS_DE *e) +{ + return Add2Ptr(e, le16_to_cpu(e->size)); +} + +static inline struct ATTR_FILE_NAME *de_get_fname(const struct NTFS_DE *e) +{ + return le16_to_cpu(e->key_size) >= SIZEOF_ATTRIBUTE_FILENAME ? + Add2Ptr(e, sizeof(struct NTFS_DE)) : + NULL; +} + +static inline bool de_is_last(const struct NTFS_DE *e) +{ + return e->flags & NTFS_IE_LAST; +} + +static inline bool de_has_vcn(const struct NTFS_DE *e) +{ + return e->flags & NTFS_IE_HAS_SUBNODES; +} + +static inline bool de_has_vcn_ex(const struct NTFS_DE *e) +{ + return (e->flags & NTFS_IE_HAS_SUBNODES) && + (u64)(-1) != *((u64 *)Add2Ptr(e, le16_to_cpu(e->size) - + sizeof(__le64))); +} + +#define MAX_BYTES_PER_NAME_ENTRY \ + QuadAlign(sizeof(struct NTFS_DE) + \ + offsetof(struct ATTR_FILE_NAME, name) + \ + NTFS_NAME_LEN * sizeof(short)) + +struct INDEX_HDR { + __le32 de_off; // 0x00: The offset from the start of this structure + // to the first NTFS_DE + __le32 used; // 0x04: The size of this structure plus all + // entries (quad-word aligned) + __le32 total; // 0x08: The allocated size of for this structure plus all entries + u8 flags; // 0x0C: 0x00 = Small directory, 0x01 = Large directory + u8 res[3]; + + // + // de_off + used <= total + // +}; + +static_assert(sizeof(struct INDEX_HDR) == 0x10); + +static inline struct NTFS_DE *hdr_first_de(const struct INDEX_HDR *hdr) +{ + u32 de_off = le32_to_cpu(hdr->de_off); + u32 used = le32_to_cpu(hdr->used); + struct NTFS_DE *e = Add2Ptr(hdr, de_off); + u16 esize; + + if (de_off >= used || de_off >= le32_to_cpu(hdr->total)) + return NULL; + + esize = le16_to_cpu(e->size); + if (esize < sizeof(struct NTFS_DE) || de_off + esize > used) + return NULL; + + return e; +} + +static inline struct NTFS_DE *hdr_next_de(const struct INDEX_HDR *hdr, + const struct NTFS_DE *e) +{ + size_t off = PtrOffset(hdr, e); + u32 used = le32_to_cpu(hdr->used); + u16 esize; + + if (off >= used) + return NULL; + + esize = le16_to_cpu(e->size); + + if (esize < sizeof(struct NTFS_DE) || + off + esize + sizeof(struct NTFS_DE) > used) + return NULL; + + return Add2Ptr(e, esize); +} + +static inline bool hdr_has_subnode(const struct INDEX_HDR *hdr) +{ + return hdr->flags & 1; +} + +struct INDEX_BUFFER { + struct NTFS_RECORD_HEADER rhdr; // 'INDX' + __le64 vbn; // 0x10: vcn if index >= cluster or vsn id index < cluster + struct INDEX_HDR ihdr; // 0x18: +}; + +static_assert(sizeof(struct INDEX_BUFFER) == 0x28); + +static inline bool ib_is_empty(const struct INDEX_BUFFER *ib) +{ + const struct NTFS_DE *first = hdr_first_de(&ib->ihdr); + + return !first || de_is_last(first); +} + +static inline bool ib_is_leaf(const struct INDEX_BUFFER *ib) +{ + return !(ib->ihdr.flags & 1); +} + +/* Index root structure ( 0x90 ) */ +enum COLLATION_RULE { + NTFS_COLLATION_TYPE_BINARY = cpu_to_le32(0), + // $I30 + NTFS_COLLATION_TYPE_FILENAME = cpu_to_le32(0x01), + // $SII of $Secure and $Q of Quota + NTFS_COLLATION_TYPE_UINT = cpu_to_le32(0x10), + // $O of Quota + NTFS_COLLATION_TYPE_SID = cpu_to_le32(0x11), + // $SDH of $Secure + NTFS_COLLATION_TYPE_SECURITY_HASH = cpu_to_le32(0x12), + // $O of ObjId and "$R" for Reparse + NTFS_COLLATION_TYPE_UINTS = cpu_to_le32(0x13) +}; + +static_assert(sizeof(enum COLLATION_RULE) == 4); + +// +struct INDEX_ROOT { + enum ATTR_TYPE type; // 0x00: The type of attribute to index on + enum COLLATION_RULE rule; // 0x04: The rule + __le32 index_block_size;// 0x08: The size of index record + u8 index_block_clst; // 0x0C: The number of clusters or sectors per index + u8 res[3]; + struct INDEX_HDR ihdr; // 0x10: +}; + +static_assert(sizeof(struct INDEX_ROOT) == 0x20); +static_assert(offsetof(struct INDEX_ROOT, ihdr) == 0x10); + +#define VOLUME_FLAG_DIRTY cpu_to_le16(0x0001) +#define VOLUME_FLAG_RESIZE_LOG_FILE cpu_to_le16(0x0002) + +struct VOLUME_INFO { + __le64 res1; // 0x00 + u8 major_ver; // 0x08: NTFS major version number (before .) + u8 minor_ver; // 0x09: NTFS minor version number (after .) + __le16 flags; // 0x0A: Volume flags, see VOLUME_FLAG_XXX + +}; // sizeof=0xC + +#define SIZEOF_ATTRIBUTE_VOLUME_INFO 0xc + +#define NTFS_LABEL_MAX_LENGTH (0x100 / sizeof(short)) +#define NTFS_ATTR_INDEXABLE cpu_to_le32(0x00000002) +#define NTFS_ATTR_DUPALLOWED cpu_to_le32(0x00000004) +#define NTFS_ATTR_MUST_BE_INDEXED cpu_to_le32(0x00000010) +#define NTFS_ATTR_MUST_BE_NAMED cpu_to_le32(0x00000020) +#define NTFS_ATTR_MUST_BE_RESIDENT cpu_to_le32(0x00000040) +#define NTFS_ATTR_LOG_ALWAYS cpu_to_le32(0x00000080) + +/* $AttrDef file entry */ +struct ATTR_DEF_ENTRY { + __le16 name[0x40]; // 0x00: Attr name + enum ATTR_TYPE type; // 0x80: struct ATTRIB type + __le32 res; // 0x84: + enum COLLATION_RULE rule; // 0x88: + __le32 flags; // 0x8C: NTFS_ATTR_XXX (see above) + __le64 min_sz; // 0x90: Minimum attribute data size + __le64 max_sz; // 0x98: Maximum attribute data size +}; + +static_assert(sizeof(struct ATTR_DEF_ENTRY) == 0xa0); + +/* Object ID (0x40) */ +struct OBJECT_ID { + struct GUID ObjId; // 0x00: Unique Id assigned to file + struct GUID BirthVolumeId;// 0x10: Birth Volume Id is the Object Id of the Volume on + // which the Object Id was allocated. It never changes + struct GUID BirthObjectId; // 0x20: Birth Object Id is the first Object Id that was + // ever assigned to this MFT Record. I.e. If the Object Id + // is changed for some reason, this field will reflect the + // original value of the Object Id. + struct GUID DomainId; // 0x30: Domain Id is currently unused but it is intended to be + // used in a network environment where the local machine is + // part of a Windows 2000 Domain. This may be used in a Windows + // 2000 Advanced Server managed domain. +}; + +static_assert(sizeof(struct OBJECT_ID) == 0x40); + +/* O Directory entry structure ( rule = 0x13 ) */ +struct NTFS_DE_O { + struct NTFS_DE de; + struct GUID ObjId; // 0x10: Unique Id assigned to file + struct MFT_REF ref; // 0x20: MFT record number with this file + struct GUID BirthVolumeId; // 0x28: Birth Volume Id is the Object Id of the Volume on + // which the Object Id was allocated. It never changes + struct GUID BirthObjectId; // 0x38: Birth Object Id is the first Object Id that was + // ever assigned to this MFT Record. I.e. If the Object Id + // is changed for some reason, this field will reflect the + // original value of the Object Id. + // This field is valid if data_size == 0x48 + struct GUID BirthDomainId; // 0x48: Domain Id is currently unused but it is intended + // to be used in a network environment where the local + // machine is part of a Windows 2000 Domain. This may be + // used in a Windows 2000 Advanced Server managed domain. +}; + +static_assert(sizeof(struct NTFS_DE_O) == 0x58); + +#define NTFS_OBJECT_ENTRY_DATA_SIZE1 \ + 0x38 // struct NTFS_DE_O.BirthDomainId is not used +#define NTFS_OBJECT_ENTRY_DATA_SIZE2 \ + 0x48 // struct NTFS_DE_O.BirthDomainId is used + +/* Q Directory entry structure ( rule = 0x11 ) */ +struct NTFS_DE_Q { + struct NTFS_DE de; + __le32 owner_id; // 0x10: Unique Id assigned to file + __le32 Version; // 0x14: 0x02 + __le32 flags2; // 0x18: Quota flags, see above + __le64 BytesUsed; // 0x1C: + __le64 ChangeTime; // 0x24: + __le64 WarningLimit; // 0x28: + __le64 HardLimit; // 0x34: + __le64 ExceededTime; // 0x3C: + + // SID is placed here +}; // sizeof() = 0x44 + +#define SIZEOF_NTFS_DE_Q 0x44 + +#define SecurityDescriptorsBlockSize 0x40000 // 256K +#define SecurityDescriptorMaxSize 0x20000 // 128K +#define Log2OfSecurityDescriptorsBlockSize 18 + +struct SECURITY_KEY { + __le32 hash; // Hash value for descriptor + __le32 sec_id; // Security Id (guaranteed unique) +}; + +/* Security descriptors (the content of $Secure::SDS data stream) */ +struct SECURITY_HDR { + struct SECURITY_KEY key; // 0x00: Security Key + __le64 off; // 0x08: Offset of this entry in the file + __le32 size; // 0x10: Size of this entry, 8 byte aligned + // + // Security descriptor itself is placed here + // Total size is 16 byte aligned + // +} __packed; + +#define SIZEOF_SECURITY_HDR 0x14 + +/* SII Directory entry structure */ +struct NTFS_DE_SII { + struct NTFS_DE de; + __le32 sec_id; // 0x10: Key: sizeof(security_id) = wKeySize + struct SECURITY_HDR sec_hdr; // 0x14: +} __packed; + +#define SIZEOF_SII_DIRENTRY 0x28 + +/* SDH Directory entry structure */ +struct NTFS_DE_SDH { + struct NTFS_DE de; + struct SECURITY_KEY key; // 0x10: Key + struct SECURITY_HDR sec_hdr; // 0x18: Data + __le16 magic[2]; // 0x2C: 0x00490049 "I I" +}; + +#define SIZEOF_SDH_DIRENTRY 0x30 + +struct REPARSE_KEY { + __le32 ReparseTag; // 0x00: Reparse Tag + struct MFT_REF ref; // 0x04: MFT record number with this file +}; // sizeof() = 0x0C + +static_assert(offsetof(struct REPARSE_KEY, ref) == 0x04); +#define SIZEOF_REPARSE_KEY 0x0C + +/* Reparse Directory entry structure */ +struct NTFS_DE_R { + struct NTFS_DE de; + struct REPARSE_KEY key; // 0x10: Reparse Key + u32 zero; // 0x1c +}; // sizeof() = 0x20 + +static_assert(sizeof(struct NTFS_DE_R) == 0x20); + +/* CompressReparseBuffer.WofVersion */ +#define WOF_CURRENT_VERSION cpu_to_le32(1) +/* CompressReparseBuffer.WofProvider */ +#define WOF_PROVIDER_WIM cpu_to_le32(1) +/* CompressReparseBuffer.WofProvider */ +#define WOF_PROVIDER_SYSTEM cpu_to_le32(2) +/* CompressReparseBuffer.ProviderVer */ +#define WOF_PROVIDER_CURRENT_VERSION cpu_to_le32(1) + +#define WOF_COMPRESSION_XPRESS4K cpu_to_le32(0) // 4k +#define WOF_COMPRESSION_LZX32K cpu_to_le32(1) // 32k +#define WOF_COMPRESSION_XPRESS8K cpu_to_le32(2) // 8k +#define WOF_COMPRESSION_XPRESS16K cpu_to_le32(3) // 16k + +/* + * ATTR_REPARSE (0xC0) + * + * The reparse struct GUID structure is used by all 3rd party layered drivers to + * store data in a reparse point. For non-Microsoft tags, The struct GUID field + * cannot be GUID_NULL. + * The constraints on reparse tags are defined below. + * Microsoft tags can also be used with this format of the reparse point buffer. + */ +struct REPARSE_POINT { + __le32 ReparseTag; // 0x00: + __le16 ReparseDataLength;// 0x04: + __le16 Reserved; + + struct GUID Guid; // 0x08: + + // + // Here GenericReparseBuffer is placed + // +}; + +static_assert(sizeof(struct REPARSE_POINT) == 0x18); + +// +// Maximum allowed size of the reparse data. +// +#define MAXIMUM_REPARSE_DATA_BUFFER_SIZE (16 * 1024) + +// +// The value of the following constant needs to satisfy the following +// conditions: +// (1) Be at least as large as the largest of the reserved tags. +// (2) Be strictly smaller than all the tags in use. +// +#define IO_REPARSE_TAG_RESERVED_RANGE 1 + +// +// The reparse tags are a ULONG. The 32 bits are laid out as follows: +// +// 3 3 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 +// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 +// +-+-+-+-+-----------------------+-------------------------------+ +// |M|R|N|R| Reserved bits | Reparse Tag Value | +// +-+-+-+-+-----------------------+-------------------------------+ +// +// M is the Microsoft bit. When set to 1, it denotes a tag owned by Microsoft. +// All ISVs must use a tag with a 0 in this position. +// Note: If a Microsoft tag is used by non-Microsoft software, the +// behavior is not defined. +// +// R is reserved. Must be zero for non-Microsoft tags. +// +// N is name surrogate. When set to 1, the file represents another named +// entity in the system. +// +// The M and N bits are OR-able. +// The following macros check for the M and N bit values: +// + +// +// Macro to determine whether a reparse point tag corresponds to a tag +// owned by Microsoft. +// +#define IsReparseTagMicrosoft(_tag) (((_tag)&IO_REPARSE_TAG_MICROSOFT)) + +// +// Macro to determine whether a reparse point tag is a name surrogate +// +#define IsReparseTagNameSurrogate(_tag) (((_tag)&IO_REPARSE_TAG_NAME_SURROGATE)) + +// +// The following constant represents the bits that are valid to use in +// reparse tags. +// +#define IO_REPARSE_TAG_VALID_VALUES 0xF000FFFF + +// +// Macro to determine whether a reparse tag is a valid tag. +// +#define IsReparseTagValid(_tag) \ + (!((_tag) & ~IO_REPARSE_TAG_VALID_VALUES) && \ + ((_tag) > IO_REPARSE_TAG_RESERVED_RANGE)) + +// +// Microsoft tags for reparse points. +// + +enum IO_REPARSE_TAG { + IO_REPARSE_TAG_SYMBOLIC_LINK = cpu_to_le32(0), + IO_REPARSE_TAG_NAME_SURROGATE = cpu_to_le32(0x20000000), + IO_REPARSE_TAG_MICROSOFT = cpu_to_le32(0x80000000), + IO_REPARSE_TAG_MOUNT_POINT = cpu_to_le32(0xA0000003), + IO_REPARSE_TAG_SYMLINK = cpu_to_le32(0xA000000C), + IO_REPARSE_TAG_HSM = cpu_to_le32(0xC0000004), + IO_REPARSE_TAG_SIS = cpu_to_le32(0x80000007), + IO_REPARSE_TAG_DEDUP = cpu_to_le32(0x80000013), + IO_REPARSE_TAG_COMPRESS = cpu_to_le32(0x80000017), + + // + // The reparse tag 0x80000008 is reserved for Microsoft internal use + // (may be published in the future) + // + + // + // Microsoft reparse tag reserved for DFS + // + IO_REPARSE_TAG_DFS = cpu_to_le32(0x8000000A), + + // + // Microsoft reparse tag reserved for the file system filter manager + // + IO_REPARSE_TAG_FILTER_MANAGER = cpu_to_le32(0x8000000B), + + // + // Non-Microsoft tags for reparse points + // + + // + // Tag allocated to CONGRUENT, May 2000. Used by IFSTEST + // + IO_REPARSE_TAG_IFSTEST_CONGRUENT = cpu_to_le32(0x00000009), + + // + // Tag allocated to ARKIVIO + // + IO_REPARSE_TAG_ARKIVIO = cpu_to_le32(0x0000000C), + + // + // Tag allocated to SOLUTIONSOFT + // + IO_REPARSE_TAG_SOLUTIONSOFT = cpu_to_le32(0x2000000D), + + // + // Tag allocated to COMMVAULT + // + IO_REPARSE_TAG_COMMVAULT = cpu_to_le32(0x0000000E), + + // OneDrive?? + IO_REPARSE_TAG_CLOUD = cpu_to_le32(0x9000001A), + IO_REPARSE_TAG_CLOUD_1 = cpu_to_le32(0x9000101A), + IO_REPARSE_TAG_CLOUD_2 = cpu_to_le32(0x9000201A), + IO_REPARSE_TAG_CLOUD_3 = cpu_to_le32(0x9000301A), + IO_REPARSE_TAG_CLOUD_4 = cpu_to_le32(0x9000401A), + IO_REPARSE_TAG_CLOUD_5 = cpu_to_le32(0x9000501A), + IO_REPARSE_TAG_CLOUD_6 = cpu_to_le32(0x9000601A), + IO_REPARSE_TAG_CLOUD_7 = cpu_to_le32(0x9000701A), + IO_REPARSE_TAG_CLOUD_8 = cpu_to_le32(0x9000801A), + IO_REPARSE_TAG_CLOUD_9 = cpu_to_le32(0x9000901A), + IO_REPARSE_TAG_CLOUD_A = cpu_to_le32(0x9000A01A), + IO_REPARSE_TAG_CLOUD_B = cpu_to_le32(0x9000B01A), + IO_REPARSE_TAG_CLOUD_C = cpu_to_le32(0x9000C01A), + IO_REPARSE_TAG_CLOUD_D = cpu_to_le32(0x9000D01A), + IO_REPARSE_TAG_CLOUD_E = cpu_to_le32(0x9000E01A), + IO_REPARSE_TAG_CLOUD_F = cpu_to_le32(0x9000F01A), + +}; + +#define SYMLINK_FLAG_RELATIVE 1 + +/* Microsoft reparse buffer. (see DDK for details) */ +struct REPARSE_DATA_BUFFER { + __le32 ReparseTag; // 0x00: + __le16 ReparseDataLength; // 0x04: + __le16 Reserved; + + union { + // If ReparseTag == 0xA0000003 (IO_REPARSE_TAG_MOUNT_POINT) + struct { + __le16 SubstituteNameOffset; // 0x08 + __le16 SubstituteNameLength; // 0x0A + __le16 PrintNameOffset; // 0x0C + __le16 PrintNameLength; // 0x0E + __le16 PathBuffer[]; // 0x10 + } MountPointReparseBuffer; + + // If ReparseTag == 0xA000000C (IO_REPARSE_TAG_SYMLINK) + // https://msdn.microsoft.com/en-us/library/cc232006.aspx + struct { + __le16 SubstituteNameOffset; // 0x08 + __le16 SubstituteNameLength; // 0x0A + __le16 PrintNameOffset; // 0x0C + __le16 PrintNameLength; // 0x0E + // 0-absolute path 1- relative path, SYMLINK_FLAG_RELATIVE + __le32 Flags; // 0x10 + __le16 PathBuffer[]; // 0x14 + } SymbolicLinkReparseBuffer; + + // If ReparseTag == 0x80000017U + struct { + __le32 WofVersion; // 0x08 == 1 + /* 1 - WIM backing provider ("WIMBoot"), + * 2 - System compressed file provider + */ + __le32 WofProvider; // 0x0C + __le32 ProviderVer; // 0x10: == 1 WOF_FILE_PROVIDER_CURRENT_VERSION == 1 + __le32 CompressionFormat; // 0x14: 0, 1, 2, 3. See WOF_COMPRESSION_XXX + } CompressReparseBuffer; + + struct { + u8 DataBuffer[1]; // 0x08 + } GenericReparseBuffer; + }; +}; + +/* ATTR_EA_INFO (0xD0) */ + +#define FILE_NEED_EA 0x80 // See ntifs.h +/* FILE_NEED_EA, indicates that the file to which the EA belongs cannot be + * interpreted without understanding the associated extended attributes. + */ +struct EA_INFO { + __le16 size_pack; // 0x00: Size of buffer to hold in packed form + __le16 count; // 0x02: Count of EA's with FILE_NEED_EA bit set + __le32 size; // 0x04: Size of buffer to hold in unpacked form +}; + +static_assert(sizeof(struct EA_INFO) == 8); + +/* ATTR_EA (0xE0) */ +struct EA_FULL { + __le32 size; // 0x00: (not in packed) + u8 flags; // 0x04 + u8 name_len; // 0x05 + __le16 elength; // 0x06 + u8 name[]; // 0x08 +}; + +static_assert(offsetof(struct EA_FULL, name) == 8); + +#define ACL_REVISION 2 +#define ACL_REVISION_DS 4 + +#define SE_SELF_RELATIVE cpu_to_le16(0x8000) + +struct SECURITY_DESCRIPTOR_RELATIVE { + u8 Revision; + u8 Sbz1; + __le16 Control; + __le32 Owner; + __le32 Group; + __le32 Sacl; + __le32 Dacl; +}; +static_assert(sizeof(struct SECURITY_DESCRIPTOR_RELATIVE) == 0x14); + +struct ACE_HEADER { + u8 AceType; + u8 AceFlags; + __le16 AceSize; +}; +static_assert(sizeof(struct ACE_HEADER) == 4); + +struct ACL { + u8 AclRevision; + u8 Sbz1; + __le16 AclSize; + __le16 AceCount; + __le16 Sbz2; +}; +static_assert(sizeof(struct ACL) == 8); + +struct SID { + u8 Revision; + u8 SubAuthorityCount; + u8 IdentifierAuthority[6]; + __le32 SubAuthority[]; +}; +static_assert(offsetof(struct SID, SubAuthority) == 8); + +// clang-format on diff --git a/fs/ntfs3/ntfs_fs.h b/fs/ntfs3/ntfs_fs.h new file mode 100644 index 000000000000..5e1dd628d3cc --- /dev/null +++ b/fs/ntfs3/ntfs_fs.h @@ -0,0 +1,1085 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + */ + +// clang-format off +#define MINUS_ONE_T ((size_t)(-1)) +/* Biggest MFT / smallest cluster */ +#define MAXIMUM_BYTES_PER_MFT 4096 +#define NTFS_BLOCKS_PER_MFT_RECORD (MAXIMUM_BYTES_PER_MFT / 512) + +#define MAXIMUM_BYTES_PER_INDEX 4096 +#define NTFS_BLOCKS_PER_INODE (MAXIMUM_BYTES_PER_INDEX / 512) + +/* ntfs specific error code when fixup failed*/ +#define E_NTFS_FIXUP 555 +/* ntfs specific error code about resident->nonresident*/ +#define E_NTFS_NONRESIDENT 556 + +/* sbi->flags */ +#define NTFS_FLAGS_NODISCARD 0x00000001 +/* Set when LogFile is replaying */ +#define NTFS_FLAGS_LOG_REPLAYING 0x00000008 +/* Set when we changed first MFT's which copy must be updated in $MftMirr */ +#define NTFS_FLAGS_MFTMIRR 0x00001000 +#define NTFS_FLAGS_NEED_REPLAY 0x04000000 + + +/* ni->ni_flags */ +/* + * Data attribute is external compressed (lzx/xpress) + * 1 - WOF_COMPRESSION_XPRESS4K + * 2 - WOF_COMPRESSION_XPRESS8K + * 3 - WOF_COMPRESSION_XPRESS16K + * 4 - WOF_COMPRESSION_LZX32K + */ +#define NI_FLAG_COMPRESSED_MASK 0x0000000f +/* Data attribute is deduplicated */ +#define NI_FLAG_DEDUPLICATED 0x00000010 +#define NI_FLAG_EA 0x00000020 +#define NI_FLAG_DIR 0x00000040 +#define NI_FLAG_RESIDENT 0x00000080 +#define NI_FLAG_UPDATE_PARENT 0x00000100 +// clang-format on + +struct ntfs_mount_options { + struct nls_table *nls; + + kuid_t fs_uid; + kgid_t fs_gid; + u16 fs_fmask_inv; + u16 fs_dmask_inv; + + unsigned uid : 1, /* uid was set */ + gid : 1, /* gid was set */ + fmask : 1, /* fmask was set */ + dmask : 1, /*dmask was set*/ + sys_immutable : 1, /* immutable system files */ + discard : 1, /* issue discard requests on deletions */ + sparse : 1, /*create sparse files*/ + showmeta : 1, /*show meta files*/ + nohidden : 1, /*do not show hidden files*/ + force : 1, /*rw mount dirty volume*/ + no_acs_rules : 1, /*exclude acs rules*/ + prealloc : 1 /*preallocate space when file is growing*/ + ; +}; + +/* special value to unpack and deallocate*/ +#define RUN_DEALLOCATE ((struct runs_tree *)(size_t)1) + +/* TODO: use rb tree instead of array */ +struct runs_tree { + struct ntfs_run *runs; + size_t count; // Currently used size a ntfs_run storage. + size_t allocated; // Currently allocated ntfs_run storage size. +}; + +struct ntfs_buffers { + /* Biggest MFT / smallest cluster = 4096 / 512 = 8 */ + /* Biggest index / smallest cluster = 4096 / 512 = 8 */ + struct buffer_head *bh[PAGE_SIZE >> SECTOR_SHIFT]; + u32 bytes; + u32 nbufs; + u32 off; +}; + +enum ALLOCATE_OPT { + ALLOCATE_DEF = 0, // Allocate all clusters + ALLOCATE_MFT = 1, // Allocate for MFT +}; + +enum bitmap_mutex_classes { + BITMAP_MUTEX_CLUSTERS = 0, + BITMAP_MUTEX_MFT = 1, +}; + +struct wnd_bitmap { + struct super_block *sb; + struct rw_semaphore rw_lock; + + struct runs_tree run; + size_t nbits; + + size_t total_zeroes; // total number of free bits + u16 *free_bits; // free bits in each window + size_t nwnd; + u32 bits_last; // bits in last window + + struct rb_root start_tree; // extents, sorted by 'start' + struct rb_root count_tree; // extents, sorted by 'count + start' + size_t count; // extents count + + /* + * -1 Tree is activated but not updated (too many fragments) + * 0 - Tree is not activated + * 1 - Tree is activated and updated + */ + int uptodated; + size_t extent_min; // Minimal extent used while building + size_t extent_max; // Upper estimate of biggest free block + + /* Zone [bit, end) */ + size_t zone_bit; + size_t zone_end; + + bool set_tail; // not necessary in driver + bool inited; +}; + +typedef int (*NTFS_CMP_FUNC)(const void *key1, size_t len1, const void *key2, + size_t len2, const void *param); + +enum index_mutex_classed { + INDEX_MUTEX_I30 = 0, + INDEX_MUTEX_SII = 1, + INDEX_MUTEX_SDH = 2, + INDEX_MUTEX_SO = 3, + INDEX_MUTEX_SQ = 4, + INDEX_MUTEX_SR = 5, + INDEX_MUTEX_TOTAL +}; + +/* ntfs_index - allocation unit inside directory */ +struct ntfs_index { + struct runs_tree bitmap_run; + struct runs_tree alloc_run; + /* read/write access to 'bitmap_run'/'alloc_run' while ntfs_readdir */ + struct rw_semaphore run_lock; + + /*TODO: remove 'cmp'*/ + NTFS_CMP_FUNC cmp; + + u8 index_bits; // log2(root->index_block_size) + u8 idx2vbn_bits; // log2(root->index_block_clst) + u8 vbn2vbo_bits; // index_block_size < cluster? 9 : cluster_bits + u8 type; // index_mutex_classed +}; + +/* Minimum mft zone */ +#define NTFS_MIN_MFT_ZONE 100 + +/* ntfs file system in-core superblock data */ +struct ntfs_sb_info { + struct super_block *sb; + + u32 discard_granularity; + u64 discard_granularity_mask_inv; // ~(discard_granularity_mask_inv-1) + + u32 cluster_size; // bytes per cluster + u32 cluster_mask; // == cluster_size - 1 + u64 cluster_mask_inv; // ~(cluster_size - 1) + u32 block_mask; // sb->s_blocksize - 1 + u32 blocks_per_cluster; // cluster_size / sb->s_blocksize + + u32 record_size; + u32 sector_size; + u32 index_size; + + u8 sector_bits; + u8 cluster_bits; + u8 record_bits; + + u64 maxbytes; // Maximum size for normal files + u64 maxbytes_sparse; // Maximum size for sparse file + + u32 flags; // See NTFS_FLAGS_XXX + + CLST bad_clusters; // The count of marked bad clusters + + u16 max_bytes_per_attr; // maximum attribute size in record + u16 attr_size_tr; // attribute size threshold (320 bytes) + + /* Records in $Extend */ + CLST objid_no; + CLST quota_no; + CLST reparse_no; + CLST usn_jrnl_no; + + struct ATTR_DEF_ENTRY *def_table; // attribute definition table + u32 def_entries; + u32 ea_max_size; + + struct MFT_REC *new_rec; + + u16 *upcase; + + struct { + u64 lbo, lbo2; + struct ntfs_inode *ni; + struct wnd_bitmap bitmap; // $MFT::Bitmap + /* + * MFT records [11-24) used to expand MFT itself + * They always marked as used in $MFT::Bitmap + * 'reserved_bitmap' contains real bitmap of these records + */ + ulong reserved_bitmap; // bitmap of used records [11 - 24) + size_t next_free; // The next record to allocate from + size_t used; // mft valid size in records + u32 recs_mirr; // Number of records in MFTMirr + u8 next_reserved; + u8 reserved_bitmap_inited; + } mft; + + struct { + struct wnd_bitmap bitmap; // $Bitmap::Data + CLST next_free_lcn; + } used; + + struct { + u64 size; // in bytes + u64 blocks; // in blocks + u64 ser_num; + struct ntfs_inode *ni; + __le16 flags; // cached current VOLUME_INFO::flags, VOLUME_FLAG_DIRTY + u8 major_ver; + u8 minor_ver; + char label[65]; + bool real_dirty; /* real fs state*/ + } volume; + + struct { + struct ntfs_index index_sii; + struct ntfs_index index_sdh; + struct ntfs_inode *ni; + u32 next_id; + u64 next_off; + + __le32 def_security_id; + } security; + + struct { + struct ntfs_index index_r; + struct ntfs_inode *ni; + u64 max_size; // 16K + } reparse; + + struct { + struct ntfs_index index_o; + struct ntfs_inode *ni; + } objid; + + struct { + struct mutex mtx_lznt; + struct lznt *lznt; +#ifdef CONFIG_NTFS3_LZX_XPRESS + struct mutex mtx_xpress; + struct xpress_decompressor *xpress; + struct mutex mtx_lzx; + struct lzx_decompressor *lzx; +#endif + } compress; + + struct ntfs_mount_options options; + struct ratelimit_state msg_ratelimit; +}; + +/* + * one MFT record(usually 1024 bytes), consists of attributes + */ +struct mft_inode { + struct rb_node node; + struct ntfs_sb_info *sbi; + + struct MFT_REC *mrec; + struct ntfs_buffers nb; + + CLST rno; + bool dirty; +}; + +/* nested class for ntfs_inode::ni_lock */ +enum ntfs_inode_mutex_lock_class { + NTFS_INODE_MUTEX_DIRTY, + NTFS_INODE_MUTEX_SECURITY, + NTFS_INODE_MUTEX_OBJID, + NTFS_INODE_MUTEX_REPARSE, + NTFS_INODE_MUTEX_NORMAL, + NTFS_INODE_MUTEX_PARENT, +}; + +/* + * ntfs inode - extends linux inode. consists of one or more mft inodes + */ +struct ntfs_inode { + struct mft_inode mi; // base record + + /* + * Valid size: [0 - i_valid) - these range in file contains valid data + * Range [i_valid - inode->i_size) - contains 0 + * Usually i_valid <= inode->i_size + */ + u64 i_valid; + struct timespec64 i_crtime; + + struct mutex ni_lock; + + /* file attributes from std */ + enum FILE_ATTRIBUTE std_fa; + __le32 std_security_id; + + /* + * tree of mft_inode + * not empty when primary MFT record (usually 1024 bytes) can't save all attributes + * e.g. file becomes too fragmented or contains a lot of names + */ + struct rb_root mi_tree; + + /* + * This member is used in ntfs_readdir to ensure that all subrecords are loaded + */ + u8 mi_loaded; + + union { + struct ntfs_index dir; + struct { + struct rw_semaphore run_lock; + struct runs_tree run; +#ifdef CONFIG_NTFS3_LZX_XPRESS + struct page *offs_page; +#endif + } file; + }; + + struct { + struct runs_tree run; + struct ATTR_LIST_ENTRY *le; // 1K aligned memory + size_t size; + bool dirty; + } attr_list; + + size_t ni_flags; // NI_FLAG_XXX + + struct inode vfs_inode; +}; + +struct indx_node { + struct ntfs_buffers nb; + struct INDEX_BUFFER *index; +}; + +struct ntfs_fnd { + int level; + struct indx_node *nodes[20]; + struct NTFS_DE *de[20]; + struct NTFS_DE *root_de; +}; + +enum REPARSE_SIGN { + REPARSE_NONE = 0, + REPARSE_COMPRESSED = 1, + REPARSE_DEDUPLICATED = 2, + REPARSE_LINK = 3 +}; + +/* functions from attrib.c*/ +int attr_load_runs(struct ATTRIB *attr, struct ntfs_inode *ni, + struct runs_tree *run, const CLST *vcn); +int attr_allocate_clusters(struct ntfs_sb_info *sbi, struct runs_tree *run, + CLST vcn, CLST lcn, CLST len, CLST *pre_alloc, + enum ALLOCATE_OPT opt, CLST *alen, const size_t fr, + CLST *new_lcn); +int attr_make_nonresident(struct ntfs_inode *ni, struct ATTRIB *attr, + struct ATTR_LIST_ENTRY *le, struct mft_inode *mi, + u64 new_size, struct runs_tree *run, + struct ATTRIB **ins_attr, struct page *page); +int attr_set_size(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, struct runs_tree *run, + u64 new_size, const u64 *new_valid, bool keep_prealloc, + struct ATTRIB **ret); +int attr_data_get_block(struct ntfs_inode *ni, CLST vcn, CLST clen, CLST *lcn, + CLST *len, bool *new); +int attr_data_read_resident(struct ntfs_inode *ni, struct page *page); +int attr_data_write_resident(struct ntfs_inode *ni, struct page *page); +int attr_load_runs_vcn(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, struct runs_tree *run, + CLST vcn); +int attr_load_runs_range(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, struct runs_tree *run, + u64 from, u64 to); +int attr_wof_frame_info(struct ntfs_inode *ni, struct ATTRIB *attr, + struct runs_tree *run, u64 frame, u64 frames, + u8 frame_bits, u32 *ondisk_size, u64 *vbo_data); +int attr_is_frame_compressed(struct ntfs_inode *ni, struct ATTRIB *attr, + CLST frame, CLST *clst_data); +int attr_allocate_frame(struct ntfs_inode *ni, CLST frame, size_t compr_size, + u64 new_valid); +int attr_collapse_range(struct ntfs_inode *ni, u64 vbo, u64 bytes); +int attr_punch_hole(struct ntfs_inode *ni, u64 vbo, u64 bytes); + +/* functions from attrlist.c*/ +void al_destroy(struct ntfs_inode *ni); +bool al_verify(struct ntfs_inode *ni); +int ntfs_load_attr_list(struct ntfs_inode *ni, struct ATTRIB *attr); +struct ATTR_LIST_ENTRY *al_enumerate(struct ntfs_inode *ni, + struct ATTR_LIST_ENTRY *le); +struct ATTR_LIST_ENTRY *al_find_le(struct ntfs_inode *ni, + struct ATTR_LIST_ENTRY *le, + const struct ATTRIB *attr); +struct ATTR_LIST_ENTRY *al_find_ex(struct ntfs_inode *ni, + struct ATTR_LIST_ENTRY *le, + enum ATTR_TYPE type, const __le16 *name, + u8 name_len, const CLST *vcn); +int al_add_le(struct ntfs_inode *ni, enum ATTR_TYPE type, const __le16 *name, + u8 name_len, CLST svcn, __le16 id, const struct MFT_REF *ref, + struct ATTR_LIST_ENTRY **new_le); +bool al_remove_le(struct ntfs_inode *ni, struct ATTR_LIST_ENTRY *le); +bool al_delete_le(struct ntfs_inode *ni, enum ATTR_TYPE type, CLST vcn, + const __le16 *name, size_t name_len, + const struct MFT_REF *ref); +int al_update(struct ntfs_inode *ni); +static inline size_t al_aligned(size_t size) +{ + return (size + 1023) & ~(size_t)1023; +} + +/* globals from bitfunc.c */ +bool are_bits_clear(const ulong *map, size_t bit, size_t nbits); +bool are_bits_set(const ulong *map, size_t bit, size_t nbits); +size_t get_set_bits_ex(const ulong *map, size_t bit, size_t nbits); + +/* globals from dir.c */ +int ntfs_utf16_to_nls(struct ntfs_sb_info *sbi, const struct le_str *uni, + u8 *buf, int buf_len); +int ntfs_nls_to_utf16(struct ntfs_sb_info *sbi, const u8 *name, u32 name_len, + struct cpu_str *uni, u32 max_ulen, + enum utf16_endian endian); +struct inode *dir_search_u(struct inode *dir, const struct cpu_str *uni, + struct ntfs_fnd *fnd); +bool dir_is_empty(struct inode *dir); +extern const struct file_operations ntfs_dir_operations; + +/* globals from file.c*/ +int ntfs_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, u32 flags); +void ntfs_sparse_cluster(struct inode *inode, struct page *page0, CLST vcn, + CLST len); +int ntfs3_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr); +int ntfs_file_open(struct inode *inode, struct file *file); +int ntfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + __u64 start, __u64 len); +extern const struct inode_operations ntfs_special_inode_operations; +extern const struct inode_operations ntfs_file_inode_operations; +extern const struct file_operations ntfs_file_operations; + +/* globals from frecord.c */ +void ni_remove_mi(struct ntfs_inode *ni, struct mft_inode *mi); +struct ATTR_STD_INFO *ni_std(struct ntfs_inode *ni); +struct ATTR_STD_INFO5 *ni_std5(struct ntfs_inode *ni); +void ni_clear(struct ntfs_inode *ni); +int ni_load_mi_ex(struct ntfs_inode *ni, CLST rno, struct mft_inode **mi); +int ni_load_mi(struct ntfs_inode *ni, struct ATTR_LIST_ENTRY *le, + struct mft_inode **mi); +struct ATTRIB *ni_find_attr(struct ntfs_inode *ni, struct ATTRIB *attr, + struct ATTR_LIST_ENTRY **entry_o, + enum ATTR_TYPE type, const __le16 *name, + u8 name_len, const CLST *vcn, + struct mft_inode **mi); +struct ATTRIB *ni_enum_attr_ex(struct ntfs_inode *ni, struct ATTRIB *attr, + struct ATTR_LIST_ENTRY **le, + struct mft_inode **mi); +struct ATTRIB *ni_load_attr(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, CLST vcn, + struct mft_inode **pmi); +int ni_load_all_mi(struct ntfs_inode *ni); +bool ni_add_subrecord(struct ntfs_inode *ni, CLST rno, struct mft_inode **mi); +int ni_remove_attr(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, size_t name_len, bool base_only, + const __le16 *id); +int ni_create_attr_list(struct ntfs_inode *ni); +int ni_expand_list(struct ntfs_inode *ni); +int ni_insert_nonresident(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, + const struct runs_tree *run, CLST svcn, CLST len, + __le16 flags, struct ATTRIB **new_attr, + struct mft_inode **mi); +int ni_insert_resident(struct ntfs_inode *ni, u32 data_size, + enum ATTR_TYPE type, const __le16 *name, u8 name_len, + struct ATTRIB **new_attr, struct mft_inode **mi); +int ni_remove_attr_le(struct ntfs_inode *ni, struct ATTRIB *attr, + struct ATTR_LIST_ENTRY *le); +int ni_delete_all(struct ntfs_inode *ni); +struct ATTR_FILE_NAME *ni_fname_name(struct ntfs_inode *ni, + const struct cpu_str *uni, + const struct MFT_REF *home, + struct ATTR_LIST_ENTRY **entry); +struct ATTR_FILE_NAME *ni_fname_type(struct ntfs_inode *ni, u8 name_type, + struct ATTR_LIST_ENTRY **entry); +int ni_new_attr_flags(struct ntfs_inode *ni, enum FILE_ATTRIBUTE new_fa); +enum REPARSE_SIGN ni_parse_reparse(struct ntfs_inode *ni, struct ATTRIB *attr, + void *buffer); +int ni_write_inode(struct inode *inode, int sync, const char *hint); +#define _ni_write_inode(i, w) ni_write_inode(i, w, __func__) +int ni_fiemap(struct ntfs_inode *ni, struct fiemap_extent_info *fieinfo, + __u64 vbo, __u64 len); +int ni_readpage_cmpr(struct ntfs_inode *ni, struct page *page); +int ni_decompress_file(struct ntfs_inode *ni); +int ni_read_frame(struct ntfs_inode *ni, u64 frame_vbo, struct page **pages, + u32 pages_per_frame); +int ni_write_frame(struct ntfs_inode *ni, struct page **pages, + u32 pages_per_frame); + +/* globals from fslog.c */ +int log_replay(struct ntfs_inode *ni, bool *initialized); + +/* globals from fsntfs.c */ +bool ntfs_fix_pre_write(struct NTFS_RECORD_HEADER *rhdr, size_t bytes); +int ntfs_fix_post_read(struct NTFS_RECORD_HEADER *rhdr, size_t bytes, + bool simple); +int ntfs_extend_init(struct ntfs_sb_info *sbi); +int ntfs_loadlog_and_replay(struct ntfs_inode *ni, struct ntfs_sb_info *sbi); +const struct ATTR_DEF_ENTRY *ntfs_query_def(struct ntfs_sb_info *sbi, + enum ATTR_TYPE Type); +int ntfs_look_for_free_space(struct ntfs_sb_info *sbi, CLST lcn, CLST len, + CLST *new_lcn, CLST *new_len, + enum ALLOCATE_OPT opt); +int ntfs_look_free_mft(struct ntfs_sb_info *sbi, CLST *rno, bool mft, + struct ntfs_inode *ni, struct mft_inode **mi); +void ntfs_mark_rec_free(struct ntfs_sb_info *sbi, CLST rno); +int ntfs_clear_mft_tail(struct ntfs_sb_info *sbi, size_t from, size_t to); +int ntfs_refresh_zone(struct ntfs_sb_info *sbi); +int ntfs_update_mftmirr(struct ntfs_sb_info *sbi, int wait); +enum NTFS_DIRTY_FLAGS { + NTFS_DIRTY_CLEAR = 0, + NTFS_DIRTY_DIRTY = 1, + NTFS_DIRTY_ERROR = 2, +}; +int ntfs_set_state(struct ntfs_sb_info *sbi, enum NTFS_DIRTY_FLAGS dirty); +int ntfs_sb_read(struct super_block *sb, u64 lbo, size_t bytes, void *buffer); +int ntfs_sb_write(struct super_block *sb, u64 lbo, size_t bytes, + const void *buffer, int wait); +int ntfs_sb_write_run(struct ntfs_sb_info *sbi, const struct runs_tree *run, + u64 vbo, const void *buf, size_t bytes); +struct buffer_head *ntfs_bread_run(struct ntfs_sb_info *sbi, + const struct runs_tree *run, u64 vbo); +int ntfs_read_run_nb(struct ntfs_sb_info *sbi, const struct runs_tree *run, + u64 vbo, void *buf, u32 bytes, struct ntfs_buffers *nb); +int ntfs_read_bh(struct ntfs_sb_info *sbi, const struct runs_tree *run, u64 vbo, + struct NTFS_RECORD_HEADER *rhdr, u32 bytes, + struct ntfs_buffers *nb); +int ntfs_get_bh(struct ntfs_sb_info *sbi, const struct runs_tree *run, u64 vbo, + u32 bytes, struct ntfs_buffers *nb); +int ntfs_write_bh(struct ntfs_sb_info *sbi, struct NTFS_RECORD_HEADER *rhdr, + struct ntfs_buffers *nb, int sync); +int ntfs_bio_pages(struct ntfs_sb_info *sbi, const struct runs_tree *run, + struct page **pages, u32 nr_pages, u64 vbo, u32 bytes, + u32 op); +int ntfs_bio_fill_1(struct ntfs_sb_info *sbi, const struct runs_tree *run); +int ntfs_vbo_to_lbo(struct ntfs_sb_info *sbi, const struct runs_tree *run, + u64 vbo, u64 *lbo, u64 *bytes); +struct ntfs_inode *ntfs_new_inode(struct ntfs_sb_info *sbi, CLST nRec, + bool dir); +extern const u8 s_default_security[0x50]; +bool is_sd_valid(const struct SECURITY_DESCRIPTOR_RELATIVE *sd, u32 len); +int ntfs_security_init(struct ntfs_sb_info *sbi); +int ntfs_get_security_by_id(struct ntfs_sb_info *sbi, __le32 security_id, + struct SECURITY_DESCRIPTOR_RELATIVE **sd, + size_t *size); +int ntfs_insert_security(struct ntfs_sb_info *sbi, + const struct SECURITY_DESCRIPTOR_RELATIVE *sd, + u32 size, __le32 *security_id, bool *inserted); +int ntfs_reparse_init(struct ntfs_sb_info *sbi); +int ntfs_objid_init(struct ntfs_sb_info *sbi); +int ntfs_objid_remove(struct ntfs_sb_info *sbi, struct GUID *guid); +int ntfs_insert_reparse(struct ntfs_sb_info *sbi, __le32 rtag, + const struct MFT_REF *ref); +int ntfs_remove_reparse(struct ntfs_sb_info *sbi, __le32 rtag, + const struct MFT_REF *ref); +void mark_as_free_ex(struct ntfs_sb_info *sbi, CLST lcn, CLST len, bool trim); +int run_deallocate(struct ntfs_sb_info *sbi, struct runs_tree *run, bool trim); + +/* globals from index.c */ +int indx_used_bit(struct ntfs_index *indx, struct ntfs_inode *ni, size_t *bit); +void fnd_clear(struct ntfs_fnd *fnd); +static inline struct ntfs_fnd *fnd_get(void) +{ + return ntfs_zalloc(sizeof(struct ntfs_fnd)); +} +static inline void fnd_put(struct ntfs_fnd *fnd) +{ + if (fnd) { + fnd_clear(fnd); + ntfs_free(fnd); + } +} +void indx_clear(struct ntfs_index *idx); +int indx_init(struct ntfs_index *indx, struct ntfs_sb_info *sbi, + const struct ATTRIB *attr, enum index_mutex_classed type); +struct INDEX_ROOT *indx_get_root(struct ntfs_index *indx, struct ntfs_inode *ni, + struct ATTRIB **attr, struct mft_inode **mi); +int indx_read(struct ntfs_index *idx, struct ntfs_inode *ni, CLST vbn, + struct indx_node **node); +int indx_find(struct ntfs_index *indx, struct ntfs_inode *dir, + const struct INDEX_ROOT *root, const void *Key, size_t KeyLen, + const void *param, int *diff, struct NTFS_DE **entry, + struct ntfs_fnd *fnd); +int indx_find_sort(struct ntfs_index *indx, struct ntfs_inode *ni, + const struct INDEX_ROOT *root, struct NTFS_DE **entry, + struct ntfs_fnd *fnd); +int indx_find_raw(struct ntfs_index *indx, struct ntfs_inode *ni, + const struct INDEX_ROOT *root, struct NTFS_DE **entry, + size_t *off, struct ntfs_fnd *fnd); +int indx_insert_entry(struct ntfs_index *indx, struct ntfs_inode *ni, + const struct NTFS_DE *new_de, const void *param, + struct ntfs_fnd *fnd); +int indx_delete_entry(struct ntfs_index *indx, struct ntfs_inode *ni, + const void *key, u32 key_len, const void *param); +int indx_update_dup(struct ntfs_inode *ni, struct ntfs_sb_info *sbi, + const struct ATTR_FILE_NAME *fname, + const struct NTFS_DUP_INFO *dup, int sync); + +/* globals from inode.c */ +struct inode *ntfs_iget5(struct super_block *sb, const struct MFT_REF *ref, + const struct cpu_str *name); +int ntfs_set_size(struct inode *inode, u64 new_size); +int reset_log_file(struct inode *inode); +int ntfs_get_block(struct inode *inode, sector_t vbn, + struct buffer_head *bh_result, int create); +int ntfs3_write_inode(struct inode *inode, struct writeback_control *wbc); +int ntfs_sync_inode(struct inode *inode); +int ntfs_flush_inodes(struct super_block *sb, struct inode *i1, + struct inode *i2); +int inode_write_data(struct inode *inode, const void *data, size_t bytes); +struct inode *ntfs_create_inode(struct user_namespace *mnt_userns, + struct inode *dir, struct dentry *dentry, + const struct cpu_str *uni, umode_t mode, + dev_t dev, const char *symname, u32 size, + int excl, struct ntfs_fnd *fnd); +int ntfs_link_inode(struct inode *inode, struct dentry *dentry); +int ntfs_unlink_inode(struct inode *dir, const struct dentry *dentry); +void ntfs_evict_inode(struct inode *inode); +extern const struct inode_operations ntfs_link_inode_operations; +extern const struct address_space_operations ntfs_aops; +extern const struct address_space_operations ntfs_aops_cmpr; + +/* globals from name_i.c*/ +int fill_name_de(struct ntfs_sb_info *sbi, void *buf, const struct qstr *name, + const struct cpu_str *uni); +struct dentry *ntfs3_get_parent(struct dentry *child); + +extern const struct inode_operations ntfs_dir_inode_operations; + +/* globals from record.c */ +int mi_get(struct ntfs_sb_info *sbi, CLST rno, struct mft_inode **mi); +void mi_put(struct mft_inode *mi); +int mi_init(struct mft_inode *mi, struct ntfs_sb_info *sbi, CLST rno); +int mi_read(struct mft_inode *mi, bool is_mft); +struct ATTRIB *mi_enum_attr(struct mft_inode *mi, struct ATTRIB *attr); +// TODO: id? +struct ATTRIB *mi_find_attr(struct mft_inode *mi, struct ATTRIB *attr, + enum ATTR_TYPE type, const __le16 *name, + size_t name_len, const __le16 *id); +static inline struct ATTRIB *rec_find_attr_le(struct mft_inode *rec, + struct ATTR_LIST_ENTRY *le) +{ + return mi_find_attr(rec, NULL, le->type, le_name(le), le->name_len, + &le->id); +} +int mi_write(struct mft_inode *mi, int wait); +int mi_format_new(struct mft_inode *mi, struct ntfs_sb_info *sbi, CLST rno, + __le16 flags, bool is_mft); +void mi_mark_free(struct mft_inode *mi); +struct ATTRIB *mi_insert_attr(struct mft_inode *mi, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, u32 asize, + u16 name_off); + +bool mi_remove_attr(struct mft_inode *mi, struct ATTRIB *attr); +bool mi_resize_attr(struct mft_inode *mi, struct ATTRIB *attr, int bytes); +int mi_pack_runs(struct mft_inode *mi, struct ATTRIB *attr, + struct runs_tree *run, CLST len); +static inline bool mi_is_ref(const struct mft_inode *mi, + const struct MFT_REF *ref) +{ + if (le32_to_cpu(ref->low) != mi->rno) + return false; + if (ref->seq != mi->mrec->seq) + return false; + +#ifdef CONFIG_NTFS3_64BIT_CLUSTER + return le16_to_cpu(ref->high) == (mi->rno >> 32); +#else + return !ref->high; +#endif +} + +static inline void mi_get_ref(const struct mft_inode *mi, struct MFT_REF *ref) +{ + ref->low = cpu_to_le32(mi->rno); +#ifdef CONFIG_NTFS3_64BIT_CLUSTER + ref->high = cpu_to_le16(mi->rno >> 32); +#else + ref->high = 0; +#endif + ref->seq = mi->mrec->seq; +} + +/* globals from run.c */ +bool run_lookup_entry(const struct runs_tree *run, CLST vcn, CLST *lcn, + CLST *len, size_t *index); +void run_truncate(struct runs_tree *run, CLST vcn); +void run_truncate_head(struct runs_tree *run, CLST vcn); +void run_truncate_around(struct runs_tree *run, CLST vcn); +bool run_lookup(const struct runs_tree *run, CLST vcn, size_t *Index); +bool run_add_entry(struct runs_tree *run, CLST vcn, CLST lcn, CLST len, + bool is_mft); +bool run_collapse_range(struct runs_tree *run, CLST vcn, CLST len); +bool run_get_entry(const struct runs_tree *run, size_t index, CLST *vcn, + CLST *lcn, CLST *len); +bool run_is_mapped_full(const struct runs_tree *run, CLST svcn, CLST evcn); + +int run_pack(const struct runs_tree *run, CLST svcn, CLST len, u8 *run_buf, + u32 run_buf_size, CLST *packed_vcns); +int run_unpack(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino, + CLST svcn, CLST evcn, CLST vcn, const u8 *run_buf, + u32 run_buf_size); + +#ifdef NTFS3_CHECK_FREE_CLST +int run_unpack_ex(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino, + CLST svcn, CLST evcn, CLST vcn, const u8 *run_buf, + u32 run_buf_size); +#else +#define run_unpack_ex run_unpack +#endif +int run_get_highest_vcn(CLST vcn, const u8 *run_buf, u64 *highest_vcn); + +/* globals from super.c */ +void *ntfs_set_shared(void *ptr, u32 bytes); +void *ntfs_put_shared(void *ptr); +void ntfs_unmap_meta(struct super_block *sb, CLST lcn, CLST len); +int ntfs_discard(struct ntfs_sb_info *sbi, CLST Lcn, CLST Len); + +/* globals from bitmap.c*/ +int __init ntfs3_init_bitmap(void); +void ntfs3_exit_bitmap(void); +void wnd_close(struct wnd_bitmap *wnd); +static inline size_t wnd_zeroes(const struct wnd_bitmap *wnd) +{ + return wnd->total_zeroes; +} +int wnd_init(struct wnd_bitmap *wnd, struct super_block *sb, size_t nbits); +int wnd_set_free(struct wnd_bitmap *wnd, size_t bit, size_t bits); +int wnd_set_used(struct wnd_bitmap *wnd, size_t bit, size_t bits); +bool wnd_is_free(struct wnd_bitmap *wnd, size_t bit, size_t bits); +bool wnd_is_used(struct wnd_bitmap *wnd, size_t bit, size_t bits); + +/* Possible values for 'flags' 'wnd_find' */ +#define BITMAP_FIND_MARK_AS_USED 0x01 +#define BITMAP_FIND_FULL 0x02 +size_t wnd_find(struct wnd_bitmap *wnd, size_t to_alloc, size_t hint, + size_t flags, size_t *allocated); +int wnd_extend(struct wnd_bitmap *wnd, size_t new_bits); +void wnd_zone_set(struct wnd_bitmap *wnd, size_t Lcn, size_t Len); +int ntfs_trim_fs(struct ntfs_sb_info *sbi, struct fstrim_range *range); + +/* globals from upcase.c */ +int ntfs_cmp_names(const __le16 *s1, size_t l1, const __le16 *s2, size_t l2, + const u16 *upcase, bool bothcase); +int ntfs_cmp_names_cpu(const struct cpu_str *uni1, const struct le_str *uni2, + const u16 *upcase, bool bothcase); + +/* globals from xattr.c */ +#ifdef CONFIG_NTFS3_FS_POSIX_ACL +struct posix_acl *ntfs_get_acl(struct inode *inode, int type); +int ntfs_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type); +int ntfs_init_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct inode *dir); +#else +#define ntfs_get_acl NULL +#define ntfs_set_acl NULL +#endif + +int ntfs_acl_chmod(struct user_namespace *mnt_userns, struct inode *inode); +int ntfs_permission(struct user_namespace *mnt_userns, struct inode *inode, + int mask); +ssize_t ntfs_listxattr(struct dentry *dentry, char *buffer, size_t size); +extern const struct xattr_handler *ntfs_xattr_handlers[]; + +/* globals from lznt.c */ +struct lznt *get_lznt_ctx(int level); +size_t compress_lznt(const void *uncompressed, size_t uncompressed_size, + void *compressed, size_t compressed_size, + struct lznt *ctx); +ssize_t decompress_lznt(const void *compressed, size_t compressed_size, + void *uncompressed, size_t uncompressed_size); + +static inline bool is_ntfs3(struct ntfs_sb_info *sbi) +{ + return sbi->volume.major_ver >= 3; +} + +/*(sb->s_flags & SB_ACTIVE)*/ +static inline bool is_mounted(struct ntfs_sb_info *sbi) +{ + return !!sbi->sb->s_root; +} + +static inline bool ntfs_is_meta_file(struct ntfs_sb_info *sbi, CLST rno) +{ + return rno < MFT_REC_FREE || rno == sbi->objid_no || + rno == sbi->quota_no || rno == sbi->reparse_no || + rno == sbi->usn_jrnl_no; +} + +static inline void ntfs_unmap_page(struct page *page) +{ + kunmap(page); + put_page(page); +} + +static inline struct page *ntfs_map_page(struct address_space *mapping, + unsigned long index) +{ + struct page *page = read_mapping_page(mapping, index, NULL); + + if (!IS_ERR(page)) { + kmap(page); + if (!PageError(page)) + return page; + ntfs_unmap_page(page); + return ERR_PTR(-EIO); + } + return page; +} + +static inline size_t wnd_zone_bit(const struct wnd_bitmap *wnd) +{ + return wnd->zone_bit; +} + +static inline size_t wnd_zone_len(const struct wnd_bitmap *wnd) +{ + return wnd->zone_end - wnd->zone_bit; +} + +static inline void run_init(struct runs_tree *run) +{ + run->runs = NULL; + run->count = 0; + run->allocated = 0; +} + +static inline struct runs_tree *run_alloc(void) +{ + return ntfs_zalloc(sizeof(struct runs_tree)); +} + +static inline void run_close(struct runs_tree *run) +{ + ntfs_vfree(run->runs); + memset(run, 0, sizeof(*run)); +} + +static inline void run_free(struct runs_tree *run) +{ + if (run) { + ntfs_vfree(run->runs); + ntfs_free(run); + } +} + +static inline bool run_is_empty(struct runs_tree *run) +{ + return !run->count; +} + +/* NTFS uses quad aligned bitmaps */ +static inline size_t bitmap_size(size_t bits) +{ + return QuadAlign((bits + 7) >> 3); +} + +#define _100ns2seconds 10000000 +#define SecondsToStartOf1970 0x00000002B6109100 + +#define NTFS_TIME_GRAN 100 + +/* + * kernel2nt + * + * converts in-memory kernel timestamp into nt time + */ +static inline __le64 kernel2nt(const struct timespec64 *ts) +{ + // 10^7 units of 100 nanoseconds one second + return cpu_to_le64(_100ns2seconds * + (ts->tv_sec + SecondsToStartOf1970) + + ts->tv_nsec / NTFS_TIME_GRAN); +} + +/* + * nt2kernel + * + * converts on-disk nt time into kernel timestamp + */ +static inline void nt2kernel(const __le64 tm, struct timespec64 *ts) +{ + u64 t = le64_to_cpu(tm) - _100ns2seconds * SecondsToStartOf1970; + + // WARNING: do_div changes its first argument(!) + ts->tv_nsec = do_div(t, _100ns2seconds) * 100; + ts->tv_sec = t; +} + +static inline struct ntfs_sb_info *ntfs_sb(struct super_block *sb) +{ + return sb->s_fs_info; +} + +/* Align up on cluster boundary */ +static inline u64 ntfs_up_cluster(const struct ntfs_sb_info *sbi, u64 size) +{ + return (size + sbi->cluster_mask) & sbi->cluster_mask_inv; +} + +/* Align up on cluster boundary */ +static inline u64 ntfs_up_block(const struct super_block *sb, u64 size) +{ + return (size + sb->s_blocksize - 1) & ~(u64)(sb->s_blocksize - 1); +} + +static inline CLST bytes_to_cluster(const struct ntfs_sb_info *sbi, u64 size) +{ + return (size + sbi->cluster_mask) >> sbi->cluster_bits; +} + +static inline u64 bytes_to_block(const struct super_block *sb, u64 size) +{ + return (size + sb->s_blocksize - 1) >> sb->s_blocksize_bits; +} + +static inline struct buffer_head *ntfs_bread(struct super_block *sb, + sector_t block) +{ + struct buffer_head *bh = sb_bread(sb, block); + + if (bh) + return bh; + + ntfs_err(sb, "failed to read volume at offset 0x%llx", + (u64)block << sb->s_blocksize_bits); + return NULL; +} + +static inline bool is_power_of2(size_t v) +{ + return v && !(v & (v - 1)); +} + +static inline struct ntfs_inode *ntfs_i(struct inode *inode) +{ + return container_of(inode, struct ntfs_inode, vfs_inode); +} + +static inline bool is_compressed(const struct ntfs_inode *ni) +{ + return (ni->std_fa & FILE_ATTRIBUTE_COMPRESSED) || + (ni->ni_flags & NI_FLAG_COMPRESSED_MASK); +} + +static inline int ni_ext_compress_bits(const struct ntfs_inode *ni) +{ + return 0xb + (ni->ni_flags & NI_FLAG_COMPRESSED_MASK); +} + +/* bits - 0xc, 0xd, 0xe, 0xf, 0x10 */ +static inline void ni_set_ext_compress_bits(struct ntfs_inode *ni, u8 bits) +{ + ni->ni_flags |= (bits - 0xb) & NI_FLAG_COMPRESSED_MASK; +} + +static inline bool is_dedup(const struct ntfs_inode *ni) +{ + return ni->ni_flags & NI_FLAG_DEDUPLICATED; +} + +static inline bool is_encrypted(const struct ntfs_inode *ni) +{ + return ni->std_fa & FILE_ATTRIBUTE_ENCRYPTED; +} + +static inline bool is_sparsed(const struct ntfs_inode *ni) +{ + return ni->std_fa & FILE_ATTRIBUTE_SPARSE_FILE; +} + +static inline int is_resident(struct ntfs_inode *ni) +{ + return ni->ni_flags & NI_FLAG_RESIDENT; +} + +static inline void le16_sub_cpu(__le16 *var, u16 val) +{ + *var = cpu_to_le16(le16_to_cpu(*var) - val); +} + +static inline void le32_sub_cpu(__le32 *var, u32 val) +{ + *var = cpu_to_le32(le32_to_cpu(*var) - val); +} + +static inline void nb_put(struct ntfs_buffers *nb) +{ + u32 i, nbufs = nb->nbufs; + + if (!nbufs) + return; + + for (i = 0; i < nbufs; i++) + put_bh(nb->bh[i]); + nb->nbufs = 0; +} + +static inline void put_indx_node(struct indx_node *in) +{ + if (!in) + return; + + ntfs_free(in->index); + nb_put(&in->nb); + ntfs_free(in); +} + +static inline void mi_clear(struct mft_inode *mi) +{ + nb_put(&mi->nb); + ntfs_free(mi->mrec); + mi->mrec = NULL; +} + +static inline void ni_lock(struct ntfs_inode *ni) +{ + mutex_lock_nested(&ni->ni_lock, NTFS_INODE_MUTEX_NORMAL); +} + +static inline void ni_lock_dir(struct ntfs_inode *ni) +{ + mutex_lock_nested(&ni->ni_lock, NTFS_INODE_MUTEX_PARENT); +} + +static inline void ni_unlock(struct ntfs_inode *ni) +{ + mutex_unlock(&ni->ni_lock); +} + +static inline int ni_trylock(struct ntfs_inode *ni) +{ + return mutex_trylock(&ni->ni_lock); +} + +static inline int attr_load_runs_attr(struct ntfs_inode *ni, + struct ATTRIB *attr, + struct runs_tree *run, CLST vcn) +{ + return attr_load_runs_vcn(ni, attr->type, attr_name(attr), + attr->name_len, run, vcn); +} + +static inline void le64_sub_cpu(__le64 *var, u64 val) +{ + *var = cpu_to_le64(le64_to_cpu(*var) - val); +} diff --git a/fs/ntfs3/record.c b/fs/ntfs3/record.c new file mode 100644 index 000000000000..0d4a6251bddc --- /dev/null +++ b/fs/ntfs3/record.c @@ -0,0 +1,609 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + */ + +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +static inline int compare_attr(const struct ATTRIB *left, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, + const u16 *upcase) +{ + /* First, compare the type codes: */ + int diff = le32_to_cpu(left->type) - le32_to_cpu(type); + + if (diff) + return diff; + + /* + * They have the same type code, so we have to compare the names. + */ + return ntfs_cmp_names(attr_name(left), left->name_len, name, name_len, + upcase, true); +} + +/* + * mi_new_attt_id + * + * returns unused attribute id that is less than mrec->next_attr_id + */ +static __le16 mi_new_attt_id(struct mft_inode *mi) +{ + u16 free_id, max_id, t16; + struct MFT_REC *rec = mi->mrec; + struct ATTRIB *attr; + __le16 id; + + id = rec->next_attr_id; + free_id = le16_to_cpu(id); + if (free_id < 0x7FFF) { + rec->next_attr_id = cpu_to_le16(free_id + 1); + return id; + } + + /* One record can store up to 1024/24 ~= 42 attributes */ + free_id = 0; + max_id = 0; + + attr = NULL; + + for (;;) { + attr = mi_enum_attr(mi, attr); + if (!attr) { + rec->next_attr_id = cpu_to_le16(max_id + 1); + mi->dirty = true; + return cpu_to_le16(free_id); + } + + t16 = le16_to_cpu(attr->id); + if (t16 == free_id) { + free_id += 1; + attr = NULL; + } else if (max_id < t16) + max_id = t16; + } +} + +int mi_get(struct ntfs_sb_info *sbi, CLST rno, struct mft_inode **mi) +{ + int err; + struct mft_inode *m = ntfs_zalloc(sizeof(struct mft_inode)); + + if (!m) + return -ENOMEM; + + err = mi_init(m, sbi, rno); + if (err) { + ntfs_free(m); + return err; + } + + err = mi_read(m, false); + if (err) { + mi_put(m); + return err; + } + + *mi = m; + return 0; +} + +void mi_put(struct mft_inode *mi) +{ + mi_clear(mi); + ntfs_free(mi); +} + +int mi_init(struct mft_inode *mi, struct ntfs_sb_info *sbi, CLST rno) +{ + mi->sbi = sbi; + mi->rno = rno; + mi->mrec = ntfs_malloc(sbi->record_size); + if (!mi->mrec) + return -ENOMEM; + + return 0; +} + +/* + * mi_read + * + * reads MFT data + */ +int mi_read(struct mft_inode *mi, bool is_mft) +{ + int err; + struct MFT_REC *rec = mi->mrec; + struct ntfs_sb_info *sbi = mi->sbi; + u32 bpr = sbi->record_size; + u64 vbo = (u64)mi->rno << sbi->record_bits; + struct ntfs_inode *mft_ni = sbi->mft.ni; + struct runs_tree *run = mft_ni ? &mft_ni->file.run : NULL; + struct rw_semaphore *rw_lock = NULL; + + if (is_mounted(sbi)) { + if (!is_mft) { + rw_lock = &mft_ni->file.run_lock; + down_read(rw_lock); + } + } + + err = ntfs_read_bh(sbi, run, vbo, &rec->rhdr, bpr, &mi->nb); + if (rw_lock) + up_read(rw_lock); + if (!err) + goto ok; + + if (err == -E_NTFS_FIXUP) { + mi->dirty = true; + goto ok; + } + + if (err != -ENOENT) + goto out; + + if (rw_lock) { + ni_lock(mft_ni); + down_write(rw_lock); + } + err = attr_load_runs_vcn(mft_ni, ATTR_DATA, NULL, 0, &mft_ni->file.run, + vbo >> sbi->cluster_bits); + if (rw_lock) { + up_write(rw_lock); + ni_unlock(mft_ni); + } + if (err) + goto out; + + if (rw_lock) + down_read(rw_lock); + err = ntfs_read_bh(sbi, run, vbo, &rec->rhdr, bpr, &mi->nb); + if (rw_lock) + up_read(rw_lock); + + if (err == -E_NTFS_FIXUP) { + mi->dirty = true; + goto ok; + } + if (err) + goto out; + +ok: + /* check field 'total' only here */ + if (le32_to_cpu(rec->total) != bpr) { + err = -EINVAL; + goto out; + } + + return 0; + +out: + return err; +} + +struct ATTRIB *mi_enum_attr(struct mft_inode *mi, struct ATTRIB *attr) +{ + const struct MFT_REC *rec = mi->mrec; + u32 used = le32_to_cpu(rec->used); + u32 t32, off, asize; + u16 t16; + + if (!attr) { + u32 total = le32_to_cpu(rec->total); + + off = le16_to_cpu(rec->attr_off); + + if (used > total) + return NULL; + + if (off >= used || off < MFTRECORD_FIXUP_OFFSET_1 || + !IsDwordAligned(off)) { + return NULL; + } + + /* Skip non-resident records */ + if (!is_rec_inuse(rec)) + return NULL; + + attr = Add2Ptr(rec, off); + } else { + /* Check if input attr inside record */ + off = PtrOffset(rec, attr); + if (off >= used) + return NULL; + + asize = le32_to_cpu(attr->size); + if (asize < SIZEOF_RESIDENT) { + /* Impossible 'cause we should not return such attribute */ + return NULL; + } + + attr = Add2Ptr(attr, asize); + off += asize; + } + + asize = le32_to_cpu(attr->size); + + /* Can we use the first field (attr->type) */ + if (off + 8 > used) { + static_assert(QuadAlign(sizeof(enum ATTR_TYPE)) == 8); + return NULL; + } + + if (attr->type == ATTR_END) { + /* end of enumeration */ + return NULL; + } + + /* 0x100 is last known attribute for now*/ + t32 = le32_to_cpu(attr->type); + if ((t32 & 0xf) || (t32 > 0x100)) + return NULL; + + /* Check boundary */ + if (off + asize > used) + return NULL; + + /* Check size of attribute */ + if (!attr->non_res) { + if (asize < SIZEOF_RESIDENT) + return NULL; + + t16 = le16_to_cpu(attr->res.data_off); + + if (t16 > asize) + return NULL; + + t32 = le32_to_cpu(attr->res.data_size); + if (t16 + t32 > asize) + return NULL; + + return attr; + } + + /* Check some nonresident fields */ + if (attr->name_len && + le16_to_cpu(attr->name_off) + sizeof(short) * attr->name_len > + le16_to_cpu(attr->nres.run_off)) { + return NULL; + } + + if (attr->nres.svcn || !is_attr_ext(attr)) { + if (asize + 8 < SIZEOF_NONRESIDENT) + return NULL; + + if (attr->nres.c_unit) + return NULL; + } else if (asize + 8 < SIZEOF_NONRESIDENT_EX) + return NULL; + + return attr; +} + +/* + * mi_find_attr + * + * finds the attribute by type and name and id + */ +struct ATTRIB *mi_find_attr(struct mft_inode *mi, struct ATTRIB *attr, + enum ATTR_TYPE type, const __le16 *name, + size_t name_len, const __le16 *id) +{ + u32 type_in = le32_to_cpu(type); + u32 atype; + +next_attr: + attr = mi_enum_attr(mi, attr); + if (!attr) + return NULL; + + atype = le32_to_cpu(attr->type); + if (atype > type_in) + return NULL; + + if (atype < type_in) + goto next_attr; + + if (attr->name_len != name_len) + goto next_attr; + + if (name_len && memcmp(attr_name(attr), name, name_len * sizeof(short))) + goto next_attr; + + if (id && *id != attr->id) + goto next_attr; + + return attr; +} + +int mi_write(struct mft_inode *mi, int wait) +{ + struct MFT_REC *rec; + int err; + struct ntfs_sb_info *sbi; + + if (!mi->dirty) + return 0; + + sbi = mi->sbi; + rec = mi->mrec; + + err = ntfs_write_bh(sbi, &rec->rhdr, &mi->nb, wait); + if (err) + return err; + + if (mi->rno < sbi->mft.recs_mirr) + sbi->flags |= NTFS_FLAGS_MFTMIRR; + + mi->dirty = false; + + return 0; +} + +int mi_format_new(struct mft_inode *mi, struct ntfs_sb_info *sbi, CLST rno, + __le16 flags, bool is_mft) +{ + int err; + u16 seq = 1; + struct MFT_REC *rec; + u64 vbo = (u64)rno << sbi->record_bits; + + err = mi_init(mi, sbi, rno); + if (err) + return err; + + rec = mi->mrec; + + if (rno == MFT_REC_MFT) { + ; + } else if (rno < MFT_REC_FREE) { + seq = rno; + } else if (rno >= sbi->mft.used) { + ; + } else if (mi_read(mi, is_mft)) { + ; + } else if (rec->rhdr.sign == NTFS_FILE_SIGNATURE) { + /* Record is reused. Update its sequence number */ + seq = le16_to_cpu(rec->seq) + 1; + if (!seq) + seq = 1; + } + + memcpy(rec, sbi->new_rec, sbi->record_size); + + rec->seq = cpu_to_le16(seq); + rec->flags = RECORD_FLAG_IN_USE | flags; + + mi->dirty = true; + + if (!mi->nb.nbufs) { + struct ntfs_inode *ni = sbi->mft.ni; + bool lock = false; + + if (is_mounted(sbi) && !is_mft) { + down_read(&ni->file.run_lock); + lock = true; + } + + err = ntfs_get_bh(sbi, &ni->file.run, vbo, sbi->record_size, + &mi->nb); + if (lock) + up_read(&ni->file.run_lock); + } + + return err; +} + +/* + * mi_mark_free + * + * marks record as unused and marks it as free in bitmap + */ +void mi_mark_free(struct mft_inode *mi) +{ + CLST rno = mi->rno; + struct ntfs_sb_info *sbi = mi->sbi; + + if (rno >= MFT_REC_RESERVED && rno < MFT_REC_FREE) { + ntfs_clear_mft_tail(sbi, rno, rno + 1); + mi->dirty = false; + return; + } + + if (mi->mrec) { + clear_rec_inuse(mi->mrec); + mi->dirty = true; + mi_write(mi, 0); + } + ntfs_mark_rec_free(sbi, rno); +} + +/* + * mi_insert_attr + * + * reserves space for new attribute + * returns not full constructed attribute or NULL if not possible to create + */ +struct ATTRIB *mi_insert_attr(struct mft_inode *mi, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, u32 asize, + u16 name_off) +{ + size_t tail; + struct ATTRIB *attr; + __le16 id; + struct MFT_REC *rec = mi->mrec; + struct ntfs_sb_info *sbi = mi->sbi; + u32 used = le32_to_cpu(rec->used); + const u16 *upcase = sbi->upcase; + int diff; + + /* Can we insert mi attribute? */ + if (used + asize > mi->sbi->record_size) + return NULL; + + /* + * Scan through the list of attributes to find the point + * at which we should insert it. + */ + attr = NULL; + while ((attr = mi_enum_attr(mi, attr))) { + diff = compare_attr(attr, type, name, name_len, upcase); + if (diff > 0) + break; + if (diff < 0) + continue; + + if (!is_attr_indexed(attr)) + return NULL; + break; + } + + if (!attr) { + tail = 8; /* not used, just to suppress warning */ + attr = Add2Ptr(rec, used - 8); + } else { + tail = used - PtrOffset(rec, attr); + } + + id = mi_new_attt_id(mi); + + memmove(Add2Ptr(attr, asize), attr, tail); + memset(attr, 0, asize); + + attr->type = type; + attr->size = cpu_to_le32(asize); + attr->name_len = name_len; + attr->name_off = cpu_to_le16(name_off); + attr->id = id; + + memmove(Add2Ptr(attr, name_off), name, name_len * sizeof(short)); + rec->used = cpu_to_le32(used + asize); + + mi->dirty = true; + + return attr; +} + +/* + * mi_remove_attr + * + * removes the attribute from record + * NOTE: The source attr will point to next attribute + */ +bool mi_remove_attr(struct mft_inode *mi, struct ATTRIB *attr) +{ + struct MFT_REC *rec = mi->mrec; + u32 aoff = PtrOffset(rec, attr); + u32 used = le32_to_cpu(rec->used); + u32 asize = le32_to_cpu(attr->size); + + if (aoff + asize > used) + return false; + + used -= asize; + memmove(attr, Add2Ptr(attr, asize), used - aoff); + rec->used = cpu_to_le32(used); + mi->dirty = true; + + return true; +} + +/* bytes = "new attribute size" - "old attribute size" */ +bool mi_resize_attr(struct mft_inode *mi, struct ATTRIB *attr, int bytes) +{ + struct MFT_REC *rec = mi->mrec; + u32 aoff = PtrOffset(rec, attr); + u32 total, used = le32_to_cpu(rec->used); + u32 nsize, asize = le32_to_cpu(attr->size); + u32 rsize = le32_to_cpu(attr->res.data_size); + int tail = (int)(used - aoff - asize); + int dsize; + char *next; + + if (tail < 0 || aoff >= used) + return false; + + if (!bytes) + return true; + + total = le32_to_cpu(rec->total); + next = Add2Ptr(attr, asize); + + if (bytes > 0) { + dsize = QuadAlign(bytes); + if (used + dsize > total) + return false; + nsize = asize + dsize; + // move tail + memmove(next + dsize, next, tail); + memset(next, 0, dsize); + used += dsize; + rsize += dsize; + } else { + dsize = QuadAlign(-bytes); + if (dsize > asize) + return false; + nsize = asize - dsize; + memmove(next - dsize, next, tail); + used -= dsize; + rsize -= dsize; + } + + rec->used = cpu_to_le32(used); + attr->size = cpu_to_le32(nsize); + if (!attr->non_res) + attr->res.data_size = cpu_to_le32(rsize); + mi->dirty = true; + + return true; +} + +int mi_pack_runs(struct mft_inode *mi, struct ATTRIB *attr, + struct runs_tree *run, CLST len) +{ + int err = 0; + struct ntfs_sb_info *sbi = mi->sbi; + u32 new_run_size; + CLST plen; + struct MFT_REC *rec = mi->mrec; + CLST svcn = le64_to_cpu(attr->nres.svcn); + u32 used = le32_to_cpu(rec->used); + u32 aoff = PtrOffset(rec, attr); + u32 asize = le32_to_cpu(attr->size); + char *next = Add2Ptr(attr, asize); + u16 run_off = le16_to_cpu(attr->nres.run_off); + u32 run_size = asize - run_off; + u32 tail = used - aoff - asize; + u32 dsize = sbi->record_size - used; + + /* Make a maximum gap in current record */ + memmove(next + dsize, next, tail); + + /* Pack as much as possible */ + err = run_pack(run, svcn, len, Add2Ptr(attr, run_off), run_size + dsize, + &plen); + if (err < 0) { + memmove(next, next + dsize, tail); + return err; + } + + new_run_size = QuadAlign(err); + + memmove(next + new_run_size - run_size, next + dsize, tail); + + attr->size = cpu_to_le32(asize + new_run_size - run_size); + attr->nres.evcn = cpu_to_le64(svcn + plen - 1); + rec->used = cpu_to_le32(used + new_run_size - run_size); + mi->dirty = true; + + return 0; +} diff --git a/fs/ntfs3/run.c b/fs/ntfs3/run.c new file mode 100644 index 000000000000..5cdf6efe67e0 --- /dev/null +++ b/fs/ntfs3/run.c @@ -0,0 +1,1111 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + * TODO: try to use extents tree (instead of array) + */ + +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +/* runs_tree is a continues memory. Try to avoid big size */ +#define NTFS3_RUN_MAX_BYTES 0x10000 + +struct ntfs_run { + CLST vcn; /* virtual cluster number */ + CLST len; /* length in clusters */ + CLST lcn; /* logical cluster number */ +}; + +/* + * run_lookup + * + * Lookup the index of a MCB entry that is first <= vcn. + * case of success it will return non-zero value and set + * 'index' parameter to index of entry been found. + * case of entry missing from list 'index' will be set to + * point to insertion position for the entry question. + */ +bool run_lookup(const struct runs_tree *run, CLST vcn, size_t *index) +{ + size_t min_idx, max_idx, mid_idx; + struct ntfs_run *r; + + if (!run->count) { + *index = 0; + return false; + } + + min_idx = 0; + max_idx = run->count - 1; + + /* Check boundary cases specially, 'cause they cover the often requests */ + r = run->runs; + if (vcn < r->vcn) { + *index = 0; + return false; + } + + if (vcn < r->vcn + r->len) { + *index = 0; + return true; + } + + r += max_idx; + if (vcn >= r->vcn + r->len) { + *index = run->count; + return false; + } + + if (vcn >= r->vcn) { + *index = max_idx; + return true; + } + + do { + mid_idx = min_idx + ((max_idx - min_idx) >> 1); + r = run->runs + mid_idx; + + if (vcn < r->vcn) { + max_idx = mid_idx - 1; + if (!mid_idx) + break; + } else if (vcn >= r->vcn + r->len) { + min_idx = mid_idx + 1; + } else { + *index = mid_idx; + return true; + } + } while (min_idx <= max_idx); + + *index = max_idx + 1; + return false; +} + +/* + * run_consolidate + * + * consolidate runs starting from a given one. + */ +static void run_consolidate(struct runs_tree *run, size_t index) +{ + size_t i; + struct ntfs_run *r = run->runs + index; + + while (index + 1 < run->count) { + /* + * I should merge current run with next + * if start of the next run lies inside one being tested. + */ + struct ntfs_run *n = r + 1; + CLST end = r->vcn + r->len; + CLST dl; + + /* Stop if runs are not aligned one to another. */ + if (n->vcn > end) + break; + + dl = end - n->vcn; + + /* + * If range at index overlaps with next one + * then I will either adjust it's start position + * or (if completely matches) dust remove one from the list. + */ + if (dl > 0) { + if (n->len <= dl) + goto remove_next_range; + + n->len -= dl; + n->vcn += dl; + if (n->lcn != SPARSE_LCN) + n->lcn += dl; + dl = 0; + } + + /* + * Stop if sparse mode does not match + * both current and next runs. + */ + if ((n->lcn == SPARSE_LCN) != (r->lcn == SPARSE_LCN)) { + index += 1; + r = n; + continue; + } + + /* + * Check if volume block + * of a next run lcn does not match + * last volume block of the current run. + */ + if (n->lcn != SPARSE_LCN && n->lcn != r->lcn + r->len) + break; + + /* + * Next and current are siblings. + * Eat/join. + */ + r->len += n->len - dl; + +remove_next_range: + i = run->count - (index + 1); + if (i > 1) + memmove(n, n + 1, sizeof(*n) * (i - 1)); + + run->count -= 1; + } +} + +/* returns true if range [svcn - evcn] is mapped*/ +bool run_is_mapped_full(const struct runs_tree *run, CLST svcn, CLST evcn) +{ + size_t i; + const struct ntfs_run *r, *end; + CLST next_vcn; + + if (!run_lookup(run, svcn, &i)) + return false; + + end = run->runs + run->count; + r = run->runs + i; + + for (;;) { + next_vcn = r->vcn + r->len; + if (next_vcn > evcn) + return true; + + if (++r >= end) + return false; + + if (r->vcn != next_vcn) + return false; + } +} + +bool run_lookup_entry(const struct runs_tree *run, CLST vcn, CLST *lcn, + CLST *len, size_t *index) +{ + size_t idx; + CLST gap; + struct ntfs_run *r; + + /* Fail immediately if nrun was not touched yet. */ + if (!run->runs) + return false; + + if (!run_lookup(run, vcn, &idx)) + return false; + + r = run->runs + idx; + + if (vcn >= r->vcn + r->len) + return false; + + gap = vcn - r->vcn; + if (r->len <= gap) + return false; + + *lcn = r->lcn == SPARSE_LCN ? SPARSE_LCN : (r->lcn + gap); + + if (len) + *len = r->len - gap; + if (index) + *index = idx; + + return true; +} + +/* + * run_truncate_head + * + * decommit the range before vcn + */ +void run_truncate_head(struct runs_tree *run, CLST vcn) +{ + size_t index; + struct ntfs_run *r; + + if (run_lookup(run, vcn, &index)) { + r = run->runs + index; + + if (vcn > r->vcn) { + CLST dlen = vcn - r->vcn; + + r->vcn = vcn; + r->len -= dlen; + if (r->lcn != SPARSE_LCN) + r->lcn += dlen; + } + + if (!index) + return; + } + r = run->runs; + memmove(r, r + index, sizeof(*r) * (run->count - index)); + + run->count -= index; + + if (!run->count) { + ntfs_vfree(run->runs); + run->runs = NULL; + run->allocated = 0; + } +} + +/* + * run_truncate + * + * decommit the range after vcn + */ +void run_truncate(struct runs_tree *run, CLST vcn) +{ + size_t index; + + /* + * If I hit the range then + * I have to truncate one. + * If range to be truncated is becoming empty + * then it will entirely be removed. + */ + if (run_lookup(run, vcn, &index)) { + struct ntfs_run *r = run->runs + index; + + r->len = vcn - r->vcn; + + if (r->len > 0) + index += 1; + } + + /* + * At this point 'index' is set to + * position that should be thrown away (including index itself) + * Simple one - just set the limit. + */ + run->count = index; + + /* Do not reallocate array 'runs'. Only free if possible */ + if (!index) { + ntfs_vfree(run->runs); + run->runs = NULL; + run->allocated = 0; + } +} + +/* trim head and tail if necessary*/ +void run_truncate_around(struct runs_tree *run, CLST vcn) +{ + run_truncate_head(run, vcn); + + if (run->count >= NTFS3_RUN_MAX_BYTES / sizeof(struct ntfs_run) / 2) + run_truncate(run, (run->runs + (run->count >> 1))->vcn); +} + +/* + * run_add_entry + * + * sets location to known state. + * run to be added may overlap with existing location. + * returns false if of memory + */ +bool run_add_entry(struct runs_tree *run, CLST vcn, CLST lcn, CLST len, + bool is_mft) +{ + size_t used, index; + struct ntfs_run *r; + bool inrange; + CLST tail_vcn = 0, tail_len = 0, tail_lcn = 0; + bool should_add_tail = false; + + /* + * Lookup the insertion point. + * + * Execute bsearch for the entry containing + * start position question. + */ + inrange = run_lookup(run, vcn, &index); + + /* + * Shortcut here would be case of + * range not been found but one been added + * continues previous run. + * this case I can directly make use of + * existing range as my start point. + */ + if (!inrange && index > 0) { + struct ntfs_run *t = run->runs + index - 1; + + if (t->vcn + t->len == vcn && + (t->lcn == SPARSE_LCN) == (lcn == SPARSE_LCN) && + (lcn == SPARSE_LCN || lcn == t->lcn + t->len)) { + inrange = true; + index -= 1; + } + } + + /* + * At this point 'index' either points to the range + * containing start position or to the insertion position + * for a new range. + * So first let's check if range I'm probing is here already. + */ + if (!inrange) { +requires_new_range: + /* + * Range was not found. + * Insert at position 'index' + */ + used = run->count * sizeof(struct ntfs_run); + + /* + * Check allocated space. + * If one is not enough to get one more entry + * then it will be reallocated + */ + if (run->allocated < used + sizeof(struct ntfs_run)) { + size_t bytes; + struct ntfs_run *new_ptr; + + /* Use power of 2 for 'bytes'*/ + if (!used) { + bytes = 64; + } else if (used <= 16 * PAGE_SIZE) { + if (is_power_of2(run->allocated)) + bytes = run->allocated << 1; + else + bytes = (size_t)1 + << (2 + blksize_bits(used)); + } else { + bytes = run->allocated + (16 * PAGE_SIZE); + } + + WARN_ON(!is_mft && bytes > NTFS3_RUN_MAX_BYTES); + + new_ptr = ntfs_vmalloc(bytes); + + if (!new_ptr) + return false; + + r = new_ptr + index; + memcpy(new_ptr, run->runs, + index * sizeof(struct ntfs_run)); + memcpy(r + 1, run->runs + index, + sizeof(struct ntfs_run) * (run->count - index)); + + ntfs_vfree(run->runs); + run->runs = new_ptr; + run->allocated = bytes; + + } else { + size_t i = run->count - index; + + r = run->runs + index; + + /* memmove appears to be a bottle neck here... */ + if (i > 0) + memmove(r + 1, r, sizeof(struct ntfs_run) * i); + } + + r->vcn = vcn; + r->lcn = lcn; + r->len = len; + run->count += 1; + } else { + r = run->runs + index; + + /* + * If one of ranges was not allocated + * then I have to split location I just matched. + * and insert current one + * a common case this requires tail to be reinserted + * a recursive call. + */ + if (((lcn == SPARSE_LCN) != (r->lcn == SPARSE_LCN)) || + (lcn != SPARSE_LCN && lcn != r->lcn + (vcn - r->vcn))) { + CLST to_eat = vcn - r->vcn; + CLST Tovcn = to_eat + len; + + should_add_tail = Tovcn < r->len; + + if (should_add_tail) { + tail_lcn = r->lcn == SPARSE_LCN + ? SPARSE_LCN + : (r->lcn + Tovcn); + tail_vcn = r->vcn + Tovcn; + tail_len = r->len - Tovcn; + } + + if (to_eat > 0) { + r->len = to_eat; + inrange = false; + index += 1; + goto requires_new_range; + } + + /* lcn should match one I'm going to add. */ + r->lcn = lcn; + } + + /* + * If existing range fits then I'm done. + * Otherwise extend found one and fall back to range jocode. + */ + if (r->vcn + r->len < vcn + len) + r->len += len - ((r->vcn + r->len) - vcn); + } + + /* + * And normalize it starting from insertion point. + * It's possible that no insertion needed case if + * start point lies within the range of an entry + * that 'index' points to. + */ + if (inrange && index > 0) + index -= 1; + run_consolidate(run, index); + run_consolidate(run, index + 1); + + /* + * a special case + * I have to add extra range a tail. + */ + if (should_add_tail && + !run_add_entry(run, tail_vcn, tail_lcn, tail_len, is_mft)) + return false; + + return true; +} + +/*helper for attr_collapse_range, which is helper for fallocate(collapse_range)*/ +bool run_collapse_range(struct runs_tree *run, CLST vcn, CLST len) +{ + size_t index, eat; + struct ntfs_run *r, *e, *eat_start, *eat_end; + CLST end; + + if (WARN_ON(!run_lookup(run, vcn, &index))) + return true; /* should never be here */ + + e = run->runs + run->count; + r = run->runs + index; + end = vcn + len; + + if (vcn > r->vcn) { + if (r->vcn + r->len <= end) { + /* collapse tail of run */ + r->len = vcn - r->vcn; + } else if (r->lcn == SPARSE_LCN) { + /* collapse a middle part of sparsed run */ + r->len -= len; + } else { + /* collapse a middle part of normal run, split */ + if (!run_add_entry(run, vcn, SPARSE_LCN, len, false)) + return false; + return run_collapse_range(run, vcn, len); + } + + r += 1; + } + + eat_start = r; + eat_end = r; + + for (; r < e; r++) { + CLST d; + + if (r->vcn >= end) { + r->vcn -= len; + continue; + } + + if (r->vcn + r->len <= end) { + /* eat this run */ + eat_end = r + 1; + continue; + } + + d = end - r->vcn; + if (r->lcn != SPARSE_LCN) + r->lcn += d; + r->len -= d; + r->vcn -= len - d; + } + + eat = eat_end - eat_start; + memmove(eat_start, eat_end, (e - eat_end) * sizeof(*r)); + run->count -= eat; + + return true; +} + +/* + * run_get_entry + * + * returns index-th mapped region + */ +bool run_get_entry(const struct runs_tree *run, size_t index, CLST *vcn, + CLST *lcn, CLST *len) +{ + const struct ntfs_run *r; + + if (index >= run->count) + return false; + + r = run->runs + index; + + if (!r->len) + return false; + + if (vcn) + *vcn = r->vcn; + if (lcn) + *lcn = r->lcn; + if (len) + *len = r->len; + return true; +} + +/* + * run_packed_size + * + * calculates the size of packed int64 + */ +#ifdef __BIG_ENDIAN +static inline int run_packed_size(const s64 n) +{ + const u8 *p = (const u8 *)&n + sizeof(n) - 1; + + if (n >= 0) { + if (p[-7] || p[-6] || p[-5] || p[-4]) + p -= 4; + if (p[-3] || p[-2]) + p -= 2; + if (p[-1]) + p -= 1; + if (p[0] & 0x80) + p -= 1; + } else { + if (p[-7] != 0xff || p[-6] != 0xff || p[-5] != 0xff || + p[-4] != 0xff) + p -= 4; + if (p[-3] != 0xff || p[-2] != 0xff) + p -= 2; + if (p[-1] != 0xff) + p -= 1; + if (!(p[0] & 0x80)) + p -= 1; + } + return (const u8 *)&n + sizeof(n) - p; +} + +/* full trusted function. It does not check 'size' for errors */ +static inline void run_pack_s64(u8 *run_buf, u8 size, s64 v) +{ + const u8 *p = (u8 *)&v; + + switch (size) { + case 8: + run_buf[7] = p[0]; + fallthrough; + case 7: + run_buf[6] = p[1]; + fallthrough; + case 6: + run_buf[5] = p[2]; + fallthrough; + case 5: + run_buf[4] = p[3]; + fallthrough; + case 4: + run_buf[3] = p[4]; + fallthrough; + case 3: + run_buf[2] = p[5]; + fallthrough; + case 2: + run_buf[1] = p[6]; + fallthrough; + case 1: + run_buf[0] = p[7]; + } +} + +/* full trusted function. It does not check 'size' for errors */ +static inline s64 run_unpack_s64(const u8 *run_buf, u8 size, s64 v) +{ + u8 *p = (u8 *)&v; + + switch (size) { + case 8: + p[0] = run_buf[7]; + fallthrough; + case 7: + p[1] = run_buf[6]; + fallthrough; + case 6: + p[2] = run_buf[5]; + fallthrough; + case 5: + p[3] = run_buf[4]; + fallthrough; + case 4: + p[4] = run_buf[3]; + fallthrough; + case 3: + p[5] = run_buf[2]; + fallthrough; + case 2: + p[6] = run_buf[1]; + fallthrough; + case 1: + p[7] = run_buf[0]; + } + return v; +} + +#else + +static inline int run_packed_size(const s64 n) +{ + const u8 *p = (const u8 *)&n; + + if (n >= 0) { + if (p[7] || p[6] || p[5] || p[4]) + p += 4; + if (p[3] || p[2]) + p += 2; + if (p[1]) + p += 1; + if (p[0] & 0x80) + p += 1; + } else { + if (p[7] != 0xff || p[6] != 0xff || p[5] != 0xff || + p[4] != 0xff) + p += 4; + if (p[3] != 0xff || p[2] != 0xff) + p += 2; + if (p[1] != 0xff) + p += 1; + if (!(p[0] & 0x80)) + p += 1; + } + + return 1 + p - (const u8 *)&n; +} + +/* full trusted function. It does not check 'size' for errors */ +static inline void run_pack_s64(u8 *run_buf, u8 size, s64 v) +{ + const u8 *p = (u8 *)&v; + + /* memcpy( run_buf, &v, size); is it faster? */ + switch (size) { + case 8: + run_buf[7] = p[7]; + fallthrough; + case 7: + run_buf[6] = p[6]; + fallthrough; + case 6: + run_buf[5] = p[5]; + fallthrough; + case 5: + run_buf[4] = p[4]; + fallthrough; + case 4: + run_buf[3] = p[3]; + fallthrough; + case 3: + run_buf[2] = p[2]; + fallthrough; + case 2: + run_buf[1] = p[1]; + fallthrough; + case 1: + run_buf[0] = p[0]; + } +} + +/* full trusted function. It does not check 'size' for errors */ +static inline s64 run_unpack_s64(const u8 *run_buf, u8 size, s64 v) +{ + u8 *p = (u8 *)&v; + + /* memcpy( &v, run_buf, size); is it faster? */ + switch (size) { + case 8: + p[7] = run_buf[7]; + fallthrough; + case 7: + p[6] = run_buf[6]; + fallthrough; + case 6: + p[5] = run_buf[5]; + fallthrough; + case 5: + p[4] = run_buf[4]; + fallthrough; + case 4: + p[3] = run_buf[3]; + fallthrough; + case 3: + p[2] = run_buf[2]; + fallthrough; + case 2: + p[1] = run_buf[1]; + fallthrough; + case 1: + p[0] = run_buf[0]; + } + return v; +} +#endif + +/* + * run_pack + * + * packs runs into buffer + * packed_vcns - how much runs we have packed + * packed_size - how much bytes we have used run_buf + */ +int run_pack(const struct runs_tree *run, CLST svcn, CLST len, u8 *run_buf, + u32 run_buf_size, CLST *packed_vcns) +{ + CLST next_vcn, vcn, lcn; + CLST prev_lcn = 0; + CLST evcn1 = svcn + len; + int packed_size = 0; + size_t i; + bool ok; + s64 dlcn; + int offset_size, size_size, tmp; + + next_vcn = vcn = svcn; + + *packed_vcns = 0; + + if (!len) + goto out; + + ok = run_lookup_entry(run, vcn, &lcn, &len, &i); + + if (!ok) + goto error; + + if (next_vcn != vcn) + goto error; + + for (;;) { + next_vcn = vcn + len; + if (next_vcn > evcn1) + len = evcn1 - vcn; + + /* how much bytes required to pack len */ + size_size = run_packed_size(len); + + /* offset_size - how much bytes is packed dlcn */ + if (lcn == SPARSE_LCN) { + offset_size = 0; + dlcn = 0; + } else { + /* NOTE: lcn can be less than prev_lcn! */ + dlcn = (s64)lcn - prev_lcn; + offset_size = run_packed_size(dlcn); + prev_lcn = lcn; + } + + tmp = run_buf_size - packed_size - 2 - offset_size; + if (tmp <= 0) + goto out; + + /* can we store this entire run */ + if (tmp < size_size) + goto out; + + if (run_buf) { + /* pack run header */ + run_buf[0] = ((u8)(size_size | (offset_size << 4))); + run_buf += 1; + + /* Pack the length of run */ + run_pack_s64(run_buf, size_size, len); + + run_buf += size_size; + /* Pack the offset from previous lcn */ + run_pack_s64(run_buf, offset_size, dlcn); + run_buf += offset_size; + } + + packed_size += 1 + offset_size + size_size; + *packed_vcns += len; + + if (packed_size + 1 >= run_buf_size || next_vcn >= evcn1) + goto out; + + ok = run_get_entry(run, ++i, &vcn, &lcn, &len); + if (!ok) + goto error; + + if (next_vcn != vcn) + goto error; + } + +out: + /* Store last zero */ + if (run_buf) + run_buf[0] = 0; + + return packed_size + 1; + +error: + return -EOPNOTSUPP; +} + +/* + * run_unpack + * + * unpacks packed runs from "run_buf" + * returns error, if negative, or real used bytes + */ +int run_unpack(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino, + CLST svcn, CLST evcn, CLST vcn, const u8 *run_buf, + u32 run_buf_size) +{ + u64 prev_lcn, vcn64, lcn, next_vcn; + const u8 *run_last, *run_0; + bool is_mft = ino == MFT_REC_MFT; + + /* Check for empty */ + if (evcn + 1 == svcn) + return 0; + + if (evcn < svcn) + return -EINVAL; + + run_0 = run_buf; + run_last = run_buf + run_buf_size; + prev_lcn = 0; + vcn64 = svcn; + + /* Read all runs the chain */ + /* size_size - how much bytes is packed len */ + while (run_buf < run_last) { + /* size_size - how much bytes is packed len */ + u8 size_size = *run_buf & 0xF; + /* offset_size - how much bytes is packed dlcn */ + u8 offset_size = *run_buf++ >> 4; + u64 len; + + if (!size_size) + break; + + /* + * Unpack runs. + * NOTE: runs are stored little endian order + * "len" is unsigned value, "dlcn" is signed + * Large positive number requires to store 5 bytes + * e.g.: 05 FF 7E FF FF 00 00 00 + */ + if (size_size > 8) + return -EINVAL; + + len = run_unpack_s64(run_buf, size_size, 0); + /* skip size_size */ + run_buf += size_size; + + if (!len) + return -EINVAL; + + if (!offset_size) + lcn = SPARSE_LCN64; + else if (offset_size <= 8) { + s64 dlcn; + + /* initial value of dlcn is -1 or 0 */ + dlcn = (run_buf[offset_size - 1] & 0x80) ? (s64)-1 : 0; + dlcn = run_unpack_s64(run_buf, offset_size, dlcn); + /* skip offset_size */ + run_buf += offset_size; + + if (!dlcn) + return -EINVAL; + lcn = prev_lcn + dlcn; + prev_lcn = lcn; + } else + return -EINVAL; + + next_vcn = vcn64 + len; + /* check boundary */ + if (next_vcn > evcn + 1) + return -EINVAL; + +#ifndef CONFIG_NTFS3_64BIT_CLUSTER + if (next_vcn > 0x100000000ull || (lcn + len) > 0x100000000ull) { + ntfs_err( + sbi->sb, + "This driver is compiled whitout CONFIG_NTFS3_64BIT_CLUSTER (like windows driver).\n" + "Volume contains 64 bits run: vcn %llx, lcn %llx, len %llx.\n" + "Activate CONFIG_NTFS3_64BIT_CLUSTER to process this case", + vcn64, lcn, len); + return -EOPNOTSUPP; + } +#endif + if (lcn != SPARSE_LCN64 && lcn + len > sbi->used.bitmap.nbits) { + /* lcn range is out of volume */ + return -EINVAL; + } + + if (!run) + ; /* called from check_attr(fslog.c) to check run */ + else if (run == RUN_DEALLOCATE) { + /* called from ni_delete_all to free clusters without storing in run */ + if (lcn != SPARSE_LCN64) + mark_as_free_ex(sbi, lcn, len, true); + } else if (vcn64 >= vcn) { + if (!run_add_entry(run, vcn64, lcn, len, is_mft)) + return -ENOMEM; + } else if (next_vcn > vcn) { + u64 dlen = vcn - vcn64; + + if (!run_add_entry(run, vcn, lcn + dlen, len - dlen, + is_mft)) + return -ENOMEM; + } + + vcn64 = next_vcn; + } + + if (vcn64 != evcn + 1) { + /* not expected length of unpacked runs */ + return -EINVAL; + } + + return run_buf - run_0; +} + +#ifdef NTFS3_CHECK_FREE_CLST +/* + * run_unpack_ex + * + * unpacks packed runs from "run_buf" + * checks unpacked runs to be used in bitmap + * returns error, if negative, or real used bytes + */ +int run_unpack_ex(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino, + CLST svcn, CLST evcn, CLST vcn, const u8 *run_buf, + u32 run_buf_size) +{ + int ret, err; + CLST next_vcn, lcn, len; + size_t index; + bool ok; + struct wnd_bitmap *wnd; + + ret = run_unpack(run, sbi, ino, svcn, evcn, vcn, run_buf, run_buf_size); + if (ret <= 0) + return ret; + + if (!sbi->used.bitmap.sb || !run || run == RUN_DEALLOCATE) + return ret; + + if (ino == MFT_REC_BADCLUST) + return ret; + + next_vcn = vcn = svcn; + wnd = &sbi->used.bitmap; + + for (ok = run_lookup_entry(run, vcn, &lcn, &len, &index); + next_vcn <= evcn; + ok = run_get_entry(run, ++index, &vcn, &lcn, &len)) { + if (!ok || next_vcn != vcn) + return -EINVAL; + + next_vcn = vcn + len; + + if (lcn == SPARSE_LCN) + continue; + + if (sbi->flags & NTFS_FLAGS_NEED_REPLAY) + continue; + + down_read_nested(&wnd->rw_lock, BITMAP_MUTEX_CLUSTERS); + /* Check for free blocks */ + ok = wnd_is_used(wnd, lcn, len); + up_read(&wnd->rw_lock); + if (ok) + continue; + + /* Looks like volume is corrupted */ + ntfs_set_state(sbi, NTFS_DIRTY_ERROR); + + if (down_write_trylock(&wnd->rw_lock)) { + /* mark all zero bits as used in range [lcn, lcn+len) */ + CLST i, lcn_f = 0, len_f = 0; + + err = 0; + for (i = 0; i < len; i++) { + if (wnd_is_free(wnd, lcn + i, 1)) { + if (!len_f) + lcn_f = lcn + i; + len_f += 1; + } else if (len_f) { + err = wnd_set_used(wnd, lcn_f, len_f); + len_f = 0; + if (err) + break; + } + } + + if (len_f) + err = wnd_set_used(wnd, lcn_f, len_f); + + up_write(&wnd->rw_lock); + if (err) + return err; + } + } + + return ret; +} +#endif + +/* + * run_get_highest_vcn + * + * returns the highest vcn from a mapping pairs array + * it used while replaying log file + */ +int run_get_highest_vcn(CLST vcn, const u8 *run_buf, u64 *highest_vcn) +{ + u64 vcn64 = vcn; + u8 size_size; + + while ((size_size = *run_buf & 0xF)) { + u8 offset_size = *run_buf++ >> 4; + u64 len; + + if (size_size > 8 || offset_size > 8) + return -EINVAL; + + len = run_unpack_s64(run_buf, size_size, 0); + if (!len) + return -EINVAL; + + run_buf += size_size + offset_size; + vcn64 += len; + +#ifndef CONFIG_NTFS3_64BIT_CLUSTER + if (vcn64 > 0x100000000ull) + return -EINVAL; +#endif + } + + *highest_vcn = vcn64 - 1; + return 0; +} diff --git a/fs/ntfs3/super.c b/fs/ntfs3/super.c new file mode 100644 index 000000000000..c563431248bf --- /dev/null +++ b/fs/ntfs3/super.c @@ -0,0 +1,1500 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + * + * terminology + * + * cluster - allocation unit - 512,1K,2K,4K,...,2M + * vcn - virtual cluster number - offset inside the file in clusters + * vbo - virtual byte offset - offset inside the file in bytes + * lcn - logical cluster number - 0 based cluster in clusters heap + * lbo - logical byte offset - absolute position inside volume + * run - maps vcn to lcn - stored in attributes in packed form + * attr - attribute segment - std/name/data etc records inside MFT + * mi - mft inode - one MFT record(usually 1024 bytes or 4K), consists of attributes + * ni - ntfs inode - extends linux inode. consists of one or more mft inodes + * index - unit inside directory - 2K, 4K, <=page size, does not depend on cluster size + * + * TODO: Implement + * https://docs.microsoft.com/en-us/windows/wsl/file-permissions + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" +#ifdef CONFIG_NTFS3_LZX_XPRESS +#include "lib/lib.h" +#endif + +#ifdef CONFIG_PRINTK +/* + * Trace warnings/notices/errors + * Thanks Joe Perches for implementation + */ +void ntfs_printk(const struct super_block *sb, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + int level; + struct ntfs_sb_info *sbi = sb->s_fs_info; + + /*should we use different ratelimits for warnings/notices/errors? */ + if (!___ratelimit(&sbi->msg_ratelimit, "ntfs3")) + return; + + va_start(args, fmt); + + level = printk_get_level(fmt); + vaf.fmt = printk_skip_level(fmt); + vaf.va = &args; + printk("%c%cntfs3: %s: %pV\n", KERN_SOH_ASCII, level, sb->s_id, &vaf); + + va_end(args); +} + +static char s_name_buf[512]; +static atomic_t s_name_buf_cnt = ATOMIC_INIT(1); // 1 means 'free s_name_buf' + +/* print warnings/notices/errors about inode using name or inode number */ +void ntfs_inode_printk(struct inode *inode, const char *fmt, ...) +{ + struct super_block *sb = inode->i_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + char *name; + va_list args; + struct va_format vaf; + int level; + + if (!___ratelimit(&sbi->msg_ratelimit, "ntfs3")) + return; + + /* use static allocated buffer, if possible */ + name = atomic_dec_and_test(&s_name_buf_cnt) + ? s_name_buf + : kmalloc(sizeof(s_name_buf), GFP_NOFS); + + if (name) { + struct dentry *de = d_find_alias(inode); + const u32 name_len = ARRAY_SIZE(s_name_buf) - 1; + + if (de) { + spin_lock(&de->d_lock); + snprintf(name, name_len, " \"%s\"", de->d_name.name); + spin_unlock(&de->d_lock); + name[name_len] = 0; /* to be sure*/ + } else { + name[0] = 0; + } + dput(de); /* cocci warns if placed in branch "if (de)" */ + } + + va_start(args, fmt); + + level = printk_get_level(fmt); + vaf.fmt = printk_skip_level(fmt); + vaf.va = &args; + + printk("%c%cntfs3: %s: ino=%lx,%s %pV\n", KERN_SOH_ASCII, level, + sb->s_id, inode->i_ino, name ? name : "", &vaf); + + va_end(args); + + atomic_inc(&s_name_buf_cnt); + if (name != s_name_buf) + kfree(name); +} +#endif + +/* + * Shared memory struct. + * + * on-disk ntfs's upcase table is created by ntfs formater + * 'upcase' table is 128K bytes of memory + * we should read it into memory when mounting + * Several ntfs volumes likely use the same 'upcase' table + * It is good idea to share in-memory 'upcase' table between different volumes + * Unfortunately winxp/vista/win7 use different upcase tables + */ +static DEFINE_SPINLOCK(s_shared_lock); + +static struct { + void *ptr; + u32 len; + int cnt; +} s_shared[8]; + +/* + * ntfs_set_shared + * + * Returns 'ptr' if pointer was saved in shared memory + * Returns NULL if pointer was not shared + */ +void *ntfs_set_shared(void *ptr, u32 bytes) +{ + void *ret = NULL; + int i, j = -1; + + spin_lock(&s_shared_lock); + for (i = 0; i < ARRAY_SIZE(s_shared); i++) { + if (!s_shared[i].cnt) { + j = i; + } else if (bytes == s_shared[i].len && + !memcmp(s_shared[i].ptr, ptr, bytes)) { + s_shared[i].cnt += 1; + ret = s_shared[i].ptr; + break; + } + } + + if (!ret && j != -1) { + s_shared[j].ptr = ptr; + s_shared[j].len = bytes; + s_shared[j].cnt = 1; + ret = ptr; + } + spin_unlock(&s_shared_lock); + + return ret; +} + +/* + * ntfs_put_shared + * + * Returns 'ptr' if pointer is not shared anymore + * Returns NULL if pointer is still shared + */ +void *ntfs_put_shared(void *ptr) +{ + void *ret = ptr; + int i; + + spin_lock(&s_shared_lock); + for (i = 0; i < ARRAY_SIZE(s_shared); i++) { + if (s_shared[i].cnt && s_shared[i].ptr == ptr) { + if (--s_shared[i].cnt) + ret = NULL; + break; + } + } + spin_unlock(&s_shared_lock); + + return ret; +} + +static inline void clear_mount_options(struct ntfs_mount_options *options) +{ + unload_nls(options->nls); +} + +enum Opt { + Opt_uid, + Opt_gid, + Opt_umask, + Opt_dmask, + Opt_fmask, + Opt_immutable, + Opt_discard, + Opt_force, + Opt_sparse, + Opt_nohidden, + Opt_showmeta, + Opt_acl, + Opt_noatime, + Opt_nls, + Opt_prealloc, + Opt_no_acs_rules, + Opt_err, +}; + +static const match_table_t ntfs_tokens = { + { Opt_uid, "uid=%u" }, + { Opt_gid, "gid=%u" }, + { Opt_umask, "umask=%o" }, + { Opt_dmask, "dmask=%o" }, + { Opt_fmask, "fmask=%o" }, + { Opt_immutable, "sys_immutable" }, + { Opt_discard, "discard" }, + { Opt_force, "force" }, + { Opt_sparse, "sparse" }, + { Opt_nohidden, "nohidden" }, + { Opt_acl, "acl" }, + { Opt_noatime, "noatime" }, + { Opt_showmeta, "showmeta" }, + { Opt_nls, "nls=%s" }, + { Opt_prealloc, "prealloc" }, + { Opt_no_acs_rules, "no_acs_rules" }, + { Opt_err, NULL }, +}; + +static noinline int ntfs_parse_options(struct super_block *sb, char *options, + int silent, + struct ntfs_mount_options *opts) +{ + char *p; + substring_t args[MAX_OPT_ARGS]; + int option; + char nls_name[30]; + struct nls_table *nls; + + opts->fs_uid = current_uid(); + opts->fs_gid = current_gid(); + opts->fs_fmask_inv = opts->fs_dmask_inv = ~current_umask(); + nls_name[0] = 0; + + if (!options) + goto out; + + while ((p = strsep(&options, ","))) { + int token; + + if (!*p) + continue; + + token = match_token(p, ntfs_tokens, args); + switch (token) { + case Opt_immutable: + opts->sys_immutable = 1; + break; + case Opt_uid: + if (match_int(&args[0], &option)) + return -EINVAL; + opts->fs_uid = make_kuid(current_user_ns(), option); + if (!uid_valid(opts->fs_uid)) + return -EINVAL; + opts->uid = 1; + break; + case Opt_gid: + if (match_int(&args[0], &option)) + return -EINVAL; + opts->fs_gid = make_kgid(current_user_ns(), option); + if (!gid_valid(opts->fs_gid)) + return -EINVAL; + opts->gid = 1; + break; + case Opt_umask: + if (match_octal(&args[0], &option)) + return -EINVAL; + opts->fs_fmask_inv = opts->fs_dmask_inv = ~option; + opts->fmask = opts->dmask = 1; + break; + case Opt_dmask: + if (match_octal(&args[0], &option)) + return -EINVAL; + opts->fs_dmask_inv = ~option; + opts->dmask = 1; + break; + case Opt_fmask: + if (match_octal(&args[0], &option)) + return -EINVAL; + opts->fs_fmask_inv = ~option; + opts->fmask = 1; + break; + case Opt_discard: + opts->discard = 1; + break; + case Opt_force: + opts->force = 1; + break; + case Opt_sparse: + opts->sparse = 1; + break; + case Opt_nohidden: + opts->nohidden = 1; + break; + case Opt_acl: +#ifdef CONFIG_NTFS3_FS_POSIX_ACL + sb->s_flags |= SB_POSIXACL; + break; +#else + ntfs_err(sb, "support for ACL not compiled in!"); + return -EINVAL; +#endif + case Opt_noatime: + sb->s_flags |= SB_NOATIME; + break; + case Opt_showmeta: + opts->showmeta = 1; + break; + case Opt_nls: + match_strlcpy(nls_name, &args[0], sizeof(nls_name)); + break; + case Opt_prealloc: + opts->prealloc = 1; + break; + case Opt_no_acs_rules: + opts->no_acs_rules = 1; + break; + default: + if (!silent) + ntfs_err( + sb, + "Unrecognized mount option \"%s\" or missing value", + p); + //return -EINVAL; + } + } + +out: + if (!strcmp(nls_name[0] ? nls_name : CONFIG_NLS_DEFAULT, "utf8")) { + /* For UTF-8 use utf16s_to_utf8s/utf8s_to_utf16s instead of nls */ + nls = NULL; + } else if (nls_name[0]) { + nls = load_nls(nls_name); + if (!nls) { + ntfs_err(sb, "failed to load \"%s\"", nls_name); + return -EINVAL; + } + } else { + nls = load_nls_default(); + if (!nls) { + ntfs_err(sb, "failed to load default nls"); + return -EINVAL; + } + } + opts->nls = nls; + + return 0; +} + +static int ntfs_remount(struct super_block *sb, int *flags, char *data) +{ + int err, ro_rw; + struct ntfs_sb_info *sbi = sb->s_fs_info; + struct ntfs_mount_options old_opts; + char *orig_data = kstrdup(data, GFP_KERNEL); + + if (data && !orig_data) + return -ENOMEM; + + /* Store original options */ + memcpy(&old_opts, &sbi->options, sizeof(old_opts)); + clear_mount_options(&sbi->options); + memset(&sbi->options, 0, sizeof(sbi->options)); + + err = ntfs_parse_options(sb, data, 0, &sbi->options); + if (err) + goto restore_opts; + + ro_rw = sb_rdonly(sb) && !(*flags & SB_RDONLY); + if (ro_rw && (sbi->flags & NTFS_FLAGS_NEED_REPLAY)) { + ntfs_warn( + sb, + "Couldn't remount rw because journal is not replayed. Please umount/remount instead\n"); + err = -EINVAL; + goto restore_opts; + } + + sync_filesystem(sb); + + if (ro_rw && (sbi->volume.flags & VOLUME_FLAG_DIRTY) && + !sbi->options.force) { + ntfs_warn(sb, "volume is dirty and \"force\" flag is not set!"); + err = -EINVAL; + goto restore_opts; + } + + clear_mount_options(&old_opts); + + *flags = (*flags & ~SB_LAZYTIME) | (sb->s_flags & SB_LAZYTIME) | + SB_NODIRATIME | SB_NOATIME; + ntfs_info(sb, "re-mounted. Opts: %s", orig_data); + err = 0; + goto out; + +restore_opts: + clear_mount_options(&sbi->options); + memcpy(&sbi->options, &old_opts, sizeof(old_opts)); + +out: + kfree(orig_data); + return err; +} + +static struct kmem_cache *ntfs_inode_cachep; + +static struct inode *ntfs_alloc_inode(struct super_block *sb) +{ + struct ntfs_inode *ni = kmem_cache_alloc(ntfs_inode_cachep, GFP_NOFS); + + if (!ni) + return NULL; + + memset(ni, 0, offsetof(struct ntfs_inode, vfs_inode)); + + mutex_init(&ni->ni_lock); + + return &ni->vfs_inode; +} + +static void ntfs_i_callback(struct rcu_head *head) +{ + struct inode *inode = container_of(head, struct inode, i_rcu); + struct ntfs_inode *ni = ntfs_i(inode); + + mutex_destroy(&ni->ni_lock); + + kmem_cache_free(ntfs_inode_cachep, ni); +} + +static void ntfs_destroy_inode(struct inode *inode) +{ + call_rcu(&inode->i_rcu, ntfs_i_callback); +} + +static void init_once(void *foo) +{ + struct ntfs_inode *ni = foo; + + inode_init_once(&ni->vfs_inode); +} + +/* noinline to reduce binary size*/ +static noinline void put_ntfs(struct ntfs_sb_info *sbi) +{ + ntfs_free(sbi->new_rec); + ntfs_vfree(ntfs_put_shared(sbi->upcase)); + ntfs_free(sbi->def_table); + + wnd_close(&sbi->mft.bitmap); + wnd_close(&sbi->used.bitmap); + + if (sbi->mft.ni) + iput(&sbi->mft.ni->vfs_inode); + + if (sbi->security.ni) + iput(&sbi->security.ni->vfs_inode); + + if (sbi->reparse.ni) + iput(&sbi->reparse.ni->vfs_inode); + + if (sbi->objid.ni) + iput(&sbi->objid.ni->vfs_inode); + + if (sbi->volume.ni) + iput(&sbi->volume.ni->vfs_inode); + + ntfs_update_mftmirr(sbi, 0); + + indx_clear(&sbi->security.index_sii); + indx_clear(&sbi->security.index_sdh); + indx_clear(&sbi->reparse.index_r); + indx_clear(&sbi->objid.index_o); + ntfs_free(sbi->compress.lznt); +#ifdef CONFIG_NTFS3_LZX_XPRESS + xpress_free_decompressor(sbi->compress.xpress); + lzx_free_decompressor(sbi->compress.lzx); +#endif + clear_mount_options(&sbi->options); + + ntfs_free(sbi); +} + +static void ntfs_put_super(struct super_block *sb) +{ + struct ntfs_sb_info *sbi = sb->s_fs_info; + + /*mark rw ntfs as clear, if possible*/ + ntfs_set_state(sbi, NTFS_DIRTY_CLEAR); + + put_ntfs(sbi); + + sync_blockdev(sb->s_bdev); +} + +static int ntfs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + struct wnd_bitmap *wnd = &sbi->used.bitmap; + + buf->f_type = sb->s_magic; + buf->f_bsize = sbi->cluster_size; + buf->f_blocks = wnd->nbits; + + buf->f_bfree = buf->f_bavail = wnd_zeroes(wnd); + buf->f_fsid.val[0] = sbi->volume.ser_num; + buf->f_fsid.val[1] = (sbi->volume.ser_num >> 32); + buf->f_namelen = NTFS_NAME_LEN; + + return 0; +} + +static int ntfs_show_options(struct seq_file *m, struct dentry *root) +{ + struct super_block *sb = root->d_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + struct ntfs_mount_options *opts = &sbi->options; + struct user_namespace *user_ns = seq_user_ns(m); + + if (opts->uid) + seq_printf(m, ",uid=%u", + from_kuid_munged(user_ns, opts->fs_uid)); + if (opts->gid) + seq_printf(m, ",gid=%u", + from_kgid_munged(user_ns, opts->fs_gid)); + if (opts->fmask) + seq_printf(m, ",fmask=%04o", ~opts->fs_fmask_inv); + if (opts->dmask) + seq_printf(m, ",dmask=%04o", ~opts->fs_dmask_inv); + if (opts->nls) + seq_printf(m, ",nls=%s", opts->nls->charset); + else + seq_puts(m, ",nls=utf8"); + if (opts->sys_immutable) + seq_puts(m, ",sys_immutable"); + if (opts->discard) + seq_puts(m, ",discard"); + if (opts->sparse) + seq_puts(m, ",sparse"); + if (opts->showmeta) + seq_puts(m, ",showmeta"); + if (opts->nohidden) + seq_puts(m, ",nohidden"); + if (opts->force) + seq_puts(m, ",force"); + if (opts->no_acs_rules) + seq_puts(m, ",no_acs_rules"); + if (opts->prealloc) + seq_puts(m, ",prealloc"); + if (sb->s_flags & SB_POSIXACL) + seq_puts(m, ",acl"); + if (sb->s_flags & SB_NOATIME) + seq_puts(m, ",noatime"); + + return 0; +} + +/*super_operations::sync_fs*/ +static int ntfs_sync_fs(struct super_block *sb, int wait) +{ + int err = 0, err2; + struct ntfs_sb_info *sbi = sb->s_fs_info; + struct ntfs_inode *ni; + struct inode *inode; + + ni = sbi->security.ni; + if (ni) { + inode = &ni->vfs_inode; + err2 = _ni_write_inode(inode, wait); + if (err2 && !err) + err = err2; + } + + ni = sbi->objid.ni; + if (ni) { + inode = &ni->vfs_inode; + err2 = _ni_write_inode(inode, wait); + if (err2 && !err) + err = err2; + } + + ni = sbi->reparse.ni; + if (ni) { + inode = &ni->vfs_inode; + err2 = _ni_write_inode(inode, wait); + if (err2 && !err) + err = err2; + } + + if (!err) + ntfs_set_state(sbi, NTFS_DIRTY_CLEAR); + + ntfs_update_mftmirr(sbi, wait); + + return err; +} + +static const struct super_operations ntfs_sops = { + .alloc_inode = ntfs_alloc_inode, + .destroy_inode = ntfs_destroy_inode, + .evict_inode = ntfs_evict_inode, + .put_super = ntfs_put_super, + .statfs = ntfs_statfs, + .show_options = ntfs_show_options, + .sync_fs = ntfs_sync_fs, + .remount_fs = ntfs_remount, + .write_inode = ntfs3_write_inode, +}; + +static struct inode *ntfs_export_get_inode(struct super_block *sb, u64 ino, + u32 generation) +{ + struct MFT_REF ref; + struct inode *inode; + + ref.low = cpu_to_le32(ino); +#ifdef CONFIG_NTFS3_64BIT_CLUSTER + ref.high = cpu_to_le16(ino >> 32); +#else + ref.high = 0; +#endif + ref.seq = cpu_to_le16(generation); + + inode = ntfs_iget5(sb, &ref, NULL); + if (!IS_ERR(inode) && is_bad_inode(inode)) { + iput(inode); + inode = ERR_PTR(-ESTALE); + } + + return inode; +} + +static struct dentry *ntfs_fh_to_dentry(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + return generic_fh_to_dentry(sb, fid, fh_len, fh_type, + ntfs_export_get_inode); +} + +static struct dentry *ntfs_fh_to_parent(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + return generic_fh_to_parent(sb, fid, fh_len, fh_type, + ntfs_export_get_inode); +} + +/* TODO: == ntfs_sync_inode */ +static int ntfs_nfs_commit_metadata(struct inode *inode) +{ + return _ni_write_inode(inode, 1); +} + +static const struct export_operations ntfs_export_ops = { + .fh_to_dentry = ntfs_fh_to_dentry, + .fh_to_parent = ntfs_fh_to_parent, + .get_parent = ntfs3_get_parent, + .commit_metadata = ntfs_nfs_commit_metadata, +}; + +/* Returns Gb,Mb to print with "%u.%02u Gb" */ +static u32 format_size_gb(const u64 bytes, u32 *mb) +{ + /* Do simple right 30 bit shift of 64 bit value */ + u64 kbytes = bytes >> 10; + u32 kbytes32 = kbytes; + + *mb = (100 * (kbytes32 & 0xfffff) + 0x7ffff) >> 20; + if (*mb >= 100) + *mb = 99; + + return (kbytes32 >> 20) | (((u32)(kbytes >> 32)) << 12); +} + +static u32 true_sectors_per_clst(const struct NTFS_BOOT *boot) +{ + return boot->sectors_per_clusters <= 0x80 + ? boot->sectors_per_clusters + : (1u << (0 - boot->sectors_per_clusters)); +} + +/* inits internal info from on-disk boot sector*/ +static int ntfs_init_from_boot(struct super_block *sb, u32 sector_size, + u64 dev_size) +{ + struct ntfs_sb_info *sbi = sb->s_fs_info; + int err; + u32 mb, gb, boot_sector_size, sct_per_clst, record_size; + u64 sectors, clusters, fs_size, mlcn, mlcn2; + struct NTFS_BOOT *boot; + struct buffer_head *bh; + struct MFT_REC *rec; + u16 fn, ao; + + sbi->volume.blocks = dev_size >> PAGE_SHIFT; + + bh = ntfs_bread(sb, 0); + if (!bh) + return -EIO; + + err = -EINVAL; + boot = (struct NTFS_BOOT *)bh->b_data; + + if (memcmp(boot->system_id, "NTFS ", sizeof("NTFS ") - 1)) + goto out; + + /* 0x55AA is not mandaroty. Thanks Maxim Suhanov*/ + /*if (0x55 != boot->boot_magic[0] || 0xAA != boot->boot_magic[1]) + * goto out; + */ + + boot_sector_size = (u32)boot->bytes_per_sector[1] << 8; + if (boot->bytes_per_sector[0] || boot_sector_size < SECTOR_SIZE || + !is_power_of2(boot_sector_size)) { + goto out; + } + + /* cluster size: 512, 1K, 2K, 4K, ... 2M */ + sct_per_clst = true_sectors_per_clst(boot); + if (!is_power_of2(sct_per_clst)) + goto out; + + mlcn = le64_to_cpu(boot->mft_clst); + mlcn2 = le64_to_cpu(boot->mft2_clst); + sectors = le64_to_cpu(boot->sectors_per_volume); + + if (mlcn * sct_per_clst >= sectors) + goto out; + + if (mlcn2 * sct_per_clst >= sectors) + goto out; + + /* Check MFT record size */ + if ((boot->record_size < 0 && + SECTOR_SIZE > (2U << (-boot->record_size))) || + (boot->record_size >= 0 && !is_power_of2(boot->record_size))) { + goto out; + } + + /* Check index record size */ + if ((boot->index_size < 0 && + SECTOR_SIZE > (2U << (-boot->index_size))) || + (boot->index_size >= 0 && !is_power_of2(boot->index_size))) { + goto out; + } + + sbi->sector_size = boot_sector_size; + sbi->sector_bits = blksize_bits(boot_sector_size); + fs_size = (sectors + 1) << sbi->sector_bits; + + gb = format_size_gb(fs_size, &mb); + + /* + * - Volume formatted and mounted with the same sector size + * - Volume formatted 4K and mounted as 512 + * - Volume formatted 512 and mounted as 4K + */ + if (sbi->sector_size != sector_size) { + ntfs_warn(sb, + "Different NTFS' sector size and media sector size"); + dev_size += sector_size - 1; + } + + sbi->cluster_size = boot_sector_size * sct_per_clst; + sbi->cluster_bits = blksize_bits(sbi->cluster_size); + + sbi->mft.lbo = mlcn << sbi->cluster_bits; + sbi->mft.lbo2 = mlcn2 << sbi->cluster_bits; + + if (sbi->cluster_size < sbi->sector_size) + goto out; + + sbi->cluster_mask = sbi->cluster_size - 1; + sbi->cluster_mask_inv = ~(u64)sbi->cluster_mask; + sbi->record_size = record_size = boot->record_size < 0 + ? 1 << (-boot->record_size) + : (u32)boot->record_size + << sbi->cluster_bits; + + if (record_size > MAXIMUM_BYTES_PER_MFT) + goto out; + + sbi->record_bits = blksize_bits(record_size); + sbi->attr_size_tr = (5 * record_size >> 4); // ~320 bytes + + sbi->max_bytes_per_attr = + record_size - QuadAlign(MFTRECORD_FIXUP_OFFSET_1) - + QuadAlign(((record_size >> SECTOR_SHIFT) * sizeof(short))) - + QuadAlign(sizeof(enum ATTR_TYPE)); + + sbi->index_size = boot->index_size < 0 + ? 1u << (-boot->index_size) + : (u32)boot->index_size << sbi->cluster_bits; + + sbi->volume.ser_num = le64_to_cpu(boot->serial_num); + sbi->volume.size = sectors << sbi->sector_bits; + + /* warning if RAW volume */ + if (dev_size < fs_size) { + u32 mb0, gb0; + + gb0 = format_size_gb(dev_size, &mb0); + ntfs_warn( + sb, + "RAW NTFS volume: Filesystem size %u.%02u Gb > volume size %u.%02u Gb. Mount in read-only", + gb, mb, gb0, mb0); + sb->s_flags |= SB_RDONLY; + } + + clusters = sbi->volume.size >> sbi->cluster_bits; +#ifndef CONFIG_NTFS3_64BIT_CLUSTER + /* 32 bits per cluster */ + if (clusters >> 32) { + ntfs_notice( + sb, + "NTFS %u.%02u Gb is too big to use 32 bits per cluster", + gb, mb); + goto out; + } +#elif BITS_PER_LONG < 64 +#error "CONFIG_NTFS3_64BIT_CLUSTER incompatible in 32 bit OS" +#endif + + sbi->used.bitmap.nbits = clusters; + + rec = ntfs_zalloc(record_size); + if (!rec) { + err = -ENOMEM; + goto out; + } + + sbi->new_rec = rec; + rec->rhdr.sign = NTFS_FILE_SIGNATURE; + rec->rhdr.fix_off = cpu_to_le16(MFTRECORD_FIXUP_OFFSET_1); + fn = (sbi->record_size >> SECTOR_SHIFT) + 1; + rec->rhdr.fix_num = cpu_to_le16(fn); + ao = QuadAlign(MFTRECORD_FIXUP_OFFSET_1 + sizeof(short) * fn); + rec->attr_off = cpu_to_le16(ao); + rec->used = cpu_to_le32(ao + QuadAlign(sizeof(enum ATTR_TYPE))); + rec->total = cpu_to_le32(sbi->record_size); + ((struct ATTRIB *)Add2Ptr(rec, ao))->type = ATTR_END; + + if (sbi->cluster_size < PAGE_SIZE) + sb_set_blocksize(sb, sbi->cluster_size); + + sbi->block_mask = sb->s_blocksize - 1; + sbi->blocks_per_cluster = sbi->cluster_size >> sb->s_blocksize_bits; + sbi->volume.blocks = sbi->volume.size >> sb->s_blocksize_bits; + + /* Maximum size for normal files */ + sbi->maxbytes = (clusters << sbi->cluster_bits) - 1; + +#ifdef CONFIG_NTFS3_64BIT_CLUSTER + if (clusters >= (1ull << (64 - sbi->cluster_bits))) + sbi->maxbytes = -1; + sbi->maxbytes_sparse = -1; +#else + /* Maximum size for sparse file */ + sbi->maxbytes_sparse = (1ull << (sbi->cluster_bits + 32)) - 1; +#endif + + err = 0; + +out: + brelse(bh); + + return err; +} + +/* try to mount*/ +static int ntfs_fill_super(struct super_block *sb, void *data, int silent) +{ + int err; + struct ntfs_sb_info *sbi; + struct block_device *bdev = sb->s_bdev; + struct inode *bd_inode = bdev->bd_inode; + struct request_queue *rq = bdev_get_queue(bdev); + struct inode *inode = NULL; + struct ntfs_inode *ni; + size_t i, tt; + CLST vcn, lcn, len; + struct ATTRIB *attr; + const struct VOLUME_INFO *info; + u32 idx, done, bytes; + struct ATTR_DEF_ENTRY *t; + u16 *upcase = NULL; + u16 *shared; + bool is_ro; + struct MFT_REF ref; + + ref.high = 0; + + sbi = ntfs_zalloc(sizeof(struct ntfs_sb_info)); + if (!sbi) + return -ENOMEM; + + sb->s_fs_info = sbi; + sbi->sb = sb; + sb->s_flags |= SB_NODIRATIME; + sb->s_magic = 0x7366746e; // "ntfs" + sb->s_op = &ntfs_sops; + sb->s_export_op = &ntfs_export_ops; + sb->s_time_gran = NTFS_TIME_GRAN; // 100 nsec + sb->s_xattr = ntfs_xattr_handlers; + + ratelimit_state_init(&sbi->msg_ratelimit, DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + + err = ntfs_parse_options(sb, data, silent, &sbi->options); + if (err) + goto out; + + if (!rq || !blk_queue_discard(rq) || !rq->limits.discard_granularity) { + ; + } else { + sbi->discard_granularity = rq->limits.discard_granularity; + sbi->discard_granularity_mask_inv = + ~(u64)(sbi->discard_granularity - 1); + } + + sb_set_blocksize(sb, PAGE_SIZE); + + /* parse boot */ + err = ntfs_init_from_boot(sb, rq ? queue_logical_block_size(rq) : 512, + bd_inode->i_size); + if (err) + goto out; + +#ifdef CONFIG_NTFS3_64BIT_CLUSTER + sb->s_maxbytes = MAX_LFS_FILESIZE; +#else + sb->s_maxbytes = 0xFFFFFFFFull << sbi->cluster_bits; +#endif + + mutex_init(&sbi->compress.mtx_lznt); +#ifdef CONFIG_NTFS3_LZX_XPRESS + mutex_init(&sbi->compress.mtx_xpress); + mutex_init(&sbi->compress.mtx_lzx); +#endif + + /* + * Load $Volume. This should be done before LogFile + * 'cause 'sbi->volume.ni' is used 'ntfs_set_state' + */ + ref.low = cpu_to_le32(MFT_REC_VOL); + ref.seq = cpu_to_le16(MFT_REC_VOL); + inode = ntfs_iget5(sb, &ref, &NAME_VOLUME); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + ntfs_err(sb, "Failed to load $Volume."); + inode = NULL; + goto out; + } + + ni = ntfs_i(inode); + + /* Load and save label (not necessary) */ + attr = ni_find_attr(ni, NULL, NULL, ATTR_LABEL, NULL, 0, NULL, NULL); + + if (!attr) { + /* It is ok if no ATTR_LABEL */ + } else if (!attr->non_res && !is_attr_ext(attr)) { + /* $AttrDef allows labels to be up to 128 symbols */ + err = utf16s_to_utf8s(resident_data(attr), + le32_to_cpu(attr->res.data_size) >> 1, + UTF16_LITTLE_ENDIAN, sbi->volume.label, + sizeof(sbi->volume.label)); + if (err < 0) + sbi->volume.label[0] = 0; + } else { + /* should we break mounting here? */ + //err = -EINVAL; + //goto out; + } + + attr = ni_find_attr(ni, attr, NULL, ATTR_VOL_INFO, NULL, 0, NULL, NULL); + if (!attr || is_attr_ext(attr)) { + err = -EINVAL; + goto out; + } + + info = resident_data_ex(attr, SIZEOF_ATTRIBUTE_VOLUME_INFO); + if (!info) { + err = -EINVAL; + goto out; + } + + sbi->volume.major_ver = info->major_ver; + sbi->volume.minor_ver = info->minor_ver; + sbi->volume.flags = info->flags; + + sbi->volume.ni = ni; + inode = NULL; + + /* Load $MFTMirr to estimate recs_mirr */ + ref.low = cpu_to_le32(MFT_REC_MIRR); + ref.seq = cpu_to_le16(MFT_REC_MIRR); + inode = ntfs_iget5(sb, &ref, &NAME_MIRROR); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + ntfs_err(sb, "Failed to load $MFTMirr."); + inode = NULL; + goto out; + } + + sbi->mft.recs_mirr = + ntfs_up_cluster(sbi, inode->i_size) >> sbi->record_bits; + + iput(inode); + + /* Load LogFile to replay */ + ref.low = cpu_to_le32(MFT_REC_LOG); + ref.seq = cpu_to_le16(MFT_REC_LOG); + inode = ntfs_iget5(sb, &ref, &NAME_LOGFILE); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + ntfs_err(sb, "Failed to load \x24LogFile."); + inode = NULL; + goto out; + } + + ni = ntfs_i(inode); + + err = ntfs_loadlog_and_replay(ni, sbi); + if (err) + goto out; + + iput(inode); + inode = NULL; + + is_ro = sb_rdonly(sbi->sb); + + if (sbi->flags & NTFS_FLAGS_NEED_REPLAY) { + if (!is_ro) { + ntfs_warn(sb, + "failed to replay log file. Can't mount rw!"); + err = -EINVAL; + goto out; + } + } else if (sbi->volume.flags & VOLUME_FLAG_DIRTY) { + if (!is_ro && !sbi->options.force) { + ntfs_warn( + sb, + "volume is dirty and \"force\" flag is not set!"); + err = -EINVAL; + goto out; + } + } + + /* Load $MFT */ + ref.low = cpu_to_le32(MFT_REC_MFT); + ref.seq = cpu_to_le16(1); + + inode = ntfs_iget5(sb, &ref, &NAME_MFT); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + ntfs_err(sb, "Failed to load $MFT."); + inode = NULL; + goto out; + } + + ni = ntfs_i(inode); + + sbi->mft.used = ni->i_valid >> sbi->record_bits; + tt = inode->i_size >> sbi->record_bits; + sbi->mft.next_free = MFT_REC_USER; + + err = wnd_init(&sbi->mft.bitmap, sb, tt); + if (err) + goto out; + + err = ni_load_all_mi(ni); + if (err) + goto out; + + sbi->mft.ni = ni; + + /* Load $BadClus */ + ref.low = cpu_to_le32(MFT_REC_BADCLUST); + ref.seq = cpu_to_le16(MFT_REC_BADCLUST); + inode = ntfs_iget5(sb, &ref, &NAME_BADCLUS); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + ntfs_err(sb, "Failed to load $BadClus."); + inode = NULL; + goto out; + } + + ni = ntfs_i(inode); + + for (i = 0; run_get_entry(&ni->file.run, i, &vcn, &lcn, &len); i++) { + if (lcn == SPARSE_LCN) + continue; + + if (!sbi->bad_clusters) + ntfs_notice(sb, "Volume contains bad blocks"); + + sbi->bad_clusters += len; + } + + iput(inode); + + /* Load $Bitmap */ + ref.low = cpu_to_le32(MFT_REC_BITMAP); + ref.seq = cpu_to_le16(MFT_REC_BITMAP); + inode = ntfs_iget5(sb, &ref, &NAME_BITMAP); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + ntfs_err(sb, "Failed to load $Bitmap."); + inode = NULL; + goto out; + } + + ni = ntfs_i(inode); + +#ifndef CONFIG_NTFS3_64BIT_CLUSTER + if (inode->i_size >> 32) { + err = -EINVAL; + goto out; + } +#endif + + /* Check bitmap boundary */ + tt = sbi->used.bitmap.nbits; + if (inode->i_size < bitmap_size(tt)) { + err = -EINVAL; + goto out; + } + + /* Not necessary */ + sbi->used.bitmap.set_tail = true; + err = wnd_init(&sbi->used.bitmap, sbi->sb, tt); + if (err) + goto out; + + iput(inode); + + /* Compute the mft zone */ + err = ntfs_refresh_zone(sbi); + if (err) + goto out; + + /* Load $AttrDef */ + ref.low = cpu_to_le32(MFT_REC_ATTR); + ref.seq = cpu_to_le16(MFT_REC_ATTR); + inode = ntfs_iget5(sbi->sb, &ref, &NAME_ATTRDEF); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + ntfs_err(sb, "Failed to load $AttrDef -> %d", err); + inode = NULL; + goto out; + } + + if (inode->i_size < sizeof(struct ATTR_DEF_ENTRY)) { + err = -EINVAL; + goto out; + } + bytes = inode->i_size; + sbi->def_table = t = ntfs_malloc(bytes); + if (!t) { + err = -ENOMEM; + goto out; + } + + for (done = idx = 0; done < bytes; done += PAGE_SIZE, idx++) { + unsigned long tail = bytes - done; + struct page *page = ntfs_map_page(inode->i_mapping, idx); + + if (IS_ERR(page)) { + err = PTR_ERR(page); + goto out; + } + memcpy(Add2Ptr(t, done), page_address(page), + min(PAGE_SIZE, tail)); + ntfs_unmap_page(page); + + if (!idx && ATTR_STD != t->type) { + err = -EINVAL; + goto out; + } + } + + t += 1; + sbi->def_entries = 1; + done = sizeof(struct ATTR_DEF_ENTRY); + sbi->reparse.max_size = MAXIMUM_REPARSE_DATA_BUFFER_SIZE; + sbi->ea_max_size = 0x10000; /* default formater value */ + + while (done + sizeof(struct ATTR_DEF_ENTRY) <= bytes) { + u32 t32 = le32_to_cpu(t->type); + u64 sz = le64_to_cpu(t->max_sz); + + if ((t32 & 0xF) || le32_to_cpu(t[-1].type) >= t32) + break; + + if (t->type == ATTR_REPARSE) + sbi->reparse.max_size = sz; + else if (t->type == ATTR_EA) + sbi->ea_max_size = sz; + + done += sizeof(struct ATTR_DEF_ENTRY); + t += 1; + sbi->def_entries += 1; + } + iput(inode); + + /* Load $UpCase */ + ref.low = cpu_to_le32(MFT_REC_UPCASE); + ref.seq = cpu_to_le16(MFT_REC_UPCASE); + inode = ntfs_iget5(sb, &ref, &NAME_UPCASE); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + ntfs_err(sb, "Failed to load \x24LogFile."); + inode = NULL; + goto out; + } + + ni = ntfs_i(inode); + + if (inode->i_size != 0x10000 * sizeof(short)) { + err = -EINVAL; + goto out; + } + + sbi->upcase = upcase = ntfs_vmalloc(0x10000 * sizeof(short)); + if (!upcase) { + err = -ENOMEM; + goto out; + } + + for (idx = 0; idx < (0x10000 * sizeof(short) >> PAGE_SHIFT); idx++) { + const __le16 *src; + u16 *dst = Add2Ptr(upcase, idx << PAGE_SHIFT); + struct page *page = ntfs_map_page(inode->i_mapping, idx); + + if (IS_ERR(page)) { + err = PTR_ERR(page); + goto out; + } + + src = page_address(page); + +#ifdef __BIG_ENDIAN + for (i = 0; i < PAGE_SIZE / sizeof(u16); i++) + *dst++ = le16_to_cpu(*src++); +#else + memcpy(dst, src, PAGE_SIZE); +#endif + ntfs_unmap_page(page); + } + + shared = ntfs_set_shared(upcase, 0x10000 * sizeof(short)); + if (shared && upcase != shared) { + sbi->upcase = shared; + ntfs_vfree(upcase); + } + + iput(inode); + inode = NULL; + + if (is_ntfs3(sbi)) { + /* Load $Secure */ + err = ntfs_security_init(sbi); + if (err) + goto out; + + /* Load $Extend */ + err = ntfs_extend_init(sbi); + if (err) + goto load_root; + + /* Load $Extend\$Reparse */ + err = ntfs_reparse_init(sbi); + if (err) + goto load_root; + + /* Load $Extend\$ObjId */ + err = ntfs_objid_init(sbi); + if (err) + goto load_root; + } + +load_root: + /* Load root */ + ref.low = cpu_to_le32(MFT_REC_ROOT); + ref.seq = cpu_to_le16(MFT_REC_ROOT); + inode = ntfs_iget5(sb, &ref, &NAME_ROOT); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + ntfs_err(sb, "Failed to load root."); + inode = NULL; + goto out; + } + + ni = ntfs_i(inode); + + sb->s_root = d_make_root(inode); + + if (!sb->s_root) { + err = -EINVAL; + goto out; + } + + return 0; + +out: + iput(inode); + + if (sb->s_root) { + d_drop(sb->s_root); + sb->s_root = NULL; + } + + put_ntfs(sbi); + + sb->s_fs_info = NULL; + return err; +} + +void ntfs_unmap_meta(struct super_block *sb, CLST lcn, CLST len) +{ + struct ntfs_sb_info *sbi = sb->s_fs_info; + struct block_device *bdev = sb->s_bdev; + sector_t devblock = (u64)lcn * sbi->blocks_per_cluster; + unsigned long blocks = (u64)len * sbi->blocks_per_cluster; + unsigned long cnt = 0; + unsigned long limit = global_zone_page_state(NR_FREE_PAGES) + << (PAGE_SHIFT - sb->s_blocksize_bits); + + if (limit >= 0x2000) + limit -= 0x1000; + else if (limit < 32) + limit = 32; + else + limit >>= 1; + + while (blocks--) { + clean_bdev_aliases(bdev, devblock++, 1); + if (cnt++ >= limit) { + sync_blockdev(bdev); + cnt = 0; + } + } +} + +/* + * ntfs_discard + * + * issue a discard request (trim for SSD) + */ +int ntfs_discard(struct ntfs_sb_info *sbi, CLST lcn, CLST len) +{ + int err; + u64 lbo, bytes, start, end; + struct super_block *sb; + + if (sbi->used.next_free_lcn == lcn + len) + sbi->used.next_free_lcn = lcn; + + if (sbi->flags & NTFS_FLAGS_NODISCARD) + return -EOPNOTSUPP; + + if (!sbi->options.discard) + return -EOPNOTSUPP; + + lbo = (u64)lcn << sbi->cluster_bits; + bytes = (u64)len << sbi->cluster_bits; + + /* Align up 'start' on discard_granularity */ + start = (lbo + sbi->discard_granularity - 1) & + sbi->discard_granularity_mask_inv; + /* Align down 'end' on discard_granularity */ + end = (lbo + bytes) & sbi->discard_granularity_mask_inv; + + sb = sbi->sb; + if (start >= end) + return 0; + + err = blkdev_issue_discard(sb->s_bdev, start >> 9, (end - start) >> 9, + GFP_NOFS, 0); + + if (err == -EOPNOTSUPP) + sbi->flags |= NTFS_FLAGS_NODISCARD; + + return err; +} + +static struct dentry *ntfs_mount(struct file_system_type *fs_type, int flags, + const char *dev_name, void *data) +{ + return mount_bdev(fs_type, flags, dev_name, data, ntfs_fill_super); +} + +static struct file_system_type ntfs_fs_type = { + .owner = THIS_MODULE, + .name = "ntfs3", + .mount = ntfs_mount, + .kill_sb = kill_block_super, + .fs_flags = FS_REQUIRES_DEV, +}; + +static int __init init_ntfs_fs(void) +{ + int err; + + pr_notice("ntfs3: Index binary search\n"); + pr_notice("ntfs3: Hot fix free clusters\n"); + pr_notice("ntfs3: Max link count %u\n", NTFS_LINK_MAX); + +#ifdef CONFIG_NTFS3_FS_POSIX_ACL + pr_notice("ntfs3: Enabled Linux POSIX ACLs support\n"); +#endif +#ifdef CONFIG_NTFS3_64BIT_CLUSTER + pr_notice("ntfs3: Activated 64 bits per cluster\n"); +#else + pr_notice("ntfs3: Activated 32 bits per cluster\n"); +#endif +#ifdef CONFIG_NTFS3_LZX_XPRESS + pr_notice("ntfs3: Read-only lzx/xpress compression included\n"); +#endif + + err = ntfs3_init_bitmap(); + if (err) + return err; + + ntfs_inode_cachep = kmem_cache_create( + "ntfs_inode_cache", sizeof(struct ntfs_inode), 0, + (SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT), + init_once); + if (!ntfs_inode_cachep) { + err = -ENOMEM; + goto out1; + } + + err = register_filesystem(&ntfs_fs_type); + if (err) + goto out; + + return 0; +out: + kmem_cache_destroy(ntfs_inode_cachep); +out1: + ntfs3_exit_bitmap(); + return err; +} + +static void __exit exit_ntfs_fs(void) +{ + if (ntfs_inode_cachep) { + rcu_barrier(); + kmem_cache_destroy(ntfs_inode_cachep); + } + + unregister_filesystem(&ntfs_fs_type); + ntfs3_exit_bitmap(); +} + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("ntfs3 read/write filesystem"); +MODULE_INFO(behaviour, "Index binary search"); +MODULE_INFO(behaviour, "Hot fix free clusters"); +#ifdef CONFIG_NTFS3_FS_POSIX_ACL +MODULE_INFO(behaviour, "Enabled Linux POSIX ACLs support"); +#endif +#ifdef CONFIG_NTFS3_64BIT_CLUSTER +MODULE_INFO(cluster, "Activated 64 bits per cluster"); +#else +MODULE_INFO(cluster, "Activated 32 bits per cluster"); +#endif +#ifdef CONFIG_NTFS3_LZX_XPRESS +MODULE_INFO(compression, "Read-only lzx/xpress compression included"); +#endif + +MODULE_AUTHOR("Konstantin Komarov"); +MODULE_ALIAS_FS("ntfs3"); + +module_init(init_ntfs_fs); +module_exit(exit_ntfs_fs); diff --git a/fs/ntfs3/upcase.c b/fs/ntfs3/upcase.c new file mode 100644 index 000000000000..9617382aca64 --- /dev/null +++ b/fs/ntfs3/upcase.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + */ +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +static inline u16 upcase_unicode_char(const u16 *upcase, u16 chr) +{ + if (chr < 'a') + return chr; + + if (chr <= 'z') + return chr - ('a' - 'A'); + + return upcase[chr]; +} + +/* + * Thanks Kari Argillander for idea and implementation 'bothcase' + * + * Straigth way to compare names: + * - case insensitive + * - if name equals and 'bothcases' then + * - case sensitive + * 'Straigth way' code scans input names twice in worst case + * Optimized code scans input names only once + */ +int ntfs_cmp_names(const __le16 *s1, size_t l1, const __le16 *s2, size_t l2, + const u16 *upcase, bool bothcase) +{ + int diff1 = 0; + int diff2; + size_t len = min(l1, l2); + + if (!bothcase && upcase) + goto case_insentive; + + for (; len; s1++, s2++, len--) { + diff1 = le16_to_cpu(*s1) - le16_to_cpu(*s2); + if (diff1) { + if (bothcase && upcase) + goto case_insentive; + + return diff1; + } + } + return l1 - l2; + +case_insentive: + for (; len; s1++, s2++, len--) { + diff2 = upcase_unicode_char(upcase, le16_to_cpu(*s1)) - + upcase_unicode_char(upcase, le16_to_cpu(*s2)); + if (diff2) + return diff2; + } + + diff2 = l1 - l2; + return diff2 ? diff2 : diff1; +} + +int ntfs_cmp_names_cpu(const struct cpu_str *uni1, const struct le_str *uni2, + const u16 *upcase, bool bothcase) +{ + const u16 *s1 = uni1->name; + const __le16 *s2 = uni2->name; + size_t l1 = uni1->len; + size_t l2 = uni2->len; + size_t len = min(l1, l2); + int diff1 = 0; + int diff2; + + if (!bothcase && upcase) + goto case_insentive; + + for (; len; s1++, s2++, len--) { + diff1 = *s1 - le16_to_cpu(*s2); + if (diff1) { + if (bothcase && upcase) + goto case_insentive; + + return diff1; + } + } + return l1 - l2; + +case_insentive: + for (; len; s1++, s2++, len--) { + diff2 = upcase_unicode_char(upcase, *s1) - + upcase_unicode_char(upcase, le16_to_cpu(*s2)); + if (diff2) + return diff2; + } + + diff2 = l1 - l2; + return diff2 ? diff2 : diff1; +} diff --git a/fs/ntfs3/xattr.c b/fs/ntfs3/xattr.c new file mode 100644 index 000000000000..759df507c92c --- /dev/null +++ b/fs/ntfs3/xattr.c @@ -0,0 +1,1046 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +// clang-format off +#define SYSTEM_DOS_ATTRIB "system.dos_attrib" +#define SYSTEM_NTFS_ATTRIB "system.ntfs_attrib" +#define SYSTEM_NTFS_SECURITY "system.ntfs_security" +// clang-format on + +static inline size_t unpacked_ea_size(const struct EA_FULL *ea) +{ + return ea->size ? le32_to_cpu(ea->size) + : DwordAlign(struct_size( + ea, name, + 1 + ea->name_len + le16_to_cpu(ea->elength))); +} + +static inline size_t packed_ea_size(const struct EA_FULL *ea) +{ + return struct_size(ea, name, + 1 + ea->name_len + le16_to_cpu(ea->elength)) - + offsetof(struct EA_FULL, flags); +} + +/* + * find_ea + * + * assume there is at least one xattr in the list + */ +static inline bool find_ea(const struct EA_FULL *ea_all, u32 bytes, + const char *name, u8 name_len, u32 *off) +{ + *off = 0; + + if (!ea_all || !bytes) + return false; + + for (;;) { + const struct EA_FULL *ea = Add2Ptr(ea_all, *off); + u32 next_off = *off + unpacked_ea_size(ea); + + if (next_off > bytes) + return false; + + if (ea->name_len == name_len && + !memcmp(ea->name, name, name_len)) + return true; + + *off = next_off; + if (next_off >= bytes) + return false; + } +} + +/* + * ntfs_read_ea + * + * reads all extended attributes + * ea - new allocated memory + * info - pointer into resident data + */ +static int ntfs_read_ea(struct ntfs_inode *ni, struct EA_FULL **ea, + size_t add_bytes, const struct EA_INFO **info) +{ + int err; + struct ATTR_LIST_ENTRY *le = NULL; + struct ATTRIB *attr_info, *attr_ea; + void *ea_p; + u32 size; + + static_assert(le32_to_cpu(ATTR_EA_INFO) < le32_to_cpu(ATTR_EA)); + + *ea = NULL; + *info = NULL; + + attr_info = + ni_find_attr(ni, NULL, &le, ATTR_EA_INFO, NULL, 0, NULL, NULL); + attr_ea = + ni_find_attr(ni, attr_info, &le, ATTR_EA, NULL, 0, NULL, NULL); + + if (!attr_ea || !attr_info) + return 0; + + *info = resident_data_ex(attr_info, sizeof(struct EA_INFO)); + if (!*info) + return -EINVAL; + + /* Check Ea limit */ + size = le32_to_cpu((*info)->size); + if (size > ni->mi.sbi->ea_max_size) + return -EFBIG; + + if (attr_size(attr_ea) > ni->mi.sbi->ea_max_size) + return -EFBIG; + + /* Allocate memory for packed Ea */ + ea_p = ntfs_malloc(size + add_bytes); + if (!ea_p) + return -ENOMEM; + + if (attr_ea->non_res) { + struct runs_tree run; + + run_init(&run); + + err = attr_load_runs(attr_ea, ni, &run, NULL); + if (!err) + err = ntfs_read_run_nb(ni->mi.sbi, &run, 0, ea_p, size, + NULL); + run_close(&run); + + if (err) + goto out; + } else { + void *p = resident_data_ex(attr_ea, size); + + if (!p) { + err = -EINVAL; + goto out; + } + memcpy(ea_p, p, size); + } + + memset(Add2Ptr(ea_p, size), 0, add_bytes); + *ea = ea_p; + return 0; + +out: + ntfs_free(ea_p); + *ea = NULL; + return err; +} + +/* + * ntfs_list_ea + * + * copy a list of xattrs names into the buffer + * provided, or compute the buffer size required + * + * Returns a negative error number on failure, or the number of bytes + * used / required on success. + */ +static ssize_t ntfs_list_ea(struct ntfs_inode *ni, char *buffer, + size_t bytes_per_buffer) +{ + const struct EA_INFO *info; + struct EA_FULL *ea_all = NULL; + const struct EA_FULL *ea; + u32 off, size; + int err; + size_t ret; + + err = ntfs_read_ea(ni, &ea_all, 0, &info); + if (err) + return err; + + if (!info || !ea_all) + return 0; + + size = le32_to_cpu(info->size); + + /* Enumerate all xattrs */ + for (ret = 0, off = 0; off < size; off += unpacked_ea_size(ea)) { + ea = Add2Ptr(ea_all, off); + + if (buffer) { + if (ret + ea->name_len + 1 > bytes_per_buffer) { + err = -ERANGE; + goto out; + } + + memcpy(buffer + ret, ea->name, ea->name_len); + buffer[ret + ea->name_len] = 0; + } + + ret += ea->name_len + 1; + } + +out: + ntfs_free(ea_all); + return err ? err : ret; +} + +static int ntfs_get_ea(struct inode *inode, const char *name, size_t name_len, + void *buffer, size_t size, size_t *required) +{ + struct ntfs_inode *ni = ntfs_i(inode); + const struct EA_INFO *info; + struct EA_FULL *ea_all = NULL; + const struct EA_FULL *ea; + u32 off, len; + int err; + + if (!(ni->ni_flags & NI_FLAG_EA)) + return -ENODATA; + + if (!required) + ni_lock(ni); + + len = 0; + + if (name_len > 255) { + err = -ENAMETOOLONG; + goto out; + } + + err = ntfs_read_ea(ni, &ea_all, 0, &info); + if (err) + goto out; + + if (!info) + goto out; + + /* Enumerate all xattrs */ + if (!find_ea(ea_all, le32_to_cpu(info->size), name, name_len, &off)) { + err = -ENODATA; + goto out; + } + ea = Add2Ptr(ea_all, off); + + len = le16_to_cpu(ea->elength); + if (!buffer) { + err = 0; + goto out; + } + + if (len > size) { + err = -ERANGE; + if (required) + *required = len; + goto out; + } + + memcpy(buffer, ea->name + ea->name_len + 1, len); + err = 0; + +out: + ntfs_free(ea_all); + if (!required) + ni_unlock(ni); + + return err ? err : len; +} + +static noinline int ntfs_set_ea(struct inode *inode, const char *name, + size_t name_len, const void *value, + size_t val_size, int flags, int locked) +{ + struct ntfs_inode *ni = ntfs_i(inode); + struct ntfs_sb_info *sbi = ni->mi.sbi; + int err; + struct EA_INFO ea_info; + const struct EA_INFO *info; + struct EA_FULL *new_ea; + struct EA_FULL *ea_all = NULL; + size_t add, new_pack; + u32 off, size; + __le16 size_pack; + struct ATTRIB *attr; + struct ATTR_LIST_ENTRY *le; + struct mft_inode *mi; + struct runs_tree ea_run; + u64 new_sz; + void *p; + + if (!locked) + ni_lock(ni); + + run_init(&ea_run); + + if (name_len > 255) { + err = -ENAMETOOLONG; + goto out; + } + + add = DwordAlign(struct_size(ea_all, name, 1 + name_len + val_size)); + + err = ntfs_read_ea(ni, &ea_all, add, &info); + if (err) + goto out; + + if (!info) { + memset(&ea_info, 0, sizeof(ea_info)); + size = 0; + size_pack = 0; + } else { + memcpy(&ea_info, info, sizeof(ea_info)); + size = le32_to_cpu(ea_info.size); + size_pack = ea_info.size_pack; + } + + if (info && find_ea(ea_all, size, name, name_len, &off)) { + struct EA_FULL *ea; + size_t ea_sz; + + if (flags & XATTR_CREATE) { + err = -EEXIST; + goto out; + } + + /* Remove current xattr */ + ea = Add2Ptr(ea_all, off); + if (ea->flags & FILE_NEED_EA) + le16_add_cpu(&ea_info.count, -1); + + ea_sz = unpacked_ea_size(ea); + + le16_add_cpu(&ea_info.size_pack, 0 - packed_ea_size(ea)); + + memmove(ea, Add2Ptr(ea, ea_sz), size - off - ea_sz); + + size -= ea_sz; + memset(Add2Ptr(ea_all, size), 0, ea_sz); + + ea_info.size = cpu_to_le32(size); + + if ((flags & XATTR_REPLACE) && !val_size) + goto update_ea; + } else { + if (flags & XATTR_REPLACE) { + err = -ENODATA; + goto out; + } + + if (!ea_all) { + ea_all = ntfs_zalloc(add); + if (!ea_all) { + err = -ENOMEM; + goto out; + } + } + } + + /* append new xattr */ + new_ea = Add2Ptr(ea_all, size); + new_ea->size = cpu_to_le32(add); + new_ea->flags = 0; + new_ea->name_len = name_len; + new_ea->elength = cpu_to_le16(val_size); + memcpy(new_ea->name, name, name_len); + new_ea->name[name_len] = 0; + memcpy(new_ea->name + name_len + 1, value, val_size); + new_pack = le16_to_cpu(ea_info.size_pack) + packed_ea_size(new_ea); + + /* should fit into 16 bits */ + if (new_pack > 0xffff) { + err = -EFBIG; // -EINVAL? + goto out; + } + ea_info.size_pack = cpu_to_le16(new_pack); + + /* new size of ATTR_EA */ + size += add; + if (size > sbi->ea_max_size) { + err = -EFBIG; // -EINVAL? + goto out; + } + ea_info.size = cpu_to_le32(size); + +update_ea: + + if (!info) { + /* Create xattr */ + if (!size) { + err = 0; + goto out; + } + + err = ni_insert_resident(ni, sizeof(struct EA_INFO), + ATTR_EA_INFO, NULL, 0, NULL, NULL); + if (err) + goto out; + + err = ni_insert_resident(ni, 0, ATTR_EA, NULL, 0, NULL, NULL); + if (err) + goto out; + } + + new_sz = size; + err = attr_set_size(ni, ATTR_EA, NULL, 0, &ea_run, new_sz, &new_sz, + false, NULL); + if (err) + goto out; + + le = NULL; + attr = ni_find_attr(ni, NULL, &le, ATTR_EA_INFO, NULL, 0, NULL, &mi); + if (!attr) { + err = -EINVAL; + goto out; + } + + if (!size) { + /* delete xattr, ATTR_EA_INFO */ + err = ni_remove_attr_le(ni, attr, le); + if (err) + goto out; + } else { + p = resident_data_ex(attr, sizeof(struct EA_INFO)); + if (!p) { + err = -EINVAL; + goto out; + } + memcpy(p, &ea_info, sizeof(struct EA_INFO)); + mi->dirty = true; + } + + le = NULL; + attr = ni_find_attr(ni, NULL, &le, ATTR_EA, NULL, 0, NULL, &mi); + if (!attr) { + err = -EINVAL; + goto out; + } + + if (!size) { + /* delete xattr, ATTR_EA */ + err = ni_remove_attr_le(ni, attr, le); + if (err) + goto out; + } else if (attr->non_res) { + err = ntfs_sb_write_run(sbi, &ea_run, 0, ea_all, size); + if (err) + goto out; + } else { + p = resident_data_ex(attr, size); + if (!p) { + err = -EINVAL; + goto out; + } + memcpy(p, ea_all, size); + mi->dirty = true; + } + + /* Check if we delete the last xattr */ + if (size) + ni->ni_flags |= NI_FLAG_EA; + else + ni->ni_flags &= ~NI_FLAG_EA; + + if (ea_info.size_pack != size_pack) + ni->ni_flags |= NI_FLAG_UPDATE_PARENT; + mark_inode_dirty(&ni->vfs_inode); + +out: + if (!locked) + ni_unlock(ni); + + run_close(&ea_run); + ntfs_free(ea_all); + + return err; +} + +#ifdef CONFIG_NTFS3_FS_POSIX_ACL +static inline void ntfs_posix_acl_release(struct posix_acl *acl) +{ + if (acl && refcount_dec_and_test(&acl->a_refcount)) + kfree(acl); +} + +static struct posix_acl *ntfs_get_acl_ex(struct user_namespace *mnt_userns, + struct inode *inode, int type, + int locked) +{ + struct ntfs_inode *ni = ntfs_i(inode); + const char *name; + size_t name_len; + struct posix_acl *acl; + size_t req; + int err; + void *buf; + + /* allocate PATH_MAX bytes */ + buf = __getname(); + if (!buf) + return ERR_PTR(-ENOMEM); + + /* Possible values of 'type' was already checked above */ + if (type == ACL_TYPE_ACCESS) { + name = XATTR_NAME_POSIX_ACL_ACCESS; + name_len = sizeof(XATTR_NAME_POSIX_ACL_ACCESS) - 1; + } else { + name = XATTR_NAME_POSIX_ACL_DEFAULT; + name_len = sizeof(XATTR_NAME_POSIX_ACL_DEFAULT) - 1; + } + + if (!locked) + ni_lock(ni); + + err = ntfs_get_ea(inode, name, name_len, buf, PATH_MAX, &req); + + if (!locked) + ni_unlock(ni); + + /* Translate extended attribute to acl */ + if (err > 0) { + acl = posix_acl_from_xattr(mnt_userns, buf, err); + if (!IS_ERR(acl)) + set_cached_acl(inode, type, acl); + } else { + acl = err == -ENODATA ? NULL : ERR_PTR(err); + } + + __putname(buf); + + return acl; +} + +/* + * ntfs_get_acl + * + * inode_operations::get_acl + */ +struct posix_acl *ntfs_get_acl(struct inode *inode, int type) +{ + /* TODO: init_user_ns? */ + return ntfs_get_acl_ex(&init_user_ns, inode, type, 0); +} + +static noinline int ntfs_set_acl_ex(struct user_namespace *mnt_userns, + struct inode *inode, struct posix_acl *acl, + int type, int locked) +{ + const char *name; + size_t size, name_len; + void *value = NULL; + int err = 0; + + if (S_ISLNK(inode->i_mode)) + return -EOPNOTSUPP; + + switch (type) { + case ACL_TYPE_ACCESS: + if (acl) { + umode_t mode = inode->i_mode; + + err = posix_acl_equiv_mode(acl, &mode); + if (err < 0) + return err; + + if (inode->i_mode != mode) { + inode->i_mode = mode; + mark_inode_dirty(inode); + } + + if (!err) { + /* + * acl can be exactly represented in the + * traditional file mode permission bits + */ + acl = NULL; + goto out; + } + } + name = XATTR_NAME_POSIX_ACL_ACCESS; + name_len = sizeof(XATTR_NAME_POSIX_ACL_ACCESS) - 1; + break; + + case ACL_TYPE_DEFAULT: + if (!S_ISDIR(inode->i_mode)) + return acl ? -EACCES : 0; + name = XATTR_NAME_POSIX_ACL_DEFAULT; + name_len = sizeof(XATTR_NAME_POSIX_ACL_DEFAULT) - 1; + break; + + default: + return -EINVAL; + } + + if (!acl) + goto out; + + size = posix_acl_xattr_size(acl->a_count); + value = ntfs_malloc(size); + if (!value) + return -ENOMEM; + + err = posix_acl_to_xattr(mnt_userns, acl, value, size); + if (err) + goto out; + + err = ntfs_set_ea(inode, name, name_len, value, size, 0, locked); + if (err) + goto out; + + inode->i_flags &= ~S_NOSEC; + +out: + if (!err) + set_cached_acl(inode, type, acl); + + kfree(value); + + return err; +} + +/* + * ntfs_set_acl + * + * inode_operations::set_acl + */ +int ntfs_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type) +{ + return ntfs_set_acl_ex(mnt_userns, inode, acl, type, 0); +} + +static int ntfs_xattr_get_acl(struct user_namespace *mnt_userns, + struct inode *inode, int type, void *buffer, + size_t size) +{ + struct posix_acl *acl; + int err; + + if (!(inode->i_sb->s_flags & SB_POSIXACL)) + return -EOPNOTSUPP; + + acl = ntfs_get_acl(inode, type); + if (IS_ERR(acl)) + return PTR_ERR(acl); + + if (!acl) + return -ENODATA; + + err = posix_acl_to_xattr(mnt_userns, acl, buffer, size); + ntfs_posix_acl_release(acl); + + return err; +} + +static int ntfs_xattr_set_acl(struct user_namespace *mnt_userns, + struct inode *inode, int type, const void *value, + size_t size) +{ + struct posix_acl *acl; + int err; + + if (!(inode->i_sb->s_flags & SB_POSIXACL)) + return -EOPNOTSUPP; + + if (!inode_owner_or_capable(mnt_userns, inode)) + return -EPERM; + + if (!value) + return 0; + + acl = posix_acl_from_xattr(mnt_userns, value, size); + if (IS_ERR(acl)) + return PTR_ERR(acl); + + if (acl) { + err = posix_acl_valid(mnt_userns, acl); + if (err) + goto release_and_out; + } + + err = ntfs_set_acl(mnt_userns, inode, acl, type); + +release_and_out: + ntfs_posix_acl_release(acl); + return err; +} + +/* + * Initialize the ACLs of a new inode. Called from ntfs_create_inode. + */ +int ntfs_init_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct inode *dir) +{ + struct posix_acl *default_acl, *acl; + int err; + + /* + * TODO refactoring lock + * ni_lock(dir) ... -> posix_acl_create(dir,...) -> ntfs_get_acl -> ni_lock(dir) + */ + inode->i_default_acl = NULL; + + default_acl = ntfs_get_acl_ex(mnt_userns, dir, ACL_TYPE_DEFAULT, 1); + + if (!default_acl || default_acl == ERR_PTR(-EOPNOTSUPP)) { + inode->i_mode &= ~current_umask(); + err = 0; + goto out; + } + + if (IS_ERR(default_acl)) { + err = PTR_ERR(default_acl); + goto out; + } + + acl = default_acl; + err = __posix_acl_create(&acl, GFP_NOFS, &inode->i_mode); + if (err < 0) + goto out1; + if (!err) { + posix_acl_release(acl); + acl = NULL; + } + + if (!S_ISDIR(inode->i_mode)) { + posix_acl_release(default_acl); + default_acl = NULL; + } + + if (default_acl) + err = ntfs_set_acl_ex(mnt_userns, inode, default_acl, + ACL_TYPE_DEFAULT, 1); + + if (!acl) + inode->i_acl = NULL; + else if (!err) + err = ntfs_set_acl_ex(mnt_userns, inode, acl, ACL_TYPE_ACCESS, + 1); + + posix_acl_release(acl); +out1: + posix_acl_release(default_acl); + +out: + return err; +} +#endif + +/* + * ntfs_acl_chmod + * + * helper for 'ntfs3_setattr' + */ +int ntfs_acl_chmod(struct user_namespace *mnt_userns, struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + + if (!(sb->s_flags & SB_POSIXACL)) + return 0; + + if (S_ISLNK(inode->i_mode)) + return -EOPNOTSUPP; + + return posix_acl_chmod(mnt_userns, inode, inode->i_mode); +} + +/* + * ntfs_permission + * + * inode_operations::permission + */ +int ntfs_permission(struct user_namespace *mnt_userns, struct inode *inode, + int mask) +{ + if (ntfs_sb(inode->i_sb)->options.no_acs_rules) { + /* "no access rules" mode - allow all changes */ + return 0; + } + + return generic_permission(mnt_userns, inode, mask); +} + +/* + * ntfs_listxattr + * + * inode_operations::listxattr + */ +ssize_t ntfs_listxattr(struct dentry *dentry, char *buffer, size_t size) +{ + struct inode *inode = d_inode(dentry); + struct ntfs_inode *ni = ntfs_i(inode); + ssize_t ret; + + if (!(ni->ni_flags & NI_FLAG_EA)) { + /* no xattr in file */ + return 0; + } + + ni_lock(ni); + + ret = ntfs_list_ea(ni, buffer, size); + + ni_unlock(ni); + + return ret; +} + +static int ntfs_getxattr(const struct xattr_handler *handler, struct dentry *de, + struct inode *inode, const char *name, void *buffer, + size_t size) +{ + int err; + struct ntfs_inode *ni = ntfs_i(inode); + size_t name_len = strlen(name); + + /* Dispatch request */ + if (name_len == sizeof(SYSTEM_DOS_ATTRIB) - 1 && + !memcmp(name, SYSTEM_DOS_ATTRIB, sizeof(SYSTEM_DOS_ATTRIB))) { + /* system.dos_attrib */ + if (!buffer) { + err = sizeof(u8); + } else if (size < sizeof(u8)) { + err = -ENODATA; + } else { + err = sizeof(u8); + *(u8 *)buffer = le32_to_cpu(ni->std_fa); + } + goto out; + } + + if (name_len == sizeof(SYSTEM_NTFS_ATTRIB) - 1 && + !memcmp(name, SYSTEM_NTFS_ATTRIB, sizeof(SYSTEM_NTFS_ATTRIB))) { + /* system.ntfs_attrib */ + if (!buffer) { + err = sizeof(u32); + } else if (size < sizeof(u32)) { + err = -ENODATA; + } else { + err = sizeof(u32); + *(u32 *)buffer = le32_to_cpu(ni->std_fa); + } + goto out; + } + + if (name_len == sizeof(SYSTEM_NTFS_SECURITY) - 1 && + !memcmp(name, SYSTEM_NTFS_SECURITY, sizeof(SYSTEM_NTFS_SECURITY))) { + /* system.ntfs_security*/ + struct SECURITY_DESCRIPTOR_RELATIVE *sd = NULL; + size_t sd_size = 0; + + if (!is_ntfs3(ni->mi.sbi)) { + /* we should get nt4 security */ + err = -EINVAL; + goto out; + } else if (le32_to_cpu(ni->std_security_id) < + SECURITY_ID_FIRST) { + err = -ENOENT; + goto out; + } + + err = ntfs_get_security_by_id(ni->mi.sbi, ni->std_security_id, + &sd, &sd_size); + if (err) + goto out; + + if (!is_sd_valid(sd, sd_size)) { + ntfs_inode_warn( + inode, + "looks like you get incorrect security descriptor id=%u", + ni->std_security_id); + } + + if (!buffer) { + err = sd_size; + } else if (size < sd_size) { + err = -ENODATA; + } else { + err = sd_size; + memcpy(buffer, sd, sd_size); + } + ntfs_free(sd); + goto out; + } + +#ifdef CONFIG_NTFS3_FS_POSIX_ACL + if ((name_len == sizeof(XATTR_NAME_POSIX_ACL_ACCESS) - 1 && + !memcmp(name, XATTR_NAME_POSIX_ACL_ACCESS, + sizeof(XATTR_NAME_POSIX_ACL_ACCESS))) || + (name_len == sizeof(XATTR_NAME_POSIX_ACL_DEFAULT) - 1 && + !memcmp(name, XATTR_NAME_POSIX_ACL_DEFAULT, + sizeof(XATTR_NAME_POSIX_ACL_DEFAULT)))) { + /* TODO: init_user_ns? */ + err = ntfs_xattr_get_acl( + &init_user_ns, inode, + name_len == sizeof(XATTR_NAME_POSIX_ACL_ACCESS) - 1 + ? ACL_TYPE_ACCESS + : ACL_TYPE_DEFAULT, + buffer, size); + goto out; + } +#endif + /* deal with ntfs extended attribute */ + err = ntfs_get_ea(inode, name, name_len, buffer, size, NULL); + +out: + return err; +} + +/* + * ntfs_setxattr + * + * inode_operations::setxattr + */ +static noinline int ntfs_setxattr(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, + struct dentry *de, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +{ + int err = -EINVAL; + struct ntfs_inode *ni = ntfs_i(inode); + size_t name_len = strlen(name); + enum FILE_ATTRIBUTE new_fa; + + /* Dispatch request */ + if (name_len == sizeof(SYSTEM_DOS_ATTRIB) - 1 && + !memcmp(name, SYSTEM_DOS_ATTRIB, sizeof(SYSTEM_DOS_ATTRIB))) { + if (sizeof(u8) != size) + goto out; + new_fa = cpu_to_le32(*(u8 *)value); + goto set_new_fa; + } + + if (name_len == sizeof(SYSTEM_NTFS_ATTRIB) - 1 && + !memcmp(name, SYSTEM_NTFS_ATTRIB, sizeof(SYSTEM_NTFS_ATTRIB))) { + if (size != sizeof(u32)) + goto out; + new_fa = cpu_to_le32(*(u32 *)value); + + if (S_ISREG(inode->i_mode)) { + /* Process compressed/sparsed in special way*/ + ni_lock(ni); + err = ni_new_attr_flags(ni, new_fa); + ni_unlock(ni); + if (err) + goto out; + } +set_new_fa: + /* + * Thanks Mark Harmstone: + * keep directory bit consistency + */ + if (S_ISDIR(inode->i_mode)) + new_fa |= FILE_ATTRIBUTE_DIRECTORY; + else + new_fa &= ~FILE_ATTRIBUTE_DIRECTORY; + + if (ni->std_fa != new_fa) { + ni->std_fa = new_fa; + if (new_fa & FILE_ATTRIBUTE_READONLY) + inode->i_mode &= ~0222; + else + inode->i_mode |= 0222; + /* std attribute always in primary record */ + ni->mi.dirty = true; + mark_inode_dirty(inode); + } + err = 0; + + goto out; + } + + if (name_len == sizeof(SYSTEM_NTFS_SECURITY) - 1 && + !memcmp(name, SYSTEM_NTFS_SECURITY, sizeof(SYSTEM_NTFS_SECURITY))) { + /* system.ntfs_security*/ + __le32 security_id; + bool inserted; + struct ATTR_STD_INFO5 *std; + + if (!is_ntfs3(ni->mi.sbi)) { + /* + * we should replace ATTR_SECURE + * Skip this way cause it is nt4 feature + */ + err = -EINVAL; + goto out; + } + + if (!is_sd_valid(value, size)) { + err = -EINVAL; + ntfs_inode_warn( + inode, + "you try to set invalid security descriptor"); + goto out; + } + + err = ntfs_insert_security(ni->mi.sbi, value, size, + &security_id, &inserted); + if (err) + goto out; + + ni_lock(ni); + std = ni_std5(ni); + if (!std) { + err = -EINVAL; + } else if (std->security_id != security_id) { + std->security_id = ni->std_security_id = security_id; + /* std attribute always in primary record */ + ni->mi.dirty = true; + mark_inode_dirty(&ni->vfs_inode); + } + ni_unlock(ni); + goto out; + } + +#ifdef CONFIG_NTFS3_FS_POSIX_ACL + if ((name_len == sizeof(XATTR_NAME_POSIX_ACL_ACCESS) - 1 && + !memcmp(name, XATTR_NAME_POSIX_ACL_ACCESS, + sizeof(XATTR_NAME_POSIX_ACL_ACCESS))) || + (name_len == sizeof(XATTR_NAME_POSIX_ACL_DEFAULT) - 1 && + !memcmp(name, XATTR_NAME_POSIX_ACL_DEFAULT, + sizeof(XATTR_NAME_POSIX_ACL_DEFAULT)))) { + /* TODO: init_user_ns? */ + err = ntfs_xattr_set_acl( + &init_user_ns, inode, + name_len == sizeof(XATTR_NAME_POSIX_ACL_ACCESS) - 1 + ? ACL_TYPE_ACCESS + : ACL_TYPE_DEFAULT, + value, size); + goto out; + } +#endif + /* deal with ntfs extended attribute */ + err = ntfs_set_ea(inode, name, name_len, value, size, flags, 0); + +out: + return err; +} + +static bool ntfs_xattr_user_list(struct dentry *dentry) +{ + return true; +} + +static const struct xattr_handler ntfs_xattr_handler = { + .prefix = "", + .get = ntfs_getxattr, + .set = ntfs_setxattr, + .list = ntfs_xattr_user_list, +}; + +const struct xattr_handler *ntfs_xattr_handlers[] = { + &ntfs_xattr_handler, + NULL, +}; diff --git a/fs/overlayfs/copy_up.c b/fs/overlayfs/copy_up.c index 0b2891c6c71e..2846b943e80c 100644 --- a/fs/overlayfs/copy_up.c +++ b/fs/overlayfs/copy_up.c @@ -932,7 +932,7 @@ static int ovl_copy_up_one(struct dentry *parent, struct dentry *dentry, static int ovl_copy_up_flags(struct dentry *dentry, int flags) { int err = 0; - const struct cred *old_cred = ovl_override_creds(dentry->d_sb); + const struct cred *old_cred; bool disconnected = (dentry->d_flags & DCACHE_DISCONNECTED); /* @@ -943,6 +943,7 @@ static int ovl_copy_up_flags(struct dentry *dentry, int flags) if (WARN_ON(disconnected && d_is_dir(dentry))) return -EIO; + old_cred = ovl_override_creds(dentry->d_sb); while (!err) { struct dentry *next; struct dentry *parent = NULL; diff --git a/fs/overlayfs/namei.c b/fs/overlayfs/namei.c index 3fe05fb5d145..71e264e2f16b 100644 --- a/fs/overlayfs/namei.c +++ b/fs/overlayfs/namei.c @@ -919,6 +919,7 @@ struct dentry *ovl_lookup(struct inode *dir, struct dentry *dentry, continue; if ((uppermetacopy || d.metacopy) && !ofs->config.metacopy) { + dput(this); err = -EPERM; pr_warn_ratelimited("refusing to follow metacopy origin for (%pd2)\n", dentry); goto out_put; diff --git a/fs/overlayfs/overlayfs.h b/fs/overlayfs/overlayfs.h index 95cff83786a5..2322f854533c 100644 --- a/fs/overlayfs/overlayfs.h +++ b/fs/overlayfs/overlayfs.h @@ -319,9 +319,6 @@ int ovl_check_setxattr(struct dentry *dentry, struct dentry *upperdentry, enum ovl_xattr ox, const void *value, size_t size, int xerr); int ovl_set_impure(struct dentry *dentry, struct dentry *upperdentry); -void ovl_set_flag(unsigned long flag, struct inode *inode); -void ovl_clear_flag(unsigned long flag, struct inode *inode); -bool ovl_test_flag(unsigned long flag, struct inode *inode); bool ovl_inuse_trylock(struct dentry *dentry); void ovl_inuse_unlock(struct dentry *dentry); bool ovl_is_inuse(struct dentry *dentry); @@ -335,6 +332,21 @@ char *ovl_get_redirect_xattr(struct ovl_fs *ofs, struct dentry *dentry, int padding); int ovl_sync_status(struct ovl_fs *ofs); +static inline void ovl_set_flag(unsigned long flag, struct inode *inode) +{ + set_bit(flag, &OVL_I(inode)->flags); +} + +static inline void ovl_clear_flag(unsigned long flag, struct inode *inode) +{ + clear_bit(flag, &OVL_I(inode)->flags); +} + +static inline bool ovl_test_flag(unsigned long flag, struct inode *inode) +{ + return test_bit(flag, &OVL_I(inode)->flags); +} + static inline bool ovl_is_impuredir(struct super_block *sb, struct dentry *dentry) { @@ -439,6 +451,18 @@ int ovl_workdir_cleanup(struct inode *dir, struct vfsmount *mnt, struct dentry *dentry, int level); int ovl_indexdir_cleanup(struct ovl_fs *ofs); +/* + * Can we iterate real dir directly? + * + * Non-merge dir may contain whiteouts from a time it was a merge upper, before + * lower dir was removed under it and possibly before it was rotated from upper + * to lower layer. + */ +static inline bool ovl_dir_is_real(struct dentry *dir) +{ + return !ovl_test_flag(OVL_WHITEOUTS, d_inode(dir)); +} + /* inode.c */ int ovl_set_nlink_upper(struct dentry *dentry); int ovl_set_nlink_lower(struct dentry *dentry); diff --git a/fs/overlayfs/readdir.c b/fs/overlayfs/readdir.c index f404a78e6b60..cc1e80257064 100644 --- a/fs/overlayfs/readdir.c +++ b/fs/overlayfs/readdir.c @@ -319,18 +319,6 @@ static inline int ovl_dir_read(struct path *realpath, return err; } -/* - * Can we iterate real dir directly? - * - * Non-merge dir may contain whiteouts from a time it was a merge upper, before - * lower dir was removed under it and possibly before it was rotated from upper - * to lower layer. - */ -static bool ovl_dir_is_real(struct dentry *dir) -{ - return !ovl_test_flag(OVL_WHITEOUTS, d_inode(dir)); -} - static void ovl_dir_reset(struct file *file) { struct ovl_dir_file *od = file->private_data; diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index fdd72f1a9c5e..787ce7c38fba 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -380,6 +380,8 @@ static int ovl_show_options(struct seq_file *m, struct dentry *dentry) ofs->config.metacopy ? "on" : "off"); if (ofs->config.ovl_volatile) seq_puts(m, ",volatile"); + if (ofs->config.userxattr) + seq_puts(m, ",userxattr"); return 0; } @@ -1826,7 +1828,8 @@ static struct ovl_entry *ovl_get_lowerstack(struct super_block *sb, * - upper/work dir of any overlayfs instance */ static int ovl_check_layer(struct super_block *sb, struct ovl_fs *ofs, - struct dentry *dentry, const char *name) + struct dentry *dentry, const char *name, + bool is_lower) { struct dentry *next = dentry, *parent; int err = 0; @@ -1838,7 +1841,7 @@ static int ovl_check_layer(struct super_block *sb, struct ovl_fs *ofs, /* Walk back ancestors to root (inclusive) looking for traps */ while (!err && parent != next) { - if (ovl_lookup_trap_inode(sb, parent)) { + if (is_lower && ovl_lookup_trap_inode(sb, parent)) { err = -ELOOP; pr_err("overlapping %s path\n", name); } else if (ovl_is_inuse(parent)) { @@ -1864,7 +1867,7 @@ static int ovl_check_overlapping_layers(struct super_block *sb, if (ovl_upper_mnt(ofs)) { err = ovl_check_layer(sb, ofs, ovl_upper_mnt(ofs)->mnt_root, - "upperdir"); + "upperdir", false); if (err) return err; @@ -1875,7 +1878,8 @@ static int ovl_check_overlapping_layers(struct super_block *sb, * workbasedir. In that case, we already have their traps in * inode cache and we will catch that case on lookup. */ - err = ovl_check_layer(sb, ofs, ofs->workbasedir, "workdir"); + err = ovl_check_layer(sb, ofs, ofs->workbasedir, "workdir", + false); if (err) return err; } @@ -1883,7 +1887,7 @@ static int ovl_check_overlapping_layers(struct super_block *sb, for (i = 1; i < ofs->numlayer; i++) { err = ovl_check_layer(sb, ofs, ofs->layers[i].mnt->mnt_root, - "lowerdir"); + "lowerdir", true); if (err) return err; } diff --git a/fs/overlayfs/util.c b/fs/overlayfs/util.c index 7f5a01a11f97..404a0a32ddf6 100644 --- a/fs/overlayfs/util.c +++ b/fs/overlayfs/util.c @@ -422,18 +422,20 @@ void ovl_inode_update(struct inode *inode, struct dentry *upperdentry) } } -static void ovl_dentry_version_inc(struct dentry *dentry, bool impurity) +static void ovl_dir_version_inc(struct dentry *dentry, bool impurity) { struct inode *inode = d_inode(dentry); WARN_ON(!inode_is_locked(inode)); + WARN_ON(!d_is_dir(dentry)); /* - * Version is used by readdir code to keep cache consistent. For merge - * dirs all changes need to be noted. For non-merge dirs, cache only - * contains impure (ones which have been copied up and have origins) - * entries, so only need to note changes to impure entries. + * Version is used by readdir code to keep cache consistent. + * For merge dirs (or dirs with origin) all changes need to be noted. + * For non-merge dirs, cache contains only impure entries (i.e. ones + * which have been copied up and have origins), so only need to note + * changes to impure entries. */ - if (OVL_TYPE_MERGE(ovl_path_type(dentry)) || impurity) + if (!ovl_dir_is_real(dentry) || impurity) OVL_I(inode)->version++; } @@ -442,7 +444,7 @@ void ovl_dir_modified(struct dentry *dentry, bool impurity) /* Copy mtime/ctime */ ovl_copyattr(d_inode(ovl_dentry_upper(dentry)), d_inode(dentry)); - ovl_dentry_version_inc(dentry, impurity); + ovl_dir_version_inc(dentry, impurity); } u64 ovl_dentry_version_get(struct dentry *dentry) @@ -638,21 +640,6 @@ int ovl_set_impure(struct dentry *dentry, struct dentry *upperdentry) return err; } -void ovl_set_flag(unsigned long flag, struct inode *inode) -{ - set_bit(flag, &OVL_I(inode)->flags); -} - -void ovl_clear_flag(unsigned long flag, struct inode *inode) -{ - clear_bit(flag, &OVL_I(inode)->flags); -} - -bool ovl_test_flag(unsigned long flag, struct inode *inode) -{ - return test_bit(flag, &OVL_I(inode)->flags); -} - /** * Caller must hold a reference to inode to prevent it from being freed while * it is marked inuse. diff --git a/fs/proc/array.c b/fs/proc/array.c index bb87e4d89cd8..7ec59171f197 100644 --- a/fs/proc/array.c +++ b/fs/proc/array.c @@ -342,8 +342,10 @@ static inline void task_seccomp(struct seq_file *m, struct task_struct *p) seq_put_decimal_ull(m, "NoNewPrivs:\t", task_no_new_privs(p)); #ifdef CONFIG_SECCOMP seq_put_decimal_ull(m, "\nSeccomp:\t", p->seccomp.mode); +#ifdef CONFIG_SECCOMP_FILTER seq_put_decimal_ull(m, "\nSeccomp_filters:\t", atomic_read(&p->seccomp.filter_count)); +#endif #endif seq_puts(m, "\nSpeculation_Store_Bypass:\t"); switch (arch_prctl_spec_ctrl_get(p, PR_SPEC_STORE_BYPASS)) { diff --git a/fs/proc/base.c b/fs/proc/base.c index 3851bfcdba56..542fd084a466 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -95,6 +95,8 @@ #include #include #include +#include +#include #include #include "internal.h" #include "fd.h" @@ -3149,6 +3151,144 @@ static int proc_stack_depth(struct seq_file *m, struct pid_namespace *ns, } #endif /* CONFIG_STACKLEAK_METRICS */ +#ifdef CONFIG_KSM +static int ksm_open(struct inode *inode, struct file *file) +{ + struct task_struct *task; + struct mm_struct *mm; + int err; + + task = get_proc_task(inode); + if (!task) { + err = -ESRCH; + goto out; + } + if (task->flags & PF_KTHREAD) { + put_task_struct(task); + err = -EINVAL; + goto out; + } + + mm = mm_access(task, PTRACE_MODE_ATTACH_FSCREDS); + put_task_struct(task); + if (!mm) { + err = -EINVAL; + goto out; + } + if (IS_ERR(mm)) { + err = PTR_ERR(mm); + goto out; + } + + /* ensure this mm_struct can't be freed */ + mmgrab(mm); + /* but do not pin its memory */ + mmput(mm); + + err = 0; + file->private_data = mm; + +out: + return err; +} + +static ssize_t ksm_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + char kbuf[PROC_NUMBUF]; + char *pos; + int behaviour; + struct mm_struct *mm = file->private_data; + int err; + int last_err; + struct vm_area_struct *vma; + + if (!mm) { + err = -EINVAL; + goto out; + } + + /* Only allow a very narrow range of strings to be written */ + if ((*ppos != 0) || (count >= sizeof(kbuf))) { + err = -EINVAL; + goto out; + } + + /* What was written? */ + if (copy_from_user(kbuf, buf, count)) { + err = -EFAULT; + goto out; + } + kbuf[count] = '\0'; + pos = kbuf; + + /* What is being requested? */ + if (strncmp(pos, "merge", 5) == 0) { + pos += 5; + behaviour = MADV_MERGEABLE; + } + else if (strncmp(pos, "unmerge", 7) == 0) { + pos += 7; + behaviour = MADV_UNMERGEABLE; + } + else { + err = -EINVAL; + goto out; + } + + /* Verify there is not trailing junk on the line */ + pos = skip_spaces(pos); + if (*pos != '\0') { + err = -EINVAL; + goto out; + } + + if (!mmget_not_zero(mm)) { + err = -EINVAL; + goto out; + } + + mmap_write_lock(mm); + + err = 0; + + vma = mm->mmap; + while (vma) { + if (behaviour == MADV_MERGEABLE) + last_err = ksm_madvise_merge(vma->vm_mm, vma, &vma->vm_flags); + else + last_err = ksm_madvise_unmerge(vma, vma->vm_start, vma->vm_end, &vma->vm_flags); + if (last_err) + err = last_err; + vma = vma->vm_next; + } + + mmap_write_unlock(mm); + + mmput(mm); + +out: + return err ? err : count; +} + +static int ksm_release(struct inode *inode, struct file *file) +{ + struct mm_struct *mm = file->private_data; + + if (mm) + mmdrop(mm); + + return 0; +} + +static const struct file_operations proc_ksm_operations = { + .open = ksm_open, + .write = ksm_write, + .llseek = noop_llseek, + .release = ksm_release, +}; +#endif /* CONFIG_KSM */ + /* * Thread groups */ @@ -3265,6 +3405,9 @@ static const struct pid_entry tgid_base_stuff[] = { #ifdef CONFIG_SECCOMP_CACHE_DEBUG ONE("seccomp_cache", S_IRUSR, proc_pid_seccomp_cache), #endif +#ifdef CONFIG_KSM + REG("ksm", S_IRUGO|S_IWUSR, proc_ksm_operations), +#endif }; static int proc_tgid_base_readdir(struct file *file, struct dir_context *ctx) diff --git a/fs/proc/generic.c b/fs/proc/generic.c index bc86aa87cc41..5600da30e289 100644 --- a/fs/proc/generic.c +++ b/fs/proc/generic.c @@ -756,7 +756,7 @@ int remove_proc_subtree(const char *name, struct proc_dir_entry *parent) while (1) { next = pde_subdir_first(de); if (next) { - if (unlikely(pde_is_permanent(root))) { + if (unlikely(pde_is_permanent(next))) { write_unlock(&proc_subdir_lock); WARN(1, "removing permanent /proc entry '%s/%s'", next->parent->name, next->name); diff --git a/fs/pstore/platform.c b/fs/pstore/platform.c index d963ae7902f9..67b194ba1b03 100644 --- a/fs/pstore/platform.c +++ b/fs/pstore/platform.c @@ -218,7 +218,7 @@ static int zbufsize_842(size_t size) #if IS_ENABLED(CONFIG_PSTORE_ZSTD_COMPRESS) static int zbufsize_zstd(size_t size) { - return ZSTD_compressBound(size); + return zstd_compress_bound(size); } #endif diff --git a/fs/squashfs/file.c b/fs/squashfs/file.c index 7b1128398976..89d492916dea 100644 --- a/fs/squashfs/file.c +++ b/fs/squashfs/file.c @@ -211,11 +211,11 @@ static long long read_indexes(struct super_block *sb, int n, * If the skip factor is limited in this way then the file will use multiple * slots. */ -static inline int calculate_skip(int blocks) +static inline int calculate_skip(u64 blocks) { - int skip = blocks / ((SQUASHFS_META_ENTRIES + 1) + u64 skip = blocks / ((SQUASHFS_META_ENTRIES + 1) * SQUASHFS_META_INDEXES); - return min(SQUASHFS_CACHED_BLKS - 1, skip + 1); + return min((u64) SQUASHFS_CACHED_BLKS - 1, skip + 1); } diff --git a/fs/squashfs/zstd_wrapper.c b/fs/squashfs/zstd_wrapper.c index b7cb1faa652d..6967c0aae801 100644 --- a/fs/squashfs/zstd_wrapper.c +++ b/fs/squashfs/zstd_wrapper.c @@ -34,7 +34,7 @@ static void *zstd_init(struct squashfs_sb_info *msblk, void *buff) goto failed; wksp->window_size = max_t(size_t, msblk->block_size, SQUASHFS_METADATA_SIZE); - wksp->mem_size = ZSTD_DStreamWorkspaceBound(wksp->window_size); + wksp->mem_size = zstd_dstream_workspace_bound(wksp->window_size); wksp->mem = vmalloc(wksp->mem_size); if (wksp->mem == NULL) goto failed; @@ -63,15 +63,15 @@ static int zstd_uncompress(struct squashfs_sb_info *msblk, void *strm, struct squashfs_page_actor *output) { struct workspace *wksp = strm; - ZSTD_DStream *stream; + zstd_dstream *stream; size_t total_out = 0; int error = 0; - ZSTD_inBuffer in_buf = { NULL, 0, 0 }; - ZSTD_outBuffer out_buf = { NULL, 0, 0 }; + zstd_in_buffer in_buf = { NULL, 0, 0 }; + zstd_out_buffer out_buf = { NULL, 0, 0 }; struct bvec_iter_all iter_all = {}; struct bio_vec *bvec = bvec_init_iter_all(&iter_all); - stream = ZSTD_initDStream(wksp->window_size, wksp->mem, wksp->mem_size); + stream = zstd_init_dstream(wksp->window_size, wksp->mem, wksp->mem_size); if (!stream) { ERROR("Failed to initialize zstd decompressor\n"); @@ -116,14 +116,14 @@ static int zstd_uncompress(struct squashfs_sb_info *msblk, void *strm, } total_out -= out_buf.pos; - zstd_err = ZSTD_decompressStream(stream, &out_buf, &in_buf); + zstd_err = zstd_decompress_stream(stream, &out_buf, &in_buf); total_out += out_buf.pos; /* add the additional data produced */ if (zstd_err == 0) break; - if (ZSTD_isError(zstd_err)) { + if (zstd_is_error(zstd_err)) { ERROR("zstd decompression error: %d\n", - (int)ZSTD_getErrorCode(zstd_err)); + (int)zstd_get_error_code(zstd_err)); error = -EIO; break; } diff --git a/fs/stat.c b/fs/stat.c index fbc171d038aa..1fa38bdec1a6 100644 --- a/fs/stat.c +++ b/fs/stat.c @@ -86,12 +86,20 @@ int vfs_getattr_nosec(const struct path *path, struct kstat *stat, /* SB_NOATIME means filesystem supplies dummy atime value */ if (inode->i_sb->s_flags & SB_NOATIME) stat->result_mask &= ~STATX_ATIME; + + /* + * Note: If you add another clause to set an attribute flag, please + * update attributes_mask below. + */ if (IS_AUTOMOUNT(inode)) stat->attributes |= STATX_ATTR_AUTOMOUNT; if (IS_DAX(inode)) stat->attributes |= STATX_ATTR_DAX; + stat->attributes_mask |= (STATX_ATTR_AUTOMOUNT | + STATX_ATTR_DAX); + mnt_userns = mnt_user_ns(path->mnt); if (inode->i_op->getattr) return inode->i_op->getattr(mnt_userns, path, stat, diff --git a/fs/ubifs/replay.c b/fs/ubifs/replay.c index 0f8a6a16421b..1929ec63a0cb 100644 --- a/fs/ubifs/replay.c +++ b/fs/ubifs/replay.c @@ -223,7 +223,8 @@ static bool inode_still_linked(struct ubifs_info *c, struct replay_entry *rino) */ list_for_each_entry_reverse(r, &c->replay_list, list) { ubifs_assert(c, r->sqnum >= rino->sqnum); - if (key_inum(c, &r->key) == key_inum(c, &rino->key)) + if (key_inum(c, &r->key) == key_inum(c, &rino->key) && + key_type(c, &r->key) == UBIFS_INO_KEY) return r->deletion == 0; } diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index 472b3039eabb..902e5f7e6642 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -928,6 +928,7 @@ xfs_attr_node_addname( * Search to see if name already exists, and get back a pointer * to where it should go. */ + error = 0; retval = xfs_attr_node_hasname(args, &state); if (retval != -ENOATTR && retval != -EEXIST) goto out; diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index fcde59c65a81..cb3d6b1c655d 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -165,6 +165,8 @@ static inline struct crypto_acomp *crypto_acomp_reqtfm(struct acomp_req *req) * crypto_free_acomp() -- free ACOMPRESS tfm handle * * @tfm: ACOMPRESS tfm handle allocated with crypto_alloc_acomp() + * + * If @tfm is a NULL or error pointer, this function does nothing. */ static inline void crypto_free_acomp(struct crypto_acomp *tfm) { diff --git a/include/crypto/aead.h b/include/crypto/aead.h index fcc12c593ef8..e728469c4ccc 100644 --- a/include/crypto/aead.h +++ b/include/crypto/aead.h @@ -185,6 +185,8 @@ static inline struct crypto_tfm *crypto_aead_tfm(struct crypto_aead *tfm) /** * crypto_free_aead() - zeroize and free aead handle * @tfm: cipher handle to be freed + * + * If @tfm is a NULL or error pointer, this function does nothing. */ static inline void crypto_free_aead(struct crypto_aead *tfm) { diff --git a/include/crypto/akcipher.h b/include/crypto/akcipher.h index 1d3aa252caba..5764b46bd1ec 100644 --- a/include/crypto/akcipher.h +++ b/include/crypto/akcipher.h @@ -174,6 +174,8 @@ static inline struct crypto_akcipher *crypto_akcipher_reqtfm( * crypto_free_akcipher() - free AKCIPHER tfm handle * * @tfm: AKCIPHER tfm handle allocated with crypto_alloc_akcipher() + * + * If @tfm is a NULL or error pointer, this function does nothing. */ static inline void crypto_free_akcipher(struct crypto_akcipher *tfm) { diff --git a/include/crypto/chacha.h b/include/crypto/chacha.h index 3a1c72fdb7cf..dabaee698718 100644 --- a/include/crypto/chacha.h +++ b/include/crypto/chacha.h @@ -47,13 +47,18 @@ static inline void hchacha_block(const u32 *state, u32 *out, int nrounds) hchacha_block_generic(state, out, nrounds); } -void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv); -static inline void chacha_init_generic(u32 *state, const u32 *key, const u8 *iv) +static inline void chacha_init_consts(u32 *state) { state[0] = 0x61707865; /* "expa" */ state[1] = 0x3320646e; /* "nd 3" */ state[2] = 0x79622d32; /* "2-by" */ state[3] = 0x6b206574; /* "te k" */ +} + +void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv); +static inline void chacha_init_generic(u32 *state, const u32 *key, const u8 *iv) +{ + chacha_init_consts(state); state[4] = key[0]; state[5] = key[1]; state[6] = key[2]; diff --git a/include/crypto/hash.h b/include/crypto/hash.h index 13f8a6a54ca8..b2bc1e46e86a 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -281,6 +281,8 @@ static inline struct crypto_tfm *crypto_ahash_tfm(struct crypto_ahash *tfm) /** * crypto_free_ahash() - zeroize and free the ahash handle * @tfm: cipher handle to be freed + * + * If @tfm is a NULL or error pointer, this function does nothing. */ static inline void crypto_free_ahash(struct crypto_ahash *tfm) { @@ -724,6 +726,8 @@ static inline struct crypto_tfm *crypto_shash_tfm(struct crypto_shash *tfm) /** * crypto_free_shash() - zeroize and free the message digest handle * @tfm: cipher handle to be freed + * + * If @tfm is a NULL or error pointer, this function does nothing. */ static inline void crypto_free_shash(struct crypto_shash *tfm) { diff --git a/include/crypto/internal/poly1305.h b/include/crypto/internal/poly1305.h index 064e52ca5248..196aa769f296 100644 --- a/include/crypto/internal/poly1305.h +++ b/include/crypto/internal/poly1305.h @@ -18,7 +18,8 @@ * only the ε-almost-∆-universal hash function (not the full MAC) is computed. */ -void poly1305_core_setkey(struct poly1305_core_key *key, const u8 *raw_key); +void poly1305_core_setkey(struct poly1305_core_key *key, + const u8 raw_key[POLY1305_BLOCK_SIZE]); static inline void poly1305_core_init(struct poly1305_state *state) { *state = (struct poly1305_state){}; diff --git a/include/crypto/kpp.h b/include/crypto/kpp.h index 88b591215d5c..cccceadc164b 100644 --- a/include/crypto/kpp.h +++ b/include/crypto/kpp.h @@ -154,6 +154,8 @@ static inline void crypto_kpp_set_flags(struct crypto_kpp *tfm, u32 flags) * crypto_free_kpp() - free KPP tfm handle * * @tfm: KPP tfm handle allocated with crypto_alloc_kpp() + * + * If @tfm is a NULL or error pointer, this function does nothing. */ static inline void crypto_free_kpp(struct crypto_kpp *tfm) { diff --git a/include/crypto/poly1305.h b/include/crypto/poly1305.h index f1f67fc749cf..090692ec3bc7 100644 --- a/include/crypto/poly1305.h +++ b/include/crypto/poly1305.h @@ -58,8 +58,10 @@ struct poly1305_desc_ctx { }; }; -void poly1305_init_arch(struct poly1305_desc_ctx *desc, const u8 *key); -void poly1305_init_generic(struct poly1305_desc_ctx *desc, const u8 *key); +void poly1305_init_arch(struct poly1305_desc_ctx *desc, + const u8 key[POLY1305_KEY_SIZE]); +void poly1305_init_generic(struct poly1305_desc_ctx *desc, + const u8 key[POLY1305_KEY_SIZE]); static inline void poly1305_init(struct poly1305_desc_ctx *desc, const u8 *key) { diff --git a/include/crypto/rng.h b/include/crypto/rng.h index 8b4b844b4eef..17bb3673d3c1 100644 --- a/include/crypto/rng.h +++ b/include/crypto/rng.h @@ -111,6 +111,8 @@ static inline struct rng_alg *crypto_rng_alg(struct crypto_rng *tfm) /** * crypto_free_rng() - zeroize and free RNG handle * @tfm: cipher handle to be freed + * + * If @tfm is a NULL or error pointer, this function does nothing. */ static inline void crypto_free_rng(struct crypto_rng *tfm) { diff --git a/include/crypto/skcipher.h b/include/crypto/skcipher.h index 6a733b171a5d..ef0fc9ed4342 100644 --- a/include/crypto/skcipher.h +++ b/include/crypto/skcipher.h @@ -196,6 +196,8 @@ static inline struct crypto_tfm *crypto_skcipher_tfm( /** * crypto_free_skcipher() - zeroize and free cipher handle * @tfm: cipher handle to be freed + * + * If @tfm is a NULL or error pointer, this function does nothing. */ static inline void crypto_free_skcipher(struct crypto_skcipher *tfm) { diff --git a/include/keys/trusted-type.h b/include/keys/trusted-type.h index a94c03a61d8f..b2ed3481c6a0 100644 --- a/include/keys/trusted-type.h +++ b/include/keys/trusted-type.h @@ -30,6 +30,7 @@ struct trusted_key_options { uint16_t keytype; uint32_t keyhandle; unsigned char keyauth[TPM_DIGEST_SIZE]; + uint32_t blobauth_len; unsigned char blobauth[TPM_DIGEST_SIZE]; uint32_t pcrinfo_len; unsigned char pcrinfo[MAX_PCRINFO_SIZE]; diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 158aefae1030..0d81eed39833 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -272,6 +272,12 @@ static inline bool bio_is_passthrough(struct bio *bio) return blk_op_is_scsi(op) || blk_op_is_private(op); } +static inline bool blk_op_is_passthrough(unsigned int op) +{ + return (blk_op_is_scsi(op & REQ_OP_MASK) || + blk_op_is_private(op & REQ_OP_MASK)); +} + static inline unsigned short req_get_ioprio(struct request *req) { return req->ioprio; diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 971b33aca13d..99bc82342ca0 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -299,10 +299,11 @@ struct bpf_verifier_state_list { }; /* Possible states for alu_state member. */ -#define BPF_ALU_SANITIZE_SRC 1U -#define BPF_ALU_SANITIZE_DST 2U +#define BPF_ALU_SANITIZE_SRC (1U << 0) +#define BPF_ALU_SANITIZE_DST (1U << 1) #define BPF_ALU_NEG_VALUE (1U << 2) #define BPF_ALU_NON_POINTER (1U << 3) +#define BPF_ALU_IMMEDIATE (1U << 4) #define BPF_ALU_SANITIZE (BPF_ALU_SANITIZE_SRC | \ BPF_ALU_SANITIZE_DST) diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index f14adb882338..cc7c3fda2aa6 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -135,6 +135,7 @@ enum cpuhp_state { CPUHP_AP_RISCV_TIMER_STARTING, CPUHP_AP_CLINT_TIMER_STARTING, CPUHP_AP_CSKY_TIMER_STARTING, + CPUHP_AP_TI_GP_TIMER_STARTING, CPUHP_AP_HYPERV_TIMER_STARTING, CPUHP_AP_KVM_STARTING, CPUHP_AP_KVM_ARM_VGIC_INIT_STARTING, diff --git a/include/linux/dma-iommu.h b/include/linux/dma-iommu.h index 706b68d1359b..13d1f4c14d7b 100644 --- a/include/linux/dma-iommu.h +++ b/include/linux/dma-iommu.h @@ -40,6 +40,8 @@ void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list); void iommu_dma_free_cpu_cached_iovas(unsigned int cpu, struct iommu_domain *domain); +extern bool iommu_dma_forcedac; + #else /* CONFIG_IOMMU_DMA */ struct iommu_domain; diff --git a/include/linux/elevator.h b/include/linux/elevator.h index 1fe8e105b83b..dcb2f9022c1d 100644 --- a/include/linux/elevator.h +++ b/include/linux/elevator.h @@ -34,7 +34,7 @@ struct elevator_mq_ops { void (*depth_updated)(struct blk_mq_hw_ctx *); bool (*allow_merge)(struct request_queue *, struct request *, struct bio *); - bool (*bio_merge)(struct blk_mq_hw_ctx *, struct bio *, unsigned int); + bool (*bio_merge)(struct request_queue *, struct bio *, unsigned int); int (*request_merge)(struct request_queue *q, struct request **, struct bio *); void (*request_merged)(struct request_queue *, struct request *, enum elv_merge); void (*requests_merged)(struct request_queue *, struct request *, struct request *); diff --git a/include/linux/firmware/xlnx-zynqmp.h b/include/linux/firmware/xlnx-zynqmp.h index 71177b17eee5..66e2423d9feb 100644 --- a/include/linux/firmware/xlnx-zynqmp.h +++ b/include/linux/firmware/xlnx-zynqmp.h @@ -354,11 +354,6 @@ int zynqmp_pm_read_pggs(u32 index, u32 *value); int zynqmp_pm_system_shutdown(const u32 type, const u32 subtype); int zynqmp_pm_set_boot_health_status(u32 value); #else -static inline struct zynqmp_eemi_ops *zynqmp_pm_get_eemi_ops(void) -{ - return ERR_PTR(-ENODEV); -} - static inline int zynqmp_pm_get_api_version(u32 *version) { return -ENODEV; diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h index 286de0520574..ecf0032a0995 100644 --- a/include/linux/gpio/driver.h +++ b/include/linux/gpio/driver.h @@ -624,8 +624,17 @@ void gpiochip_irq_domain_deactivate(struct irq_domain *domain, bool gpiochip_irqchip_irq_valid(const struct gpio_chip *gc, unsigned int offset); +#ifdef CONFIG_GPIOLIB_IRQCHIP int gpiochip_irqchip_add_domain(struct gpio_chip *gc, struct irq_domain *domain); +#else +static inline int gpiochip_irqchip_add_domain(struct gpio_chip *gc, + struct irq_domain *domain) +{ + WARN_ON(1); + return -EINVAL; +} +#endif int gpiochip_generic_request(struct gpio_chip *gc, unsigned int offset); void gpiochip_generic_free(struct gpio_chip *gc, unsigned int offset); diff --git a/include/linux/hid.h b/include/linux/hid.h index ef702b3f56e3..3e33eb14118c 100644 --- a/include/linux/hid.h +++ b/include/linux/hid.h @@ -262,6 +262,8 @@ struct hid_item { #define HID_CP_SELECTION 0x000c0080 #define HID_CP_MEDIASELECTION 0x000c0087 #define HID_CP_SELECTDISC 0x000c00ba +#define HID_CP_VOLUMEUP 0x000c00e9 +#define HID_CP_VOLUMEDOWN 0x000c00ea #define HID_CP_PLAYBACKSPEED 0x000c00f1 #define HID_CP_PROXIMITY 0x000c0109 #define HID_CP_SPEAKERSYSTEM 0x000c0160 diff --git a/include/linux/i2c.h b/include/linux/i2c.h index 56622658b215..a670ae129f4b 100644 --- a/include/linux/i2c.h +++ b/include/linux/i2c.h @@ -687,6 +687,8 @@ struct i2c_adapter_quirks { #define I2C_AQ_NO_ZERO_LEN_READ BIT(5) #define I2C_AQ_NO_ZERO_LEN_WRITE BIT(6) #define I2C_AQ_NO_ZERO_LEN (I2C_AQ_NO_ZERO_LEN_READ | I2C_AQ_NO_ZERO_LEN_WRITE) +/* adapter cannot do repeated START */ +#define I2C_AQ_NO_REP_START BIT(7) /* * i2c_adapter is the structure used to identify a physical i2c bus along diff --git a/include/linux/initrd.h b/include/linux/initrd.h index 85c15717af34..1bbe9af48dc3 100644 --- a/include/linux/initrd.h +++ b/include/linux/initrd.h @@ -20,8 +20,10 @@ extern void free_initrd_mem(unsigned long, unsigned long); #ifdef CONFIG_BLK_DEV_INITRD extern void __init reserve_initrd_mem(void); +extern void wait_for_initramfs(void); #else static inline void __init reserve_initrd_mem(void) {} +static inline void wait_for_initramfs(void) {} #endif extern phys_addr_t phys_initrd_start; diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index 1bc46b88711a..d1f32b33415a 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -372,6 +372,7 @@ enum { /* PASID cache invalidation granu */ #define QI_PC_ALL_PASIDS 0 #define QI_PC_PASID_SEL 1 +#define QI_PC_GLOBAL 3 #define QI_EIOTLB_ADDR(addr) ((u64)(addr) & VTD_PAGE_MASK) #define QI_EIOTLB_IH(ih) (((u64)ih) << 6) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 5e7fe519430a..9ca6e6b8084d 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -547,7 +547,7 @@ static inline void iommu_iotlb_gather_add_page(struct iommu_domain *domain, * structure can be rewritten. */ if (gather->pgsize != size || - end < gather->start || start > gather->end) { + end + 1 < gather->start || start > gather->end + 1) { if (gather->pgsize) iommu_iotlb_sync(domain, gather); gather->pgsize = size; diff --git a/include/linux/ioport.h b/include/linux/ioport.h index 55de385c839c..647744d8514e 100644 --- a/include/linux/ioport.h +++ b/include/linux/ioport.h @@ -331,7 +331,7 @@ static inline void irqresource_disabled(struct resource *res, u32 irq) { res->start = irq; res->end = irq; - res->flags = IORESOURCE_IRQ | IORESOURCE_DISABLED | IORESOURCE_UNSET; + res->flags |= IORESOURCE_IRQ | IORESOURCE_DISABLED | IORESOURCE_UNSET; } extern struct address_space *iomem_get_mapping(void); diff --git a/include/linux/irqdesc.h b/include/linux/irqdesc.h index 891b323266df..adb4d3b761b0 100644 --- a/include/linux/irqdesc.h +++ b/include/linux/irqdesc.h @@ -159,6 +159,7 @@ static inline void generic_handle_irq_desc(struct irq_desc *desc) } int generic_handle_irq(unsigned int irq); +int generic_dispatch_irq(unsigned int irq); #ifdef CONFIG_HANDLE_DOMAIN_IRQ /* diff --git a/include/linux/ksm.h b/include/linux/ksm.h index 161e8164abcf..a5993f7b4345 100644 --- a/include/linux/ksm.h +++ b/include/linux/ksm.h @@ -19,6 +19,10 @@ struct stable_node; struct mem_cgroup; #ifdef CONFIG_KSM +int ksm_madvise_merge(struct mm_struct *mm, struct vm_area_struct *vma, + unsigned long *vm_flags); +int ksm_madvise_unmerge(struct vm_area_struct *vma, unsigned long start, + unsigned long end, unsigned long *vm_flags); int ksm_madvise(struct vm_area_struct *vma, unsigned long start, unsigned long end, int advice, unsigned long *vm_flags); int __ksm_enter(struct mm_struct *mm); diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 1b65e7204344..99dccea4293c 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -192,8 +192,8 @@ int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, int len, void *val); int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, int len, struct kvm_io_device *dev); -void kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, - struct kvm_io_device *dev); +int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, + struct kvm_io_device *dev); struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr); diff --git a/include/linux/mfd/da9063/registers.h b/include/linux/mfd/da9063/registers.h index 1dbabf1b3cb8..6e0f66a2e727 100644 --- a/include/linux/mfd/da9063/registers.h +++ b/include/linux/mfd/da9063/registers.h @@ -1037,6 +1037,9 @@ #define DA9063_NONKEY_PIN_AUTODOWN 0x02 #define DA9063_NONKEY_PIN_AUTOFLPRT 0x03 +/* DA9063_REG_CONFIG_J (addr=0x10F) */ +#define DA9063_TWOWIRE_TO 0x40 + /* DA9063_REG_MON_REG_5 (addr=0x116) */ #define DA9063_MON_A8_IDX_MASK 0x07 #define DA9063_MON_A8_IDX_NONE 0x00 diff --git a/include/linux/mfd/intel-m10-bmc.h b/include/linux/mfd/intel-m10-bmc.h index 74d4e193966a..9b54ca13eac3 100644 --- a/include/linux/mfd/intel-m10-bmc.h +++ b/include/linux/mfd/intel-m10-bmc.h @@ -11,7 +11,7 @@ #define M10BMC_LEGACY_SYS_BASE 0x300400 #define M10BMC_SYS_BASE 0x300800 -#define M10BMC_MEM_END 0x200000fc +#define M10BMC_MEM_END 0x1fffffff /* Register offset of system registers */ #define NIOS2_FW_VERSION 0x0 diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 53b89631a1d9..ab07f09f2bad 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -1226,7 +1226,7 @@ enum { MLX5_TRIGGERED_CMD_COMP = (u64)1 << 32, }; -static inline bool mlx5_is_roce_enabled(struct mlx5_core_dev *dev) +static inline bool mlx5_is_roce_init_enabled(struct mlx5_core_dev *dev) { struct devlink *devlink = priv_to_devlink(dev); union devlink_param_value val; diff --git a/include/linux/mm.h b/include/linux/mm.h index 8ba434287387..6c1b29bb3563 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3170,5 +3170,37 @@ extern int sysctl_nr_trim_pages; void mem_dump_obj(void *object); +/** + * seal_check_future_write - Check for F_SEAL_FUTURE_WRITE flag and handle it + * @seals: the seals to check + * @vma: the vma to operate on + * + * Check whether F_SEAL_FUTURE_WRITE is set; if so, do proper check/handling on + * the vma flags. Return 0 if check pass, or <0 for errors. + */ +static inline int seal_check_future_write(int seals, struct vm_area_struct *vma) +{ + if (seals & F_SEAL_FUTURE_WRITE) { + /* + * New PROT_WRITE and MAP_SHARED mmaps are not allowed when + * "future write" seal active. + */ + if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_WRITE)) + return -EPERM; + + /* + * Since an F_SEAL_FUTURE_WRITE sealed memfd can be mapped as + * MAP_SHARED and read-only, take care to not allow mprotect to + * revert protections on such mappings. Do this only for shared + * mappings. For private mappings, don't need to mask + * VM_MAYWRITE as we still want them to be COW-writable. + */ + if (vma->vm_flags & VM_SHARED) + vma->vm_flags &= ~(VM_MAYWRITE); + } + + return 0; +} + #endif /* __KERNEL__ */ #endif /* _LINUX_MM_H */ diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 6613b26a8894..5aacc1c10a45 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -97,10 +97,10 @@ struct page { }; struct { /* page_pool used by netstack */ /** - * @dma_addr: might require a 64-bit value even on + * @dma_addr: might require a 64-bit value on * 32-bit architectures. */ - dma_addr_t dma_addr; + unsigned long dma_addr[2]; }; struct { /* slab, slob and slub */ union { diff --git a/include/linux/mmc/host.h b/include/linux/mmc/host.h index 26a3c7bc29ae..a3a4e374f802 100644 --- a/include/linux/mmc/host.h +++ b/include/linux/mmc/host.h @@ -302,9 +302,6 @@ struct mmc_host { u32 ocr_avail_sdio; /* SDIO-specific OCR */ u32 ocr_avail_sd; /* SD-specific OCR */ u32 ocr_avail_mmc; /* MMC-specific OCR */ -#ifdef CONFIG_PM_SLEEP - struct notifier_block pm_notify; -#endif struct wakeup_source *ws; /* Enable consume of uevents */ u32 max_current_330; u32 max_current_300; diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index 3327239fa2f9..cc29dee508f7 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -15,6 +15,8 @@ #define NFS_DEF_FILE_IO_SIZE (4096U) #define NFS_MIN_FILE_IO_SIZE (1024U) +#define NFS_BITMASK_SZ 3 + struct nfs4_string { unsigned int len; char *data; @@ -525,7 +527,8 @@ struct nfs_closeargs { struct nfs_seqid * seqid; fmode_t fmode; u32 share_access; - u32 * bitmask; + const u32 * bitmask; + u32 bitmask_store[NFS_BITMASK_SZ]; struct nfs4_layoutreturn_args *lr_args; }; @@ -608,7 +611,8 @@ struct nfs4_delegreturnargs { struct nfs4_sequence_args seq_args; const struct nfs_fh *fhandle; const nfs4_stateid *stateid; - u32 * bitmask; + const u32 *bitmask; + u32 bitmask_store[NFS_BITMASK_SZ]; struct nfs4_layoutreturn_args *lr_args; }; @@ -648,7 +652,8 @@ struct nfs_pgio_args { union { unsigned int replen; /* used by read */ struct { - u32 * bitmask; /* used by write */ + const u32 * bitmask; /* used by write */ + u32 bitmask_store[NFS_BITMASK_SZ]; /* used by write */ enum nfs3_stable_how stable; /* used by write */ }; }; diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 3f7f89ea5e51..3d478abf411c 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -607,6 +607,7 @@ struct swevent_hlist { #define PERF_ATTACH_TASK_DATA 0x08 #define PERF_ATTACH_ITRACE 0x10 #define PERF_ATTACH_SCHED_CB 0x20 +#define PERF_ATTACH_CHILD 0x40 struct perf_cgroup; struct perf_buffer; diff --git a/include/linux/phy.h b/include/linux/phy.h index 1a12e4436b5b..8644b097dea3 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -493,6 +493,7 @@ struct macsec_ops; * @loopback_enabled: Set true if this PHY has been loopbacked successfully. * @downshifted_rate: Set true if link speed has been downshifted. * @is_on_sfp_module: Set true if PHY is located on an SFP module. + * @mac_managed_pm: Set true if MAC driver takes of suspending/resuming PHY * @state: State of the PHY for management purposes * @dev_flags: Device-specific flags used by the PHY driver. * @irq: IRQ number of the PHY's interrupt (-1 if none) @@ -567,6 +568,7 @@ struct phy_device { unsigned loopback_enabled:1; unsigned downshifted_rate:1; unsigned is_on_sfp_module:1; + unsigned mac_managed_pm:1; unsigned autoneg:1; /* The most recently read link state */ diff --git a/include/linux/platform_device.h b/include/linux/platform_device.h index 3f23f6e430bf..cd81e060863c 100644 --- a/include/linux/platform_device.h +++ b/include/linux/platform_device.h @@ -359,4 +359,7 @@ static inline int is_sh_early_platform_device(struct platform_device *pdev) } #endif /* CONFIG_SUPERH */ +/* For now only SuperH uses it */ +void early_platform_cleanup(void); + #endif /* _PLATFORM_DEVICE_H_ */ diff --git a/include/linux/pm.h b/include/linux/pm.h index 482313a8ccfc..628718697679 100644 --- a/include/linux/pm.h +++ b/include/linux/pm.h @@ -602,6 +602,7 @@ struct dev_pm_info { unsigned int idle_notification:1; unsigned int request_pending:1; unsigned int deferred_resume:1; + unsigned int needs_force_resume:1; unsigned int runtime_auto:1; bool ignore_children:1; unsigned int no_callbacks:1; diff --git a/include/linux/pm_runtime.h b/include/linux/pm_runtime.h index b492ae00cc90..6c08a085367b 100644 --- a/include/linux/pm_runtime.h +++ b/include/linux/pm_runtime.h @@ -265,7 +265,7 @@ static inline void pm_runtime_no_callbacks(struct device *dev) {} static inline void pm_runtime_irq_safe(struct device *dev) {} static inline bool pm_runtime_is_irq_safe(struct device *dev) { return false; } -static inline bool pm_runtime_callbacks_present(struct device *dev) { return false; } +static inline bool pm_runtime_has_no_callbacks(struct device *dev) { return false; } static inline void pm_runtime_mark_last_busy(struct device *dev) {} static inline void __pm_runtime_use_autosuspend(struct device *dev, bool use) {} diff --git a/include/linux/power/bq27xxx_battery.h b/include/linux/power/bq27xxx_battery.h index 111a40d0d3d5..8d5f4f40fb41 100644 --- a/include/linux/power/bq27xxx_battery.h +++ b/include/linux/power/bq27xxx_battery.h @@ -53,7 +53,6 @@ struct bq27xxx_reg_cache { int capacity; int energy; int flags; - int power_avg; int health; }; diff --git a/include/linux/reset.h b/include/linux/reset.h index b9109efa2a5c..9700124affa3 100644 --- a/include/linux/reset.h +++ b/include/linux/reset.h @@ -47,6 +47,11 @@ static inline int reset_control_reset(struct reset_control *rstc) return 0; } +static inline int reset_control_rearm(struct reset_control *rstc) +{ + return 0; +} + static inline int reset_control_assert(struct reset_control *rstc) { return 0; diff --git a/include/linux/smp.h b/include/linux/smp.h index 70c6f6284dcf..238a3f97a415 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -73,7 +73,7 @@ void on_each_cpu_cond(smp_cond_func_t cond_func, smp_call_func_t func, void on_each_cpu_cond_mask(smp_cond_func_t cond_func, smp_call_func_t func, void *info, bool wait, const struct cpumask *mask); -int smp_call_function_single_async(int cpu, call_single_data_t *csd); +int smp_call_function_single_async(int cpu, struct __call_single_data *csd); #ifdef CONFIG_SMP diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index 592897fa4f03..643139b1eafe 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -510,6 +510,9 @@ struct spi_controller { #define SPI_MASTER_GPIO_SS BIT(5) /* GPIO CS must select slave */ + /* flag indicating this is a non-devres managed controller */ + bool devm_allocated; + /* flag indicating this is an SPI slave controller */ bool slave; diff --git a/include/linux/tcp.h b/include/linux/tcp.h index 48d8a363319e..1bd559c69e83 100644 --- a/include/linux/tcp.h +++ b/include/linux/tcp.h @@ -225,7 +225,8 @@ struct tcp_sock { u8 compressed_ack; u8 dup_ack_counter:2, tlp_retrans:1, /* TLP is a retransmission */ - unused:5; + fast_ack_mode:2, /* which fast ack mode ? */ + unused:3; u32 chrono_start; /* Start time in jiffies of a TCP chrono */ u32 chrono_stat[3]; /* Time in jiffies for chrono_stat stats */ u8 chrono_type:2, /* current chronograph type */ diff --git a/include/linux/tty_driver.h b/include/linux/tty_driver.h index 61c3372d3f32..2f719b471d52 100644 --- a/include/linux/tty_driver.h +++ b/include/linux/tty_driver.h @@ -228,7 +228,7 @@ * * Called when the device receives a TIOCGICOUNT ioctl. Passed a kernel * structure to complete. This method is optional and will only be called - * if provided (otherwise EINVAL will be returned). + * if provided (otherwise ENOTTY will be returned). */ #include diff --git a/include/linux/udp.h b/include/linux/udp.h index aa84597bdc33..ae58ff3b6b5b 100644 --- a/include/linux/udp.h +++ b/include/linux/udp.h @@ -51,7 +51,9 @@ struct udp_sock { * different encapsulation layer set * this */ - gro_enabled:1; /* Can accept GRO packets */ + gro_enabled:1, /* Request GRO aggregation */ + accept_udp_l4:1, + accept_udp_fraglist:1; /* * Following member retains the information to create a UDP header * when the socket is uncorked. @@ -131,8 +133,16 @@ static inline void udp_cmsg_recv(struct msghdr *msg, struct sock *sk, static inline bool udp_unexpected_gso(struct sock *sk, struct sk_buff *skb) { - return !udp_sk(sk)->gro_enabled && skb_is_gso(skb) && - skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4; + if (!skb_is_gso(skb)) + return false; + + if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4 && !udp_sk(sk)->accept_udp_l4) + return true; + + if (skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST && !udp_sk(sk)->accept_udp_fraglist) + return true; + + return false; } #define udp_portaddr_for_each_entry(__sk, list) \ diff --git a/include/linux/usb/pd.h b/include/linux/usb/pd.h index 70d681918d01..bf00259493e0 100644 --- a/include/linux/usb/pd.h +++ b/include/linux/usb/pd.h @@ -493,4 +493,6 @@ static inline unsigned int rdo_max_power(u32 rdo) #define PD_N_CAPS_COUNT (PD_T_NO_RESPONSE / PD_T_SEND_SOURCE_CAP) #define PD_N_HARD_RESET_COUNT 2 +#define PD_P_SNK_STDBY_MW 2500 /* 2500 mW */ + #endif /* __LINUX_USB_PD_H */ diff --git a/include/linux/user_namespace.h b/include/linux/user_namespace.h index f6c5f784be5a..07d194f23587 100644 --- a/include/linux/user_namespace.h +++ b/include/linux/user_namespace.h @@ -108,6 +108,8 @@ void dec_ucount(struct ucounts *ucounts, enum ucount_type type); #ifdef CONFIG_USER_NS +extern int unprivileged_userns_clone; + static inline struct user_namespace *get_user_ns(struct user_namespace *ns) { if (ns) @@ -141,6 +143,8 @@ extern bool current_in_userns(const struct user_namespace *target_ns); struct ns_common *ns_get_owner(struct ns_common *ns); #else +#define unprivileged_userns_clone 0 + static inline struct user_namespace *get_user_ns(struct user_namespace *ns) { return &init_user_ns; diff --git a/include/linux/zstd.h b/include/linux/zstd.h index e87f78c9b19c..446ecabcdd02 100644 --- a/include/linux/zstd.h +++ b/include/linux/zstd.h @@ -1,138 +1,97 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) Yann Collet, Facebook, Inc. * All rights reserved. * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of https://github.com/facebook/zstd. - * An additional grant of patent rights can be found in the PATENTS file in the - * same directory. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of https://github.com/facebook/zstd) and + * the GPLv2 (found in the COPYING file in the root directory of + * https://github.com/facebook/zstd). You may select, at your option, one of the + * above-listed licenses. */ -#ifndef ZSTD_H -#define ZSTD_H +#ifndef LINUX_ZSTD_H +#define LINUX_ZSTD_H -/* ====== Dependency ======*/ -#include /* size_t */ +/** + * This is a kernel-style API that wraps the upstream zstd API, which cannot be + * used directly because the symbols aren't exported. It exposes the minimal + * functionality which is currently required by users of zstd in the kernel. + * Expose extra functions from lib/zstd/zstd.h as needed. + */ +/* ====== Dependency ====== */ +#include +#include +#include -/*-***************************************************************************** - * Introduction +/* ====== Helper Functions ====== */ +/** + * zstd_compress_bound() - maximum compressed size in worst case scenario + * @src_size: The size of the data to compress. * - * zstd, short for Zstandard, is a fast lossless compression algorithm, - * targeting real-time compression scenarios at zlib-level and better - * compression ratios. The zstd compression library provides in-memory - * compression and decompression functions. The library supports compression - * levels from 1 up to ZSTD_maxCLevel() which is 22. Levels >= 20, labeled - * ultra, should be used with caution, as they require more memory. - * Compression can be done in: - * - a single step, reusing a context (described as Explicit memory management) - * - unbounded multiple steps (described as Streaming compression) - * The compression ratio achievable on small data can be highly improved using - * compression with a dictionary in: - * - a single step (described as Simple dictionary API) - * - a single step, reusing a dictionary (described as Fast dictionary API) - ******************************************************************************/ - -/*====== Helper functions ======*/ + * Return: The maximum compressed size in the worst case scenario. + */ +size_t zstd_compress_bound(size_t src_size); /** - * enum ZSTD_ErrorCode - zstd error codes + * zstd_is_error() - tells if a size_t function result is an error code + * @code: The function result to check for error. * - * Functions that return size_t can be checked for errors using ZSTD_isError() - * and the ZSTD_ErrorCode can be extracted using ZSTD_getErrorCode(). + * Return: Non-zero iff the code is an error. + */ +unsigned int zstd_is_error(size_t code); + +/** + * enum zstd_error_code - zstd error codes */ -typedef enum { - ZSTD_error_no_error, - ZSTD_error_GENERIC, - ZSTD_error_prefix_unknown, - ZSTD_error_version_unsupported, - ZSTD_error_parameter_unknown, - ZSTD_error_frameParameter_unsupported, - ZSTD_error_frameParameter_unsupportedBy32bits, - ZSTD_error_frameParameter_windowTooLarge, - ZSTD_error_compressionParameter_unsupported, - ZSTD_error_init_missing, - ZSTD_error_memory_allocation, - ZSTD_error_stage_wrong, - ZSTD_error_dstSize_tooSmall, - ZSTD_error_srcSize_wrong, - ZSTD_error_corruption_detected, - ZSTD_error_checksum_wrong, - ZSTD_error_tableLog_tooLarge, - ZSTD_error_maxSymbolValue_tooLarge, - ZSTD_error_maxSymbolValue_tooSmall, - ZSTD_error_dictionary_corrupted, - ZSTD_error_dictionary_wrong, - ZSTD_error_dictionaryCreation_failed, - ZSTD_error_maxCode -} ZSTD_ErrorCode; +typedef ZSTD_ErrorCode zstd_error_code; /** - * ZSTD_maxCLevel() - maximum compression level available + * zstd_get_error_code() - translates an error function result to an error code + * @code: The function result for which zstd_is_error(code) is true. * - * Return: Maximum compression level available. + * Return: A unique error code for this error. */ -int ZSTD_maxCLevel(void); +zstd_error_code zstd_get_error_code(size_t code); + /** - * ZSTD_compressBound() - maximum compressed size in worst case scenario - * @srcSize: The size of the data to compress. + * zstd_get_error_name() - translates an error function result to a string + * @code: The function result for which zstd_is_error(code) is true. * - * Return: The maximum compressed size in the worst case scenario. + * Return: An error string corresponding to the error code. */ -size_t ZSTD_compressBound(size_t srcSize); +const char *zstd_get_error_name(size_t code); + /** - * ZSTD_isError() - tells if a size_t function result is an error code - * @code: The function result to check for error. + * zstd_min_clevel() - minimum allowed compression level * - * Return: Non-zero iff the code is an error. + * Return: The minimum allowed compression level. */ -static __attribute__((unused)) unsigned int ZSTD_isError(size_t code) -{ - return code > (size_t)-ZSTD_error_maxCode; -} +int zstd_min_clevel(void); + /** - * ZSTD_getErrorCode() - translates an error function result to a ZSTD_ErrorCode - * @functionResult: The result of a function for which ZSTD_isError() is true. + * zstd_max_clevel() - maximum allowed compression level * - * Return: The ZSTD_ErrorCode corresponding to the functionResult or 0 - * if the functionResult isn't an error. + * Return: The maximum allowed compression level. */ -static __attribute__((unused)) ZSTD_ErrorCode ZSTD_getErrorCode( - size_t functionResult) -{ - if (!ZSTD_isError(functionResult)) - return (ZSTD_ErrorCode)0; - return (ZSTD_ErrorCode)(0 - functionResult); -} +int zstd_max_clevel(void); + +/* ====== Parameter Selection ====== */ /** - * enum ZSTD_strategy - zstd compression search strategy + * enum zstd_strategy - zstd compression search strategy * - * From faster to stronger. + * From faster to stronger. See zstd_lib.h. */ -typedef enum { - ZSTD_fast, - ZSTD_dfast, - ZSTD_greedy, - ZSTD_lazy, - ZSTD_lazy2, - ZSTD_btlazy2, - ZSTD_btopt, - ZSTD_btopt2 -} ZSTD_strategy; +typedef ZSTD_strategy zstd_strategy; /** - * struct ZSTD_compressionParameters - zstd compression parameters + * struct zstd_compression_parameters - zstd compression parameters * @windowLog: Log of the largest match distance. Larger means more * compression, and more memory needed during decompression. - * @chainLog: Fully searched segment. Larger means more compression, slower, - * and more memory (useless for fast). + * @chainLog: Fully searched segment. Larger means more compression, + * slower, and more memory (useless for fast). * @hashLog: Dispatch table. Larger means more compression, * slower, and more memory. * @searchLog: Number of searches. Larger means more compression and slower. @@ -141,1017 +100,348 @@ typedef enum { * @targetLength: Acceptable match size for optimal parser (only). Larger means * more compression, and slower. * @strategy: The zstd compression strategy. + * + * See zstd_lib.h. */ -typedef struct { - unsigned int windowLog; - unsigned int chainLog; - unsigned int hashLog; - unsigned int searchLog; - unsigned int searchLength; - unsigned int targetLength; - ZSTD_strategy strategy; -} ZSTD_compressionParameters; +typedef ZSTD_compressionParameters zstd_compression_parameters; /** - * struct ZSTD_frameParameters - zstd frame parameters - * @contentSizeFlag: Controls whether content size will be present in the frame - * header (when known). - * @checksumFlag: Controls whether a 32-bit checksum is generated at the end - * of the frame for error detection. - * @noDictIDFlag: Controls whether dictID will be saved into the frame header - * when using dictionary compression. + * struct zstd_frame_parameters - zstd frame parameters + * @contentSizeFlag: Controls whether content size will be present in the + * frame header (when known). + * @checksumFlag: Controls whether a 32-bit checksum is generated at the + * end of the frame for error detection. + * @noDictIDFlag: Controls whether dictID will be saved into the frame + * header when using dictionary compression. * - * The default value is all fields set to 0. + * The default value is all fields set to 0. See zstd_lib.h. */ -typedef struct { - unsigned int contentSizeFlag; - unsigned int checksumFlag; - unsigned int noDictIDFlag; -} ZSTD_frameParameters; +typedef ZSTD_frameParameters zstd_frame_parameters; /** - * struct ZSTD_parameters - zstd parameters + * struct zstd_parameters - zstd parameters * @cParams: The compression parameters. * @fParams: The frame parameters. */ -typedef struct { - ZSTD_compressionParameters cParams; - ZSTD_frameParameters fParams; -} ZSTD_parameters; +typedef ZSTD_parameters zstd_parameters; /** - * ZSTD_getCParams() - returns ZSTD_compressionParameters for selected level - * @compressionLevel: The compression level from 1 to ZSTD_maxCLevel(). - * @estimatedSrcSize: The estimated source size to compress or 0 if unknown. - * @dictSize: The dictionary size or 0 if a dictionary isn't being used. + * zstd_get_params() - returns zstd_parameters for selected level + * @level: The compression level + * @estimated_src_size: The estimated source size to compress or 0 + * if unknown. * - * Return: The selected ZSTD_compressionParameters. + * Return: The selected zstd_parameters. */ -ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, - unsigned long long estimatedSrcSize, size_t dictSize); +zstd_parameters zstd_get_params(int level, + unsigned long long estimated_src_size); -/** - * ZSTD_getParams() - returns ZSTD_parameters for selected level - * @compressionLevel: The compression level from 1 to ZSTD_maxCLevel(). - * @estimatedSrcSize: The estimated source size to compress or 0 if unknown. - * @dictSize: The dictionary size or 0 if a dictionary isn't being used. - * - * The same as ZSTD_getCParams() except also selects the default frame - * parameters (all zero). - * - * Return: The selected ZSTD_parameters. - */ -ZSTD_parameters ZSTD_getParams(int compressionLevel, - unsigned long long estimatedSrcSize, size_t dictSize); +/* ====== Single-pass Compression ====== */ -/*-************************************* - * Explicit memory management - **************************************/ +typedef ZSTD_CCtx zstd_cctx; /** - * ZSTD_CCtxWorkspaceBound() - amount of memory needed to initialize a ZSTD_CCtx - * @cParams: The compression parameters to be used for compression. + * zstd_cctx_workspace_bound() - max memory needed to initialize a zstd_cctx + * @parameters: The compression parameters to be used. * * If multiple compression parameters might be used, the caller must call - * ZSTD_CCtxWorkspaceBound() for each set of parameters and use the maximum + * zstd_cctx_workspace_bound() for each set of parameters and use the maximum * size. * - * Return: A lower bound on the size of the workspace that is passed to - * ZSTD_initCCtx(). + * Return: A lower bound on the size of the workspace that is passed to + * zstd_init_cctx(). */ -size_t ZSTD_CCtxWorkspaceBound(ZSTD_compressionParameters cParams); +size_t zstd_cctx_workspace_bound(const zstd_compression_parameters *parameters); /** - * struct ZSTD_CCtx - the zstd compression context - * - * When compressing many times it is recommended to allocate a context just once - * and reuse it for each successive compression operation. - */ -typedef struct ZSTD_CCtx_s ZSTD_CCtx; -/** - * ZSTD_initCCtx() - initialize a zstd compression context - * @workspace: The workspace to emplace the context into. It must outlive - * the returned context. - * @workspaceSize: The size of workspace. Use ZSTD_CCtxWorkspaceBound() to - * determine how large the workspace must be. - * - * Return: A compression context emplaced into workspace. - */ -ZSTD_CCtx *ZSTD_initCCtx(void *workspace, size_t workspaceSize); - -/** - * ZSTD_compressCCtx() - compress src into dst - * @ctx: The context. Must have been initialized with a workspace at - * least as large as ZSTD_CCtxWorkspaceBound(params.cParams). - * @dst: The buffer to compress src into. - * @dstCapacity: The size of the destination buffer. May be any size, but - * ZSTD_compressBound(srcSize) is guaranteed to be large enough. - * @src: The data to compress. - * @srcSize: The size of the data to compress. - * @params: The parameters to use for compression. See ZSTD_getParams(). - * - * Return: The compressed size or an error, which can be checked using - * ZSTD_isError(). - */ -size_t ZSTD_compressCCtx(ZSTD_CCtx *ctx, void *dst, size_t dstCapacity, - const void *src, size_t srcSize, ZSTD_parameters params); - -/** - * ZSTD_DCtxWorkspaceBound() - amount of memory needed to initialize a ZSTD_DCtx - * - * Return: A lower bound on the size of the workspace that is passed to - * ZSTD_initDCtx(). - */ -size_t ZSTD_DCtxWorkspaceBound(void); - -/** - * struct ZSTD_DCtx - the zstd decompression context - * - * When decompressing many times it is recommended to allocate a context just - * once and reuse it for each successive decompression operation. - */ -typedef struct ZSTD_DCtx_s ZSTD_DCtx; -/** - * ZSTD_initDCtx() - initialize a zstd decompression context - * @workspace: The workspace to emplace the context into. It must outlive - * the returned context. - * @workspaceSize: The size of workspace. Use ZSTD_DCtxWorkspaceBound() to - * determine how large the workspace must be. - * - * Return: A decompression context emplaced into workspace. - */ -ZSTD_DCtx *ZSTD_initDCtx(void *workspace, size_t workspaceSize); - -/** - * ZSTD_decompressDCtx() - decompress zstd compressed src into dst - * @ctx: The decompression context. - * @dst: The buffer to decompress src into. - * @dstCapacity: The size of the destination buffer. Must be at least as large - * as the decompressed size. If the caller cannot upper bound the - * decompressed size, then it's better to use the streaming API. - * @src: The zstd compressed data to decompress. Multiple concatenated - * frames and skippable frames are allowed. - * @srcSize: The exact size of the data to decompress. - * - * Return: The decompressed size or an error, which can be checked using - * ZSTD_isError(). - */ -size_t ZSTD_decompressDCtx(ZSTD_DCtx *ctx, void *dst, size_t dstCapacity, - const void *src, size_t srcSize); - -/*-************************ - * Simple dictionary API - **************************/ - -/** - * ZSTD_compress_usingDict() - compress src into dst using a dictionary - * @ctx: The context. Must have been initialized with a workspace at - * least as large as ZSTD_CCtxWorkspaceBound(params.cParams). - * @dst: The buffer to compress src into. - * @dstCapacity: The size of the destination buffer. May be any size, but - * ZSTD_compressBound(srcSize) is guaranteed to be large enough. - * @src: The data to compress. - * @srcSize: The size of the data to compress. - * @dict: The dictionary to use for compression. - * @dictSize: The size of the dictionary. - * @params: The parameters to use for compression. See ZSTD_getParams(). - * - * Compression using a predefined dictionary. The same dictionary must be used - * during decompression. - * - * Return: The compressed size or an error, which can be checked using - * ZSTD_isError(). - */ -size_t ZSTD_compress_usingDict(ZSTD_CCtx *ctx, void *dst, size_t dstCapacity, - const void *src, size_t srcSize, const void *dict, size_t dictSize, - ZSTD_parameters params); - -/** - * ZSTD_decompress_usingDict() - decompress src into dst using a dictionary - * @ctx: The decompression context. - * @dst: The buffer to decompress src into. - * @dstCapacity: The size of the destination buffer. Must be at least as large - * as the decompressed size. If the caller cannot upper bound the - * decompressed size, then it's better to use the streaming API. - * @src: The zstd compressed data to decompress. Multiple concatenated - * frames and skippable frames are allowed. - * @srcSize: The exact size of the data to decompress. - * @dict: The dictionary to use for decompression. The same dictionary - * must've been used to compress the data. - * @dictSize: The size of the dictionary. - * - * Return: The decompressed size or an error, which can be checked using - * ZSTD_isError(). - */ -size_t ZSTD_decompress_usingDict(ZSTD_DCtx *ctx, void *dst, size_t dstCapacity, - const void *src, size_t srcSize, const void *dict, size_t dictSize); - -/*-************************** - * Fast dictionary API - ***************************/ - -/** - * ZSTD_CDictWorkspaceBound() - memory needed to initialize a ZSTD_CDict - * @cParams: The compression parameters to be used for compression. + * zstd_init_cctx() - initialize a zstd compression context + * @workspace: The workspace to emplace the context into. It must outlive + * the returned context. + * @workspace_size: The size of workspace. Use zstd_cctx_workspace_bound() to + * determine how large the workspace must be. * - * Return: A lower bound on the size of the workspace that is passed to - * ZSTD_initCDict(). - */ -size_t ZSTD_CDictWorkspaceBound(ZSTD_compressionParameters cParams); - -/** - * struct ZSTD_CDict - a digested dictionary to be used for compression + * Return: A zstd compression context or NULL on error. */ -typedef struct ZSTD_CDict_s ZSTD_CDict; +zstd_cctx *zstd_init_cctx(void *workspace, size_t workspace_size); /** - * ZSTD_initCDict() - initialize a digested dictionary for compression - * @dictBuffer: The dictionary to digest. The buffer is referenced by the - * ZSTD_CDict so it must outlive the returned ZSTD_CDict. - * @dictSize: The size of the dictionary. - * @params: The parameters to use for compression. See ZSTD_getParams(). - * @workspace: The workspace. It must outlive the returned ZSTD_CDict. - * @workspaceSize: The workspace size. Must be at least - * ZSTD_CDictWorkspaceBound(params.cParams). + * zstd_compress_cctx() - compress src into dst with the initialized parameters + * @cctx: The context. Must have been initialized with zstd_init_cctx(). + * @dst: The buffer to compress src into. + * @dst_capacity: The size of the destination buffer. May be any size, but + * ZSTD_compressBound(srcSize) is guaranteed to be large enough. + * @src: The data to compress. + * @src_size: The size of the data to compress. + * @parameters: The compression parameters to be used. * - * When compressing multiple messages / blocks with the same dictionary it is - * recommended to load it just once. The ZSTD_CDict merely references the - * dictBuffer, so it must outlive the returned ZSTD_CDict. - * - * Return: The digested dictionary emplaced into workspace. + * Return: The compressed size or an error, which can be checked using + * zstd_is_error(). */ -ZSTD_CDict *ZSTD_initCDict(const void *dictBuffer, size_t dictSize, - ZSTD_parameters params, void *workspace, size_t workspaceSize); +size_t zstd_compress_cctx(zstd_cctx *cctx, void *dst, size_t dst_capacity, + const void *src, size_t src_size, const zstd_parameters *parameters); -/** - * ZSTD_compress_usingCDict() - compress src into dst using a ZSTD_CDict - * @ctx: The context. Must have been initialized with a workspace at - * least as large as ZSTD_CCtxWorkspaceBound(cParams) where - * cParams are the compression parameters used to initialize the - * cdict. - * @dst: The buffer to compress src into. - * @dstCapacity: The size of the destination buffer. May be any size, but - * ZSTD_compressBound(srcSize) is guaranteed to be large enough. - * @src: The data to compress. - * @srcSize: The size of the data to compress. - * @cdict: The digested dictionary to use for compression. - * @params: The parameters to use for compression. See ZSTD_getParams(). - * - * Compression using a digested dictionary. The same dictionary must be used - * during decompression. - * - * Return: The compressed size or an error, which can be checked using - * ZSTD_isError(). - */ -size_t ZSTD_compress_usingCDict(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, - const void *src, size_t srcSize, const ZSTD_CDict *cdict); +/* ====== Single-pass Decompression ====== */ +typedef ZSTD_DCtx zstd_dctx; /** - * ZSTD_DDictWorkspaceBound() - memory needed to initialize a ZSTD_DDict + * zstd_dctx_workspace_bound() - max memory needed to initialize a zstd_dctx * - * Return: A lower bound on the size of the workspace that is passed to - * ZSTD_initDDict(). - */ -size_t ZSTD_DDictWorkspaceBound(void); - -/** - * struct ZSTD_DDict - a digested dictionary to be used for decompression + * Return: A lower bound on the size of the workspace that is passed to + * zstd_init_dctx(). */ -typedef struct ZSTD_DDict_s ZSTD_DDict; +size_t zstd_dctx_workspace_bound(void); /** - * ZSTD_initDDict() - initialize a digested dictionary for decompression - * @dictBuffer: The dictionary to digest. The buffer is referenced by the - * ZSTD_DDict so it must outlive the returned ZSTD_DDict. - * @dictSize: The size of the dictionary. - * @workspace: The workspace. It must outlive the returned ZSTD_DDict. - * @workspaceSize: The workspace size. Must be at least - * ZSTD_DDictWorkspaceBound(). - * - * When decompressing multiple messages / blocks with the same dictionary it is - * recommended to load it just once. The ZSTD_DDict merely references the - * dictBuffer, so it must outlive the returned ZSTD_DDict. + * zstd_init_dctx() - initialize a zstd decompression context + * @workspace: The workspace to emplace the context into. It must outlive + * the returned context. + * @workspace_size: The size of workspace. Use zstd_dctx_workspace_bound() to + * determine how large the workspace must be. * - * Return: The digested dictionary emplaced into workspace. + * Return: A zstd decompression context or NULL on error. */ -ZSTD_DDict *ZSTD_initDDict(const void *dictBuffer, size_t dictSize, - void *workspace, size_t workspaceSize); +zstd_dctx *zstd_init_dctx(void *workspace, size_t workspace_size); /** - * ZSTD_decompress_usingDDict() - decompress src into dst using a ZSTD_DDict - * @ctx: The decompression context. - * @dst: The buffer to decompress src into. - * @dstCapacity: The size of the destination buffer. Must be at least as large - * as the decompressed size. If the caller cannot upper bound the - * decompressed size, then it's better to use the streaming API. - * @src: The zstd compressed data to decompress. Multiple concatenated - * frames and skippable frames are allowed. - * @srcSize: The exact size of the data to decompress. - * @ddict: The digested dictionary to use for decompression. The same - * dictionary must've been used to compress the data. + * zstd_decompress_dctx() - decompress zstd compressed src into dst + * @dctx: The decompression context. + * @dst: The buffer to decompress src into. + * @dst_capacity: The size of the destination buffer. Must be at least as large + * as the decompressed size. If the caller cannot upper bound the + * decompressed size, then it's better to use the streaming API. + * @src: The zstd compressed data to decompress. Multiple concatenated + * frames and skippable frames are allowed. + * @src_size: The exact size of the data to decompress. * - * Return: The decompressed size or an error, which can be checked using - * ZSTD_isError(). + * Return: The decompressed size or an error, which can be checked using + * zstd_is_error(). */ -size_t ZSTD_decompress_usingDDict(ZSTD_DCtx *dctx, void *dst, - size_t dstCapacity, const void *src, size_t srcSize, - const ZSTD_DDict *ddict); +size_t zstd_decompress_dctx(zstd_dctx *dctx, void *dst, size_t dst_capacity, + const void *src, size_t src_size); - -/*-************************** - * Streaming - ***************************/ +/* ====== Streaming Buffers ====== */ /** - * struct ZSTD_inBuffer - input buffer for streaming + * struct zstd_in_buffer - input buffer for streaming * @src: Start of the input buffer. * @size: Size of the input buffer. * @pos: Position where reading stopped. Will be updated. * Necessarily 0 <= pos <= size. + * + * See zstd_lib.h. */ -typedef struct ZSTD_inBuffer_s { - const void *src; - size_t size; - size_t pos; -} ZSTD_inBuffer; +typedef ZSTD_inBuffer zstd_in_buffer; /** - * struct ZSTD_outBuffer - output buffer for streaming + * struct zstd_out_buffer - output buffer for streaming * @dst: Start of the output buffer. * @size: Size of the output buffer. * @pos: Position where writing stopped. Will be updated. * Necessarily 0 <= pos <= size. + * + * See zstd_lib.h. */ -typedef struct ZSTD_outBuffer_s { - void *dst; - size_t size; - size_t pos; -} ZSTD_outBuffer; +typedef ZSTD_outBuffer zstd_out_buffer; +/* ====== Streaming Compression ====== */ - -/*-***************************************************************************** - * Streaming compression - HowTo - * - * A ZSTD_CStream object is required to track streaming operation. - * Use ZSTD_initCStream() to initialize a ZSTD_CStream object. - * ZSTD_CStream objects can be reused multiple times on consecutive compression - * operations. It is recommended to re-use ZSTD_CStream in situations where many - * streaming operations will be achieved consecutively. Use one separate - * ZSTD_CStream per thread for parallel execution. - * - * Use ZSTD_compressStream() repetitively to consume input stream. - * The function will automatically update both `pos` fields. - * Note that it may not consume the entire input, in which case `pos < size`, - * and it's up to the caller to present again remaining data. - * It returns a hint for the preferred number of bytes to use as an input for - * the next function call. - * - * At any moment, it's possible to flush whatever data remains within internal - * buffer, using ZSTD_flushStream(). `output->pos` will be updated. There might - * still be some content left within the internal buffer if `output->size` is - * too small. It returns the number of bytes left in the internal buffer and - * must be called until it returns 0. - * - * ZSTD_endStream() instructs to finish a frame. It will perform a flush and - * write frame epilogue. The epilogue is required for decoders to consider a - * frame completed. Similar to ZSTD_flushStream(), it may not be able to flush - * the full content if `output->size` is too small. In which case, call again - * ZSTD_endStream() to complete the flush. It returns the number of bytes left - * in the internal buffer and must be called until it returns 0. - ******************************************************************************/ +typedef ZSTD_CStream zstd_cstream; /** - * ZSTD_CStreamWorkspaceBound() - memory needed to initialize a ZSTD_CStream - * @cParams: The compression parameters to be used for compression. + * zstd_cstream_workspace_bound() - memory needed to initialize a zstd_cstream + * @cparams: The compression parameters to be used for compression. * * Return: A lower bound on the size of the workspace that is passed to - * ZSTD_initCStream() and ZSTD_initCStream_usingCDict(). - */ -size_t ZSTD_CStreamWorkspaceBound(ZSTD_compressionParameters cParams); - -/** - * struct ZSTD_CStream - the zstd streaming compression context - */ -typedef struct ZSTD_CStream_s ZSTD_CStream; - -/*===== ZSTD_CStream management functions =====*/ -/** - * ZSTD_initCStream() - initialize a zstd streaming compression context - * @params: The zstd compression parameters. - * @pledgedSrcSize: If params.fParams.contentSizeFlag == 1 then the caller must - * pass the source size (zero means empty source). Otherwise, - * the caller may optionally pass the source size, or zero if - * unknown. - * @workspace: The workspace to emplace the context into. It must outlive - * the returned context. - * @workspaceSize: The size of workspace. - * Use ZSTD_CStreamWorkspaceBound(params.cParams) to determine - * how large the workspace must be. - * - * Return: The zstd streaming compression context. + * zstd_init_cstream(). */ -ZSTD_CStream *ZSTD_initCStream(ZSTD_parameters params, - unsigned long long pledgedSrcSize, void *workspace, - size_t workspaceSize); +size_t zstd_cstream_workspace_bound(const zstd_compression_parameters *cparams); /** - * ZSTD_initCStream_usingCDict() - initialize a streaming compression context - * @cdict: The digested dictionary to use for compression. - * @pledgedSrcSize: Optionally the source size, or zero if unknown. - * @workspace: The workspace to emplace the context into. It must outlive - * the returned context. - * @workspaceSize: The size of workspace. Call ZSTD_CStreamWorkspaceBound() - * with the cParams used to initialize the cdict to determine - * how large the workspace must be. + * zstd_init_cstream() - initialize a zstd streaming compression context + * @parameters The zstd parameters to use for compression. + * @pledged_src_size: If params.fParams.contentSizeFlag == 1 then the caller + * must pass the source size (zero means empty source). + * Otherwise, the caller may optionally pass the source + * size, or zero if unknown. + * @workspace: The workspace to emplace the context into. It must outlive + * the returned context. + * @workspace_size: The size of workspace. + * Use zstd_cstream_workspace_bound(params->cparams) to + * determine how large the workspace must be. * - * Return: The zstd streaming compression context. + * Return: The zstd streaming compression context or NULL on error. */ -ZSTD_CStream *ZSTD_initCStream_usingCDict(const ZSTD_CDict *cdict, - unsigned long long pledgedSrcSize, void *workspace, - size_t workspaceSize); +zstd_cstream *zstd_init_cstream(const zstd_parameters *parameters, + unsigned long long pledged_src_size, void *workspace, size_t workspace_size); -/*===== Streaming compression functions =====*/ /** - * ZSTD_resetCStream() - reset the context using parameters from creation - * @zcs: The zstd streaming compression context to reset. - * @pledgedSrcSize: Optionally the source size, or zero if unknown. + * zstd_reset_cstream() - reset the context using parameters from creation + * @cstream: The zstd streaming compression context to reset. + * @pledged_src_size: Optionally the source size, or zero if unknown. * * Resets the context using the parameters from creation. Skips dictionary - * loading, since it can be reused. If `pledgedSrcSize` is non-zero the frame + * loading, since it can be reused. If `pledged_src_size` is non-zero the frame * content size is always written into the frame header. * - * Return: Zero or an error, which can be checked using ZSTD_isError(). + * Return: Zero or an error, which can be checked using + * zstd_is_error(). */ -size_t ZSTD_resetCStream(ZSTD_CStream *zcs, unsigned long long pledgedSrcSize); +size_t zstd_reset_cstream(zstd_cstream *cstream, + unsigned long long pledged_src_size); + /** - * ZSTD_compressStream() - streaming compress some of input into output - * @zcs: The zstd streaming compression context. - * @output: Destination buffer. `output->pos` is updated to indicate how much - * compressed data was written. - * @input: Source buffer. `input->pos` is updated to indicate how much data was - * read. Note that it may not consume the entire input, in which case - * `input->pos < input->size`, and it's up to the caller to present - * remaining data again. + * zstd_compress_stream() - streaming compress some of input into output + * @cstream: The zstd streaming compression context. + * @output: Destination buffer. `output->pos` is updated to indicate how much + * compressed data was written. + * @input: Source buffer. `input->pos` is updated to indicate how much data + * was read. Note that it may not consume the entire input, in which + * case `input->pos < input->size`, and it's up to the caller to + * present remaining data again. * * The `input` and `output` buffers may be any size. Guaranteed to make some * forward progress if `input` and `output` are not empty. * - * Return: A hint for the number of bytes to use as the input for the next - * function call or an error, which can be checked using - * ZSTD_isError(). + * Return: A hint for the number of bytes to use as the input for the next + * function call or an error, which can be checked using + * zstd_is_error(). */ -size_t ZSTD_compressStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output, - ZSTD_inBuffer *input); +size_t zstd_compress_stream(zstd_cstream *cstream, zstd_out_buffer *output, + zstd_in_buffer *input); + /** - * ZSTD_flushStream() - flush internal buffers into output - * @zcs: The zstd streaming compression context. - * @output: Destination buffer. `output->pos` is updated to indicate how much - * compressed data was written. + * zstd_flush_stream() - flush internal buffers into output + * @cstream: The zstd streaming compression context. + * @output: Destination buffer. `output->pos` is updated to indicate how much + * compressed data was written. * - * ZSTD_flushStream() must be called until it returns 0, meaning all the data - * has been flushed. Since ZSTD_flushStream() causes a block to be ended, + * zstd_flush_stream() must be called until it returns 0, meaning all the data + * has been flushed. Since zstd_flush_stream() causes a block to be ended, * calling it too often will degrade the compression ratio. * - * Return: The number of bytes still present within internal buffers or an - * error, which can be checked using ZSTD_isError(). + * Return: The number of bytes still present within internal buffers or an + * error, which can be checked using zstd_is_error(). */ -size_t ZSTD_flushStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output); -/** - * ZSTD_endStream() - flush internal buffers into output and end the frame - * @zcs: The zstd streaming compression context. - * @output: Destination buffer. `output->pos` is updated to indicate how much - * compressed data was written. - * - * ZSTD_endStream() must be called until it returns 0, meaning all the data has - * been flushed and the frame epilogue has been written. - * - * Return: The number of bytes still present within internal buffers or an - * error, which can be checked using ZSTD_isError(). - */ -size_t ZSTD_endStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output); +size_t zstd_flush_stream(zstd_cstream *cstream, zstd_out_buffer *output); /** - * ZSTD_CStreamInSize() - recommended size for the input buffer - * - * Return: The recommended size for the input buffer. - */ -size_t ZSTD_CStreamInSize(void); -/** - * ZSTD_CStreamOutSize() - recommended size for the output buffer + * zstd_end_stream() - flush internal buffers into output and end the frame + * @cstream: The zstd streaming compression context. + * @output: Destination buffer. `output->pos` is updated to indicate how much + * compressed data was written. * - * When the output buffer is at least this large, it is guaranteed to be large - * enough to flush at least one complete compressed block. + * zstd_end_stream() must be called until it returns 0, meaning all the data has + * been flushed and the frame epilogue has been written. * - * Return: The recommended size for the output buffer. + * Return: The number of bytes still present within internal buffers or an + * error, which can be checked using zstd_is_error(). */ -size_t ZSTD_CStreamOutSize(void); +size_t zstd_end_stream(zstd_cstream *cstream, zstd_out_buffer *output); +/* ====== Streaming Decompression ====== */ - -/*-***************************************************************************** - * Streaming decompression - HowTo - * - * A ZSTD_DStream object is required to track streaming operations. - * Use ZSTD_initDStream() to initialize a ZSTD_DStream object. - * ZSTD_DStream objects can be re-used multiple times. - * - * Use ZSTD_decompressStream() repetitively to consume your input. - * The function will update both `pos` fields. - * If `input->pos < input->size`, some input has not been consumed. - * It's up to the caller to present again remaining data. - * If `output->pos < output->size`, decoder has flushed everything it could. - * Returns 0 iff a frame is completely decoded and fully flushed. - * Otherwise it returns a suggested next input size that will never load more - * than the current frame. - ******************************************************************************/ +typedef ZSTD_DStream zstd_dstream; /** - * ZSTD_DStreamWorkspaceBound() - memory needed to initialize a ZSTD_DStream - * @maxWindowSize: The maximum window size allowed for compressed frames. + * zstd_dstream_workspace_bound() - memory needed to initialize a zstd_dstream + * @max_window_size: The maximum window size allowed for compressed frames. * - * Return: A lower bound on the size of the workspace that is passed to - * ZSTD_initDStream() and ZSTD_initDStream_usingDDict(). + * Return: A lower bound on the size of the workspace that is passed + * to zstd_init_dstream(). */ -size_t ZSTD_DStreamWorkspaceBound(size_t maxWindowSize); +size_t zstd_dstream_workspace_bound(size_t max_window_size); /** - * struct ZSTD_DStream - the zstd streaming decompression context - */ -typedef struct ZSTD_DStream_s ZSTD_DStream; -/*===== ZSTD_DStream management functions =====*/ -/** - * ZSTD_initDStream() - initialize a zstd streaming decompression context - * @maxWindowSize: The maximum window size allowed for compressed frames. - * @workspace: The workspace to emplace the context into. It must outlive - * the returned context. - * @workspaceSize: The size of workspace. - * Use ZSTD_DStreamWorkspaceBound(maxWindowSize) to determine - * how large the workspace must be. - * - * Return: The zstd streaming decompression context. - */ -ZSTD_DStream *ZSTD_initDStream(size_t maxWindowSize, void *workspace, - size_t workspaceSize); -/** - * ZSTD_initDStream_usingDDict() - initialize streaming decompression context - * @maxWindowSize: The maximum window size allowed for compressed frames. - * @ddict: The digested dictionary to use for decompression. - * @workspace: The workspace to emplace the context into. It must outlive - * the returned context. - * @workspaceSize: The size of workspace. - * Use ZSTD_DStreamWorkspaceBound(maxWindowSize) to determine - * how large the workspace must be. + * zstd_init_dstream() - initialize a zstd streaming decompression context + * @max_window_size: The maximum window size allowed for compressed frames. + * @workspace: The workspace to emplace the context into. It must outlive + * the returned context. + * @workspaceSize: The size of workspace. + * Use zstd_dstream_workspace_bound(max_window_size) to + * determine how large the workspace must be. * - * Return: The zstd streaming decompression context. + * Return: The zstd streaming decompression context. */ -ZSTD_DStream *ZSTD_initDStream_usingDDict(size_t maxWindowSize, - const ZSTD_DDict *ddict, void *workspace, size_t workspaceSize); +zstd_dstream *zstd_init_dstream(size_t max_window_size, void *workspace, + size_t workspace_size); -/*===== Streaming decompression functions =====*/ /** - * ZSTD_resetDStream() - reset the context using parameters from creation - * @zds: The zstd streaming decompression context to reset. + * zstd_reset_dstream() - reset the context using parameters from creation + * @dstream: The zstd streaming decompression context to reset. * * Resets the context using the parameters from creation. Skips dictionary * loading, since it can be reused. * - * Return: Zero or an error, which can be checked using ZSTD_isError(). + * Return: Zero or an error, which can be checked using zstd_is_error(). */ -size_t ZSTD_resetDStream(ZSTD_DStream *zds); +size_t zstd_reset_dstream(zstd_dstream *dstream); + /** - * ZSTD_decompressStream() - streaming decompress some of input into output - * @zds: The zstd streaming decompression context. - * @output: Destination buffer. `output.pos` is updated to indicate how much - * decompressed data was written. - * @input: Source buffer. `input.pos` is updated to indicate how much data was - * read. Note that it may not consume the entire input, in which case - * `input.pos < input.size`, and it's up to the caller to present - * remaining data again. + * zstd_decompress_stream() - streaming decompress some of input into output + * @dstream: The zstd streaming decompression context. + * @output: Destination buffer. `output.pos` is updated to indicate how much + * decompressed data was written. + * @input: Source buffer. `input.pos` is updated to indicate how much data was + * read. Note that it may not consume the entire input, in which case + * `input.pos < input.size`, and it's up to the caller to present + * remaining data again. * * The `input` and `output` buffers may be any size. Guaranteed to make some * forward progress if `input` and `output` are not empty. - * ZSTD_decompressStream() will not consume the last byte of the frame until + * zstd_decompress_stream() will not consume the last byte of the frame until * the entire frame is flushed. * - * Return: Returns 0 iff a frame is completely decoded and fully flushed. - * Otherwise returns a hint for the number of bytes to use as the input - * for the next function call or an error, which can be checked using - * ZSTD_isError(). The size hint will never load more than the frame. + * Return: Returns 0 iff a frame is completely decoded and fully flushed. + * Otherwise returns a hint for the number of bytes to use as the + * input for the next function call or an error, which can be checked + * using zstd_is_error(). The size hint will never load more than the + * frame. */ -size_t ZSTD_decompressStream(ZSTD_DStream *zds, ZSTD_outBuffer *output, - ZSTD_inBuffer *input); +size_t zstd_decompress_stream(zstd_dstream *dstream, zstd_out_buffer *output, + zstd_in_buffer *input); -/** - * ZSTD_DStreamInSize() - recommended size for the input buffer - * - * Return: The recommended size for the input buffer. - */ -size_t ZSTD_DStreamInSize(void); -/** - * ZSTD_DStreamOutSize() - recommended size for the output buffer - * - * When the output buffer is at least this large, it is guaranteed to be large - * enough to flush at least one complete decompressed block. - * - * Return: The recommended size for the output buffer. - */ -size_t ZSTD_DStreamOutSize(void); - - -/* --- Constants ---*/ -#define ZSTD_MAGICNUMBER 0xFD2FB528 /* >= v0.8.0 */ -#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U - -#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) -#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) - -#define ZSTD_WINDOWLOG_MAX_32 27 -#define ZSTD_WINDOWLOG_MAX_64 27 -#define ZSTD_WINDOWLOG_MAX \ - ((unsigned int)(sizeof(size_t) == 4 \ - ? ZSTD_WINDOWLOG_MAX_32 \ - : ZSTD_WINDOWLOG_MAX_64)) -#define ZSTD_WINDOWLOG_MIN 10 -#define ZSTD_HASHLOG_MAX ZSTD_WINDOWLOG_MAX -#define ZSTD_HASHLOG_MIN 6 -#define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1) -#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN -#define ZSTD_HASHLOG3_MAX 17 -#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) -#define ZSTD_SEARCHLOG_MIN 1 -/* only for ZSTD_fast, other strategies are limited to 6 */ -#define ZSTD_SEARCHLENGTH_MAX 7 -/* only for ZSTD_btopt, other strategies are limited to 4 */ -#define ZSTD_SEARCHLENGTH_MIN 3 -#define ZSTD_TARGETLENGTH_MIN 4 -#define ZSTD_TARGETLENGTH_MAX 999 - -/* for static allocation */ -#define ZSTD_FRAMEHEADERSIZE_MAX 18 -#define ZSTD_FRAMEHEADERSIZE_MIN 6 -#define ZSTD_frameHeaderSize_prefix 5 -#define ZSTD_frameHeaderSize_min ZSTD_FRAMEHEADERSIZE_MIN -#define ZSTD_frameHeaderSize_max ZSTD_FRAMEHEADERSIZE_MAX -/* magic number + skippable frame length */ -#define ZSTD_skippableHeaderSize 8 - - -/*-************************************* - * Compressed size functions - **************************************/ - -/** - * ZSTD_findFrameCompressedSize() - returns the size of a compressed frame - * @src: Source buffer. It should point to the start of a zstd encoded frame - * or a skippable frame. - * @srcSize: The size of the source buffer. It must be at least as large as the - * size of the frame. - * - * Return: The compressed size of the frame pointed to by `src` or an error, - * which can be check with ZSTD_isError(). - * Suitable to pass to ZSTD_decompress() or similar functions. - */ -size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize); - -/*-************************************* - * Decompressed size functions - **************************************/ -/** - * ZSTD_getFrameContentSize() - returns the content size in a zstd frame header - * @src: It should point to the start of a zstd encoded frame. - * @srcSize: The size of the source buffer. It must be at least as large as the - * frame header. `ZSTD_frameHeaderSize_max` is always large enough. - * - * Return: The frame content size stored in the frame header if known. - * `ZSTD_CONTENTSIZE_UNKNOWN` if the content size isn't stored in the - * frame header. `ZSTD_CONTENTSIZE_ERROR` on invalid input. - */ -unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); +/* ====== Frame Inspection Functions ====== */ /** - * ZSTD_findDecompressedSize() - returns decompressed size of a series of frames - * @src: It should point to the start of a series of zstd encoded and/or - * skippable frames. - * @srcSize: The exact size of the series of frames. + * zstd_find_frame_compressed_size() - returns the size of a compressed frame + * @src: Source buffer. It should point to the start of a zstd encoded + * frame or a skippable frame. + * @src_size: The size of the source buffer. It must be at least as large as the + * size of the frame. * - * If any zstd encoded frame in the series doesn't have the frame content size - * set, `ZSTD_CONTENTSIZE_UNKNOWN` is returned. But frame content size is always - * set when using ZSTD_compress(). The decompressed size can be very large. - * If the source is untrusted, the decompressed size could be wrong or - * intentionally modified. Always ensure the result fits within the - * application's authorized limits. ZSTD_findDecompressedSize() handles multiple - * frames, and so it must traverse the input to read each frame header. This is - * efficient as most of the data is skipped, however it does mean that all frame - * data must be present and valid. - * - * Return: Decompressed size of all the data contained in the frames if known. - * `ZSTD_CONTENTSIZE_UNKNOWN` if the decompressed size is unknown. - * `ZSTD_CONTENTSIZE_ERROR` if an error occurred. - */ -unsigned long long ZSTD_findDecompressedSize(const void *src, size_t srcSize); - -/*-************************************* - * Advanced compression functions - **************************************/ -/** - * ZSTD_checkCParams() - ensure parameter values remain within authorized range - * @cParams: The zstd compression parameters. - * - * Return: Zero or an error, which can be checked using ZSTD_isError(). + * Return: The compressed size of the frame pointed to by `src` or an error, + * which can be check with zstd_is_error(). + * Suitable to pass to ZSTD_decompress() or similar functions. */ -size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams); +size_t zstd_find_frame_compressed_size(const void *src, size_t src_size); /** - * ZSTD_adjustCParams() - optimize parameters for a given srcSize and dictSize - * @srcSize: Optionally the estimated source size, or zero if unknown. - * @dictSize: Optionally the estimated dictionary size, or zero if unknown. - * - * Return: The optimized parameters. - */ -ZSTD_compressionParameters ZSTD_adjustCParams( - ZSTD_compressionParameters cParams, unsigned long long srcSize, - size_t dictSize); - -/*--- Advanced decompression functions ---*/ - -/** - * ZSTD_isFrame() - returns true iff the buffer starts with a valid frame - * @buffer: The source buffer to check. - * @size: The size of the source buffer, must be at least 4 bytes. - * - * Return: True iff the buffer starts with a zstd or skippable frame identifier. - */ -unsigned int ZSTD_isFrame(const void *buffer, size_t size); - -/** - * ZSTD_getDictID_fromDict() - returns the dictionary id stored in a dictionary - * @dict: The dictionary buffer. - * @dictSize: The size of the dictionary buffer. - * - * Return: The dictionary id stored within the dictionary or 0 if the - * dictionary is not a zstd dictionary. If it returns 0 the - * dictionary can still be loaded as a content-only dictionary. - */ -unsigned int ZSTD_getDictID_fromDict(const void *dict, size_t dictSize); - -/** - * ZSTD_getDictID_fromDDict() - returns the dictionary id stored in a ZSTD_DDict - * @ddict: The ddict to find the id of. - * - * Return: The dictionary id stored within `ddict` or 0 if the dictionary is not - * a zstd dictionary. If it returns 0 `ddict` will be loaded as a - * content-only dictionary. - */ -unsigned int ZSTD_getDictID_fromDDict(const ZSTD_DDict *ddict); - -/** - * ZSTD_getDictID_fromFrame() - returns the dictionary id stored in a zstd frame - * @src: Source buffer. It must be a zstd encoded frame. - * @srcSize: The size of the source buffer. It must be at least as large as the - * frame header. `ZSTD_frameHeaderSize_max` is always large enough. - * - * Return: The dictionary id required to decompress the frame stored within - * `src` or 0 if the dictionary id could not be decoded. It can return - * 0 if the frame does not require a dictionary, the dictionary id - * wasn't stored in the frame, `src` is not a zstd frame, or `srcSize` - * is too small. - */ -unsigned int ZSTD_getDictID_fromFrame(const void *src, size_t srcSize); - -/** - * struct ZSTD_frameParams - zstd frame parameters stored in the frame header - * @frameContentSize: The frame content size, or 0 if not present. + * struct zstd_frame_params - zstd frame parameters stored in the frame header + * @frameContentSize: The frame content size, or ZSTD_CONTENTSIZE_UNKNOWN if not + * present. * @windowSize: The window size, or 0 if the frame is a skippable frame. + * @blockSizeMax: The maximum block size. + * @frameType: The frame type (zstd or skippable) + * @headerSize: The size of the frame header. * @dictID: The dictionary id, or 0 if not present. * @checksumFlag: Whether a checksum was used. + * + * See zstd_lib.h. */ -typedef struct { - unsigned long long frameContentSize; - unsigned int windowSize; - unsigned int dictID; - unsigned int checksumFlag; -} ZSTD_frameParams; +typedef ZSTD_frameHeader zstd_frame_header; /** - * ZSTD_getFrameParams() - extracts parameters from a zstd or skippable frame - * @fparamsPtr: On success the frame parameters are written here. - * @src: The source buffer. It must point to a zstd or skippable frame. - * @srcSize: The size of the source buffer. `ZSTD_frameHeaderSize_max` is - * always large enough to succeed. + * zstd_get_frame_header() - extracts parameters from a zstd or skippable frame + * @params: On success the frame parameters are written here. + * @src: The source buffer. It must point to a zstd or skippable frame. + * @src_size: The size of the source buffer. * - * Return: 0 on success. If more data is required it returns how many bytes - * must be provided to make forward progress. Otherwise it returns - * an error, which can be checked using ZSTD_isError(). + * Return: 0 on success. If more data is required it returns how many bytes + * must be provided to make forward progress. Otherwise it returns + * an error, which can be checked using zstd_is_error(). */ -size_t ZSTD_getFrameParams(ZSTD_frameParams *fparamsPtr, const void *src, - size_t srcSize); - -/*-***************************************************************************** - * Buffer-less and synchronous inner streaming functions - * - * This is an advanced API, giving full control over buffer management, for - * users which need direct control over memory. - * But it's also a complex one, with many restrictions (documented below). - * Prefer using normal streaming API for an easier experience - ******************************************************************************/ - -/*-***************************************************************************** - * Buffer-less streaming compression (synchronous mode) - * - * A ZSTD_CCtx object is required to track streaming operations. - * Use ZSTD_initCCtx() to initialize a context. - * ZSTD_CCtx object can be re-used multiple times within successive compression - * operations. - * - * Start by initializing a context. - * Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary - * compression, - * or ZSTD_compressBegin_advanced(), for finer parameter control. - * It's also possible to duplicate a reference context which has already been - * initialized, using ZSTD_copyCCtx() - * - * Then, consume your input using ZSTD_compressContinue(). - * There are some important considerations to keep in mind when using this - * advanced function : - * - ZSTD_compressContinue() has no internal buffer. It uses externally provided - * buffer only. - * - Interface is synchronous : input is consumed entirely and produce 1+ - * (or more) compressed blocks. - * - Caller must ensure there is enough space in `dst` to store compressed data - * under worst case scenario. Worst case evaluation is provided by - * ZSTD_compressBound(). - * ZSTD_compressContinue() doesn't guarantee recover after a failed - * compression. - * - ZSTD_compressContinue() presumes prior input ***is still accessible and - * unmodified*** (up to maximum distance size, see WindowLog). - * It remembers all previous contiguous blocks, plus one separated memory - * segment (which can itself consists of multiple contiguous blocks) - * - ZSTD_compressContinue() detects that prior input has been overwritten when - * `src` buffer overlaps. In which case, it will "discard" the relevant memory - * section from its history. - * - * Finish a frame with ZSTD_compressEnd(), which will write the last block(s) - * and optional checksum. It's possible to use srcSize==0, in which case, it - * will write a final empty block to end the frame. Without last block mark, - * frames will be considered unfinished (corrupted) by decoders. - * - * `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress some new - * frame. - ******************************************************************************/ - -/*===== Buffer-less streaming compression functions =====*/ -size_t ZSTD_compressBegin(ZSTD_CCtx *cctx, int compressionLevel); -size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx *cctx, const void *dict, - size_t dictSize, int compressionLevel); -size_t ZSTD_compressBegin_advanced(ZSTD_CCtx *cctx, const void *dict, - size_t dictSize, ZSTD_parameters params, - unsigned long long pledgedSrcSize); -size_t ZSTD_copyCCtx(ZSTD_CCtx *cctx, const ZSTD_CCtx *preparedCCtx, - unsigned long long pledgedSrcSize); -size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx *cctx, const ZSTD_CDict *cdict, - unsigned long long pledgedSrcSize); -size_t ZSTD_compressContinue(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, - const void *src, size_t srcSize); -size_t ZSTD_compressEnd(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, - const void *src, size_t srcSize); - - - -/*-***************************************************************************** - * Buffer-less streaming decompression (synchronous mode) - * - * A ZSTD_DCtx object is required to track streaming operations. - * Use ZSTD_initDCtx() to initialize a context. - * A ZSTD_DCtx object can be re-used multiple times. - * - * First typical operation is to retrieve frame parameters, using - * ZSTD_getFrameParams(). It fills a ZSTD_frameParams structure which provide - * important information to correctly decode the frame, such as the minimum - * rolling buffer size to allocate to decompress data (`windowSize`), and the - * dictionary ID used. - * Note: content size is optional, it may not be present. 0 means unknown. - * Note that these values could be wrong, either because of data malformation, - * or because an attacker is spoofing deliberate false information. As a - * consequence, check that values remain within valid application range, - * especially `windowSize`, before allocation. Each application can set its own - * limit, depending on local restrictions. For extended interoperability, it is - * recommended to support at least 8 MB. - * Frame parameters are extracted from the beginning of the compressed frame. - * Data fragment must be large enough to ensure successful decoding, typically - * `ZSTD_frameHeaderSize_max` bytes. - * Result: 0: successful decoding, the `ZSTD_frameParams` structure is filled. - * >0: `srcSize` is too small, provide at least this many bytes. - * errorCode, which can be tested using ZSTD_isError(). - * - * Start decompression, with ZSTD_decompressBegin() or - * ZSTD_decompressBegin_usingDict(). Alternatively, you can copy a prepared - * context, using ZSTD_copyDCtx(). - * - * Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() - * alternatively. - * ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' - * to ZSTD_decompressContinue(). - * ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will - * fail. - * - * The result of ZSTD_decompressContinue() is the number of bytes regenerated - * within 'dst' (necessarily <= dstCapacity). It can be zero, which is not an - * error; it just means ZSTD_decompressContinue() has decoded some metadata - * item. It can also be an error code, which can be tested with ZSTD_isError(). - * - * ZSTD_decompressContinue() needs previous data blocks during decompression, up - * to `windowSize`. They should preferably be located contiguously, prior to - * current block. Alternatively, a round buffer of sufficient size is also - * possible. Sufficient size is determined by frame parameters. - * ZSTD_decompressContinue() is very sensitive to contiguity, if 2 blocks don't - * follow each other, make sure that either the compressor breaks contiguity at - * the same place, or that previous contiguous segment is large enough to - * properly handle maximum back-reference. - * - * A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. - * Context can then be reset to start a new decompression. - * - * Note: it's possible to know if next input to present is a header or a block, - * using ZSTD_nextInputType(). This information is not required to properly - * decode a frame. - * - * == Special case: skippable frames == - * - * Skippable frames allow integration of user-defined data into a flow of - * concatenated frames. Skippable frames will be ignored (skipped) by a - * decompressor. The format of skippable frames is as follows: - * a) Skippable frame ID - 4 Bytes, Little endian format, any value from - * 0x184D2A50 to 0x184D2A5F - * b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits - * c) Frame Content - any content (User Data) of length equal to Frame Size - * For skippable frames ZSTD_decompressContinue() always returns 0. - * For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 - * what means that a frame is skippable. - * Note: If fparamsPtr->frameContentSize==0, it is ambiguous: the frame might - * actually be a zstd encoded frame with no content. For purposes of - * decompression, it is valid in both cases to skip the frame using - * ZSTD_findFrameCompressedSize() to find its size in bytes. - * It also returns frame size as fparamsPtr->frameContentSize. - ******************************************************************************/ - -/*===== Buffer-less streaming decompression functions =====*/ -size_t ZSTD_decompressBegin(ZSTD_DCtx *dctx); -size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx *dctx, const void *dict, - size_t dictSize); -void ZSTD_copyDCtx(ZSTD_DCtx *dctx, const ZSTD_DCtx *preparedDCtx); -size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx *dctx); -size_t ZSTD_decompressContinue(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, - const void *src, size_t srcSize); -typedef enum { - ZSTDnit_frameHeader, - ZSTDnit_blockHeader, - ZSTDnit_block, - ZSTDnit_lastBlock, - ZSTDnit_checksum, - ZSTDnit_skippableFrame -} ZSTD_nextInputType_e; -ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx *dctx); - -/*-***************************************************************************** - * Block functions - * - * Block functions produce and decode raw zstd blocks, without frame metadata. - * Frame metadata cost is typically ~18 bytes, which can be non-negligible for - * very small blocks (< 100 bytes). User will have to take in charge required - * information to regenerate data, such as compressed and content sizes. - * - * A few rules to respect: - * - Compressing and decompressing require a context structure - * + Use ZSTD_initCCtx() and ZSTD_initDCtx() - * - It is necessary to init context before starting - * + compression : ZSTD_compressBegin() - * + decompression : ZSTD_decompressBegin() - * + variants _usingDict() are also allowed - * + copyCCtx() and copyDCtx() work too - * - Block size is limited, it must be <= ZSTD_getBlockSizeMax() - * + If you need to compress more, cut data into multiple blocks - * + Consider using the regular ZSTD_compress() instead, as frame metadata - * costs become negligible when source size is large. - * - When a block is considered not compressible enough, ZSTD_compressBlock() - * result will be zero. In which case, nothing is produced into `dst`. - * + User must test for such outcome and deal directly with uncompressed data - * + ZSTD_decompressBlock() doesn't accept uncompressed data as input!!! - * + In case of multiple successive blocks, decoder must be informed of - * uncompressed block existence to follow proper history. Use - * ZSTD_insertBlock() in such a case. - ******************************************************************************/ - -/* Define for static allocation */ -#define ZSTD_BLOCKSIZE_ABSOLUTEMAX (128 * 1024) -/*===== Raw zstd block functions =====*/ -size_t ZSTD_getBlockSizeMax(ZSTD_CCtx *cctx); -size_t ZSTD_compressBlock(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, - const void *src, size_t srcSize); -size_t ZSTD_decompressBlock(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, - const void *src, size_t srcSize); -size_t ZSTD_insertBlock(ZSTD_DCtx *dctx, const void *blockStart, - size_t blockSize); +size_t zstd_get_frame_header(zstd_frame_header *params, const void *src, + size_t src_size); -#endif /* ZSTD_H */ +#endif /* LINUX_ZSTD_H */ diff --git a/include/linux/zstd_errors.h b/include/linux/zstd_errors.h new file mode 100644 index 000000000000..58b6dd45a969 --- /dev/null +++ b/include/linux/zstd_errors.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_ERRORS_H_398273423 +#define ZSTD_ERRORS_H_398273423 + + +/*===== dependency =====*/ +#include /* size_t */ + + +/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */ +#define ZSTDERRORLIB_VISIBILITY +#define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY + +/*-********************************************* + * Error codes list + *-********************************************* + * Error codes _values_ are pinned down since v1.3.1 only. + * Therefore, don't rely on values if you may link to any version < v1.3.1. + * + * Only values < 100 are considered stable. + * + * note 1 : this API shall be used with static linking only. + * dynamic linking is not yet officially supported. + * note 2 : Prefer relying on the enum than on its value whenever possible + * This is the only supported way to use the error list < v1.3.1 + * note 3 : ZSTD_isError() is always correct, whatever the library version. + **********************************************/ +typedef enum { + ZSTD_error_no_error = 0, + ZSTD_error_GENERIC = 1, + ZSTD_error_prefix_unknown = 10, + ZSTD_error_version_unsupported = 12, + ZSTD_error_frameParameter_unsupported = 14, + ZSTD_error_frameParameter_windowTooLarge = 16, + ZSTD_error_corruption_detected = 20, + ZSTD_error_checksum_wrong = 22, + ZSTD_error_dictionary_corrupted = 30, + ZSTD_error_dictionary_wrong = 32, + ZSTD_error_dictionaryCreation_failed = 34, + ZSTD_error_parameter_unsupported = 40, + ZSTD_error_parameter_outOfBound = 42, + ZSTD_error_tableLog_tooLarge = 44, + ZSTD_error_maxSymbolValue_tooLarge = 46, + ZSTD_error_maxSymbolValue_tooSmall = 48, + ZSTD_error_stage_wrong = 60, + ZSTD_error_init_missing = 62, + ZSTD_error_memory_allocation = 64, + ZSTD_error_workSpace_tooSmall= 66, + ZSTD_error_dstSize_tooSmall = 70, + ZSTD_error_srcSize_wrong = 72, + ZSTD_error_dstBuffer_null = 74, + /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */ + ZSTD_error_frameIndex_tooLarge = 100, + ZSTD_error_seekableIO = 102, + ZSTD_error_dstBuffer_wrong = 104, + ZSTD_error_srcBuffer_wrong = 105, + ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */ +} ZSTD_ErrorCode; + +/*! ZSTD_getErrorCode() : + convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, + which can be used to compare with enum list published above */ +ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); +ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /*< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */ + + + +#endif /* ZSTD_ERRORS_H_398273423 */ diff --git a/include/linux/zstd_lib.h b/include/linux/zstd_lib.h new file mode 100644 index 000000000000..b8c7dbf98390 --- /dev/null +++ b/include/linux/zstd_lib.h @@ -0,0 +1,2432 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_H_235446 +#define ZSTD_H_235446 + +/* ====== Dependency ======*/ +#include /* INT_MAX */ +#include /* size_t */ + + +/* ===== ZSTDLIB_API : control library symbols visibility ===== */ +#define ZSTDLIB_VISIBILITY +#define ZSTDLIB_API ZSTDLIB_VISIBILITY + + +/* ***************************************************************************** + Introduction + + zstd, short for Zstandard, is a fast lossless compression algorithm, targeting + real-time compression scenarios at zlib-level and better compression ratios. + The zstd compression library provides in-memory compression and decompression + functions. + + The library supports regular compression levels from 1 up to ZSTD_maxCLevel(), + which is currently 22. Levels >= 20, labeled `--ultra`, should be used with + caution, as they require more memory. The library also offers negative + compression levels, which extend the range of speed vs. ratio preferences. + The lower the level, the faster the speed (at the cost of compression). + + Compression can be done in: + - a single step (described as Simple API) + - a single step, reusing a context (described as Explicit context) + - unbounded multiple steps (described as Streaming compression) + + The compression ratio achievable on small data can be highly improved using + a dictionary. Dictionary compression can be performed in: + - a single step (described as Simple dictionary API) + - a single step, reusing a dictionary (described as Bulk-processing + dictionary API) + + Advanced experimental functions can be accessed using + `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h. + + Advanced experimental APIs should never be used with a dynamically-linked + library. They are not "stable"; their definitions or signatures may change in + the future. Only static linking is allowed. +*******************************************************************************/ + +/*------ Version ------*/ +#define ZSTD_VERSION_MAJOR 1 +#define ZSTD_VERSION_MINOR 4 +#define ZSTD_VERSION_RELEASE 10 +#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) + +/*! ZSTD_versionNumber() : + * Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). */ +ZSTDLIB_API unsigned ZSTD_versionNumber(void); + +#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE +#define ZSTD_QUOTE(str) #str +#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) +#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) + +/*! ZSTD_versionString() : + * Return runtime library version, like "1.4.5". Requires v1.3.0+. */ +ZSTDLIB_API const char* ZSTD_versionString(void); + +/* ************************************* + * Default constant + ***************************************/ +#ifndef ZSTD_CLEVEL_DEFAULT +# define ZSTD_CLEVEL_DEFAULT 3 +#endif + +/* ************************************* + * Constants + ***************************************/ + +/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */ +#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */ +#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */ +#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */ +#define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0 + +#define ZSTD_BLOCKSIZELOG_MAX 17 +#define ZSTD_BLOCKSIZE_MAX (1<= `ZSTD_compressBound(srcSize)`. + * @return : compressed size written into `dst` (<= `dstCapacity), + * or an error code if it fails (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel); + +/*! ZSTD_decompress() : + * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. + * `dstCapacity` is an upper bound of originalSize to regenerate. + * If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. + * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), + * or an errorCode if it fails (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, + const void* src, size_t compressedSize); + +/*! ZSTD_getFrameContentSize() : requires v1.3.0+ + * `src` should point to the start of a ZSTD encoded frame. + * `srcSize` must be at least as large as the frame header. + * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. + * @return : - decompressed size of `src` frame content, if known + * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined + * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) + * note 1 : a 0 return value means the frame is valid but "empty". + * note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode. + * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. + * In which case, it's necessary to use streaming mode to decompress data. + * Optionally, application can rely on some implicit limit, + * as ZSTD_decompress() only needs an upper bound of decompressed size. + * (For example, data could be necessarily cut into blocks <= 16 KB). + * note 3 : decompressed size is always present when compression is completed using single-pass functions, + * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict(). + * note 4 : decompressed size can be very large (64-bits value), + * potentially larger than what local system can handle as a single memory segment. + * In which case, it's necessary to use streaming mode to decompress data. + * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified. + * Always ensure return value fits within application's authorized limits. + * Each application can set its own limits. + * note 6 : This function replaces ZSTD_getDecompressedSize() */ +#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) +#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) +ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); + +/*! ZSTD_getDecompressedSize() : + * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize(). + * Both functions work the same way, but ZSTD_getDecompressedSize() blends + * "empty", "unknown" and "error" results to the same return value (0), + * while ZSTD_getFrameContentSize() gives them separate return values. + * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */ +ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); + +/*! ZSTD_findFrameCompressedSize() : + * `src` should point to the start of a ZSTD frame or skippable frame. + * `srcSize` must be >= first frame size + * @return : the compressed size of the first frame starting at `src`, + * suitable to pass as `srcSize` to `ZSTD_decompress` or similar, + * or an error code if input is invalid */ +ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); + + +/*====== Helper functions ======*/ +#define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ +ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ +ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ +ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */ +ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed */ +ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ + + +/* ************************************* +* Explicit context +***************************************/ +/*= Compression context + * When compressing many times, + * it is recommended to allocate a context just once, + * and re-use it for each successive compression operation. + * This will make workload friendlier for system's memory. + * Note : re-using context is just a speed / resource optimization. + * It doesn't change the compression ratio, which remains identical. + * Note 2 : In multi-threaded environments, + * use one different context per thread for parallel execution. + */ +typedef struct ZSTD_CCtx_s ZSTD_CCtx; +ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); +ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /* accept NULL pointer */ + +/*! ZSTD_compressCCtx() : + * Same as ZSTD_compress(), using an explicit ZSTD_CCtx. + * Important : in order to behave similarly to `ZSTD_compress()`, + * this function compresses at requested compression level, + * __ignoring any other parameter__ . + * If any advanced parameter was set using the advanced API, + * they will all be reset. Only `compressionLevel` remains. + */ +ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel); + +/*= Decompression context + * When decompressing many times, + * it is recommended to allocate a context only once, + * and re-use it for each successive compression operation. + * This will make workload friendlier for system's memory. + * Use one context per thread for parallel execution. */ +typedef struct ZSTD_DCtx_s ZSTD_DCtx; +ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); +ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /* accept NULL pointer */ + +/*! ZSTD_decompressDCtx() : + * Same as ZSTD_decompress(), + * requires an allocated ZSTD_DCtx. + * Compatible with sticky parameters. + */ +ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +/* ************************************* +* Advanced compression API +***************************************/ + +/* API design : + * Parameters are pushed one by one into an existing context, + * using ZSTD_CCtx_set*() functions. + * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame. + * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` ! + * __They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()__ . + * + * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset(). + * + * This API supercedes all other "advanced" API entry points in the experimental section. + * In the future, we expect to remove from experimental API entry points which are redundant with this API. + */ + + +/* Compression strategies, listed from fastest to strongest */ +typedef enum { ZSTD_fast=1, + ZSTD_dfast=2, + ZSTD_greedy=3, + ZSTD_lazy=4, + ZSTD_lazy2=5, + ZSTD_btlazy2=6, + ZSTD_btopt=7, + ZSTD_btultra=8, + ZSTD_btultra2=9 + /* note : new strategies _might_ be added in the future. + Only the order (from fast to strong) is guaranteed */ +} ZSTD_strategy; + + +typedef enum { + + /* compression parameters + * Note: When compressing with a ZSTD_CDict these parameters are superseded + * by the parameters used to construct the ZSTD_CDict. + * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */ + ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table. + * Note that exact compression parameters are dynamically determined, + * depending on both compression level and srcSize (when known). + * Default level is ZSTD_CLEVEL_DEFAULT==3. + * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. + * Note 1 : it's possible to pass a negative compression level. + * Note 2 : setting a level does not automatically set all other compression parameters + * to default. Setting this will however eventually dynamically impact the compression + * parameters which have not been manually set. The manually set + * ones will 'stick'. */ + /* Advanced compression parameters : + * It's possible to pin down compression parameters to some specific values. + * In which case, these values are no longer dynamically selected by the compressor */ + ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2. + * This will set a memory budget for streaming decompression, + * with larger values requiring more memory + * and typically compressing more. + * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX. + * Special: value 0 means "use default windowLog". + * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT + * requires explicitly allowing such size at streaming decompression stage. */ + ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2. + * Resulting memory usage is (1 << (hashLog+2)). + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. + * Larger tables improve compression ratio of strategies <= dFast, + * and improve speed of strategies > dFast. + * Special: value 0 means "use default hashLog". */ + ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2. + * Resulting memory usage is (1 << (chainLog+2)). + * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX. + * Larger tables result in better and slower compression. + * This parameter is useless for "fast" strategy. + * It's still useful when using "dfast" strategy, + * in which case it defines a secondary probe table. + * Special: value 0 means "use default chainLog". */ + ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2. + * More attempts result in better and slower compression. + * This parameter is useless for "fast" and "dFast" strategies. + * Special: value 0 means "use default searchLog". */ + ZSTD_c_minMatch=105, /* Minimum size of searched matches. + * Note that Zstandard can still find matches of smaller size, + * it just tweaks its search algorithm to look for this size and larger. + * Larger values increase compression and decompression speed, but decrease ratio. + * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX. + * Note that currently, for all strategies < btopt, effective minimum is 4. + * , for all strategies > fast, effective maximum is 6. + * Special: value 0 means "use default minMatchLength". */ + ZSTD_c_targetLength=106, /* Impact of this field depends on strategy. + * For strategies btopt, btultra & btultra2: + * Length of Match considered "good enough" to stop search. + * Larger values make compression stronger, and slower. + * For strategy fast: + * Distance between match sampling. + * Larger values make compression faster, and weaker. + * Special: value 0 means "use default targetLength". */ + ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition. + * The higher the value of selected strategy, the more complex it is, + * resulting in stronger and slower compression. + * Special: value 0 means "use default strategy". */ + + /* LDM mode parameters */ + ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching. + * This parameter is designed to improve compression ratio + * for large inputs, by finding large matches at long distance. + * It increases memory usage and window size. + * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB + * except when expressly set to a different value. + * Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and + * compression strategy >= ZSTD_btopt (== compression level 16+) */ + ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2. + * Larger values increase memory usage and compression ratio, + * but decrease compression speed. + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX + * default: windowlog - 7. + * Special: value 0 means "automatically determine hashlog". */ + ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher. + * Larger/too small values usually decrease compression ratio. + * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX. + * Special: value 0 means "use default value" (default: 64). */ + ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution. + * Larger values improve collision resolution but decrease compression speed. + * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX. + * Special: value 0 means "use default value" (default: 3). */ + ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table. + * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN). + * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage. + * Larger values improve compression speed. + * Deviating far from default value will likely result in a compression ratio decrease. + * Special: value 0 means "automatically determine hashRateLog". */ + + /* frame parameters */ + ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1) + * Content size must be known at the beginning of compression. + * This is automatically the case when using ZSTD_compress2(), + * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */ + ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */ + ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */ + + /* multi-threading parameters */ + /* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD). + * Otherwise, trying to set any other value than default (0) will be a no-op and return an error. + * In a situation where it's unknown if the linked library supports multi-threading or not, + * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property. + */ + ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel. + * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() : + * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller, + * while compression is performed in parallel, within worker thread(s). + * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end : + * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call). + * More workers improve speed, but also increase memory usage. + * Default value is `0`, aka "single-threaded mode" : no worker is spawned, + * compression is performed inside Caller's thread, and all invocations are blocking */ + ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1. + * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads. + * 0 means default, which is dynamically determined based on compression parameters. + * Job size must be a minimum of overlap size, or 1 MB, whichever is largest. + * The minimum size is automatically and transparently enforced. */ + ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size. + * The overlap size is an amount of data reloaded from previous job at the beginning of a new job. + * It helps preserve compression ratio, while each job is compressed in parallel. + * This value is enforced only when nbWorkers >= 1. + * Larger values increase compression ratio, but decrease speed. + * Possible values range from 0 to 9 : + * - 0 means "default" : value will be determined by the library, depending on strategy + * - 1 means "no overlap" + * - 9 means "full overlap", using a full window size. + * Each intermediate rank increases/decreases load size by a factor 2 : + * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default + * default value varies between 6 and 9, depending on strategy */ + + /* note : additional experimental parameters are also available + * within the experimental section of the API. + * At the time of this writing, they include : + * ZSTD_c_rsyncable + * ZSTD_c_format + * ZSTD_c_forceMaxWindow + * ZSTD_c_forceAttachDict + * ZSTD_c_literalCompressionMode + * ZSTD_c_targetCBlockSize + * ZSTD_c_srcSizeHint + * ZSTD_c_enableDedicatedDictSearch + * ZSTD_c_stableInBuffer + * ZSTD_c_stableOutBuffer + * ZSTD_c_blockDelimiters + * ZSTD_c_validateSequences + * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. + * note : never ever use experimentalParam? names directly; + * also, the enums values themselves are unstable and can still change. + */ + ZSTD_c_experimentalParam1=500, + ZSTD_c_experimentalParam2=10, + ZSTD_c_experimentalParam3=1000, + ZSTD_c_experimentalParam4=1001, + ZSTD_c_experimentalParam5=1002, + ZSTD_c_experimentalParam6=1003, + ZSTD_c_experimentalParam7=1004, + ZSTD_c_experimentalParam8=1005, + ZSTD_c_experimentalParam9=1006, + ZSTD_c_experimentalParam10=1007, + ZSTD_c_experimentalParam11=1008, + ZSTD_c_experimentalParam12=1009 +} ZSTD_cParameter; + +typedef struct { + size_t error; + int lowerBound; + int upperBound; +} ZSTD_bounds; + +/*! ZSTD_cParam_getBounds() : + * All parameters must belong to an interval with lower and upper bounds, + * otherwise they will either trigger an error or be automatically clamped. + * @return : a structure, ZSTD_bounds, which contains + * - an error status field, which must be tested using ZSTD_isError() + * - lower and upper bounds, both inclusive + */ +ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam); + +/*! ZSTD_CCtx_setParameter() : + * Set one compression parameter, selected by enum ZSTD_cParameter. + * All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds(). + * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). + * Setting a parameter is generally only possible during frame initialization (before starting compression). + * Exception : when using multi-threading mode (nbWorkers >= 1), + * the following parameters can be updated _during_ compression (within same frame): + * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy. + * new parameters will be active for next job only (after a flush()). + * @return : an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value); + +/*! ZSTD_CCtx_setPledgedSrcSize() : + * Total input data size to be compressed as a single frame. + * Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag. + * This value will also be controlled at end of frame, and trigger an error if not respected. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame. + * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN. + * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame. + * Note 2 : pledgedSrcSize is only valid once, for the next frame. + * It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN. + * Note 3 : Whenever all input data is provided and consumed in a single round, + * for example with ZSTD_compress2(), + * or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end), + * this value is automatically overridden by srcSize instead. + */ +ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize); + +typedef enum { + ZSTD_reset_session_only = 1, + ZSTD_reset_parameters = 2, + ZSTD_reset_session_and_parameters = 3 +} ZSTD_ResetDirective; + +/*! ZSTD_CCtx_reset() : + * There are 2 different things that can be reset, independently or jointly : + * - The session : will stop compressing current frame, and make CCtx ready to start a new one. + * Useful after an error, or to interrupt any ongoing compression. + * Any internal data not yet flushed is cancelled. + * Compression parameters and dictionary remain unchanged. + * They will be used to compress next frame. + * Resetting session never fails. + * - The parameters : changes all parameters back to "default". + * This removes any reference to any dictionary too. + * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing) + * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError()) + * - Both : similar to resetting the session, followed by resetting parameters. + */ +ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset); + +/*! ZSTD_compress2() : + * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API. + * ZSTD_compress2() always starts a new frame. + * Should cctx hold data from a previously unfinished frame, everything about it is forgotten. + * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() + * - The function is always blocking, returns when compression is completed. + * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. + * @return : compressed size written into `dst` (<= `dstCapacity), + * or an error code if it fails (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +/* ************************************* +* Advanced decompression API +***************************************/ + +/* The advanced API pushes parameters one by one into an existing DCtx context. + * Parameters are sticky, and remain valid for all following frames + * using the same DCtx context. + * It's possible to reset parameters to default values using ZSTD_DCtx_reset(). + * Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream(). + * Therefore, no new decompression function is necessary. + */ + +typedef enum { + + ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which + * the streaming API will refuse to allocate memory buffer + * in order to protect the host from unreasonable memory requirements. + * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. + * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT). + * Special: value 0 means "use default maximum windowLog". */ + + /* note : additional experimental parameters are also available + * within the experimental section of the API. + * At the time of this writing, they include : + * ZSTD_d_format + * ZSTD_d_stableOutBuffer + * ZSTD_d_forceIgnoreChecksum + * ZSTD_d_refMultipleDDicts + * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. + * note : never ever use experimentalParam? names directly + */ + ZSTD_d_experimentalParam1=1000, + ZSTD_d_experimentalParam2=1001, + ZSTD_d_experimentalParam3=1002, + ZSTD_d_experimentalParam4=1003 + +} ZSTD_dParameter; + +/*! ZSTD_dParam_getBounds() : + * All parameters must belong to an interval with lower and upper bounds, + * otherwise they will either trigger an error or be automatically clamped. + * @return : a structure, ZSTD_bounds, which contains + * - an error status field, which must be tested using ZSTD_isError() + * - both lower and upper bounds, inclusive + */ +ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam); + +/*! ZSTD_DCtx_setParameter() : + * Set one compression parameter, selected by enum ZSTD_dParameter. + * All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds(). + * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). + * Setting a parameter is only possible during frame initialization (before starting decompression). + * @return : 0, or an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value); + +/*! ZSTD_DCtx_reset() : + * Return a DCtx to clean state. + * Session and parameters can be reset jointly or separately. + * Parameters can only be reset when no active frame is being decompressed. + * @return : 0, or an error code, which can be tested with ZSTD_isError() + */ +ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset); + + +/* ************************** +* Streaming +****************************/ + +typedef struct ZSTD_inBuffer_s { + const void* src; /*< start of input buffer */ + size_t size; /*< size of input buffer */ + size_t pos; /*< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_inBuffer; + +typedef struct ZSTD_outBuffer_s { + void* dst; /*< start of output buffer */ + size_t size; /*< size of output buffer */ + size_t pos; /*< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_outBuffer; + + + +/*-*********************************************************************** +* Streaming compression - HowTo +* +* A ZSTD_CStream object is required to track streaming operation. +* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. +* ZSTD_CStream objects can be reused multiple times on consecutive compression operations. +* It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory. +* +* For parallel execution, use one separate ZSTD_CStream per thread. +* +* note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing. +* +* Parameters are sticky : when starting a new compression on the same context, +* it will re-use the same sticky parameters as previous compression session. +* When in doubt, it's recommended to fully initialize the context before usage. +* Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(), +* ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to +* set more specific parameters, the pledged source size, or load a dictionary. +* +* Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to +* consume input stream. The function will automatically update both `pos` +* fields within `input` and `output`. +* Note that the function may not consume the entire input, for example, because +* the output buffer is already full, in which case `input.pos < input.size`. +* The caller must check if input has been entirely consumed. +* If not, the caller must make some room to receive more compressed data, +* and then present again remaining input data. +* note: ZSTD_e_continue is guaranteed to make some forward progress when called, +* but doesn't guarantee maximal forward progress. This is especially relevant +* when compressing with multiple threads. The call won't block if it can +* consume some input, but if it can't it will wait for some, but not all, +* output to be flushed. +* @return : provides a minimum amount of data remaining to be flushed from internal buffers +* or an error code, which can be tested using ZSTD_isError(). +* +* At any moment, it's possible to flush whatever data might remain stuck within internal buffer, +* using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated. +* Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0). +* In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush. +* You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the +* operation. +* note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will +* block until the flush is complete or the output buffer is full. +* @return : 0 if internal buffers are entirely flushed, +* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), +* or an error code, which can be tested using ZSTD_isError(). +* +* Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame. +* It will perform a flush and write frame epilogue. +* The epilogue is required for decoders to consider a frame completed. +* flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush. +* You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to +* start a new frame. +* note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will +* block until the flush is complete or the output buffer is full. +* @return : 0 if frame fully completed and fully flushed, +* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), +* or an error code, which can be tested using ZSTD_isError(). +* +* *******************************************************************/ + +typedef ZSTD_CCtx ZSTD_CStream; /*< CCtx and CStream are now effectively same object (>= v1.3.0) */ + /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */ +/*===== ZSTD_CStream management functions =====*/ +ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); +ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); /* accept NULL pointer */ + +/*===== Streaming compression functions =====*/ +typedef enum { + ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */ + ZSTD_e_flush=1, /* flush any data provided so far, + * it creates (at least) one new block, that can be decoded immediately on reception; + * frame will continue: any future data can still reference previously compressed data, improving compression. + * note : multithreaded compression will block to flush as much output as possible. */ + ZSTD_e_end=2 /* flush any remaining data _and_ close current frame. + * note that frame is only closed after compressed data is fully flushed (return value == 0). + * After that point, any additional data starts a new frame. + * note : each frame is independent (does not reference any content from previous frame). + : note : multithreaded compression will block to flush as much output as possible. */ +} ZSTD_EndDirective; + +/*! ZSTD_compressStream2() : + * Behaves about the same as ZSTD_compressStream, with additional control on end directive. + * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() + * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode) + * - output->pos must be <= dstCapacity, input->pos must be <= srcSize + * - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. + * - endOp must be a valid directive + * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller. + * - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available, + * and then immediately returns, just indicating that there is some data remaining to be flushed. + * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte. + * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking. + * - @return provides a minimum amount of data remaining to be flushed from internal buffers + * or an error code, which can be tested using ZSTD_isError(). + * if @return != 0, flush is not fully completed, there is still some data left within internal buffers. + * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers. + * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed. + * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0), + * only ZSTD_e_end or ZSTD_e_flush operations are allowed. + * Before starting a new compression job, or changing compression parameters, + * it is required to fully flush internal buffers. + */ +ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp); + + +/* These buffer sizes are softly recommended. + * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output. + * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(), + * reducing the amount of memory shuffling and buffering, resulting in minor performance savings. + * + * However, note that these recommendations are from the perspective of a C caller program. + * If the streaming interface is invoked from some other language, + * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo, + * a major performance rule is to reduce crossing such interface to an absolute minimum. + * It's not rare that performance ends being spent more into the interface, rather than compression itself. + * In which cases, prefer using large buffers, as large as practical, + * for both input and output, to reduce the nb of roundtrips. + */ +ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /*< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /*< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */ + + +/* ***************************************************************************** + * This following is a legacy streaming API. + * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2(). + * It is redundant, but remains fully supported. + * Advanced parameters and dictionary compression can only be used through the + * new API. + ******************************************************************************/ + +/*! + * Equivalent to: + * + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + */ +ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); +/*! + * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue). + * NOTE: The return value is different. ZSTD_compressStream() returns a hint for + * the next read size (if non-zero and not an error). ZSTD_compressStream2() + * returns the minimum nb of bytes left to flush (if non-zero and not an error). + */ +ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */ +ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */ +ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); + + +/*-*************************************************************************** +* Streaming decompression - HowTo +* +* A ZSTD_DStream object is required to track streaming operations. +* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. +* ZSTD_DStream objects can be re-used multiple times. +* +* Use ZSTD_initDStream() to start a new decompression operation. +* @return : recommended first input size +* Alternatively, use advanced API to set specific properties. +* +* Use ZSTD_decompressStream() repetitively to consume your input. +* The function will update both `pos` fields. +* If `input.pos < input.size`, some input has not been consumed. +* It's up to the caller to present again remaining data. +* The function tries to flush all data decoded immediately, respecting output buffer size. +* If `output.pos < output.size`, decoder has flushed everything it could. +* But if `output.pos == output.size`, there might be some data left within internal buffers., +* In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer. +* Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX. +* @return : 0 when a frame is completely decoded and fully flushed, +* or an error code, which can be tested using ZSTD_isError(), +* or any other value > 0, which means there is still some decoding or flushing to do to complete current frame : +* the return value is a suggested next input size (just a hint for better latency) +* that will never request more than the remaining frame size. +* *******************************************************************************/ + +typedef ZSTD_DCtx ZSTD_DStream; /*< DCtx and DStream are now effectively same object (>= v1.3.0) */ + /* For compatibility with versions <= v1.2.0, prefer differentiating them. */ +/*===== ZSTD_DStream management functions =====*/ +ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); +ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); /* accept NULL pointer */ + +/*===== Streaming decompression functions =====*/ + +/* This function is redundant with the advanced API and equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_refDDict(zds, NULL); + */ +ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); + +ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); + +ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ + + +/* ************************ +* Simple dictionary API +***************************/ +/*! ZSTD_compress_usingDict() : + * Compression at an explicit compression level using a Dictionary. + * A dictionary can be any arbitrary data segment (also called a prefix), + * or a buffer with specified information (see dictBuilder/zdict.h). + * Note : This function loads the dictionary, resulting in significant startup delay. + * It's intended for a dictionary used only once. + * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + int compressionLevel); + +/*! ZSTD_decompress_usingDict() : + * Decompression using a known Dictionary. + * Dictionary must be identical to the one used during compression. + * Note : This function loads the dictionary, resulting in significant startup delay. + * It's intended for a dictionary used only once. + * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize); + + +/* ********************************* + * Bulk processing dictionary API + **********************************/ +typedef struct ZSTD_CDict_s ZSTD_CDict; + +/*! ZSTD_createCDict() : + * When compressing multiple messages or blocks using the same dictionary, + * it's recommended to digest the dictionary only once, since it's a costly operation. + * ZSTD_createCDict() will create a state from digesting a dictionary. + * The resulting state can be used for future compression operations with very limited startup cost. + * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. + * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict. + * Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content. + * Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer, + * in which case the only thing that it transports is the @compressionLevel. + * This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively, + * expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */ +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, + int compressionLevel); + +/*! ZSTD_freeCDict() : + * Function frees memory allocated by ZSTD_createCDict(). + * If a NULL pointer is passed, no operation is performed. */ +ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); + +/*! ZSTD_compress_usingCDict() : + * Compression using a digested Dictionary. + * Recommended when same dictionary is used multiple times. + * Note : compression level is _decided at dictionary creation time_, + * and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */ +ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict); + + +typedef struct ZSTD_DDict_s ZSTD_DDict; + +/*! ZSTD_createDDict() : + * Create a digested dictionary, ready to start decompression operation without startup delay. + * dictBuffer can be released after DDict creation, as its content is copied inside DDict. */ +ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); + +/*! ZSTD_freeDDict() : + * Function frees memory allocated with ZSTD_createDDict() + * If a NULL pointer is passed, no operation is performed. */ +ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); + +/*! ZSTD_decompress_usingDDict() : + * Decompression using a digested Dictionary. + * Recommended when same dictionary is used multiple times. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict); + + +/* ****************************** + * Dictionary helper functions + *******************************/ + +/*! ZSTD_getDictID_fromDict() : + * Provides the dictID stored within dictionary. + * if @return == 0, the dictionary is not conformant with Zstandard specification. + * It can still be loaded, but as a content-only dictionary. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); + +/*! ZSTD_getDictID_fromDDict() : + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); + +/*! ZSTD_getDictID_fromFrame() : + * Provides the dictID required to decompressed the frame stored within `src`. + * If @return == 0, the dictID could not be decoded. + * This could for one of the following reasons : + * - The frame does not require a dictionary to be decoded (most common case). + * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information. + * Note : this use case also happens when using a non-conformant dictionary. + * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). + * - This is not a Zstandard frame. + * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); + + +/* ***************************************************************************** + * Advanced dictionary and prefix API + * + * This API allows dictionaries to be used with ZSTD_compress2(), + * ZSTD_compressStream2(), and ZSTD_decompress(). Dictionaries are sticky, and + * only reset with the context is reset with ZSTD_reset_parameters or + * ZSTD_reset_session_and_parameters. Prefixes are single-use. + ******************************************************************************/ + + +/*! ZSTD_CCtx_loadDictionary() : + * Create an internal CDict from `dict` buffer. + * Decompression will have to use same dictionary. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, + * meaning "return to no-dictionary mode". + * Note 1 : Dictionary is sticky, it will be used for all future compressed frames. + * To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters). + * Note 2 : Loading a dictionary involves building tables. + * It's also a CPU consuming operation, with non-negligible impact on latency. + * Tables are dependent on compression parameters, and for this reason, + * compression parameters can no longer be changed after loading a dictionary. + * Note 3 :`dict` content will be copied internally. + * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead. + * In such a case, dictionary buffer must outlive its users. + * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() + * to precisely select how dictionary content must be interpreted. */ +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + +/*! ZSTD_CCtx_refCDict() : + * Reference a prepared dictionary, to be used for all next compressed frames. + * Note that compression parameters are enforced from within CDict, + * and supersede any compression parameter previously set within CCtx. + * The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs. + * The ignored parameters will be used again if the CCtx is returned to no-dictionary mode. + * The dictionary will remain valid for future compressed frames using same CCtx. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special : Referencing a NULL CDict means "return to no-dictionary mode". + * Note 1 : Currently, only one dictionary can be managed. + * Referencing a new dictionary effectively "discards" any previous one. + * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */ +ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); + +/*! ZSTD_CCtx_refPrefix() : + * Reference a prefix (single-usage dictionary) for next compressed frame. + * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end). + * Decompression will need same prefix to properly regenerate data. + * Compressing with a prefix is similar in outcome as performing a diff and compressing it, + * but performs much faster, especially during decompression (compression speed is tunable with compression level). + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary + * Note 1 : Prefix buffer is referenced. It **must** outlive compression. + * Its content must remain unmodified during compression. + * Note 2 : If the intention is to diff some large src data blob with some prior version of itself, + * ensure that the window size is large enough to contain the entire source. + * See ZSTD_c_windowLog. + * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. + * It's a CPU consuming operation, with non-negligible impact on latency. + * If there is a need to use the same prefix multiple times, consider loadDictionary instead. + * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent). + * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */ +ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, + const void* prefix, size_t prefixSize); + +/*! ZSTD_DCtx_loadDictionary() : + * Create an internal DDict from dict buffer, + * to be used to decompress next frames. + * The dictionary remains valid for all future frames, until explicitly invalidated. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, + * meaning "return to no-dictionary mode". + * Note 1 : Loading a dictionary involves building tables, + * which has a non-negligible impact on CPU usage and latency. + * It's recommended to "load once, use many times", to amortize the cost + * Note 2 :`dict` content will be copied internally, so `dict` can be released after loading. + * Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead. + * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of + * how dictionary content is loaded and interpreted. + */ +ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); + +/*! ZSTD_DCtx_refDDict() : + * Reference a prepared dictionary, to be used to decompress next frames. + * The dictionary remains active for decompression of future frames using same DCtx. + * + * If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function + * will store the DDict references in a table, and the DDict used for decompression + * will be determined at decompression time, as per the dict ID in the frame. + * The memory for the table is allocated on the first call to refDDict, and can be + * freed with ZSTD_freeDCtx(). + * + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : Currently, only one dictionary can be managed. + * Referencing a new dictionary effectively "discards" any previous one. + * Special: referencing a NULL DDict means "return to no-dictionary mode". + * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx. + */ +ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + +/*! ZSTD_DCtx_refPrefix() : + * Reference a prefix (single-usage dictionary) to decompress next frame. + * This is the reverse operation of ZSTD_CCtx_refPrefix(), + * and must use the same prefix as the one used during compression. + * Prefix is **only used once**. Reference is discarded at end of frame. + * End of frame is reached when ZSTD_decompressStream() returns 0. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary + * Note 2 : Prefix buffer is referenced. It **must** outlive decompression. + * Prefix buffer must remain unmodified up to the end of frame, + * reached when ZSTD_decompressStream() returns 0. + * Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent). + * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section) + * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. + * A full dictionary is more costly, as it requires building tables. + */ +ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, + const void* prefix, size_t prefixSize); + +/* === Memory management === */ + +/*! ZSTD_sizeof_*() : + * These functions give the _current_ memory usage of selected object. + * Note that object memory usage can evolve (increase or decrease) over time. */ +ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); +ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); +ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); +ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); +ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); + +#endif /* ZSTD_H_235446 */ + + +/* ************************************************************************************** + * ADVANCED AND EXPERIMENTAL FUNCTIONS + **************************************************************************************** + * The definitions in the following section are considered experimental. + * They are provided for advanced scenarios. + * They should never be used with a dynamic library, as prototypes may change in the future. + * Use them only in association with static linking. + * ***************************************************************************************/ + +#if !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) +#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY + +/* ************************************************************************************** + * experimental API (static linking only) + **************************************************************************************** + * The following symbols and constants + * are not planned to join "stable API" status in the near future. + * They can still change in future versions. + * Some of them are planned to remain in the static_only section indefinitely. + * Some of them might be removed in the future (especially when redundant with existing stable functions) + * ***************************************************************************************/ + +#define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */ +#define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2) +#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */ +#define ZSTD_SKIPPABLEHEADERSIZE 8 + +/* compression parameter bounds */ +#define ZSTD_WINDOWLOG_MAX_32 30 +#define ZSTD_WINDOWLOG_MAX_64 31 +#define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) +#define ZSTD_WINDOWLOG_MIN 10 +#define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30) +#define ZSTD_HASHLOG_MIN 6 +#define ZSTD_CHAINLOG_MAX_32 29 +#define ZSTD_CHAINLOG_MAX_64 30 +#define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64)) +#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN +#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) +#define ZSTD_SEARCHLOG_MIN 1 +#define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ +#define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */ +#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX +#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */ +#define ZSTD_STRATEGY_MIN ZSTD_fast +#define ZSTD_STRATEGY_MAX ZSTD_btultra2 + + +#define ZSTD_OVERLAPLOG_MIN 0 +#define ZSTD_OVERLAPLOG_MAX 9 + +#define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame + * requiring larger than (1< 0: + * If litLength != 0: + * rep == 1 --> offset == repeat_offset_1 + * rep == 2 --> offset == repeat_offset_2 + * rep == 3 --> offset == repeat_offset_3 + * If litLength == 0: + * rep == 1 --> offset == repeat_offset_2 + * rep == 2 --> offset == repeat_offset_3 + * rep == 3 --> offset == repeat_offset_1 - 1 + * + * Note: This field is optional. ZSTD_generateSequences() will calculate the value of + * 'rep', but repeat offsets do not necessarily need to be calculated from an external + * sequence provider's perspective. For example, ZSTD_compressSequences() does not + * use this 'rep' field at all (as of now). + */ +} ZSTD_Sequence; + +typedef struct { + unsigned windowLog; /*< largest match distance : larger == more compression, more memory needed during decompression */ + unsigned chainLog; /*< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ + unsigned hashLog; /*< dispatch table : larger == faster, more memory */ + unsigned searchLog; /*< nb of searches : larger == more compression, slower */ + unsigned minMatch; /*< match length searched : larger == faster decompression, sometimes less compression */ + unsigned targetLength; /*< acceptable match size for optimal parser (only) : larger == more compression, slower */ + ZSTD_strategy strategy; /*< see ZSTD_strategy definition above */ +} ZSTD_compressionParameters; + +typedef struct { + int contentSizeFlag; /*< 1: content size will be in frame header (when known) */ + int checksumFlag; /*< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */ + int noDictIDFlag; /*< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */ +} ZSTD_frameParameters; + +typedef struct { + ZSTD_compressionParameters cParams; + ZSTD_frameParameters fParams; +} ZSTD_parameters; + +typedef enum { + ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */ + ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */ + ZSTD_dct_fullDict = 2 /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */ +} ZSTD_dictContentType_e; + +typedef enum { + ZSTD_dlm_byCopy = 0, /*< Copy dictionary content internally */ + ZSTD_dlm_byRef = 1 /*< Reference dictionary content -- the dictionary buffer must outlive its users. */ +} ZSTD_dictLoadMethod_e; + +typedef enum { + ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */ + ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number. + * Useful to save 4 bytes per generated frame. + * Decoder cannot recognise automatically this format, requiring this instruction. */ +} ZSTD_format_e; + +typedef enum { + /* Note: this enum controls ZSTD_d_forceIgnoreChecksum */ + ZSTD_d_validateChecksum = 0, + ZSTD_d_ignoreChecksum = 1 +} ZSTD_forceIgnoreChecksum_e; + +typedef enum { + /* Note: this enum controls ZSTD_d_refMultipleDDicts */ + ZSTD_rmd_refSingleDDict = 0, + ZSTD_rmd_refMultipleDDicts = 1 +} ZSTD_refMultipleDDicts_e; + +typedef enum { + /* Note: this enum and the behavior it controls are effectively internal + * implementation details of the compressor. They are expected to continue + * to evolve and should be considered only in the context of extremely + * advanced performance tuning. + * + * Zstd currently supports the use of a CDict in three ways: + * + * - The contents of the CDict can be copied into the working context. This + * means that the compression can search both the dictionary and input + * while operating on a single set of internal tables. This makes + * the compression faster per-byte of input. However, the initial copy of + * the CDict's tables incurs a fixed cost at the beginning of the + * compression. For small compressions (< 8 KB), that copy can dominate + * the cost of the compression. + * + * - The CDict's tables can be used in-place. In this model, compression is + * slower per input byte, because the compressor has to search two sets of + * tables. However, this model incurs no start-up cost (as long as the + * working context's tables can be reused). For small inputs, this can be + * faster than copying the CDict's tables. + * + * - The CDict's tables are not used at all, and instead we use the working + * context alone to reload the dictionary and use params based on the source + * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict(). + * This method is effective when the dictionary sizes are very small relative + * to the input size, and the input size is fairly large to begin with. + * + * Zstd has a simple internal heuristic that selects which strategy to use + * at the beginning of a compression. However, if experimentation shows that + * Zstd is making poor choices, it is possible to override that choice with + * this enum. + */ + ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ + ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ + ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ + ZSTD_dictForceLoad = 3 /* Always reload the dictionary */ +} ZSTD_dictAttachPref_e; + +typedef enum { + ZSTD_lcm_auto = 0, /*< Automatically determine the compression mode based on the compression level. + * Negative compression levels will be uncompressed, and positive compression + * levels will be compressed. */ + ZSTD_lcm_huffman = 1, /*< Always attempt Huffman compression. Uncompressed literals will still be + * emitted if Huffman compression is not profitable. */ + ZSTD_lcm_uncompressed = 2 /*< Always emit uncompressed literals. */ +} ZSTD_literalCompressionMode_e; + + +/* ************************************* +* Frame size functions +***************************************/ + +/*! ZSTD_findDecompressedSize() : + * `src` should point to the start of a series of ZSTD encoded and/or skippable frames + * `srcSize` must be the _exact_ size of this series + * (i.e. there should be a frame boundary at `src + srcSize`) + * @return : - decompressed size of all data in all successive frames + * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN + * - if an error occurred: ZSTD_CONTENTSIZE_ERROR + * + * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. + * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. + * In which case, it's necessary to use streaming mode to decompress data. + * note 2 : decompressed size is always present when compression is done with ZSTD_compress() + * note 3 : decompressed size can be very large (64-bits value), + * potentially larger than what local system can handle as a single memory segment. + * In which case, it's necessary to use streaming mode to decompress data. + * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. + * Always ensure result fits within application's authorized limits. + * Each application can set its own limits. + * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to + * read each contained frame header. This is fast as most of the data is skipped, + * however it does mean that all frame data must be present and valid. */ +ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); + +/*! ZSTD_decompressBound() : + * `src` should point to the start of a series of ZSTD encoded and/or skippable frames + * `srcSize` must be the _exact_ size of this series + * (i.e. there should be a frame boundary at `src + srcSize`) + * @return : - upper-bound for the decompressed size of all data in all successive frames + * - if an error occurred: ZSTD_CONTENTSIZE_ERROR + * + * note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame. + * note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`. + * in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value. + * note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by: + * upper-bound = # blocks * min(128 KB, Window_Size) + */ +ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize); + +/*! ZSTD_frameHeaderSize() : + * srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX. + * @return : size of the Frame Header, + * or an error code (if srcSize is too small) */ +ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); + +typedef enum { + ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */ + ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */ +} ZSTD_sequenceFormat_e; + +/*! ZSTD_generateSequences() : + * Generate sequences using ZSTD_compress2, given a source buffer. + * + * Each block will end with a dummy sequence + * with offset == 0, matchLength == 0, and litLength == length of last literals. + * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0) + * simply acts as a block delimiter. + * + * zc can be used to insert custom compression params. + * This function invokes ZSTD_compress2 + * + * The output of this function can be fed into ZSTD_compressSequences() with CCtx + * setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters + * @return : number of sequences generated + */ + +ZSTDLIB_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, + size_t outSeqsSize, const void* src, size_t srcSize); + +/*! ZSTD_mergeBlockDelimiters() : + * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals + * by merging them into into the literals of the next sequence. + * + * As such, the final generated result has no explicit representation of block boundaries, + * and the final last literals segment is not represented in the sequences. + * + * The output of this function can be fed into ZSTD_compressSequences() with CCtx + * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters + * @return : number of sequences left after merging + */ +ZSTDLIB_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize); + +/*! ZSTD_compressSequences() : + * Compress an array of ZSTD_Sequence, generated from the original source buffer, into dst. + * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.) + * The entire source is compressed into a single frame. + * + * The compression behavior changes based on cctx params. In particular: + * If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain + * no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on + * the block size derived from the cctx, and sequences may be split. This is the default setting. + * + * If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain + * block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided. + * + * If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined + * behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for + * specifics regarding offset/matchlength requirements) then the function will bail out and return an error. + * + * In addition to the two adjustable experimental params, there are other important cctx params. + * - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN. + * - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression. + * - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset + * is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md + * + * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused. + * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly, + * and cannot emit an RLE block that disagrees with the repcode history + * @return : final compressed size or a ZSTD error. + */ +ZSTDLIB_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize); + + +/*! ZSTD_writeSkippableFrame() : + * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer. + * + * Skippable frames begin with a a 4-byte magic number. There are 16 possible choices of magic number, + * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15. + * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so + * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant. + * + * Returns an error if destination buffer is not large enough, if the source size is not representable + * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid). + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, unsigned magicVariant); + + +/* ************************************* +* Memory management +***************************************/ + +/*! ZSTD_estimate*() : + * These functions make it possible to estimate memory usage + * of a future {D,C}Ctx, before its creation. + * + * ZSTD_estimateCCtxSize() will provide a memory budget large enough + * for any compression level up to selected one. + * Note : Unlike ZSTD_estimateCStreamSize*(), this estimate + * does not include space for a window buffer. + * Therefore, the estimation is only guaranteed for single-shot compressions, not streaming. + * The estimate will assume the input may be arbitrarily large, + * which is the worst case. + * + * When srcSize can be bound by a known and rather "small" value, + * this fact can be used to provide a tighter estimation + * because the CCtx compression context will need less memory. + * This tighter estimation can be provided by more advanced functions + * ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(), + * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter(). + * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits. + * + * Note 2 : only single-threaded compression is supported. + * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. + */ +ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel); +ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void); + +/*! ZSTD_estimateCStreamSize() : + * ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one. + * It will also consider src size to be arbitrarily "large", which is worst case. + * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation. + * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. + * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. + * Note : CStream size estimation is only correct for single-threaded compression. + * ZSTD_DStream memory budget depends on window Size. + * This information can be passed manually, using ZSTD_estimateDStreamSize, + * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame(); + * Note : if streaming is init with function ZSTD_init?Stream_usingDict(), + * an internal ?Dict will be created, which additional size is not estimated here. + * In this case, get total size by adding ZSTD_estimate?DictSize */ +ZSTDLIB_API size_t ZSTD_estimateCStreamSize(int compressionLevel); +ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_API size_t ZSTD_estimateDStreamSize(size_t windowSize); +ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize); + +/*! ZSTD_estimate?DictSize() : + * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict(). + * ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced(). + * Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller. + */ +ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); +ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod); +ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod); + +/*! ZSTD_initStatic*() : + * Initialize an object using a pre-allocated fixed-size buffer. + * workspace: The memory area to emplace the object into. + * Provided pointer *must be 8-bytes aligned*. + * Buffer must outlive object. + * workspaceSize: Use ZSTD_estimate*Size() to determine + * how large workspace must be to support target scenario. + * @return : pointer to object (same address as workspace, just different type), + * or NULL if error (size too small, incorrect alignment, etc.) + * Note : zstd will never resize nor malloc() when using a static buffer. + * If the object requires more memory than available, + * zstd will just error out (typically ZSTD_error_memory_allocation). + * Note 2 : there is no corresponding "free" function. + * Since workspace is allocated externally, it must be freed externally too. + * Note 3 : cParams : use ZSTD_getCParams() to convert a compression level + * into its associated cParams. + * Limitation 1 : currently not compatible with internal dictionary creation, triggered by + * ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict(). + * Limitation 2 : static cctx currently not compatible with multi-threading. + * Limitation 3 : static dctx is incompatible with legacy support. + */ +ZSTDLIB_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /*< same as ZSTD_initStaticCCtx() */ + +ZSTDLIB_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /*< same as ZSTD_initStaticDCtx() */ + +ZSTDLIB_API const ZSTD_CDict* ZSTD_initStaticCDict( + void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams); + +ZSTDLIB_API const ZSTD_DDict* ZSTD_initStaticDDict( + void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType); + + +/*! Custom memory allocation : + * These prototypes make it possible to pass your own allocation/free functions. + * ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below. + * All allocation/free operations will be completed using these custom variants instead of regular ones. + */ +typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); +typedef void (*ZSTD_freeFunction) (void* opaque, void* address); +typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; +static +__attribute__((__unused__)) +ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /*< this constant defers to stdlib's functions */ + +ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); +ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); + +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams, + ZSTD_customMem customMem); + +/* ! Thread pool : + * These prototypes make it possible to share a thread pool among multiple compression contexts. + * This can limit resources for applications with multiple threads where each one uses + * a threaded compression mode (via ZSTD_c_nbWorkers parameter). + * ZSTD_createThreadPool creates a new thread pool with a given number of threads. + * Note that the lifetime of such pool must exist while being used. + * ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value + * to use an internal thread pool). + * ZSTD_freeThreadPool frees a thread pool, accepts NULL pointer. + */ +typedef struct POOL_ctx_s ZSTD_threadPool; +ZSTDLIB_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads); +ZSTDLIB_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); /* accept NULL pointer */ +ZSTDLIB_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool); + + +/* + * This API is temporary and is expected to change or disappear in the future! + */ +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + const ZSTD_CCtx_params* cctxParams, + ZSTD_customMem customMem); + +ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_customMem customMem); + + +/* ************************************* +* Advanced compression functions +***************************************/ + +/*! ZSTD_createCDict_byReference() : + * Create a digested dictionary for compression + * Dictionary content is just referenced, not duplicated. + * As a consequence, `dictBuffer` **must** outlive CDict, + * and its content must remain unmodified throughout the lifetime of CDict. + * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */ +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); + +/*! ZSTD_getDictID_fromCDict() : + * Provides the dictID of the dictionary loaded into `cdict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict); + +/*! ZSTD_getCParams() : + * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. + * `estimatedSrcSize` value is optional, select 0 if not known */ +ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); + +/*! ZSTD_getParams() : + * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. + * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */ +ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); + +/*! ZSTD_checkCParams() : + * Ensure param values remain within authorized range. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */ +ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); + +/*! ZSTD_adjustCParams() : + * optimize params for a given `srcSize` and `dictSize`. + * `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN. + * `dictSize` must be `0` when there is no dictionary. + * cPar can be invalid : all parameters will be clamped within valid range in the @return struct. + * This function never fails (wide contract) */ +ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); + +/*! ZSTD_compress_advanced() : + * Note : this function is now DEPRECATED. + * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters. + * This prototype will be marked as deprecated and generate compilation warning on reaching v1.5.x */ +ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params); + +/*! ZSTD_compress_usingCDict_advanced() : + * Note : this function is now REDUNDANT. + * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters. + * This prototype will be marked as deprecated and generate compilation warning in some future version */ +ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams); + + +/*! ZSTD_CCtx_loadDictionary_byReference() : + * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx. + * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */ +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + +/*! ZSTD_CCtx_loadDictionary_advanced() : + * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over + * how to load the dictionary (by copy ? by reference ?) + * and how to interpret it (automatic ? force raw mode ? full mode only ?) */ +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_CCtx_refPrefix_advanced() : + * Same as ZSTD_CCtx_refPrefix(), but gives finer control over + * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ +ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); + +/* === experimental parameters === */ +/* these parameters can be used with ZSTD_setParameter() + * they are not guaranteed to remain supported in the future */ + + /* Enables rsyncable mode, + * which makes compressed files more rsync friendly + * by adding periodic synchronization points to the compressed data. + * The target average block size is ZSTD_c_jobSize / 2. + * It's possible to modify the job size to increase or decrease + * the granularity of the synchronization point. + * Once the jobSize is smaller than the window size, + * it will result in compression ratio degradation. + * NOTE 1: rsyncable mode only works when multithreading is enabled. + * NOTE 2: rsyncable performs poorly in combination with long range mode, + * since it will decrease the effectiveness of synchronization points, + * though mileage may vary. + * NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s. + * If the selected compression level is already running significantly slower, + * the overall speed won't be significantly impacted. + */ + #define ZSTD_c_rsyncable ZSTD_c_experimentalParam1 + +/* Select a compression format. + * The value must be of type ZSTD_format_e. + * See ZSTD_format_e enum definition for details */ +#define ZSTD_c_format ZSTD_c_experimentalParam2 + +/* Force back-reference distances to remain < windowSize, + * even when referencing into Dictionary content (default:0) */ +#define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3 + +/* Controls whether the contents of a CDict + * are used in place, or copied into the working context. + * Accepts values from the ZSTD_dictAttachPref_e enum. + * See the comments on that enum for an explanation of the feature. */ +#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4 + +/* Controls how the literals are compressed (default is auto). + * The value must be of type ZSTD_literalCompressionMode_e. + * See ZSTD_literalCompressionMode_t enum definition for details. + */ +#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5 + +/* Tries to fit compressed block size to be around targetCBlockSize. + * No target when targetCBlockSize == 0. + * There is no guarantee on compressed block size (default:0) */ +#define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6 + +/* User's best guess of source size. + * Hint is not valid when srcSizeHint == 0. + * There is no guarantee that hint is close to actual source size, + * but compression ratio may regress significantly if guess considerably underestimates */ +#define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7 + +/* Controls whether the new and experimental "dedicated dictionary search + * structure" can be used. This feature is still rough around the edges, be + * prepared for surprising behavior! + * + * How to use it: + * + * When using a CDict, whether to use this feature or not is controlled at + * CDict creation, and it must be set in a CCtxParams set passed into that + * construction (via ZSTD_createCDict_advanced2()). A compression will then + * use the feature or not based on how the CDict was constructed; the value of + * this param, set in the CCtx, will have no effect. + * + * However, when a dictionary buffer is passed into a CCtx, such as via + * ZSTD_CCtx_loadDictionary(), this param can be set on the CCtx to control + * whether the CDict that is created internally can use the feature or not. + * + * What it does: + * + * Normally, the internal data structures of the CDict are analogous to what + * would be stored in a CCtx after compressing the contents of a dictionary. + * To an approximation, a compression using a dictionary can then use those + * data structures to simply continue what is effectively a streaming + * compression where the simulated compression of the dictionary left off. + * Which is to say, the search structures in the CDict are normally the same + * format as in the CCtx. + * + * It is possible to do better, since the CDict is not like a CCtx: the search + * structures are written once during CDict creation, and then are only read + * after that, while the search structures in the CCtx are both read and + * written as the compression goes along. This means we can choose a search + * structure for the dictionary that is read-optimized. + * + * This feature enables the use of that different structure. + * + * Note that some of the members of the ZSTD_compressionParameters struct have + * different semantics and constraints in the dedicated search structure. It is + * highly recommended that you simply set a compression level in the CCtxParams + * you pass into the CDict creation call, and avoid messing with the cParams + * directly. + * + * Effects: + * + * This will only have any effect when the selected ZSTD_strategy + * implementation supports this feature. Currently, that's limited to + * ZSTD_greedy, ZSTD_lazy, and ZSTD_lazy2. + * + * Note that this means that the CDict tables can no longer be copied into the + * CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be + * useable. The dictionary can only be attached or reloaded. + * + * In general, you should expect compression to be faster--sometimes very much + * so--and CDict creation to be slightly slower. Eventually, we will probably + * make this mode the default. + */ +#define ZSTD_c_enableDedicatedDictSearch ZSTD_c_experimentalParam8 + +/* ZSTD_c_stableInBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells the compressor that the ZSTD_inBuffer will ALWAYS be the same + * between calls, except for the modifications that zstd makes to pos (the + * caller must not modify pos). This is checked by the compressor, and + * compression will fail if it ever changes. This means the only flush + * mode that makes sense is ZSTD_e_end, so zstd will error if ZSTD_e_end + * is not used. The data in the ZSTD_inBuffer in the range [src, src + pos) + * MUST not be modified during compression or you will get data corruption. + * + * When this flag is enabled zstd won't allocate an input window buffer, + * because the user guarantees it can reference the ZSTD_inBuffer until + * the frame is complete. But, it will still allocate an output buffer + * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also + * avoid the memcpy() from the input buffer to the input window buffer. + * + * NOTE: ZSTD_compressStream2() will error if ZSTD_e_end is not used. + * That means this flag cannot be used with ZSTD_compressStream(). + * + * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using + * this flag is ALWAYS memory safe, and will never access out-of-bounds + * memory. However, compression WILL fail if you violate the preconditions. + * + * WARNING: The data in the ZSTD_inBuffer in the range [dst, dst + pos) MUST + * not be modified during compression or you will get data corruption. This + * is because zstd needs to reference data in the ZSTD_inBuffer to find + * matches. Normally zstd maintains its own window buffer for this purpose, + * but passing this flag tells zstd to use the user provided buffer. + */ +#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9 + +/* ZSTD_c_stableOutBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells he compressor that the ZSTD_outBuffer will not be resized between + * calls. Specifically: (out.size - out.pos) will never grow. This gives the + * compressor the freedom to say: If the compressed data doesn't fit in the + * output buffer then return ZSTD_error_dstSizeTooSmall. This allows us to + * always decompress directly into the output buffer, instead of decompressing + * into an internal buffer and copying to the output buffer. + * + * When this flag is enabled zstd won't allocate an output buffer, because + * it can write directly to the ZSTD_outBuffer. It will still allocate the + * input window buffer (see ZSTD_c_stableInBuffer). + * + * Zstd will check that (out.size - out.pos) never grows and return an error + * if it does. While not strictly necessary, this should prevent surprises. + */ +#define ZSTD_c_stableOutBuffer ZSTD_c_experimentalParam10 + +/* ZSTD_c_blockDelimiters + * Default is 0 == ZSTD_sf_noBlockDelimiters. + * + * For use with sequence compression API: ZSTD_compressSequences(). + * + * Designates whether or not the given array of ZSTD_Sequence contains block delimiters + * and last literals, which are defined as sequences with offset == 0 and matchLength == 0. + * See the definition of ZSTD_Sequence for more specifics. + */ +#define ZSTD_c_blockDelimiters ZSTD_c_experimentalParam11 + +/* ZSTD_c_validateSequences + * Default is 0 == disabled. Set to 1 to enable sequence validation. + * + * For use with sequence compression API: ZSTD_compressSequences(). + * Designates whether or not we validate sequences provided to ZSTD_compressSequences() + * during function execution. + * + * Without validation, providing a sequence that does not conform to the zstd spec will cause + * undefined behavior, and may produce a corrupted block. + * + * With validation enabled, a if sequence is invalid (see doc/zstd_compression_format.md for + * specifics regarding offset/matchlength requirements) then the function will bail out and + * return an error. + * + */ +#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12 + +/*! ZSTD_CCtx_getParameter() : + * Get the requested compression parameter value, selected by enum ZSTD_cParameter, + * and store it into int* value. + * @return : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value); + + +/*! ZSTD_CCtx_params : + * Quick howto : + * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure + * - ZSTD_CCtxParams_setParameter() : Push parameters one by one into + * an existing ZSTD_CCtx_params structure. + * This is similar to + * ZSTD_CCtx_setParameter(). + * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to + * an existing CCtx. + * These parameters will be applied to + * all subsequent frames. + * - ZSTD_compressStream2() : Do compression using the CCtx. + * - ZSTD_freeCCtxParams() : Free the memory, accept NULL pointer. + * + * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams() + * for static allocation of CCtx for single-threaded compression. + */ +ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); +ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); /* accept NULL pointer */ + +/*! ZSTD_CCtxParams_reset() : + * Reset params to default values. + */ +ZSTDLIB_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params); + +/*! ZSTD_CCtxParams_init() : + * Initializes the compression parameters of cctxParams according to + * compression level. All other parameters are reset to their default values. + */ +ZSTDLIB_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel); + +/*! ZSTD_CCtxParams_init_advanced() : + * Initializes the compression and frame parameters of cctxParams according to + * params. All other parameters are reset to their default values. + */ +ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); + +/*! ZSTD_CCtxParams_setParameter() : + * Similar to ZSTD_CCtx_setParameter. + * Set one compression parameter, selected by enum ZSTD_cParameter. + * Parameters must be applied to a ZSTD_CCtx using + * ZSTD_CCtx_setParametersUsingCCtxParams(). + * @result : a code representing success or failure (which can be tested with + * ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); + +/*! ZSTD_CCtxParams_getParameter() : + * Similar to ZSTD_CCtx_getParameter. + * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); + +/*! ZSTD_CCtx_setParametersUsingCCtxParams() : + * Apply a set of ZSTD_CCtx_params to the compression context. + * This can be done even after compression is started, + * if nbWorkers==0, this will have no impact until a new compression is started. + * if nbWorkers>=1, new parameters will be picked up at next job, + * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated). + */ +ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( + ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params); + +/*! ZSTD_compressStream2_simpleArgs() : + * Same as ZSTD_compressStream2(), + * but using only integral types as arguments. + * This variant might be helpful for binders from dynamic languages + * which have troubles handling structures containing memory pointers. + */ +ZSTDLIB_API size_t ZSTD_compressStream2_simpleArgs ( + ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos, + ZSTD_EndDirective endOp); + + +/* ************************************* +* Advanced decompression functions +***************************************/ + +/*! ZSTD_isFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. + * Note 3 : Skippable Frame Identifiers are considered valid. */ +ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size); + +/*! ZSTD_createDDict_byReference() : + * Create a digested dictionary, ready to start decompression operation without startup delay. + * Dictionary content is referenced, and therefore stays in dictBuffer. + * It is important that dictBuffer outlives DDict, + * it must remain read accessible throughout the lifetime of DDict */ +ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); + +/*! ZSTD_DCtx_loadDictionary_byReference() : + * Same as ZSTD_DCtx_loadDictionary(), + * but references `dict` content instead of copying it into `dctx`. + * This saves memory if `dict` remains around., + * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */ +ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); + +/*! ZSTD_DCtx_loadDictionary_advanced() : + * Same as ZSTD_DCtx_loadDictionary(), + * but gives direct control over + * how to load the dictionary (by copy ? by reference ?) + * and how to interpret it (automatic ? force raw mode ? full mode only ?). */ +ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_DCtx_refPrefix_advanced() : + * Same as ZSTD_DCtx_refPrefix(), but gives finer control over + * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ +ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_DCtx_setMaxWindowSize() : + * Refuses allocating internal buffers for frames requiring a window size larger than provided limit. + * This protects a decoder context from reserving too much memory for itself (potential attack scenario). + * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. + * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + * @return : 0, or an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); + +/*! ZSTD_DCtx_getParameter() : + * Get the requested decompression parameter value, selected by enum ZSTD_dParameter, + * and store it into int* value. + * @return : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value); + +/* ZSTD_d_format + * experimental parameter, + * allowing selection between ZSTD_format_e input compression formats + */ +#define ZSTD_d_format ZSTD_d_experimentalParam1 +/* ZSTD_d_stableOutBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same + * between calls, except for the modifications that zstd makes to pos (the + * caller must not modify pos). This is checked by the decompressor, and + * decompression will fail if it ever changes. Therefore the ZSTD_outBuffer + * MUST be large enough to fit the entire decompressed frame. This will be + * checked when the frame content size is known. The data in the ZSTD_outBuffer + * in the range [dst, dst + pos) MUST not be modified during decompression + * or you will get data corruption. + * + * When this flags is enabled zstd won't allocate an output buffer, because + * it can write directly to the ZSTD_outBuffer, but it will still allocate + * an input buffer large enough to fit any compressed block. This will also + * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer. + * If you need to avoid the input buffer allocation use the buffer-less + * streaming API. + * + * NOTE: So long as the ZSTD_outBuffer always points to valid memory, using + * this flag is ALWAYS memory safe, and will never access out-of-bounds + * memory. However, decompression WILL fail if you violate the preconditions. + * + * WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST + * not be modified during decompression or you will get data corruption. This + * is because zstd needs to reference data in the ZSTD_outBuffer to regenerate + * matches. Normally zstd maintains its own buffer for this purpose, but passing + * this flag tells zstd to use the user provided buffer. + */ +#define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2 + +/* ZSTD_d_forceIgnoreChecksum + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable + * + * Tells the decompressor to skip checksum validation during decompression, regardless + * of whether checksumming was specified during compression. This offers some + * slight performance benefits, and may be useful for debugging. + * Param has values of type ZSTD_forceIgnoreChecksum_e + */ +#define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3 + +/* ZSTD_d_refMultipleDDicts + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable + * + * If enabled and dctx is allocated on the heap, then additional memory will be allocated + * to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict() + * using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead + * store all references. At decompression time, the appropriate dictID is selected + * from the set of DDicts based on the dictID in the frame. + * + * Usage is simply calling ZSTD_refDDict() on multiple dict buffers. + * + * Param has values of byte ZSTD_refMultipleDDicts_e + * + * WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory + * allocation for the hash table. ZSTD_freeDCtx() also frees this memory. + * Memory is allocated as per ZSTD_DCtx::customMem. + * + * Although this function allocates memory for the table, the user is still responsible for + * memory management of the underlying ZSTD_DDict* themselves. + */ +#define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4 + + +/*! ZSTD_DCtx_setFormat() : + * Instruct the decoder context about what kind of data to decode next. + * This instruction is mandatory to decode data without a fully-formed header, + * such ZSTD_f_zstd1_magicless for example. + * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); + +/*! ZSTD_decompressStream_simpleArgs() : + * Same as ZSTD_decompressStream(), + * but using only integral types as arguments. + * This can be helpful for binders from dynamic languages + * which have troubles handling structures containing memory pointers. + */ +ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs ( + ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos); + + +/* ****************************************************************** +* Advanced streaming functions +* Warning : most of these functions are now redundant with the Advanced API. +* Once Advanced API reaches "stable" status, +* redundant functions will be deprecated, and then at some point removed. +********************************************************************/ + +/*===== Advanced Streaming compression functions =====*/ + +/*! ZSTD_initCStream_srcSize() : + * This function is deprecated, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * + * pledgedSrcSize must be correct. If it is not known at init time, use + * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, + * "0" also disables frame content size field. It may be enabled in the future. + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + */ +ZSTDLIB_API size_t +ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, + int compressionLevel, + unsigned long long pledgedSrcSize); + +/*! ZSTD_initCStream_usingDict() : + * This function is deprecated, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * Creates of an internal CDict (incompatible with static CCtx), except if + * dict == NULL or dictSize < 8, in which case no dict is used. + * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if + * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + */ +ZSTDLIB_API size_t +ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + int compressionLevel); + +/*! ZSTD_initCStream_advanced() : + * This function is deprecated, and is approximately equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * // Pseudocode: Set each zstd parameter and leave the rest as-is. + * for ((param, value) : params) { + * ZSTD_CCtx_setParameter(zcs, param, value); + * } + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy. + * pledgedSrcSize must be correct. + * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + */ +ZSTDLIB_API size_t +ZSTD_initCStream_advanced(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, + unsigned long long pledgedSrcSize); + +/*! ZSTD_initCStream_usingCDict() : + * This function is deprecated, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * note : cdict will just be referenced, and must outlive compression session + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + */ +ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); + +/*! ZSTD_initCStream_usingCDict_advanced() : + * This function is DEPRECATED, and is approximately equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * // Pseudocode: Set each zstd frame parameter and leave the rest as-is. + * for ((fParam, value) : fParams) { + * ZSTD_CCtx_setParameter(zcs, fParam, value); + * } + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * same as ZSTD_initCStream_usingCDict(), with control over frame parameters. + * pledgedSrcSize must be correct. If srcSize is not known at init time, use + * value ZSTD_CONTENTSIZE_UNKNOWN. + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + */ +ZSTDLIB_API size_t +ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams, + unsigned long long pledgedSrcSize); + +/*! ZSTD_resetCStream() : + * This function is deprecated, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * + * start a new frame, using same parameters from previous frame. + * This is typically useful to skip dictionary loading stage, since it will re-use it in-place. + * Note that zcs must be init at least once before using ZSTD_resetCStream(). + * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. + * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. + * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, + * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. + * @return : 0, or an error code (which can be tested using ZSTD_isError()) + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + */ +ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); + + +typedef struct { + unsigned long long ingested; /* nb input bytes read and buffered */ + unsigned long long consumed; /* nb input bytes actually compressed */ + unsigned long long produced; /* nb of compressed bytes generated and buffered */ + unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */ + unsigned currentJobID; /* MT only : latest started job nb */ + unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */ +} ZSTD_frameProgression; + +/* ZSTD_getFrameProgression() : + * tells how much data has been ingested (read from input) + * consumed (input actually compressed) and produced (output) for current frame. + * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed. + * Aggregates progression inside active worker threads. + */ +ZSTDLIB_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx); + +/*! ZSTD_toFlushNow() : + * Tell how many bytes are ready to be flushed immediately. + * Useful for multithreading scenarios (nbWorkers >= 1). + * Probe the oldest active job, defined as oldest job not yet entirely flushed, + * and check its output buffer. + * @return : amount of data stored in oldest job and ready to be flushed immediately. + * if @return == 0, it means either : + * + there is no active job (could be checked with ZSTD_frameProgression()), or + * + oldest job is still actively compressing data, + * but everything it has produced has also been flushed so far, + * therefore flush speed is limited by production speed of oldest job + * irrespective of the speed of concurrent (and newer) jobs. + */ +ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); + + +/*===== Advanced Streaming decompression functions =====*/ + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_loadDictionary(zds, dict, dictSize); + * + * note: no dictionary will be used if dict == NULL or dictSize < 8 + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + */ +ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_refDDict(zds, ddict); + * + * note : ddict is referenced, it must outlive decompression session + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + */ +ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * + * re-use decompression parameters from previous init; saves dictionary loading + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + */ +ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); + + +/* ******************************************************************* +* Buffer-less and synchronous inner streaming functions +* +* This is an advanced API, giving full control over buffer management, for users which need direct control over memory. +* But it's also a complex one, with several restrictions, documented below. +* Prefer normal streaming API for an easier experience. +********************************************************************* */ + +/* + Buffer-less streaming compression (synchronous mode) + + A ZSTD_CCtx object is required to track streaming operations. + Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. + ZSTD_CCtx object can be re-used multiple times within successive compression operations. + + Start by initializing a context. + Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression, + or ZSTD_compressBegin_advanced(), for finer parameter control. + It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx() + + Then, consume your input using ZSTD_compressContinue(). + There are some important considerations to keep in mind when using this advanced function : + - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only. + - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks. + - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario. + Worst case evaluation is provided by ZSTD_compressBound(). + ZSTD_compressContinue() doesn't guarantee recover after a failed compression. + - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog). + It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks) + - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps. + In which case, it will "discard" the relevant memory section from its history. + + Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum. + It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame. + Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders. + + `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again. +*/ + +/*===== Buffer-less streaming compression functions =====*/ +ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /*< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ +ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /*< note: fails if cdict==NULL */ +ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ +ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /*< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ + +ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + + +/* + Buffer-less streaming decompression (synchronous mode) + + A ZSTD_DCtx object is required to track streaming operations. + Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. + A ZSTD_DCtx object can be re-used multiple times. + + First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader(). + Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough. + Data fragment must be large enough to ensure successful decoding. + `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough. + @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled. + >0 : `srcSize` is too small, please provide at least @result bytes on next attempt. + errorCode, which can be tested using ZSTD_isError(). + + It fills a ZSTD_frameHeader structure with important information to correctly decode the frame, + such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`). + Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information. + As a consequence, check that values remain within valid application range. + For example, do not allocate memory blindly, check that `windowSize` is within expectation. + Each application can set its own limits, depending on local restrictions. + For extended interoperability, it is recommended to support `windowSize` of at least 8 MB. + + ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes. + ZSTD_decompressContinue() is very sensitive to contiguity, + if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place, + or that previous contiguous segment is large enough to properly handle maximum back-reference distance. + There are multiple ways to guarantee this condition. + + The most memory efficient way is to use a round buffer of sufficient size. + Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(), + which can @return an error code if required value is too large for current system (in 32-bits mode). + In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one, + up to the moment there is not enough room left in the buffer to guarantee decoding another full block, + which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`. + At which point, decoding can resume from the beginning of the buffer. + Note that already decoded data stored in the buffer should be flushed before being overwritten. + + There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory. + + Finally, if you control the compression process, you can also ignore all buffer size rules, + as long as the encoder and decoder progress in "lock-step", + aka use exactly the same buffer sizes, break contiguity at the same place, etc. + + Once buffers are setup, start decompression, with ZSTD_decompressBegin(). + If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict(). + + Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. + ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue(). + ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail. + + @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). + It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item. + It can also be an error code, which can be tested with ZSTD_isError(). + + A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. + Context can then be reset to start a new decompression. + + Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType(). + This information is not required to properly decode a frame. + + == Special case : skippable frames == + + Skippable frames allow integration of user-defined data into a flow of concatenated frames. + Skippable frames will be ignored (skipped) by decompressor. + The format of skippable frames is as follows : + a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F + b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits + c) Frame Content - any content (User Data) of length equal to Frame Size + For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame. + For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content. +*/ + +/*===== Buffer-less streaming decompression functions =====*/ +typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e; +typedef struct { + unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */ + unsigned long long windowSize; /* can be very large, up to <= frameContentSize */ + unsigned blockSizeMax; + ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */ + unsigned headerSize; + unsigned dictID; + unsigned checksumFlag; +} ZSTD_frameHeader; + +/*! ZSTD_getFrameHeader() : + * decode Frame Header, or requires larger `srcSize`. + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, + * or an error code, which can be tested using ZSTD_isError() */ +ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /*< doesn't consume input */ +/*! ZSTD_getFrameHeader_advanced() : + * same as ZSTD_getFrameHeader(), + * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ +ZSTDLIB_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); +ZSTDLIB_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /*< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ + +ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIB_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + +ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* misc */ +ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); +typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; +ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); + + + + +/* ============================ */ +/* Block level API */ +/* ============================ */ + +/*! + Block functions produce and decode raw zstd blocks, without frame metadata. + Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes). + But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes. + + A few rules to respect : + - Compressing and decompressing require a context structure + + Use ZSTD_createCCtx() and ZSTD_createDCtx() + - It is necessary to init context before starting + + compression : any ZSTD_compressBegin*() variant, including with dictionary + + decompression : any ZSTD_decompressBegin*() variant, including with dictionary + + copyCCtx() and copyDCtx() can be used too + - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB + + If input is larger than a block size, it's necessary to split input data into multiple blocks + + For inputs larger than a single block, consider using regular ZSTD_compress() instead. + Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block. + - When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) ! + ===> In which case, nothing is produced into `dst` ! + + User __must__ test for such outcome and deal directly with uncompressed data + + A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0. + Doing so would mess up with statistics history, leading to potential data corruption. + + ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !! + + In case of multiple successive blocks, should some of them be uncompressed, + decoder must be informed of their existence in order to follow proper history. + Use ZSTD_insertBlock() for such a case. +*/ + +/*===== Raw zstd block functions =====*/ +ZSTDLIB_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx); +ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /*< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ + + +#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ + diff --git a/include/media/v4l2-ctrls.h b/include/media/v4l2-ctrls.h index 167ca8c8424f..2fe4019b749f 100644 --- a/include/media/v4l2-ctrls.h +++ b/include/media/v4l2-ctrls.h @@ -301,12 +301,14 @@ struct v4l2_ctrl { * the control has been applied. This prevents applying controls * from a cluster with multiple controls twice (when the first * control of a cluster is applied, they all are). - * @req: If set, this refers to another request that sets this control. + * @valid_p_req: If set, then p_req contains the control value for the request. * @p_req: If the control handler containing this control reference * is bound to a media request, then this points to the - * value of the control that should be applied when the request + * value of the control that must be applied when the request * is executed, or to the value of the control at the time - * that the request was completed. + * that the request was completed. If @valid_p_req is false, + * then this control was never set for this request and the + * control will not be updated when this request is applied. * * Each control handler has a list of these refs. The list_head is used to * keep a sorted-by-control-ID list of all controls, while the next pointer @@ -319,7 +321,7 @@ struct v4l2_ctrl_ref { struct v4l2_ctrl_helper *helper; bool from_other_dev; bool req_done; - struct v4l2_ctrl_ref *req; + bool valid_p_req; union v4l2_ctrl_ptr p_req; }; @@ -346,7 +348,7 @@ struct v4l2_ctrl_ref { * @error: The error code of the first failed control addition. * @request_is_queued: True if the request was queued. * @requests: List to keep track of open control handler request objects. - * For the parent control handler (@req_obj.req == NULL) this + * For the parent control handler (@req_obj.ops == NULL) this * is the list header. When the parent control handler is * removed, it has to unbind and put all these requests since * they refer to the parent. diff --git a/include/net/addrconf.h b/include/net/addrconf.h index 18f783dcd55f..78ea3e332688 100644 --- a/include/net/addrconf.h +++ b/include/net/addrconf.h @@ -233,7 +233,6 @@ void ipv6_mc_unmap(struct inet6_dev *idev); void ipv6_mc_remap(struct inet6_dev *idev); void ipv6_mc_init_dev(struct inet6_dev *idev); void ipv6_mc_destroy_dev(struct inet6_dev *idev); -int ipv6_mc_check_icmpv6(struct sk_buff *skb); int ipv6_mc_check_mld(struct sk_buff *skb); void addrconf_dad_failure(struct sk_buff *skb, struct inet6_ifaddr *ifp); diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h index ebdd4afe30d2..ca4ac6603b9a 100644 --- a/include/net/bluetooth/hci_core.h +++ b/include/net/bluetooth/hci_core.h @@ -704,6 +704,7 @@ struct hci_chan { struct sk_buff_head data_q; unsigned int sent; __u8 state; + bool amp; }; struct hci_conn_params { diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h index 3c8c59471bc1..2cdc5a0709fe 100644 --- a/include/net/inet_connection_sock.h +++ b/include/net/inet_connection_sock.h @@ -134,8 +134,9 @@ struct inet_connection_sock { u32 icsk_probes_tstamp; u32 icsk_user_timeout; - u64 icsk_ca_priv[104 / sizeof(u64)]; -#define ICSK_CA_PRIV_SIZE (13 * sizeof(u64)) +/* XXX inflated by temporary internal debugging info */ +#define ICSK_CA_PRIV_SIZE (216) + u64 icsk_ca_priv[ICSK_CA_PRIV_SIZE / sizeof(u64)]; }; #define ICSK_TIME_RETRANS 1 /* Retransmit timer */ diff --git a/include/net/netfilter/nf_tables_offload.h b/include/net/netfilter/nf_tables_offload.h index 1d34fe154fe0..434a6158852f 100644 --- a/include/net/netfilter/nf_tables_offload.h +++ b/include/net/netfilter/nf_tables_offload.h @@ -4,11 +4,16 @@ #include #include +enum nft_offload_reg_flags { + NFT_OFFLOAD_F_NETWORK2HOST = (1 << 0), +}; + struct nft_offload_reg { u32 key; u32 len; u32 base_offset; u32 offset; + u32 flags; struct nft_data data; struct nft_data mask; }; @@ -45,6 +50,7 @@ struct nft_flow_key { struct flow_dissector_key_ports tp; struct flow_dissector_key_ip ip; struct flow_dissector_key_vlan vlan; + struct flow_dissector_key_vlan cvlan; struct flow_dissector_key_eth_addrs eth_addrs; struct flow_dissector_key_meta meta; } __aligned(BITS_PER_LONG / 8); /* Ensure that we can do comparisons as longs. */ @@ -71,13 +77,17 @@ struct nft_flow_rule *nft_flow_rule_create(struct net *net, const struct nft_rul void nft_flow_rule_destroy(struct nft_flow_rule *flow); int nft_flow_rule_offload_commit(struct net *net); -#define NFT_OFFLOAD_MATCH(__key, __base, __field, __len, __reg) \ +#define NFT_OFFLOAD_MATCH_FLAGS(__key, __base, __field, __len, __reg, __flags) \ (__reg)->base_offset = \ offsetof(struct nft_flow_key, __base); \ (__reg)->offset = \ offsetof(struct nft_flow_key, __base.__field); \ (__reg)->len = __len; \ (__reg)->key = __key; \ + (__reg)->flags = __flags; + +#define NFT_OFFLOAD_MATCH(__key, __base, __field, __len, __reg) \ + NFT_OFFLOAD_MATCH_FLAGS(__key, __base, __field, __len, __reg, 0) #define NFT_OFFLOAD_MATCH_EXACT(__key, __base, __field, __len, __reg) \ NFT_OFFLOAD_MATCH(__key, __base, __field, __len, __reg) \ diff --git a/include/net/page_pool.h b/include/net/page_pool.h index b5b195305346..e05744b9a1bc 100644 --- a/include/net/page_pool.h +++ b/include/net/page_pool.h @@ -198,7 +198,17 @@ static inline void page_pool_recycle_direct(struct page_pool *pool, static inline dma_addr_t page_pool_get_dma_addr(struct page *page) { - return page->dma_addr; + dma_addr_t ret = page->dma_addr[0]; + if (sizeof(dma_addr_t) > sizeof(unsigned long)) + ret |= (dma_addr_t)page->dma_addr[1] << 16 << 16; + return ret; +} + +static inline void page_pool_set_dma_addr(struct page *page, dma_addr_t addr) +{ + page->dma_addr[0] = addr; + if (sizeof(dma_addr_t) > sizeof(unsigned long)) + page->dma_addr[1] = upper_32_bits(addr); } static inline bool is_page_pool_compiled_in(void) diff --git a/include/net/tcp.h b/include/net/tcp.h index 963cd86d12dd..5a86fa1d2ff1 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -799,6 +799,11 @@ static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0) return max_t(s64, t1 - t0, 0); } +static inline u32 tcp_stamp32_us_delta(u32 t1, u32 t0) +{ + return max_t(s32, t1 - t0, 0); +} + static inline u32 tcp_skb_timestamp(const struct sk_buff *skb) { return tcp_ns_to_ts(skb->skb_mstamp_ns); @@ -866,16 +871,22 @@ struct tcp_skb_cb { __u32 ack_seq; /* Sequence number ACK'd */ union { struct { +#define TCPCB_DELIVERED_CE_MASK ((1U<<20) - 1) /* There is space for up to 24 bytes */ - __u32 in_flight:30,/* Bytes in flight at transmit */ - is_app_limited:1, /* cwnd not fully used? */ - unused:1; + __u32 is_app_limited:1, /* cwnd not fully used? */ + delivered_ce:20, + unused:11; /* pkts S/ACKed so far upon tx of skb, incl retrans: */ __u32 delivered; /* start of send pipeline phase */ - u64 first_tx_mstamp; + u32 first_tx_mstamp; /* when we reached the "delivered" count */ - u64 delivered_mstamp; + u32 delivered_mstamp; +#define TCPCB_IN_FLIGHT_BITS 20 +#define TCPCB_IN_FLIGHT_MAX ((1U << TCPCB_IN_FLIGHT_BITS) - 1) + u32 in_flight:20, /* packets in flight at transmit */ + unused2:12; + u32 lost; /* packets lost so far upon tx of skb */ } tx; /* only used for outgoing skbs */ union { struct inet_skb_parm h4; @@ -1025,7 +1036,11 @@ enum tcp_ca_ack_event_flags { #define TCP_CONG_NON_RESTRICTED 0x1 /* Requires ECN/ECT set on all packets */ #define TCP_CONG_NEEDS_ECN 0x2 -#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN) +/* Wants notification of CE events (CA_EVENT_ECN_IS_CE, CA_EVENT_ECN_NO_CE). */ +#define TCP_CONG_WANTS_CE_EVENTS 0x4 +#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | \ + TCP_CONG_NEEDS_ECN | \ + TCP_CONG_WANTS_CE_EVENTS) union tcp_cc_info; @@ -1045,8 +1060,13 @@ struct ack_sample { */ struct rate_sample { u64 prior_mstamp; /* starting timestamp for interval */ + u32 prior_lost; /* tp->lost at "prior_mstamp" */ u32 prior_delivered; /* tp->delivered at "prior_mstamp" */ + u32 prior_delivered_ce;/* tp->delivered_ce at "prior_mstamp" */ + u32 tx_in_flight; /* packets in flight at starting timestamp */ + s32 lost; /* number of packets lost over interval */ s32 delivered; /* number of packets delivered over interval */ + s32 delivered_ce; /* packets delivered w/ CE mark over interval */ long interval_us; /* time for tp->delivered to incr "delivered" */ u32 snd_interval_us; /* snd interval for delivered packets */ u32 rcv_interval_us; /* rcv interval for delivered packets */ @@ -1057,6 +1077,7 @@ struct rate_sample { bool is_app_limited; /* is sample from packet with bubble in pipe? */ bool is_retrans; /* is sample from retransmission? */ bool is_ack_delayed; /* is this (likely) a delayed ACK? */ + bool is_ece; /* did this ACK have ECN marked? */ }; struct tcp_congestion_ops { @@ -1083,10 +1104,12 @@ struct tcp_congestion_ops { u32 (*undo_cwnd)(struct sock *sk); /* hook for packet ack accounting (optional) */ void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample); - /* override sysctl_tcp_min_tso_segs */ - u32 (*min_tso_segs)(struct sock *sk); + /* pick target number of segments per TSO/GSO skb (optional): */ + u32 (*tso_segs)(struct sock *sk, unsigned int mss_now); /* returns the multiplier used in tcp_sndbuf_expand (optional) */ u32 (*sndbuf_expand)(struct sock *sk); + /* react to a specific lost skb (optional) */ + void (*skb_marked_lost)(struct sock *sk, const struct sk_buff *skb); /* call when packets are delivered to update cwnd and pacing rate, * after all the ca_state processing. (optional) */ @@ -1132,6 +1155,14 @@ static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer) } #endif +static inline bool tcp_ca_wants_ce_events(const struct sock *sk) +{ + const struct inet_connection_sock *icsk = inet_csk(sk); + + return icsk->icsk_ca_ops->flags & (TCP_CONG_NEEDS_ECN | + TCP_CONG_WANTS_CE_EVENTS); +} + static inline bool tcp_ca_needs_ecn(const struct sock *sk) { const struct inet_connection_sock *icsk = inet_csk(sk); @@ -1157,6 +1188,7 @@ static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event) } /* From tcp_rate.c */ +void tcp_set_tx_in_flight(struct sock *sk, struct sk_buff *skb); void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb); void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, struct rate_sample *rs); diff --git a/include/scsi/libfcoe.h b/include/scsi/libfcoe.h index 2568cb0627ec..fac8e89aed81 100644 --- a/include/scsi/libfcoe.h +++ b/include/scsi/libfcoe.h @@ -249,7 +249,7 @@ int fcoe_ctlr_recv_flogi(struct fcoe_ctlr *, struct fc_lport *, struct fc_frame *); /* libfcoe funcs */ -u64 fcoe_wwn_from_mac(unsigned char mac[], unsigned int, unsigned int); +u64 fcoe_wwn_from_mac(unsigned char mac[MAX_ADDR_LEN], unsigned int, unsigned int); int fcoe_libfc_config(struct fc_lport *, struct fcoe_ctlr *, const struct libfc_function_template *, int init_fcp); u32 fcoe_fc_crc(struct fc_frame *fp); diff --git a/include/trace/events/sunrpc.h b/include/trace/events/sunrpc.h index 036eb1f5c133..2f01314de73a 100644 --- a/include/trace/events/sunrpc.h +++ b/include/trace/events/sunrpc.h @@ -1141,7 +1141,6 @@ DECLARE_EVENT_CLASS(xprt_writelock_event, DEFINE_WRITELOCK_EVENT(reserve_xprt); DEFINE_WRITELOCK_EVENT(release_xprt); -DEFINE_WRITELOCK_EVENT(transmit_queued); DECLARE_EVENT_CLASS(xprt_cong_event, TP_PROTO( diff --git a/include/uapi/linux/inet_diag.h b/include/uapi/linux/inet_diag.h index 20ee93f0f876..96d52dd9c48a 100644 --- a/include/uapi/linux/inet_diag.h +++ b/include/uapi/linux/inet_diag.h @@ -231,9 +231,42 @@ struct tcp_bbr_info { __u32 bbr_cwnd_gain; /* cwnd gain shifted left 8 bits */ }; +/* Phase as reported in netlink/ss stats. */ +enum tcp_bbr2_phase { + BBR2_PHASE_INVALID = 0, + BBR2_PHASE_STARTUP = 1, + BBR2_PHASE_DRAIN = 2, + BBR2_PHASE_PROBE_RTT = 3, + BBR2_PHASE_PROBE_BW_UP = 4, + BBR2_PHASE_PROBE_BW_DOWN = 5, + BBR2_PHASE_PROBE_BW_CRUISE = 6, + BBR2_PHASE_PROBE_BW_REFILL = 7 +}; + +struct tcp_bbr2_info { + /* u64 bw: bandwidth (app throughput) estimate in Byte per sec: */ + __u32 bbr_bw_lsb; /* lower 32 bits of bw */ + __u32 bbr_bw_msb; /* upper 32 bits of bw */ + __u32 bbr_min_rtt; /* min-filtered RTT in uSec */ + __u32 bbr_pacing_gain; /* pacing gain shifted left 8 bits */ + __u32 bbr_cwnd_gain; /* cwnd gain shifted left 8 bits */ + __u32 bbr_bw_hi_lsb; /* lower 32 bits of bw_hi */ + __u32 bbr_bw_hi_msb; /* upper 32 bits of bw_hi */ + __u32 bbr_bw_lo_lsb; /* lower 32 bits of bw_lo */ + __u32 bbr_bw_lo_msb; /* upper 32 bits of bw_lo */ + __u8 bbr_mode; /* current bbr_mode in state machine */ + __u8 bbr_phase; /* current state machine phase */ + __u8 unused1; /* alignment padding; not used yet */ + __u8 bbr_version; /* MUST be at this offset in struct */ + __u32 bbr_inflight_lo; /* lower/short-term data volume bound */ + __u32 bbr_inflight_hi; /* higher/long-term data volume bound */ + __u32 bbr_extra_acked; /* max excess packets ACKed in epoch */ +}; + union tcp_cc_info { struct tcpvegas_info vegas; struct tcp_dctcp_info dctcp; struct tcp_bbr_info bbr; + struct tcp_bbr2_info bbr2; }; #endif /* _UAPI_INET_DIAG_H_ */ diff --git a/include/uapi/linux/netfilter/xt_SECMARK.h b/include/uapi/linux/netfilter/xt_SECMARK.h index 1f2a708413f5..beb2cadba8a9 100644 --- a/include/uapi/linux/netfilter/xt_SECMARK.h +++ b/include/uapi/linux/netfilter/xt_SECMARK.h @@ -20,4 +20,10 @@ struct xt_secmark_target_info { char secctx[SECMARK_SECCTX_MAX]; }; +struct xt_secmark_target_info_v1 { + __u8 mode; + char secctx[SECMARK_SECCTX_MAX]; + __u32 secid; +}; + #endif /*_XT_SECMARK_H_target */ diff --git a/include/uapi/linux/tty_flags.h b/include/uapi/linux/tty_flags.h index 900a32e63424..6a3ac496a56c 100644 --- a/include/uapi/linux/tty_flags.h +++ b/include/uapi/linux/tty_flags.h @@ -39,7 +39,7 @@ * WARNING: These flags are no longer used and have been superceded by the * TTY_PORT_ flags in the iflags field (and not userspace-visible) */ -#ifndef _KERNEL_ +#ifndef __KERNEL__ #define ASYNCB_INITIALIZED 31 /* Serial port was initialized */ #define ASYNCB_SUSPENDED 30 /* Serial port is suspended */ #define ASYNCB_NORMAL_ACTIVE 29 /* Normal device is active */ @@ -81,7 +81,7 @@ #define ASYNC_SPD_WARP (ASYNC_SPD_HI|ASYNC_SPD_SHI) #define ASYNC_SPD_MASK (ASYNC_SPD_HI|ASYNC_SPD_VHI|ASYNC_SPD_SHI) -#ifndef _KERNEL_ +#ifndef __KERNEL__ /* These flags are no longer used (and were always masked from userspace) */ #define ASYNC_INITIALIZED (1U << ASYNCB_INITIALIZED) #define ASYNC_NORMAL_ACTIVE (1U << ASYNCB_NORMAL_ACTIVE) diff --git a/include/uapi/linux/usb/video.h b/include/uapi/linux/usb/video.h index d854cb19c42c..bfdae12cdacf 100644 --- a/include/uapi/linux/usb/video.h +++ b/include/uapi/linux/usb/video.h @@ -302,9 +302,10 @@ struct uvc_processing_unit_descriptor { __u8 bControlSize; __u8 bmControls[2]; __u8 iProcessing; + __u8 bmVideoStandards; } __attribute__((__packed__)); -#define UVC_DT_PROCESSING_UNIT_SIZE(n) (9+(n)) +#define UVC_DT_PROCESSING_UNIT_SIZE(n) (10+(n)) /* 3.7.2.6. Extension Unit Descriptor */ struct uvc_extension_unit_descriptor { diff --git a/init/Kconfig b/init/Kconfig index 5f5c776ef192..e942f19baf29 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1177,6 +1177,22 @@ config USER_NS If unsure, say N. +config USER_NS_UNPRIVILEGED + bool "Allow unprivileged users to create namespaces" + default y + depends on USER_NS + help + When disabled, unprivileged users will not be able to create + new namespaces. Allowing users to create their own namespaces + has been part of several recent local privilege escalation + exploits, so if you need user namespaces but are + paranoid^Wsecurity-conscious you want to disable this. + + This setting can be overridden at runtime via the + kernel.unprivileged_userns_clone sysctl. + + If unsure, say Y. + config PID_NS bool "PID Namespaces" default y @@ -1316,7 +1332,6 @@ config CC_OPTIMIZE_FOR_PERFORMANCE config CC_OPTIMIZE_FOR_PERFORMANCE_O3 bool "Optimize more for performance (-O3)" - depends on ARC help Choosing this option will pass "-O3" to your compiler to optimize the kernel yet more for performance. @@ -2217,8 +2232,8 @@ config MODULE_COMPRESS bool "Compress modules on installation" help - Compresses kernel modules when 'make modules_install' is run; gzip or - xz depending on "Compression algorithm" below. + Compresses kernel modules when 'make modules_install' is run; gzip, + xz, or zstd depending on "Compression algorithm" below. module-init-tools MAY support gzip, and kmod MAY support gzip and xz. @@ -2240,7 +2255,7 @@ choice This determines which sort of compression will be used during 'make modules_install'. - GZIP (default) and XZ are supported. + GZIP (default), XZ, and ZSTD are supported. config MODULE_COMPRESS_GZIP bool "GZIP" @@ -2248,6 +2263,9 @@ config MODULE_COMPRESS_GZIP config MODULE_COMPRESS_XZ bool "XZ" +config MODULE_COMPRESS_ZSTD + bool "ZSTD" + endchoice config MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS @@ -2263,6 +2281,18 @@ config MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS If unsure, say N. +config MODPROBE_PATH + string "Path to modprobe binary" + default "/sbin/modprobe" + help + When kernel code requests a module, it does so by calling + the "modprobe" userspace utility. This option allows you to + set the path where that binary is found. This can be changed + at runtime via the sysctl file + /proc/sys/kernel/modprobe. Setting this to the empty string + removes the kernel's ability to request modules (but + userspace can still load modules explicitly). + config TRIM_UNUSED_KSYMS bool "Trim unused exported kernel symbols" if EXPERT depends on !COMPILE_TEST diff --git a/init/init_task.c b/init/init_task.c index 3711cdaafed2..8b08c2e19cbb 100644 --- a/init/init_task.c +++ b/init/init_task.c @@ -210,7 +210,7 @@ struct task_struct init_task #ifdef CONFIG_SECURITY .security = NULL, #endif -#ifdef CONFIG_SECCOMP +#ifdef CONFIG_SECCOMP_FILTER .seccomp = { .filter_count = ATOMIC_INIT(0) }, #endif }; diff --git a/init/initramfs.c b/init/initramfs.c index d677e8e717f1..af27abc59643 100644 --- a/init/initramfs.c +++ b/init/initramfs.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include #include #include @@ -541,6 +542,14 @@ static int __init keepinitrd_setup(char *__unused) __setup("keepinitrd", keepinitrd_setup); #endif +static bool __initdata initramfs_async = true; +static int __init initramfs_async_setup(char *str) +{ + strtobool(str, &initramfs_async); + return 1; +} +__setup("initramfs_async=", initramfs_async_setup); + extern char __initramfs_start[]; extern unsigned long __initramfs_size; #include @@ -658,7 +667,7 @@ static void __init populate_initrd_image(char *err) } #endif /* CONFIG_BLK_DEV_RAM */ -static int __init populate_rootfs(void) +static void __init do_populate_rootfs(void *unused, async_cookie_t cookie) { /* Load the built in initramfs */ char *err = unpack_to_rootfs(__initramfs_start, __initramfs_size); @@ -693,6 +702,33 @@ static int __init populate_rootfs(void) initrd_end = 0; flush_delayed_fput(); +} + +static ASYNC_DOMAIN_EXCLUSIVE(initramfs_domain); +static async_cookie_t initramfs_cookie; + +void wait_for_initramfs(void) +{ + if (!initramfs_cookie) { + /* + * Something before rootfs_initcall wants to access + * the filesystem/initramfs. Probably a bug. Make a + * note, avoid deadlocking the machine, and let the + * caller's access fail as it used to. + */ + pr_warn_once("wait_for_initramfs() called before rootfs_initcalls\n"); + return; + } + async_synchronize_cookie_domain(initramfs_cookie + 1, &initramfs_domain); +} +EXPORT_SYMBOL_GPL(wait_for_initramfs); + +static int __init populate_rootfs(void) +{ + initramfs_cookie = async_schedule_domain(do_populate_rootfs, NULL, + &initramfs_domain); + if (!initramfs_async) + wait_for_initramfs(); return 0; } rootfs_initcall(populate_rootfs); diff --git a/init/main.c b/init/main.c index 53b278845b88..64253b181a84 100644 --- a/init/main.c +++ b/init/main.c @@ -1538,6 +1538,7 @@ static noinline void __init kernel_init_freeable(void) kunit_run_all_tests(); + wait_for_initramfs(); console_on_rootfs(); /* diff --git a/kernel/.gitignore b/kernel/.gitignore index 78701ea37c97..5518835ac35c 100644 --- a/kernel/.gitignore +++ b/kernel/.gitignore @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only +/config_data kheaders.md5 timeconst.h hz.bc diff --git a/kernel/Makefile b/kernel/Makefile index 320f1f3941b7..605ec3e70cb7 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -138,10 +138,15 @@ obj-$(CONFIG_SCF_TORTURE_TEST) += scftorture.o $(obj)/configs.o: $(obj)/config_data.gz -targets += config_data.gz -$(obj)/config_data.gz: $(KCONFIG_CONFIG) FORCE +targets += config_data config_data.gz +$(obj)/config_data.gz: $(obj)/config_data FORCE $(call if_changed,gzip) +filechk_cat = cat $< + +$(obj)/config_data: $(KCONFIG_CONFIG) FORCE + $(call filechk,cat) + $(obj)/kheaders.o: $(obj)/kheaders_data.tar.xz quiet_cmd_genikh = CHK $(obj)/kheaders_data.tar.xz diff --git a/kernel/bpf/ringbuf.c b/kernel/bpf/ringbuf.c index f25b719ac786..84b3b35fc0d0 100644 --- a/kernel/bpf/ringbuf.c +++ b/kernel/bpf/ringbuf.c @@ -221,25 +221,20 @@ static int ringbuf_map_get_next_key(struct bpf_map *map, void *key, return -ENOTSUPP; } -static size_t bpf_ringbuf_mmap_page_cnt(const struct bpf_ringbuf *rb) -{ - size_t data_pages = (rb->mask + 1) >> PAGE_SHIFT; - - /* consumer page + producer page + 2 x data pages */ - return RINGBUF_POS_PAGES + 2 * data_pages; -} - static int ringbuf_map_mmap(struct bpf_map *map, struct vm_area_struct *vma) { struct bpf_ringbuf_map *rb_map; - size_t mmap_sz; rb_map = container_of(map, struct bpf_ringbuf_map, map); - mmap_sz = bpf_ringbuf_mmap_page_cnt(rb_map->rb) << PAGE_SHIFT; - - if (vma->vm_pgoff * PAGE_SIZE + (vma->vm_end - vma->vm_start) > mmap_sz) - return -EINVAL; + if (vma->vm_flags & VM_WRITE) { + /* allow writable mapping for the consumer_pos only */ + if (vma->vm_pgoff != 0 || vma->vm_end - vma->vm_start != PAGE_SIZE) + return -EPERM; + } else { + vma->vm_flags &= ~VM_MAYWRITE; + } + /* remap_vmalloc_range() checks size and offset constraints */ return remap_vmalloc_range(vma, rb_map->rb, vma->vm_pgoff + RINGBUF_PGOFF); } @@ -315,6 +310,9 @@ static void *__bpf_ringbuf_reserve(struct bpf_ringbuf *rb, u64 size) return NULL; len = round_up(size + BPF_RINGBUF_HDR_SZ, 8); + if (len > rb->mask + 1) + return NULL; + cons_pos = smp_load_acquire(&rb->consumer_pos); if (in_nmi()) { diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 0399ac092b36..7fa6fc6bedf1 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -1362,9 +1362,7 @@ static bool __reg64_bound_s32(s64 a) static bool __reg64_bound_u32(u64 a) { - if (a > U32_MIN && a < U32_MAX) - return true; - return false; + return a > U32_MIN && a < U32_MAX; } static void __reg_combine_64_into_32(struct bpf_reg_state *reg) @@ -1375,10 +1373,10 @@ static void __reg_combine_64_into_32(struct bpf_reg_state *reg) reg->s32_min_value = (s32)reg->smin_value; reg->s32_max_value = (s32)reg->smax_value; } - if (__reg64_bound_u32(reg->umin_value)) + if (__reg64_bound_u32(reg->umin_value) && __reg64_bound_u32(reg->umax_value)) { reg->u32_min_value = (u32)reg->umin_value; - if (__reg64_bound_u32(reg->umax_value)) reg->u32_max_value = (u32)reg->umax_value; + } /* Intersecting with the old var_off might have improved our bounds * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), @@ -5952,6 +5950,7 @@ static int sanitize_ptr_alu(struct bpf_verifier_env *env, { struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : tmp_aux; struct bpf_verifier_state *vstate = env->cur_state; + bool off_is_imm = tnum_is_const(off_reg->var_off); bool off_is_neg = off_reg->smin_value < 0; bool ptr_is_dst_reg = ptr_reg == dst_reg; u8 opcode = BPF_OP(insn->code); @@ -5982,6 +5981,7 @@ static int sanitize_ptr_alu(struct bpf_verifier_env *env, alu_limit = abs(tmp_aux->alu_limit - alu_limit); } else { alu_state = off_is_neg ? BPF_ALU_NEG_VALUE : 0; + alu_state |= off_is_imm ? BPF_ALU_IMMEDIATE : 0; alu_state |= ptr_is_dst_reg ? BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST; } @@ -6538,11 +6538,10 @@ static void scalar32_min_max_and(struct bpf_reg_state *dst_reg, s32 smin_val = src_reg->s32_min_value; u32 umax_val = src_reg->u32_max_value; - /* Assuming scalar64_min_max_and will be called so its safe - * to skip updating register for known 32-bit case. - */ - if (src_known && dst_known) + if (src_known && dst_known) { + __mark_reg32_known(dst_reg, var32_off.value); return; + } /* We get our minimum from the var_off, since that's inherently * bitwise. Our maximum is the minimum of the operands' maxima. @@ -6562,7 +6561,6 @@ static void scalar32_min_max_and(struct bpf_reg_state *dst_reg, dst_reg->s32_min_value = dst_reg->u32_min_value; dst_reg->s32_max_value = dst_reg->u32_max_value; } - } static void scalar_min_max_and(struct bpf_reg_state *dst_reg, @@ -6609,11 +6607,10 @@ static void scalar32_min_max_or(struct bpf_reg_state *dst_reg, s32 smin_val = src_reg->s32_min_value; u32 umin_val = src_reg->u32_min_value; - /* Assuming scalar64_min_max_or will be called so it is safe - * to skip updating register for known case. - */ - if (src_known && dst_known) + if (src_known && dst_known) { + __mark_reg32_known(dst_reg, var32_off.value); return; + } /* We get our maximum from the var_off, and our minimum is the * maximum of the operands' minima @@ -6678,11 +6675,10 @@ static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg, struct tnum var32_off = tnum_subreg(dst_reg->var_off); s32 smin_val = src_reg->s32_min_value; - /* Assuming scalar64_min_max_xor will be called so it is safe - * to skip updating register for known case. - */ - if (src_known && dst_known) + if (src_known && dst_known) { + __mark_reg32_known(dst_reg, var32_off.value); return; + } /* We get both minimum and maximum from the var32_off. */ dst_reg->u32_min_value = var32_off.value; @@ -11740,7 +11736,7 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env) const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X; struct bpf_insn insn_buf[16]; struct bpf_insn *patch = &insn_buf[0]; - bool issrc, isneg; + bool issrc, isneg, isimm; u32 off_reg; aux = &env->insn_aux_data[i + delta]; @@ -11751,28 +11747,29 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env) isneg = aux->alu_state & BPF_ALU_NEG_VALUE; issrc = (aux->alu_state & BPF_ALU_SANITIZE) == BPF_ALU_SANITIZE_SRC; + isimm = aux->alu_state & BPF_ALU_IMMEDIATE; off_reg = issrc ? insn->src_reg : insn->dst_reg; - if (isneg) - *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); - *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); - *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg); - *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg); - *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0); - *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63); - if (issrc) { - *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, - off_reg); - insn->src_reg = BPF_REG_AX; + if (isimm) { + *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); } else { - *patch++ = BPF_ALU64_REG(BPF_AND, off_reg, - BPF_REG_AX); + if (isneg) + *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); + *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); + *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg); + *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg); + *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0); + *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63); + *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg); } + if (!issrc) + *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg); + insn->src_reg = BPF_REG_AX; if (isneg) insn->code = insn->code == code_add ? code_sub : code_add; *patch++ = *insn; - if (issrc && isneg) + if (issrc && isneg && !isimm) *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); cnt = patch - insn_buf; diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index c10e855a03bc..fe4c01c14ab2 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -608,7 +608,8 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr, enum dma_data_direction dir, unsigned long attrs) { unsigned int offset = swiotlb_align_offset(dev, orig_addr); - unsigned int index, i; + unsigned int i; + int index; phys_addr_t tlb_addr; if (no_iotlb_memory) diff --git a/kernel/events/core.c b/kernel/events/core.c index 03db40f6cba9..c24ea952e7ae 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -2204,6 +2204,26 @@ static void perf_group_detach(struct perf_event *event) perf_event__header_size(leader); } +static void sync_child_event(struct perf_event *child_event); + +static void perf_child_detach(struct perf_event *event) +{ + struct perf_event *parent_event = event->parent; + + if (!(event->attach_state & PERF_ATTACH_CHILD)) + return; + + event->attach_state &= ~PERF_ATTACH_CHILD; + + if (WARN_ON_ONCE(!parent_event)) + return; + + lockdep_assert_held(&parent_event->child_mutex); + + sync_child_event(event); + list_del_init(&event->child_list); +} + static bool is_orphaned_event(struct perf_event *event) { return event->state == PERF_EVENT_STATE_DEAD; @@ -2311,6 +2331,7 @@ group_sched_out(struct perf_event *group_event, } #define DETACH_GROUP 0x01UL +#define DETACH_CHILD 0x02UL /* * Cross CPU call to remove a performance event @@ -2334,6 +2355,8 @@ __perf_remove_from_context(struct perf_event *event, event_sched_out(event, cpuctx, ctx); if (flags & DETACH_GROUP) perf_group_detach(event); + if (flags & DETACH_CHILD) + perf_child_detach(event); list_del_event(event, ctx); if (!ctx->nr_events && ctx->is_active) { @@ -2362,25 +2385,21 @@ static void perf_remove_from_context(struct perf_event *event, unsigned long fla lockdep_assert_held(&ctx->mutex); - event_function_call(event, __perf_remove_from_context, (void *)flags); - /* - * The above event_function_call() can NO-OP when it hits - * TASK_TOMBSTONE. In that case we must already have been detached - * from the context (by perf_event_exit_event()) but the grouping - * might still be in-tact. + * Because of perf_event_exit_task(), perf_remove_from_context() ought + * to work in the face of TASK_TOMBSTONE, unlike every other + * event_function_call() user. */ - WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT); - if ((flags & DETACH_GROUP) && - (event->attach_state & PERF_ATTACH_GROUP)) { - /* - * Since in that case we cannot possibly be scheduled, simply - * detach now. - */ - raw_spin_lock_irq(&ctx->lock); - perf_group_detach(event); + raw_spin_lock_irq(&ctx->lock); + if (!ctx->is_active) { + __perf_remove_from_context(event, __get_cpu_context(ctx), + ctx, (void *)flags); raw_spin_unlock_irq(&ctx->lock); + return; } + raw_spin_unlock_irq(&ctx->lock); + + event_function_call(event, __perf_remove_from_context, (void *)flags); } /* @@ -11829,12 +11848,12 @@ SYSCALL_DEFINE5(perf_event_open, return err; } - err = security_locked_down(LOCKDOWN_PERF); - if (err && (attr.sample_type & PERF_SAMPLE_REGS_INTR)) - /* REGS_INTR can leak data, lockdown must prevent this */ - return err; - - err = 0; + /* REGS_INTR can leak data, lockdown must prevent this */ + if (attr.sample_type & PERF_SAMPLE_REGS_INTR) { + err = security_locked_down(LOCKDOWN_PERF); + if (err) + return err; + } /* * In cgroup mode, the pid argument is used to pass the fd @@ -12373,14 +12392,17 @@ void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu) } EXPORT_SYMBOL_GPL(perf_pmu_migrate_context); -static void sync_child_event(struct perf_event *child_event, - struct task_struct *child) +static void sync_child_event(struct perf_event *child_event) { struct perf_event *parent_event = child_event->parent; u64 child_val; - if (child_event->attr.inherit_stat) - perf_event_read_event(child_event, child); + if (child_event->attr.inherit_stat) { + struct task_struct *task = child_event->ctx->task; + + if (task && task != TASK_TOMBSTONE) + perf_event_read_event(child_event, task); + } child_val = perf_event_count(child_event); @@ -12395,60 +12417,53 @@ static void sync_child_event(struct perf_event *child_event, } static void -perf_event_exit_event(struct perf_event *child_event, - struct perf_event_context *child_ctx, - struct task_struct *child) +perf_event_exit_event(struct perf_event *event, struct perf_event_context *ctx) { - struct perf_event *parent_event = child_event->parent; + struct perf_event *parent_event = event->parent; + unsigned long detach_flags = 0; - /* - * Do not destroy the 'original' grouping; because of the context - * switch optimization the original events could've ended up in a - * random child task. - * - * If we were to destroy the original group, all group related - * operations would cease to function properly after this random - * child dies. - * - * Do destroy all inherited groups, we don't care about those - * and being thorough is better. - */ - raw_spin_lock_irq(&child_ctx->lock); - WARN_ON_ONCE(child_ctx->is_active); + if (parent_event) { + /* + * Do not destroy the 'original' grouping; because of the + * context switch optimization the original events could've + * ended up in a random child task. + * + * If we were to destroy the original group, all group related + * operations would cease to function properly after this + * random child dies. + * + * Do destroy all inherited groups, we don't care about those + * and being thorough is better. + */ + detach_flags = DETACH_GROUP | DETACH_CHILD; + mutex_lock(&parent_event->child_mutex); + } - if (parent_event) - perf_group_detach(child_event); - list_del_event(child_event, child_ctx); - perf_event_set_state(child_event, PERF_EVENT_STATE_EXIT); /* is_event_hup() */ - raw_spin_unlock_irq(&child_ctx->lock); + perf_remove_from_context(event, detach_flags); + + raw_spin_lock_irq(&ctx->lock); + if (event->state > PERF_EVENT_STATE_EXIT) + perf_event_set_state(event, PERF_EVENT_STATE_EXIT); + raw_spin_unlock_irq(&ctx->lock); /* - * Parent events are governed by their filedesc, retain them. + * Child events can be freed. */ - if (!parent_event) { - perf_event_wakeup(child_event); + if (parent_event) { + mutex_unlock(&parent_event->child_mutex); + /* + * Kick perf_poll() for is_event_hup(); + */ + perf_event_wakeup(parent_event); + free_event(event); + put_event(parent_event); return; } - /* - * Child events can be cleaned up. - */ - - sync_child_event(child_event, child); /* - * Remove this event from the parent's list - */ - WARN_ON_ONCE(parent_event->ctx->parent_ctx); - mutex_lock(&parent_event->child_mutex); - list_del_init(&child_event->child_list); - mutex_unlock(&parent_event->child_mutex); - - /* - * Kick perf_poll() for is_event_hup(). + * Parent events are governed by their filedesc, retain them. */ - perf_event_wakeup(parent_event); - free_event(child_event); - put_event(parent_event); + perf_event_wakeup(event); } static void perf_event_exit_task_context(struct task_struct *child, int ctxn) @@ -12505,7 +12520,7 @@ static void perf_event_exit_task_context(struct task_struct *child, int ctxn) perf_event_task(child, child_ctx, 0); list_for_each_entry_safe(child_event, next, &child_ctx->event_list, event_entry) - perf_event_exit_event(child_event, child_ctx, child); + perf_event_exit_event(child_event, child_ctx); mutex_unlock(&child_ctx->mutex); @@ -12765,6 +12780,7 @@ inherit_event(struct perf_event *parent_event, */ raw_spin_lock_irqsave(&child_ctx->lock, flags); add_event_to_ctx(child_event, child_ctx); + child_event->attach_state |= PERF_ATTACH_CHILD; raw_spin_unlock_irqrestore(&child_ctx->lock, flags); /* diff --git a/kernel/fork.c b/kernel/fork.c index 426cd0c51f9e..e83a79860d58 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -97,6 +97,10 @@ #include #include +#ifdef CONFIG_USER_NS +#include +#endif + #include #include #include @@ -1872,6 +1876,10 @@ static __latent_entropy struct task_struct *copy_process( if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS)) return ERR_PTR(-EINVAL); + if ((clone_flags & CLONE_NEWUSER) && !unprivileged_userns_clone) + if (!capable(CAP_SYS_ADMIN)) + return ERR_PTR(-EPERM); + /* * Thread groups must share signals as well, and detached threads * can only be started up within the thread group. @@ -2971,6 +2979,12 @@ int ksys_unshare(unsigned long unshare_flags) if (unshare_flags & CLONE_NEWNS) unshare_flags |= CLONE_FS; + if ((unshare_flags & CLONE_NEWUSER) && !unprivileged_userns_clone) { + err = -EPERM; + if (!capable(CAP_SYS_ADMIN)) + goto bad_unshare_out; + } + err = check_unshare_flags(unshare_flags); if (err) goto bad_unshare_out; diff --git a/kernel/futex.c b/kernel/futex.c index 00febd6dea9c..a8629b695d38 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -3711,8 +3711,7 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, if (op & FUTEX_CLOCK_REALTIME) { flags |= FLAGS_CLOCKRT; - if (cmd != FUTEX_WAIT && cmd != FUTEX_WAIT_BITSET && \ - cmd != FUTEX_WAIT_REQUEUE_PI) + if (cmd != FUTEX_WAIT_BITSET && cmd != FUTEX_WAIT_REQUEUE_PI) return -ENOSYS; } @@ -3782,7 +3781,7 @@ SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val, t = timespec64_to_ktime(ts); if (cmd == FUTEX_WAIT) t = ktime_add_safe(ktime_get(), t); - else if (!(op & FUTEX_CLOCK_REALTIME)) + else if (cmd != FUTEX_LOCK_PI && !(op & FUTEX_CLOCK_REALTIME)) t = timens_ktime_to_host(CLOCK_MONOTONIC, t); tp = &t; } @@ -3976,7 +3975,7 @@ SYSCALL_DEFINE6(futex_time32, u32 __user *, uaddr, int, op, u32, val, t = timespec64_to_ktime(ts); if (cmd == FUTEX_WAIT) t = ktime_add_safe(ktime_get(), t); - else if (!(op & FUTEX_CLOCK_REALTIME)) + else if (cmd != FUTEX_LOCK_PI && !(op & FUTEX_CLOCK_REALTIME)) t = timens_ktime_to_host(CLOCK_MONOTONIC, t); tp = &t; } diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c index cc1a09406c6e..6c2a75eb67bd 100644 --- a/kernel/irq/irqdesc.c +++ b/kernel/irq/irqdesc.c @@ -654,6 +654,27 @@ int generic_handle_irq(unsigned int irq) } EXPORT_SYMBOL_GPL(generic_handle_irq); +/** + * generic_dispatch_irq - Dispatch an interrupt from an interrupt handler + * @irq: The irq number to handle + * + * A wrapper around generic_handle_irq() which ensures that interrupts are + * disabled when the primary handler of the dispatched irq is invoked. + * This is useful for interrupt handlers with dispatching to be safe for + * the forced threaded case. + */ +int generic_dispatch_irq(unsigned int irq) +{ + unsigned long flags; + int ret; + + local_irq_save(flags); + ret = generic_handle_irq(irq); + local_irq_restore(flags); + return ret; +} +EXPORT_SYMBOL_GPL(generic_dispatch_irq); + #ifdef CONFIG_HANDLE_DOMAIN_IRQ /** * __handle_domain_irq - Invoke the handler for a HW irq belonging to a domain diff --git a/kernel/irq/matrix.c b/kernel/irq/matrix.c index 651a4ad6d711..8e586858bcf4 100644 --- a/kernel/irq/matrix.c +++ b/kernel/irq/matrix.c @@ -423,7 +423,9 @@ void irq_matrix_free(struct irq_matrix *m, unsigned int cpu, if (WARN_ON_ONCE(bit < m->alloc_start || bit >= m->alloc_end)) return; - clear_bit(bit, cm->alloc_map); + if (WARN_ON_ONCE(!test_and_clear_bit(bit, cm->alloc_map))) + return; + cm->allocated--; if(managed) cm->managed_allocated--; diff --git a/kernel/kcsan/core.c b/kernel/kcsan/core.c index 3bf98db9c702..23e7acb5c667 100644 --- a/kernel/kcsan/core.c +++ b/kernel/kcsan/core.c @@ -639,8 +639,6 @@ void __init kcsan_init(void) BUG_ON(!in_task()); - kcsan_debugfs_init(); - for_each_possible_cpu(cpu) per_cpu(kcsan_rand_state, cpu) = (u32)get_cycles(); diff --git a/kernel/kcsan/debugfs.c b/kernel/kcsan/debugfs.c index 3c8093a371b1..209ad8dcfcec 100644 --- a/kernel/kcsan/debugfs.c +++ b/kernel/kcsan/debugfs.c @@ -261,7 +261,9 @@ static const struct file_operations debugfs_ops = .release = single_release }; -void __init kcsan_debugfs_init(void) +static void __init kcsan_debugfs_init(void) { debugfs_create_file("kcsan", 0644, NULL, NULL, &debugfs_ops); } + +late_initcall(kcsan_debugfs_init); diff --git a/kernel/kcsan/kcsan.h b/kernel/kcsan/kcsan.h index 8d4bf3431b3c..87ccdb3b051f 100644 --- a/kernel/kcsan/kcsan.h +++ b/kernel/kcsan/kcsan.h @@ -30,11 +30,6 @@ extern bool kcsan_enabled; void kcsan_save_irqtrace(struct task_struct *task); void kcsan_restore_irqtrace(struct task_struct *task); -/* - * Initialize debugfs file. - */ -void kcsan_debugfs_init(void); - /* * Statistics counters displayed via debugfs; should only be modified in * slow-paths. diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index 5c3447cf7ad5..33400ff051a8 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -740,8 +740,10 @@ static int kexec_calculate_store_digests(struct kimage *image) sha_region_sz = KEXEC_SEGMENT_MAX * sizeof(struct kexec_sha_region); sha_regions = vzalloc(sha_region_sz); - if (!sha_regions) + if (!sha_regions) { + ret = -ENOMEM; goto out_free_desc; + } desc->tfm = tfm; diff --git a/kernel/kmod.c b/kernel/kmod.c index 3cd075ce2a1e..b717134ebe17 100644 --- a/kernel/kmod.c +++ b/kernel/kmod.c @@ -58,7 +58,7 @@ static DECLARE_WAIT_QUEUE_HEAD(kmod_wq); /* modprobe_path is set via /proc/sys. */ -char modprobe_path[KMOD_PATH_LEN] = "/sbin/modprobe"; +char modprobe_path[KMOD_PATH_LEN] = CONFIG_MODPROBE_PATH; static void free_modprobe_argv(struct subprocess_info *info) { diff --git a/kernel/kthread.c b/kernel/kthread.c index 1578973c5740..6d3c488a0f82 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -84,6 +84,25 @@ static inline struct kthread *to_kthread(struct task_struct *k) return (__force void *)k->set_child_tid; } +/* + * Variant of to_kthread() that doesn't assume @p is a kthread. + * + * Per construction; when: + * + * (p->flags & PF_KTHREAD) && p->set_child_tid + * + * the task is both a kthread and struct kthread is persistent. However + * PF_KTHREAD on it's own is not, kernel_thread() can exec() (See umh.c and + * begin_new_exec()). + */ +static inline struct kthread *__to_kthread(struct task_struct *p) +{ + void *kthread = (__force void *)p->set_child_tid; + if (kthread && !(p->flags & PF_KTHREAD)) + kthread = NULL; + return kthread; +} + void free_kthread_struct(struct task_struct *k) { struct kthread *kthread; @@ -168,8 +187,9 @@ EXPORT_SYMBOL_GPL(kthread_freezable_should_stop); */ void *kthread_func(struct task_struct *task) { - if (task->flags & PF_KTHREAD) - return to_kthread(task)->threadfn; + struct kthread *kthread = __to_kthread(task); + if (kthread) + return kthread->threadfn; return NULL; } EXPORT_SYMBOL_GPL(kthread_func); @@ -199,10 +219,11 @@ EXPORT_SYMBOL_GPL(kthread_data); */ void *kthread_probe_data(struct task_struct *task) { - struct kthread *kthread = to_kthread(task); + struct kthread *kthread = __to_kthread(task); void *data = NULL; - copy_from_kernel_nofault(&data, &kthread->data, sizeof(data)); + if (kthread) + copy_from_kernel_nofault(&data, &kthread->data, sizeof(data)); return data; } @@ -514,9 +535,9 @@ void kthread_set_per_cpu(struct task_struct *k, int cpu) set_bit(KTHREAD_IS_PER_CPU, &kthread->flags); } -bool kthread_is_per_cpu(struct task_struct *k) +bool kthread_is_per_cpu(struct task_struct *p) { - struct kthread *kthread = to_kthread(k); + struct kthread *kthread = __to_kthread(p); if (!kthread) return false; diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 575a34b88936..77ae2704e979 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -1494,6 +1494,7 @@ static int syslog_print_all(char __user *buf, int size, bool clear) struct printk_info info; unsigned int line_count; struct printk_record r; + u64 max_seq; char *text; int len = 0; u64 seq; @@ -1512,9 +1513,15 @@ static int syslog_print_all(char __user *buf, int size, bool clear) prb_for_each_info(clear_seq, prb, seq, &info, &line_count) len += get_record_print_text_size(&info, line_count, true, time); + /* + * Set an upper bound for the next loop to avoid subtracting lengths + * that were never added. + */ + max_seq = seq; + /* move first record forward until length fits into the buffer */ prb_for_each_info(clear_seq, prb, seq, &info, &line_count) { - if (len <= size) + if (len <= size || info.seq >= max_seq) break; len -= get_record_print_text_size(&info, line_count, true, time); } diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index da6f5213fb74..7356764e49a0 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -1077,7 +1077,6 @@ noinstr void rcu_nmi_enter(void) } else if (!in_nmi()) { instrumentation_begin(); rcu_irq_enter_check_tick(); - instrumentation_end(); } else { instrumentation_begin(); } @@ -3464,7 +3463,7 @@ static void fill_page_cache_func(struct work_struct *work) for (i = 0; i < rcu_min_cached_objs; i++) { bnode = (struct kvfree_rcu_bulk_data *) - __get_free_page(GFP_KERNEL | __GFP_NOWARN); + __get_free_page(GFP_KERNEL | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN); if (bnode) { raw_spin_lock_irqsave(&krcp->lock, flags); diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 2d603771c7dc..0796a75b6e0e 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -1646,7 +1646,11 @@ static bool wake_nocb_gp(struct rcu_data *rdp, bool force, rcu_nocb_unlock_irqrestore(rdp, flags); return false; } - del_timer(&rdp->nocb_timer); + + if (READ_ONCE(rdp->nocb_defer_wakeup) > RCU_NOCB_WAKE_NOT) { + WRITE_ONCE(rdp->nocb_defer_wakeup, RCU_NOCB_WAKE_NOT); + del_timer(&rdp->nocb_timer); + } rcu_nocb_unlock_irqrestore(rdp, flags); raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags); if (force || READ_ONCE(rdp_gp->nocb_gp_sleep)) { @@ -2265,7 +2269,6 @@ static bool do_nocb_deferred_wakeup_common(struct rcu_data *rdp) return false; } ndw = READ_ONCE(rdp->nocb_defer_wakeup); - WRITE_ONCE(rdp->nocb_defer_wakeup, RCU_NOCB_WAKE_NOT); ret = wake_nocb_gp(rdp, ndw == RCU_NOCB_WAKE_FORCE, flags); trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DeferredWake")); diff --git a/kernel/resource.c b/kernel/resource.c index 627e61b0c124..16e0c7e8ed24 100644 --- a/kernel/resource.c +++ b/kernel/resource.c @@ -457,7 +457,7 @@ int walk_system_ram_res(u64 start, u64 end, void *arg, { unsigned long flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; - return __walk_iomem_res_desc(start, end, flags, IORES_DESC_NONE, true, + return __walk_iomem_res_desc(start, end, flags, IORES_DESC_NONE, false, arg, func); } @@ -470,7 +470,7 @@ int walk_mem_res(u64 start, u64 end, void *arg, { unsigned long flags = IORESOURCE_MEM | IORESOURCE_BUSY; - return __walk_iomem_res_desc(start, end, flags, IORES_DESC_NONE, true, + return __walk_iomem_res_desc(start, end, flags, IORES_DESC_NONE, false, arg, func); } diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 98191218d891..814200541f8f 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -928,7 +928,7 @@ DEFINE_STATIC_KEY_FALSE(sched_uclamp_used); static inline unsigned int uclamp_bucket_id(unsigned int clamp_value) { - return clamp_value / UCLAMP_BUCKET_DELTA; + return min_t(unsigned int, clamp_value / UCLAMP_BUCKET_DELTA, UCLAMP_BUCKETS - 1); } static inline unsigned int uclamp_none(enum uclamp_id clamp_id) @@ -7652,7 +7652,7 @@ static void balance_push(struct rq *rq) * histerical raisins. */ if (rq->idle == push_task || - ((push_task->flags & PF_KTHREAD) && kthread_is_per_cpu(push_task)) || + kthread_is_per_cpu(push_task) || is_migration_disabled(push_task)) { /* diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 486f403a778b..9c8b3ed2199a 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -8,8 +8,6 @@ */ #include "sched.h" -static DEFINE_SPINLOCK(sched_debug_lock); - /* * This allows printing both to /proc/sched_debug and * to the console @@ -470,16 +468,37 @@ static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_group #endif #ifdef CONFIG_CGROUP_SCHED +static DEFINE_SPINLOCK(sched_debug_lock); static char group_path[PATH_MAX]; -static char *task_group_path(struct task_group *tg) +static void task_group_path(struct task_group *tg, char *path, int plen) { - if (autogroup_path(tg, group_path, PATH_MAX)) - return group_path; + if (autogroup_path(tg, path, plen)) + return; - cgroup_path(tg->css.cgroup, group_path, PATH_MAX); + cgroup_path(tg->css.cgroup, path, plen); +} - return group_path; +/* + * Only 1 SEQ_printf_task_group_path() caller can use the full length + * group_path[] for cgroup path. Other simultaneous callers will have + * to use a shorter stack buffer. A "..." suffix is appended at the end + * of the stack buffer so that it will show up in case the output length + * matches the given buffer size to indicate possible path name truncation. + */ +#define SEQ_printf_task_group_path(m, tg, fmt...) \ +{ \ + if (spin_trylock(&sched_debug_lock)) { \ + task_group_path(tg, group_path, sizeof(group_path)); \ + SEQ_printf(m, fmt, group_path); \ + spin_unlock(&sched_debug_lock); \ + } else { \ + char buf[128]; \ + char *bufend = buf + sizeof(buf) - 3; \ + task_group_path(tg, buf, bufend - buf); \ + strcpy(bufend - 1, "..."); \ + SEQ_printf(m, fmt, buf); \ + } \ } #endif @@ -506,7 +525,7 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p) SEQ_printf(m, " %d %d", task_node(p), task_numa_group_id(p)); #endif #ifdef CONFIG_CGROUP_SCHED - SEQ_printf(m, " %s", task_group_path(task_group(p))); + SEQ_printf_task_group_path(m, task_group(p), " %s") #endif SEQ_printf(m, "\n"); @@ -543,7 +562,7 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) #ifdef CONFIG_FAIR_GROUP_SCHED SEQ_printf(m, "\n"); - SEQ_printf(m, "cfs_rq[%d]:%s\n", cpu, task_group_path(cfs_rq->tg)); + SEQ_printf_task_group_path(m, cfs_rq->tg, "cfs_rq[%d]:%s\n", cpu); #else SEQ_printf(m, "\n"); SEQ_printf(m, "cfs_rq[%d]:\n", cpu); @@ -614,7 +633,7 @@ void print_rt_rq(struct seq_file *m, int cpu, struct rt_rq *rt_rq) { #ifdef CONFIG_RT_GROUP_SCHED SEQ_printf(m, "\n"); - SEQ_printf(m, "rt_rq[%d]:%s\n", cpu, task_group_path(rt_rq->tg)); + SEQ_printf_task_group_path(m, rt_rq->tg, "rt_rq[%d]:%s\n", cpu); #else SEQ_printf(m, "\n"); SEQ_printf(m, "rt_rq[%d]:\n", cpu); @@ -666,7 +685,6 @@ void print_dl_rq(struct seq_file *m, int cpu, struct dl_rq *dl_rq) static void print_cpu(struct seq_file *m, int cpu) { struct rq *rq = cpu_rq(cpu); - unsigned long flags; #ifdef CONFIG_X86 { @@ -717,13 +735,11 @@ do { \ } #undef P - spin_lock_irqsave(&sched_debug_lock, flags); print_cfs_stats(m, cpu); print_rt_stats(m, cpu); print_dl_stats(m, cpu); print_rq(m, rq, cpu); - spin_unlock_irqrestore(&sched_debug_lock, flags); SEQ_printf(m, "\n"); } diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 794c2cb945f8..a073a839cd06 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -682,7 +682,13 @@ static u64 __sched_period(unsigned long nr_running) */ static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) { - u64 slice = __sched_period(cfs_rq->nr_running + !se->on_rq); + unsigned int nr_running = cfs_rq->nr_running; + u64 slice; + + if (sched_feat(ALT_PERIOD)) + nr_running = rq_of(cfs_rq)->cfs.h_nr_running; + + slice = __sched_period(nr_running + !se->on_rq); for_each_sched_entity(se) { struct load_weight *load; @@ -699,6 +705,10 @@ static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) } slice = __calc_delta(slice, se->load.weight, load); } + + if (sched_feat(BASE_SLICE)) + slice = max(slice, (u64)sysctl_sched_min_granularity); + return slice; } @@ -3941,6 +3951,8 @@ static inline void util_est_dequeue(struct cfs_rq *cfs_rq, trace_sched_util_est_cfs_tp(cfs_rq); } +#define UTIL_EST_MARGIN (SCHED_CAPACITY_SCALE / 100) + /* * Check if a (signed) value is within a specified (unsigned) margin, * based on the observation that: @@ -3958,7 +3970,7 @@ static inline void util_est_update(struct cfs_rq *cfs_rq, struct task_struct *p, bool task_sleep) { - long last_ewma_diff; + long last_ewma_diff, last_enqueued_diff; struct util_est ue; if (!sched_feat(UTIL_EST)) @@ -3979,6 +3991,8 @@ static inline void util_est_update(struct cfs_rq *cfs_rq, if (ue.enqueued & UTIL_AVG_UNCHANGED) return; + last_enqueued_diff = ue.enqueued; + /* * Reset EWMA on utilization increases, the moving average is used only * to smooth utilization decreases. @@ -3992,12 +4006,17 @@ static inline void util_est_update(struct cfs_rq *cfs_rq, } /* - * Skip update of task's estimated utilization when its EWMA is + * Skip update of task's estimated utilization when its members are * already ~1% close to its last activation value. */ last_ewma_diff = ue.enqueued - ue.ewma; - if (within_margin(last_ewma_diff, (SCHED_CAPACITY_SCALE / 100))) + last_enqueued_diff -= ue.enqueued; + if (within_margin(last_ewma_diff, UTIL_EST_MARGIN)) { + if (!within_margin(last_enqueued_diff, UTIL_EST_MARGIN)) + goto done; + return; + } /* * To avoid overestimation of actual task utilization, skip updates if @@ -6098,6 +6117,24 @@ static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpu return -1; } +/* + * Scan the local SMT mask for idle CPUs. + */ +static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target) +{ + int cpu; + + for_each_cpu(cpu, cpu_smt_mask(target)) { + if (!cpumask_test_cpu(cpu, p->cpus_ptr) || + !cpumask_test_cpu(cpu, sched_domain_span(sd))) + continue; + if (available_idle_cpu(cpu) || sched_idle_cpu(cpu)) + return cpu; + } + + return -1; +} + #else /* CONFIG_SCHED_SMT */ static inline void set_idle_cores(int cpu, int val) @@ -6114,6 +6151,11 @@ static inline int select_idle_core(struct task_struct *p, int core, struct cpuma return __select_idle_cpu(core); } +static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target) +{ + return -1; +} + #endif /* CONFIG_SCHED_SMT */ /* @@ -6121,11 +6163,10 @@ static inline int select_idle_core(struct task_struct *p, int core, struct cpuma * comparing the average scan cost (tracked in sd->avg_scan_cost) against the * average idle time for this rq (as found in rq->avg_idle). */ -static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int target) +static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool has_idle_core, int target) { struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_idle_mask); int i, cpu, idle_cpu = -1, nr = INT_MAX; - bool smt = test_idle_cores(target, false); int this = smp_processor_id(); struct sched_domain *this_sd; u64 time; @@ -6136,7 +6177,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr); - if (sched_feat(SIS_PROP) && !smt) { + if (sched_feat(SIS_PROP) && !has_idle_core) { u64 avg_cost, avg_idle, span_avg; /* @@ -6156,7 +6197,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t } for_each_cpu_wrap(cpu, cpus, target) { - if (smt) { + if (has_idle_core) { i = select_idle_core(p, cpu, cpus, &idle_cpu); if ((unsigned int)i < nr_cpumask_bits) return i; @@ -6170,10 +6211,10 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t } } - if (smt) - set_idle_cores(this, false); + if (has_idle_core) + set_idle_cores(target, false); - if (sched_feat(SIS_PROP) && !smt) { + if (sched_feat(SIS_PROP) && !has_idle_core) { time = cpu_clock(this) - time; update_avg(&this_sd->avg_scan_cost, time); } @@ -6228,6 +6269,7 @@ static inline bool asym_fits_capacity(int task_util, int cpu) */ static int select_idle_sibling(struct task_struct *p, int prev, int target) { + bool has_idle_core = false; struct sched_domain *sd; unsigned long task_util; int i, recent_used_cpu; @@ -6307,7 +6349,17 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) if (!sd) return target; - i = select_idle_cpu(p, sd, target); + if (sched_smt_active()) { + has_idle_core = test_idle_cores(target, false); + + if (!has_idle_core && cpus_share_cache(prev, target)) { + i = select_idle_smt(p, sd, prev); + if ((unsigned int)i < nr_cpumask_bits) + return i; + } + } + + i = select_idle_cpu(p, sd, has_idle_core, target); if ((unsigned)i < nr_cpumask_bits) return i; @@ -6518,8 +6570,24 @@ compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd) * its pd list and will not be accounted by compute_energy(). */ for_each_cpu_and(cpu, pd_mask, cpu_online_mask) { - unsigned long cpu_util, util_cfs = cpu_util_next(cpu, p, dst_cpu); - struct task_struct *tsk = cpu == dst_cpu ? p : NULL; + unsigned long util_freq = cpu_util_next(cpu, p, dst_cpu); + unsigned long cpu_util, util_running = util_freq; + struct task_struct *tsk = NULL; + + /* + * When @p is placed on @cpu: + * + * util_running = max(cpu_util, cpu_util_est) + + * max(task_util, _task_util_est) + * + * while cpu_util_next is: max(cpu_util + task_util, + * cpu_util_est + _task_util_est) + */ + if (cpu == dst_cpu) { + tsk = p; + util_running = + cpu_util_next(cpu, p, -1) + task_util_est(p); + } /* * Busy time computation: utilization clamping is not @@ -6527,7 +6595,7 @@ compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd) * is already enough to scale the EM reported power * consumption at the (eventually clamped) cpu_capacity. */ - sum_util += effective_cpu_util(cpu, util_cfs, cpu_cap, + sum_util += effective_cpu_util(cpu, util_running, cpu_cap, ENERGY_UTIL, NULL); /* @@ -6537,7 +6605,7 @@ compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd) * NOTE: in case RT tasks are running, by default the * FREQUENCY_UTIL's utilization can be max OPP. */ - cpu_util = effective_cpu_util(cpu, util_cfs, cpu_cap, + cpu_util = effective_cpu_util(cpu, util_freq, cpu_cap, FREQUENCY_UTIL, tsk); max_util = max(max_util, cpu_util); } @@ -7539,6 +7607,10 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) if (throttled_lb_pair(task_group(p), env->src_cpu, env->dst_cpu)) return 0; + /* Disregard pcpu kthreads; they are where they need to be. */ + if (kthread_is_per_cpu(p)) + return 0; + if (!cpumask_test_cpu(env->dst_cpu, p->cpus_ptr)) { int cpu; @@ -7708,8 +7780,7 @@ static int detach_tasks(struct lb_env *env) * scheduler fails to find a good waiting task to * migrate. */ - - if ((load >> env->sd->nr_balance_failed) > env->imbalance) + if (shr_bound(load, env->sd->nr_balance_failed) > env->imbalance) goto next; env->imbalance -= load; @@ -10844,16 +10915,22 @@ static void propagate_entity_cfs_rq(struct sched_entity *se) { struct cfs_rq *cfs_rq; + list_add_leaf_cfs_rq(cfs_rq_of(se)); + /* Start to propagate at parent */ se = se->parent; for_each_sched_entity(se) { cfs_rq = cfs_rq_of(se); - if (cfs_rq_throttled(cfs_rq)) - break; + if (!cfs_rq_throttled(cfs_rq)){ + update_load_avg(cfs_rq, se, UPDATE_TG); + list_add_leaf_cfs_rq(cfs_rq); + continue; + } - update_load_avg(cfs_rq, se, UPDATE_TG); + if (list_add_leaf_cfs_rq(cfs_rq)) + break; } } #else diff --git a/kernel/sched/features.h b/kernel/sched/features.h index 1bc2b158fc51..e911111df83a 100644 --- a/kernel/sched/features.h +++ b/kernel/sched/features.h @@ -90,3 +90,6 @@ SCHED_FEAT(WA_BIAS, true) */ SCHED_FEAT(UTIL_EST, true) SCHED_FEAT(UTIL_EST_FASTUP, true) + +SCHED_FEAT(ALT_PERIOD, true) +SCHED_FEAT(BASE_SLICE, true) diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c index 967732c0766c..651218ded981 100644 --- a/kernel/sched/psi.c +++ b/kernel/sched/psi.c @@ -711,14 +711,15 @@ static void psi_group_change(struct psi_group *group, int cpu, for (t = 0, m = clear; m; m &= ~(1 << t), t++) { if (!(m & (1 << t))) continue; - if (groupc->tasks[t] == 0 && !psi_bug) { + if (groupc->tasks[t]) { + groupc->tasks[t]--; + } else if (!psi_bug) { printk_deferred(KERN_ERR "psi: task underflow! cpu=%d t=%d tasks=[%u %u %u %u] clear=%x set=%x\n", cpu, t, groupc->tasks[0], groupc->tasks[1], groupc->tasks[2], groupc->tasks[3], clear, set); psi_bug = 1; } - groupc->tasks[t]--; } for (t = 0; set; set &= ~(1 << t), t++) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 10a1522b1e30..e4e4f47cee6a 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -204,6 +204,13 @@ static inline void update_avg(u64 *avg, u64 sample) *avg += diff / 8; } +/* + * Shifting a value by an exponent greater *or equal* to the size of said value + * is UB; cap at size-1. + */ +#define shr_bound(val, shift) \ + (val >> min_t(typeof(shift), shift, BITS_PER_TYPE(typeof(val)) - 1)) + /* * !! For sched_setattr_nocheck() (kernel) only !! * diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 09d35044bd88..12f80587e127 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -723,35 +723,6 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) for (tmp = sd; tmp; tmp = tmp->parent) numa_distance += !!(tmp->flags & SD_NUMA); - /* - * FIXME: Diameter >=3 is misrepresented. - * - * Smallest diameter=3 topology is: - * - * node 0 1 2 3 - * 0: 10 20 30 40 - * 1: 20 10 20 30 - * 2: 30 20 10 20 - * 3: 40 30 20 10 - * - * 0 --- 1 --- 2 --- 3 - * - * NUMA-3 0-3 N/A N/A 0-3 - * groups: {0-2},{1-3} {1-3},{0-2} - * - * NUMA-2 0-2 0-3 0-3 1-3 - * groups: {0-1},{1-3} {0-2},{2-3} {1-3},{0-1} {2-3},{0-2} - * - * NUMA-1 0-1 0-2 1-3 2-3 - * groups: {0},{1} {1},{2},{0} {2},{3},{1} {3},{2} - * - * NUMA-0 0 1 2 3 - * - * The NUMA-2 groups for nodes 0 and 3 are obviously buggered, as the - * group span isn't a subset of the domain span. - */ - WARN_ONCE(numa_distance > 2, "Shortest NUMA path spans too many nodes\n"); - sched_domain_debug(sd, cpu); rq_attach_root(rq, rd); @@ -982,6 +953,31 @@ static void init_overlap_sched_group(struct sched_domain *sd, sg->sgc->max_capacity = SCHED_CAPACITY_SCALE; } +static struct sched_domain * +find_descended_sibling(struct sched_domain *sd, struct sched_domain *sibling) +{ + /* + * The proper descendant would be the one whose child won't span out + * of sd + */ + while (sibling->child && + !cpumask_subset(sched_domain_span(sibling->child), + sched_domain_span(sd))) + sibling = sibling->child; + + /* + * As we are referencing sgc across different topology level, we need + * to go down to skip those sched_domains which don't contribute to + * scheduling because they will be degenerated in cpu_attach_domain + */ + while (sibling->child && + cpumask_equal(sched_domain_span(sibling->child), + sched_domain_span(sibling))) + sibling = sibling->child; + + return sibling; +} + static int build_overlap_sched_groups(struct sched_domain *sd, int cpu) { @@ -1015,6 +1011,41 @@ build_overlap_sched_groups(struct sched_domain *sd, int cpu) if (!cpumask_test_cpu(i, sched_domain_span(sibling))) continue; + /* + * Usually we build sched_group by sibling's child sched_domain + * But for machines whose NUMA diameter are 3 or above, we move + * to build sched_group by sibling's proper descendant's child + * domain because sibling's child sched_domain will span out of + * the sched_domain being built as below. + * + * Smallest diameter=3 topology is: + * + * node 0 1 2 3 + * 0: 10 20 30 40 + * 1: 20 10 20 30 + * 2: 30 20 10 20 + * 3: 40 30 20 10 + * + * 0 --- 1 --- 2 --- 3 + * + * NUMA-3 0-3 N/A N/A 0-3 + * groups: {0-2},{1-3} {1-3},{0-2} + * + * NUMA-2 0-2 0-3 0-3 1-3 + * groups: {0-1},{1-3} {0-2},{2-3} {1-3},{0-1} {2-3},{0-2} + * + * NUMA-1 0-1 0-2 1-3 2-3 + * groups: {0},{1} {1},{2},{0} {2},{3},{1} {3},{2} + * + * NUMA-0 0 1 2 3 + * + * The NUMA-2 groups for nodes 0 and 3 are obviously buggered, as the + * group span isn't a subset of the domain span. + */ + if (sibling->child && + !cpumask_subset(sched_domain_span(sibling->child), span)) + sibling = find_descended_sibling(sd, sibling); + sg = build_group_from_child_sched_domain(sibling, cpu); if (!sg) goto fail; @@ -1022,7 +1053,7 @@ build_overlap_sched_groups(struct sched_domain *sd, int cpu) sg_span = sched_group_span(sg); cpumask_or(covered, covered, sg_span); - init_overlap_sched_group(sd, sg); + init_overlap_sched_group(sibling, sg); if (!first) first = sg; diff --git a/kernel/smp.c b/kernel/smp.c index aeb0adfa0606..c678589fbb76 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -110,7 +110,7 @@ static DEFINE_PER_CPU(void *, cur_csd_info); static atomic_t csd_bug_count = ATOMIC_INIT(0); /* Record current CSD work for current CPU, NULL to erase. */ -static void csd_lock_record(call_single_data_t *csd) +static void csd_lock_record(struct __call_single_data *csd) { if (!csd) { smp_mb(); /* NULL cur_csd after unlock. */ @@ -125,7 +125,7 @@ static void csd_lock_record(call_single_data_t *csd) /* Or before unlock, as the case may be. */ } -static __always_inline int csd_lock_wait_getcpu(call_single_data_t *csd) +static __always_inline int csd_lock_wait_getcpu(struct __call_single_data *csd) { unsigned int csd_type; @@ -140,7 +140,7 @@ static __always_inline int csd_lock_wait_getcpu(call_single_data_t *csd) * the CSD_TYPE_SYNC/ASYNC types provide the destination CPU, * so waiting on other types gets much less information. */ -static __always_inline bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, int *bug_id) +static __always_inline bool csd_lock_wait_toolong(struct __call_single_data *csd, u64 ts0, u64 *ts1, int *bug_id) { int cpu = -1; int cpux; @@ -204,7 +204,7 @@ static __always_inline bool csd_lock_wait_toolong(call_single_data_t *csd, u64 t * previous function call. For multi-cpu calls its even more interesting * as we'll have to ensure no other cpu is observing our csd. */ -static __always_inline void csd_lock_wait(call_single_data_t *csd) +static __always_inline void csd_lock_wait(struct __call_single_data *csd) { int bug_id = 0; u64 ts0, ts1; @@ -219,17 +219,17 @@ static __always_inline void csd_lock_wait(call_single_data_t *csd) } #else -static void csd_lock_record(call_single_data_t *csd) +static void csd_lock_record(struct __call_single_data *csd) { } -static __always_inline void csd_lock_wait(call_single_data_t *csd) +static __always_inline void csd_lock_wait(struct __call_single_data *csd) { smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK)); } #endif -static __always_inline void csd_lock(call_single_data_t *csd) +static __always_inline void csd_lock(struct __call_single_data *csd) { csd_lock_wait(csd); csd->node.u_flags |= CSD_FLAG_LOCK; @@ -242,7 +242,7 @@ static __always_inline void csd_lock(call_single_data_t *csd) smp_wmb(); } -static __always_inline void csd_unlock(call_single_data_t *csd) +static __always_inline void csd_unlock(struct __call_single_data *csd) { WARN_ON(!(csd->node.u_flags & CSD_FLAG_LOCK)); @@ -276,7 +276,7 @@ void __smp_call_single_queue(int cpu, struct llist_node *node) * for execution on the given CPU. data must already have * ->func, ->info, and ->flags set. */ -static int generic_exec_single(int cpu, call_single_data_t *csd) +static int generic_exec_single(int cpu, struct __call_single_data *csd) { if (cpu == smp_processor_id()) { smp_call_func_t func = csd->func; @@ -542,7 +542,7 @@ EXPORT_SYMBOL(smp_call_function_single); * NOTE: Be careful, there is unfortunately no current debugging facility to * validate the correctness of this serialization. */ -int smp_call_function_single_async(int cpu, call_single_data_t *csd) +int smp_call_function_single_async(int cpu, struct __call_single_data *csd) { int err = 0; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 62fbd09b5dc1..0b17fdb1b1ff 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -103,6 +103,9 @@ #ifdef CONFIG_LOCKUP_DETECTOR #include #endif +#ifdef CONFIG_USER_NS +#include +#endif #if defined(CONFIG_SYSCTL) @@ -111,6 +114,22 @@ static int sixty = 60; #endif +#if defined(CONFIG_UNEVICTABLE_FILE) +#if CONFIG_UNEVICTABLE_FILE_KBYTES_LOW < 0 +#error "CONFIG_UNEVICTABLE_FILE_KBYTES_LOW should be >= 0" +#endif +#if CONFIG_UNEVICTABLE_FILE_KBYTES_MIN < 0 +#error "CONFIG_UNEVICTABLE_FILE_KBYTES_MIN should be >= 0" +#endif +#if CONFIG_UNEVICTABLE_FILE_KBYTES_LOW < CONFIG_UNEVICTABLE_FILE_KBYTES_MIN +#error "CONFIG_UNEVICTABLE_FILE_KBYTES_LOW should be >= CONFIG_UNEVICTABLE_FILE_KBYTES_MIN" +#endif +unsigned long sysctl_unevictable_file_kbytes_low __read_mostly = + CONFIG_UNEVICTABLE_FILE_KBYTES_LOW; +unsigned long sysctl_unevictable_file_kbytes_min __read_mostly = + CONFIG_UNEVICTABLE_FILE_KBYTES_MIN; +#endif + static int __maybe_unused neg_one = -1; static int __maybe_unused two = 2; static int __maybe_unused four = 4; @@ -1902,6 +1921,15 @@ static struct ctl_table kern_table[] = { .proc_handler = proc_dointvec, }, #endif +#ifdef CONFIG_USER_NS + { + .procname = "unprivileged_userns_clone", + .data = &unprivileged_userns_clone, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +#endif #ifdef CONFIG_PROC_SYSCTL { .procname = "tainted", @@ -3092,6 +3120,25 @@ static struct ctl_table vm_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, +#endif +#if defined(CONFIG_UNEVICTABLE_FILE) + { + .procname = "unevictable_file_kbytes_low", + .data = &sysctl_unevictable_file_kbytes_low, + .maxlen = sizeof(sysctl_unevictable_file_kbytes_low), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + .extra1 = &sysctl_unevictable_file_kbytes_min, + }, + { + .procname = "unevictable_file_kbytes_min", + .data = &sysctl_unevictable_file_kbytes_min, + .maxlen = sizeof(sysctl_unevictable_file_kbytes_min), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + .extra1 = &zero_ul, + .extra2 = &sysctl_unevictable_file_kbytes_low, + }, #endif { .procname = "user_reserve_kbytes", diff --git a/kernel/time/alarmtimer.c b/kernel/time/alarmtimer.c index 4d94e2b5499d..a7924fedf479 100644 --- a/kernel/time/alarmtimer.c +++ b/kernel/time/alarmtimer.c @@ -92,7 +92,7 @@ static int alarmtimer_rtc_add_device(struct device *dev, if (rtcdev) return -EBUSY; - if (!rtc->ops->set_alarm) + if (!test_bit(RTC_FEATURE_ALARM, rtc->features)) return -1; if (!device_may_wakeup(rtc->dev.parent)) return -1; diff --git a/kernel/time/posix-timers.c b/kernel/time/posix-timers.c index bf540f5a4115..dd5697d7347b 100644 --- a/kernel/time/posix-timers.c +++ b/kernel/time/posix-timers.c @@ -1191,8 +1191,8 @@ SYSCALL_DEFINE2(clock_adjtime32, clockid_t, which_clock, err = do_clock_adjtime(which_clock, &ktx); - if (err >= 0) - err = put_old_timex32(utp, &ktx); + if (err >= 0 && put_old_timex32(utp, &ktx)) + return -EFAULT; return err; } diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 3ba52d4e1314..826b88b727a6 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -5631,7 +5631,10 @@ int ftrace_regex_release(struct inode *inode, struct file *file) parser = &iter->parser; if (trace_parser_loaded(parser)) { - ftrace_match_records(iter->hash, parser->buffer, parser->idx); + int enable = !(iter->flags & FTRACE_ITER_NOTRACE); + + ftrace_process_regex(iter, parser->buffer, + parser->idx, enable); } trace_parser_put(parser); diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index c0c9aa5cd8e2..67c01dc5cdeb 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -2390,14 +2390,13 @@ static void tracing_stop_tr(struct trace_array *tr) static int trace_save_cmdline(struct task_struct *tsk) { - unsigned pid, idx; + unsigned tpid, idx; /* treat recording of idle task as a success */ if (!tsk->pid) return 1; - if (unlikely(tsk->pid > PID_MAX_DEFAULT)) - return 0; + tpid = tsk->pid & (PID_MAX_DEFAULT - 1); /* * It's not the end of the world if we don't get @@ -2408,26 +2407,15 @@ static int trace_save_cmdline(struct task_struct *tsk) if (!arch_spin_trylock(&trace_cmdline_lock)) return 0; - idx = savedcmd->map_pid_to_cmdline[tsk->pid]; + idx = savedcmd->map_pid_to_cmdline[tpid]; if (idx == NO_CMDLINE_MAP) { idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num; - /* - * Check whether the cmdline buffer at idx has a pid - * mapped. We are going to overwrite that entry so we - * need to clear the map_pid_to_cmdline. Otherwise we - * would read the new comm for the old pid. - */ - pid = savedcmd->map_cmdline_to_pid[idx]; - if (pid != NO_CMDLINE_MAP) - savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP; - - savedcmd->map_cmdline_to_pid[idx] = tsk->pid; - savedcmd->map_pid_to_cmdline[tsk->pid] = idx; - + savedcmd->map_pid_to_cmdline[tpid] = idx; savedcmd->cmdline_idx = idx; } + savedcmd->map_cmdline_to_pid[idx] = tsk->pid; set_cmdline(idx, tsk->comm); arch_spin_unlock(&trace_cmdline_lock); @@ -2438,6 +2426,7 @@ static int trace_save_cmdline(struct task_struct *tsk) static void __trace_find_cmdline(int pid, char comm[]) { unsigned map; + int tpid; if (!pid) { strcpy(comm, ""); @@ -2449,16 +2438,16 @@ static void __trace_find_cmdline(int pid, char comm[]) return; } - if (pid > PID_MAX_DEFAULT) { - strcpy(comm, "<...>"); - return; + tpid = pid & (PID_MAX_DEFAULT - 1); + map = savedcmd->map_pid_to_cmdline[tpid]; + if (map != NO_CMDLINE_MAP) { + tpid = savedcmd->map_cmdline_to_pid[map]; + if (tpid == pid) { + strlcpy(comm, get_saved_cmdlines(map), TASK_COMM_LEN); + return; + } } - - map = savedcmd->map_pid_to_cmdline[pid]; - if (map != NO_CMDLINE_MAP) - strlcpy(comm, get_saved_cmdlines(map), TASK_COMM_LEN); - else - strcpy(comm, "<...>"); + strcpy(comm, "<...>"); } void trace_find_cmdline(int pid, char comm[]) diff --git a/kernel/trace/trace_clock.c b/kernel/trace/trace_clock.c index aaf6793ededa..c1637f90c8a3 100644 --- a/kernel/trace/trace_clock.c +++ b/kernel/trace/trace_clock.c @@ -95,33 +95,49 @@ u64 notrace trace_clock_global(void) { unsigned long flags; int this_cpu; - u64 now; + u64 now, prev_time; raw_local_irq_save(flags); this_cpu = raw_smp_processor_id(); - now = sched_clock_cpu(this_cpu); + /* - * If in an NMI context then dont risk lockups and return the - * cpu_clock() time: + * The global clock "guarantees" that the events are ordered + * between CPUs. But if two events on two different CPUS call + * trace_clock_global at roughly the same time, it really does + * not matter which one gets the earlier time. Just make sure + * that the same CPU will always show a monotonic clock. + * + * Use a read memory barrier to get the latest written + * time that was recorded. */ - if (unlikely(in_nmi())) - goto out; + smp_rmb(); + prev_time = READ_ONCE(trace_clock_struct.prev_time); + now = sched_clock_cpu(this_cpu); - arch_spin_lock(&trace_clock_struct.lock); + /* Make sure that now is always greater than prev_time */ + if ((s64)(now - prev_time) < 0) + now = prev_time + 1; /* - * TODO: if this happens often then maybe we should reset - * my_scd->clock to prev_time+1, to make sure - * we start ticking with the local clock from now on? + * If in an NMI context then dont risk lockups and simply return + * the current time. */ - if ((s64)(now - trace_clock_struct.prev_time) < 0) - now = trace_clock_struct.prev_time + 1; + if (unlikely(in_nmi())) + goto out; - trace_clock_struct.prev_time = now; + /* Tracing can cause strange recursion, always use a try lock */ + if (arch_spin_trylock(&trace_clock_struct.lock)) { + /* Reread prev_time in case it was already updated */ + prev_time = READ_ONCE(trace_clock_struct.prev_time); + if ((s64)(now - prev_time) < 0) + now = prev_time + 1; - arch_spin_unlock(&trace_clock_struct.lock); + trace_clock_struct.prev_time = now; + /* The unlock acts as the wmb for the above rmb */ + arch_spin_unlock(&trace_clock_struct.lock); + } out: raw_local_irq_restore(flags); diff --git a/kernel/umh.c b/kernel/umh.c index 3f646613a9d3..61f6b82c354b 100644 --- a/kernel/umh.c +++ b/kernel/umh.c @@ -27,6 +27,7 @@ #include #include #include +#include #include @@ -107,6 +108,7 @@ static int call_usermodehelper_exec_async(void *data) commit_creds(new); + wait_for_initramfs(); retval = kernel_execve(sub_info->path, (const char *const *)sub_info->argv, (const char *const *)sub_info->envp); diff --git a/kernel/up.c b/kernel/up.c index c6f323dcd45b..4edd5493eba2 100644 --- a/kernel/up.c +++ b/kernel/up.c @@ -25,7 +25,7 @@ int smp_call_function_single(int cpu, void (*func) (void *info), void *info, } EXPORT_SYMBOL(smp_call_function_single); -int smp_call_function_single_async(int cpu, call_single_data_t *csd) +int smp_call_function_single_async(int cpu, struct __call_single_data *csd) { unsigned long flags; diff --git a/kernel/user_namespace.c b/kernel/user_namespace.c index 9a4b980d695b..4388ca13ea3f 100644 --- a/kernel/user_namespace.c +++ b/kernel/user_namespace.c @@ -21,6 +21,13 @@ #include #include +/* sysctl */ +#ifdef CONFIG_USER_NS_UNPRIVILEGED +int unprivileged_userns_clone = 1; +#else +int unprivileged_userns_clone; +#endif + static struct kmem_cache *user_ns_cachep __read_mostly; static DEFINE_MUTEX(userns_state_mutex); diff --git a/kernel/watchdog.c b/kernel/watchdog.c index 107bc38b1945..8cf0678378d2 100644 --- a/kernel/watchdog.c +++ b/kernel/watchdog.c @@ -154,7 +154,11 @@ static void lockup_detector_update_enable(void) #ifdef CONFIG_SOFTLOCKUP_DETECTOR -#define SOFTLOCKUP_RESET ULONG_MAX +/* + * Delay the soflockup report when running a known slow code. + * It does _not_ affect the timestamp of the last successdul reschedule. + */ +#define SOFTLOCKUP_DELAY_REPORT ULONG_MAX #ifdef CONFIG_SMP int __read_mostly sysctl_softlockup_all_cpu_backtrace; @@ -169,10 +173,12 @@ unsigned int __read_mostly softlockup_panic = static bool softlockup_initialized __read_mostly; static u64 __read_mostly sample_period; +/* Timestamp taken after the last successful reschedule. */ static DEFINE_PER_CPU(unsigned long, watchdog_touch_ts); +/* Timestamp of the last softlockup report. */ +static DEFINE_PER_CPU(unsigned long, watchdog_report_ts); static DEFINE_PER_CPU(struct hrtimer, watchdog_hrtimer); static DEFINE_PER_CPU(bool, softlockup_touch_sync); -static DEFINE_PER_CPU(bool, soft_watchdog_warn); static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts); static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts_saved); static unsigned long soft_lockup_nmi_warn; @@ -235,10 +241,16 @@ static void set_sample_period(void) watchdog_update_hrtimer_threshold(sample_period); } +static void update_report_ts(void) +{ + __this_cpu_write(watchdog_report_ts, get_timestamp()); +} + /* Commands for resetting the watchdog */ -static void __touch_watchdog(void) +static void update_touch_ts(void) { __this_cpu_write(watchdog_touch_ts, get_timestamp()); + update_report_ts(); } /** @@ -252,10 +264,10 @@ static void __touch_watchdog(void) notrace void touch_softlockup_watchdog_sched(void) { /* - * Preemption can be enabled. It doesn't matter which CPU's timestamp - * gets zeroed here, so use the raw_ operation. + * Preemption can be enabled. It doesn't matter which CPU's watchdog + * report period gets restarted here, so use the raw_ operation. */ - raw_cpu_write(watchdog_touch_ts, SOFTLOCKUP_RESET); + raw_cpu_write(watchdog_report_ts, SOFTLOCKUP_DELAY_REPORT); } notrace void touch_softlockup_watchdog(void) @@ -279,7 +291,7 @@ void touch_all_softlockup_watchdogs(void) * the softlockup check. */ for_each_cpu(cpu, &watchdog_allowed_mask) { - per_cpu(watchdog_touch_ts, cpu) = SOFTLOCKUP_RESET; + per_cpu(watchdog_report_ts, cpu) = SOFTLOCKUP_DELAY_REPORT; wq_watchdog_touch(cpu); } } @@ -287,16 +299,16 @@ void touch_all_softlockup_watchdogs(void) void touch_softlockup_watchdog_sync(void) { __this_cpu_write(softlockup_touch_sync, true); - __this_cpu_write(watchdog_touch_ts, SOFTLOCKUP_RESET); + __this_cpu_write(watchdog_report_ts, SOFTLOCKUP_DELAY_REPORT); } -static int is_softlockup(unsigned long touch_ts) +static int is_softlockup(unsigned long touch_ts, unsigned long period_ts) { unsigned long now = get_timestamp(); if ((watchdog_enabled & SOFT_WATCHDOG_ENABLED) && watchdog_thresh){ /* Warn about unreasonable delays. */ - if (time_after(now, touch_ts + get_softlockup_thresh())) + if (time_after(now, period_ts + get_softlockup_thresh())) return now - touch_ts; } return 0; @@ -332,7 +344,7 @@ static DEFINE_PER_CPU(struct cpu_stop_work, softlockup_stop_work); */ static int softlockup_fn(void *data) { - __touch_watchdog(); + update_touch_ts(); complete(this_cpu_ptr(&softlockup_completion)); return 0; @@ -342,6 +354,7 @@ static int softlockup_fn(void *data) static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) { unsigned long touch_ts = __this_cpu_read(watchdog_touch_ts); + unsigned long period_ts = __this_cpu_read(watchdog_report_ts); struct pt_regs *regs = get_irq_regs(); int duration; int softlockup_all_cpu_backtrace = sysctl_softlockup_all_cpu_backtrace; @@ -363,7 +376,8 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) /* .. and repeat */ hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period)); - if (touch_ts == SOFTLOCKUP_RESET) { + /* Reset the interval when touched externally by a known slow code. */ + if (period_ts == SOFTLOCKUP_DELAY_REPORT) { if (unlikely(__this_cpu_read(softlockup_touch_sync))) { /* * If the time stamp was touched atomically @@ -375,7 +389,8 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) /* Clear the guest paused flag on watchdog reset */ kvm_check_and_clear_guest_paused(); - __touch_watchdog(); + update_report_ts(); + return HRTIMER_RESTART; } @@ -385,7 +400,7 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) * indicate it is getting cpu time. If it hasn't then * this is a good indication some task is hogging the cpu */ - duration = is_softlockup(touch_ts); + duration = is_softlockup(touch_ts, period_ts); if (unlikely(duration)) { /* * If a virtual machine is stopped by the host it can look to @@ -395,21 +410,18 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) if (kvm_check_and_clear_guest_paused()) return HRTIMER_RESTART; - /* only warn once */ - if (__this_cpu_read(soft_watchdog_warn) == true) - return HRTIMER_RESTART; - + /* + * Prevent multiple soft-lockup reports if one cpu is already + * engaged in dumping all cpu back traces. + */ if (softlockup_all_cpu_backtrace) { - /* Prevent multiple soft-lockup reports if one cpu is already - * engaged in dumping cpu back traces - */ - if (test_and_set_bit(0, &soft_lockup_nmi_warn)) { - /* Someone else will report us. Let's give up */ - __this_cpu_write(soft_watchdog_warn, true); + if (test_and_set_bit_lock(0, &soft_lockup_nmi_warn)) return HRTIMER_RESTART; - } } + /* Start period for the next softlockup warning. */ + update_report_ts(); + pr_emerg("BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n", smp_processor_id(), duration, current->comm, task_pid_nr(current)); @@ -421,22 +433,14 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) dump_stack(); if (softlockup_all_cpu_backtrace) { - /* Avoid generating two back traces for current - * given that one is already made above - */ trigger_allbutself_cpu_backtrace(); - - clear_bit(0, &soft_lockup_nmi_warn); - /* Barrier to sync with other cpus */ - smp_mb__after_atomic(); + clear_bit_unlock(0, &soft_lockup_nmi_warn); } add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK); if (softlockup_panic) panic("softlockup: hung tasks"); - __this_cpu_write(soft_watchdog_warn, true); - } else - __this_cpu_write(soft_watchdog_warn, false); + } return HRTIMER_RESTART; } @@ -461,7 +465,7 @@ static void watchdog_enable(unsigned int cpu) HRTIMER_MODE_REL_PINNED_HARD); /* Initialize timestamp */ - __touch_watchdog(); + update_touch_ts(); /* Enable the perf event */ if (watchdog_enabled & NMI_WATCHDOG_ENABLED) watchdog_nmi_enable(cpu); diff --git a/lib/Kconfig.kfence b/lib/Kconfig.kfence index 78f50ccb3b45..e641add33947 100644 --- a/lib/Kconfig.kfence +++ b/lib/Kconfig.kfence @@ -7,6 +7,7 @@ menuconfig KFENCE bool "KFENCE: low-overhead sampling-based memory safety error detector" depends on HAVE_ARCH_KFENCE && (SLAB || SLUB) select STACKTRACE + select IRQ_WORK help KFENCE is a low-overhead sampling-based detector of heap out-of-bounds access, use-after-free, and invalid-free errors. KFENCE is designed diff --git a/lib/bug.c b/lib/bug.c index 8f9d537bfb2a..b92da1f6e21b 100644 --- a/lib/bug.c +++ b/lib/bug.c @@ -155,30 +155,27 @@ enum bug_trap_type report_bug(unsigned long bugaddr, struct pt_regs *regs) file = NULL; line = 0; - warning = 0; - if (bug) { #ifdef CONFIG_DEBUG_BUGVERBOSE #ifndef CONFIG_GENERIC_BUG_RELATIVE_POINTERS - file = bug->file; + file = bug->file; #else - file = (const char *)bug + bug->file_disp; + file = (const char *)bug + bug->file_disp; #endif - line = bug->line; + line = bug->line; #endif - warning = (bug->flags & BUGFLAG_WARNING) != 0; - once = (bug->flags & BUGFLAG_ONCE) != 0; - done = (bug->flags & BUGFLAG_DONE) != 0; - - if (warning && once) { - if (done) - return BUG_TRAP_TYPE_WARN; - - /* - * Since this is the only store, concurrency is not an issue. - */ - bug->flags |= BUGFLAG_DONE; - } + warning = (bug->flags & BUGFLAG_WARNING) != 0; + once = (bug->flags & BUGFLAG_ONCE) != 0; + done = (bug->flags & BUGFLAG_DONE) != 0; + + if (warning && once) { + if (done) + return BUG_TRAP_TYPE_WARN; + + /* + * Since this is the only store, concurrency is not an issue. + */ + bug->flags |= BUGFLAG_DONE; } /* diff --git a/lib/crypto/poly1305-donna32.c b/lib/crypto/poly1305-donna32.c index 3cc77d94390b..7fb71845cc84 100644 --- a/lib/crypto/poly1305-donna32.c +++ b/lib/crypto/poly1305-donna32.c @@ -10,7 +10,8 @@ #include #include -void poly1305_core_setkey(struct poly1305_core_key *key, const u8 raw_key[16]) +void poly1305_core_setkey(struct poly1305_core_key *key, + const u8 raw_key[POLY1305_BLOCK_SIZE]) { /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ key->key.r[0] = (get_unaligned_le32(&raw_key[0])) & 0x3ffffff; diff --git a/lib/crypto/poly1305-donna64.c b/lib/crypto/poly1305-donna64.c index 6ae181bb4345..d34cf4053668 100644 --- a/lib/crypto/poly1305-donna64.c +++ b/lib/crypto/poly1305-donna64.c @@ -12,7 +12,8 @@ typedef __uint128_t u128; -void poly1305_core_setkey(struct poly1305_core_key *key, const u8 raw_key[16]) +void poly1305_core_setkey(struct poly1305_core_key *key, + const u8 raw_key[POLY1305_BLOCK_SIZE]) { u64 t0, t1; diff --git a/lib/crypto/poly1305.c b/lib/crypto/poly1305.c index 9d2d14df0fee..26d87fc3823e 100644 --- a/lib/crypto/poly1305.c +++ b/lib/crypto/poly1305.c @@ -12,7 +12,8 @@ #include #include -void poly1305_init_generic(struct poly1305_desc_ctx *desc, const u8 *key) +void poly1305_init_generic(struct poly1305_desc_ctx *desc, + const u8 key[POLY1305_KEY_SIZE]) { poly1305_core_setkey(&desc->core_r, key); desc->s[0] = get_unaligned_le32(key + 16); diff --git a/lib/decompress_unzstd.c b/lib/decompress_unzstd.c index 790abc472f5b..6e5ecfba0a8d 100644 --- a/lib/decompress_unzstd.c +++ b/lib/decompress_unzstd.c @@ -68,11 +68,7 @@ #ifdef STATIC # define UNZSTD_PREBOOT # include "xxhash.c" -# include "zstd/entropy_common.c" -# include "zstd/fse_decompress.c" -# include "zstd/huf_decompress.c" -# include "zstd/zstd_common.c" -# include "zstd/decompress.c" +# include "zstd/decompress_sources.h" #endif #include @@ -91,11 +87,15 @@ static int INIT handle_zstd_error(size_t ret, void (*error)(char *x)) { - const int err = ZSTD_getErrorCode(ret); + const zstd_error_code err = zstd_get_error_code(ret); - if (!ZSTD_isError(ret)) + if (!zstd_is_error(ret)) return 0; + /* + * zstd_get_error_name() cannot be used because error takes a char * + * not a const char * + */ switch (err) { case ZSTD_error_memory_allocation: error("ZSTD decompressor ran out of memory"); @@ -124,28 +124,28 @@ static int INIT decompress_single(const u8 *in_buf, long in_len, u8 *out_buf, long out_len, long *in_pos, void (*error)(char *x)) { - const size_t wksp_size = ZSTD_DCtxWorkspaceBound(); + const size_t wksp_size = zstd_dctx_workspace_bound(); void *wksp = large_malloc(wksp_size); - ZSTD_DCtx *dctx = ZSTD_initDCtx(wksp, wksp_size); + zstd_dctx *dctx = zstd_init_dctx(wksp, wksp_size); int err; size_t ret; if (dctx == NULL) { - error("Out of memory while allocating ZSTD_DCtx"); + error("Out of memory while allocating zstd_dctx"); err = -1; goto out; } /* * Find out how large the frame actually is, there may be junk at - * the end of the frame that ZSTD_decompressDCtx() can't handle. + * the end of the frame that zstd_decompress_dctx() can't handle. */ - ret = ZSTD_findFrameCompressedSize(in_buf, in_len); + ret = zstd_find_frame_compressed_size(in_buf, in_len); err = handle_zstd_error(ret, error); if (err) goto out; in_len = (long)ret; - ret = ZSTD_decompressDCtx(dctx, out_buf, out_len, in_buf, in_len); + ret = zstd_decompress_dctx(dctx, out_buf, out_len, in_buf, in_len); err = handle_zstd_error(ret, error); if (err) goto out; @@ -167,14 +167,14 @@ static int INIT __unzstd(unsigned char *in_buf, long in_len, long *in_pos, void (*error)(char *x)) { - ZSTD_inBuffer in; - ZSTD_outBuffer out; - ZSTD_frameParams params; + zstd_in_buffer in; + zstd_out_buffer out; + zstd_frame_header header; void *in_allocated = NULL; void *out_allocated = NULL; void *wksp = NULL; size_t wksp_size; - ZSTD_DStream *dstream; + zstd_dstream *dstream; int err; size_t ret; @@ -238,13 +238,13 @@ static int INIT __unzstd(unsigned char *in_buf, long in_len, out.size = out_len; /* - * We need to know the window size to allocate the ZSTD_DStream. + * We need to know the window size to allocate the zstd_dstream. * Since we are streaming, we need to allocate a buffer for the sliding * window. The window size varies from 1 KB to ZSTD_WINDOWSIZE_MAX * (8 MB), so it is important to use the actual value so as not to * waste memory when it is smaller. */ - ret = ZSTD_getFrameParams(¶ms, in.src, in.size); + ret = zstd_get_frame_header(&header, in.src, in.size); err = handle_zstd_error(ret, error); if (err) goto out; @@ -253,19 +253,19 @@ static int INIT __unzstd(unsigned char *in_buf, long in_len, err = -1; goto out; } - if (params.windowSize > ZSTD_WINDOWSIZE_MAX) { + if (header.windowSize > ZSTD_WINDOWSIZE_MAX) { error("ZSTD-compressed data has too large a window size"); err = -1; goto out; } /* - * Allocate the ZSTD_DStream now that we know how much memory is + * Allocate the zstd_dstream now that we know how much memory is * required. */ - wksp_size = ZSTD_DStreamWorkspaceBound(params.windowSize); + wksp_size = zstd_dstream_workspace_bound(header.windowSize); wksp = large_malloc(wksp_size); - dstream = ZSTD_initDStream(params.windowSize, wksp, wksp_size); + dstream = zstd_init_dstream(header.windowSize, wksp, wksp_size); if (dstream == NULL) { error("Out of memory while allocating ZSTD_DStream"); err = -1; @@ -298,7 +298,7 @@ static int INIT __unzstd(unsigned char *in_buf, long in_len, in.size = in_len; } /* Returns zero when the frame is complete. */ - ret = ZSTD_decompressStream(dstream, &out, &in); + ret = zstd_decompress_stream(dstream, &out, &in); err = handle_zstd_error(ret, error); if (err) goto out; diff --git a/lib/dynamic_debug.c b/lib/dynamic_debug.c index c70d6347afa2..921d0a654243 100644 --- a/lib/dynamic_debug.c +++ b/lib/dynamic_debug.c @@ -396,7 +396,7 @@ static int ddebug_parse_query(char *words[], int nwords, /* tail :$info is function or line-range */ fline = strchr(query->filename, ':'); if (!fline) - break; + continue; *fline++ = '\0'; if (isalpha(*fline) || *fline == '*' || *fline == '?') { /* take as function name */ diff --git a/lib/kobject_uevent.c b/lib/kobject_uevent.c index 7998affa45d4..c87d5b6a8a55 100644 --- a/lib/kobject_uevent.c +++ b/lib/kobject_uevent.c @@ -251,12 +251,13 @@ static int kobj_usermode_filter(struct kobject *kobj) static int init_uevent_argv(struct kobj_uevent_env *env, const char *subsystem) { + int buffer_size = sizeof(env->buf) - env->buflen; int len; - len = strlcpy(&env->buf[env->buflen], subsystem, - sizeof(env->buf) - env->buflen); - if (len >= (sizeof(env->buf) - env->buflen)) { - WARN(1, KERN_ERR "init_uevent_argv: buffer size too small\n"); + len = strlcpy(&env->buf[env->buflen], subsystem, buffer_size); + if (len >= buffer_size) { + pr_warn("init_uevent_argv: buffer size of %d too small, needed %d\n", + buffer_size, len); return -ENOMEM; } diff --git a/lib/nlattr.c b/lib/nlattr.c index 5b6116e81f9f..1d051ef66afe 100644 --- a/lib/nlattr.c +++ b/lib/nlattr.c @@ -828,7 +828,7 @@ int nla_strcmp(const struct nlattr *nla, const char *str) int attrlen = nla_len(nla); int d; - if (attrlen > 0 && buf[attrlen - 1] == '\0') + while (attrlen > 0 && buf[attrlen - 1] == '\0') attrlen--; d = attrlen - len; diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 49f67a0c6e5d..df9179f4f441 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -71,7 +71,7 @@ static void *stack_slabs[STACK_ALLOC_MAX_SLABS]; static int depot_index; static int next_slab_inited; static size_t depot_offset; -static DEFINE_SPINLOCK(depot_lock); +static DEFINE_RAW_SPINLOCK(depot_lock); static bool init_stack_slab(void **prealloc) { @@ -305,7 +305,7 @@ depot_stack_handle_t stack_depot_save(unsigned long *entries, prealloc = page_address(page); } - spin_lock_irqsave(&depot_lock, flags); + raw_spin_lock_irqsave(&depot_lock, flags); found = find_stack(*bucket, entries, nr_entries, hash); if (!found) { @@ -329,7 +329,7 @@ depot_stack_handle_t stack_depot_save(unsigned long *entries, WARN_ON(!init_stack_slab(&prealloc)); } - spin_unlock_irqrestore(&depot_lock, flags); + raw_spin_unlock_irqrestore(&depot_lock, flags); exit: if (prealloc) { /* Nobody used this memory, ok to free it. */ diff --git a/lib/test_kasan.c b/lib/test_kasan.c index e5647d147b35..be69c3aa615a 100644 --- a/lib/test_kasan.c +++ b/lib/test_kasan.c @@ -646,8 +646,20 @@ static char global_array[10]; static void kasan_global_oob(struct kunit *test) { - volatile int i = 3; - char *p = &global_array[ARRAY_SIZE(global_array) + i]; + /* + * Deliberate out-of-bounds access. To prevent CONFIG_UBSAN_LOCAL_BOUNDS + * from failing here and panicing the kernel, access the array via a + * volatile pointer, which will prevent the compiler from being able to + * determine the array bounds. + * + * This access uses a volatile pointer to char (char *volatile) rather + * than the more conventional pointer to volatile char (volatile char *) + * because we want to prevent the compiler from making inferences about + * the pointer itself (i.e. its array bounds), not the data that it + * refers to. + */ + char *volatile array = global_array; + char *p = &array[ARRAY_SIZE(global_array) + 3]; /* Only generic mode instruments globals. */ KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_GENERIC); @@ -695,8 +707,9 @@ static void ksize_uaf(struct kunit *test) static void kasan_stack_oob(struct kunit *test) { char stack_array[10]; - volatile int i = OOB_TAG_OFF; - char *p = &stack_array[ARRAY_SIZE(stack_array) + i]; + /* See comment in kasan_global_oob. */ + char *volatile array = stack_array; + char *p = &array[ARRAY_SIZE(stack_array) + OOB_TAG_OFF]; KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_STACK); @@ -707,7 +720,9 @@ static void kasan_alloca_oob_left(struct kunit *test) { volatile int i = 10; char alloca_array[i]; - char *p = alloca_array - 1; + /* See comment in kasan_global_oob. */ + char *volatile array = alloca_array; + char *p = array - 1; /* Only generic mode instruments dynamic allocas. */ KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_GENERIC); @@ -720,7 +735,9 @@ static void kasan_alloca_oob_right(struct kunit *test) { volatile int i = 10; char alloca_array[i]; - char *p = alloca_array + i; + /* See comment in kasan_global_oob. */ + char *volatile array = alloca_array; + char *p = array + i; /* Only generic mode instruments dynamic allocas. */ KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_GENERIC); diff --git a/lib/vsprintf.c b/lib/vsprintf.c index 41ddc353ebb8..39ef2e314da5 100644 --- a/lib/vsprintf.c +++ b/lib/vsprintf.c @@ -3135,8 +3135,6 @@ int bstr_printf(char *buf, size_t size, const char *fmt, const u32 *bin_buf) switch (*fmt) { case 'S': case 's': - case 'F': - case 'f': case 'x': case 'K': case 'e': diff --git a/lib/zstd/Makefile b/lib/zstd/Makefile index f5d778e7e5c7..19485e3cc7c9 100644 --- a/lib/zstd/Makefile +++ b/lib/zstd/Makefile @@ -1,10 +1,46 @@ # SPDX-License-Identifier: GPL-2.0-only +# ################################################################ +# Copyright (c) Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# You may select, at your option, one of the above-listed licenses. +# ################################################################ obj-$(CONFIG_ZSTD_COMPRESS) += zstd_compress.o obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd_decompress.o ccflags-y += -O3 -zstd_compress-y := fse_compress.o huf_compress.o compress.o \ - entropy_common.o fse_decompress.o zstd_common.o -zstd_decompress-y := huf_decompress.o decompress.o \ - entropy_common.o fse_decompress.o zstd_common.o +zstd_compress-y := \ + zstd_compress_module.o \ + common/debug.o \ + common/entropy_common.o \ + common/error_private.o \ + common/fse_decompress.o \ + common/zstd_common.o \ + compress/fse_compress.o \ + compress/hist.o \ + compress/huf_compress.o \ + compress/zstd_compress.o \ + compress/zstd_compress_literals.o \ + compress/zstd_compress_sequences.o \ + compress/zstd_compress_superblock.o \ + compress/zstd_double_fast.o \ + compress/zstd_fast.o \ + compress/zstd_lazy.o \ + compress/zstd_ldm.o \ + compress/zstd_opt.o \ + +zstd_decompress-y := \ + zstd_decompress_module.o \ + common/debug.o \ + common/entropy_common.o \ + common/error_private.o \ + common/fse_decompress.o \ + common/zstd_common.o \ + decompress/huf_decompress.o \ + decompress/zstd_ddict.o \ + decompress/zstd_decompress.o \ + decompress/zstd_decompress_block.o \ diff --git a/lib/zstd/bitstream.h b/lib/zstd/bitstream.h deleted file mode 100644 index 5d6343c1a909..000000000000 --- a/lib/zstd/bitstream.h +++ /dev/null @@ -1,380 +0,0 @@ -/* - * bitstream - * Part of FSE library - * header file (to include) - * Copyright (C) 2013-2016, Yann Collet. - * - * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - * - * You can contact the author at : - * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy - */ -#ifndef BITSTREAM_H_MODULE -#define BITSTREAM_H_MODULE - -/* -* This API consists of small unitary functions, which must be inlined for best performance. -* Since link-time-optimization is not available for all compilers, -* these functions are defined into a .h to be included. -*/ - -/*-**************************************** -* Dependencies -******************************************/ -#include "error_private.h" /* error codes and messages */ -#include "mem.h" /* unaligned access routines */ - -/*========================================= -* Target specific -=========================================*/ -#define STREAM_ACCUMULATOR_MIN_32 25 -#define STREAM_ACCUMULATOR_MIN_64 57 -#define STREAM_ACCUMULATOR_MIN ((U32)(ZSTD_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64)) - -/*-****************************************** -* bitStream encoding API (write forward) -********************************************/ -/* bitStream can mix input from multiple sources. -* A critical property of these streams is that they encode and decode in **reverse** direction. -* So the first bit sequence you add will be the last to be read, like a LIFO stack. -*/ -typedef struct { - size_t bitContainer; - int bitPos; - char *startPtr; - char *ptr; - char *endPtr; -} BIT_CStream_t; - -ZSTD_STATIC size_t BIT_initCStream(BIT_CStream_t *bitC, void *dstBuffer, size_t dstCapacity); -ZSTD_STATIC void BIT_addBits(BIT_CStream_t *bitC, size_t value, unsigned nbBits); -ZSTD_STATIC void BIT_flushBits(BIT_CStream_t *bitC); -ZSTD_STATIC size_t BIT_closeCStream(BIT_CStream_t *bitC); - -/* Start with initCStream, providing the size of buffer to write into. -* bitStream will never write outside of this buffer. -* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code. -* -* bits are first added to a local register. -* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. -* Writing data into memory is an explicit operation, performed by the flushBits function. -* Hence keep track how many bits are potentially stored into local register to avoid register overflow. -* After a flushBits, a maximum of 7 bits might still be stored into local register. -* -* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers. -* -* Last operation is to close the bitStream. -* The function returns the final size of CStream in bytes. -* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable) -*/ - -/*-******************************************** -* bitStream decoding API (read backward) -**********************************************/ -typedef struct { - size_t bitContainer; - unsigned bitsConsumed; - const char *ptr; - const char *start; -} BIT_DStream_t; - -typedef enum { - BIT_DStream_unfinished = 0, - BIT_DStream_endOfBuffer = 1, - BIT_DStream_completed = 2, - BIT_DStream_overflow = 3 -} BIT_DStream_status; /* result of BIT_reloadDStream() */ -/* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ - -ZSTD_STATIC size_t BIT_initDStream(BIT_DStream_t *bitD, const void *srcBuffer, size_t srcSize); -ZSTD_STATIC size_t BIT_readBits(BIT_DStream_t *bitD, unsigned nbBits); -ZSTD_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t *bitD); -ZSTD_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t *bitD); - -/* Start by invoking BIT_initDStream(). -* A chunk of the bitStream is then stored into a local register. -* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). -* You can then retrieve bitFields stored into the local register, **in reverse order**. -* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. -* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. -* Otherwise, it can be less than that, so proceed accordingly. -* Checking if DStream has reached its end can be performed with BIT_endOfDStream(). -*/ - -/*-**************************************** -* unsafe API -******************************************/ -ZSTD_STATIC void BIT_addBitsFast(BIT_CStream_t *bitC, size_t value, unsigned nbBits); -/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */ - -ZSTD_STATIC void BIT_flushBitsFast(BIT_CStream_t *bitC); -/* unsafe version; does not check buffer overflow */ - -ZSTD_STATIC size_t BIT_readBitsFast(BIT_DStream_t *bitD, unsigned nbBits); -/* faster, but works only if nbBits >= 1 */ - -/*-************************************************************** -* Internal functions -****************************************************************/ -ZSTD_STATIC unsigned BIT_highbit32(register U32 val) { return 31 - __builtin_clz(val); } - -/*===== Local Constants =====*/ -static const unsigned BIT_mask[] = {0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, - 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, - 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF}; /* up to 26 bits */ - -/*-************************************************************** -* bitStream encoding -****************************************************************/ -/*! BIT_initCStream() : - * `dstCapacity` must be > sizeof(void*) - * @return : 0 if success, - otherwise an error code (can be tested using ERR_isError() ) */ -ZSTD_STATIC size_t BIT_initCStream(BIT_CStream_t *bitC, void *startPtr, size_t dstCapacity) -{ - bitC->bitContainer = 0; - bitC->bitPos = 0; - bitC->startPtr = (char *)startPtr; - bitC->ptr = bitC->startPtr; - bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->ptr); - if (dstCapacity <= sizeof(bitC->ptr)) - return ERROR(dstSize_tooSmall); - return 0; -} - -/*! BIT_addBits() : - can add up to 26 bits into `bitC`. - Does not check for register overflow ! */ -ZSTD_STATIC void BIT_addBits(BIT_CStream_t *bitC, size_t value, unsigned nbBits) -{ - bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos; - bitC->bitPos += nbBits; -} - -/*! BIT_addBitsFast() : - * works only if `value` is _clean_, meaning all high bits above nbBits are 0 */ -ZSTD_STATIC void BIT_addBitsFast(BIT_CStream_t *bitC, size_t value, unsigned nbBits) -{ - bitC->bitContainer |= value << bitC->bitPos; - bitC->bitPos += nbBits; -} - -/*! BIT_flushBitsFast() : - * unsafe version; does not check buffer overflow */ -ZSTD_STATIC void BIT_flushBitsFast(BIT_CStream_t *bitC) -{ - size_t const nbBytes = bitC->bitPos >> 3; - ZSTD_writeLEST(bitC->ptr, bitC->bitContainer); - bitC->ptr += nbBytes; - bitC->bitPos &= 7; - bitC->bitContainer >>= nbBytes * 8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */ -} - -/*! BIT_flushBits() : - * safe version; check for buffer overflow, and prevents it. - * note : does not signal buffer overflow. This will be revealed later on using BIT_closeCStream() */ -ZSTD_STATIC void BIT_flushBits(BIT_CStream_t *bitC) -{ - size_t const nbBytes = bitC->bitPos >> 3; - ZSTD_writeLEST(bitC->ptr, bitC->bitContainer); - bitC->ptr += nbBytes; - if (bitC->ptr > bitC->endPtr) - bitC->ptr = bitC->endPtr; - bitC->bitPos &= 7; - bitC->bitContainer >>= nbBytes * 8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */ -} - -/*! BIT_closeCStream() : - * @return : size of CStream, in bytes, - or 0 if it could not fit into dstBuffer */ -ZSTD_STATIC size_t BIT_closeCStream(BIT_CStream_t *bitC) -{ - BIT_addBitsFast(bitC, 1, 1); /* endMark */ - BIT_flushBits(bitC); - - if (bitC->ptr >= bitC->endPtr) - return 0; /* doesn't fit within authorized budget : cancel */ - - return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); -} - -/*-******************************************************** -* bitStream decoding -**********************************************************/ -/*! BIT_initDStream() : -* Initialize a BIT_DStream_t. -* `bitD` : a pointer to an already allocated BIT_DStream_t structure. -* `srcSize` must be the *exact* size of the bitStream, in bytes. -* @return : size of stream (== srcSize) or an errorCode if a problem is detected -*/ -ZSTD_STATIC size_t BIT_initDStream(BIT_DStream_t *bitD, const void *srcBuffer, size_t srcSize) -{ - if (srcSize < 1) { - memset(bitD, 0, sizeof(*bitD)); - return ERROR(srcSize_wrong); - } - - if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ - bitD->start = (const char *)srcBuffer; - bitD->ptr = (const char *)srcBuffer + srcSize - sizeof(bitD->bitContainer); - bitD->bitContainer = ZSTD_readLEST(bitD->ptr); - { - BYTE const lastByte = ((const BYTE *)srcBuffer)[srcSize - 1]; - bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ - if (lastByte == 0) - return ERROR(GENERIC); /* endMark not present */ - } - } else { - bitD->start = (const char *)srcBuffer; - bitD->ptr = bitD->start; - bitD->bitContainer = *(const BYTE *)(bitD->start); - switch (srcSize) { - case 7: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[6]) << (sizeof(bitD->bitContainer) * 8 - 16); - fallthrough; - case 6: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[5]) << (sizeof(bitD->bitContainer) * 8 - 24); - fallthrough; - case 5: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[4]) << (sizeof(bitD->bitContainer) * 8 - 32); - fallthrough; - case 4: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[3]) << 24; - fallthrough; - case 3: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[2]) << 16; - fallthrough; - case 2: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[1]) << 8; - fallthrough; - default:; - } - { - BYTE const lastByte = ((const BYTE *)srcBuffer)[srcSize - 1]; - bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; - if (lastByte == 0) - return ERROR(GENERIC); /* endMark not present */ - } - bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize) * 8; - } - - return srcSize; -} - -ZSTD_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start) { return bitContainer >> start; } - -ZSTD_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) { return (bitContainer >> start) & BIT_mask[nbBits]; } - -ZSTD_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) { return bitContainer & BIT_mask[nbBits]; } - -/*! BIT_lookBits() : - * Provides next n bits from local register. - * local register is not modified. - * On 32-bits, maxNbBits==24. - * On 64-bits, maxNbBits==56. - * @return : value extracted - */ -ZSTD_STATIC size_t BIT_lookBits(const BIT_DStream_t *bitD, U32 nbBits) -{ - U32 const bitMask = sizeof(bitD->bitContainer) * 8 - 1; - return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask - nbBits) & bitMask); -} - -/*! BIT_lookBitsFast() : -* unsafe version; only works only if nbBits >= 1 */ -ZSTD_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t *bitD, U32 nbBits) -{ - U32 const bitMask = sizeof(bitD->bitContainer) * 8 - 1; - return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask + 1) - nbBits) & bitMask); -} - -ZSTD_STATIC void BIT_skipBits(BIT_DStream_t *bitD, U32 nbBits) { bitD->bitsConsumed += nbBits; } - -/*! BIT_readBits() : - * Read (consume) next n bits from local register and update. - * Pay attention to not read more than nbBits contained into local register. - * @return : extracted value. - */ -ZSTD_STATIC size_t BIT_readBits(BIT_DStream_t *bitD, U32 nbBits) -{ - size_t const value = BIT_lookBits(bitD, nbBits); - BIT_skipBits(bitD, nbBits); - return value; -} - -/*! BIT_readBitsFast() : -* unsafe version; only works only if nbBits >= 1 */ -ZSTD_STATIC size_t BIT_readBitsFast(BIT_DStream_t *bitD, U32 nbBits) -{ - size_t const value = BIT_lookBitsFast(bitD, nbBits); - BIT_skipBits(bitD, nbBits); - return value; -} - -/*! BIT_reloadDStream() : -* Refill `bitD` from buffer previously set in BIT_initDStream() . -* This function is safe, it guarantees it will not read beyond src buffer. -* @return : status of `BIT_DStream_t` internal register. - if status == BIT_DStream_unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */ -ZSTD_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t *bitD) -{ - if (bitD->bitsConsumed > (sizeof(bitD->bitContainer) * 8)) /* should not happen => corruption detected */ - return BIT_DStream_overflow; - - if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) { - bitD->ptr -= bitD->bitsConsumed >> 3; - bitD->bitsConsumed &= 7; - bitD->bitContainer = ZSTD_readLEST(bitD->ptr); - return BIT_DStream_unfinished; - } - if (bitD->ptr == bitD->start) { - if (bitD->bitsConsumed < sizeof(bitD->bitContainer) * 8) - return BIT_DStream_endOfBuffer; - return BIT_DStream_completed; - } - { - U32 nbBytes = bitD->bitsConsumed >> 3; - BIT_DStream_status result = BIT_DStream_unfinished; - if (bitD->ptr - nbBytes < bitD->start) { - nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ - result = BIT_DStream_endOfBuffer; - } - bitD->ptr -= nbBytes; - bitD->bitsConsumed -= nbBytes * 8; - bitD->bitContainer = ZSTD_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */ - return result; - } -} - -/*! BIT_endOfDStream() : -* @return Tells if DStream has exactly reached its end (all bits consumed). -*/ -ZSTD_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t *DStream) -{ - return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer) * 8)); -} - -#endif /* BITSTREAM_H_MODULE */ diff --git a/lib/zstd/common/bitstream.h b/lib/zstd/common/bitstream.h new file mode 100644 index 000000000000..2d6c95b4f40c --- /dev/null +++ b/lib/zstd/common/bitstream.h @@ -0,0 +1,437 @@ +/* ****************************************************************** + * bitstream + * Part of FSE library + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ +#ifndef BITSTREAM_H_MODULE +#define BITSTREAM_H_MODULE + +/* +* This API consists of small unitary functions, which must be inlined for best performance. +* Since link-time-optimization is not available for all compilers, +* these functions are defined into a .h to be included. +*/ + +/*-**************************************** +* Dependencies +******************************************/ +#include "mem.h" /* unaligned access routines */ +#include "compiler.h" /* UNLIKELY() */ +#include "debug.h" /* assert(), DEBUGLOG(), RAWLOG() */ +#include "error_private.h" /* error codes and messages */ + + +/*========================================= +* Target specific +=========================================*/ + +#define STREAM_ACCUMULATOR_MIN_32 25 +#define STREAM_ACCUMULATOR_MIN_64 57 +#define STREAM_ACCUMULATOR_MIN ((U32)(MEM_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64)) + + +/*-****************************************** +* bitStream encoding API (write forward) +********************************************/ +/* bitStream can mix input from multiple sources. + * A critical property of these streams is that they encode and decode in **reverse** direction. + * So the first bit sequence you add will be the last to be read, like a LIFO stack. + */ +typedef struct { + size_t bitContainer; + unsigned bitPos; + char* startPtr; + char* ptr; + char* endPtr; +} BIT_CStream_t; + +MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity); +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC); +MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); + +/* Start with initCStream, providing the size of buffer to write into. +* bitStream will never write outside of this buffer. +* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code. +* +* bits are first added to a local register. +* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. +* Writing data into memory is an explicit operation, performed by the flushBits function. +* Hence keep track how many bits are potentially stored into local register to avoid register overflow. +* After a flushBits, a maximum of 7 bits might still be stored into local register. +* +* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers. +* +* Last operation is to close the bitStream. +* The function returns the final size of CStream in bytes. +* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable) +*/ + + +/*-******************************************** +* bitStream decoding API (read backward) +**********************************************/ +typedef struct { + size_t bitContainer; + unsigned bitsConsumed; + const char* ptr; + const char* start; + const char* limitPtr; +} BIT_DStream_t; + +typedef enum { BIT_DStream_unfinished = 0, + BIT_DStream_endOfBuffer = 1, + BIT_DStream_completed = 2, + BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */ + /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ + +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize); +MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); +MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD); +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); + + +/* Start by invoking BIT_initDStream(). +* A chunk of the bitStream is then stored into a local register. +* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). +* You can then retrieve bitFields stored into the local register, **in reverse order**. +* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. +* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. +* Otherwise, it can be less than that, so proceed accordingly. +* Checking if DStream has reached its end can be performed with BIT_endOfDStream(). +*/ + + +/*-**************************************** +* unsafe API +******************************************/ +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */ + +MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); +/* unsafe version; does not check buffer overflow */ + +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); +/* faster, but works only if nbBits >= 1 */ + + + +/*-************************************************************** +* Internal functions +****************************************************************/ +MEM_STATIC unsigned BIT_highbit32 (U32 val) +{ + assert(val != 0); + { +# if (__GNUC__ >= 3) /* Use GCC Intrinsic */ + return __builtin_clz (val) ^ 31; +# else /* Software version */ + static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, + 11, 14, 16, 18, 22, 25, 3, 30, + 8, 12, 20, 28, 15, 17, 24, 7, + 19, 27, 23, 6, 26, 5, 4, 31 }; + U32 v = val; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27]; +# endif + } +} + +/*===== Local Constants =====*/ +static const unsigned BIT_mask[] = { + 0, 1, 3, 7, 0xF, 0x1F, + 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, + 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, + 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, + 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, 0x7FFFFFF, 0xFFFFFFF, 0x1FFFFFFF, + 0x3FFFFFFF, 0x7FFFFFFF}; /* up to 31 bits */ +#define BIT_MASK_SIZE (sizeof(BIT_mask) / sizeof(BIT_mask[0])) + +/*-************************************************************** +* bitStream encoding +****************************************************************/ +/*! BIT_initCStream() : + * `dstCapacity` must be > sizeof(size_t) + * @return : 0 if success, + * otherwise an error code (can be tested using ERR_isError()) */ +MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, + void* startPtr, size_t dstCapacity) +{ + bitC->bitContainer = 0; + bitC->bitPos = 0; + bitC->startPtr = (char*)startPtr; + bitC->ptr = bitC->startPtr; + bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer); + if (dstCapacity <= sizeof(bitC->bitContainer)) return ERROR(dstSize_tooSmall); + return 0; +} + +/*! BIT_addBits() : + * can add up to 31 bits into `bitC`. + * Note : does not check for register overflow ! */ +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, + size_t value, unsigned nbBits) +{ + DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32); + assert(nbBits < BIT_MASK_SIZE); + assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); + bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos; + bitC->bitPos += nbBits; +} + +/*! BIT_addBitsFast() : + * works only if `value` is _clean_, + * meaning all high bits above nbBits are 0 */ +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, + size_t value, unsigned nbBits) +{ + assert((value>>nbBits) == 0); + assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); + bitC->bitContainer |= value << bitC->bitPos; + bitC->bitPos += nbBits; +} + +/*! BIT_flushBitsFast() : + * assumption : bitContainer has not overflowed + * unsafe version; does not check buffer overflow */ +MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC) +{ + size_t const nbBytes = bitC->bitPos >> 3; + assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); + assert(bitC->ptr <= bitC->endPtr); + MEM_writeLEST(bitC->ptr, bitC->bitContainer); + bitC->ptr += nbBytes; + bitC->bitPos &= 7; + bitC->bitContainer >>= nbBytes*8; +} + +/*! BIT_flushBits() : + * assumption : bitContainer has not overflowed + * safe version; check for buffer overflow, and prevents it. + * note : does not signal buffer overflow. + * overflow will be revealed later on using BIT_closeCStream() */ +MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC) +{ + size_t const nbBytes = bitC->bitPos >> 3; + assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); + assert(bitC->ptr <= bitC->endPtr); + MEM_writeLEST(bitC->ptr, bitC->bitContainer); + bitC->ptr += nbBytes; + if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; + bitC->bitPos &= 7; + bitC->bitContainer >>= nbBytes*8; +} + +/*! BIT_closeCStream() : + * @return : size of CStream, in bytes, + * or 0 if it could not fit into dstBuffer */ +MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) +{ + BIT_addBitsFast(bitC, 1, 1); /* endMark */ + BIT_flushBits(bitC); + if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */ + return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); +} + + +/*-******************************************************** +* bitStream decoding +**********************************************************/ +/*! BIT_initDStream() : + * Initialize a BIT_DStream_t. + * `bitD` : a pointer to an already allocated BIT_DStream_t structure. + * `srcSize` must be the *exact* size of the bitStream, in bytes. + * @return : size of stream (== srcSize), or an errorCode if a problem is detected + */ +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize) +{ + if (srcSize < 1) { ZSTD_memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } + + bitD->start = (const char*)srcBuffer; + bitD->limitPtr = bitD->start + sizeof(bitD->bitContainer); + + if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ + bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); + bitD->bitContainer = MEM_readLEST(bitD->ptr); + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ + if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } + } else { + bitD->ptr = bitD->start; + bitD->bitContainer = *(const BYTE*)(bitD->start); + switch(srcSize) + { + case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); + /* fall-through */ + + case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); + /* fall-through */ + + case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); + /* fall-through */ + + case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; + /* fall-through */ + + case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; + /* fall-through */ + + case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; + /* fall-through */ + + default: break; + } + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; + if (lastByte == 0) return ERROR(corruption_detected); /* endMark not present */ + } + bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; + } + + return srcSize; +} + +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getUpperBits(size_t bitContainer, U32 const start) +{ + return bitContainer >> start; +} + +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) +{ + U32 const regMask = sizeof(bitContainer)*8 - 1; + /* if start > regMask, bitstream is corrupted, and result is undefined */ + assert(nbBits < BIT_MASK_SIZE); + return (bitContainer >> (start & regMask)) & BIT_mask[nbBits]; +} + +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) +{ + assert(nbBits < BIT_MASK_SIZE); + return bitContainer & BIT_mask[nbBits]; +} + +/*! BIT_lookBits() : + * Provides next n bits from local register. + * local register is not modified. + * On 32-bits, maxNbBits==24. + * On 64-bits, maxNbBits==56. + * @return : value extracted */ +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) +{ + /* arbitrate between double-shift and shift+mask */ +#if 1 + /* if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8, + * bitstream is likely corrupted, and result is undefined */ + return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits); +#else + /* this code path is slower on my os-x laptop */ + U32 const regMask = sizeof(bitD->bitContainer)*8 - 1; + return ((bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> 1) >> ((regMask-nbBits) & regMask); +#endif +} + +/*! BIT_lookBitsFast() : + * unsafe version; only works if nbBits >= 1 */ +MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) +{ + U32 const regMask = sizeof(bitD->bitContainer)*8 - 1; + assert(nbBits >= 1); + return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask); +} + +MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) +{ + bitD->bitsConsumed += nbBits; +} + +/*! BIT_readBits() : + * Read (consume) next n bits from local register and update. + * Pay attention to not read more than nbBits contained into local register. + * @return : extracted value. */ +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits) +{ + size_t const value = BIT_lookBits(bitD, nbBits); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_readBitsFast() : + * unsafe version; only works only if nbBits >= 1 */ +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits) +{ + size_t const value = BIT_lookBitsFast(bitD, nbBits); + assert(nbBits >= 1); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_reloadDStreamFast() : + * Similar to BIT_reloadDStream(), but with two differences: + * 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold! + * 2. Returns BIT_DStream_overflow when bitD->ptr < bitD->limitPtr, at this + * point you must use BIT_reloadDStream() to reload. + */ +MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD) +{ + if (UNLIKELY(bitD->ptr < bitD->limitPtr)) + return BIT_DStream_overflow; + assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8); + bitD->ptr -= bitD->bitsConsumed >> 3; + bitD->bitsConsumed &= 7; + bitD->bitContainer = MEM_readLEST(bitD->ptr); + return BIT_DStream_unfinished; +} + +/*! BIT_reloadDStream() : + * Refill `bitD` from buffer previously set in BIT_initDStream() . + * This function is safe, it guarantees it will not read beyond src buffer. + * @return : status of `BIT_DStream_t` internal register. + * when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */ +MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) +{ + if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* overflow detected, like end of stream */ + return BIT_DStream_overflow; + + if (bitD->ptr >= bitD->limitPtr) { + return BIT_reloadDStreamFast(bitD); + } + if (bitD->ptr == bitD->start) { + if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; + return BIT_DStream_completed; + } + /* start < ptr < limitPtr */ + { U32 nbBytes = bitD->bitsConsumed >> 3; + BIT_DStream_status result = BIT_DStream_unfinished; + if (bitD->ptr - nbBytes < bitD->start) { + nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ + result = BIT_DStream_endOfBuffer; + } + bitD->ptr -= nbBytes; + bitD->bitsConsumed -= nbBytes*8; + bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD->bitContainer), otherwise bitD->ptr == bitD->start */ + return result; + } +} + +/*! BIT_endOfDStream() : + * @return : 1 if DStream has _exactly_ reached its end (all bits consumed). + */ +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) +{ + return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8)); +} + + +#endif /* BITSTREAM_H_MODULE */ diff --git a/lib/zstd/common/compiler.h b/lib/zstd/common/compiler.h new file mode 100644 index 000000000000..a8286d8c4244 --- /dev/null +++ b/lib/zstd/common/compiler.h @@ -0,0 +1,151 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPILER_H +#define ZSTD_COMPILER_H + +/*-******************************************************* +* Compiler specifics +*********************************************************/ +/* force inlining */ + +#if (defined(__GNUC__) && !defined(__STRICT_ANSI__)) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# define INLINE_KEYWORD inline +#else +# define INLINE_KEYWORD +#endif + +#define FORCE_INLINE_ATTR __attribute__((always_inline)) + + +/* + On MSVC qsort requires that functions passed into it use the __cdecl calling conversion(CC). + This explictly marks such functions as __cdecl so that the code will still compile + if a CC other than __cdecl has been made the default. +*/ +#define WIN_CDECL + +/* + * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant + * parameters. They must be inlined for the compiler to eliminate the constant + * branches. + */ +#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR +/* + * HINT_INLINE is used to help the compiler generate better code. It is *not* + * used for "templates", so it can be tweaked based on the compilers + * performance. + * + * gcc-4.8 and gcc-4.9 have been shown to benefit from leaving off the + * always_inline attribute. + * + * clang up to 5.0.0 (trunk) benefit tremendously from the always_inline + * attribute. + */ +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5 +# define HINT_INLINE static INLINE_KEYWORD +#else +# define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR +#endif + +/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */ +#define UNUSED_ATTR __attribute__((unused)) + +/* force no inlining */ +#define FORCE_NOINLINE static __attribute__((__noinline__)) + + +/* target attribute */ +#ifndef __has_attribute + #define __has_attribute(x) 0 /* Compatibility with non-clang compilers. */ +#endif +#define TARGET_ATTRIBUTE(target) __attribute__((__target__(target))) + +/* Enable runtime BMI2 dispatch based on the CPU. + * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default. + */ +#ifndef DYNAMIC_BMI2 + #if ((defined(__clang__) && __has_attribute(__target__)) \ + || (defined(__GNUC__) \ + && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \ + && (defined(__x86_64__) || defined(_M_X86)) \ + && !defined(__BMI2__) + # define DYNAMIC_BMI2 1 + #else + # define DYNAMIC_BMI2 0 + #endif +#endif + +/* prefetch + * can be disabled, by declaring NO_PREFETCH build macro */ +#if ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */) +#elif defined(__aarch64__) +# define PREFETCH_L1(ptr) __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr))) +# define PREFETCH_L2(ptr) __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr))) +#else +# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ +# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ +#endif /* NO_PREFETCH */ + +#define CACHELINE_SIZE 64 + +#define PREFETCH_AREA(p, s) { \ + const char* const _ptr = (const char*)(p); \ + size_t const _size = (size_t)(s); \ + size_t _pos; \ + for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \ + PREFETCH_L2(_ptr + _pos); \ + } \ +} + +/* vectorization + * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax */ +#if !defined(__INTEL_COMPILER) && !defined(__clang__) && defined(__GNUC__) +# if (__GNUC__ == 4 && __GNUC_MINOR__ > 3) || (__GNUC__ >= 5) +# define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize"))) +# else +# define DONT_VECTORIZE _Pragma("GCC optimize(\"no-tree-vectorize\")") +# endif +#else +# define DONT_VECTORIZE +#endif + +/* Tell the compiler that a branch is likely or unlikely. + * Only use these macros if it causes the compiler to generate better code. + * If you can remove a LIKELY/UNLIKELY annotation without speed changes in gcc + * and clang, please do. + */ +#define LIKELY(x) (__builtin_expect((x), 1)) +#define UNLIKELY(x) (__builtin_expect((x), 0)) + +/* disable warnings */ + +/*Like DYNAMIC_BMI2 but for compile time determination of BMI2 support*/ + + +/* compat. with non-clang compilers */ +#ifndef __has_builtin +# define __has_builtin(x) 0 +#endif + +/* compat. with non-clang compilers */ +#ifndef __has_feature +# define __has_feature(x) 0 +#endif + +/* detects whether we are being compiled under msan */ + + +/* detects whether we are being compiled under asan */ + + +#endif /* ZSTD_COMPILER_H */ diff --git a/lib/zstd/common/cpu.h b/lib/zstd/common/cpu.h new file mode 100644 index 000000000000..0db7b42407ee --- /dev/null +++ b/lib/zstd/common/cpu.h @@ -0,0 +1,194 @@ +/* + * Copyright (c) Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMMON_CPU_H +#define ZSTD_COMMON_CPU_H + +/* + * Implementation taken from folly/CpuId.h + * https://github.com/facebook/folly/blob/master/folly/CpuId.h + */ + +#include "mem.h" + + +typedef struct { + U32 f1c; + U32 f1d; + U32 f7b; + U32 f7c; +} ZSTD_cpuid_t; + +MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) { + U32 f1c = 0; + U32 f1d = 0; + U32 f7b = 0; + U32 f7c = 0; +#if defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__) + /* The following block like the normal cpuid branch below, but gcc + * reserves ebx for use of its pic register so we must specially + * handle the save and restore to avoid clobbering the register + */ + U32 n; + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "popl %%ebx\n\t" + : "=a"(n) + : "a"(0) + : "ecx", "edx"); + if (n >= 1) { + U32 f1a; + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "popl %%ebx\n\t" + : "=a"(f1a), "=c"(f1c), "=d"(f1d) + : "a"(1)); + } + if (n >= 7) { + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "movl %%ebx, %%eax\n\t" + "popl %%ebx" + : "=a"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "edx"); + } +#elif defined(__x86_64__) || defined(_M_X64) || defined(__i386__) + U32 n; + __asm__("cpuid" : "=a"(n) : "a"(0) : "ebx", "ecx", "edx"); + if (n >= 1) { + U32 f1a; + __asm__("cpuid" : "=a"(f1a), "=c"(f1c), "=d"(f1d) : "a"(1) : "ebx"); + } + if (n >= 7) { + U32 f7a; + __asm__("cpuid" + : "=a"(f7a), "=b"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "edx"); + } +#endif + { + ZSTD_cpuid_t cpuid; + cpuid.f1c = f1c; + cpuid.f1d = f1d; + cpuid.f7b = f7b; + cpuid.f7c = f7c; + return cpuid; + } +} + +#define X(name, r, bit) \ + MEM_STATIC int ZSTD_cpuid_##name(ZSTD_cpuid_t const cpuid) { \ + return ((cpuid.r) & (1U << bit)) != 0; \ + } + +/* cpuid(1): Processor Info and Feature Bits. */ +#define C(name, bit) X(name, f1c, bit) + C(sse3, 0) + C(pclmuldq, 1) + C(dtes64, 2) + C(monitor, 3) + C(dscpl, 4) + C(vmx, 5) + C(smx, 6) + C(eist, 7) + C(tm2, 8) + C(ssse3, 9) + C(cnxtid, 10) + C(fma, 12) + C(cx16, 13) + C(xtpr, 14) + C(pdcm, 15) + C(pcid, 17) + C(dca, 18) + C(sse41, 19) + C(sse42, 20) + C(x2apic, 21) + C(movbe, 22) + C(popcnt, 23) + C(tscdeadline, 24) + C(aes, 25) + C(xsave, 26) + C(osxsave, 27) + C(avx, 28) + C(f16c, 29) + C(rdrand, 30) +#undef C +#define D(name, bit) X(name, f1d, bit) + D(fpu, 0) + D(vme, 1) + D(de, 2) + D(pse, 3) + D(tsc, 4) + D(msr, 5) + D(pae, 6) + D(mce, 7) + D(cx8, 8) + D(apic, 9) + D(sep, 11) + D(mtrr, 12) + D(pge, 13) + D(mca, 14) + D(cmov, 15) + D(pat, 16) + D(pse36, 17) + D(psn, 18) + D(clfsh, 19) + D(ds, 21) + D(acpi, 22) + D(mmx, 23) + D(fxsr, 24) + D(sse, 25) + D(sse2, 26) + D(ss, 27) + D(htt, 28) + D(tm, 29) + D(pbe, 31) +#undef D + +/* cpuid(7): Extended Features. */ +#define B(name, bit) X(name, f7b, bit) + B(bmi1, 3) + B(hle, 4) + B(avx2, 5) + B(smep, 7) + B(bmi2, 8) + B(erms, 9) + B(invpcid, 10) + B(rtm, 11) + B(mpx, 14) + B(avx512f, 16) + B(avx512dq, 17) + B(rdseed, 18) + B(adx, 19) + B(smap, 20) + B(avx512ifma, 21) + B(pcommit, 22) + B(clflushopt, 23) + B(clwb, 24) + B(avx512pf, 26) + B(avx512er, 27) + B(avx512cd, 28) + B(sha, 29) + B(avx512bw, 30) + B(avx512vl, 31) +#undef B +#define C(name, bit) X(name, f7c, bit) + C(prefetchwt1, 0) + C(avx512vbmi, 1) +#undef C + +#undef X + +#endif /* ZSTD_COMMON_CPU_H */ diff --git a/lib/zstd/common/debug.c b/lib/zstd/common/debug.c new file mode 100644 index 000000000000..bb863c9ea616 --- /dev/null +++ b/lib/zstd/common/debug.c @@ -0,0 +1,24 @@ +/* ****************************************************************** + * debug + * Part of FSE library + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + + +/* + * This module only hosts one global variable + * which can be used to dynamically influence the verbosity of traces, + * such as DEBUGLOG and RAWLOG + */ + +#include "debug.h" + +int g_debuglevel = DEBUGLEVEL; diff --git a/lib/zstd/common/debug.h b/lib/zstd/common/debug.h new file mode 100644 index 000000000000..6dd88d1fbd02 --- /dev/null +++ b/lib/zstd/common/debug.h @@ -0,0 +1,101 @@ +/* ****************************************************************** + * debug + * Part of FSE library + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + + +/* + * The purpose of this header is to enable debug functions. + * They regroup assert(), DEBUGLOG() and RAWLOG() for run-time, + * and DEBUG_STATIC_ASSERT() for compile-time. + * + * By default, DEBUGLEVEL==0, which means run-time debug is disabled. + * + * Level 1 enables assert() only. + * Starting level 2, traces can be generated and pushed to stderr. + * The higher the level, the more verbose the traces. + * + * It's possible to dynamically adjust level using variable g_debug_level, + * which is only declared if DEBUGLEVEL>=2, + * and is a global variable, not multi-thread protected (use with care) + */ + +#ifndef DEBUG_H_12987983217 +#define DEBUG_H_12987983217 + + + +/* static assert is triggered at compile time, leaving no runtime artefact. + * static assert only works with compile-time constants. + * Also, this variant can only be used inside a function. */ +#define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1]) + + +/* DEBUGLEVEL is expected to be defined externally, + * typically through compiler command line. + * Value must be a number. */ +#ifndef DEBUGLEVEL +# define DEBUGLEVEL 0 +#endif + + +/* recommended values for DEBUGLEVEL : + * 0 : release mode, no debug, all run-time checks disabled + * 1 : enables assert() only, no display + * 2 : reserved, for currently active debug path + * 3 : events once per object lifetime (CCtx, CDict, etc.) + * 4 : events once per frame + * 5 : events once per block + * 6 : events once per sequence (verbose) + * 7+: events at every position (*very* verbose) + * + * It's generally inconvenient to output traces > 5. + * In which case, it's possible to selectively trigger high verbosity levels + * by modifying g_debug_level. + */ + +#if (DEBUGLEVEL>=1) +# define ZSTD_DEPS_NEED_ASSERT +# include "zstd_deps.h" +#else +# ifndef assert /* assert may be already defined, due to prior #include */ +# define assert(condition) ((void)0) /* disable assert (default) */ +# endif +#endif + +#if (DEBUGLEVEL>=2) +# define ZSTD_DEPS_NEED_IO +# include "zstd_deps.h" +extern int g_debuglevel; /* the variable is only declared, + it actually lives in debug.c, + and is shared by the whole process. + It's not thread-safe. + It's useful when enabling very verbose levels + on selective conditions (such as position in src) */ + +# define RAWLOG(l, ...) { \ + if (l<=g_debuglevel) { \ + ZSTD_DEBUG_PRINT(__VA_ARGS__); \ + } } +# define DEBUGLOG(l, ...) { \ + if (l<=g_debuglevel) { \ + ZSTD_DEBUG_PRINT(__FILE__ ": " __VA_ARGS__); \ + ZSTD_DEBUG_PRINT(" \n"); \ + } } +#else +# define RAWLOG(l, ...) {} /* disabled */ +# define DEBUGLOG(l, ...) {} /* disabled */ +#endif + + + +#endif /* DEBUG_H_12987983217 */ diff --git a/lib/zstd/common/entropy_common.c b/lib/zstd/common/entropy_common.c new file mode 100644 index 000000000000..53b47a2b52ff --- /dev/null +++ b/lib/zstd/common/entropy_common.c @@ -0,0 +1,357 @@ +/* ****************************************************************** + * Common functions of New Generation Entropy library + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* ************************************* +* Dependencies +***************************************/ +#include "mem.h" +#include "error_private.h" /* ERR_*, ERROR */ +#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */ +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */ +#include "huf.h" + + +/*=== Version ===*/ +unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; } + + +/*=== Error Management ===*/ +unsigned FSE_isError(size_t code) { return ERR_isError(code); } +const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); } + +unsigned HUF_isError(size_t code) { return ERR_isError(code); } +const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } + + +/*-************************************************************** +* FSE NCount encoding-decoding +****************************************************************/ +static U32 FSE_ctz(U32 val) +{ + assert(val != 0); + { +# if (__GNUC__ >= 3) /* GCC Intrinsic */ + return __builtin_ctz(val); +# else /* Software version */ + U32 count = 0; + while ((val & 1) == 0) { + val >>= 1; + ++count; + } + return count; +# endif + } +} + +FORCE_INLINE_TEMPLATE +size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + const BYTE* const istart = (const BYTE*) headerBuffer; + const BYTE* const iend = istart + hbSize; + const BYTE* ip = istart; + int nbBits; + int remaining; + int threshold; + U32 bitStream; + int bitCount; + unsigned charnum = 0; + unsigned const maxSV1 = *maxSVPtr + 1; + int previous0 = 0; + + if (hbSize < 8) { + /* This function only works when hbSize >= 8 */ + char buffer[8] = {0}; + ZSTD_memcpy(buffer, headerBuffer, hbSize); + { size_t const countSize = FSE_readNCount(normalizedCounter, maxSVPtr, tableLogPtr, + buffer, sizeof(buffer)); + if (FSE_isError(countSize)) return countSize; + if (countSize > hbSize) return ERROR(corruption_detected); + return countSize; + } } + assert(hbSize >= 8); + + /* init */ + ZSTD_memset(normalizedCounter, 0, (*maxSVPtr+1) * sizeof(normalizedCounter[0])); /* all symbols not present in NCount have a frequency of 0 */ + bitStream = MEM_readLE32(ip); + nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ + if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); + bitStream >>= 4; + bitCount = 4; + *tableLogPtr = nbBits; + remaining = (1<> 1; + while (repeats >= 12) { + charnum += 3 * 12; + if (LIKELY(ip <= iend-7)) { + ip += 3; + } else { + bitCount -= (int)(8 * (iend - 7 - ip)); + bitCount &= 31; + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> bitCount; + repeats = FSE_ctz(~bitStream | 0x80000000) >> 1; + } + charnum += 3 * repeats; + bitStream >>= 2 * repeats; + bitCount += 2 * repeats; + + /* Add the final repeat which isn't 0b11. */ + assert((bitStream & 3) < 3); + charnum += bitStream & 3; + bitCount += 2; + + /* This is an error, but break and return an error + * at the end, because returning out of a loop makes + * it harder for the compiler to optimize. + */ + if (charnum >= maxSV1) break; + + /* We don't need to set the normalized count to 0 + * because we already memset the whole buffer to 0. + */ + + if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + assert((bitCount >> 3) <= 3); /* For first condition to work */ + ip += bitCount>>3; + bitCount &= 7; + } else { + bitCount -= (int)(8 * (iend - 4 - ip)); + bitCount &= 31; + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> bitCount; + } + { + int const max = (2*threshold-1) - remaining; + int count; + + if ((bitStream & (threshold-1)) < (U32)max) { + count = bitStream & (threshold-1); + bitCount += nbBits-1; + } else { + count = bitStream & (2*threshold-1); + if (count >= threshold) count -= max; + bitCount += nbBits; + } + + count--; /* extra accuracy */ + /* When it matters (small blocks), this is a + * predictable branch, because we don't use -1. + */ + if (count >= 0) { + remaining -= count; + } else { + assert(count == -1); + remaining += count; + } + normalizedCounter[charnum++] = (short)count; + previous0 = !count; + + assert(threshold > 1); + if (remaining < threshold) { + /* This branch can be folded into the + * threshold update condition because we + * know that threshold > 1. + */ + if (remaining <= 1) break; + nbBits = BIT_highbit32(remaining) + 1; + threshold = 1 << (nbBits - 1); + } + if (charnum >= maxSV1) break; + + if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + ip += bitCount>>3; + bitCount &= 7; + } else { + bitCount -= (int)(8 * (iend - 4 - ip)); + bitCount &= 31; + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> bitCount; + } } + if (remaining != 1) return ERROR(corruption_detected); + /* Only possible when there are too many zeros. */ + if (charnum > maxSV1) return ERROR(maxSymbolValue_tooSmall); + if (bitCount > 32) return ERROR(corruption_detected); + *maxSVPtr = charnum-1; + + ip += (bitCount+7)>>3; + return ip-istart; +} + +/* Avoids the FORCE_INLINE of the _body() function. */ +static size_t FSE_readNCount_body_default( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); +} + +#if DYNAMIC_BMI2 +TARGET_ATTRIBUTE("bmi2") static size_t FSE_readNCount_body_bmi2( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); +} +#endif + +size_t FSE_readNCount_bmi2( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize, int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { + return FSE_readNCount_body_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); + } +#endif + (void)bmi2; + return FSE_readNCount_body_default(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); +} + +size_t FSE_readNCount( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + return FSE_readNCount_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize, /* bmi2 */ 0); +} + + +/*! HUF_readStats() : + Read compact Huffman tree, saved by HUF_writeCTable(). + `huffWeight` is destination buffer. + `rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32. + @return : size read from `src` , or an error Code . + Note : Needed by HUF_readCTable() and HUF_readDTableX?() . +*/ +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize) +{ + U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; + return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* bmi2 */ 0); +} + +FORCE_INLINE_TEMPLATE size_t +HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, + int bmi2) +{ + U32 weightTotal; + const BYTE* ip = (const BYTE*) src; + size_t iSize; + size_t oSize; + + if (!srcSize) return ERROR(srcSize_wrong); + iSize = ip[0]; + /* ZSTD_memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */ + + if (iSize >= 128) { /* special header */ + oSize = iSize - 127; + iSize = ((oSize+1)/2); + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + if (oSize >= hwSize) return ERROR(corruption_detected); + ip += 1; + { U32 n; + for (n=0; n> 4; + huffWeight[n+1] = ip[n/2] & 15; + } } } + else { /* header compressed with FSE (normal case) */ + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + /* max (hwSize-1) values decoded, as last one is implied */ + oSize = FSE_decompress_wksp_bmi2(huffWeight, hwSize-1, ip+1, iSize, 6, workSpace, wkspSize, bmi2); + if (FSE_isError(oSize)) return oSize; + } + + /* collect weight stats */ + ZSTD_memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32)); + weightTotal = 0; + { U32 n; for (n=0; n= HUF_TABLELOG_MAX) return ERROR(corruption_detected); + rankStats[huffWeight[n]]++; + weightTotal += (1 << huffWeight[n]) >> 1; + } } + if (weightTotal == 0) return ERROR(corruption_detected); + + /* get last non-null symbol weight (implied, total must be 2^n) */ + { U32 const tableLog = BIT_highbit32(weightTotal) + 1; + if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected); + *tableLogPtr = tableLog; + /* determine last weight */ + { U32 const total = 1 << tableLog; + U32 const rest = total - weightTotal; + U32 const verif = 1 << BIT_highbit32(rest); + U32 const lastWeight = BIT_highbit32(rest) + 1; + if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ + huffWeight[oSize] = (BYTE)lastWeight; + rankStats[lastWeight]++; + } } + + /* check tree construction validity */ + if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ + + /* results */ + *nbSymbolsPtr = (U32)(oSize+1); + return iSize+1; +} + +/* Avoids the FORCE_INLINE of the _body() function. */ +static size_t HUF_readStats_body_default(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize) +{ + return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 0); +} + +#if DYNAMIC_BMI2 +static TARGET_ATTRIBUTE("bmi2") size_t HUF_readStats_body_bmi2(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize) +{ + return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 1); +} +#endif + +size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, + int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { + return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); + } +#endif + (void)bmi2; + return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); +} diff --git a/lib/zstd/common/error_private.c b/lib/zstd/common/error_private.c new file mode 100644 index 000000000000..6d1135f8c373 --- /dev/null +++ b/lib/zstd/common/error_private.c @@ -0,0 +1,56 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* The purpose of this file is to have a single list of error strings embedded in binary */ + +#include "error_private.h" + +const char* ERR_getErrorString(ERR_enum code) +{ +#ifdef ZSTD_STRIP_ERROR_STRINGS + (void)code; + return "Error strings stripped"; +#else + static const char* const notErrorCode = "Unspecified error code"; + switch( code ) + { + case PREFIX(no_error): return "No error detected"; + case PREFIX(GENERIC): return "Error (generic)"; + case PREFIX(prefix_unknown): return "Unknown frame descriptor"; + case PREFIX(version_unsupported): return "Version not supported"; + case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; + case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding"; + case PREFIX(corruption_detected): return "Corrupted block detected"; + case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; + case PREFIX(parameter_unsupported): return "Unsupported parameter"; + case PREFIX(parameter_outOfBound): return "Parameter is out of bound"; + case PREFIX(init_missing): return "Context should be init first"; + case PREFIX(memory_allocation): return "Allocation error : not enough memory"; + case PREFIX(workSpace_tooSmall): return "workSpace buffer is not large enough"; + case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; + case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; + case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; + case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; + case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; + case PREFIX(dictionary_wrong): return "Dictionary mismatch"; + case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples"; + case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; + case PREFIX(srcSize_wrong): return "Src size is incorrect"; + case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer"; + /* following error codes are not stable and may be removed or changed in a future version */ + case PREFIX(frameIndex_tooLarge): return "Frame index is too large"; + case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking"; + case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong"; + case PREFIX(srcBuffer_wrong): return "Source buffer is wrong"; + case PREFIX(maxCode): + default: return notErrorCode; + } +#endif +} diff --git a/lib/zstd/common/error_private.h b/lib/zstd/common/error_private.h new file mode 100644 index 000000000000..d14e686adf95 --- /dev/null +++ b/lib/zstd/common/error_private.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* Note : this module is expected to remain private, do not expose it */ + +#ifndef ERROR_H_MODULE +#define ERROR_H_MODULE + + + +/* **************************************** +* Dependencies +******************************************/ +#include "zstd_deps.h" /* size_t */ +#include /* enum list */ + + +/* **************************************** +* Compiler-specific +******************************************/ +#define ERR_STATIC static __attribute__((unused)) + + +/*-**************************************** +* Customization (error_public.h) +******************************************/ +typedef ZSTD_ErrorCode ERR_enum; +#define PREFIX(name) ZSTD_error_##name + + +/*-**************************************** +* Error codes handling +******************************************/ +#undef ERROR /* already defined on Visual Studio */ +#define ERROR(name) ZSTD_ERROR(name) +#define ZSTD_ERROR(name) ((size_t)-PREFIX(name)) + +ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } + +ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } + +/* check and forward error code */ +#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e +#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } + + +/*-**************************************** +* Error Strings +******************************************/ + +const char* ERR_getErrorString(ERR_enum code); /* error_private.c */ + +ERR_STATIC const char* ERR_getErrorName(size_t code) +{ + return ERR_getErrorString(ERR_getErrorCode(code)); +} + + +#endif /* ERROR_H_MODULE */ diff --git a/lib/zstd/common/fse.h b/lib/zstd/common/fse.h new file mode 100644 index 000000000000..0bb174c2c367 --- /dev/null +++ b/lib/zstd/common/fse.h @@ -0,0 +1,710 @@ +/* ****************************************************************** + * FSE : Finite State Entropy codec + * Public Prototypes declaration + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + + +#ifndef FSE_H +#define FSE_H + + +/*-***************************************** +* Dependencies +******************************************/ +#include "zstd_deps.h" /* size_t, ptrdiff_t */ + + +/*-***************************************** +* FSE_PUBLIC_API : control library symbols visibility +******************************************/ +#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4) +# define FSE_PUBLIC_API __attribute__ ((visibility ("default"))) +#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */ +# define FSE_PUBLIC_API __declspec(dllexport) +#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1) +# define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define FSE_PUBLIC_API +#endif + +/*------ Version ------*/ +#define FSE_VERSION_MAJOR 0 +#define FSE_VERSION_MINOR 9 +#define FSE_VERSION_RELEASE 0 + +#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE +#define FSE_QUOTE(str) #str +#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str) +#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION) + +#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE) +FSE_PUBLIC_API unsigned FSE_versionNumber(void); /*< library version number; to be used when checking dll version */ + + +/*-**************************************** +* FSE simple functions +******************************************/ +/*! FSE_compress() : + Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'. + 'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize). + @return : size of compressed data (<= dstCapacity). + Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! + if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead. + if FSE_isError(return), compression failed (more details using FSE_getErrorName()) +*/ +FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +/*! FSE_decompress(): + Decompress FSE data from buffer 'cSrc', of size 'cSrcSize', + into already allocated destination buffer 'dst', of size 'dstCapacity'. + @return : size of regenerated data (<= maxDstSize), + or an error code, which can be tested using FSE_isError() . + + ** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!! + Why ? : making this distinction requires a header. + Header management is intentionally delegated to the user layer, which can better manage special cases. +*/ +FSE_PUBLIC_API size_t FSE_decompress(void* dst, size_t dstCapacity, + const void* cSrc, size_t cSrcSize); + + +/*-***************************************** +* Tool functions +******************************************/ +FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */ + +/* Error Management */ +FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */ +FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */ + + +/*-***************************************** +* FSE advanced functions +******************************************/ +/*! FSE_compress2() : + Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog' + Both parameters can be defined as '0' to mean : use default value + @return : size of compressed data + Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!! + if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression. + if FSE_isError(return), it's an error code. +*/ +FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); + + +/*-***************************************** +* FSE detailed API +******************************************/ +/*! +FSE_compress() does the following: +1. count symbol occurrence from source[] into table count[] (see hist.h) +2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog) +3. save normalized counters to memory buffer using writeNCount() +4. build encoding table 'CTable' from normalized counters +5. encode the data stream using encoding table 'CTable' + +FSE_decompress() does the following: +1. read normalized counters with readNCount() +2. build decoding table 'DTable' from normalized counters +3. decode the data stream using decoding table 'DTable' + +The following API allows targeting specific sub-functions for advanced tasks. +For example, it's possible to compress several blocks using the same 'CTable', +or to save and provide normalized distribution using external method. +*/ + +/* *** COMPRESSION *** */ + +/*! FSE_optimalTableLog(): + dynamically downsize 'tableLog' when conditions are met. + It saves CPU time, by using smaller tables, while preserving or even improving compression ratio. + @return : recommended tableLog (necessarily <= 'maxTableLog') */ +FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); + +/*! FSE_normalizeCount(): + normalize counts so that sum(count[]) == Power_of_2 (2^tableLog) + 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). + useLowProbCount is a boolean parameter which trades off compressed size for + faster header decoding. When it is set to 1, the compressed data will be slightly + smaller. And when it is set to 0, FSE_readNCount() and FSE_buildDTable() will be + faster. If you are compressing a small amount of data (< 2 KB) then useLowProbCount=0 + is a good default, since header deserialization makes a big speed difference. + Otherwise, useLowProbCount=1 is a good default, since the speed difference is small. + @return : tableLog, + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, + const unsigned* count, size_t srcSize, unsigned maxSymbolValue, unsigned useLowProbCount); + +/*! FSE_NCountWriteBound(): + Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'. + Typically useful for allocation purpose. */ +FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_writeNCount(): + Compactly save 'normalizedCounter' into 'buffer'. + @return : size of the compressed table, + or an errorCode, which can be tested using FSE_isError(). */ +FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, + const short* normalizedCounter, + unsigned maxSymbolValue, unsigned tableLog); + +/*! Constructor and Destructor of FSE_CTable. + Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ +typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ +FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog); +FSE_PUBLIC_API void FSE_freeCTable (FSE_CTable* ct); + +/*! FSE_buildCTable(): + Builds `ct`, which must be already allocated, using FSE_createCTable(). + @return : 0, or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_compress_usingCTable(): + Compress `src` using `ct` into `dst` which must be already allocated. + @return : size of compressed data (<= `dstCapacity`), + or 0 if compressed data could not fit into `dst`, + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct); + +/*! +Tutorial : +---------- +The first step is to count all symbols. FSE_count() does this job very fast. +Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells. +'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0] +maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value) +FSE_count() will return the number of occurrence of the most frequent symbol. +This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). + +The next step is to normalize the frequencies. +FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'. +It also guarantees a minimum of 1 to any Symbol with frequency >= 1. +You can use 'tableLog'==0 to mean "use default tableLog value". +If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(), +which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default"). + +The result of FSE_normalizeCount() will be saved into a table, +called 'normalizedCounter', which is a table of signed short. +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells. +The return value is tableLog if everything proceeded as expected. +It is 0 if there is a single symbol within distribution. +If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()). + +'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount(). +'buffer' must be already allocated. +For guaranteed success, buffer size must be at least FSE_headerBound(). +The result of the function is the number of bytes written into 'buffer'. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small). + +'normalizedCounter' can then be used to create the compression table 'CTable'. +The space required by 'CTable' must be already allocated, using FSE_createCTable(). +You can then use FSE_buildCTable() to fill 'CTable'. +If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()). + +'CTable' can then be used to compress 'src', with FSE_compress_usingCTable(). +Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize' +The function returns the size of compressed data (without header), necessarily <= `dstCapacity`. +If it returns '0', compressed data could not fit into 'dst'. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). +*/ + + +/* *** DECOMPRESSION *** */ + +/*! FSE_readNCount(): + Read compactly saved 'normalizedCounter' from 'rBuffer'. + @return : size read from 'rBuffer', + or an errorCode, which can be tested using FSE_isError(). + maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ +FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter, + unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, + const void* rBuffer, size_t rBuffSize); + +/*! FSE_readNCount_bmi2(): + * Same as FSE_readNCount() but pass bmi2=1 when your CPU supports BMI2 and 0 otherwise. + */ +FSE_PUBLIC_API size_t FSE_readNCount_bmi2(short* normalizedCounter, + unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, + const void* rBuffer, size_t rBuffSize, int bmi2); + +/*! Constructor and Destructor of FSE_DTable. + Note that its size depends on 'tableLog' */ +typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ +FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog); +FSE_PUBLIC_API void FSE_freeDTable(FSE_DTable* dt); + +/*! FSE_buildDTable(): + Builds 'dt', which must be already allocated, using FSE_createDTable(). + return : 0, or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_decompress_usingDTable(): + Decompress compressed source `cSrc` of size `cSrcSize` using `dt` + into `dst` which must be already allocated. + @return : size of regenerated data (necessarily <= `dstCapacity`), + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt); + +/*! +Tutorial : +---------- +(Note : these functions only decompress FSE-compressed blocks. + If block is uncompressed, use memcpy() instead + If block is a single repeated byte, use memset() instead ) + +The first step is to obtain the normalized frequencies of symbols. +This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount(). +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short. +In practice, that means it's necessary to know 'maxSymbolValue' beforehand, +or size the table to handle worst case situations (typically 256). +FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'. +The result of FSE_readNCount() is the number of bytes read from 'rBuffer'. +Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that. +If there is an error, the function will return an error code, which can be tested using FSE_isError(). + +The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'. +This is performed by the function FSE_buildDTable(). +The space required by 'FSE_DTable' must be already allocated using FSE_createDTable(). +If there is an error, the function will return an error code, which can be tested using FSE_isError(). + +`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable(). +`cSrcSize` must be strictly correct, otherwise decompression will fail. +FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`). +If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small) +*/ + +#endif /* FSE_H */ + +#if !defined(FSE_H_FSE_STATIC_LINKING_ONLY) +#define FSE_H_FSE_STATIC_LINKING_ONLY + +/* *** Dependency *** */ +#include "bitstream.h" + + +/* ***************************************** +* Static allocation +*******************************************/ +/* FSE buffer bounds */ +#define FSE_NCOUNTBOUND 512 +#define FSE_BLOCKBOUND(size) ((size) + ((size)>>7) + 4 /* fse states */ + sizeof(size_t) /* bitContainer */) +#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */ +#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<((maxTableLog)-1)) + (((maxSymbolValue)+1)*2)) +#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<(maxTableLog))) + +/* or use the size to malloc() space directly. Pay attention to alignment restrictions though */ +#define FSE_CTABLE_SIZE(maxTableLog, maxSymbolValue) (FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(FSE_CTable)) +#define FSE_DTABLE_SIZE(maxTableLog) (FSE_DTABLE_SIZE_U32(maxTableLog) * sizeof(FSE_DTable)) + + +/* ***************************************** + * FSE advanced API + ***************************************** */ + +unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); +/*< same as FSE_optimalTableLog(), which used `minus==2` */ + +/* FSE_compress_wksp() : + * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). + * FSE_COMPRESS_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable. + */ +#define FSE_COMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) ) +size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); + +size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits); +/*< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */ + +size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue); +/*< build a fake FSE_CTable, designed to compress always the same symbolValue */ + +/* FSE_buildCTable_wksp() : + * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). + * `wkspSize` must be >= `FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)` of `unsigned`. + */ +#define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (maxSymbolValue + 2 + (1ull << (tableLog - 2))) +#define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)) +size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); + +#define FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) (sizeof(short) * (maxSymbolValue + 1) + (1ULL << maxTableLog) + 8) +#define FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ((FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) + sizeof(unsigned) - 1) / sizeof(unsigned)) +FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); +/*< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` */ + +size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits); +/*< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */ + +size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue); +/*< build a fake FSE_DTable, designed to always generate the same symbolValue */ + +#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1) +#define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned)) +size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize); +/*< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)` */ + +size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2); +/*< Same as FSE_decompress_wksp() but with dynamic BMI2 support. Pass 1 if your CPU supports BMI2 or 0 if it doesn't. */ + +typedef enum { + FSE_repeat_none, /*< Cannot use the previous table */ + FSE_repeat_check, /*< Can use the previous table but it must be checked */ + FSE_repeat_valid /*< Can use the previous table and it is assumed to be valid */ + } FSE_repeat; + +/* ***************************************** +* FSE symbol compression API +*******************************************/ +/*! + This API consists of small unitary functions, which highly benefit from being inlined. + Hence their body are included in next section. +*/ +typedef struct { + ptrdiff_t value; + const void* stateTable; + const void* symbolTT; + unsigned stateLog; +} FSE_CState_t; + +static void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct); + +static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol); + +static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr); + +/*< +These functions are inner components of FSE_compress_usingCTable(). +They allow the creation of custom streams, mixing multiple tables and bit sources. + +A key property to keep in mind is that encoding and decoding are done **in reverse direction**. +So the first symbol you will encode is the last you will decode, like a LIFO stack. + +You will need a few variables to track your CStream. They are : + +FSE_CTable ct; // Provided by FSE_buildCTable() +BIT_CStream_t bitStream; // bitStream tracking structure +FSE_CState_t state; // State tracking structure (can have several) + + +The first thing to do is to init bitStream and state. + size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize); + FSE_initCState(&state, ct); + +Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError(); +You can then encode your input data, byte after byte. +FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time. +Remember decoding will be done in reverse direction. + FSE_encodeByte(&bitStream, &state, symbol); + +At any time, you can also add any bit sequence. +Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders + BIT_addBits(&bitStream, bitField, nbBits); + +The above methods don't commit data to memory, they just store it into local register, for speed. +Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). +Writing data to memory is a manual operation, performed by the flushBits function. + BIT_flushBits(&bitStream); + +Your last FSE encoding operation shall be to flush your last state value(s). + FSE_flushState(&bitStream, &state); + +Finally, you must close the bitStream. +The function returns the size of CStream in bytes. +If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible) +If there is an error, it returns an errorCode (which can be tested using FSE_isError()). + size_t size = BIT_closeCStream(&bitStream); +*/ + + +/* ***************************************** +* FSE symbol decompression API +*******************************************/ +typedef struct { + size_t state; + const void* table; /* precise table may vary, depending on U16 */ +} FSE_DState_t; + + +static void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt); + +static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); + +static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr); + +/*< +Let's now decompose FSE_decompress_usingDTable() into its unitary components. +You will decode FSE-encoded symbols from the bitStream, +and also any other bitFields you put in, **in reverse order**. + +You will need a few variables to track your bitStream. They are : + +BIT_DStream_t DStream; // Stream context +FSE_DState_t DState; // State context. Multiple ones are possible +FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable() + +The first thing to do is to init the bitStream. + errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize); + +You should then retrieve your initial state(s) +(in reverse flushing order if you have several ones) : + errorCode = FSE_initDState(&DState, &DStream, DTablePtr); + +You can then decode your data, symbol after symbol. +For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'. +Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out). + unsigned char symbol = FSE_decodeSymbol(&DState, &DStream); + +You can retrieve any bitfield you eventually stored into the bitStream (in reverse order) +Note : maximum allowed nbBits is 25, for 32-bits compatibility + size_t bitField = BIT_readBits(&DStream, nbBits); + +All above operations only read from local register (which size depends on size_t). +Refueling the register from memory is manually performed by the reload method. + endSignal = FSE_reloadDStream(&DStream); + +BIT_reloadDStream() result tells if there is still some more data to read from DStream. +BIT_DStream_unfinished : there is still some data left into the DStream. +BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled. +BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed. +BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted. + +When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop, +to properly detect the exact end of stream. +After each decoded symbol, check if DStream is fully consumed using this simple test : + BIT_reloadDStream(&DStream) >= BIT_DStream_completed + +When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. +Checking if DStream has reached its end is performed by : + BIT_endOfDStream(&DStream); +Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible. + FSE_endOfDState(&DState); +*/ + + +/* ***************************************** +* FSE unsafe API +*******************************************/ +static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); +/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ + + +/* ***************************************** +* Implementation of inlined functions +*******************************************/ +typedef struct { + int deltaFindState; + U32 deltaNbBits; +} FSE_symbolCompressionTransform; /* total 8 bytes */ + +MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct) +{ + const void* ptr = ct; + const U16* u16ptr = (const U16*) ptr; + const U32 tableLog = MEM_read16(ptr); + statePtr->value = (ptrdiff_t)1<stateTable = u16ptr+2; + statePtr->symbolTT = ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1); + statePtr->stateLog = tableLog; +} + + +/*! FSE_initCState2() : +* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read) +* uses the smallest state value possible, saving the cost of this symbol */ +MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol) +{ + FSE_initCState(statePtr, ct); + { const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* stateTable = (const U16*)(statePtr->stateTable); + U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16); + statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits; + statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; + } +} + +MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, unsigned symbol) +{ + FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* const stateTable = (const U16*)(statePtr->stateTable); + U32 const nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); + BIT_addBits(bitC, statePtr->value, nbBitsOut); + statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; +} + +MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr) +{ + BIT_addBits(bitC, statePtr->value, statePtr->stateLog); + BIT_flushBits(bitC); +} + + +/* FSE_getMaxNbBits() : + * Approximate maximum cost of a symbol, in bits. + * Fractional get rounded up (i.e : a symbol with a normalized frequency of 3 gives the same result as a frequency of 2) + * note 1 : assume symbolValue is valid (<= maxSymbolValue) + * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */ +MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue) +{ + const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr; + return (symbolTT[symbolValue].deltaNbBits + ((1<<16)-1)) >> 16; +} + +/* FSE_bitCost() : + * Approximate symbol cost, as fractional value, using fixed-point format (accuracyLog fractional bits) + * note 1 : assume symbolValue is valid (<= maxSymbolValue) + * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */ +MEM_STATIC U32 FSE_bitCost(const void* symbolTTPtr, U32 tableLog, U32 symbolValue, U32 accuracyLog) +{ + const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr; + U32 const minNbBits = symbolTT[symbolValue].deltaNbBits >> 16; + U32 const threshold = (minNbBits+1) << 16; + assert(tableLog < 16); + assert(accuracyLog < 31-tableLog); /* ensure enough room for renormalization double shift */ + { U32 const tableSize = 1 << tableLog; + U32 const deltaFromThreshold = threshold - (symbolTT[symbolValue].deltaNbBits + tableSize); + U32 const normalizedDeltaFromThreshold = (deltaFromThreshold << accuracyLog) >> tableLog; /* linear interpolation (very approximate) */ + U32 const bitMultiplier = 1 << accuracyLog; + assert(symbolTT[symbolValue].deltaNbBits + tableSize <= threshold); + assert(normalizedDeltaFromThreshold <= bitMultiplier); + return (minNbBits+1)*bitMultiplier - normalizedDeltaFromThreshold; + } +} + + +/* ====== Decompression ====== */ + +typedef struct { + U16 tableLog; + U16 fastMode; +} FSE_DTableHeader; /* sizeof U32 */ + +typedef struct +{ + unsigned short newState; + unsigned char symbol; + unsigned char nbBits; +} FSE_decode_t; /* size == U32 */ + +MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt) +{ + const void* ptr = dt; + const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr; + DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + BIT_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + return DInfo.symbol; +} + +MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = DInfo.newState + lowBits; +} + +MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBits(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +/*! FSE_decodeSymbolFast() : + unsafe, only works if no symbol has a probability > 50% */ +MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBitsFast(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) +{ + return DStatePtr->state == 0; +} + + + +#ifndef FSE_COMMONDEFS_ONLY + +/* ************************************************************** +* Tuning parameters +****************************************************************/ +/*!MEMORY_USAGE : +* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) +* Increasing memory usage improves compression ratio +* Reduced memory usage can improve speed, due to cache effect +* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ +#ifndef FSE_MAX_MEMORY_USAGE +# define FSE_MAX_MEMORY_USAGE 14 +#endif +#ifndef FSE_DEFAULT_MEMORY_USAGE +# define FSE_DEFAULT_MEMORY_USAGE 13 +#endif +#if (FSE_DEFAULT_MEMORY_USAGE > FSE_MAX_MEMORY_USAGE) +# error "FSE_DEFAULT_MEMORY_USAGE must be <= FSE_MAX_MEMORY_USAGE" +#endif + +/*!FSE_MAX_SYMBOL_VALUE : +* Maximum symbol value authorized. +* Required for proper stack allocation */ +#ifndef FSE_MAX_SYMBOL_VALUE +# define FSE_MAX_SYMBOL_VALUE 255 +#endif + +/* ************************************************************** +* template functions type & suffix +****************************************************************/ +#define FSE_FUNCTION_TYPE BYTE +#define FSE_FUNCTION_EXTENSION +#define FSE_DECODE_TYPE FSE_decode_t + + +#endif /* !FSE_COMMONDEFS_ONLY */ + + +/* *************************************************************** +* Constants +*****************************************************************/ +#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2) +#define FSE_MAX_TABLESIZE (1U< FSE_TABLELOG_ABSOLUTE_MAX +# error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" +#endif + +#define FSE_TABLESTEP(tableSize) (((tableSize)>>1) + ((tableSize)>>3) + 3) + + +#endif /* FSE_STATIC_LINKING_ONLY */ + + diff --git a/lib/zstd/common/fse_decompress.c b/lib/zstd/common/fse_decompress.c new file mode 100644 index 000000000000..2c8bbe3e4c14 --- /dev/null +++ b/lib/zstd/common/fse_decompress.c @@ -0,0 +1,390 @@ +/* ****************************************************************** + * FSE : Finite State Entropy decoder + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + + +/* ************************************************************** +* Includes +****************************************************************/ +#include "debug.h" /* assert */ +#include "bitstream.h" +#include "compiler.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#include "error_private.h" +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define FSE_isError ERR_isError +#define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */ + + +/* ************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +# error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +# error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X,Y) X##Y +#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) +#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) + + +/* Function templates */ +FSE_DTable* FSE_createDTable (unsigned tableLog) +{ + if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; + return (FSE_DTable*)ZSTD_malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); +} + +void FSE_freeDTable (FSE_DTable* dt) +{ + ZSTD_free(dt); +} + +static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize) +{ + void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ + FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr); + U16* symbolNext = (U16*)workSpace; + BYTE* spread = (BYTE*)(symbolNext + maxSymbolValue + 1); + + U32 const maxSV1 = maxSymbolValue + 1; + U32 const tableSize = 1 << tableLog; + U32 highThreshold = tableSize-1; + + /* Sanity Checks */ + if (FSE_BUILD_DTABLE_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(maxSymbolValue_tooLarge); + if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + + /* Init, lay down lowprob symbols */ + { FSE_DTableHeader DTableH; + DTableH.tableLog = (U16)tableLog; + DTableH.fastMode = 1; + { S16 const largeLimit= (S16)(1 << (tableLog-1)); + U32 s; + for (s=0; s= largeLimit) DTableH.fastMode=0; + symbolNext[s] = normalizedCounter[s]; + } } } + ZSTD_memcpy(dt, &DTableH, sizeof(DTableH)); + } + + /* Spread symbols */ + if (highThreshold == tableSize - 1) { + size_t const tableMask = tableSize-1; + size_t const step = FSE_TABLESTEP(tableSize); + /* First lay down the symbols in order. + * We use a uint64_t to lay down 8 bytes at a time. This reduces branch + * misses since small blocks generally have small table logs, so nearly + * all symbols have counts <= 8. We ensure we have 8 bytes at the end of + * our buffer to handle the over-write. + */ + { + U64 const add = 0x0101010101010101ull; + size_t pos = 0; + U64 sv = 0; + U32 s; + for (s=0; s highThreshold) position = (position + step) & tableMask; /* lowprob area */ + } } + if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { U32 u; + for (u=0; utableLog = 0; + DTableH->fastMode = 0; + + cell->newState = 0; + cell->symbol = symbolValue; + cell->nbBits = 0; + + return 0; +} + + +size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) +{ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + void* dPtr = dt + 1; + FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr; + const unsigned tableSize = 1 << nbBits; + const unsigned tableMask = tableSize - 1; + const unsigned maxSV1 = tableMask+1; + unsigned s; + + /* Sanity checks */ + if (nbBits < 1) return ERROR(GENERIC); /* min size */ + + /* Build Decoding Table */ + DTableH->tableLog = (U16)nbBits; + DTableH->fastMode = 1; + for (s=0; s sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[1] = FSE_GETSYMBOL(&state2); + + if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } } + + op[2] = FSE_GETSYMBOL(&state1); + + if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[3] = FSE_GETSYMBOL(&state2); + } + + /* tail */ + /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ + while (1) { + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + *op++ = FSE_GETSYMBOL(&state1); + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state2); + break; + } + + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + *op++ = FSE_GETSYMBOL(&state2); + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state1); + break; + } } + + return op-ostart; +} + + +size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, + const void* cSrc, size_t cSrcSize, + const FSE_DTable* dt) +{ + const void* ptr = dt; + const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; + const U32 fastMode = DTableH->fastMode; + + /* select fast mode (static) */ + if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); + return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); +} + + +size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) +{ + return FSE_decompress_wksp_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, /* bmi2 */ 0); +} + +typedef struct { + short ncount[FSE_MAX_SYMBOL_VALUE + 1]; + FSE_DTable dtable[1]; /* Dynamically sized */ +} FSE_DecompressWksp; + + +FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body( + void* dst, size_t dstCapacity, + const void* cSrc, size_t cSrcSize, + unsigned maxLog, void* workSpace, size_t wkspSize, + int bmi2) +{ + const BYTE* const istart = (const BYTE*)cSrc; + const BYTE* ip = istart; + unsigned tableLog; + unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; + FSE_DecompressWksp* const wksp = (FSE_DecompressWksp*)workSpace; + + DEBUG_STATIC_ASSERT((FSE_MAX_SYMBOL_VALUE + 1) % 2 == 0); + if (wkspSize < sizeof(*wksp)) return ERROR(GENERIC); + + /* normal FSE decoding mode */ + { + size_t const NCountLength = FSE_readNCount_bmi2(wksp->ncount, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2); + if (FSE_isError(NCountLength)) return NCountLength; + if (tableLog > maxLog) return ERROR(tableLog_tooLarge); + assert(NCountLength <= cSrcSize); + ip += NCountLength; + cSrcSize -= NCountLength; + } + + if (FSE_DECOMPRESS_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(tableLog_tooLarge); + workSpace = wksp->dtable + FSE_DTABLE_SIZE_U32(tableLog); + wkspSize -= sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog); + + CHECK_F( FSE_buildDTable_internal(wksp->dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) ); + + { + const void* ptr = wksp->dtable; + const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; + const U32 fastMode = DTableH->fastMode; + + /* select fast mode (static) */ + if (fastMode) return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 1); + return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 0); + } +} + +/* Avoids the FORCE_INLINE of the _body() function. */ +static size_t FSE_decompress_wksp_body_default(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) +{ + return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 0); +} + +#if DYNAMIC_BMI2 +TARGET_ATTRIBUTE("bmi2") static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) +{ + return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 1); +} +#endif + +size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { + return FSE_decompress_wksp_body_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); + } +#endif + (void)bmi2; + return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); +} + + +typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; + + + +#endif /* FSE_COMMONDEFS_ONLY */ diff --git a/lib/zstd/common/huf.h b/lib/zstd/common/huf.h new file mode 100644 index 000000000000..88c5586646aa --- /dev/null +++ b/lib/zstd/common/huf.h @@ -0,0 +1,356 @@ +/* ****************************************************************** + * huff0 huffman codec, + * part of Finite State Entropy library + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + + +#ifndef HUF_H_298734234 +#define HUF_H_298734234 + +/* *** Dependencies *** */ +#include "zstd_deps.h" /* size_t */ + + +/* *** library symbols visibility *** */ +/* Note : when linking with -fvisibility=hidden on gcc, or by default on Visual, + * HUF symbols remain "private" (internal symbols for library only). + * Set macro FSE_DLL_EXPORT to 1 if you want HUF symbols visible on DLL interface */ +#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4) +# define HUF_PUBLIC_API __attribute__ ((visibility ("default"))) +#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */ +# define HUF_PUBLIC_API __declspec(dllexport) +#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1) +# define HUF_PUBLIC_API __declspec(dllimport) /* not required, just to generate faster code (saves a function pointer load from IAT and an indirect jump) */ +#else +# define HUF_PUBLIC_API +#endif + + +/* ========================== */ +/* *** simple functions *** */ +/* ========================== */ + +/* HUF_compress() : + * Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'. + * 'dst' buffer must be already allocated. + * Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize). + * `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB. + * @return : size of compressed data (<= `dstCapacity`). + * Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! + * if HUF_isError(return), compression failed (more details using HUF_getErrorName()) + */ +HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +/* HUF_decompress() : + * Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', + * into already allocated buffer 'dst', of minimum size 'dstSize'. + * `originalSize` : **must** be the ***exact*** size of original (uncompressed) data. + * Note : in contrast with FSE, HUF_decompress can regenerate + * RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, + * because it knows size to regenerate (originalSize). + * @return : size of regenerated data (== originalSize), + * or an error code, which can be tested using HUF_isError() + */ +HUF_PUBLIC_API size_t HUF_decompress(void* dst, size_t originalSize, + const void* cSrc, size_t cSrcSize); + + +/* *** Tool functions *** */ +#define HUF_BLOCKSIZE_MAX (128 * 1024) /*< maximum input size for a single block compressed with HUF_compress */ +HUF_PUBLIC_API size_t HUF_compressBound(size_t size); /*< maximum compressed size (worst case) */ + +/* Error Management */ +HUF_PUBLIC_API unsigned HUF_isError(size_t code); /*< tells if a return value is an error code */ +HUF_PUBLIC_API const char* HUF_getErrorName(size_t code); /*< provides error code string (useful for debugging) */ + + +/* *** Advanced function *** */ + +/* HUF_compress2() : + * Same as HUF_compress(), but offers control over `maxSymbolValue` and `tableLog`. + * `maxSymbolValue` must be <= HUF_SYMBOLVALUE_MAX . + * `tableLog` must be `<= HUF_TABLELOG_MAX` . */ +HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned tableLog); + +/* HUF_compress4X_wksp() : + * Same as HUF_compress2(), but uses externally allocated `workSpace`. + * `workspace` must have minimum alignment of 4, and be at least as large as HUF_WORKSPACE_SIZE */ +#define HUF_WORKSPACE_SIZE ((6 << 10) + 256) +#define HUF_WORKSPACE_SIZE_U32 (HUF_WORKSPACE_SIZE / sizeof(U32)) +HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned tableLog, + void* workSpace, size_t wkspSize); + +#endif /* HUF_H_298734234 */ + +/* ****************************************************************** + * WARNING !! + * The following section contains advanced and experimental definitions + * which shall never be used in the context of a dynamic library, + * because they are not guaranteed to remain stable in the future. + * Only consider them in association with static linking. + * *****************************************************************/ +#if !defined(HUF_H_HUF_STATIC_LINKING_ONLY) +#define HUF_H_HUF_STATIC_LINKING_ONLY + +/* *** Dependencies *** */ +#include "mem.h" /* U32 */ +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" + + +/* *** Constants *** */ +#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ +#define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */ +#define HUF_SYMBOLVALUE_MAX 255 + +#define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ +#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) +# error "HUF_TABLELOG_MAX is too large !" +#endif + + +/* **************************************** +* Static allocation +******************************************/ +/* HUF buffer bounds */ +#define HUF_CTABLEBOUND 129 +#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true when incompressible is pre-filtered with fast heuristic */ +#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* static allocation of HUF's Compression Table */ +/* this is a private definition, just exposed for allocation and strict aliasing purpose. never EVER access its members directly */ +struct HUF_CElt_s { + U16 val; + BYTE nbBits; +}; /* typedef'd to HUF_CElt */ +typedef struct HUF_CElt_s HUF_CElt; /* consider it an incomplete type */ +#define HUF_CTABLE_SIZE_U32(maxSymbolValue) ((maxSymbolValue)+1) /* Use tables of U32, for proper alignment */ +#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_U32(maxSymbolValue) * sizeof(U32)) +#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ + HUF_CElt name[HUF_CTABLE_SIZE_U32(maxSymbolValue)] /* no final ; */ + +/* static allocation of HUF's DTable */ +typedef U32 HUF_DTable; +#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog))) +#define HUF_CREATE_STATIC_DTABLEX1(DTable, maxTableLog) \ + HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) } +#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \ + HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) } + + +/* **************************************** +* Advanced decompression functions +******************************************/ +size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /*< single-symbol decoder */ +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /*< double-symbols decoder */ +#endif + +size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /*< decodes RLE and uncompressed */ +size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /*< considers RLE and uncompressed as errors */ +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /*< considers RLE and uncompressed as errors */ +size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /*< single-symbol decoder */ +size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /*< single-symbol decoder */ +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /*< double-symbols decoder */ +size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /*< double-symbols decoder */ +#endif + + +/* **************************************** + * HUF detailed API + * ****************************************/ + +/*! HUF_compress() does the following: + * 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h") + * 2. (optional) refine tableLog using HUF_optimalTableLog() + * 3. build Huffman table from count using HUF_buildCTable() + * 4. save Huffman table to memory buffer using HUF_writeCTable() + * 5. encode the data stream using HUF_compress4X_usingCTable() + * + * The following API allows targeting specific sub-functions for advanced tasks. + * For example, it's possible to compress several blocks using the same 'CTable', + * or to save and regenerate 'CTable' using external methods. + */ +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); +size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */ +size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog); +size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize); +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); +size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); +int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); + +typedef enum { + HUF_repeat_none, /*< Cannot use the previous table */ + HUF_repeat_check, /*< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */ + HUF_repeat_valid /*< Can use the previous table and it is assumed to be valid */ + } HUF_repeat; +/* HUF_compress4X_repeat() : + * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. + * If it uses hufTable it does not modify hufTable or repeat. + * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. + * If preferRepeat then the old table will always be used if valid. */ +size_t HUF_compress4X_repeat(void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned tableLog, + void* workSpace, size_t wkspSize, /*< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ + HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2); + +/* HUF_buildCTable_wksp() : + * Same as HUF_buildCTable(), but using externally allocated scratch buffer. + * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE. + */ +#define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1) +#define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned)) +size_t HUF_buildCTable_wksp (HUF_CElt* tree, + const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, + void* workSpace, size_t wkspSize); + +/*! HUF_readStats() : + * Read compact Huffman tree, saved by HUF_writeCTable(). + * `huffWeight` is destination buffer. + * @return : size read from `src` , or an error Code . + * Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */ +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, + U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize); + +/*! HUF_readStats_wksp() : + * Same as HUF_readStats() but takes an external workspace which must be + * 4-byte aligned and its size must be >= HUF_READ_STATS_WORKSPACE_SIZE. + * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0. + */ +#define HUF_READ_STATS_WORKSPACE_SIZE_U32 FSE_DECOMPRESS_WKSP_SIZE_U32(6, HUF_TABLELOG_MAX-1) +#define HUF_READ_STATS_WORKSPACE_SIZE (HUF_READ_STATS_WORKSPACE_SIZE_U32 * sizeof(unsigned)) +size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, + U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workspace, size_t wkspSize, + int bmi2); + +/* HUF_readCTable() : + * Loading a CTable saved with HUF_writeCTable() */ +size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned *hasZeroWeights); + +/* HUF_getNbBits() : + * Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX + * Note 1 : is not inlined, as HUF_CElt definition is private + * Note 2 : const void* used, so that it can provide a statically allocated table as argument (which uses type U32) */ +U32 HUF_getNbBits(const void* symbolTable, U32 symbolValue); + +/* + * HUF_decompress() does the following: + * 1. select the decompression algorithm (X1, X2) based on pre-computed heuristics + * 2. build Huffman table from save, using HUF_readDTableX?() + * 3. decode 1 or 4 segments in parallel using HUF_decompress?X?_usingDTable() + */ + +/* HUF_selectDecoder() : + * Tells which decoder is likely to decode faster, + * based on a set of pre-computed metrics. + * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 . + * Assumption : 0 < dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize); + +/* + * The minimum workspace size for the `workSpace` used in + * HUF_readDTableX1_wksp() and HUF_readDTableX2_wksp(). + * + * The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when + * HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15. + * Buffer overflow errors may potentially occur if code modifications result in + * a required workspace size greater than that specified in the following + * macro. + */ +#define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9)) +#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32)) + +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize); +size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize); +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize); +size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize); +#endif + +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +#endif + + +/* ====================== */ +/* single stream variants */ +/* ====================== */ + +size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); +size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /*< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */ +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); +/* HUF_compress1X_repeat() : + * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. + * If it uses hufTable it does not modify hufTable or repeat. + * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. + * If preferRepeat then the old table will always be used if valid. */ +size_t HUF_compress1X_repeat(void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned tableLog, + void* workSpace, size_t wkspSize, /*< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ + HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2); + +size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ +#endif + +size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); +size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /*< single-symbol decoder */ +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /*< single-symbol decoder */ +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /*< double-symbols decoder */ +size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /*< double-symbols decoder */ +#endif + +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /*< automatic selection of sing or double symbol decoder, based on DTable */ +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +#endif + +/* BMI2 variants. + * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0. + */ +size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2); +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2); +#endif +size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2); +size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2); +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2); +#endif + +#endif /* HUF_STATIC_LINKING_ONLY */ + diff --git a/lib/zstd/common/mem.h b/lib/zstd/common/mem.h new file mode 100644 index 000000000000..4b5db5756a6f --- /dev/null +++ b/lib/zstd/common/mem.h @@ -0,0 +1,259 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef MEM_H_MODULE +#define MEM_H_MODULE + +/*-**************************************** +* Dependencies +******************************************/ +#include /* get_unaligned, put_unaligned* */ +#include /* inline */ +#include /* swab32, swab64 */ +#include /* size_t, ptrdiff_t */ +#include "debug.h" /* DEBUG_STATIC_ASSERT */ + +/*-**************************************** +* Compiler specifics +******************************************/ +#define MEM_STATIC static inline + +/*-************************************************************** +* Basic Types +*****************************************************************/ +typedef uint8_t BYTE; +typedef uint16_t U16; +typedef int16_t S16; +typedef uint32_t U32; +typedef int32_t S32; +typedef uint64_t U64; +typedef int64_t S64; + +/*-************************************************************** +* Memory I/O API +*****************************************************************/ +/*=== Static platform detection ===*/ +MEM_STATIC unsigned MEM_32bits(void); +MEM_STATIC unsigned MEM_64bits(void); +MEM_STATIC unsigned MEM_isLittleEndian(void); + +/*=== Native unaligned read/write ===*/ +MEM_STATIC U16 MEM_read16(const void* memPtr); +MEM_STATIC U32 MEM_read32(const void* memPtr); +MEM_STATIC U64 MEM_read64(const void* memPtr); +MEM_STATIC size_t MEM_readST(const void* memPtr); + +MEM_STATIC void MEM_write16(void* memPtr, U16 value); +MEM_STATIC void MEM_write32(void* memPtr, U32 value); +MEM_STATIC void MEM_write64(void* memPtr, U64 value); + +/*=== Little endian unaligned read/write ===*/ +MEM_STATIC U16 MEM_readLE16(const void* memPtr); +MEM_STATIC U32 MEM_readLE24(const void* memPtr); +MEM_STATIC U32 MEM_readLE32(const void* memPtr); +MEM_STATIC U64 MEM_readLE64(const void* memPtr); +MEM_STATIC size_t MEM_readLEST(const void* memPtr); + +MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val); +MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val); +MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32); +MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64); +MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val); + +/*=== Big endian unaligned read/write ===*/ +MEM_STATIC U32 MEM_readBE32(const void* memPtr); +MEM_STATIC U64 MEM_readBE64(const void* memPtr); +MEM_STATIC size_t MEM_readBEST(const void* memPtr); + +MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32); +MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64); +MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val); + +/*=== Byteswap ===*/ +MEM_STATIC U32 MEM_swap32(U32 in); +MEM_STATIC U64 MEM_swap64(U64 in); +MEM_STATIC size_t MEM_swapST(size_t in); + +/*-************************************************************** +* Memory I/O Implementation +*****************************************************************/ +MEM_STATIC unsigned MEM_32bits(void) +{ + return sizeof(size_t) == 4; +} + +MEM_STATIC unsigned MEM_64bits(void) +{ + return sizeof(size_t) == 8; +} + +#if defined(__LITTLE_ENDIAN) +#define MEM_LITTLE_ENDIAN 1 +#else +#define MEM_LITTLE_ENDIAN 0 +#endif + +MEM_STATIC unsigned MEM_isLittleEndian(void) +{ + return MEM_LITTLE_ENDIAN; +} + +MEM_STATIC U16 MEM_read16(const void *memPtr) +{ + return get_unaligned((const U16 *)memPtr); +} + +MEM_STATIC U32 MEM_read32(const void *memPtr) +{ + return get_unaligned((const U32 *)memPtr); +} + +MEM_STATIC U64 MEM_read64(const void *memPtr) +{ + return get_unaligned((const U64 *)memPtr); +} + +MEM_STATIC size_t MEM_readST(const void *memPtr) +{ + return get_unaligned((const size_t *)memPtr); +} + +MEM_STATIC void MEM_write16(void *memPtr, U16 value) +{ + put_unaligned(value, (U16 *)memPtr); +} + +MEM_STATIC void MEM_write32(void *memPtr, U32 value) +{ + put_unaligned(value, (U32 *)memPtr); +} + +MEM_STATIC void MEM_write64(void *memPtr, U64 value) +{ + put_unaligned(value, (U64 *)memPtr); +} + +/*=== Little endian r/w ===*/ + +MEM_STATIC U16 MEM_readLE16(const void *memPtr) +{ + return get_unaligned_le16(memPtr); +} + +MEM_STATIC void MEM_writeLE16(void *memPtr, U16 val) +{ + put_unaligned_le16(val, memPtr); +} + +MEM_STATIC U32 MEM_readLE24(const void *memPtr) +{ + return MEM_readLE16(memPtr) + (((const BYTE *)memPtr)[2] << 16); +} + +MEM_STATIC void MEM_writeLE24(void *memPtr, U32 val) +{ + MEM_writeLE16(memPtr, (U16)val); + ((BYTE *)memPtr)[2] = (BYTE)(val >> 16); +} + +MEM_STATIC U32 MEM_readLE32(const void *memPtr) +{ + return get_unaligned_le32(memPtr); +} + +MEM_STATIC void MEM_writeLE32(void *memPtr, U32 val32) +{ + put_unaligned_le32(val32, memPtr); +} + +MEM_STATIC U64 MEM_readLE64(const void *memPtr) +{ + return get_unaligned_le64(memPtr); +} + +MEM_STATIC void MEM_writeLE64(void *memPtr, U64 val64) +{ + put_unaligned_le64(val64, memPtr); +} + +MEM_STATIC size_t MEM_readLEST(const void *memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readLE32(memPtr); + else + return (size_t)MEM_readLE64(memPtr); +} + +MEM_STATIC void MEM_writeLEST(void *memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeLE32(memPtr, (U32)val); + else + MEM_writeLE64(memPtr, (U64)val); +} + +/*=== Big endian r/w ===*/ + +MEM_STATIC U32 MEM_readBE32(const void *memPtr) +{ + return get_unaligned_be32(memPtr); +} + +MEM_STATIC void MEM_writeBE32(void *memPtr, U32 val32) +{ + put_unaligned_be32(val32, memPtr); +} + +MEM_STATIC U64 MEM_readBE64(const void *memPtr) +{ + return get_unaligned_be64(memPtr); +} + +MEM_STATIC void MEM_writeBE64(void *memPtr, U64 val64) +{ + put_unaligned_be64(val64, memPtr); +} + +MEM_STATIC size_t MEM_readBEST(const void *memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readBE32(memPtr); + else + return (size_t)MEM_readBE64(memPtr); +} + +MEM_STATIC void MEM_writeBEST(void *memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeBE32(memPtr, (U32)val); + else + MEM_writeBE64(memPtr, (U64)val); +} + +MEM_STATIC U32 MEM_swap32(U32 in) +{ + return swab32(in); +} + +MEM_STATIC U64 MEM_swap64(U64 in) +{ + return swab64(in); +} + +MEM_STATIC size_t MEM_swapST(size_t in) +{ + if (MEM_32bits()) + return (size_t)MEM_swap32((U32)in); + else + return (size_t)MEM_swap64((U64)in); +} + +#endif /* MEM_H_MODULE */ diff --git a/lib/zstd/common/zstd_common.c b/lib/zstd/common/zstd_common.c new file mode 100644 index 000000000000..3d7e35b309b5 --- /dev/null +++ b/lib/zstd/common/zstd_common.c @@ -0,0 +1,83 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + + +/*-************************************* +* Dependencies +***************************************/ +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */ +#include "error_private.h" +#include "zstd_internal.h" + + +/*-**************************************** +* Version +******************************************/ +unsigned ZSTD_versionNumber(void) { return ZSTD_VERSION_NUMBER; } + +const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; } + + +/*-**************************************** +* ZSTD Error Management +******************************************/ +#undef ZSTD_isError /* defined within zstd_internal.h */ +/*! ZSTD_isError() : + * tells if a return value is an error code + * symbol is required for external callers */ +unsigned ZSTD_isError(size_t code) { return ERR_isError(code); } + +/*! ZSTD_getErrorName() : + * provides error code string from function result (useful for debugging) */ +const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } + +/*! ZSTD_getError() : + * convert a `size_t` function result into a proper ZSTD_errorCode enum */ +ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); } + +/*! ZSTD_getErrorString() : + * provides error code string from enum */ +const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); } + + + +/*=************************************************************** +* Custom allocator +****************************************************************/ +void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) + return customMem.customAlloc(customMem.opaque, size); + return ZSTD_malloc(size); +} + +void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) { + /* calloc implemented as malloc+memset; + * not as efficient as calloc, but next best guess for custom malloc */ + void* const ptr = customMem.customAlloc(customMem.opaque, size); + ZSTD_memset(ptr, 0, size); + return ptr; + } + return ZSTD_calloc(1, size); +} + +void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) +{ + if (ptr!=NULL) { + if (customMem.customFree) + customMem.customFree(customMem.opaque, ptr); + else + ZSTD_free(ptr); + } +} diff --git a/lib/zstd/common/zstd_deps.h b/lib/zstd/common/zstd_deps.h new file mode 100644 index 000000000000..853b72426215 --- /dev/null +++ b/lib/zstd/common/zstd_deps.h @@ -0,0 +1,125 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* + * This file provides common libc dependencies that zstd requires. + * The purpose is to allow replacing this file with a custom implementation + * to compile zstd without libc support. + */ + +/* Need: + * NULL + * INT_MAX + * UINT_MAX + * ZSTD_memcpy() + * ZSTD_memset() + * ZSTD_memmove() + */ +#ifndef ZSTD_DEPS_COMMON +#define ZSTD_DEPS_COMMON + +#include +#include + +#define ZSTD_memcpy(d,s,n) __builtin_memcpy((d),(s),(n)) +#define ZSTD_memmove(d,s,n) __builtin_memmove((d),(s),(n)) +#define ZSTD_memset(d,s,n) __builtin_memset((d),(s),(n)) + +#endif /* ZSTD_DEPS_COMMON */ + +/* + * Define malloc as always failing. That means the user must + * either use ZSTD_customMem or statically allocate memory. + * Need: + * ZSTD_malloc() + * ZSTD_free() + * ZSTD_calloc() + */ +#ifdef ZSTD_DEPS_NEED_MALLOC +#ifndef ZSTD_DEPS_MALLOC +#define ZSTD_DEPS_MALLOC + +#define ZSTD_malloc(s) ({ (void)(s); NULL; }) +#define ZSTD_free(p) ((void)(p)) +#define ZSTD_calloc(n,s) ({ (void)(n); (void)(s); NULL; }) + +#endif /* ZSTD_DEPS_MALLOC */ +#endif /* ZSTD_DEPS_NEED_MALLOC */ + +/* + * Provides 64-bit math support. + * Need: + * U64 ZSTD_div64(U64 dividend, U32 divisor) + */ +#ifdef ZSTD_DEPS_NEED_MATH64 +#ifndef ZSTD_DEPS_MATH64 +#define ZSTD_DEPS_MATH64 + +#include + +static uint64_t ZSTD_div64(uint64_t dividend, uint32_t divisor) { + return div_u64(dividend, divisor); +} + +#endif /* ZSTD_DEPS_MATH64 */ +#endif /* ZSTD_DEPS_NEED_MATH64 */ + +/* + * This is only requested when DEBUGLEVEL >= 1, meaning + * it is disabled in production. + * Need: + * assert() + */ +#ifdef ZSTD_DEPS_NEED_ASSERT +#ifndef ZSTD_DEPS_ASSERT +#define ZSTD_DEPS_ASSERT + +#include + +#define assert(x) WARN_ON((x)) + +#endif /* ZSTD_DEPS_ASSERT */ +#endif /* ZSTD_DEPS_NEED_ASSERT */ + +/* + * This is only requested when DEBUGLEVEL >= 2, meaning + * it is disabled in production. + * Need: + * ZSTD_DEBUG_PRINT() + */ +#ifdef ZSTD_DEPS_NEED_IO +#ifndef ZSTD_DEPS_IO +#define ZSTD_DEPS_IO + +#include + +#define ZSTD_DEBUG_PRINT(...) pr_debug(__VA_ARGS__) + +#endif /* ZSTD_DEPS_IO */ +#endif /* ZSTD_DEPS_NEED_IO */ + +/* + * Only requested when MSAN is enabled. + * Need: + * intptr_t + */ +#ifdef ZSTD_DEPS_NEED_STDINT +#ifndef ZSTD_DEPS_STDINT +#define ZSTD_DEPS_STDINT + +/* + * The Linux Kernel doesn't provide intptr_t, only uintptr_t, which + * is an unsigned long. + */ +typedef long intptr_t; + +#endif /* ZSTD_DEPS_STDINT */ +#endif /* ZSTD_DEPS_NEED_STDINT */ diff --git a/lib/zstd/common/zstd_internal.h b/lib/zstd/common/zstd_internal.h new file mode 100644 index 000000000000..fc6f3a9b40c0 --- /dev/null +++ b/lib/zstd/common/zstd_internal.h @@ -0,0 +1,450 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_CCOMMON_H_MODULE +#define ZSTD_CCOMMON_H_MODULE + +/* this module contains definitions which must be identical + * across compression, decompression and dictBuilder. + * It also contains a few functions useful to at least 2 of them + * and which benefit from being inlined */ + +/*-************************************* +* Dependencies +***************************************/ +#include "compiler.h" +#include "mem.h" +#include "debug.h" /* assert, DEBUGLOG, RAWLOG, g_debuglevel */ +#include "error_private.h" +#define ZSTD_STATIC_LINKING_ONLY +#include +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#include /* XXH_reset, update, digest */ +#define ZSTD_TRACE 0 + + +/* ---- static assert (debug) --- */ +#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) +#define ZSTD_isError ERR_isError /* for inlining */ +#define FSE_isError ERR_isError +#define HUF_isError ERR_isError + + +/*-************************************* +* shared macros +***************************************/ +#undef MIN +#undef MAX +#define MIN(a,b) ((a)<(b) ? (a) : (b)) +#define MAX(a,b) ((a)>(b) ? (a) : (b)) + +/* + * Ignore: this is an internal helper. + * + * This is a helper function to help force C99-correctness during compilation. + * Under strict compilation modes, variadic macro arguments can't be empty. + * However, variadic function arguments can be. Using a function therefore lets + * us statically check that at least one (string) argument was passed, + * independent of the compilation flags. + */ +static INLINE_KEYWORD UNUSED_ATTR +void _force_has_format_string(const char *format, ...) { + (void)format; +} + +/* + * Ignore: this is an internal helper. + * + * We want to force this function invocation to be syntactically correct, but + * we don't want to force runtime evaluation of its arguments. + */ +#define _FORCE_HAS_FORMAT_STRING(...) \ + if (0) { \ + _force_has_format_string(__VA_ARGS__); \ + } + +/* + * Return the specified error if the condition evaluates to true. + * + * In debug modes, prints additional information. + * In order to do that (particularly, printing the conditional that failed), + * this can't just wrap RETURN_ERROR(). + */ +#define RETURN_ERROR_IF(cond, err, ...) \ + if (cond) { \ + RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \ + __FILE__, __LINE__, ZSTD_QUOTE(cond), ZSTD_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } + +/* + * Unconditionally return the specified error. + * + * In debug modes, prints additional information. + */ +#define RETURN_ERROR(err, ...) \ + do { \ + RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \ + __FILE__, __LINE__, ZSTD_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } while(0); + +/* + * If the provided expression evaluates to an error code, returns that error code. + * + * In debug modes, prints additional information. + */ +#define FORWARD_IF_ERROR(err, ...) \ + do { \ + size_t const err_code = (err); \ + if (ERR_isError(err_code)) { \ + RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \ + __FILE__, __LINE__, ZSTD_QUOTE(err), ERR_getErrorName(err_code)); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return err_code; \ + } \ + } while(0); + + +/*-************************************* +* Common constants +***************************************/ +#define ZSTD_OPT_NUM (1<<12) + +#define ZSTD_REP_NUM 3 /* number of repcodes */ +#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1) +static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define BIT7 128 +#define BIT6 64 +#define BIT5 32 +#define BIT4 16 +#define BIT1 2 +#define BIT0 1 + +#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 +static UNUSED_ATTR const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; +static UNUSED_ATTR const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; + +#define ZSTD_FRAMEIDSIZE 4 /* magic number size */ + +#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ +static UNUSED_ATTR const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; +typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; + +#define ZSTD_FRAMECHECKSUMSIZE 4 + +#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ +#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */ + +#define HufLog 12 +typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e; + +#define LONGNBSEQ 0x7F00 + +#define MINMATCH 3 + +#define Litbits 8 +#define MaxLit ((1<= 8 || (ovtype == ZSTD_no_overlap && diff <= -WILDCOPY_VECLEN)); + + if (ovtype == ZSTD_overlap_src_before_dst && diff < WILDCOPY_VECLEN) { + /* Handle short offset copies. */ + do { + COPY8(op, ip) + } while (op < oend); + } else { + assert(diff >= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN); + /* Separate out the first COPY16() call because the copy length is + * almost certain to be short, so the branches have different + * probabilities. Since it is almost certain to be short, only do + * one COPY16() in the first call. Then, do two calls per loop since + * at that point it is more likely to have a high trip count. + */ +#ifdef __aarch64__ + do { + COPY16(op, ip); + } + while (op < oend); +#else + ZSTD_copy16(op, ip); + if (16 >= length) return; + op += 16; + ip += 16; + do { + COPY16(op, ip); + COPY16(op, ip); + } + while (op < oend); +#endif + } +} + +MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + if (length > 0) { + ZSTD_memcpy(dst, src, length); + } + return length; +} + +/* define "workspace is too large" as this number of times larger than needed */ +#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 + +/* when workspace is continuously too large + * during at least this number of times, + * context's memory usage is considered wasteful, + * because it's sized to handle a worst case scenario which rarely happens. + * In which case, resize it down to free some memory */ +#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 + +/* Controls whether the input/output buffer is buffered or stable. */ +typedef enum { + ZSTD_bm_buffered = 0, /* Buffer the input/output */ + ZSTD_bm_stable = 1 /* ZSTD_inBuffer/ZSTD_outBuffer is stable */ +} ZSTD_bufferMode_e; + + +/*-******************************************* +* Private declarations +*********************************************/ +typedef struct seqDef_s { + U32 offset; /* Offset code of the sequence */ + U16 litLength; + U16 matchLength; +} seqDef; + +typedef struct { + seqDef* sequencesStart; + seqDef* sequences; /* ptr to end of sequences */ + BYTE* litStart; + BYTE* lit; /* ptr to end of literals */ + BYTE* llCode; + BYTE* mlCode; + BYTE* ofCode; + size_t maxNbSeq; + size_t maxNbLit; + + /* longLengthPos and longLengthID to allow us to represent either a single litLength or matchLength + * in the seqStore that has a value larger than U16 (if it exists). To do so, we increment + * the existing value of the litLength or matchLength by 0x10000. + */ + U32 longLengthID; /* 0 == no longLength; 1 == Represent the long literal; 2 == Represent the long match; */ + U32 longLengthPos; /* Index of the sequence to apply long length modification to */ +} seqStore_t; + +typedef struct { + U32 litLength; + U32 matchLength; +} ZSTD_sequenceLength; + +/* + * Returns the ZSTD_sequenceLength for the given sequences. It handles the decoding of long sequences + * indicated by longLengthPos and longLengthID, and adds MINMATCH back to matchLength. + */ +MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore, seqDef const* seq) +{ + ZSTD_sequenceLength seqLen; + seqLen.litLength = seq->litLength; + seqLen.matchLength = seq->matchLength + MINMATCH; + if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) { + if (seqStore->longLengthID == 1) { + seqLen.litLength += 0xFFFF; + } + if (seqStore->longLengthID == 2) { + seqLen.matchLength += 0xFFFF; + } + } + return seqLen; +} + +/* + * Contains the compressed frame size and an upper-bound for the decompressed frame size. + * Note: before using `compressedSize`, check for errors using ZSTD_isError(). + * similarly, before using `decompressedBound`, check for errors using: + * `decompressedBound != ZSTD_CONTENTSIZE_ERROR` + */ +typedef struct { + size_t compressedSize; + unsigned long long decompressedBound; +} ZSTD_frameSizeInfo; /* decompress & legacy */ + +const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */ +void ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */ + +/* custom memory allocation functions */ +void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem); +void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem); +void ZSTD_customFree(void* ptr, ZSTD_customMem customMem); + + +MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */ +{ + assert(val != 0); + { +# if (__GNUC__ >= 3) /* GCC Intrinsic */ + return __builtin_clz (val) ^ 31; +# else /* Software version */ + static const U32 DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; + U32 v = val; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + return DeBruijnClz[(v * 0x07C4ACDDU) >> 27]; +# endif + } +} + + +/* ZSTD_invalidateRepCodes() : + * ensures next compression will not use repcodes from previous block. + * Note : only works with regular variant; + * do not use with extDict variant ! */ +void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); /* zstdmt, adaptive_compression (shouldn't get this definition from here) */ + + +typedef struct { + blockType_e blockType; + U32 lastBlock; + U32 origSize; +} blockProperties_t; /* declared here for decompress and fullbench */ + +/*! ZSTD_getcBlockSize() : + * Provides the size of compressed block from block header `src` */ +/* Used by: decompress, fullbench (does not get its definition from here) */ +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, + blockProperties_t* bpPtr); + +/*! ZSTD_decodeSeqHeaders() : + * decode sequence header from src */ +/* Used by: decompress, fullbench (does not get its definition from here) */ +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, + const void* src, size_t srcSize); + + + +#endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/lib/zstd/compress.c b/lib/zstd/compress.c deleted file mode 100644 index b080264ed3ad..000000000000 --- a/lib/zstd/compress.c +++ /dev/null @@ -1,3485 +0,0 @@ -/** - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of https://github.com/facebook/zstd. - * An additional grant of patent rights can be found in the PATENTS file in the - * same directory. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - */ - -/*-************************************* -* Dependencies -***************************************/ -#include "fse.h" -#include "huf.h" -#include "mem.h" -#include "zstd_internal.h" /* includes zstd.h */ -#include -#include -#include /* memset */ - -/*-************************************* -* Constants -***************************************/ -static const U32 g_searchStrength = 8; /* control skip over incompressible data */ -#define HASH_READ_SIZE 8 -typedef enum { ZSTDcs_created = 0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e; - -/*-************************************* -* Helper functions -***************************************/ -size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + 12; } - -/*-************************************* -* Sequence storage -***************************************/ -static void ZSTD_resetSeqStore(seqStore_t *ssPtr) -{ - ssPtr->lit = ssPtr->litStart; - ssPtr->sequences = ssPtr->sequencesStart; - ssPtr->longLengthID = 0; -} - -/*-************************************* -* Context memory management -***************************************/ -struct ZSTD_CCtx_s { - const BYTE *nextSrc; /* next block here to continue on curr prefix */ - const BYTE *base; /* All regular indexes relative to this position */ - const BYTE *dictBase; /* extDict indexes relative to this position */ - U32 dictLimit; /* below that point, need extDict */ - U32 lowLimit; /* below that point, no more data */ - U32 nextToUpdate; /* index from which to continue dictionary update */ - U32 nextToUpdate3; /* index from which to continue dictionary update */ - U32 hashLog3; /* dispatch table : larger == faster, more memory */ - U32 loadedDictEnd; /* index of end of dictionary */ - U32 forceWindow; /* force back-references to respect limit of 1< 3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, cParams.windowLog); - size_t const h3Size = ((size_t)1) << hashLog3; - size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); - size_t const optSpace = - ((MaxML + 1) + (MaxLL + 1) + (MaxOff + 1) + (1 << Litbits)) * sizeof(U32) + (ZSTD_OPT_NUM + 1) * (sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t)); - size_t const workspaceSize = tableSpace + (256 * sizeof(U32)) /* huffTable */ + tokenSpace + - (((cParams.strategy == ZSTD_btopt) || (cParams.strategy == ZSTD_btopt2)) ? optSpace : 0); - - return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_CCtx)) + ZSTD_ALIGN(workspaceSize); -} - -static ZSTD_CCtx *ZSTD_createCCtx_advanced(ZSTD_customMem customMem) -{ - ZSTD_CCtx *cctx; - if (!customMem.customAlloc || !customMem.customFree) - return NULL; - cctx = (ZSTD_CCtx *)ZSTD_malloc(sizeof(ZSTD_CCtx), customMem); - if (!cctx) - return NULL; - memset(cctx, 0, sizeof(ZSTD_CCtx)); - cctx->customMem = customMem; - return cctx; -} - -ZSTD_CCtx *ZSTD_initCCtx(void *workspace, size_t workspaceSize) -{ - ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize); - ZSTD_CCtx *cctx = ZSTD_createCCtx_advanced(stackMem); - if (cctx) { - cctx->workSpace = ZSTD_stackAllocAll(cctx->customMem.opaque, &cctx->workSpaceSize); - } - return cctx; -} - -size_t ZSTD_freeCCtx(ZSTD_CCtx *cctx) -{ - if (cctx == NULL) - return 0; /* support free on NULL */ - ZSTD_free(cctx->workSpace, cctx->customMem); - ZSTD_free(cctx, cctx->customMem); - return 0; /* reserved as a potential error code in the future */ -} - -const seqStore_t *ZSTD_getSeqStore(const ZSTD_CCtx *ctx) /* hidden interface */ { return &(ctx->seqStore); } - -static ZSTD_parameters ZSTD_getParamsFromCCtx(const ZSTD_CCtx *cctx) { return cctx->params; } - -/** ZSTD_checkParams() : - ensure param values remain within authorized range. - @return : 0, or an error code if one value is beyond authorized range */ -size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) -{ -#define CLAMPCHECK(val, min, max) \ - { \ - if ((val < min) | (val > max)) \ - return ERROR(compressionParameter_unsupported); \ - } - CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); - CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); - CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); - CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); - CLAMPCHECK(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); - CLAMPCHECK(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX); - if ((U32)(cParams.strategy) > (U32)ZSTD_btopt2) - return ERROR(compressionParameter_unsupported); - return 0; -} - -/** ZSTD_cycleLog() : - * condition for correct operation : hashLog > 1 */ -static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) -{ - U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2); - return hashLog - btScale; -} - -/** ZSTD_adjustCParams() : - optimize `cPar` for a given input (`srcSize` and `dictSize`). - mostly downsizing to reduce memory consumption and initialization. - Both `srcSize` and `dictSize` are optional (use 0 if unknown), - but if both are 0, no optimization can be done. - Note : cPar is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */ -ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize) -{ - if (srcSize + dictSize == 0) - return cPar; /* no size information available : no adjustment */ - - /* resize params, to use less memory when necessary */ - { - U32 const minSrcSize = (srcSize == 0) ? 500 : 0; - U64 const rSize = srcSize + dictSize + minSrcSize; - if (rSize < ((U64)1 << ZSTD_WINDOWLOG_MAX)) { - U32 const srcLog = MAX(ZSTD_HASHLOG_MIN, ZSTD_highbit32((U32)(rSize)-1) + 1); - if (cPar.windowLog > srcLog) - cPar.windowLog = srcLog; - } - } - if (cPar.hashLog > cPar.windowLog) - cPar.hashLog = cPar.windowLog; - { - U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy); - if (cycleLog > cPar.windowLog) - cPar.chainLog -= (cycleLog - cPar.windowLog); - } - - if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) - cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */ - - return cPar; -} - -static U32 ZSTD_equivalentParams(ZSTD_parameters param1, ZSTD_parameters param2) -{ - return (param1.cParams.hashLog == param2.cParams.hashLog) & (param1.cParams.chainLog == param2.cParams.chainLog) & - (param1.cParams.strategy == param2.cParams.strategy) & ((param1.cParams.searchLength == 3) == (param2.cParams.searchLength == 3)); -} - -/*! ZSTD_continueCCtx() : - reuse CCtx without reset (note : requires no dictionary) */ -static size_t ZSTD_continueCCtx(ZSTD_CCtx *cctx, ZSTD_parameters params, U64 frameContentSize) -{ - U32 const end = (U32)(cctx->nextSrc - cctx->base); - cctx->params = params; - cctx->frameContentSize = frameContentSize; - cctx->lowLimit = end; - cctx->dictLimit = end; - cctx->nextToUpdate = end + 1; - cctx->stage = ZSTDcs_init; - cctx->dictID = 0; - cctx->loadedDictEnd = 0; - { - int i; - for (i = 0; i < ZSTD_REP_NUM; i++) - cctx->rep[i] = repStartValue[i]; - } - cctx->seqStore.litLengthSum = 0; /* force reset of btopt stats */ - xxh64_reset(&cctx->xxhState, 0); - return 0; -} - -typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_fullReset } ZSTD_compResetPolicy_e; - -/*! ZSTD_resetCCtx_advanced() : - note : `params` must be validated */ -static size_t ZSTD_resetCCtx_advanced(ZSTD_CCtx *zc, ZSTD_parameters params, U64 frameContentSize, ZSTD_compResetPolicy_e const crp) -{ - if (crp == ZSTDcrp_continue) - if (ZSTD_equivalentParams(params, zc->params)) { - zc->flagStaticTables = 0; - zc->flagStaticHufTable = HUF_repeat_none; - return ZSTD_continueCCtx(zc, params, frameContentSize); - } - - { - size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog); - U32 const divider = (params.cParams.searchLength == 3) ? 3 : 4; - size_t const maxNbSeq = blockSize / divider; - size_t const tokenSpace = blockSize + 11 * maxNbSeq; - size_t const chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog); - size_t const hSize = ((size_t)1) << params.cParams.hashLog; - U32 const hashLog3 = (params.cParams.searchLength > 3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, params.cParams.windowLog); - size_t const h3Size = ((size_t)1) << hashLog3; - size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); - void *ptr; - - /* Check if workSpace is large enough, alloc a new one if needed */ - { - size_t const optSpace = ((MaxML + 1) + (MaxLL + 1) + (MaxOff + 1) + (1 << Litbits)) * sizeof(U32) + - (ZSTD_OPT_NUM + 1) * (sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t)); - size_t const neededSpace = tableSpace + (256 * sizeof(U32)) /* huffTable */ + tokenSpace + - (((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) ? optSpace : 0); - if (zc->workSpaceSize < neededSpace) { - ZSTD_free(zc->workSpace, zc->customMem); - zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem); - if (zc->workSpace == NULL) - return ERROR(memory_allocation); - zc->workSpaceSize = neededSpace; - } - } - - if (crp != ZSTDcrp_noMemset) - memset(zc->workSpace, 0, tableSpace); /* reset tables only */ - xxh64_reset(&zc->xxhState, 0); - zc->hashLog3 = hashLog3; - zc->hashTable = (U32 *)(zc->workSpace); - zc->chainTable = zc->hashTable + hSize; - zc->hashTable3 = zc->chainTable + chainSize; - ptr = zc->hashTable3 + h3Size; - zc->hufTable = (HUF_CElt *)ptr; - zc->flagStaticTables = 0; - zc->flagStaticHufTable = HUF_repeat_none; - ptr = ((U32 *)ptr) + 256; /* note : HUF_CElt* is incomplete type, size is simulated using U32 */ - - zc->nextToUpdate = 1; - zc->nextSrc = NULL; - zc->base = NULL; - zc->dictBase = NULL; - zc->dictLimit = 0; - zc->lowLimit = 0; - zc->params = params; - zc->blockSize = blockSize; - zc->frameContentSize = frameContentSize; - { - int i; - for (i = 0; i < ZSTD_REP_NUM; i++) - zc->rep[i] = repStartValue[i]; - } - - if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) { - zc->seqStore.litFreq = (U32 *)ptr; - zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1 << Litbits); - zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL + 1); - zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (MaxML + 1); - ptr = zc->seqStore.offCodeFreq + (MaxOff + 1); - zc->seqStore.matchTable = (ZSTD_match_t *)ptr; - ptr = zc->seqStore.matchTable + ZSTD_OPT_NUM + 1; - zc->seqStore.priceTable = (ZSTD_optimal_t *)ptr; - ptr = zc->seqStore.priceTable + ZSTD_OPT_NUM + 1; - zc->seqStore.litLengthSum = 0; - } - zc->seqStore.sequencesStart = (seqDef *)ptr; - ptr = zc->seqStore.sequencesStart + maxNbSeq; - zc->seqStore.llCode = (BYTE *)ptr; - zc->seqStore.mlCode = zc->seqStore.llCode + maxNbSeq; - zc->seqStore.ofCode = zc->seqStore.mlCode + maxNbSeq; - zc->seqStore.litStart = zc->seqStore.ofCode + maxNbSeq; - - zc->stage = ZSTDcs_init; - zc->dictID = 0; - zc->loadedDictEnd = 0; - - return 0; - } -} - -/* ZSTD_invalidateRepCodes() : - * ensures next compression will not use repcodes from previous block. - * Note : only works with regular variant; - * do not use with extDict variant ! */ -void ZSTD_invalidateRepCodes(ZSTD_CCtx *cctx) -{ - int i; - for (i = 0; i < ZSTD_REP_NUM; i++) - cctx->rep[i] = 0; -} - -/*! ZSTD_copyCCtx() : -* Duplicate an existing context `srcCCtx` into another one `dstCCtx`. -* Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). -* @return : 0, or an error code */ -size_t ZSTD_copyCCtx(ZSTD_CCtx *dstCCtx, const ZSTD_CCtx *srcCCtx, unsigned long long pledgedSrcSize) -{ - if (srcCCtx->stage != ZSTDcs_init) - return ERROR(stage_wrong); - - memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); - { - ZSTD_parameters params = srcCCtx->params; - params.fParams.contentSizeFlag = (pledgedSrcSize > 0); - ZSTD_resetCCtx_advanced(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset); - } - - /* copy tables */ - { - size_t const chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog); - size_t const hSize = ((size_t)1) << srcCCtx->params.cParams.hashLog; - size_t const h3Size = (size_t)1 << srcCCtx->hashLog3; - size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); - memcpy(dstCCtx->workSpace, srcCCtx->workSpace, tableSpace); - } - - /* copy dictionary offsets */ - dstCCtx->nextToUpdate = srcCCtx->nextToUpdate; - dstCCtx->nextToUpdate3 = srcCCtx->nextToUpdate3; - dstCCtx->nextSrc = srcCCtx->nextSrc; - dstCCtx->base = srcCCtx->base; - dstCCtx->dictBase = srcCCtx->dictBase; - dstCCtx->dictLimit = srcCCtx->dictLimit; - dstCCtx->lowLimit = srcCCtx->lowLimit; - dstCCtx->loadedDictEnd = srcCCtx->loadedDictEnd; - dstCCtx->dictID = srcCCtx->dictID; - - /* copy entropy tables */ - dstCCtx->flagStaticTables = srcCCtx->flagStaticTables; - dstCCtx->flagStaticHufTable = srcCCtx->flagStaticHufTable; - if (srcCCtx->flagStaticTables) { - memcpy(dstCCtx->litlengthCTable, srcCCtx->litlengthCTable, sizeof(dstCCtx->litlengthCTable)); - memcpy(dstCCtx->matchlengthCTable, srcCCtx->matchlengthCTable, sizeof(dstCCtx->matchlengthCTable)); - memcpy(dstCCtx->offcodeCTable, srcCCtx->offcodeCTable, sizeof(dstCCtx->offcodeCTable)); - } - if (srcCCtx->flagStaticHufTable) { - memcpy(dstCCtx->hufTable, srcCCtx->hufTable, 256 * 4); - } - - return 0; -} - -/*! ZSTD_reduceTable() : -* reduce table indexes by `reducerValue` */ -static void ZSTD_reduceTable(U32 *const table, U32 const size, U32 const reducerValue) -{ - U32 u; - for (u = 0; u < size; u++) { - if (table[u] < reducerValue) - table[u] = 0; - else - table[u] -= reducerValue; - } -} - -/*! ZSTD_reduceIndex() : -* rescale all indexes to avoid future overflow (indexes are U32) */ -static void ZSTD_reduceIndex(ZSTD_CCtx *zc, const U32 reducerValue) -{ - { - U32 const hSize = 1 << zc->params.cParams.hashLog; - ZSTD_reduceTable(zc->hashTable, hSize, reducerValue); - } - - { - U32 const chainSize = (zc->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << zc->params.cParams.chainLog); - ZSTD_reduceTable(zc->chainTable, chainSize, reducerValue); - } - - { - U32 const h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0; - ZSTD_reduceTable(zc->hashTable3, h3Size, reducerValue); - } -} - -/*-******************************************************* -* Block entropic compression -*********************************************************/ - -/* See doc/zstd_compression_format.md for detailed format description */ - -size_t ZSTD_noCompressBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - if (srcSize + ZSTD_blockHeaderSize > dstCapacity) - return ERROR(dstSize_tooSmall); - memcpy((BYTE *)dst + ZSTD_blockHeaderSize, src, srcSize); - ZSTD_writeLE24(dst, (U32)(srcSize << 2) + (U32)bt_raw); - return ZSTD_blockHeaderSize + srcSize; -} - -static size_t ZSTD_noCompressLiterals(void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - BYTE *const ostart = (BYTE * const)dst; - U32 const flSize = 1 + (srcSize > 31) + (srcSize > 4095); - - if (srcSize + flSize > dstCapacity) - return ERROR(dstSize_tooSmall); - - switch (flSize) { - case 1: /* 2 - 1 - 5 */ ostart[0] = (BYTE)((U32)set_basic + (srcSize << 3)); break; - case 2: /* 2 - 2 - 12 */ ZSTD_writeLE16(ostart, (U16)((U32)set_basic + (1 << 2) + (srcSize << 4))); break; - default: /*note : should not be necessary : flSize is within {1,2,3} */ - case 3: /* 2 - 2 - 20 */ ZSTD_writeLE32(ostart, (U32)((U32)set_basic + (3 << 2) + (srcSize << 4))); break; - } - - memcpy(ostart + flSize, src, srcSize); - return srcSize + flSize; -} - -static size_t ZSTD_compressRleLiteralsBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - BYTE *const ostart = (BYTE * const)dst; - U32 const flSize = 1 + (srcSize > 31) + (srcSize > 4095); - - (void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */ - - switch (flSize) { - case 1: /* 2 - 1 - 5 */ ostart[0] = (BYTE)((U32)set_rle + (srcSize << 3)); break; - case 2: /* 2 - 2 - 12 */ ZSTD_writeLE16(ostart, (U16)((U32)set_rle + (1 << 2) + (srcSize << 4))); break; - default: /*note : should not be necessary : flSize is necessarily within {1,2,3} */ - case 3: /* 2 - 2 - 20 */ ZSTD_writeLE32(ostart, (U32)((U32)set_rle + (3 << 2) + (srcSize << 4))); break; - } - - ostart[flSize] = *(const BYTE *)src; - return flSize + 1; -} - -static size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 2; } - -static size_t ZSTD_compressLiterals(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - size_t const minGain = ZSTD_minGain(srcSize); - size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB); - BYTE *const ostart = (BYTE *)dst; - U32 singleStream = srcSize < 256; - symbolEncodingType_e hType = set_compressed; - size_t cLitSize; - -/* small ? don't even attempt compression (speed opt) */ -#define LITERAL_NOENTROPY 63 - { - size_t const minLitSize = zc->flagStaticHufTable == HUF_repeat_valid ? 6 : LITERAL_NOENTROPY; - if (srcSize <= minLitSize) - return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); - } - - if (dstCapacity < lhSize + 1) - return ERROR(dstSize_tooSmall); /* not enough space for compression */ - { - HUF_repeat repeat = zc->flagStaticHufTable; - int const preferRepeat = zc->params.cParams.strategy < ZSTD_lazy ? srcSize <= 1024 : 0; - if (repeat == HUF_repeat_valid && lhSize == 3) - singleStream = 1; - cLitSize = singleStream ? HUF_compress1X_repeat(ostart + lhSize, dstCapacity - lhSize, src, srcSize, 255, 11, zc->tmpCounters, - sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat) - : HUF_compress4X_repeat(ostart + lhSize, dstCapacity - lhSize, src, srcSize, 255, 11, zc->tmpCounters, - sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat); - if (repeat != HUF_repeat_none) { - hType = set_repeat; - } /* reused the existing table */ - else { - zc->flagStaticHufTable = HUF_repeat_check; - } /* now have a table to reuse */ - } - - if ((cLitSize == 0) | (cLitSize >= srcSize - minGain)) { - zc->flagStaticHufTable = HUF_repeat_none; - return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); - } - if (cLitSize == 1) { - zc->flagStaticHufTable = HUF_repeat_none; - return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); - } - - /* Build header */ - switch (lhSize) { - case 3: /* 2 - 2 - 10 - 10 */ - { - U32 const lhc = hType + ((!singleStream) << 2) + ((U32)srcSize << 4) + ((U32)cLitSize << 14); - ZSTD_writeLE24(ostart, lhc); - break; - } - case 4: /* 2 - 2 - 14 - 14 */ - { - U32 const lhc = hType + (2 << 2) + ((U32)srcSize << 4) + ((U32)cLitSize << 18); - ZSTD_writeLE32(ostart, lhc); - break; - } - default: /* should not be necessary, lhSize is only {3,4,5} */ - case 5: /* 2 - 2 - 18 - 18 */ - { - U32 const lhc = hType + (3 << 2) + ((U32)srcSize << 4) + ((U32)cLitSize << 22); - ZSTD_writeLE32(ostart, lhc); - ostart[4] = (BYTE)(cLitSize >> 10); - break; - } - } - return lhSize + cLitSize; -} - -static const BYTE LL_Code[64] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, 17, 17, 18, 18, - 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, - 23, 23, 23, 23, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24}; - -static const BYTE ML_Code[128] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, - 26, 27, 28, 29, 30, 31, 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, 38, 38, 38, 38, - 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, - 40, 40, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 42, 42, 42, 42, 42, 42, 42, 42, - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42}; - -void ZSTD_seqToCodes(const seqStore_t *seqStorePtr) -{ - BYTE const LL_deltaCode = 19; - BYTE const ML_deltaCode = 36; - const seqDef *const sequences = seqStorePtr->sequencesStart; - BYTE *const llCodeTable = seqStorePtr->llCode; - BYTE *const ofCodeTable = seqStorePtr->ofCode; - BYTE *const mlCodeTable = seqStorePtr->mlCode; - U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); - U32 u; - for (u = 0; u < nbSeq; u++) { - U32 const llv = sequences[u].litLength; - U32 const mlv = sequences[u].matchLength; - llCodeTable[u] = (llv > 63) ? (BYTE)ZSTD_highbit32(llv) + LL_deltaCode : LL_Code[llv]; - ofCodeTable[u] = (BYTE)ZSTD_highbit32(sequences[u].offset); - mlCodeTable[u] = (mlv > 127) ? (BYTE)ZSTD_highbit32(mlv) + ML_deltaCode : ML_Code[mlv]; - } - if (seqStorePtr->longLengthID == 1) - llCodeTable[seqStorePtr->longLengthPos] = MaxLL; - if (seqStorePtr->longLengthID == 2) - mlCodeTable[seqStorePtr->longLengthPos] = MaxML; -} - -ZSTD_STATIC size_t ZSTD_compressSequences_internal(ZSTD_CCtx *zc, void *dst, size_t dstCapacity) -{ - const int longOffsets = zc->params.cParams.windowLog > STREAM_ACCUMULATOR_MIN; - const seqStore_t *seqStorePtr = &(zc->seqStore); - FSE_CTable *CTable_LitLength = zc->litlengthCTable; - FSE_CTable *CTable_OffsetBits = zc->offcodeCTable; - FSE_CTable *CTable_MatchLength = zc->matchlengthCTable; - U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */ - const seqDef *const sequences = seqStorePtr->sequencesStart; - const BYTE *const ofCodeTable = seqStorePtr->ofCode; - const BYTE *const llCodeTable = seqStorePtr->llCode; - const BYTE *const mlCodeTable = seqStorePtr->mlCode; - BYTE *const ostart = (BYTE *)dst; - BYTE *const oend = ostart + dstCapacity; - BYTE *op = ostart; - size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart; - BYTE *seqHead; - - U32 *count; - S16 *norm; - U32 *workspace; - size_t workspaceSize = sizeof(zc->tmpCounters); - { - size_t spaceUsed32 = 0; - count = (U32 *)zc->tmpCounters + spaceUsed32; - spaceUsed32 += MaxSeq + 1; - norm = (S16 *)((U32 *)zc->tmpCounters + spaceUsed32); - spaceUsed32 += ALIGN(sizeof(S16) * (MaxSeq + 1), sizeof(U32)) >> 2; - - workspace = (U32 *)zc->tmpCounters + spaceUsed32; - workspaceSize -= (spaceUsed32 << 2); - } - - /* Compress literals */ - { - const BYTE *const literals = seqStorePtr->litStart; - size_t const litSize = seqStorePtr->lit - literals; - size_t const cSize = ZSTD_compressLiterals(zc, op, dstCapacity, literals, litSize); - if (ZSTD_isError(cSize)) - return cSize; - op += cSize; - } - - /* Sequences Header */ - if ((oend - op) < 3 /*max nbSeq Size*/ + 1 /*seqHead */) - return ERROR(dstSize_tooSmall); - if (nbSeq < 0x7F) - *op++ = (BYTE)nbSeq; - else if (nbSeq < LONGNBSEQ) - op[0] = (BYTE)((nbSeq >> 8) + 0x80), op[1] = (BYTE)nbSeq, op += 2; - else - op[0] = 0xFF, ZSTD_writeLE16(op + 1, (U16)(nbSeq - LONGNBSEQ)), op += 3; - if (nbSeq == 0) - return op - ostart; - - /* seqHead : flags for FSE encoding type */ - seqHead = op++; - -#define MIN_SEQ_FOR_DYNAMIC_FSE 64 -#define MAX_SEQ_FOR_STATIC_FSE 1000 - - /* convert length/distances into codes */ - ZSTD_seqToCodes(seqStorePtr); - - /* CTable for Literal Lengths */ - { - U32 max = MaxLL; - size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, workspace); - if ((mostFrequent == nbSeq) && (nbSeq > 2)) { - *op++ = llCodeTable[0]; - FSE_buildCTable_rle(CTable_LitLength, (BYTE)max); - LLtype = set_rle; - } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { - LLtype = set_repeat; - } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (LL_defaultNormLog - 1)))) { - FSE_buildCTable_wksp(CTable_LitLength, LL_defaultNorm, MaxLL, LL_defaultNormLog, workspace, workspaceSize); - LLtype = set_basic; - } else { - size_t nbSeq_1 = nbSeq; - const U32 tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max); - if (count[llCodeTable[nbSeq - 1]] > 1) { - count[llCodeTable[nbSeq - 1]]--; - nbSeq_1--; - } - FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); - { - size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */ - if (FSE_isError(NCountSize)) - return NCountSize; - op += NCountSize; - } - FSE_buildCTable_wksp(CTable_LitLength, norm, max, tableLog, workspace, workspaceSize); - LLtype = set_compressed; - } - } - - /* CTable for Offsets */ - { - U32 max = MaxOff; - size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, workspace); - if ((mostFrequent == nbSeq) && (nbSeq > 2)) { - *op++ = ofCodeTable[0]; - FSE_buildCTable_rle(CTable_OffsetBits, (BYTE)max); - Offtype = set_rle; - } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { - Offtype = set_repeat; - } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (OF_defaultNormLog - 1)))) { - FSE_buildCTable_wksp(CTable_OffsetBits, OF_defaultNorm, MaxOff, OF_defaultNormLog, workspace, workspaceSize); - Offtype = set_basic; - } else { - size_t nbSeq_1 = nbSeq; - const U32 tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max); - if (count[ofCodeTable[nbSeq - 1]] > 1) { - count[ofCodeTable[nbSeq - 1]]--; - nbSeq_1--; - } - FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); - { - size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */ - if (FSE_isError(NCountSize)) - return NCountSize; - op += NCountSize; - } - FSE_buildCTable_wksp(CTable_OffsetBits, norm, max, tableLog, workspace, workspaceSize); - Offtype = set_compressed; - } - } - - /* CTable for MatchLengths */ - { - U32 max = MaxML; - size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, workspace); - if ((mostFrequent == nbSeq) && (nbSeq > 2)) { - *op++ = *mlCodeTable; - FSE_buildCTable_rle(CTable_MatchLength, (BYTE)max); - MLtype = set_rle; - } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { - MLtype = set_repeat; - } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (ML_defaultNormLog - 1)))) { - FSE_buildCTable_wksp(CTable_MatchLength, ML_defaultNorm, MaxML, ML_defaultNormLog, workspace, workspaceSize); - MLtype = set_basic; - } else { - size_t nbSeq_1 = nbSeq; - const U32 tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max); - if (count[mlCodeTable[nbSeq - 1]] > 1) { - count[mlCodeTable[nbSeq - 1]]--; - nbSeq_1--; - } - FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); - { - size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */ - if (FSE_isError(NCountSize)) - return NCountSize; - op += NCountSize; - } - FSE_buildCTable_wksp(CTable_MatchLength, norm, max, tableLog, workspace, workspaceSize); - MLtype = set_compressed; - } - } - - *seqHead = (BYTE)((LLtype << 6) + (Offtype << 4) + (MLtype << 2)); - zc->flagStaticTables = 0; - - /* Encoding Sequences */ - { - BIT_CStream_t blockStream; - FSE_CState_t stateMatchLength; - FSE_CState_t stateOffsetBits; - FSE_CState_t stateLitLength; - - CHECK_E(BIT_initCStream(&blockStream, op, oend - op), dstSize_tooSmall); /* not enough space remaining */ - - /* first symbols */ - FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq - 1]); - FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq - 1]); - FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq - 1]); - BIT_addBits(&blockStream, sequences[nbSeq - 1].litLength, LL_bits[llCodeTable[nbSeq - 1]]); - if (ZSTD_32bits()) - BIT_flushBits(&blockStream); - BIT_addBits(&blockStream, sequences[nbSeq - 1].matchLength, ML_bits[mlCodeTable[nbSeq - 1]]); - if (ZSTD_32bits()) - BIT_flushBits(&blockStream); - if (longOffsets) { - U32 const ofBits = ofCodeTable[nbSeq - 1]; - int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN - 1); - if (extraBits) { - BIT_addBits(&blockStream, sequences[nbSeq - 1].offset, extraBits); - BIT_flushBits(&blockStream); - } - BIT_addBits(&blockStream, sequences[nbSeq - 1].offset >> extraBits, ofBits - extraBits); - } else { - BIT_addBits(&blockStream, sequences[nbSeq - 1].offset, ofCodeTable[nbSeq - 1]); - } - BIT_flushBits(&blockStream); - - { - size_t n; - for (n = nbSeq - 2; n < nbSeq; n--) { /* intentional underflow */ - BYTE const llCode = llCodeTable[n]; - BYTE const ofCode = ofCodeTable[n]; - BYTE const mlCode = mlCodeTable[n]; - U32 const llBits = LL_bits[llCode]; - U32 const ofBits = ofCode; /* 32b*/ /* 64b*/ - U32 const mlBits = ML_bits[mlCode]; - /* (7)*/ /* (7)*/ - FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 15 */ /* 15 */ - FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 24 */ /* 24 */ - if (ZSTD_32bits()) - BIT_flushBits(&blockStream); /* (7)*/ - FSE_encodeSymbol(&blockStream, &stateLitLength, llCode); /* 16 */ /* 33 */ - if (ZSTD_32bits() || (ofBits + mlBits + llBits >= 64 - 7 - (LLFSELog + MLFSELog + OffFSELog))) - BIT_flushBits(&blockStream); /* (7)*/ - BIT_addBits(&blockStream, sequences[n].litLength, llBits); - if (ZSTD_32bits() && ((llBits + mlBits) > 24)) - BIT_flushBits(&blockStream); - BIT_addBits(&blockStream, sequences[n].matchLength, mlBits); - if (ZSTD_32bits()) - BIT_flushBits(&blockStream); /* (7)*/ - if (longOffsets) { - int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN - 1); - if (extraBits) { - BIT_addBits(&blockStream, sequences[n].offset, extraBits); - BIT_flushBits(&blockStream); /* (7)*/ - } - BIT_addBits(&blockStream, sequences[n].offset >> extraBits, ofBits - extraBits); /* 31 */ - } else { - BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */ - } - BIT_flushBits(&blockStream); /* (7)*/ - } - } - - FSE_flushCState(&blockStream, &stateMatchLength); - FSE_flushCState(&blockStream, &stateOffsetBits); - FSE_flushCState(&blockStream, &stateLitLength); - - { - size_t const streamSize = BIT_closeCStream(&blockStream); - if (streamSize == 0) - return ERROR(dstSize_tooSmall); /* not enough space */ - op += streamSize; - } - } - return op - ostart; -} - -ZSTD_STATIC size_t ZSTD_compressSequences(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, size_t srcSize) -{ - size_t const cSize = ZSTD_compressSequences_internal(zc, dst, dstCapacity); - size_t const minGain = ZSTD_minGain(srcSize); - size_t const maxCSize = srcSize - minGain; - /* If the srcSize <= dstCapacity, then there is enough space to write a - * raw uncompressed block. Since we ran out of space, the block must not - * be compressible, so fall back to a raw uncompressed block. - */ - int const uncompressibleError = cSize == ERROR(dstSize_tooSmall) && srcSize <= dstCapacity; - int i; - - if (ZSTD_isError(cSize) && !uncompressibleError) - return cSize; - if (cSize >= maxCSize || uncompressibleError) { - zc->flagStaticHufTable = HUF_repeat_none; - return 0; - } - /* confirm repcodes */ - for (i = 0; i < ZSTD_REP_NUM; i++) - zc->rep[i] = zc->repToConfirm[i]; - return cSize; -} - -/*! ZSTD_storeSeq() : - Store a sequence (literal length, literals, offset code and match length code) into seqStore_t. - `offsetCode` : distance to match, or 0 == repCode. - `matchCode` : matchLength - MINMATCH -*/ -ZSTD_STATIC void ZSTD_storeSeq(seqStore_t *seqStorePtr, size_t litLength, const void *literals, U32 offsetCode, size_t matchCode) -{ - /* copy Literals */ - ZSTD_wildcopy(seqStorePtr->lit, literals, litLength); - seqStorePtr->lit += litLength; - - /* literal Length */ - if (litLength > 0xFFFF) { - seqStorePtr->longLengthID = 1; - seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); - } - seqStorePtr->sequences[0].litLength = (U16)litLength; - - /* match offset */ - seqStorePtr->sequences[0].offset = offsetCode + 1; - - /* match Length */ - if (matchCode > 0xFFFF) { - seqStorePtr->longLengthID = 2; - seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); - } - seqStorePtr->sequences[0].matchLength = (U16)matchCode; - - seqStorePtr->sequences++; -} - -/*-************************************* -* Match length counter -***************************************/ -static unsigned ZSTD_NbCommonBytes(register size_t val) -{ - if (ZSTD_isLittleEndian()) { - if (ZSTD_64bits()) { - return (__builtin_ctzll((U64)val) >> 3); - } else { /* 32 bits */ - return (__builtin_ctz((U32)val) >> 3); - } - } else { /* Big Endian CPU */ - if (ZSTD_64bits()) { - return (__builtin_clzll(val) >> 3); - } else { /* 32 bits */ - return (__builtin_clz((U32)val) >> 3); - } - } -} - -static size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch, const BYTE *const pInLimit) -{ - const BYTE *const pStart = pIn; - const BYTE *const pInLoopLimit = pInLimit - (sizeof(size_t) - 1); - - while (pIn < pInLoopLimit) { - size_t const diff = ZSTD_readST(pMatch) ^ ZSTD_readST(pIn); - if (!diff) { - pIn += sizeof(size_t); - pMatch += sizeof(size_t); - continue; - } - pIn += ZSTD_NbCommonBytes(diff); - return (size_t)(pIn - pStart); - } - if (ZSTD_64bits()) - if ((pIn < (pInLimit - 3)) && (ZSTD_read32(pMatch) == ZSTD_read32(pIn))) { - pIn += 4; - pMatch += 4; - } - if ((pIn < (pInLimit - 1)) && (ZSTD_read16(pMatch) == ZSTD_read16(pIn))) { - pIn += 2; - pMatch += 2; - } - if ((pIn < pInLimit) && (*pMatch == *pIn)) - pIn++; - return (size_t)(pIn - pStart); -} - -/** ZSTD_count_2segments() : -* can count match length with `ip` & `match` in 2 different segments. -* convention : on reaching mEnd, match count continue starting from iStart -*/ -static size_t ZSTD_count_2segments(const BYTE *ip, const BYTE *match, const BYTE *iEnd, const BYTE *mEnd, const BYTE *iStart) -{ - const BYTE *const vEnd = MIN(ip + (mEnd - match), iEnd); - size_t const matchLength = ZSTD_count(ip, match, vEnd); - if (match + matchLength != mEnd) - return matchLength; - return matchLength + ZSTD_count(ip + matchLength, iStart, iEnd); -} - -/*-************************************* -* Hashes -***************************************/ -static const U32 prime3bytes = 506832829U; -static U32 ZSTD_hash3(U32 u, U32 h) { return ((u << (32 - 24)) * prime3bytes) >> (32 - h); } -ZSTD_STATIC size_t ZSTD_hash3Ptr(const void *ptr, U32 h) { return ZSTD_hash3(ZSTD_readLE32(ptr), h); } /* only in zstd_opt.h */ - -static const U32 prime4bytes = 2654435761U; -static U32 ZSTD_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32 - h); } -static size_t ZSTD_hash4Ptr(const void *ptr, U32 h) { return ZSTD_hash4(ZSTD_read32(ptr), h); } - -static const U64 prime5bytes = 889523592379ULL; -static size_t ZSTD_hash5(U64 u, U32 h) { return (size_t)(((u << (64 - 40)) * prime5bytes) >> (64 - h)); } -static size_t ZSTD_hash5Ptr(const void *p, U32 h) { return ZSTD_hash5(ZSTD_readLE64(p), h); } - -static const U64 prime6bytes = 227718039650203ULL; -static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64 - 48)) * prime6bytes) >> (64 - h)); } -static size_t ZSTD_hash6Ptr(const void *p, U32 h) { return ZSTD_hash6(ZSTD_readLE64(p), h); } - -static const U64 prime7bytes = 58295818150454627ULL; -static size_t ZSTD_hash7(U64 u, U32 h) { return (size_t)(((u << (64 - 56)) * prime7bytes) >> (64 - h)); } -static size_t ZSTD_hash7Ptr(const void *p, U32 h) { return ZSTD_hash7(ZSTD_readLE64(p), h); } - -static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; -static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u)*prime8bytes) >> (64 - h)); } -static size_t ZSTD_hash8Ptr(const void *p, U32 h) { return ZSTD_hash8(ZSTD_readLE64(p), h); } - -static size_t ZSTD_hashPtr(const void *p, U32 hBits, U32 mls) -{ - switch (mls) { - // case 3: return ZSTD_hash3Ptr(p, hBits); - default: - case 4: return ZSTD_hash4Ptr(p, hBits); - case 5: return ZSTD_hash5Ptr(p, hBits); - case 6: return ZSTD_hash6Ptr(p, hBits); - case 7: return ZSTD_hash7Ptr(p, hBits); - case 8: return ZSTD_hash8Ptr(p, hBits); - } -} - -/*-************************************* -* Fast Scan -***************************************/ -static void ZSTD_fillHashTable(ZSTD_CCtx *zc, const void *end, const U32 mls) -{ - U32 *const hashTable = zc->hashTable; - U32 const hBits = zc->params.cParams.hashLog; - const BYTE *const base = zc->base; - const BYTE *ip = base + zc->nextToUpdate; - const BYTE *const iend = ((const BYTE *)end) - HASH_READ_SIZE; - const size_t fastHashFillStep = 3; - - while (ip <= iend) { - hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip - base); - ip += fastHashFillStep; - } -} - -FORCE_INLINE -void ZSTD_compressBlock_fast_generic(ZSTD_CCtx *cctx, const void *src, size_t srcSize, const U32 mls) -{ - U32 *const hashTable = cctx->hashTable; - U32 const hBits = cctx->params.cParams.hashLog; - seqStore_t *seqStorePtr = &(cctx->seqStore); - const BYTE *const base = cctx->base; - const BYTE *const istart = (const BYTE *)src; - const BYTE *ip = istart; - const BYTE *anchor = istart; - const U32 lowestIndex = cctx->dictLimit; - const BYTE *const lowest = base + lowestIndex; - const BYTE *const iend = istart + srcSize; - const BYTE *const ilimit = iend - HASH_READ_SIZE; - U32 offset_1 = cctx->rep[0], offset_2 = cctx->rep[1]; - U32 offsetSaved = 0; - - /* init */ - ip += (ip == lowest); - { - U32 const maxRep = (U32)(ip - lowest); - if (offset_2 > maxRep) - offsetSaved = offset_2, offset_2 = 0; - if (offset_1 > maxRep) - offsetSaved = offset_1, offset_1 = 0; - } - - /* Main Search Loop */ - while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ - size_t mLength; - size_t const h = ZSTD_hashPtr(ip, hBits, mls); - U32 const curr = (U32)(ip - base); - U32 const matchIndex = hashTable[h]; - const BYTE *match = base + matchIndex; - hashTable[h] = curr; /* update hash table */ - - if ((offset_1 > 0) & (ZSTD_read32(ip + 1 - offset_1) == ZSTD_read32(ip + 1))) { - mLength = ZSTD_count(ip + 1 + 4, ip + 1 + 4 - offset_1, iend) + 4; - ip++; - ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH); - } else { - U32 offset; - if ((matchIndex <= lowestIndex) || (ZSTD_read32(match) != ZSTD_read32(ip))) { - ip += ((ip - anchor) >> g_searchStrength) + 1; - continue; - } - mLength = ZSTD_count(ip + 4, match + 4, iend) + 4; - offset = (U32)(ip - match); - while (((ip > anchor) & (match > lowest)) && (ip[-1] == match[-1])) { - ip--; - match--; - mLength++; - } /* catch up */ - offset_2 = offset_1; - offset_1 = offset; - - ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH); - } - - /* match found */ - ip += mLength; - anchor = ip; - - if (ip <= ilimit) { - /* Fill Table */ - hashTable[ZSTD_hashPtr(base + curr + 2, hBits, mls)] = curr + 2; /* here because curr+2 could be > iend-8 */ - hashTable[ZSTD_hashPtr(ip - 2, hBits, mls)] = (U32)(ip - 2 - base); - /* check immediate repcode */ - while ((ip <= ilimit) && ((offset_2 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)))) { - /* store sequence */ - size_t const rLength = ZSTD_count(ip + 4, ip + 4 - offset_2, iend) + 4; - { - U32 const tmpOff = offset_2; - offset_2 = offset_1; - offset_1 = tmpOff; - } /* swap offset_2 <=> offset_1 */ - hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip - base); - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength - MINMATCH); - ip += rLength; - anchor = ip; - continue; /* faster when present ... (?) */ - } - } - } - - /* save reps for next block */ - cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved; - cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved; - - /* Last Literals */ - { - size_t const lastLLSize = iend - anchor; - memcpy(seqStorePtr->lit, anchor, lastLLSize); - seqStorePtr->lit += lastLLSize; - } -} - -static void ZSTD_compressBlock_fast(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -{ - const U32 mls = ctx->params.cParams.searchLength; - switch (mls) { - default: /* includes case 3 */ - case 4: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 4); return; - case 5: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 5); return; - case 6: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 6); return; - case 7: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 7); return; - } -} - -static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 mls) -{ - U32 *hashTable = ctx->hashTable; - const U32 hBits = ctx->params.cParams.hashLog; - seqStore_t *seqStorePtr = &(ctx->seqStore); - const BYTE *const base = ctx->base; - const BYTE *const dictBase = ctx->dictBase; - const BYTE *const istart = (const BYTE *)src; - const BYTE *ip = istart; - const BYTE *anchor = istart; - const U32 lowestIndex = ctx->lowLimit; - const BYTE *const dictStart = dictBase + lowestIndex; - const U32 dictLimit = ctx->dictLimit; - const BYTE *const lowPrefixPtr = base + dictLimit; - const BYTE *const dictEnd = dictBase + dictLimit; - const BYTE *const iend = istart + srcSize; - const BYTE *const ilimit = iend - 8; - U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1]; - - /* Search Loop */ - while (ip < ilimit) { /* < instead of <=, because (ip+1) */ - const size_t h = ZSTD_hashPtr(ip, hBits, mls); - const U32 matchIndex = hashTable[h]; - const BYTE *matchBase = matchIndex < dictLimit ? dictBase : base; - const BYTE *match = matchBase + matchIndex; - const U32 curr = (U32)(ip - base); - const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */ - const BYTE *repBase = repIndex < dictLimit ? dictBase : base; - const BYTE *repMatch = repBase + repIndex; - size_t mLength; - hashTable[h] = curr; /* update hash table */ - - if ((((U32)((dictLimit - 1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) && - (ZSTD_read32(repMatch) == ZSTD_read32(ip + 1))) { - const BYTE *repMatchEnd = repIndex < dictLimit ? dictEnd : iend; - mLength = ZSTD_count_2segments(ip + 1 + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repMatchEnd, lowPrefixPtr) + EQUAL_READ32; - ip++; - ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH); - } else { - if ((matchIndex < lowestIndex) || (ZSTD_read32(match) != ZSTD_read32(ip))) { - ip += ((ip - anchor) >> g_searchStrength) + 1; - continue; - } - { - const BYTE *matchEnd = matchIndex < dictLimit ? dictEnd : iend; - const BYTE *lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr; - U32 offset; - mLength = ZSTD_count_2segments(ip + EQUAL_READ32, match + EQUAL_READ32, iend, matchEnd, lowPrefixPtr) + EQUAL_READ32; - while (((ip > anchor) & (match > lowMatchPtr)) && (ip[-1] == match[-1])) { - ip--; - match--; - mLength++; - } /* catch up */ - offset = curr - matchIndex; - offset_2 = offset_1; - offset_1 = offset; - ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH); - } - } - - /* found a match : store it */ - ip += mLength; - anchor = ip; - - if (ip <= ilimit) { - /* Fill Table */ - hashTable[ZSTD_hashPtr(base + curr + 2, hBits, mls)] = curr + 2; - hashTable[ZSTD_hashPtr(ip - 2, hBits, mls)] = (U32)(ip - 2 - base); - /* check immediate repcode */ - while (ip <= ilimit) { - U32 const curr2 = (U32)(ip - base); - U32 const repIndex2 = curr2 - offset_2; - const BYTE *repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2; - if ((((U32)((dictLimit - 1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */ - && (ZSTD_read32(repMatch2) == ZSTD_read32(ip))) { - const BYTE *const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend; - size_t repLength2 = - ZSTD_count_2segments(ip + EQUAL_READ32, repMatch2 + EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32; - U32 tmpOffset = offset_2; - offset_2 = offset_1; - offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2 - MINMATCH); - hashTable[ZSTD_hashPtr(ip, hBits, mls)] = curr2; - ip += repLength2; - anchor = ip; - continue; - } - break; - } - } - } - - /* save reps for next block */ - ctx->repToConfirm[0] = offset_1; - ctx->repToConfirm[1] = offset_2; - - /* Last Literals */ - { - size_t const lastLLSize = iend - anchor; - memcpy(seqStorePtr->lit, anchor, lastLLSize); - seqStorePtr->lit += lastLLSize; - } -} - -static void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -{ - U32 const mls = ctx->params.cParams.searchLength; - switch (mls) { - default: /* includes case 3 */ - case 4: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 4); return; - case 5: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 5); return; - case 6: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 6); return; - case 7: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 7); return; - } -} - -/*-************************************* -* Double Fast -***************************************/ -static void ZSTD_fillDoubleHashTable(ZSTD_CCtx *cctx, const void *end, const U32 mls) -{ - U32 *const hashLarge = cctx->hashTable; - U32 const hBitsL = cctx->params.cParams.hashLog; - U32 *const hashSmall = cctx->chainTable; - U32 const hBitsS = cctx->params.cParams.chainLog; - const BYTE *const base = cctx->base; - const BYTE *ip = base + cctx->nextToUpdate; - const BYTE *const iend = ((const BYTE *)end) - HASH_READ_SIZE; - const size_t fastHashFillStep = 3; - - while (ip <= iend) { - hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip - base); - hashLarge[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip - base); - ip += fastHashFillStep; - } -} - -FORCE_INLINE -void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx *cctx, const void *src, size_t srcSize, const U32 mls) -{ - U32 *const hashLong = cctx->hashTable; - const U32 hBitsL = cctx->params.cParams.hashLog; - U32 *const hashSmall = cctx->chainTable; - const U32 hBitsS = cctx->params.cParams.chainLog; - seqStore_t *seqStorePtr = &(cctx->seqStore); - const BYTE *const base = cctx->base; - const BYTE *const istart = (const BYTE *)src; - const BYTE *ip = istart; - const BYTE *anchor = istart; - const U32 lowestIndex = cctx->dictLimit; - const BYTE *const lowest = base + lowestIndex; - const BYTE *const iend = istart + srcSize; - const BYTE *const ilimit = iend - HASH_READ_SIZE; - U32 offset_1 = cctx->rep[0], offset_2 = cctx->rep[1]; - U32 offsetSaved = 0; - - /* init */ - ip += (ip == lowest); - { - U32 const maxRep = (U32)(ip - lowest); - if (offset_2 > maxRep) - offsetSaved = offset_2, offset_2 = 0; - if (offset_1 > maxRep) - offsetSaved = offset_1, offset_1 = 0; - } - - /* Main Search Loop */ - while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ - size_t mLength; - size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8); - size_t const h = ZSTD_hashPtr(ip, hBitsS, mls); - U32 const curr = (U32)(ip - base); - U32 const matchIndexL = hashLong[h2]; - U32 const matchIndexS = hashSmall[h]; - const BYTE *matchLong = base + matchIndexL; - const BYTE *match = base + matchIndexS; - hashLong[h2] = hashSmall[h] = curr; /* update hash tables */ - - if ((offset_1 > 0) & (ZSTD_read32(ip + 1 - offset_1) == ZSTD_read32(ip + 1))) { /* note : by construction, offset_1 <= curr */ - mLength = ZSTD_count(ip + 1 + 4, ip + 1 + 4 - offset_1, iend) + 4; - ip++; - ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH); - } else { - U32 offset; - if ((matchIndexL > lowestIndex) && (ZSTD_read64(matchLong) == ZSTD_read64(ip))) { - mLength = ZSTD_count(ip + 8, matchLong + 8, iend) + 8; - offset = (U32)(ip - matchLong); - while (((ip > anchor) & (matchLong > lowest)) && (ip[-1] == matchLong[-1])) { - ip--; - matchLong--; - mLength++; - } /* catch up */ - } else if ((matchIndexS > lowestIndex) && (ZSTD_read32(match) == ZSTD_read32(ip))) { - size_t const h3 = ZSTD_hashPtr(ip + 1, hBitsL, 8); - U32 const matchIndex3 = hashLong[h3]; - const BYTE *match3 = base + matchIndex3; - hashLong[h3] = curr + 1; - if ((matchIndex3 > lowestIndex) && (ZSTD_read64(match3) == ZSTD_read64(ip + 1))) { - mLength = ZSTD_count(ip + 9, match3 + 8, iend) + 8; - ip++; - offset = (U32)(ip - match3); - while (((ip > anchor) & (match3 > lowest)) && (ip[-1] == match3[-1])) { - ip--; - match3--; - mLength++; - } /* catch up */ - } else { - mLength = ZSTD_count(ip + 4, match + 4, iend) + 4; - offset = (U32)(ip - match); - while (((ip > anchor) & (match > lowest)) && (ip[-1] == match[-1])) { - ip--; - match--; - mLength++; - } /* catch up */ - } - } else { - ip += ((ip - anchor) >> g_searchStrength) + 1; - continue; - } - - offset_2 = offset_1; - offset_1 = offset; - - ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH); - } - - /* match found */ - ip += mLength; - anchor = ip; - - if (ip <= ilimit) { - /* Fill Table */ - hashLong[ZSTD_hashPtr(base + curr + 2, hBitsL, 8)] = hashSmall[ZSTD_hashPtr(base + curr + 2, hBitsS, mls)] = - curr + 2; /* here because curr+2 could be > iend-8 */ - hashLong[ZSTD_hashPtr(ip - 2, hBitsL, 8)] = hashSmall[ZSTD_hashPtr(ip - 2, hBitsS, mls)] = (U32)(ip - 2 - base); - - /* check immediate repcode */ - while ((ip <= ilimit) && ((offset_2 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)))) { - /* store sequence */ - size_t const rLength = ZSTD_count(ip + 4, ip + 4 - offset_2, iend) + 4; - { - U32 const tmpOff = offset_2; - offset_2 = offset_1; - offset_1 = tmpOff; - } /* swap offset_2 <=> offset_1 */ - hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip - base); - hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip - base); - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength - MINMATCH); - ip += rLength; - anchor = ip; - continue; /* faster when present ... (?) */ - } - } - } - - /* save reps for next block */ - cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved; - cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved; - - /* Last Literals */ - { - size_t const lastLLSize = iend - anchor; - memcpy(seqStorePtr->lit, anchor, lastLLSize); - seqStorePtr->lit += lastLLSize; - } -} - -static void ZSTD_compressBlock_doubleFast(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -{ - const U32 mls = ctx->params.cParams.searchLength; - switch (mls) { - default: /* includes case 3 */ - case 4: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 4); return; - case 5: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 5); return; - case 6: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 6); return; - case 7: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 7); return; - } -} - -static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 mls) -{ - U32 *const hashLong = ctx->hashTable; - U32 const hBitsL = ctx->params.cParams.hashLog; - U32 *const hashSmall = ctx->chainTable; - U32 const hBitsS = ctx->params.cParams.chainLog; - seqStore_t *seqStorePtr = &(ctx->seqStore); - const BYTE *const base = ctx->base; - const BYTE *const dictBase = ctx->dictBase; - const BYTE *const istart = (const BYTE *)src; - const BYTE *ip = istart; - const BYTE *anchor = istart; - const U32 lowestIndex = ctx->lowLimit; - const BYTE *const dictStart = dictBase + lowestIndex; - const U32 dictLimit = ctx->dictLimit; - const BYTE *const lowPrefixPtr = base + dictLimit; - const BYTE *const dictEnd = dictBase + dictLimit; - const BYTE *const iend = istart + srcSize; - const BYTE *const ilimit = iend - 8; - U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1]; - - /* Search Loop */ - while (ip < ilimit) { /* < instead of <=, because (ip+1) */ - const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls); - const U32 matchIndex = hashSmall[hSmall]; - const BYTE *matchBase = matchIndex < dictLimit ? dictBase : base; - const BYTE *match = matchBase + matchIndex; - - const size_t hLong = ZSTD_hashPtr(ip, hBitsL, 8); - const U32 matchLongIndex = hashLong[hLong]; - const BYTE *matchLongBase = matchLongIndex < dictLimit ? dictBase : base; - const BYTE *matchLong = matchLongBase + matchLongIndex; - - const U32 curr = (U32)(ip - base); - const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */ - const BYTE *repBase = repIndex < dictLimit ? dictBase : base; - const BYTE *repMatch = repBase + repIndex; - size_t mLength; - hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */ - - if ((((U32)((dictLimit - 1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) && - (ZSTD_read32(repMatch) == ZSTD_read32(ip + 1))) { - const BYTE *repMatchEnd = repIndex < dictLimit ? dictEnd : iend; - mLength = ZSTD_count_2segments(ip + 1 + 4, repMatch + 4, iend, repMatchEnd, lowPrefixPtr) + 4; - ip++; - ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH); - } else { - if ((matchLongIndex > lowestIndex) && (ZSTD_read64(matchLong) == ZSTD_read64(ip))) { - const BYTE *matchEnd = matchLongIndex < dictLimit ? dictEnd : iend; - const BYTE *lowMatchPtr = matchLongIndex < dictLimit ? dictStart : lowPrefixPtr; - U32 offset; - mLength = ZSTD_count_2segments(ip + 8, matchLong + 8, iend, matchEnd, lowPrefixPtr) + 8; - offset = curr - matchLongIndex; - while (((ip > anchor) & (matchLong > lowMatchPtr)) && (ip[-1] == matchLong[-1])) { - ip--; - matchLong--; - mLength++; - } /* catch up */ - offset_2 = offset_1; - offset_1 = offset; - ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH); - - } else if ((matchIndex > lowestIndex) && (ZSTD_read32(match) == ZSTD_read32(ip))) { - size_t const h3 = ZSTD_hashPtr(ip + 1, hBitsL, 8); - U32 const matchIndex3 = hashLong[h3]; - const BYTE *const match3Base = matchIndex3 < dictLimit ? dictBase : base; - const BYTE *match3 = match3Base + matchIndex3; - U32 offset; - hashLong[h3] = curr + 1; - if ((matchIndex3 > lowestIndex) && (ZSTD_read64(match3) == ZSTD_read64(ip + 1))) { - const BYTE *matchEnd = matchIndex3 < dictLimit ? dictEnd : iend; - const BYTE *lowMatchPtr = matchIndex3 < dictLimit ? dictStart : lowPrefixPtr; - mLength = ZSTD_count_2segments(ip + 9, match3 + 8, iend, matchEnd, lowPrefixPtr) + 8; - ip++; - offset = curr + 1 - matchIndex3; - while (((ip > anchor) & (match3 > lowMatchPtr)) && (ip[-1] == match3[-1])) { - ip--; - match3--; - mLength++; - } /* catch up */ - } else { - const BYTE *matchEnd = matchIndex < dictLimit ? dictEnd : iend; - const BYTE *lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr; - mLength = ZSTD_count_2segments(ip + 4, match + 4, iend, matchEnd, lowPrefixPtr) + 4; - offset = curr - matchIndex; - while (((ip > anchor) & (match > lowMatchPtr)) && (ip[-1] == match[-1])) { - ip--; - match--; - mLength++; - } /* catch up */ - } - offset_2 = offset_1; - offset_1 = offset; - ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH); - - } else { - ip += ((ip - anchor) >> g_searchStrength) + 1; - continue; - } - } - - /* found a match : store it */ - ip += mLength; - anchor = ip; - - if (ip <= ilimit) { - /* Fill Table */ - hashSmall[ZSTD_hashPtr(base + curr + 2, hBitsS, mls)] = curr + 2; - hashLong[ZSTD_hashPtr(base + curr + 2, hBitsL, 8)] = curr + 2; - hashSmall[ZSTD_hashPtr(ip - 2, hBitsS, mls)] = (U32)(ip - 2 - base); - hashLong[ZSTD_hashPtr(ip - 2, hBitsL, 8)] = (U32)(ip - 2 - base); - /* check immediate repcode */ - while (ip <= ilimit) { - U32 const curr2 = (U32)(ip - base); - U32 const repIndex2 = curr2 - offset_2; - const BYTE *repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2; - if ((((U32)((dictLimit - 1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */ - && (ZSTD_read32(repMatch2) == ZSTD_read32(ip))) { - const BYTE *const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend; - size_t const repLength2 = - ZSTD_count_2segments(ip + EQUAL_READ32, repMatch2 + EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32; - U32 tmpOffset = offset_2; - offset_2 = offset_1; - offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2 - MINMATCH); - hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = curr2; - hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = curr2; - ip += repLength2; - anchor = ip; - continue; - } - break; - } - } - } - - /* save reps for next block */ - ctx->repToConfirm[0] = offset_1; - ctx->repToConfirm[1] = offset_2; - - /* Last Literals */ - { - size_t const lastLLSize = iend - anchor; - memcpy(seqStorePtr->lit, anchor, lastLLSize); - seqStorePtr->lit += lastLLSize; - } -} - -static void ZSTD_compressBlock_doubleFast_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -{ - U32 const mls = ctx->params.cParams.searchLength; - switch (mls) { - default: /* includes case 3 */ - case 4: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 4); return; - case 5: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 5); return; - case 6: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 6); return; - case 7: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 7); return; - } -} - -/*-************************************* -* Binary Tree search -***************************************/ -/** ZSTD_insertBt1() : add one or multiple positions to tree. -* ip : assumed <= iend-8 . -* @return : nb of positions added */ -static U32 ZSTD_insertBt1(ZSTD_CCtx *zc, const BYTE *const ip, const U32 mls, const BYTE *const iend, U32 nbCompares, U32 extDict) -{ - U32 *const hashTable = zc->hashTable; - U32 const hashLog = zc->params.cParams.hashLog; - size_t const h = ZSTD_hashPtr(ip, hashLog, mls); - U32 *const bt = zc->chainTable; - U32 const btLog = zc->params.cParams.chainLog - 1; - U32 const btMask = (1 << btLog) - 1; - U32 matchIndex = hashTable[h]; - size_t commonLengthSmaller = 0, commonLengthLarger = 0; - const BYTE *const base = zc->base; - const BYTE *const dictBase = zc->dictBase; - const U32 dictLimit = zc->dictLimit; - const BYTE *const dictEnd = dictBase + dictLimit; - const BYTE *const prefixStart = base + dictLimit; - const BYTE *match; - const U32 curr = (U32)(ip - base); - const U32 btLow = btMask >= curr ? 0 : curr - btMask; - U32 *smallerPtr = bt + 2 * (curr & btMask); - U32 *largerPtr = smallerPtr + 1; - U32 dummy32; /* to be nullified at the end */ - U32 const windowLow = zc->lowLimit; - U32 matchEndIdx = curr + 8; - size_t bestLength = 8; - - hashTable[h] = curr; /* Update Hash Table */ - - while (nbCompares-- && (matchIndex > windowLow)) { - U32 *const nextPtr = bt + 2 * (matchIndex & btMask); - size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ - - if ((!extDict) || (matchIndex + matchLength >= dictLimit)) { - match = base + matchIndex; - if (match[matchLength] == ip[matchLength]) - matchLength += ZSTD_count(ip + matchLength + 1, match + matchLength + 1, iend) + 1; - } else { - match = dictBase + matchIndex; - matchLength += ZSTD_count_2segments(ip + matchLength, match + matchLength, iend, dictEnd, prefixStart); - if (matchIndex + matchLength >= dictLimit) - match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ - } - - if (matchLength > bestLength) { - bestLength = matchLength; - if (matchLength > matchEndIdx - matchIndex) - matchEndIdx = matchIndex + (U32)matchLength; - } - - if (ip + matchLength == iend) /* equal : no way to know if inf or sup */ - break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */ - - if (match[matchLength] < ip[matchLength]) { /* necessarily within correct buffer */ - /* match is smaller than curr */ - *smallerPtr = matchIndex; /* update smaller idx */ - commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ - if (matchIndex <= btLow) { - smallerPtr = &dummy32; - break; - } /* beyond tree size, stop the search */ - smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */ - matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */ - } else { - /* match is larger than curr */ - *largerPtr = matchIndex; - commonLengthLarger = matchLength; - if (matchIndex <= btLow) { - largerPtr = &dummy32; - break; - } /* beyond tree size, stop the search */ - largerPtr = nextPtr; - matchIndex = nextPtr[0]; - } - } - - *smallerPtr = *largerPtr = 0; - if (bestLength > 384) - return MIN(192, (U32)(bestLength - 384)); /* speed optimization */ - if (matchEndIdx > curr + 8) - return matchEndIdx - curr - 8; - return 1; -} - -static size_t ZSTD_insertBtAndFindBestMatch(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iend, size_t *offsetPtr, U32 nbCompares, const U32 mls, - U32 extDict) -{ - U32 *const hashTable = zc->hashTable; - U32 const hashLog = zc->params.cParams.hashLog; - size_t const h = ZSTD_hashPtr(ip, hashLog, mls); - U32 *const bt = zc->chainTable; - U32 const btLog = zc->params.cParams.chainLog - 1; - U32 const btMask = (1 << btLog) - 1; - U32 matchIndex = hashTable[h]; - size_t commonLengthSmaller = 0, commonLengthLarger = 0; - const BYTE *const base = zc->base; - const BYTE *const dictBase = zc->dictBase; - const U32 dictLimit = zc->dictLimit; - const BYTE *const dictEnd = dictBase + dictLimit; - const BYTE *const prefixStart = base + dictLimit; - const U32 curr = (U32)(ip - base); - const U32 btLow = btMask >= curr ? 0 : curr - btMask; - const U32 windowLow = zc->lowLimit; - U32 *smallerPtr = bt + 2 * (curr & btMask); - U32 *largerPtr = bt + 2 * (curr & btMask) + 1; - U32 matchEndIdx = curr + 8; - U32 dummy32; /* to be nullified at the end */ - size_t bestLength = 0; - - hashTable[h] = curr; /* Update Hash Table */ - - while (nbCompares-- && (matchIndex > windowLow)) { - U32 *const nextPtr = bt + 2 * (matchIndex & btMask); - size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ - const BYTE *match; - - if ((!extDict) || (matchIndex + matchLength >= dictLimit)) { - match = base + matchIndex; - if (match[matchLength] == ip[matchLength]) - matchLength += ZSTD_count(ip + matchLength + 1, match + matchLength + 1, iend) + 1; - } else { - match = dictBase + matchIndex; - matchLength += ZSTD_count_2segments(ip + matchLength, match + matchLength, iend, dictEnd, prefixStart); - if (matchIndex + matchLength >= dictLimit) - match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ - } - - if (matchLength > bestLength) { - if (matchLength > matchEndIdx - matchIndex) - matchEndIdx = matchIndex + (U32)matchLength; - if ((4 * (int)(matchLength - bestLength)) > (int)(ZSTD_highbit32(curr - matchIndex + 1) - ZSTD_highbit32((U32)offsetPtr[0] + 1))) - bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + curr - matchIndex; - if (ip + matchLength == iend) /* equal : no way to know if inf or sup */ - break; /* drop, to guarantee consistency (miss a little bit of compression) */ - } - - if (match[matchLength] < ip[matchLength]) { - /* match is smaller than curr */ - *smallerPtr = matchIndex; /* update smaller idx */ - commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ - if (matchIndex <= btLow) { - smallerPtr = &dummy32; - break; - } /* beyond tree size, stop the search */ - smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */ - matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */ - } else { - /* match is larger than curr */ - *largerPtr = matchIndex; - commonLengthLarger = matchLength; - if (matchIndex <= btLow) { - largerPtr = &dummy32; - break; - } /* beyond tree size, stop the search */ - largerPtr = nextPtr; - matchIndex = nextPtr[0]; - } - } - - *smallerPtr = *largerPtr = 0; - - zc->nextToUpdate = (matchEndIdx > curr + 8) ? matchEndIdx - 8 : curr + 1; - return bestLength; -} - -static void ZSTD_updateTree(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iend, const U32 nbCompares, const U32 mls) -{ - const BYTE *const base = zc->base; - const U32 target = (U32)(ip - base); - U32 idx = zc->nextToUpdate; - - while (idx < target) - idx += ZSTD_insertBt1(zc, base + idx, mls, iend, nbCompares, 0); -} - -/** ZSTD_BtFindBestMatch() : Tree updater, providing best match */ -static size_t ZSTD_BtFindBestMatch(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, const U32 mls) -{ - if (ip < zc->base + zc->nextToUpdate) - return 0; /* skipped area */ - ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls); - return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 0); -} - -static size_t ZSTD_BtFindBestMatch_selectMLS(ZSTD_CCtx *zc, /* Index table will be updated */ - const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, const U32 matchLengthSearch) -{ - switch (matchLengthSearch) { - default: /* includes case 3 */ - case 4: return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4); - case 5: return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5); - case 7: - case 6: return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6); - } -} - -static void ZSTD_updateTree_extDict(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iend, const U32 nbCompares, const U32 mls) -{ - const BYTE *const base = zc->base; - const U32 target = (U32)(ip - base); - U32 idx = zc->nextToUpdate; - - while (idx < target) - idx += ZSTD_insertBt1(zc, base + idx, mls, iend, nbCompares, 1); -} - -/** Tree updater, providing best match */ -static size_t ZSTD_BtFindBestMatch_extDict(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, - const U32 mls) -{ - if (ip < zc->base + zc->nextToUpdate) - return 0; /* skipped area */ - ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls); - return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 1); -} - -static size_t ZSTD_BtFindBestMatch_selectMLS_extDict(ZSTD_CCtx *zc, /* Index table will be updated */ - const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, - const U32 matchLengthSearch) -{ - switch (matchLengthSearch) { - default: /* includes case 3 */ - case 4: return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4); - case 5: return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5); - case 7: - case 6: return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6); - } -} - -/* ********************************* -* Hash Chain -***********************************/ -#define NEXT_IN_CHAIN(d, mask) chainTable[(d)&mask] - -/* Update chains up to ip (excluded) - Assumption : always within prefix (i.e. not within extDict) */ -FORCE_INLINE -U32 ZSTD_insertAndFindFirstIndex(ZSTD_CCtx *zc, const BYTE *ip, U32 mls) -{ - U32 *const hashTable = zc->hashTable; - const U32 hashLog = zc->params.cParams.hashLog; - U32 *const chainTable = zc->chainTable; - const U32 chainMask = (1 << zc->params.cParams.chainLog) - 1; - const BYTE *const base = zc->base; - const U32 target = (U32)(ip - base); - U32 idx = zc->nextToUpdate; - - while (idx < target) { /* catch up */ - size_t const h = ZSTD_hashPtr(base + idx, hashLog, mls); - NEXT_IN_CHAIN(idx, chainMask) = hashTable[h]; - hashTable[h] = idx; - idx++; - } - - zc->nextToUpdate = target; - return hashTable[ZSTD_hashPtr(ip, hashLog, mls)]; -} - -/* inlining is important to hardwire a hot branch (template emulation) */ -FORCE_INLINE -size_t ZSTD_HcFindBestMatch_generic(ZSTD_CCtx *zc, /* Index table will be updated */ - const BYTE *const ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, const U32 mls, - const U32 extDict) -{ - U32 *const chainTable = zc->chainTable; - const U32 chainSize = (1 << zc->params.cParams.chainLog); - const U32 chainMask = chainSize - 1; - const BYTE *const base = zc->base; - const BYTE *const dictBase = zc->dictBase; - const U32 dictLimit = zc->dictLimit; - const BYTE *const prefixStart = base + dictLimit; - const BYTE *const dictEnd = dictBase + dictLimit; - const U32 lowLimit = zc->lowLimit; - const U32 curr = (U32)(ip - base); - const U32 minChain = curr > chainSize ? curr - chainSize : 0; - int nbAttempts = maxNbAttempts; - size_t ml = EQUAL_READ32 - 1; - - /* HC4 match finder */ - U32 matchIndex = ZSTD_insertAndFindFirstIndex(zc, ip, mls); - - for (; (matchIndex > lowLimit) & (nbAttempts > 0); nbAttempts--) { - const BYTE *match; - size_t currMl = 0; - if ((!extDict) || matchIndex >= dictLimit) { - match = base + matchIndex; - if (match[ml] == ip[ml]) /* potentially better */ - currMl = ZSTD_count(ip, match, iLimit); - } else { - match = dictBase + matchIndex; - if (ZSTD_read32(match) == ZSTD_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ - currMl = ZSTD_count_2segments(ip + EQUAL_READ32, match + EQUAL_READ32, iLimit, dictEnd, prefixStart) + EQUAL_READ32; - } - - /* save best solution */ - if (currMl > ml) { - ml = currMl; - *offsetPtr = curr - matchIndex + ZSTD_REP_MOVE; - if (ip + currMl == iLimit) - break; /* best possible, and avoid read overflow*/ - } - - if (matchIndex <= minChain) - break; - matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask); - } - - return ml; -} - -FORCE_INLINE size_t ZSTD_HcFindBestMatch_selectMLS(ZSTD_CCtx *zc, const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, - const U32 matchLengthSearch) -{ - switch (matchLengthSearch) { - default: /* includes case 3 */ - case 4: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 0); - case 5: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 0); - case 7: - case 6: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 0); - } -} - -FORCE_INLINE size_t ZSTD_HcFindBestMatch_extDict_selectMLS(ZSTD_CCtx *zc, const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, - const U32 matchLengthSearch) -{ - switch (matchLengthSearch) { - default: /* includes case 3 */ - case 4: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 1); - case 5: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 1); - case 7: - case 6: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 1); - } -} - -/* ******************************* -* Common parser - lazy strategy -*********************************/ -FORCE_INLINE -void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 searchMethod, const U32 depth) -{ - seqStore_t *seqStorePtr = &(ctx->seqStore); - const BYTE *const istart = (const BYTE *)src; - const BYTE *ip = istart; - const BYTE *anchor = istart; - const BYTE *const iend = istart + srcSize; - const BYTE *const ilimit = iend - 8; - const BYTE *const base = ctx->base + ctx->dictLimit; - - U32 const maxSearches = 1 << ctx->params.cParams.searchLog; - U32 const mls = ctx->params.cParams.searchLength; - - typedef size_t (*searchMax_f)(ZSTD_CCtx * zc, const BYTE *ip, const BYTE *iLimit, size_t *offsetPtr, U32 maxNbAttempts, U32 matchLengthSearch); - searchMax_f const searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS; - U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1], savedOffset = 0; - - /* init */ - ip += (ip == base); - ctx->nextToUpdate3 = ctx->nextToUpdate; - { - U32 const maxRep = (U32)(ip - base); - if (offset_2 > maxRep) - savedOffset = offset_2, offset_2 = 0; - if (offset_1 > maxRep) - savedOffset = offset_1, offset_1 = 0; - } - - /* Match Loop */ - while (ip < ilimit) { - size_t matchLength = 0; - size_t offset = 0; - const BYTE *start = ip + 1; - - /* check repCode */ - if ((offset_1 > 0) & (ZSTD_read32(ip + 1) == ZSTD_read32(ip + 1 - offset_1))) { - /* repcode : we take it */ - matchLength = ZSTD_count(ip + 1 + EQUAL_READ32, ip + 1 + EQUAL_READ32 - offset_1, iend) + EQUAL_READ32; - if (depth == 0) - goto _storeSequence; - } - - /* first search (depth 0) */ - { - size_t offsetFound = 99999999; - size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls); - if (ml2 > matchLength) - matchLength = ml2, start = ip, offset = offsetFound; - } - - if (matchLength < EQUAL_READ32) { - ip += ((ip - anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */ - continue; - } - - /* let's try to find a better solution */ - if (depth >= 1) - while (ip < ilimit) { - ip++; - if ((offset) && ((offset_1 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_1)))) { - size_t const mlRep = ZSTD_count(ip + EQUAL_READ32, ip + EQUAL_READ32 - offset_1, iend) + EQUAL_READ32; - int const gain2 = (int)(mlRep * 3); - int const gain1 = (int)(matchLength * 3 - ZSTD_highbit32((U32)offset + 1) + 1); - if ((mlRep >= EQUAL_READ32) && (gain2 > gain1)) - matchLength = mlRep, offset = 0, start = ip; - } - { - size_t offset2 = 99999999; - size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); - int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */ - int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 4); - if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { - matchLength = ml2, offset = offset2, start = ip; - continue; /* search a better one */ - } - } - - /* let's find an even better one */ - if ((depth == 2) && (ip < ilimit)) { - ip++; - if ((offset) && ((offset_1 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_1)))) { - size_t const ml2 = ZSTD_count(ip + EQUAL_READ32, ip + EQUAL_READ32 - offset_1, iend) + EQUAL_READ32; - int const gain2 = (int)(ml2 * 4); - int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 1); - if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) - matchLength = ml2, offset = 0, start = ip; - } - { - size_t offset2 = 99999999; - size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); - int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */ - int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 7); - if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { - matchLength = ml2, offset = offset2, start = ip; - continue; - } - } - } - break; /* nothing found : store previous solution */ - } - - /* NOTE: - * start[-offset+ZSTD_REP_MOVE-1] is undefined behavior. - * (-offset+ZSTD_REP_MOVE-1) is unsigned, and is added to start, which - * overflows the pointer, which is undefined behavior. - */ - /* catch up */ - if (offset) { - while ((start > anchor) && (start > base + offset - ZSTD_REP_MOVE) && - (start[-1] == (start-offset+ZSTD_REP_MOVE)[-1])) /* only search for offset within prefix */ - { - start--; - matchLength++; - } - offset_2 = offset_1; - offset_1 = (U32)(offset - ZSTD_REP_MOVE); - } - - /* store sequence */ -_storeSequence: - { - size_t const litLength = start - anchor; - ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength - MINMATCH); - anchor = ip = start + matchLength; - } - - /* check immediate repcode */ - while ((ip <= ilimit) && ((offset_2 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)))) { - /* store sequence */ - matchLength = ZSTD_count(ip + EQUAL_READ32, ip + EQUAL_READ32 - offset_2, iend) + EQUAL_READ32; - offset = offset_2; - offset_2 = offset_1; - offset_1 = (U32)offset; /* swap repcodes */ - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength - MINMATCH); - ip += matchLength; - anchor = ip; - continue; /* faster when present ... (?) */ - } - } - - /* Save reps for next block */ - ctx->repToConfirm[0] = offset_1 ? offset_1 : savedOffset; - ctx->repToConfirm[1] = offset_2 ? offset_2 : savedOffset; - - /* Last Literals */ - { - size_t const lastLLSize = iend - anchor; - memcpy(seqStorePtr->lit, anchor, lastLLSize); - seqStorePtr->lit += lastLLSize; - } -} - -static void ZSTD_compressBlock_btlazy2(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 1, 2); } - -static void ZSTD_compressBlock_lazy2(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 2); } - -static void ZSTD_compressBlock_lazy(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 1); } - -static void ZSTD_compressBlock_greedy(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 0); } - -FORCE_INLINE -void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 searchMethod, const U32 depth) -{ - seqStore_t *seqStorePtr = &(ctx->seqStore); - const BYTE *const istart = (const BYTE *)src; - const BYTE *ip = istart; - const BYTE *anchor = istart; - const BYTE *const iend = istart + srcSize; - const BYTE *const ilimit = iend - 8; - const BYTE *const base = ctx->base; - const U32 dictLimit = ctx->dictLimit; - const U32 lowestIndex = ctx->lowLimit; - const BYTE *const prefixStart = base + dictLimit; - const BYTE *const dictBase = ctx->dictBase; - const BYTE *const dictEnd = dictBase + dictLimit; - const BYTE *const dictStart = dictBase + ctx->lowLimit; - - const U32 maxSearches = 1 << ctx->params.cParams.searchLog; - const U32 mls = ctx->params.cParams.searchLength; - - typedef size_t (*searchMax_f)(ZSTD_CCtx * zc, const BYTE *ip, const BYTE *iLimit, size_t *offsetPtr, U32 maxNbAttempts, U32 matchLengthSearch); - searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS_extDict : ZSTD_HcFindBestMatch_extDict_selectMLS; - - U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1]; - - /* init */ - ctx->nextToUpdate3 = ctx->nextToUpdate; - ip += (ip == prefixStart); - - /* Match Loop */ - while (ip < ilimit) { - size_t matchLength = 0; - size_t offset = 0; - const BYTE *start = ip + 1; - U32 curr = (U32)(ip - base); - - /* check repCode */ - { - const U32 repIndex = (U32)(curr + 1 - offset_1); - const BYTE *const repBase = repIndex < dictLimit ? dictBase : base; - const BYTE *const repMatch = repBase + repIndex; - if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ - if (ZSTD_read32(ip + 1) == ZSTD_read32(repMatch)) { - /* repcode detected we should take it */ - const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend; - matchLength = - ZSTD_count_2segments(ip + 1 + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; - if (depth == 0) - goto _storeSequence; - } - } - - /* first search (depth 0) */ - { - size_t offsetFound = 99999999; - size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls); - if (ml2 > matchLength) - matchLength = ml2, start = ip, offset = offsetFound; - } - - if (matchLength < EQUAL_READ32) { - ip += ((ip - anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */ - continue; - } - - /* let's try to find a better solution */ - if (depth >= 1) - while (ip < ilimit) { - ip++; - curr++; - /* check repCode */ - if (offset) { - const U32 repIndex = (U32)(curr - offset_1); - const BYTE *const repBase = repIndex < dictLimit ? dictBase : base; - const BYTE *const repMatch = repBase + repIndex; - if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ - if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) { - /* repcode detected */ - const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend; - size_t const repLength = - ZSTD_count_2segments(ip + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repEnd, prefixStart) + - EQUAL_READ32; - int const gain2 = (int)(repLength * 3); - int const gain1 = (int)(matchLength * 3 - ZSTD_highbit32((U32)offset + 1) + 1); - if ((repLength >= EQUAL_READ32) && (gain2 > gain1)) - matchLength = repLength, offset = 0, start = ip; - } - } - - /* search match, depth 1 */ - { - size_t offset2 = 99999999; - size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); - int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */ - int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 4); - if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { - matchLength = ml2, offset = offset2, start = ip; - continue; /* search a better one */ - } - } - - /* let's find an even better one */ - if ((depth == 2) && (ip < ilimit)) { - ip++; - curr++; - /* check repCode */ - if (offset) { - const U32 repIndex = (U32)(curr - offset_1); - const BYTE *const repBase = repIndex < dictLimit ? dictBase : base; - const BYTE *const repMatch = repBase + repIndex; - if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ - if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) { - /* repcode detected */ - const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend; - size_t repLength = ZSTD_count_2segments(ip + EQUAL_READ32, repMatch + EQUAL_READ32, iend, - repEnd, prefixStart) + - EQUAL_READ32; - int gain2 = (int)(repLength * 4); - int gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 1); - if ((repLength >= EQUAL_READ32) && (gain2 > gain1)) - matchLength = repLength, offset = 0, start = ip; - } - } - - /* search match, depth 2 */ - { - size_t offset2 = 99999999; - size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); - int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */ - int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 7); - if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { - matchLength = ml2, offset = offset2, start = ip; - continue; - } - } - } - break; /* nothing found : store previous solution */ - } - - /* catch up */ - if (offset) { - U32 const matchIndex = (U32)((start - base) - (offset - ZSTD_REP_MOVE)); - const BYTE *match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex; - const BYTE *const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart; - while ((start > anchor) && (match > mStart) && (start[-1] == match[-1])) { - start--; - match--; - matchLength++; - } /* catch up */ - offset_2 = offset_1; - offset_1 = (U32)(offset - ZSTD_REP_MOVE); - } - - /* store sequence */ - _storeSequence : { - size_t const litLength = start - anchor; - ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength - MINMATCH); - anchor = ip = start + matchLength; - } - - /* check immediate repcode */ - while (ip <= ilimit) { - const U32 repIndex = (U32)((ip - base) - offset_2); - const BYTE *const repBase = repIndex < dictLimit ? dictBase : base; - const BYTE *const repMatch = repBase + repIndex; - if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ - if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) { - /* repcode detected we should take it */ - const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend; - matchLength = - ZSTD_count_2segments(ip + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; - offset = offset_2; - offset_2 = offset_1; - offset_1 = (U32)offset; /* swap offset history */ - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength - MINMATCH); - ip += matchLength; - anchor = ip; - continue; /* faster when present ... (?) */ - } - break; - } - } - - /* Save reps for next block */ - ctx->repToConfirm[0] = offset_1; - ctx->repToConfirm[1] = offset_2; - - /* Last Literals */ - { - size_t const lastLLSize = iend - anchor; - memcpy(seqStorePtr->lit, anchor, lastLLSize); - seqStorePtr->lit += lastLLSize; - } -} - -void ZSTD_compressBlock_greedy_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 0); } - -static void ZSTD_compressBlock_lazy_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -{ - ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 1); -} - -static void ZSTD_compressBlock_lazy2_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -{ - ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 2); -} - -static void ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -{ - ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 1, 2); -} - -/* The optimal parser */ -#include "zstd_opt.h" - -static void ZSTD_compressBlock_btopt(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -{ -#ifdef ZSTD_OPT_H_91842398743 - ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 0); -#else - (void)ctx; - (void)src; - (void)srcSize; - return; -#endif -} - -static void ZSTD_compressBlock_btopt2(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -{ -#ifdef ZSTD_OPT_H_91842398743 - ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 1); -#else - (void)ctx; - (void)src; - (void)srcSize; - return; -#endif -} - -static void ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -{ -#ifdef ZSTD_OPT_H_91842398743 - ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 0); -#else - (void)ctx; - (void)src; - (void)srcSize; - return; -#endif -} - -static void ZSTD_compressBlock_btopt2_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -{ -#ifdef ZSTD_OPT_H_91842398743 - ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 1); -#else - (void)ctx; - (void)src; - (void)srcSize; - return; -#endif -} - -typedef void (*ZSTD_blockCompressor)(ZSTD_CCtx *ctx, const void *src, size_t srcSize); - -static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict) -{ - static const ZSTD_blockCompressor blockCompressor[2][8] = { - {ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, - ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt, ZSTD_compressBlock_btopt2}, - {ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict, - ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btopt2_extDict}}; - - return blockCompressor[extDict][(U32)strat]; -} - -static size_t ZSTD_compressBlock_internal(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->params.cParams.strategy, zc->lowLimit < zc->dictLimit); - const BYTE *const base = zc->base; - const BYTE *const istart = (const BYTE *)src; - const U32 curr = (U32)(istart - base); - if (srcSize < MIN_CBLOCK_SIZE + ZSTD_blockHeaderSize + 1) - return 0; /* don't even attempt compression below a certain srcSize */ - ZSTD_resetSeqStore(&(zc->seqStore)); - if (curr > zc->nextToUpdate + 384) - zc->nextToUpdate = curr - MIN(192, (U32)(curr - zc->nextToUpdate - 384)); /* update tree not updated after finding very long rep matches */ - blockCompressor(zc, src, srcSize); - return ZSTD_compressSequences(zc, dst, dstCapacity, srcSize); -} - -/*! ZSTD_compress_generic() : -* Compress a chunk of data into one or multiple blocks. -* All blocks will be terminated, all input will be consumed. -* Function will issue an error if there is not enough `dstCapacity` to hold the compressed content. -* Frame is supposed already started (header already produced) -* @return : compressed size, or an error code -*/ -static size_t ZSTD_compress_generic(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, U32 lastFrameChunk) -{ - size_t blockSize = cctx->blockSize; - size_t remaining = srcSize; - const BYTE *ip = (const BYTE *)src; - BYTE *const ostart = (BYTE *)dst; - BYTE *op = ostart; - U32 const maxDist = 1 << cctx->params.cParams.windowLog; - - if (cctx->params.fParams.checksumFlag && srcSize) - xxh64_update(&cctx->xxhState, src, srcSize); - - while (remaining) { - U32 const lastBlock = lastFrameChunk & (blockSize >= remaining); - size_t cSize; - - if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) - return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ - if (remaining < blockSize) - blockSize = remaining; - - /* preemptive overflow correction */ - if (cctx->lowLimit > (3U << 29)) { - U32 const cycleMask = (1 << ZSTD_cycleLog(cctx->params.cParams.hashLog, cctx->params.cParams.strategy)) - 1; - U32 const curr = (U32)(ip - cctx->base); - U32 const newCurr = (curr & cycleMask) + (1 << cctx->params.cParams.windowLog); - U32 const correction = curr - newCurr; - ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_64 <= 30); - ZSTD_reduceIndex(cctx, correction); - cctx->base += correction; - cctx->dictBase += correction; - cctx->lowLimit -= correction; - cctx->dictLimit -= correction; - if (cctx->nextToUpdate < correction) - cctx->nextToUpdate = 0; - else - cctx->nextToUpdate -= correction; - } - - if ((U32)(ip + blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) { - /* enforce maxDist */ - U32 const newLowLimit = (U32)(ip + blockSize - cctx->base) - maxDist; - if (cctx->lowLimit < newLowLimit) - cctx->lowLimit = newLowLimit; - if (cctx->dictLimit < cctx->lowLimit) - cctx->dictLimit = cctx->lowLimit; - } - - cSize = ZSTD_compressBlock_internal(cctx, op + ZSTD_blockHeaderSize, dstCapacity - ZSTD_blockHeaderSize, ip, blockSize); - if (ZSTD_isError(cSize)) - return cSize; - - if (cSize == 0) { /* block is not compressible */ - U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw) << 1) + (U32)(blockSize << 3); - if (blockSize + ZSTD_blockHeaderSize > dstCapacity) - return ERROR(dstSize_tooSmall); - ZSTD_writeLE32(op, cBlockHeader24); /* no pb, 4th byte will be overwritten */ - memcpy(op + ZSTD_blockHeaderSize, ip, blockSize); - cSize = ZSTD_blockHeaderSize + blockSize; - } else { - U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed) << 1) + (U32)(cSize << 3); - ZSTD_writeLE24(op, cBlockHeader24); - cSize += ZSTD_blockHeaderSize; - } - - remaining -= blockSize; - dstCapacity -= cSize; - ip += blockSize; - op += cSize; - } - - if (lastFrameChunk && (op > ostart)) - cctx->stage = ZSTDcs_ending; - return op - ostart; -} - -static size_t ZSTD_writeFrameHeader(void *dst, size_t dstCapacity, ZSTD_parameters params, U64 pledgedSrcSize, U32 dictID) -{ - BYTE *const op = (BYTE *)dst; - U32 const dictIDSizeCode = (dictID > 0) + (dictID >= 256) + (dictID >= 65536); /* 0-3 */ - U32 const checksumFlag = params.fParams.checksumFlag > 0; - U32 const windowSize = 1U << params.cParams.windowLog; - U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize); - BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); - U32 const fcsCode = - params.fParams.contentSizeFlag ? (pledgedSrcSize >= 256) + (pledgedSrcSize >= 65536 + 256) + (pledgedSrcSize >= 0xFFFFFFFFU) : 0; /* 0-3 */ - BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag << 2) + (singleSegment << 5) + (fcsCode << 6)); - size_t pos; - - if (dstCapacity < ZSTD_frameHeaderSize_max) - return ERROR(dstSize_tooSmall); - - ZSTD_writeLE32(dst, ZSTD_MAGICNUMBER); - op[4] = frameHeaderDecriptionByte; - pos = 5; - if (!singleSegment) - op[pos++] = windowLogByte; - switch (dictIDSizeCode) { - default: /* impossible */ - case 0: break; - case 1: - op[pos] = (BYTE)(dictID); - pos++; - break; - case 2: - ZSTD_writeLE16(op + pos, (U16)dictID); - pos += 2; - break; - case 3: - ZSTD_writeLE32(op + pos, dictID); - pos += 4; - break; - } - switch (fcsCode) { - default: /* impossible */ - case 0: - if (singleSegment) - op[pos++] = (BYTE)(pledgedSrcSize); - break; - case 1: - ZSTD_writeLE16(op + pos, (U16)(pledgedSrcSize - 256)); - pos += 2; - break; - case 2: - ZSTD_writeLE32(op + pos, (U32)(pledgedSrcSize)); - pos += 4; - break; - case 3: - ZSTD_writeLE64(op + pos, (U64)(pledgedSrcSize)); - pos += 8; - break; - } - return pos; -} - -static size_t ZSTD_compressContinue_internal(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, U32 frame, U32 lastFrameChunk) -{ - const BYTE *const ip = (const BYTE *)src; - size_t fhSize = 0; - - if (cctx->stage == ZSTDcs_created) - return ERROR(stage_wrong); /* missing init (ZSTD_compressBegin) */ - - if (frame && (cctx->stage == ZSTDcs_init)) { - fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, cctx->frameContentSize, cctx->dictID); - if (ZSTD_isError(fhSize)) - return fhSize; - dstCapacity -= fhSize; - dst = (char *)dst + fhSize; - cctx->stage = ZSTDcs_ongoing; - } - - /* Check if blocks follow each other */ - if (src != cctx->nextSrc) { - /* not contiguous */ - ptrdiff_t const delta = cctx->nextSrc - ip; - cctx->lowLimit = cctx->dictLimit; - cctx->dictLimit = (U32)(cctx->nextSrc - cctx->base); - cctx->dictBase = cctx->base; - cctx->base -= delta; - cctx->nextToUpdate = cctx->dictLimit; - if (cctx->dictLimit - cctx->lowLimit < HASH_READ_SIZE) - cctx->lowLimit = cctx->dictLimit; /* too small extDict */ - } - - /* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */ - if ((ip + srcSize > cctx->dictBase + cctx->lowLimit) & (ip < cctx->dictBase + cctx->dictLimit)) { - ptrdiff_t const highInputIdx = (ip + srcSize) - cctx->dictBase; - U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)cctx->dictLimit) ? cctx->dictLimit : (U32)highInputIdx; - cctx->lowLimit = lowLimitMax; - } - - cctx->nextSrc = ip + srcSize; - - if (srcSize) { - size_t const cSize = frame ? ZSTD_compress_generic(cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) - : ZSTD_compressBlock_internal(cctx, dst, dstCapacity, src, srcSize); - if (ZSTD_isError(cSize)) - return cSize; - return cSize + fhSize; - } else - return fhSize; -} - -size_t ZSTD_compressContinue(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 0); -} - -size_t ZSTD_getBlockSizeMax(ZSTD_CCtx *cctx) { return MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, 1 << cctx->params.cParams.windowLog); } - -size_t ZSTD_compressBlock(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - size_t const blockSizeMax = ZSTD_getBlockSizeMax(cctx); - if (srcSize > blockSizeMax) - return ERROR(srcSize_wrong); - return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0, 0); -} - -/*! ZSTD_loadDictionaryContent() : - * @return : 0, or an error code - */ -static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx *zc, const void *src, size_t srcSize) -{ - const BYTE *const ip = (const BYTE *)src; - const BYTE *const iend = ip + srcSize; - - /* input becomes curr prefix */ - zc->lowLimit = zc->dictLimit; - zc->dictLimit = (U32)(zc->nextSrc - zc->base); - zc->dictBase = zc->base; - zc->base += ip - zc->nextSrc; - zc->nextToUpdate = zc->dictLimit; - zc->loadedDictEnd = zc->forceWindow ? 0 : (U32)(iend - zc->base); - - zc->nextSrc = iend; - if (srcSize <= HASH_READ_SIZE) - return 0; - - switch (zc->params.cParams.strategy) { - case ZSTD_fast: ZSTD_fillHashTable(zc, iend, zc->params.cParams.searchLength); break; - - case ZSTD_dfast: ZSTD_fillDoubleHashTable(zc, iend, zc->params.cParams.searchLength); break; - - case ZSTD_greedy: - case ZSTD_lazy: - case ZSTD_lazy2: - if (srcSize >= HASH_READ_SIZE) - ZSTD_insertAndFindFirstIndex(zc, iend - HASH_READ_SIZE, zc->params.cParams.searchLength); - break; - - case ZSTD_btlazy2: - case ZSTD_btopt: - case ZSTD_btopt2: - if (srcSize >= HASH_READ_SIZE) - ZSTD_updateTree(zc, iend - HASH_READ_SIZE, iend, 1 << zc->params.cParams.searchLog, zc->params.cParams.searchLength); - break; - - default: - return ERROR(GENERIC); /* strategy doesn't exist; impossible */ - } - - zc->nextToUpdate = (U32)(iend - zc->base); - return 0; -} - -/* Dictionaries that assign zero probability to symbols that show up causes problems - when FSE encoding. Refuse dictionaries that assign zero probability to symbols - that we may encounter during compression. - NOTE: This behavior is not standard and could be improved in the future. */ -static size_t ZSTD_checkDictNCount(short *normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue) -{ - U32 s; - if (dictMaxSymbolValue < maxSymbolValue) - return ERROR(dictionary_corrupted); - for (s = 0; s <= maxSymbolValue; ++s) { - if (normalizedCounter[s] == 0) - return ERROR(dictionary_corrupted); - } - return 0; -} - -/* Dictionary format : - * See : - * https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#dictionary-format - */ -/*! ZSTD_loadZstdDictionary() : - * @return : 0, or an error code - * assumptions : magic number supposed already checked - * dictSize supposed > 8 - */ -static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx *cctx, const void *dict, size_t dictSize) -{ - const BYTE *dictPtr = (const BYTE *)dict; - const BYTE *const dictEnd = dictPtr + dictSize; - short offcodeNCount[MaxOff + 1]; - unsigned offcodeMaxValue = MaxOff; - - dictPtr += 4; /* skip magic number */ - cctx->dictID = cctx->params.fParams.noDictIDFlag ? 0 : ZSTD_readLE32(dictPtr); - dictPtr += 4; - - { - size_t const hufHeaderSize = HUF_readCTable_wksp(cctx->hufTable, 255, dictPtr, dictEnd - dictPtr, cctx->tmpCounters, sizeof(cctx->tmpCounters)); - if (HUF_isError(hufHeaderSize)) - return ERROR(dictionary_corrupted); - dictPtr += hufHeaderSize; - } - - { - unsigned offcodeLog; - size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd - dictPtr); - if (FSE_isError(offcodeHeaderSize)) - return ERROR(dictionary_corrupted); - if (offcodeLog > OffFSELog) - return ERROR(dictionary_corrupted); - /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */ - CHECK_E(FSE_buildCTable_wksp(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog, cctx->tmpCounters, sizeof(cctx->tmpCounters)), - dictionary_corrupted); - dictPtr += offcodeHeaderSize; - } - - { - short matchlengthNCount[MaxML + 1]; - unsigned matchlengthMaxValue = MaxML, matchlengthLog; - size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd - dictPtr); - if (FSE_isError(matchlengthHeaderSize)) - return ERROR(dictionary_corrupted); - if (matchlengthLog > MLFSELog) - return ERROR(dictionary_corrupted); - /* Every match length code must have non-zero probability */ - CHECK_F(ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML)); - CHECK_E( - FSE_buildCTable_wksp(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, cctx->tmpCounters, sizeof(cctx->tmpCounters)), - dictionary_corrupted); - dictPtr += matchlengthHeaderSize; - } - - { - short litlengthNCount[MaxLL + 1]; - unsigned litlengthMaxValue = MaxLL, litlengthLog; - size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd - dictPtr); - if (FSE_isError(litlengthHeaderSize)) - return ERROR(dictionary_corrupted); - if (litlengthLog > LLFSELog) - return ERROR(dictionary_corrupted); - /* Every literal length code must have non-zero probability */ - CHECK_F(ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL)); - CHECK_E(FSE_buildCTable_wksp(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, cctx->tmpCounters, sizeof(cctx->tmpCounters)), - dictionary_corrupted); - dictPtr += litlengthHeaderSize; - } - - if (dictPtr + 12 > dictEnd) - return ERROR(dictionary_corrupted); - cctx->rep[0] = ZSTD_readLE32(dictPtr + 0); - cctx->rep[1] = ZSTD_readLE32(dictPtr + 4); - cctx->rep[2] = ZSTD_readLE32(dictPtr + 8); - dictPtr += 12; - - { - size_t const dictContentSize = (size_t)(dictEnd - dictPtr); - U32 offcodeMax = MaxOff; - if (dictContentSize <= ((U32)-1) - 128 KB) { - U32 const maxOffset = (U32)dictContentSize + 128 KB; /* The maximum offset that must be supported */ - offcodeMax = ZSTD_highbit32(maxOffset); /* Calculate minimum offset code required to represent maxOffset */ - } - /* All offset values <= dictContentSize + 128 KB must be representable */ - CHECK_F(ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff))); - /* All repCodes must be <= dictContentSize and != 0*/ - { - U32 u; - for (u = 0; u < 3; u++) { - if (cctx->rep[u] == 0) - return ERROR(dictionary_corrupted); - if (cctx->rep[u] > dictContentSize) - return ERROR(dictionary_corrupted); - } - } - - cctx->flagStaticTables = 1; - cctx->flagStaticHufTable = HUF_repeat_valid; - return ZSTD_loadDictionaryContent(cctx, dictPtr, dictContentSize); - } -} - -/** ZSTD_compress_insertDictionary() : -* @return : 0, or an error code */ -static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx *cctx, const void *dict, size_t dictSize) -{ - if ((dict == NULL) || (dictSize <= 8)) - return 0; - - /* dict as pure content */ - if ((ZSTD_readLE32(dict) != ZSTD_DICT_MAGIC) || (cctx->forceRawDict)) - return ZSTD_loadDictionaryContent(cctx, dict, dictSize); - - /* dict as zstd dictionary */ - return ZSTD_loadZstdDictionary(cctx, dict, dictSize); -} - -/*! ZSTD_compressBegin_internal() : -* @return : 0, or an error code */ -static size_t ZSTD_compressBegin_internal(ZSTD_CCtx *cctx, const void *dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize) -{ - ZSTD_compResetPolicy_e const crp = dictSize ? ZSTDcrp_fullReset : ZSTDcrp_continue; - CHECK_F(ZSTD_resetCCtx_advanced(cctx, params, pledgedSrcSize, crp)); - return ZSTD_compress_insertDictionary(cctx, dict, dictSize); -} - -/*! ZSTD_compressBegin_advanced() : -* @return : 0, or an error code */ -size_t ZSTD_compressBegin_advanced(ZSTD_CCtx *cctx, const void *dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize) -{ - /* compression parameters verification and optimization */ - CHECK_F(ZSTD_checkCParams(params.cParams)); - return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, pledgedSrcSize); -} - -size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx *cctx, const void *dict, size_t dictSize, int compressionLevel) -{ - ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); - return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, 0); -} - -size_t ZSTD_compressBegin(ZSTD_CCtx *cctx, int compressionLevel) { return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel); } - -/*! ZSTD_writeEpilogue() : -* Ends a frame. -* @return : nb of bytes written into dst (or an error code) */ -static size_t ZSTD_writeEpilogue(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity) -{ - BYTE *const ostart = (BYTE *)dst; - BYTE *op = ostart; - size_t fhSize = 0; - - if (cctx->stage == ZSTDcs_created) - return ERROR(stage_wrong); /* init missing */ - - /* special case : empty frame */ - if (cctx->stage == ZSTDcs_init) { - fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, 0, 0); - if (ZSTD_isError(fhSize)) - return fhSize; - dstCapacity -= fhSize; - op += fhSize; - cctx->stage = ZSTDcs_ongoing; - } - - if (cctx->stage != ZSTDcs_ending) { - /* write one last empty block, make it the "last" block */ - U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw) << 1) + 0; - if (dstCapacity < 4) - return ERROR(dstSize_tooSmall); - ZSTD_writeLE32(op, cBlockHeader24); - op += ZSTD_blockHeaderSize; - dstCapacity -= ZSTD_blockHeaderSize; - } - - if (cctx->params.fParams.checksumFlag) { - U32 const checksum = (U32)xxh64_digest(&cctx->xxhState); - if (dstCapacity < 4) - return ERROR(dstSize_tooSmall); - ZSTD_writeLE32(op, checksum); - op += 4; - } - - cctx->stage = ZSTDcs_created; /* return to "created but no init" status */ - return op - ostart; -} - -size_t ZSTD_compressEnd(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - size_t endResult; - size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 1); - if (ZSTD_isError(cSize)) - return cSize; - endResult = ZSTD_writeEpilogue(cctx, (char *)dst + cSize, dstCapacity - cSize); - if (ZSTD_isError(endResult)) - return endResult; - return cSize + endResult; -} - -static size_t ZSTD_compress_internal(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize, - ZSTD_parameters params) -{ - CHECK_F(ZSTD_compressBegin_internal(cctx, dict, dictSize, params, srcSize)); - return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); -} - -size_t ZSTD_compress_usingDict(ZSTD_CCtx *ctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize, - ZSTD_parameters params) -{ - return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params); -} - -size_t ZSTD_compressCCtx(ZSTD_CCtx *ctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, ZSTD_parameters params) -{ - return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, NULL, 0, params); -} - -/* ===== Dictionary API ===== */ - -struct ZSTD_CDict_s { - void *dictBuffer; - const void *dictContent; - size_t dictContentSize; - ZSTD_CCtx *refContext; -}; /* typedef'd tp ZSTD_CDict within "zstd.h" */ - -size_t ZSTD_CDictWorkspaceBound(ZSTD_compressionParameters cParams) { return ZSTD_CCtxWorkspaceBound(cParams) + ZSTD_ALIGN(sizeof(ZSTD_CDict)); } - -static ZSTD_CDict *ZSTD_createCDict_advanced(const void *dictBuffer, size_t dictSize, unsigned byReference, ZSTD_parameters params, ZSTD_customMem customMem) -{ - if (!customMem.customAlloc || !customMem.customFree) - return NULL; - - { - ZSTD_CDict *const cdict = (ZSTD_CDict *)ZSTD_malloc(sizeof(ZSTD_CDict), customMem); - ZSTD_CCtx *const cctx = ZSTD_createCCtx_advanced(customMem); - - if (!cdict || !cctx) { - ZSTD_free(cdict, customMem); - ZSTD_freeCCtx(cctx); - return NULL; - } - - if ((byReference) || (!dictBuffer) || (!dictSize)) { - cdict->dictBuffer = NULL; - cdict->dictContent = dictBuffer; - } else { - void *const internalBuffer = ZSTD_malloc(dictSize, customMem); - if (!internalBuffer) { - ZSTD_free(cctx, customMem); - ZSTD_free(cdict, customMem); - return NULL; - } - memcpy(internalBuffer, dictBuffer, dictSize); - cdict->dictBuffer = internalBuffer; - cdict->dictContent = internalBuffer; - } - - { - size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0); - if (ZSTD_isError(errorCode)) { - ZSTD_free(cdict->dictBuffer, customMem); - ZSTD_free(cdict, customMem); - ZSTD_freeCCtx(cctx); - return NULL; - } - } - - cdict->refContext = cctx; - cdict->dictContentSize = dictSize; - return cdict; - } -} - -ZSTD_CDict *ZSTD_initCDict(const void *dict, size_t dictSize, ZSTD_parameters params, void *workspace, size_t workspaceSize) -{ - ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize); - return ZSTD_createCDict_advanced(dict, dictSize, 1, params, stackMem); -} - -size_t ZSTD_freeCDict(ZSTD_CDict *cdict) -{ - if (cdict == NULL) - return 0; /* support free on NULL */ - { - ZSTD_customMem const cMem = cdict->refContext->customMem; - ZSTD_freeCCtx(cdict->refContext); - ZSTD_free(cdict->dictBuffer, cMem); - ZSTD_free(cdict, cMem); - return 0; - } -} - -static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict *cdict) { return ZSTD_getParamsFromCCtx(cdict->refContext); } - -size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx *cctx, const ZSTD_CDict *cdict, unsigned long long pledgedSrcSize) -{ - if (cdict->dictContentSize) - CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize)) - else { - ZSTD_parameters params = cdict->refContext->params; - params.fParams.contentSizeFlag = (pledgedSrcSize > 0); - CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, params, pledgedSrcSize)); - } - return 0; -} - -/*! ZSTD_compress_usingCDict() : -* Compression using a digested Dictionary. -* Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. -* Note that compression level is decided during dictionary creation */ -size_t ZSTD_compress_usingCDict(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const ZSTD_CDict *cdict) -{ - CHECK_F(ZSTD_compressBegin_usingCDict(cctx, cdict, srcSize)); - - if (cdict->refContext->params.fParams.contentSizeFlag == 1) { - cctx->params.fParams.contentSizeFlag = 1; - cctx->frameContentSize = srcSize; - } else { - cctx->params.fParams.contentSizeFlag = 0; - } - - return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); -} - -/* ****************************************************************** -* Streaming -********************************************************************/ - -typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage; - -struct ZSTD_CStream_s { - ZSTD_CCtx *cctx; - ZSTD_CDict *cdictLocal; - const ZSTD_CDict *cdict; - char *inBuff; - size_t inBuffSize; - size_t inToCompress; - size_t inBuffPos; - size_t inBuffTarget; - size_t blockSize; - char *outBuff; - size_t outBuffSize; - size_t outBuffContentSize; - size_t outBuffFlushedSize; - ZSTD_cStreamStage stage; - U32 checksum; - U32 frameEnded; - U64 pledgedSrcSize; - U64 inputProcessed; - ZSTD_parameters params; - ZSTD_customMem customMem; -}; /* typedef'd to ZSTD_CStream within "zstd.h" */ - -size_t ZSTD_CStreamWorkspaceBound(ZSTD_compressionParameters cParams) -{ - size_t const inBuffSize = (size_t)1 << cParams.windowLog; - size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, inBuffSize); - size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1; - - return ZSTD_CCtxWorkspaceBound(cParams) + ZSTD_ALIGN(sizeof(ZSTD_CStream)) + ZSTD_ALIGN(inBuffSize) + ZSTD_ALIGN(outBuffSize); -} - -ZSTD_CStream *ZSTD_createCStream_advanced(ZSTD_customMem customMem) -{ - ZSTD_CStream *zcs; - - if (!customMem.customAlloc || !customMem.customFree) - return NULL; - - zcs = (ZSTD_CStream *)ZSTD_malloc(sizeof(ZSTD_CStream), customMem); - if (zcs == NULL) - return NULL; - memset(zcs, 0, sizeof(ZSTD_CStream)); - memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem)); - zcs->cctx = ZSTD_createCCtx_advanced(customMem); - if (zcs->cctx == NULL) { - ZSTD_freeCStream(zcs); - return NULL; - } - return zcs; -} - -size_t ZSTD_freeCStream(ZSTD_CStream *zcs) -{ - if (zcs == NULL) - return 0; /* support free on NULL */ - { - ZSTD_customMem const cMem = zcs->customMem; - ZSTD_freeCCtx(zcs->cctx); - zcs->cctx = NULL; - ZSTD_freeCDict(zcs->cdictLocal); - zcs->cdictLocal = NULL; - ZSTD_free(zcs->inBuff, cMem); - zcs->inBuff = NULL; - ZSTD_free(zcs->outBuff, cMem); - zcs->outBuff = NULL; - ZSTD_free(zcs, cMem); - return 0; - } -} - -/*====== Initialization ======*/ - -size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } -size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */; } - -static size_t ZSTD_resetCStream_internal(ZSTD_CStream *zcs, unsigned long long pledgedSrcSize) -{ - if (zcs->inBuffSize == 0) - return ERROR(stage_wrong); /* zcs has not been init at least once => can't reset */ - - if (zcs->cdict) - CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize)) - else - CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize)); - - zcs->inToCompress = 0; - zcs->inBuffPos = 0; - zcs->inBuffTarget = zcs->blockSize; - zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; - zcs->stage = zcss_load; - zcs->frameEnded = 0; - zcs->pledgedSrcSize = pledgedSrcSize; - zcs->inputProcessed = 0; - return 0; /* ready to go */ -} - -size_t ZSTD_resetCStream(ZSTD_CStream *zcs, unsigned long long pledgedSrcSize) -{ - - zcs->params.fParams.contentSizeFlag = (pledgedSrcSize > 0); - - return ZSTD_resetCStream_internal(zcs, pledgedSrcSize); -} - -static size_t ZSTD_initCStream_advanced(ZSTD_CStream *zcs, const void *dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize) -{ - /* allocate buffers */ - { - size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; - if (zcs->inBuffSize < neededInBuffSize) { - zcs->inBuffSize = neededInBuffSize; - ZSTD_free(zcs->inBuff, zcs->customMem); - zcs->inBuff = (char *)ZSTD_malloc(neededInBuffSize, zcs->customMem); - if (zcs->inBuff == NULL) - return ERROR(memory_allocation); - } - zcs->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize); - } - if (zcs->outBuffSize < ZSTD_compressBound(zcs->blockSize) + 1) { - zcs->outBuffSize = ZSTD_compressBound(zcs->blockSize) + 1; - ZSTD_free(zcs->outBuff, zcs->customMem); - zcs->outBuff = (char *)ZSTD_malloc(zcs->outBuffSize, zcs->customMem); - if (zcs->outBuff == NULL) - return ERROR(memory_allocation); - } - - if (dict && dictSize >= 8) { - ZSTD_freeCDict(zcs->cdictLocal); - zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0, params, zcs->customMem); - if (zcs->cdictLocal == NULL) - return ERROR(memory_allocation); - zcs->cdict = zcs->cdictLocal; - } else - zcs->cdict = NULL; - - zcs->checksum = params.fParams.checksumFlag > 0; - zcs->params = params; - - return ZSTD_resetCStream_internal(zcs, pledgedSrcSize); -} - -ZSTD_CStream *ZSTD_initCStream(ZSTD_parameters params, unsigned long long pledgedSrcSize, void *workspace, size_t workspaceSize) -{ - ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize); - ZSTD_CStream *const zcs = ZSTD_createCStream_advanced(stackMem); - if (zcs) { - size_t const code = ZSTD_initCStream_advanced(zcs, NULL, 0, params, pledgedSrcSize); - if (ZSTD_isError(code)) { - return NULL; - } - } - return zcs; -} - -ZSTD_CStream *ZSTD_initCStream_usingCDict(const ZSTD_CDict *cdict, unsigned long long pledgedSrcSize, void *workspace, size_t workspaceSize) -{ - ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict); - ZSTD_CStream *const zcs = ZSTD_initCStream(params, pledgedSrcSize, workspace, workspaceSize); - if (zcs) { - zcs->cdict = cdict; - if (ZSTD_isError(ZSTD_resetCStream_internal(zcs, pledgedSrcSize))) { - return NULL; - } - } - return zcs; -} - -/*====== Compression ======*/ - -typedef enum { zsf_gather, zsf_flush, zsf_end } ZSTD_flush_e; - -ZSTD_STATIC size_t ZSTD_limitCopy(void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - size_t const length = MIN(dstCapacity, srcSize); - memcpy(dst, src, length); - return length; -} - -static size_t ZSTD_compressStream_generic(ZSTD_CStream *zcs, void *dst, size_t *dstCapacityPtr, const void *src, size_t *srcSizePtr, ZSTD_flush_e const flush) -{ - U32 someMoreWork = 1; - const char *const istart = (const char *)src; - const char *const iend = istart + *srcSizePtr; - const char *ip = istart; - char *const ostart = (char *)dst; - char *const oend = ostart + *dstCapacityPtr; - char *op = ostart; - - while (someMoreWork) { - switch (zcs->stage) { - case zcss_init: - return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */ - - case zcss_load: - /* complete inBuffer */ - { - size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos; - size_t const loaded = ZSTD_limitCopy(zcs->inBuff + zcs->inBuffPos, toLoad, ip, iend - ip); - zcs->inBuffPos += loaded; - ip += loaded; - if ((zcs->inBuffPos == zcs->inToCompress) || (!flush && (toLoad != loaded))) { - someMoreWork = 0; - break; /* not enough input to get a full block : stop there, wait for more */ - } - } - /* compress curr block (note : this stage cannot be stopped in the middle) */ - { - void *cDst; - size_t cSize; - size_t const iSize = zcs->inBuffPos - zcs->inToCompress; - size_t oSize = oend - op; - if (oSize >= ZSTD_compressBound(iSize)) - cDst = op; /* compress directly into output buffer (avoid flush stage) */ - else - cDst = zcs->outBuff, oSize = zcs->outBuffSize; - cSize = (flush == zsf_end) ? ZSTD_compressEnd(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) - : ZSTD_compressContinue(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize); - if (ZSTD_isError(cSize)) - return cSize; - if (flush == zsf_end) - zcs->frameEnded = 1; - /* prepare next block */ - zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize; - if (zcs->inBuffTarget > zcs->inBuffSize) - zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; /* note : inBuffSize >= blockSize */ - zcs->inToCompress = zcs->inBuffPos; - if (cDst == op) { - op += cSize; - break; - } /* no need to flush */ - zcs->outBuffContentSize = cSize; - zcs->outBuffFlushedSize = 0; - zcs->stage = zcss_flush; /* pass-through to flush stage */ - } - fallthrough; - - case zcss_flush: { - size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; - size_t const flushed = ZSTD_limitCopy(op, oend - op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush); - op += flushed; - zcs->outBuffFlushedSize += flushed; - if (toFlush != flushed) { - someMoreWork = 0; - break; - } /* dst too small to store flushed data : stop there */ - zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; - zcs->stage = zcss_load; - break; - } - - case zcss_final: - someMoreWork = 0; /* do nothing */ - break; - - default: - return ERROR(GENERIC); /* impossible */ - } - } - - *srcSizePtr = ip - istart; - *dstCapacityPtr = op - ostart; - zcs->inputProcessed += *srcSizePtr; - if (zcs->frameEnded) - return 0; - { - size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos; - if (hintInSize == 0) - hintInSize = zcs->blockSize; - return hintInSize; - } -} - -size_t ZSTD_compressStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output, ZSTD_inBuffer *input) -{ - size_t sizeRead = input->size - input->pos; - size_t sizeWritten = output->size - output->pos; - size_t const result = - ZSTD_compressStream_generic(zcs, (char *)(output->dst) + output->pos, &sizeWritten, (const char *)(input->src) + input->pos, &sizeRead, zsf_gather); - input->pos += sizeRead; - output->pos += sizeWritten; - return result; -} - -/*====== Finalize ======*/ - -/*! ZSTD_flushStream() : -* @return : amount of data remaining to flush */ -size_t ZSTD_flushStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output) -{ - size_t srcSize = 0; - size_t sizeWritten = output->size - output->pos; - size_t const result = ZSTD_compressStream_generic(zcs, (char *)(output->dst) + output->pos, &sizeWritten, &srcSize, - &srcSize, /* use a valid src address instead of NULL */ - zsf_flush); - output->pos += sizeWritten; - if (ZSTD_isError(result)) - return result; - return zcs->outBuffContentSize - zcs->outBuffFlushedSize; /* remaining to flush */ -} - -size_t ZSTD_endStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output) -{ - BYTE *const ostart = (BYTE *)(output->dst) + output->pos; - BYTE *const oend = (BYTE *)(output->dst) + output->size; - BYTE *op = ostart; - - if ((zcs->pledgedSrcSize) && (zcs->inputProcessed != zcs->pledgedSrcSize)) - return ERROR(srcSize_wrong); /* pledgedSrcSize not respected */ - - if (zcs->stage != zcss_final) { - /* flush whatever remains */ - size_t srcSize = 0; - size_t sizeWritten = output->size - output->pos; - size_t const notEnded = - ZSTD_compressStream_generic(zcs, ostart, &sizeWritten, &srcSize, &srcSize, zsf_end); /* use a valid src address instead of NULL */ - size_t const remainingToFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; - op += sizeWritten; - if (remainingToFlush) { - output->pos += sizeWritten; - return remainingToFlush + ZSTD_BLOCKHEADERSIZE /* final empty block */ + (zcs->checksum * 4); - } - /* create epilogue */ - zcs->stage = zcss_final; - zcs->outBuffContentSize = !notEnded ? 0 : ZSTD_compressEnd(zcs->cctx, zcs->outBuff, zcs->outBuffSize, NULL, - 0); /* write epilogue, including final empty block, into outBuff */ - } - - /* flush epilogue */ - { - size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; - size_t const flushed = ZSTD_limitCopy(op, oend - op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush); - op += flushed; - zcs->outBuffFlushedSize += flushed; - output->pos += op - ostart; - if (toFlush == flushed) - zcs->stage = zcss_init; /* end reached */ - return toFlush - flushed; - } -} - -/*-===== Pre-defined compression levels =====-*/ - -#define ZSTD_DEFAULT_CLEVEL 1 -#define ZSTD_MAX_CLEVEL 22 -int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } - -static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL + 1] = { - { - /* "default" */ - /* W, C, H, S, L, TL, strat */ - {18, 12, 12, 1, 7, 16, ZSTD_fast}, /* level 0 - never used */ - {19, 13, 14, 1, 7, 16, ZSTD_fast}, /* level 1 */ - {19, 15, 16, 1, 6, 16, ZSTD_fast}, /* level 2 */ - {20, 16, 17, 1, 5, 16, ZSTD_dfast}, /* level 3.*/ - {20, 18, 18, 1, 5, 16, ZSTD_dfast}, /* level 4.*/ - {20, 15, 18, 3, 5, 16, ZSTD_greedy}, /* level 5 */ - {21, 16, 19, 2, 5, 16, ZSTD_lazy}, /* level 6 */ - {21, 17, 20, 3, 5, 16, ZSTD_lazy}, /* level 7 */ - {21, 18, 20, 3, 5, 16, ZSTD_lazy2}, /* level 8 */ - {21, 20, 20, 3, 5, 16, ZSTD_lazy2}, /* level 9 */ - {21, 19, 21, 4, 5, 16, ZSTD_lazy2}, /* level 10 */ - {22, 20, 22, 4, 5, 16, ZSTD_lazy2}, /* level 11 */ - {22, 20, 22, 5, 5, 16, ZSTD_lazy2}, /* level 12 */ - {22, 21, 22, 5, 5, 16, ZSTD_lazy2}, /* level 13 */ - {22, 21, 22, 6, 5, 16, ZSTD_lazy2}, /* level 14 */ - {22, 21, 21, 5, 5, 16, ZSTD_btlazy2}, /* level 15 */ - {23, 22, 22, 5, 5, 16, ZSTD_btlazy2}, /* level 16 */ - {23, 21, 22, 4, 5, 24, ZSTD_btopt}, /* level 17 */ - {23, 23, 22, 6, 5, 32, ZSTD_btopt}, /* level 18 */ - {23, 23, 22, 6, 3, 48, ZSTD_btopt}, /* level 19 */ - {25, 25, 23, 7, 3, 64, ZSTD_btopt2}, /* level 20 */ - {26, 26, 23, 7, 3, 256, ZSTD_btopt2}, /* level 21 */ - {27, 27, 25, 9, 3, 512, ZSTD_btopt2}, /* level 22 */ - }, - { - /* for srcSize <= 256 KB */ - /* W, C, H, S, L, T, strat */ - {0, 0, 0, 0, 0, 0, ZSTD_fast}, /* level 0 - not used */ - {18, 13, 14, 1, 6, 8, ZSTD_fast}, /* level 1 */ - {18, 14, 13, 1, 5, 8, ZSTD_dfast}, /* level 2 */ - {18, 16, 15, 1, 5, 8, ZSTD_dfast}, /* level 3 */ - {18, 15, 17, 1, 5, 8, ZSTD_greedy}, /* level 4.*/ - {18, 16, 17, 4, 5, 8, ZSTD_greedy}, /* level 5.*/ - {18, 16, 17, 3, 5, 8, ZSTD_lazy}, /* level 6.*/ - {18, 17, 17, 4, 4, 8, ZSTD_lazy}, /* level 7 */ - {18, 17, 17, 4, 4, 8, ZSTD_lazy2}, /* level 8 */ - {18, 17, 17, 5, 4, 8, ZSTD_lazy2}, /* level 9 */ - {18, 17, 17, 6, 4, 8, ZSTD_lazy2}, /* level 10 */ - {18, 18, 17, 6, 4, 8, ZSTD_lazy2}, /* level 11.*/ - {18, 18, 17, 7, 4, 8, ZSTD_lazy2}, /* level 12.*/ - {18, 19, 17, 6, 4, 8, ZSTD_btlazy2}, /* level 13 */ - {18, 18, 18, 4, 4, 16, ZSTD_btopt}, /* level 14.*/ - {18, 18, 18, 4, 3, 16, ZSTD_btopt}, /* level 15.*/ - {18, 19, 18, 6, 3, 32, ZSTD_btopt}, /* level 16.*/ - {18, 19, 18, 8, 3, 64, ZSTD_btopt}, /* level 17.*/ - {18, 19, 18, 9, 3, 128, ZSTD_btopt}, /* level 18.*/ - {18, 19, 18, 10, 3, 256, ZSTD_btopt}, /* level 19.*/ - {18, 19, 18, 11, 3, 512, ZSTD_btopt2}, /* level 20.*/ - {18, 19, 18, 12, 3, 512, ZSTD_btopt2}, /* level 21.*/ - {18, 19, 18, 13, 3, 512, ZSTD_btopt2}, /* level 22.*/ - }, - { - /* for srcSize <= 128 KB */ - /* W, C, H, S, L, T, strat */ - {17, 12, 12, 1, 7, 8, ZSTD_fast}, /* level 0 - not used */ - {17, 12, 13, 1, 6, 8, ZSTD_fast}, /* level 1 */ - {17, 13, 16, 1, 5, 8, ZSTD_fast}, /* level 2 */ - {17, 16, 16, 2, 5, 8, ZSTD_dfast}, /* level 3 */ - {17, 13, 15, 3, 4, 8, ZSTD_greedy}, /* level 4 */ - {17, 15, 17, 4, 4, 8, ZSTD_greedy}, /* level 5 */ - {17, 16, 17, 3, 4, 8, ZSTD_lazy}, /* level 6 */ - {17, 15, 17, 4, 4, 8, ZSTD_lazy2}, /* level 7 */ - {17, 17, 17, 4, 4, 8, ZSTD_lazy2}, /* level 8 */ - {17, 17, 17, 5, 4, 8, ZSTD_lazy2}, /* level 9 */ - {17, 17, 17, 6, 4, 8, ZSTD_lazy2}, /* level 10 */ - {17, 17, 17, 7, 4, 8, ZSTD_lazy2}, /* level 11 */ - {17, 17, 17, 8, 4, 8, ZSTD_lazy2}, /* level 12 */ - {17, 18, 17, 6, 4, 8, ZSTD_btlazy2}, /* level 13.*/ - {17, 17, 17, 7, 3, 8, ZSTD_btopt}, /* level 14.*/ - {17, 17, 17, 7, 3, 16, ZSTD_btopt}, /* level 15.*/ - {17, 18, 17, 7, 3, 32, ZSTD_btopt}, /* level 16.*/ - {17, 18, 17, 7, 3, 64, ZSTD_btopt}, /* level 17.*/ - {17, 18, 17, 7, 3, 256, ZSTD_btopt}, /* level 18.*/ - {17, 18, 17, 8, 3, 256, ZSTD_btopt}, /* level 19.*/ - {17, 18, 17, 9, 3, 256, ZSTD_btopt2}, /* level 20.*/ - {17, 18, 17, 10, 3, 256, ZSTD_btopt2}, /* level 21.*/ - {17, 18, 17, 11, 3, 512, ZSTD_btopt2}, /* level 22.*/ - }, - { - /* for srcSize <= 16 KB */ - /* W, C, H, S, L, T, strat */ - {14, 12, 12, 1, 7, 6, ZSTD_fast}, /* level 0 - not used */ - {14, 14, 14, 1, 6, 6, ZSTD_fast}, /* level 1 */ - {14, 14, 14, 1, 4, 6, ZSTD_fast}, /* level 2 */ - {14, 14, 14, 1, 4, 6, ZSTD_dfast}, /* level 3.*/ - {14, 14, 14, 4, 4, 6, ZSTD_greedy}, /* level 4.*/ - {14, 14, 14, 3, 4, 6, ZSTD_lazy}, /* level 5.*/ - {14, 14, 14, 4, 4, 6, ZSTD_lazy2}, /* level 6 */ - {14, 14, 14, 5, 4, 6, ZSTD_lazy2}, /* level 7 */ - {14, 14, 14, 6, 4, 6, ZSTD_lazy2}, /* level 8.*/ - {14, 15, 14, 6, 4, 6, ZSTD_btlazy2}, /* level 9.*/ - {14, 15, 14, 3, 3, 6, ZSTD_btopt}, /* level 10.*/ - {14, 15, 14, 6, 3, 8, ZSTD_btopt}, /* level 11.*/ - {14, 15, 14, 6, 3, 16, ZSTD_btopt}, /* level 12.*/ - {14, 15, 14, 6, 3, 24, ZSTD_btopt}, /* level 13.*/ - {14, 15, 15, 6, 3, 48, ZSTD_btopt}, /* level 14.*/ - {14, 15, 15, 6, 3, 64, ZSTD_btopt}, /* level 15.*/ - {14, 15, 15, 6, 3, 96, ZSTD_btopt}, /* level 16.*/ - {14, 15, 15, 6, 3, 128, ZSTD_btopt}, /* level 17.*/ - {14, 15, 15, 6, 3, 256, ZSTD_btopt}, /* level 18.*/ - {14, 15, 15, 7, 3, 256, ZSTD_btopt}, /* level 19.*/ - {14, 15, 15, 8, 3, 256, ZSTD_btopt2}, /* level 20.*/ - {14, 15, 15, 9, 3, 256, ZSTD_btopt2}, /* level 21.*/ - {14, 15, 15, 10, 3, 256, ZSTD_btopt2}, /* level 22.*/ - }, -}; - -/*! ZSTD_getCParams() : -* @return ZSTD_compressionParameters structure for a selected compression level, `srcSize` and `dictSize`. -* Size values are optional, provide 0 if not known or unused */ -ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize) -{ - ZSTD_compressionParameters cp; - size_t const addedSize = srcSize ? 0 : 500; - U64 const rSize = srcSize + dictSize ? srcSize + dictSize + addedSize : (U64)-1; - U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ - if (compressionLevel <= 0) - compressionLevel = ZSTD_DEFAULT_CLEVEL; /* 0 == default; no negative compressionLevel yet */ - if (compressionLevel > ZSTD_MAX_CLEVEL) - compressionLevel = ZSTD_MAX_CLEVEL; - cp = ZSTD_defaultCParameters[tableID][compressionLevel]; - if (ZSTD_32bits()) { /* auto-correction, for 32-bits mode */ - if (cp.windowLog > ZSTD_WINDOWLOG_MAX) - cp.windowLog = ZSTD_WINDOWLOG_MAX; - if (cp.chainLog > ZSTD_CHAINLOG_MAX) - cp.chainLog = ZSTD_CHAINLOG_MAX; - if (cp.hashLog > ZSTD_HASHLOG_MAX) - cp.hashLog = ZSTD_HASHLOG_MAX; - } - cp = ZSTD_adjustCParams(cp, srcSize, dictSize); - return cp; -} - -/*! ZSTD_getParams() : -* same as ZSTD_getCParams(), but @return a `ZSTD_parameters` object (instead of `ZSTD_compressionParameters`). -* All fields of `ZSTD_frameParameters` are set to default (0) */ -ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize) -{ - ZSTD_parameters params; - ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize); - memset(¶ms, 0, sizeof(params)); - params.cParams = cParams; - return params; -} - -EXPORT_SYMBOL(ZSTD_maxCLevel); -EXPORT_SYMBOL(ZSTD_compressBound); - -EXPORT_SYMBOL(ZSTD_CCtxWorkspaceBound); -EXPORT_SYMBOL(ZSTD_initCCtx); -EXPORT_SYMBOL(ZSTD_compressCCtx); -EXPORT_SYMBOL(ZSTD_compress_usingDict); - -EXPORT_SYMBOL(ZSTD_CDictWorkspaceBound); -EXPORT_SYMBOL(ZSTD_initCDict); -EXPORT_SYMBOL(ZSTD_compress_usingCDict); - -EXPORT_SYMBOL(ZSTD_CStreamWorkspaceBound); -EXPORT_SYMBOL(ZSTD_initCStream); -EXPORT_SYMBOL(ZSTD_initCStream_usingCDict); -EXPORT_SYMBOL(ZSTD_resetCStream); -EXPORT_SYMBOL(ZSTD_compressStream); -EXPORT_SYMBOL(ZSTD_flushStream); -EXPORT_SYMBOL(ZSTD_endStream); -EXPORT_SYMBOL(ZSTD_CStreamInSize); -EXPORT_SYMBOL(ZSTD_CStreamOutSize); - -EXPORT_SYMBOL(ZSTD_getCParams); -EXPORT_SYMBOL(ZSTD_getParams); -EXPORT_SYMBOL(ZSTD_checkCParams); -EXPORT_SYMBOL(ZSTD_adjustCParams); - -EXPORT_SYMBOL(ZSTD_compressBegin); -EXPORT_SYMBOL(ZSTD_compressBegin_usingDict); -EXPORT_SYMBOL(ZSTD_compressBegin_advanced); -EXPORT_SYMBOL(ZSTD_copyCCtx); -EXPORT_SYMBOL(ZSTD_compressBegin_usingCDict); -EXPORT_SYMBOL(ZSTD_compressContinue); -EXPORT_SYMBOL(ZSTD_compressEnd); - -EXPORT_SYMBOL(ZSTD_getBlockSizeMax); -EXPORT_SYMBOL(ZSTD_compressBlock); - -MODULE_LICENSE("Dual BSD/GPL"); -MODULE_DESCRIPTION("Zstd Compressor"); diff --git a/lib/zstd/compress/fse_compress.c b/lib/zstd/compress/fse_compress.c new file mode 100644 index 000000000000..436985b620e5 --- /dev/null +++ b/lib/zstd/compress/fse_compress.c @@ -0,0 +1,625 @@ +/* ****************************************************************** + * FSE : Finite State Entropy encoder + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* ************************************************************** +* Includes +****************************************************************/ +#include "../common/compiler.h" +#include "../common/mem.h" /* U32, U16, etc. */ +#include "../common/debug.h" /* assert, DEBUGLOG */ +#include "hist.h" /* HIST_count_wksp */ +#include "../common/bitstream.h" +#define FSE_STATIC_LINKING_ONLY +#include "../common/fse.h" +#include "../common/error_private.h" +#define ZSTD_DEPS_NEED_MALLOC +#define ZSTD_DEPS_NEED_MATH64 +#include "../common/zstd_deps.h" /* ZSTD_malloc, ZSTD_free, ZSTD_memcpy, ZSTD_memset */ + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define FSE_isError ERR_isError + + +/* ************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +# error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +# error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X,Y) X##Y +#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) +#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) + + +/* Function templates */ + +/* FSE_buildCTable_wksp() : + * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). + * wkspSize should be sized to handle worst case situation, which is `1<>1 : 1) ; + FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); + U32 const step = FSE_TABLESTEP(tableSize); + + U32* cumul = (U32*)workSpace; + FSE_FUNCTION_TYPE* tableSymbol = (FSE_FUNCTION_TYPE*)(cumul + (maxSymbolValue + 2)); + + U32 highThreshold = tableSize-1; + + if ((size_t)workSpace & 3) return ERROR(GENERIC); /* Must be 4 byte aligned */ + if (FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) > wkspSize) return ERROR(tableLog_tooLarge); + /* CTable header */ + tableU16[-2] = (U16) tableLog; + tableU16[-1] = (U16) maxSymbolValue; + assert(tableLog < 16); /* required for threshold strategy to work */ + + /* For explanations on how to distribute symbol values over the table : + * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ + + #ifdef __clang_analyzer__ + ZSTD_memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */ + #endif + + /* symbol start positions */ + { U32 u; + cumul[0] = 0; + for (u=1; u <= maxSymbolValue+1; u++) { + if (normalizedCounter[u-1]==-1) { /* Low proba symbol */ + cumul[u] = cumul[u-1] + 1; + tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1); + } else { + cumul[u] = cumul[u-1] + normalizedCounter[u-1]; + } } + cumul[maxSymbolValue+1] = tableSize+1; + } + + /* Spread symbols */ + { U32 position = 0; + U32 symbol; + for (symbol=0; symbol<=maxSymbolValue; symbol++) { + int nbOccurrences; + int const freq = normalizedCounter[symbol]; + for (nbOccurrences=0; nbOccurrences highThreshold) + position = (position + step) & tableMask; /* Low proba area */ + } } + + assert(position==0); /* Must have initialized all positions */ + } + + /* Build table */ + { U32 u; for (u=0; u> 3) + 3; + return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ +} + +static size_t +FSE_writeNCount_generic (void* header, size_t headerBufferSize, + const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, + unsigned writeIsSafe) +{ + BYTE* const ostart = (BYTE*) header; + BYTE* out = ostart; + BYTE* const oend = ostart + headerBufferSize; + int nbBits; + const int tableSize = 1 << tableLog; + int remaining; + int threshold; + U32 bitStream = 0; + int bitCount = 0; + unsigned symbol = 0; + unsigned const alphabetSize = maxSymbolValue + 1; + int previousIs0 = 0; + + /* Table Size */ + bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount; + bitCount += 4; + + /* Init */ + remaining = tableSize+1; /* +1 for extra accuracy */ + threshold = tableSize; + nbBits = tableLog+1; + + while ((symbol < alphabetSize) && (remaining>1)) { /* stops at 1 */ + if (previousIs0) { + unsigned start = symbol; + while ((symbol < alphabetSize) && !normalizedCounter[symbol]) symbol++; + if (symbol == alphabetSize) break; /* incorrect distribution */ + while (symbol >= start+24) { + start+=24; + bitStream += 0xFFFFU << bitCount; + if ((!writeIsSafe) && (out > oend-2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE) bitStream; + out[1] = (BYTE)(bitStream>>8); + out+=2; + bitStream>>=16; + } + while (symbol >= start+3) { + start+=3; + bitStream += 3 << bitCount; + bitCount += 2; + } + bitStream += (symbol-start) << bitCount; + bitCount += 2; + if (bitCount>16) { + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out += 2; + bitStream >>= 16; + bitCount -= 16; + } } + { int count = normalizedCounter[symbol++]; + int const max = (2*threshold-1) - remaining; + remaining -= count < 0 ? -count : count; + count++; /* +1 for extra accuracy */ + if (count>=threshold) + count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ + bitStream += count << bitCount; + bitCount += nbBits; + bitCount -= (count>=1; } + } + if (bitCount>16) { + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out += 2; + bitStream >>= 16; + bitCount -= 16; + } } + + if (remaining != 1) + return ERROR(GENERIC); /* incorrect normalized distribution */ + assert(symbol <= alphabetSize); + + /* flush remaining bitStream */ + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out+= (bitCount+7) /8; + + return (out-ostart); +} + + +size_t FSE_writeNCount (void* buffer, size_t bufferSize, + const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +{ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported */ + if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */ + + if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) + return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); + + return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1 /* write in buffer is safe */); +} + + +/*-************************************************************** +* FSE Compression Code +****************************************************************/ + +FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog) +{ + size_t size; + if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; + size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32); + return (FSE_CTable*)ZSTD_malloc(size); +} + +void FSE_freeCTable (FSE_CTable* ct) { ZSTD_free(ct); } + +/* provides the minimum logSize to safely represent a distribution */ +static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) +{ + U32 minBitsSrc = BIT_highbit32((U32)(srcSize)) + 1; + U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2; + U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; + assert(srcSize > 1); /* Not supported, RLE should be used instead */ + return minBits; +} + +unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus) +{ + U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus; + U32 tableLog = maxTableLog; + U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); + assert(srcSize > 1); /* Not supported, RLE should be used instead */ + if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; + if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */ + if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */ + if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG; + if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG; + return tableLog; +} + +unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) +{ + return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2); +} + +/* Secondary normalization method. + To be used when primary method fails. */ + +static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue, short lowProbCount) +{ + short const NOT_YET_ASSIGNED = -2; + U32 s; + U32 distributed = 0; + U32 ToDistribute; + + /* Init */ + U32 const lowThreshold = (U32)(total >> tableLog); + U32 lowOne = (U32)((total * 3) >> (tableLog + 1)); + + for (s=0; s<=maxSymbolValue; s++) { + if (count[s] == 0) { + norm[s]=0; + continue; + } + if (count[s] <= lowThreshold) { + norm[s] = lowProbCount; + distributed++; + total -= count[s]; + continue; + } + if (count[s] <= lowOne) { + norm[s] = 1; + distributed++; + total -= count[s]; + continue; + } + + norm[s]=NOT_YET_ASSIGNED; + } + ToDistribute = (1 << tableLog) - distributed; + + if (ToDistribute == 0) + return 0; + + if ((total / ToDistribute) > lowOne) { + /* risk of rounding to zero */ + lowOne = (U32)((total * 3) / (ToDistribute * 2)); + for (s=0; s<=maxSymbolValue; s++) { + if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) { + norm[s] = 1; + distributed++; + total -= count[s]; + continue; + } } + ToDistribute = (1 << tableLog) - distributed; + } + + if (distributed == maxSymbolValue+1) { + /* all values are pretty poor; + probably incompressible data (should have already been detected); + find max, then give all remaining points to max */ + U32 maxV = 0, maxC = 0; + for (s=0; s<=maxSymbolValue; s++) + if (count[s] > maxC) { maxV=s; maxC=count[s]; } + norm[maxV] += (short)ToDistribute; + return 0; + } + + if (total == 0) { + /* all of the symbols were low enough for the lowOne or lowThreshold */ + for (s=0; ToDistribute > 0; s = (s+1)%(maxSymbolValue+1)) + if (norm[s] > 0) { ToDistribute--; norm[s]++; } + return 0; + } + + { U64 const vStepLog = 62 - tableLog; + U64 const mid = (1ULL << (vStepLog-1)) - 1; + U64 const rStep = ZSTD_div64((((U64)1<> vStepLog); + U32 const sEnd = (U32)(end >> vStepLog); + U32 const weight = sEnd - sStart; + if (weight < 1) + return ERROR(GENERIC); + norm[s] = (short)weight; + tmpTotal = end; + } } } + + return 0; +} + +size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, + const unsigned* count, size_t total, + unsigned maxSymbolValue, unsigned useLowProbCount) +{ + /* Sanity checks */ + if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; + if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported size */ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported size */ + if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */ + + { static U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 }; + short const lowProbCount = useLowProbCount ? -1 : 1; + U64 const scale = 62 - tableLog; + U64 const step = ZSTD_div64((U64)1<<62, (U32)total); /* <== here, one division ! */ + U64 const vStep = 1ULL<<(scale-20); + int stillToDistribute = 1<> tableLog); + + for (s=0; s<=maxSymbolValue; s++) { + if (count[s] == total) return 0; /* rle special case */ + if (count[s] == 0) { normalizedCounter[s]=0; continue; } + if (count[s] <= lowThreshold) { + normalizedCounter[s] = lowProbCount; + stillToDistribute--; + } else { + short proba = (short)((count[s]*step) >> scale); + if (proba<8) { + U64 restToBeat = vStep * rtbTable[proba]; + proba += (count[s]*step) - ((U64)proba< restToBeat; + } + if (proba > largestP) { largestP=proba; largest=s; } + normalizedCounter[s] = proba; + stillToDistribute -= proba; + } } + if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) { + /* corner case, need another normalization method */ + size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue, lowProbCount); + if (FSE_isError(errorCode)) return errorCode; + } + else normalizedCounter[largest] += (short)stillToDistribute; + } + +#if 0 + { /* Print Table (debug) */ + U32 s; + U32 nTotal = 0; + for (s=0; s<=maxSymbolValue; s++) + RAWLOG(2, "%3i: %4i \n", s, normalizedCounter[s]); + for (s=0; s<=maxSymbolValue; s++) + nTotal += abs(normalizedCounter[s]); + if (nTotal != (1U<>1); /* assumption : tableLog >= 1 */ + FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); + unsigned s; + + /* Sanity checks */ + if (nbBits < 1) return ERROR(GENERIC); /* min size */ + + /* header */ + tableU16[-2] = (U16) nbBits; + tableU16[-1] = (U16) maxSymbolValue; + + /* Build table */ + for (s=0; s FSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) { /* test bit 2 */ + FSE_encodeSymbol(&bitC, &CState2, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + FSE_FLUSHBITS(&bitC); + } + + /* 2 or 4 encoding per loop */ + while ( ip>istart ) { + + FSE_encodeSymbol(&bitC, &CState2, *--ip); + + if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */ + FSE_FLUSHBITS(&bitC); + + FSE_encodeSymbol(&bitC, &CState1, *--ip); + + if (sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) { /* this test must be static */ + FSE_encodeSymbol(&bitC, &CState2, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + } + + FSE_FLUSHBITS(&bitC); + } + + FSE_flushCState(&bitC, &CState2); + FSE_flushCState(&bitC, &CState1); + return BIT_closeCStream(&bitC); +} + +size_t FSE_compress_usingCTable (void* dst, size_t dstSize, + const void* src, size_t srcSize, + const FSE_CTable* ct) +{ + unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize)); + + if (fast) + return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1); + else + return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0); +} + + +size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); } + + +#endif /* FSE_COMMONDEFS_ONLY */ diff --git a/lib/zstd/compress/hist.c b/lib/zstd/compress/hist.c new file mode 100644 index 000000000000..3ddc6dfb6894 --- /dev/null +++ b/lib/zstd/compress/hist.c @@ -0,0 +1,165 @@ +/* ****************************************************************** + * hist : Histogram functions + * part of Finite State Entropy project + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* --- dependencies --- */ +#include "../common/mem.h" /* U32, BYTE, etc. */ +#include "../common/debug.h" /* assert, DEBUGLOG */ +#include "../common/error_private.h" /* ERROR */ +#include "hist.h" + + +/* --- Error management --- */ +unsigned HIST_isError(size_t code) { return ERR_isError(code); } + +/*-************************************************************** + * Histogram functions + ****************************************************************/ +unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + const BYTE* const end = ip + srcSize; + unsigned maxSymbolValue = *maxSymbolValuePtr; + unsigned largestCount=0; + + ZSTD_memset(count, 0, (maxSymbolValue+1) * sizeof(*count)); + if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; } + + while (ip largestCount) largestCount = count[s]; + } + + return largestCount; +} + +typedef enum { trustInput, checkMaxSymbolValue } HIST_checkInput_e; + +/* HIST_count_parallel_wksp() : + * store histogram into 4 intermediate tables, recombined at the end. + * this design makes better use of OoO cpus, + * and is noticeably faster when some values are heavily repeated. + * But it needs some additional workspace for intermediate tables. + * `workSpace` must be a U32 table of size >= HIST_WKSP_SIZE_U32. + * @return : largest histogram frequency, + * or an error code (notably when histogram's alphabet is larger than *maxSymbolValuePtr) */ +static size_t HIST_count_parallel_wksp( + unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, + HIST_checkInput_e check, + U32* const workSpace) +{ + const BYTE* ip = (const BYTE*)source; + const BYTE* const iend = ip+sourceSize; + size_t const countSize = (*maxSymbolValuePtr + 1) * sizeof(*count); + unsigned max=0; + U32* const Counting1 = workSpace; + U32* const Counting2 = Counting1 + 256; + U32* const Counting3 = Counting2 + 256; + U32* const Counting4 = Counting3 + 256; + + /* safety checks */ + assert(*maxSymbolValuePtr <= 255); + if (!sourceSize) { + ZSTD_memset(count, 0, countSize); + *maxSymbolValuePtr = 0; + return 0; + } + ZSTD_memset(workSpace, 0, 4*256*sizeof(unsigned)); + + /* by stripes of 16 bytes */ + { U32 cached = MEM_read32(ip); ip += 4; + while (ip < iend-15) { + U32 c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + } + ip-=4; + } + + /* finish last symbols */ + while (ip max) max = Counting1[s]; + } } + + { unsigned maxSymbolValue = 255; + while (!Counting1[maxSymbolValue]) maxSymbolValue--; + if (check && maxSymbolValue > *maxSymbolValuePtr) return ERROR(maxSymbolValue_tooSmall); + *maxSymbolValuePtr = maxSymbolValue; + ZSTD_memmove(count, Counting1, countSize); /* in case count & Counting1 are overlapping */ + } + return (size_t)max; +} + +/* HIST_countFast_wksp() : + * Same as HIST_countFast(), but using an externally provided scratch buffer. + * `workSpace` is a writable buffer which must be 4-bytes aligned, + * `workSpaceSize` must be >= HIST_WKSP_SIZE + */ +size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, + void* workSpace, size_t workSpaceSize) +{ + if (sourceSize < 1500) /* heuristic threshold */ + return HIST_count_simple(count, maxSymbolValuePtr, source, sourceSize); + if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ + if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall); + return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, trustInput, (U32*)workSpace); +} + +/* HIST_count_wksp() : + * Same as HIST_count(), but using an externally provided scratch buffer. + * `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */ +size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, + void* workSpace, size_t workSpaceSize) +{ + if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ + if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall); + if (*maxSymbolValuePtr < 255) + return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, checkMaxSymbolValue, (U32*)workSpace); + *maxSymbolValuePtr = 255; + return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace, workSpaceSize); +} + diff --git a/lib/zstd/compress/hist.h b/lib/zstd/compress/hist.h new file mode 100644 index 000000000000..fc1830abc9c6 --- /dev/null +++ b/lib/zstd/compress/hist.h @@ -0,0 +1,75 @@ +/* ****************************************************************** + * hist : Histogram functions + * part of Finite State Entropy project + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* --- dependencies --- */ +#include "../common/zstd_deps.h" /* size_t */ + + +/* --- simple histogram functions --- */ + +/*! HIST_count(): + * Provides the precise count of each byte within a table 'count'. + * 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1). + * Updates *maxSymbolValuePtr with actual largest symbol value detected. + * @return : count of the most frequent symbol (which isn't identified). + * or an error code, which can be tested using HIST_isError(). + * note : if return == srcSize, there is only one symbol. + */ +size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize); + +unsigned HIST_isError(size_t code); /*< tells if a return value is an error code */ + + +/* --- advanced histogram functions --- */ + +#define HIST_WKSP_SIZE_U32 1024 +#define HIST_WKSP_SIZE (HIST_WKSP_SIZE_U32 * sizeof(unsigned)) +/* HIST_count_wksp() : + * Same as HIST_count(), but using an externally provided scratch buffer. + * Benefit is this function will use very little stack space. + * `workSpace` is a writable buffer which must be 4-bytes aligned, + * `workSpaceSize` must be >= HIST_WKSP_SIZE + */ +size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize, + void* workSpace, size_t workSpaceSize); + +/* HIST_countFast() : + * same as HIST_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr. + * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` + */ +size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize); + +/* HIST_countFast_wksp() : + * Same as HIST_countFast(), but using an externally provided scratch buffer. + * `workSpace` is a writable buffer which must be 4-bytes aligned, + * `workSpaceSize` must be >= HIST_WKSP_SIZE + */ +size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize, + void* workSpace, size_t workSpaceSize); + +/*! HIST_count_simple() : + * Same as HIST_countFast(), this function is unsafe, + * and will segfault if any value within `src` is `> *maxSymbolValuePtr`. + * It is also a bit slower for large inputs. + * However, it does not need any additional memory (not even on stack). + * @return : count of the most frequent symbol. + * Note this function doesn't produce any error (i.e. it must succeed). + */ +unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize); diff --git a/lib/zstd/compress/huf_compress.c b/lib/zstd/compress/huf_compress.c new file mode 100644 index 000000000000..122e812642e2 --- /dev/null +++ b/lib/zstd/compress/huf_compress.c @@ -0,0 +1,902 @@ +/* ****************************************************************** + * Huffman encoder, part of New Generation Entropy library + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* ************************************************************** +* Compiler specifics +****************************************************************/ + + +/* ************************************************************** +* Includes +****************************************************************/ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */ +#include "../common/compiler.h" +#include "../common/bitstream.h" +#include "hist.h" +#define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */ +#include "../common/fse.h" /* header compression */ +#define HUF_STATIC_LINKING_ONLY +#include "../common/huf.h" +#include "../common/error_private.h" + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_isError ERR_isError +#define HUF_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */ + + +/* ************************************************************** +* Utils +****************************************************************/ +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) +{ + return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); +} + + +/* ******************************************************* +* HUF : Huffman block compression +*********************************************************/ +/* HUF_compressWeights() : + * Same as FSE_compress(), but dedicated to huff0's weights compression. + * The use case needs much less stack memory. + * Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX. + */ +#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6 + +typedef struct { + FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)]; + U32 scratchBuffer[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(HUF_TABLELOG_MAX, MAX_FSE_TABLELOG_FOR_HUFF_HEADER)]; + unsigned count[HUF_TABLELOG_MAX+1]; + S16 norm[HUF_TABLELOG_MAX+1]; +} HUF_CompressWeightsWksp; + +static size_t HUF_compressWeights(void* dst, size_t dstSize, const void* weightTable, size_t wtSize, void* workspace, size_t workspaceSize) +{ + BYTE* const ostart = (BYTE*) dst; + BYTE* op = ostart; + BYTE* const oend = ostart + dstSize; + + unsigned maxSymbolValue = HUF_TABLELOG_MAX; + U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER; + HUF_CompressWeightsWksp* wksp = (HUF_CompressWeightsWksp*)workspace; + + if (workspaceSize < sizeof(HUF_CompressWeightsWksp)) return ERROR(GENERIC); + + /* init conditions */ + if (wtSize <= 1) return 0; /* Not compressible */ + + /* Scan input and build symbol stats */ + { unsigned const maxCount = HIST_count_simple(wksp->count, &maxSymbolValue, weightTable, wtSize); /* never fails */ + if (maxCount == wtSize) return 1; /* only a single symbol in src : rle */ + if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */ + } + + tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue); + CHECK_F( FSE_normalizeCount(wksp->norm, tableLog, wksp->count, wtSize, maxSymbolValue, /* useLowProbCount */ 0) ); + + /* Write table description header */ + { CHECK_V_F(hSize, FSE_writeNCount(op, (size_t)(oend-op), wksp->norm, maxSymbolValue, tableLog) ); + op += hSize; + } + + /* Compress */ + CHECK_F( FSE_buildCTable_wksp(wksp->CTable, wksp->norm, maxSymbolValue, tableLog, wksp->scratchBuffer, sizeof(wksp->scratchBuffer)) ); + { CHECK_V_F(cSize, FSE_compress_usingCTable(op, (size_t)(oend - op), weightTable, wtSize, wksp->CTable) ); + if (cSize == 0) return 0; /* not enough space for compressed data */ + op += cSize; + } + + return (size_t)(op-ostart); +} + + +typedef struct { + HUF_CompressWeightsWksp wksp; + BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */ + BYTE huffWeight[HUF_SYMBOLVALUE_MAX]; +} HUF_WriteCTableWksp; + +size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, + const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, + void* workspace, size_t workspaceSize) +{ + BYTE* op = (BYTE*)dst; + U32 n; + HUF_WriteCTableWksp* wksp = (HUF_WriteCTableWksp*)workspace; + + /* check conditions */ + if (workspaceSize < sizeof(HUF_WriteCTableWksp)) return ERROR(GENERIC); + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge); + + /* convert to weight */ + wksp->bitsToWeight[0] = 0; + for (n=1; nbitsToWeight[n] = (BYTE)(huffLog + 1 - n); + for (n=0; nhuffWeight[n] = wksp->bitsToWeight[CTable[n].nbBits]; + + /* attempt weights compression by FSE */ + { CHECK_V_F(hSize, HUF_compressWeights(op+1, maxDstSize-1, wksp->huffWeight, maxSymbolValue, &wksp->wksp, sizeof(wksp->wksp)) ); + if ((hSize>1) & (hSize < maxSymbolValue/2)) { /* FSE compressed */ + op[0] = (BYTE)hSize; + return hSize+1; + } } + + /* write raw values as 4-bits (max : 15) */ + if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */ + if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */ + op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1)); + wksp->huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */ + for (n=0; nhuffWeight[n] << 4) + wksp->huffWeight[n+1]); + return ((maxSymbolValue+1)/2) + 1; +} + +/*! HUF_writeCTable() : + `CTable` : Huffman tree to save, using huf representation. + @return : size of saved CTable */ +size_t HUF_writeCTable (void* dst, size_t maxDstSize, + const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog) +{ + HUF_WriteCTableWksp wksp; + return HUF_writeCTable_wksp(dst, maxDstSize, CTable, maxSymbolValue, huffLog, &wksp, sizeof(wksp)); +} + + +size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* hasZeroWeights) +{ + BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; /* init not required, even though some static analyzer may complain */ + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */ + U32 tableLog = 0; + U32 nbSymbols = 0; + + /* get symbol weights */ + CHECK_V_F(readSize, HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize)); + *hasZeroWeights = (rankVal[0] > 0); + + /* check result */ + if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + if (nbSymbols > *maxSymbolValuePtr+1) return ERROR(maxSymbolValue_tooSmall); + + /* Prepare base value per rank */ + { U32 n, nextRankStart = 0; + for (n=1; n<=tableLog; n++) { + U32 curr = nextRankStart; + nextRankStart += (rankVal[n] << (n-1)); + rankVal[n] = curr; + } } + + /* fill nbBits */ + { U32 n; for (n=0; nn=tableLog+1 */ + U16 valPerRank[HUF_TABLELOG_MAX+2] = {0}; + { U32 n; for (n=0; n0; n--) { /* start at n=tablelog <-> w=1 */ + valPerRank[n] = min; /* get starting value within each rank */ + min += nbPerRank[n]; + min >>= 1; + } } + /* assign value within rank, symbol order */ + { U32 n; for (n=0; n maxNbBits to be maxNbBits. Then it adjusts + * the tree to so that it is a valid canonical Huffman tree. + * + * @pre The sum of the ranks of each symbol == 2^largestBits, + * where largestBits == huffNode[lastNonNull].nbBits. + * @post The sum of the ranks of each symbol == 2^largestBits, + * where largestBits is the return value <= maxNbBits. + * + * @param huffNode The Huffman tree modified in place to enforce maxNbBits. + * @param lastNonNull The symbol with the lowest count in the Huffman tree. + * @param maxNbBits The maximum allowed number of bits, which the Huffman tree + * may not respect. After this function the Huffman tree will + * respect maxNbBits. + * @return The maximum number of bits of the Huffman tree after adjustment, + * necessarily no more than maxNbBits. + */ +static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) +{ + const U32 largestBits = huffNode[lastNonNull].nbBits; + /* early exit : no elt > maxNbBits, so the tree is already valid. */ + if (largestBits <= maxNbBits) return largestBits; + + /* there are several too large elements (at least >= 2) */ + { int totalCost = 0; + const U32 baseCost = 1 << (largestBits - maxNbBits); + int n = (int)lastNonNull; + + /* Adjust any ranks > maxNbBits to maxNbBits. + * Compute totalCost, which is how far the sum of the ranks is + * we are over 2^largestBits after adjust the offending ranks. + */ + while (huffNode[n].nbBits > maxNbBits) { + totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); + huffNode[n].nbBits = (BYTE)maxNbBits; + n--; + } + /* n stops at huffNode[n].nbBits <= maxNbBits */ + assert(huffNode[n].nbBits <= maxNbBits); + /* n end at index of smallest symbol using < maxNbBits */ + while (huffNode[n].nbBits == maxNbBits) --n; + + /* renorm totalCost from 2^largestBits to 2^maxNbBits + * note : totalCost is necessarily a multiple of baseCost */ + assert((totalCost & (baseCost - 1)) == 0); + totalCost >>= (largestBits - maxNbBits); + assert(totalCost > 0); + + /* repay normalized cost */ + { U32 const noSymbol = 0xF0F0F0F0; + U32 rankLast[HUF_TABLELOG_MAX+2]; + + /* Get pos of last (smallest = lowest cum. count) symbol per rank */ + ZSTD_memset(rankLast, 0xF0, sizeof(rankLast)); + { U32 currentNbBits = maxNbBits; + int pos; + for (pos=n ; pos >= 0; pos--) { + if (huffNode[pos].nbBits >= currentNbBits) continue; + currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */ + rankLast[maxNbBits-currentNbBits] = (U32)pos; + } } + + while (totalCost > 0) { + /* Try to reduce the next power of 2 above totalCost because we + * gain back half the rank. + */ + U32 nBitsToDecrease = BIT_highbit32((U32)totalCost) + 1; + for ( ; nBitsToDecrease > 1; nBitsToDecrease--) { + U32 const highPos = rankLast[nBitsToDecrease]; + U32 const lowPos = rankLast[nBitsToDecrease-1]; + if (highPos == noSymbol) continue; + /* Decrease highPos if no symbols of lowPos or if it is + * not cheaper to remove 2 lowPos than highPos. + */ + if (lowPos == noSymbol) break; + { U32 const highTotal = huffNode[highPos].count; + U32 const lowTotal = 2 * huffNode[lowPos].count; + if (highTotal <= lowTotal) break; + } } + /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ + assert(rankLast[nBitsToDecrease] != noSymbol || nBitsToDecrease == 1); + /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ + while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) + nBitsToDecrease++; + assert(rankLast[nBitsToDecrease] != noSymbol); + /* Increase the number of bits to gain back half the rank cost. */ + totalCost -= 1 << (nBitsToDecrease-1); + huffNode[rankLast[nBitsToDecrease]].nbBits++; + + /* Fix up the new rank. + * If the new rank was empty, this symbol is now its smallest. + * Otherwise, this symbol will be the largest in the new rank so no adjustment. + */ + if (rankLast[nBitsToDecrease-1] == noSymbol) + rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; + /* Fix up the old rank. + * If the symbol was at position 0, meaning it was the highest weight symbol in the tree, + * it must be the only symbol in its rank, so the old rank now has no symbols. + * Otherwise, since the Huffman nodes are sorted by count, the previous position is now + * the smallest node in the rank. If the previous position belongs to a different rank, + * then the rank is now empty. + */ + if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */ + rankLast[nBitsToDecrease] = noSymbol; + else { + rankLast[nBitsToDecrease]--; + if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease) + rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ + } + } /* while (totalCost > 0) */ + + /* If we've removed too much weight, then we have to add it back. + * To avoid overshooting again, we only adjust the smallest rank. + * We take the largest nodes from the lowest rank 0 and move them + * to rank 1. There's guaranteed to be enough rank 0 symbols because + * TODO. + */ + while (totalCost < 0) { /* Sometimes, cost correction overshoot */ + /* special case : no rank 1 symbol (using maxNbBits-1); + * let's create one from largest rank 0 (using maxNbBits). + */ + if (rankLast[1] == noSymbol) { + while (huffNode[n].nbBits == maxNbBits) n--; + huffNode[n+1].nbBits--; + assert(n >= 0); + rankLast[1] = (U32)(n+1); + totalCost++; + continue; + } + huffNode[ rankLast[1] + 1 ].nbBits--; + rankLast[1]++; + totalCost ++; + } + } /* repay normalized cost */ + } /* there are several too large elements (at least >= 2) */ + + return maxNbBits; +} + +typedef struct { + U32 base; + U32 curr; +} rankPos; + +typedef nodeElt huffNodeTable[HUF_CTABLE_WORKSPACE_SIZE_U32]; + +#define RANK_POSITION_TABLE_SIZE 32 + +typedef struct { + huffNodeTable huffNodeTbl; + rankPos rankPosition[RANK_POSITION_TABLE_SIZE]; +} HUF_buildCTable_wksp_tables; + +/* + * HUF_sort(): + * Sorts the symbols [0, maxSymbolValue] by count[symbol] in decreasing order. + * + * @param[out] huffNode Sorted symbols by decreasing count. Only members `.count` and `.byte` are filled. + * Must have (maxSymbolValue + 1) entries. + * @param[in] count Histogram of the symbols. + * @param[in] maxSymbolValue Maximum symbol value. + * @param rankPosition This is a scratch workspace. Must have RANK_POSITION_TABLE_SIZE entries. + */ +static void HUF_sort(nodeElt* huffNode, const unsigned* count, U32 maxSymbolValue, rankPos* rankPosition) +{ + int n; + int const maxSymbolValue1 = (int)maxSymbolValue + 1; + + /* Compute base and set curr to base. + * For symbol s let lowerRank = BIT_highbit32(count[n]+1) and rank = lowerRank + 1. + * Then 2^lowerRank <= count[n]+1 <= 2^rank. + * We attribute each symbol to lowerRank's base value, because we want to know where + * each rank begins in the output, so for rank R we want to count ranks R+1 and above. + */ + ZSTD_memset(rankPosition, 0, sizeof(*rankPosition) * RANK_POSITION_TABLE_SIZE); + for (n = 0; n < maxSymbolValue1; ++n) { + U32 lowerRank = BIT_highbit32(count[n] + 1); + rankPosition[lowerRank].base++; + } + assert(rankPosition[RANK_POSITION_TABLE_SIZE - 1].base == 0); + for (n = RANK_POSITION_TABLE_SIZE - 1; n > 0; --n) { + rankPosition[n-1].base += rankPosition[n].base; + rankPosition[n-1].curr = rankPosition[n-1].base; + } + /* Sort */ + for (n = 0; n < maxSymbolValue1; ++n) { + U32 const c = count[n]; + U32 const r = BIT_highbit32(c+1) + 1; + U32 pos = rankPosition[r].curr++; + /* Insert into the correct position in the rank. + * We have at most 256 symbols, so this insertion should be fine. + */ + while ((pos > rankPosition[r].base) && (c > huffNode[pos-1].count)) { + huffNode[pos] = huffNode[pos-1]; + pos--; + } + huffNode[pos].count = c; + huffNode[pos].byte = (BYTE)n; + } +} + + +/* HUF_buildCTable_wksp() : + * Same as HUF_buildCTable(), but using externally allocated scratch buffer. + * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as sizeof(HUF_buildCTable_wksp_tables). + */ +#define STARTNODE (HUF_SYMBOLVALUE_MAX+1) + +/* HUF_buildTree(): + * Takes the huffNode array sorted by HUF_sort() and builds an unlimited-depth Huffman tree. + * + * @param huffNode The array sorted by HUF_sort(). Builds the Huffman tree in this array. + * @param maxSymbolValue The maximum symbol value. + * @return The smallest node in the Huffman tree (by count). + */ +static int HUF_buildTree(nodeElt* huffNode, U32 maxSymbolValue) +{ + nodeElt* const huffNode0 = huffNode - 1; + int nonNullRank; + int lowS, lowN; + int nodeNb = STARTNODE; + int n, nodeRoot; + /* init for parents */ + nonNullRank = (int)maxSymbolValue; + while(huffNode[nonNullRank].count == 0) nonNullRank--; + lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb; + huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count; + huffNode[lowS].parent = huffNode[lowS-1].parent = (U16)nodeNb; + nodeNb++; lowS-=2; + for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30); + huffNode0[0].count = (U32)(1U<<31); /* fake entry, strong barrier */ + + /* create parents */ + while (nodeNb <= nodeRoot) { + int const n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + int const n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count; + huffNode[n1].parent = huffNode[n2].parent = (U16)nodeNb; + nodeNb++; + } + + /* distribute weights (unlimited tree height) */ + huffNode[nodeRoot].nbBits = 0; + for (n=nodeRoot-1; n>=STARTNODE; n--) + huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; + for (n=0; n<=nonNullRank; n++) + huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; + + return nonNullRank; +} + +/* + * HUF_buildCTableFromTree(): + * Build the CTable given the Huffman tree in huffNode. + * + * @param[out] CTable The output Huffman CTable. + * @param huffNode The Huffman tree. + * @param nonNullRank The last and smallest node in the Huffman tree. + * @param maxSymbolValue The maximum symbol value. + * @param maxNbBits The exact maximum number of bits used in the Huffman tree. + */ +static void HUF_buildCTableFromTree(HUF_CElt* CTable, nodeElt const* huffNode, int nonNullRank, U32 maxSymbolValue, U32 maxNbBits) +{ + /* fill result into ctable (val, nbBits) */ + int n; + U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0}; + U16 valPerRank[HUF_TABLELOG_MAX+1] = {0}; + int const alphabetSize = (int)(maxSymbolValue + 1); + for (n=0; n<=nonNullRank; n++) + nbPerRank[huffNode[n].nbBits]++; + /* determine starting value per rank */ + { U16 min = 0; + for (n=(int)maxNbBits; n>0; n--) { + valPerRank[n] = min; /* get starting value within each rank */ + min += nbPerRank[n]; + min >>= 1; + } } + for (n=0; nhuffNodeTbl; + nodeElt* const huffNode = huffNode0+1; + int nonNullRank; + + /* safety checks */ + if (((size_t)workSpace & 3) != 0) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ + if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) + return ERROR(workSpace_tooSmall); + if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT; + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) + return ERROR(maxSymbolValue_tooLarge); + ZSTD_memset(huffNode0, 0, sizeof(huffNodeTable)); + + /* sort, decreasing order */ + HUF_sort(huffNode, count, maxSymbolValue, wksp_tables->rankPosition); + + /* build tree */ + nonNullRank = HUF_buildTree(huffNode, maxSymbolValue); + + /* enforce maxTableLog */ + maxNbBits = HUF_setMaxHeight(huffNode, (U32)nonNullRank, maxNbBits); + if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */ + + HUF_buildCTableFromTree(tree, huffNode, nonNullRank, maxSymbolValue, maxNbBits); + + return maxNbBits; +} + +size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) +{ + size_t nbBits = 0; + int s; + for (s = 0; s <= (int)maxSymbolValue; ++s) { + nbBits += CTable[s].nbBits * count[s]; + } + return nbBits >> 3; +} + +int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) { + int bad = 0; + int s; + for (s = 0; s <= (int)maxSymbolValue; ++s) { + bad |= (count[s] != 0) & (CTable[s].nbBits == 0); + } + return !bad; +} + +size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); } + +FORCE_INLINE_TEMPLATE void +HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable) +{ + BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits); +} + +#define HUF_FLUSHBITS(s) BIT_flushBits(s) + +#define HUF_FLUSHBITS_1(stream) \ + if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*2+7) HUF_FLUSHBITS(stream) + +#define HUF_FLUSHBITS_2(stream) \ + if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*4+7) HUF_FLUSHBITS(stream) + +FORCE_INLINE_TEMPLATE size_t +HUF_compress1X_usingCTable_internal_body(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable) +{ + const BYTE* ip = (const BYTE*) src; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + size_t n; + BIT_CStream_t bitC; + + /* init */ + if (dstSize < 8) return 0; /* not enough space to compress */ + { size_t const initErr = BIT_initCStream(&bitC, op, (size_t)(oend-op)); + if (HUF_isError(initErr)) return 0; } + + n = srcSize & ~3; /* join to mod 4 */ + switch (srcSize & 3) + { + case 3 : HUF_encodeSymbol(&bitC, ip[n+ 2], CTable); + HUF_FLUSHBITS_2(&bitC); + /* fall-through */ + case 2 : HUF_encodeSymbol(&bitC, ip[n+ 1], CTable); + HUF_FLUSHBITS_1(&bitC); + /* fall-through */ + case 1 : HUF_encodeSymbol(&bitC, ip[n+ 0], CTable); + HUF_FLUSHBITS(&bitC); + /* fall-through */ + case 0 : /* fall-through */ + default: break; + } + + for (; n>0; n-=4) { /* note : n&3==0 at this stage */ + HUF_encodeSymbol(&bitC, ip[n- 1], CTable); + HUF_FLUSHBITS_1(&bitC); + HUF_encodeSymbol(&bitC, ip[n- 2], CTable); + HUF_FLUSHBITS_2(&bitC); + HUF_encodeSymbol(&bitC, ip[n- 3], CTable); + HUF_FLUSHBITS_1(&bitC); + HUF_encodeSymbol(&bitC, ip[n- 4], CTable); + HUF_FLUSHBITS(&bitC); + } + + return BIT_closeCStream(&bitC); +} + +#if DYNAMIC_BMI2 + +static TARGET_ATTRIBUTE("bmi2") size_t +HUF_compress1X_usingCTable_internal_bmi2(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable) +{ + return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); +} + +static size_t +HUF_compress1X_usingCTable_internal_default(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable) +{ + return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); +} + +static size_t +HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable, const int bmi2) +{ + if (bmi2) { + return HUF_compress1X_usingCTable_internal_bmi2(dst, dstSize, src, srcSize, CTable); + } + return HUF_compress1X_usingCTable_internal_default(dst, dstSize, src, srcSize, CTable); +} + +#else + +static size_t +HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable, const int bmi2) +{ + (void)bmi2; + return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); +} + +#endif + +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) +{ + return HUF_compress1X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, /* bmi2 */ 0); +} + + +static size_t +HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable, int bmi2) +{ + size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */ + const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + + if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */ + if (srcSize < 12) return 0; /* no saving possible : too small input */ + op += 6; /* jumpTable */ + + assert(op <= oend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) ); + if (cSize==0) return 0; + assert(cSize <= 65535); + MEM_writeLE16(ostart, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + assert(op <= oend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) ); + if (cSize==0) return 0; + assert(cSize <= 65535); + MEM_writeLE16(ostart+2, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + assert(op <= oend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) ); + if (cSize==0) return 0; + assert(cSize <= 65535); + MEM_writeLE16(ostart+4, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + assert(op <= oend); + assert(ip <= iend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, (size_t)(iend-ip), CTable, bmi2) ); + if (cSize==0) return 0; + op += cSize; + } + + return (size_t)(op-ostart); +} + +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) +{ + return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, /* bmi2 */ 0); +} + +typedef enum { HUF_singleStream, HUF_fourStreams } HUF_nbStreams_e; + +static size_t HUF_compressCTable_internal( + BYTE* const ostart, BYTE* op, BYTE* const oend, + const void* src, size_t srcSize, + HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int bmi2) +{ + size_t const cSize = (nbStreams==HUF_singleStream) ? + HUF_compress1X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, bmi2) : + HUF_compress4X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, bmi2); + if (HUF_isError(cSize)) { return cSize; } + if (cSize==0) { return 0; } /* uncompressible */ + op += cSize; + /* check compressibility */ + assert(op >= ostart); + if ((size_t)(op-ostart) >= srcSize-1) { return 0; } + return (size_t)(op-ostart); +} + +typedef struct { + unsigned count[HUF_SYMBOLVALUE_MAX + 1]; + HUF_CElt CTable[HUF_SYMBOLVALUE_MAX + 1]; + union { + HUF_buildCTable_wksp_tables buildCTable_wksp; + HUF_WriteCTableWksp writeCTable_wksp; + } wksps; +} HUF_compress_tables_t; + +/* HUF_compress_internal() : + * `workSpace_align4` must be aligned on 4-bytes boundaries, + * and occupies the same space as a table of HUF_WORKSPACE_SIZE_U32 unsigned */ +static size_t +HUF_compress_internal (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + HUF_nbStreams_e nbStreams, + void* workSpace_align4, size_t wkspSize, + HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat, + const int bmi2) +{ + HUF_compress_tables_t* const table = (HUF_compress_tables_t*)workSpace_align4; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + + HUF_STATIC_ASSERT(sizeof(*table) <= HUF_WORKSPACE_SIZE); + assert(((size_t)workSpace_align4 & 3) == 0); /* must be aligned on 4-bytes boundaries */ + + /* checks & inits */ + if (wkspSize < HUF_WORKSPACE_SIZE) return ERROR(workSpace_tooSmall); + if (!srcSize) return 0; /* Uncompressed */ + if (!dstSize) return 0; /* cannot fit anything within dst budget */ + if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */ + if (huffLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge); + if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX; + if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT; + + /* Heuristic : If old table is valid, use it for small inputs */ + if (preferRepeat && repeat && *repeat == HUF_repeat_valid) { + return HUF_compressCTable_internal(ostart, op, oend, + src, srcSize, + nbStreams, oldHufTable, bmi2); + } + + /* Scan input and build symbol stats */ + { CHECK_V_F(largest, HIST_count_wksp (table->count, &maxSymbolValue, (const BYTE*)src, srcSize, workSpace_align4, wkspSize) ); + if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */ + if (largest <= (srcSize >> 7)+4) return 0; /* heuristic : probably not compressible enough */ + } + + /* Check validity of previous table */ + if ( repeat + && *repeat == HUF_repeat_check + && !HUF_validateCTable(oldHufTable, table->count, maxSymbolValue)) { + *repeat = HUF_repeat_none; + } + /* Heuristic : use existing table for small inputs */ + if (preferRepeat && repeat && *repeat != HUF_repeat_none) { + return HUF_compressCTable_internal(ostart, op, oend, + src, srcSize, + nbStreams, oldHufTable, bmi2); + } + + /* Build Huffman Tree */ + huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); + { size_t const maxBits = HUF_buildCTable_wksp(table->CTable, table->count, + maxSymbolValue, huffLog, + &table->wksps.buildCTable_wksp, sizeof(table->wksps.buildCTable_wksp)); + CHECK_F(maxBits); + huffLog = (U32)maxBits; + /* Zero unused symbols in CTable, so we can check it for validity */ + ZSTD_memset(table->CTable + (maxSymbolValue + 1), 0, + sizeof(table->CTable) - ((maxSymbolValue + 1) * sizeof(HUF_CElt))); + } + + /* Write table description header */ + { CHECK_V_F(hSize, HUF_writeCTable_wksp(op, dstSize, table->CTable, maxSymbolValue, huffLog, + &table->wksps.writeCTable_wksp, sizeof(table->wksps.writeCTable_wksp)) ); + /* Check if using previous huffman table is beneficial */ + if (repeat && *repeat != HUF_repeat_none) { + size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, table->count, maxSymbolValue); + size_t const newSize = HUF_estimateCompressedSize(table->CTable, table->count, maxSymbolValue); + if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) { + return HUF_compressCTable_internal(ostart, op, oend, + src, srcSize, + nbStreams, oldHufTable, bmi2); + } } + + /* Use the new huffman table */ + if (hSize + 12ul >= srcSize) { return 0; } + op += hSize; + if (repeat) { *repeat = HUF_repeat_none; } + if (oldHufTable) + ZSTD_memcpy(oldHufTable, table->CTable, sizeof(table->CTable)); /* Save new table */ + } + return HUF_compressCTable_internal(ostart, op, oend, + src, srcSize, + nbStreams, table->CTable, bmi2); +} + + +size_t HUF_compress1X_wksp (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + void* workSpace, size_t wkspSize) +{ + return HUF_compress_internal(dst, dstSize, src, srcSize, + maxSymbolValue, huffLog, HUF_singleStream, + workSpace, wkspSize, + NULL, NULL, 0, 0 /*bmi2*/); +} + +size_t HUF_compress1X_repeat (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + void* workSpace, size_t wkspSize, + HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2) +{ + return HUF_compress_internal(dst, dstSize, src, srcSize, + maxSymbolValue, huffLog, HUF_singleStream, + workSpace, wkspSize, hufTable, + repeat, preferRepeat, bmi2); +} + +/* HUF_compress4X_repeat(): + * compress input using 4 streams. + * provide workspace to generate compression tables */ +size_t HUF_compress4X_wksp (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + void* workSpace, size_t wkspSize) +{ + return HUF_compress_internal(dst, dstSize, src, srcSize, + maxSymbolValue, huffLog, HUF_fourStreams, + workSpace, wkspSize, + NULL, NULL, 0, 0 /*bmi2*/); +} + +/* HUF_compress4X_repeat(): + * compress input using 4 streams. + * re-use an existing huffman compression table */ +size_t HUF_compress4X_repeat (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + void* workSpace, size_t wkspSize, + HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2) +{ + return HUF_compress_internal(dst, dstSize, src, srcSize, + maxSymbolValue, huffLog, HUF_fourStreams, + workSpace, wkspSize, + hufTable, repeat, preferRepeat, bmi2); +} + diff --git a/lib/zstd/compress/zstd_compress.c b/lib/zstd/compress/zstd_compress.c new file mode 100644 index 000000000000..f5b37ad65022 --- /dev/null +++ b/lib/zstd/compress/zstd_compress.c @@ -0,0 +1,5105 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/*-************************************* +* Dependencies +***************************************/ +#include "../common/zstd_deps.h" /* INT_MAX, ZSTD_memset, ZSTD_memcpy */ +#include "../common/cpu.h" +#include "../common/mem.h" +#include "hist.h" /* HIST_countFast_wksp */ +#define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */ +#include "../common/fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "../common/huf.h" +#include "zstd_compress_internal.h" +#include "zstd_compress_sequences.h" +#include "zstd_compress_literals.h" +#include "zstd_fast.h" +#include "zstd_double_fast.h" +#include "zstd_lazy.h" +#include "zstd_opt.h" +#include "zstd_ldm.h" +#include "zstd_compress_superblock.h" + +/* *************************************************************** +* Tuning parameters +*****************************************************************/ +/*! + * COMPRESS_HEAPMODE : + * Select how default decompression function ZSTD_compress() allocates its context, + * on stack (0, default), or into heap (1). + * Note that functions with explicit context such as ZSTD_compressCCtx() are unaffected. + */ + + +/*-************************************* +* Helper functions +***************************************/ +/* ZSTD_compressBound() + * Note that the result from this function is only compatible with the "normal" + * full-block strategy. + * When there are a lot of small blocks due to frequent flush in streaming mode + * the overhead of headers can make the compressed data to be larger than the + * return value of ZSTD_compressBound(). + */ +size_t ZSTD_compressBound(size_t srcSize) { + return ZSTD_COMPRESSBOUND(srcSize); +} + + +/*-************************************* +* Context memory management +***************************************/ +struct ZSTD_CDict_s { + const void* dictContent; + size_t dictContentSize; + ZSTD_dictContentType_e dictContentType; /* The dictContentType the CDict was created with */ + U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */ + ZSTD_cwksp workspace; + ZSTD_matchState_t matchState; + ZSTD_compressedBlockState_t cBlockState; + ZSTD_customMem customMem; + U32 dictID; + int compressionLevel; /* 0 indicates that advanced API was used to select CDict params */ +}; /* typedef'd to ZSTD_CDict within "zstd.h" */ + +ZSTD_CCtx* ZSTD_createCCtx(void) +{ + return ZSTD_createCCtx_advanced(ZSTD_defaultCMem); +} + +static void ZSTD_initCCtx(ZSTD_CCtx* cctx, ZSTD_customMem memManager) +{ + assert(cctx != NULL); + ZSTD_memset(cctx, 0, sizeof(*cctx)); + cctx->customMem = memManager; + cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); + { size_t const err = ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters); + assert(!ZSTD_isError(err)); + (void)err; + } +} + +ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) +{ + ZSTD_STATIC_ASSERT(zcss_init==0); + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==(0ULL - 1)); + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + { ZSTD_CCtx* const cctx = (ZSTD_CCtx*)ZSTD_customMalloc(sizeof(ZSTD_CCtx), customMem); + if (!cctx) return NULL; + ZSTD_initCCtx(cctx, customMem); + return cctx; + } +} + +ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize) +{ + ZSTD_cwksp ws; + ZSTD_CCtx* cctx; + if (workspaceSize <= sizeof(ZSTD_CCtx)) return NULL; /* minimum size */ + if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */ + ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_static_alloc); + + cctx = (ZSTD_CCtx*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CCtx)); + if (cctx == NULL) return NULL; + + ZSTD_memset(cctx, 0, sizeof(ZSTD_CCtx)); + ZSTD_cwksp_move(&cctx->workspace, &ws); + cctx->staticSize = workspaceSize; + + /* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */ + if (!ZSTD_cwksp_check_available(&cctx->workspace, ENTROPY_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL; + cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); + cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); + cctx->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cctx->workspace, ENTROPY_WORKSPACE_SIZE); + cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); + return cctx; +} + +/* + * Clears and frees all of the dictionaries in the CCtx. + */ +static void ZSTD_clearAllDicts(ZSTD_CCtx* cctx) +{ + ZSTD_customFree(cctx->localDict.dictBuffer, cctx->customMem); + ZSTD_freeCDict(cctx->localDict.cdict); + ZSTD_memset(&cctx->localDict, 0, sizeof(cctx->localDict)); + ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); + cctx->cdict = NULL; +} + +static size_t ZSTD_sizeof_localDict(ZSTD_localDict dict) +{ + size_t const bufferSize = dict.dictBuffer != NULL ? dict.dictSize : 0; + size_t const cdictSize = ZSTD_sizeof_CDict(dict.cdict); + return bufferSize + cdictSize; +} + +static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) +{ + assert(cctx != NULL); + assert(cctx->staticSize == 0); + ZSTD_clearAllDicts(cctx); + ZSTD_cwksp_free(&cctx->workspace, cctx->customMem); +} + +size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) +{ + if (cctx==NULL) return 0; /* support free on NULL */ + RETURN_ERROR_IF(cctx->staticSize, memory_allocation, + "not compatible with static CCtx"); + { + int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx); + ZSTD_freeCCtxContent(cctx); + if (!cctxInWorkspace) { + ZSTD_customFree(cctx, cctx->customMem); + } + } + return 0; +} + + +static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx) +{ + (void)cctx; + return 0; +} + + +size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) +{ + if (cctx==NULL) return 0; /* support sizeof on NULL */ + /* cctx may be in the workspace */ + return (cctx->workspace.workspace == cctx ? 0 : sizeof(*cctx)) + + ZSTD_cwksp_sizeof(&cctx->workspace) + + ZSTD_sizeof_localDict(cctx->localDict) + + ZSTD_sizeof_mtctx(cctx); +} + +size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) +{ + return ZSTD_sizeof_CCtx(zcs); /* same object */ +} + +/* private API call, for dictBuilder only */ +const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); } + +/* Returns 1 if compression parameters are such that we should + * enable long distance matching (wlog >= 27, strategy >= btopt). + * Returns 0 otherwise. + */ +static U32 ZSTD_CParams_shouldEnableLdm(const ZSTD_compressionParameters* const cParams) { + return cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 27; +} + +static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams( + ZSTD_compressionParameters cParams) +{ + ZSTD_CCtx_params cctxParams; + /* should not matter, as all cParams are presumed properly defined */ + ZSTD_CCtxParams_init(&cctxParams, ZSTD_CLEVEL_DEFAULT); + cctxParams.cParams = cParams; + + if (ZSTD_CParams_shouldEnableLdm(&cParams)) { + DEBUGLOG(4, "ZSTD_makeCCtxParamsFromCParams(): Including LDM into cctx params"); + cctxParams.ldmParams.enableLdm = 1; + /* LDM is enabled by default for optimal parser and window size >= 128MB */ + ZSTD_ldm_adjustParameters(&cctxParams.ldmParams, &cParams); + assert(cctxParams.ldmParams.hashLog >= cctxParams.ldmParams.bucketSizeLog); + assert(cctxParams.ldmParams.hashRateLog < 32); + } + + assert(!ZSTD_checkCParams(cParams)); + return cctxParams; +} + +static ZSTD_CCtx_params* ZSTD_createCCtxParams_advanced( + ZSTD_customMem customMem) +{ + ZSTD_CCtx_params* params; + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + params = (ZSTD_CCtx_params*)ZSTD_customCalloc( + sizeof(ZSTD_CCtx_params), customMem); + if (!params) { return NULL; } + ZSTD_CCtxParams_init(params, ZSTD_CLEVEL_DEFAULT); + params->customMem = customMem; + return params; +} + +ZSTD_CCtx_params* ZSTD_createCCtxParams(void) +{ + return ZSTD_createCCtxParams_advanced(ZSTD_defaultCMem); +} + +size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params) +{ + if (params == NULL) { return 0; } + ZSTD_customFree(params, params->customMem); + return 0; +} + +size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params) +{ + return ZSTD_CCtxParams_init(params, ZSTD_CLEVEL_DEFAULT); +} + +size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel) { + RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!"); + ZSTD_memset(cctxParams, 0, sizeof(*cctxParams)); + cctxParams->compressionLevel = compressionLevel; + cctxParams->fParams.contentSizeFlag = 1; + return 0; +} + +#define ZSTD_NO_CLEVEL 0 + +/* + * Initializes the cctxParams from params and compressionLevel. + * @param compressionLevel If params are derived from a compression level then that compression level, otherwise ZSTD_NO_CLEVEL. + */ +static void ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, ZSTD_parameters const* params, int compressionLevel) +{ + assert(!ZSTD_checkCParams(params->cParams)); + ZSTD_memset(cctxParams, 0, sizeof(*cctxParams)); + cctxParams->cParams = params->cParams; + cctxParams->fParams = params->fParams; + /* Should not matter, as all cParams are presumed properly defined. + * But, set it for tracing anyway. + */ + cctxParams->compressionLevel = compressionLevel; +} + +size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params) +{ + RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!"); + FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) , ""); + ZSTD_CCtxParams_init_internal(cctxParams, ¶ms, ZSTD_NO_CLEVEL); + return 0; +} + +/* + * Sets cctxParams' cParams and fParams from params, but otherwise leaves them alone. + * @param param Validated zstd parameters. + */ +static void ZSTD_CCtxParams_setZstdParams( + ZSTD_CCtx_params* cctxParams, const ZSTD_parameters* params) +{ + assert(!ZSTD_checkCParams(params->cParams)); + cctxParams->cParams = params->cParams; + cctxParams->fParams = params->fParams; + /* Should not matter, as all cParams are presumed properly defined. + * But, set it for tracing anyway. + */ + cctxParams->compressionLevel = ZSTD_NO_CLEVEL; +} + +ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) +{ + ZSTD_bounds bounds = { 0, 0, 0 }; + + switch(param) + { + case ZSTD_c_compressionLevel: + bounds.lowerBound = ZSTD_minCLevel(); + bounds.upperBound = ZSTD_maxCLevel(); + return bounds; + + case ZSTD_c_windowLog: + bounds.lowerBound = ZSTD_WINDOWLOG_MIN; + bounds.upperBound = ZSTD_WINDOWLOG_MAX; + return bounds; + + case ZSTD_c_hashLog: + bounds.lowerBound = ZSTD_HASHLOG_MIN; + bounds.upperBound = ZSTD_HASHLOG_MAX; + return bounds; + + case ZSTD_c_chainLog: + bounds.lowerBound = ZSTD_CHAINLOG_MIN; + bounds.upperBound = ZSTD_CHAINLOG_MAX; + return bounds; + + case ZSTD_c_searchLog: + bounds.lowerBound = ZSTD_SEARCHLOG_MIN; + bounds.upperBound = ZSTD_SEARCHLOG_MAX; + return bounds; + + case ZSTD_c_minMatch: + bounds.lowerBound = ZSTD_MINMATCH_MIN; + bounds.upperBound = ZSTD_MINMATCH_MAX; + return bounds; + + case ZSTD_c_targetLength: + bounds.lowerBound = ZSTD_TARGETLENGTH_MIN; + bounds.upperBound = ZSTD_TARGETLENGTH_MAX; + return bounds; + + case ZSTD_c_strategy: + bounds.lowerBound = ZSTD_STRATEGY_MIN; + bounds.upperBound = ZSTD_STRATEGY_MAX; + return bounds; + + case ZSTD_c_contentSizeFlag: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_checksumFlag: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_dictIDFlag: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_nbWorkers: + bounds.lowerBound = 0; + bounds.upperBound = 0; + return bounds; + + case ZSTD_c_jobSize: + bounds.lowerBound = 0; + bounds.upperBound = 0; + return bounds; + + case ZSTD_c_overlapLog: + bounds.lowerBound = 0; + bounds.upperBound = 0; + return bounds; + + case ZSTD_c_enableDedicatedDictSearch: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_enableLongDistanceMatching: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_ldmHashLog: + bounds.lowerBound = ZSTD_LDM_HASHLOG_MIN; + bounds.upperBound = ZSTD_LDM_HASHLOG_MAX; + return bounds; + + case ZSTD_c_ldmMinMatch: + bounds.lowerBound = ZSTD_LDM_MINMATCH_MIN; + bounds.upperBound = ZSTD_LDM_MINMATCH_MAX; + return bounds; + + case ZSTD_c_ldmBucketSizeLog: + bounds.lowerBound = ZSTD_LDM_BUCKETSIZELOG_MIN; + bounds.upperBound = ZSTD_LDM_BUCKETSIZELOG_MAX; + return bounds; + + case ZSTD_c_ldmHashRateLog: + bounds.lowerBound = ZSTD_LDM_HASHRATELOG_MIN; + bounds.upperBound = ZSTD_LDM_HASHRATELOG_MAX; + return bounds; + + /* experimental parameters */ + case ZSTD_c_rsyncable: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_forceMaxWindow : + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_format: + ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless); + bounds.lowerBound = ZSTD_f_zstd1; + bounds.upperBound = ZSTD_f_zstd1_magicless; /* note : how to ensure at compile time that this is the highest value enum ? */ + return bounds; + + case ZSTD_c_forceAttachDict: + ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceLoad); + bounds.lowerBound = ZSTD_dictDefaultAttach; + bounds.upperBound = ZSTD_dictForceLoad; /* note : how to ensure at compile time that this is the highest value enum ? */ + return bounds; + + case ZSTD_c_literalCompressionMode: + ZSTD_STATIC_ASSERT(ZSTD_lcm_auto < ZSTD_lcm_huffman && ZSTD_lcm_huffman < ZSTD_lcm_uncompressed); + bounds.lowerBound = ZSTD_lcm_auto; + bounds.upperBound = ZSTD_lcm_uncompressed; + return bounds; + + case ZSTD_c_targetCBlockSize: + bounds.lowerBound = ZSTD_TARGETCBLOCKSIZE_MIN; + bounds.upperBound = ZSTD_TARGETCBLOCKSIZE_MAX; + return bounds; + + case ZSTD_c_srcSizeHint: + bounds.lowerBound = ZSTD_SRCSIZEHINT_MIN; + bounds.upperBound = ZSTD_SRCSIZEHINT_MAX; + return bounds; + + case ZSTD_c_stableInBuffer: + case ZSTD_c_stableOutBuffer: + bounds.lowerBound = (int)ZSTD_bm_buffered; + bounds.upperBound = (int)ZSTD_bm_stable; + return bounds; + + case ZSTD_c_blockDelimiters: + bounds.lowerBound = (int)ZSTD_sf_noBlockDelimiters; + bounds.upperBound = (int)ZSTD_sf_explicitBlockDelimiters; + return bounds; + + case ZSTD_c_validateSequences: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + default: + bounds.error = ERROR(parameter_unsupported); + return bounds; + } +} + +/* ZSTD_cParam_clampBounds: + * Clamps the value into the bounded range. + */ +static size_t ZSTD_cParam_clampBounds(ZSTD_cParameter cParam, int* value) +{ + ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); + if (ZSTD_isError(bounds.error)) return bounds.error; + if (*value < bounds.lowerBound) *value = bounds.lowerBound; + if (*value > bounds.upperBound) *value = bounds.upperBound; + return 0; +} + +#define BOUNDCHECK(cParam, val) { \ + RETURN_ERROR_IF(!ZSTD_cParam_withinBounds(cParam,val), \ + parameter_outOfBound, "Param out of bounds"); \ +} + + +static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) +{ + switch(param) + { + case ZSTD_c_compressionLevel: + case ZSTD_c_hashLog: + case ZSTD_c_chainLog: + case ZSTD_c_searchLog: + case ZSTD_c_minMatch: + case ZSTD_c_targetLength: + case ZSTD_c_strategy: + return 1; + + case ZSTD_c_format: + case ZSTD_c_windowLog: + case ZSTD_c_contentSizeFlag: + case ZSTD_c_checksumFlag: + case ZSTD_c_dictIDFlag: + case ZSTD_c_forceMaxWindow : + case ZSTD_c_nbWorkers: + case ZSTD_c_jobSize: + case ZSTD_c_overlapLog: + case ZSTD_c_rsyncable: + case ZSTD_c_enableDedicatedDictSearch: + case ZSTD_c_enableLongDistanceMatching: + case ZSTD_c_ldmHashLog: + case ZSTD_c_ldmMinMatch: + case ZSTD_c_ldmBucketSizeLog: + case ZSTD_c_ldmHashRateLog: + case ZSTD_c_forceAttachDict: + case ZSTD_c_literalCompressionMode: + case ZSTD_c_targetCBlockSize: + case ZSTD_c_srcSizeHint: + case ZSTD_c_stableInBuffer: + case ZSTD_c_stableOutBuffer: + case ZSTD_c_blockDelimiters: + case ZSTD_c_validateSequences: + default: + return 0; + } +} + +size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) +{ + DEBUGLOG(4, "ZSTD_CCtx_setParameter (%i, %i)", (int)param, value); + if (cctx->streamStage != zcss_init) { + if (ZSTD_isUpdateAuthorized(param)) { + cctx->cParamsChanged = 1; + } else { + RETURN_ERROR(stage_wrong, "can only set params in ctx init stage"); + } } + + switch(param) + { + case ZSTD_c_nbWorkers: + RETURN_ERROR_IF((value!=0) && cctx->staticSize, parameter_unsupported, + "MT not compatible with static alloc"); + break; + + case ZSTD_c_compressionLevel: + case ZSTD_c_windowLog: + case ZSTD_c_hashLog: + case ZSTD_c_chainLog: + case ZSTD_c_searchLog: + case ZSTD_c_minMatch: + case ZSTD_c_targetLength: + case ZSTD_c_strategy: + case ZSTD_c_ldmHashRateLog: + case ZSTD_c_format: + case ZSTD_c_contentSizeFlag: + case ZSTD_c_checksumFlag: + case ZSTD_c_dictIDFlag: + case ZSTD_c_forceMaxWindow: + case ZSTD_c_forceAttachDict: + case ZSTD_c_literalCompressionMode: + case ZSTD_c_jobSize: + case ZSTD_c_overlapLog: + case ZSTD_c_rsyncable: + case ZSTD_c_enableDedicatedDictSearch: + case ZSTD_c_enableLongDistanceMatching: + case ZSTD_c_ldmHashLog: + case ZSTD_c_ldmMinMatch: + case ZSTD_c_ldmBucketSizeLog: + case ZSTD_c_targetCBlockSize: + case ZSTD_c_srcSizeHint: + case ZSTD_c_stableInBuffer: + case ZSTD_c_stableOutBuffer: + case ZSTD_c_blockDelimiters: + case ZSTD_c_validateSequences: + break; + + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); + } + return ZSTD_CCtxParams_setParameter(&cctx->requestedParams, param, value); +} + +size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, + ZSTD_cParameter param, int value) +{ + DEBUGLOG(4, "ZSTD_CCtxParams_setParameter (%i, %i)", (int)param, value); + switch(param) + { + case ZSTD_c_format : + BOUNDCHECK(ZSTD_c_format, value); + CCtxParams->format = (ZSTD_format_e)value; + return (size_t)CCtxParams->format; + + case ZSTD_c_compressionLevel : { + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); + if (value == 0) + CCtxParams->compressionLevel = ZSTD_CLEVEL_DEFAULT; /* 0 == default */ + else + CCtxParams->compressionLevel = value; + if (CCtxParams->compressionLevel >= 0) return (size_t)CCtxParams->compressionLevel; + return 0; /* return type (size_t) cannot represent negative values */ + } + + case ZSTD_c_windowLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_windowLog, value); + CCtxParams->cParams.windowLog = (U32)value; + return CCtxParams->cParams.windowLog; + + case ZSTD_c_hashLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_hashLog, value); + CCtxParams->cParams.hashLog = (U32)value; + return CCtxParams->cParams.hashLog; + + case ZSTD_c_chainLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_chainLog, value); + CCtxParams->cParams.chainLog = (U32)value; + return CCtxParams->cParams.chainLog; + + case ZSTD_c_searchLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_searchLog, value); + CCtxParams->cParams.searchLog = (U32)value; + return (size_t)value; + + case ZSTD_c_minMatch : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_minMatch, value); + CCtxParams->cParams.minMatch = value; + return CCtxParams->cParams.minMatch; + + case ZSTD_c_targetLength : + BOUNDCHECK(ZSTD_c_targetLength, value); + CCtxParams->cParams.targetLength = value; + return CCtxParams->cParams.targetLength; + + case ZSTD_c_strategy : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_strategy, value); + CCtxParams->cParams.strategy = (ZSTD_strategy)value; + return (size_t)CCtxParams->cParams.strategy; + + case ZSTD_c_contentSizeFlag : + /* Content size written in frame header _when known_ (default:1) */ + DEBUGLOG(4, "set content size flag = %u", (value!=0)); + CCtxParams->fParams.contentSizeFlag = value != 0; + return CCtxParams->fParams.contentSizeFlag; + + case ZSTD_c_checksumFlag : + /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */ + CCtxParams->fParams.checksumFlag = value != 0; + return CCtxParams->fParams.checksumFlag; + + case ZSTD_c_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */ + DEBUGLOG(4, "set dictIDFlag = %u", (value!=0)); + CCtxParams->fParams.noDictIDFlag = !value; + return !CCtxParams->fParams.noDictIDFlag; + + case ZSTD_c_forceMaxWindow : + CCtxParams->forceWindow = (value != 0); + return CCtxParams->forceWindow; + + case ZSTD_c_forceAttachDict : { + const ZSTD_dictAttachPref_e pref = (ZSTD_dictAttachPref_e)value; + BOUNDCHECK(ZSTD_c_forceAttachDict, pref); + CCtxParams->attachDictPref = pref; + return CCtxParams->attachDictPref; + } + + case ZSTD_c_literalCompressionMode : { + const ZSTD_literalCompressionMode_e lcm = (ZSTD_literalCompressionMode_e)value; + BOUNDCHECK(ZSTD_c_literalCompressionMode, lcm); + CCtxParams->literalCompressionMode = lcm; + return CCtxParams->literalCompressionMode; + } + + case ZSTD_c_nbWorkers : + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); + return 0; + + case ZSTD_c_jobSize : + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); + return 0; + + case ZSTD_c_overlapLog : + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); + return 0; + + case ZSTD_c_rsyncable : + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); + return 0; + + case ZSTD_c_enableDedicatedDictSearch : + CCtxParams->enableDedicatedDictSearch = (value!=0); + return CCtxParams->enableDedicatedDictSearch; + + case ZSTD_c_enableLongDistanceMatching : + CCtxParams->ldmParams.enableLdm = (value!=0); + return CCtxParams->ldmParams.enableLdm; + + case ZSTD_c_ldmHashLog : + if (value!=0) /* 0 ==> auto */ + BOUNDCHECK(ZSTD_c_ldmHashLog, value); + CCtxParams->ldmParams.hashLog = value; + return CCtxParams->ldmParams.hashLog; + + case ZSTD_c_ldmMinMatch : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_ldmMinMatch, value); + CCtxParams->ldmParams.minMatchLength = value; + return CCtxParams->ldmParams.minMatchLength; + + case ZSTD_c_ldmBucketSizeLog : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_ldmBucketSizeLog, value); + CCtxParams->ldmParams.bucketSizeLog = value; + return CCtxParams->ldmParams.bucketSizeLog; + + case ZSTD_c_ldmHashRateLog : + RETURN_ERROR_IF(value > ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN, + parameter_outOfBound, "Param out of bounds!"); + CCtxParams->ldmParams.hashRateLog = value; + return CCtxParams->ldmParams.hashRateLog; + + case ZSTD_c_targetCBlockSize : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_targetCBlockSize, value); + CCtxParams->targetCBlockSize = value; + return CCtxParams->targetCBlockSize; + + case ZSTD_c_srcSizeHint : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_srcSizeHint, value); + CCtxParams->srcSizeHint = value; + return CCtxParams->srcSizeHint; + + case ZSTD_c_stableInBuffer: + BOUNDCHECK(ZSTD_c_stableInBuffer, value); + CCtxParams->inBufferMode = (ZSTD_bufferMode_e)value; + return CCtxParams->inBufferMode; + + case ZSTD_c_stableOutBuffer: + BOUNDCHECK(ZSTD_c_stableOutBuffer, value); + CCtxParams->outBufferMode = (ZSTD_bufferMode_e)value; + return CCtxParams->outBufferMode; + + case ZSTD_c_blockDelimiters: + BOUNDCHECK(ZSTD_c_blockDelimiters, value); + CCtxParams->blockDelimiters = (ZSTD_sequenceFormat_e)value; + return CCtxParams->blockDelimiters; + + case ZSTD_c_validateSequences: + BOUNDCHECK(ZSTD_c_validateSequences, value); + CCtxParams->validateSequences = value; + return CCtxParams->validateSequences; + + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); + } +} + +size_t ZSTD_CCtx_getParameter(ZSTD_CCtx const* cctx, ZSTD_cParameter param, int* value) +{ + return ZSTD_CCtxParams_getParameter(&cctx->requestedParams, param, value); +} + +size_t ZSTD_CCtxParams_getParameter( + ZSTD_CCtx_params const* CCtxParams, ZSTD_cParameter param, int* value) +{ + switch(param) + { + case ZSTD_c_format : + *value = CCtxParams->format; + break; + case ZSTD_c_compressionLevel : + *value = CCtxParams->compressionLevel; + break; + case ZSTD_c_windowLog : + *value = (int)CCtxParams->cParams.windowLog; + break; + case ZSTD_c_hashLog : + *value = (int)CCtxParams->cParams.hashLog; + break; + case ZSTD_c_chainLog : + *value = (int)CCtxParams->cParams.chainLog; + break; + case ZSTD_c_searchLog : + *value = CCtxParams->cParams.searchLog; + break; + case ZSTD_c_minMatch : + *value = CCtxParams->cParams.minMatch; + break; + case ZSTD_c_targetLength : + *value = CCtxParams->cParams.targetLength; + break; + case ZSTD_c_strategy : + *value = (unsigned)CCtxParams->cParams.strategy; + break; + case ZSTD_c_contentSizeFlag : + *value = CCtxParams->fParams.contentSizeFlag; + break; + case ZSTD_c_checksumFlag : + *value = CCtxParams->fParams.checksumFlag; + break; + case ZSTD_c_dictIDFlag : + *value = !CCtxParams->fParams.noDictIDFlag; + break; + case ZSTD_c_forceMaxWindow : + *value = CCtxParams->forceWindow; + break; + case ZSTD_c_forceAttachDict : + *value = CCtxParams->attachDictPref; + break; + case ZSTD_c_literalCompressionMode : + *value = CCtxParams->literalCompressionMode; + break; + case ZSTD_c_nbWorkers : + assert(CCtxParams->nbWorkers == 0); + *value = CCtxParams->nbWorkers; + break; + case ZSTD_c_jobSize : + RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); + case ZSTD_c_overlapLog : + RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); + case ZSTD_c_rsyncable : + RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); + case ZSTD_c_enableDedicatedDictSearch : + *value = CCtxParams->enableDedicatedDictSearch; + break; + case ZSTD_c_enableLongDistanceMatching : + *value = CCtxParams->ldmParams.enableLdm; + break; + case ZSTD_c_ldmHashLog : + *value = CCtxParams->ldmParams.hashLog; + break; + case ZSTD_c_ldmMinMatch : + *value = CCtxParams->ldmParams.minMatchLength; + break; + case ZSTD_c_ldmBucketSizeLog : + *value = CCtxParams->ldmParams.bucketSizeLog; + break; + case ZSTD_c_ldmHashRateLog : + *value = CCtxParams->ldmParams.hashRateLog; + break; + case ZSTD_c_targetCBlockSize : + *value = (int)CCtxParams->targetCBlockSize; + break; + case ZSTD_c_srcSizeHint : + *value = (int)CCtxParams->srcSizeHint; + break; + case ZSTD_c_stableInBuffer : + *value = (int)CCtxParams->inBufferMode; + break; + case ZSTD_c_stableOutBuffer : + *value = (int)CCtxParams->outBufferMode; + break; + case ZSTD_c_blockDelimiters : + *value = (int)CCtxParams->blockDelimiters; + break; + case ZSTD_c_validateSequences : + *value = (int)CCtxParams->validateSequences; + break; + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); + } + return 0; +} + +/* ZSTD_CCtx_setParametersUsingCCtxParams() : + * just applies `params` into `cctx` + * no action is performed, parameters are merely stored. + * If ZSTDMT is enabled, parameters are pushed to cctx->mtctx. + * This is possible even if a compression is ongoing. + * In which case, new parameters will be applied on the fly, starting with next compression job. + */ +size_t ZSTD_CCtx_setParametersUsingCCtxParams( + ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params) +{ + DEBUGLOG(4, "ZSTD_CCtx_setParametersUsingCCtxParams"); + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "The context is in the wrong stage!"); + RETURN_ERROR_IF(cctx->cdict, stage_wrong, + "Can't override parameters with cdict attached (some must " + "be inherited from the cdict)."); + + cctx->requestedParams = *params; + return 0; +} + +ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %u bytes", (U32)pledgedSrcSize); + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't set pledgedSrcSize when not in init stage."); + cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; + return 0; +} + +static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams( + int const compressionLevel, + size_t const dictSize); +static int ZSTD_dedicatedDictSearch_isSupported( + const ZSTD_compressionParameters* cParams); +static void ZSTD_dedicatedDictSearch_revertCParams( + ZSTD_compressionParameters* cParams); + +/* + * Initializes the local dict using the requested parameters. + * NOTE: This does not use the pledged src size, because it may be used for more + * than one compression. + */ +static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx) +{ + ZSTD_localDict* const dl = &cctx->localDict; + if (dl->dict == NULL) { + /* No local dictionary. */ + assert(dl->dictBuffer == NULL); + assert(dl->cdict == NULL); + assert(dl->dictSize == 0); + return 0; + } + if (dl->cdict != NULL) { + assert(cctx->cdict == dl->cdict); + /* Local dictionary already initialized. */ + return 0; + } + assert(dl->dictSize > 0); + assert(cctx->cdict == NULL); + assert(cctx->prefixDict.dict == NULL); + + dl->cdict = ZSTD_createCDict_advanced2( + dl->dict, + dl->dictSize, + ZSTD_dlm_byRef, + dl->dictContentType, + &cctx->requestedParams, + cctx->customMem); + RETURN_ERROR_IF(!dl->cdict, memory_allocation, "ZSTD_createCDict_advanced failed"); + cctx->cdict = dl->cdict; + return 0; +} + +size_t ZSTD_CCtx_loadDictionary_advanced( + ZSTD_CCtx* cctx, const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType) +{ + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't load a dictionary when ctx is not in init stage."); + DEBUGLOG(4, "ZSTD_CCtx_loadDictionary_advanced (size: %u)", (U32)dictSize); + ZSTD_clearAllDicts(cctx); /* in case one already exists */ + if (dict == NULL || dictSize == 0) /* no dictionary mode */ + return 0; + if (dictLoadMethod == ZSTD_dlm_byRef) { + cctx->localDict.dict = dict; + } else { + void* dictBuffer; + RETURN_ERROR_IF(cctx->staticSize, memory_allocation, + "no malloc for static CCtx"); + dictBuffer = ZSTD_customMalloc(dictSize, cctx->customMem); + RETURN_ERROR_IF(!dictBuffer, memory_allocation, "NULL pointer!"); + ZSTD_memcpy(dictBuffer, dict, dictSize); + cctx->localDict.dictBuffer = dictBuffer; + cctx->localDict.dict = dictBuffer; + } + cctx->localDict.dictSize = dictSize; + cctx->localDict.dictContentType = dictContentType; + return 0; +} + +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference( + ZSTD_CCtx* cctx, const void* dict, size_t dictSize) +{ + return ZSTD_CCtx_loadDictionary_advanced( + cctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto); +} + +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) +{ + return ZSTD_CCtx_loadDictionary_advanced( + cctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); +} + + +size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) +{ + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't ref a dict when ctx not in init stage."); + /* Free the existing local cdict (if any) to save memory. */ + ZSTD_clearAllDicts(cctx); + cctx->cdict = cdict; + return 0; +} + +size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool) +{ + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't ref a pool when ctx not in init stage."); + cctx->pool = pool; + return 0; +} + +size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize) +{ + return ZSTD_CCtx_refPrefix_advanced(cctx, prefix, prefixSize, ZSTD_dct_rawContent); +} + +size_t ZSTD_CCtx_refPrefix_advanced( + ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType) +{ + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't ref a prefix when ctx not in init stage."); + ZSTD_clearAllDicts(cctx); + if (prefix != NULL && prefixSize > 0) { + cctx->prefixDict.dict = prefix; + cctx->prefixDict.dictSize = prefixSize; + cctx->prefixDict.dictContentType = dictContentType; + } + return 0; +} + +/*! ZSTD_CCtx_reset() : + * Also dumps dictionary */ +size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset) +{ + if ( (reset == ZSTD_reset_session_only) + || (reset == ZSTD_reset_session_and_parameters) ) { + cctx->streamStage = zcss_init; + cctx->pledgedSrcSizePlusOne = 0; + } + if ( (reset == ZSTD_reset_parameters) + || (reset == ZSTD_reset_session_and_parameters) ) { + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't reset parameters only when not in init stage."); + ZSTD_clearAllDicts(cctx); + return ZSTD_CCtxParams_reset(&cctx->requestedParams); + } + return 0; +} + + +/* ZSTD_checkCParams() : + control CParam values remain within authorized range. + @return : 0, or an error code if one value is beyond authorized range */ +size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) +{ + BOUNDCHECK(ZSTD_c_windowLog, (int)cParams.windowLog); + BOUNDCHECK(ZSTD_c_chainLog, (int)cParams.chainLog); + BOUNDCHECK(ZSTD_c_hashLog, (int)cParams.hashLog); + BOUNDCHECK(ZSTD_c_searchLog, (int)cParams.searchLog); + BOUNDCHECK(ZSTD_c_minMatch, (int)cParams.minMatch); + BOUNDCHECK(ZSTD_c_targetLength,(int)cParams.targetLength); + BOUNDCHECK(ZSTD_c_strategy, cParams.strategy); + return 0; +} + +/* ZSTD_clampCParams() : + * make CParam values within valid range. + * @return : valid CParams */ +static ZSTD_compressionParameters +ZSTD_clampCParams(ZSTD_compressionParameters cParams) +{ +# define CLAMP_TYPE(cParam, val, type) { \ + ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); \ + if ((int)valbounds.upperBound) val=(type)bounds.upperBound; \ + } +# define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, unsigned) + CLAMP(ZSTD_c_windowLog, cParams.windowLog); + CLAMP(ZSTD_c_chainLog, cParams.chainLog); + CLAMP(ZSTD_c_hashLog, cParams.hashLog); + CLAMP(ZSTD_c_searchLog, cParams.searchLog); + CLAMP(ZSTD_c_minMatch, cParams.minMatch); + CLAMP(ZSTD_c_targetLength,cParams.targetLength); + CLAMP_TYPE(ZSTD_c_strategy,cParams.strategy, ZSTD_strategy); + return cParams; +} + +/* ZSTD_cycleLog() : + * condition for correct operation : hashLog > 1 */ +U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) +{ + U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2); + return hashLog - btScale; +} + +/* ZSTD_dictAndWindowLog() : + * Returns an adjusted window log that is large enough to fit the source and the dictionary. + * The zstd format says that the entire dictionary is valid if one byte of the dictionary + * is within the window. So the hashLog and chainLog should be large enough to reference both + * the dictionary and the window. So we must use this adjusted dictAndWindowLog when downsizing + * the hashLog and windowLog. + * NOTE: srcSize must not be ZSTD_CONTENTSIZE_UNKNOWN. + */ +static U32 ZSTD_dictAndWindowLog(U32 windowLog, U64 srcSize, U64 dictSize) +{ + const U64 maxWindowSize = 1ULL << ZSTD_WINDOWLOG_MAX; + /* No dictionary ==> No change */ + if (dictSize == 0) { + return windowLog; + } + assert(windowLog <= ZSTD_WINDOWLOG_MAX); + assert(srcSize != ZSTD_CONTENTSIZE_UNKNOWN); /* Handled in ZSTD_adjustCParams_internal() */ + { + U64 const windowSize = 1ULL << windowLog; + U64 const dictAndWindowSize = dictSize + windowSize; + /* If the window size is already large enough to fit both the source and the dictionary + * then just use the window size. Otherwise adjust so that it fits the dictionary and + * the window. + */ + if (windowSize >= dictSize + srcSize) { + return windowLog; /* Window size large enough already */ + } else if (dictAndWindowSize >= maxWindowSize) { + return ZSTD_WINDOWLOG_MAX; /* Larger than max window log */ + } else { + return ZSTD_highbit32((U32)dictAndWindowSize - 1) + 1; + } + } +} + +/* ZSTD_adjustCParams_internal() : + * optimize `cPar` for a specified input (`srcSize` and `dictSize`). + * mostly downsize to reduce memory consumption and initialization latency. + * `srcSize` can be ZSTD_CONTENTSIZE_UNKNOWN when not known. + * `mode` is the mode for parameter adjustment. See docs for `ZSTD_cParamMode_e`. + * note : `srcSize==0` means 0! + * condition : cPar is presumed validated (can be checked using ZSTD_checkCParams()). */ +static ZSTD_compressionParameters +ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, + unsigned long long srcSize, + size_t dictSize, + ZSTD_cParamMode_e mode) +{ + const U64 minSrcSize = 513; /* (1<<9) + 1 */ + const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1); + assert(ZSTD_checkCParams(cPar)==0); + + switch (mode) { + case ZSTD_cpm_unknown: + case ZSTD_cpm_noAttachDict: + /* If we don't know the source size, don't make any + * assumptions about it. We will already have selected + * smaller parameters if a dictionary is in use. + */ + break; + case ZSTD_cpm_createCDict: + /* Assume a small source size when creating a dictionary + * with an unkown source size. + */ + if (dictSize && srcSize == ZSTD_CONTENTSIZE_UNKNOWN) + srcSize = minSrcSize; + break; + case ZSTD_cpm_attachDict: + /* Dictionary has its own dedicated parameters which have + * already been selected. We are selecting parameters + * for only the source. + */ + dictSize = 0; + break; + default: + assert(0); + break; + } + + /* resize windowLog if input is small enough, to use less memory */ + if ( (srcSize < maxWindowResize) + && (dictSize < maxWindowResize) ) { + U32 const tSize = (U32)(srcSize + dictSize); + static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN; + U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN : + ZSTD_highbit32(tSize-1) + 1; + if (cPar.windowLog > srcLog) cPar.windowLog = srcLog; + } + if (srcSize != ZSTD_CONTENTSIZE_UNKNOWN) { + U32 const dictAndWindowLog = ZSTD_dictAndWindowLog(cPar.windowLog, (U64)srcSize, (U64)dictSize); + U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy); + if (cPar.hashLog > dictAndWindowLog+1) cPar.hashLog = dictAndWindowLog+1; + if (cycleLog > dictAndWindowLog) + cPar.chainLog -= (cycleLog - dictAndWindowLog); + } + + if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) + cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* minimum wlog required for valid frame header */ + + return cPar; +} + +ZSTD_compressionParameters +ZSTD_adjustCParams(ZSTD_compressionParameters cPar, + unsigned long long srcSize, + size_t dictSize) +{ + cPar = ZSTD_clampCParams(cPar); /* resulting cPar is necessarily valid (all parameters within range) */ + if (srcSize == 0) srcSize = ZSTD_CONTENTSIZE_UNKNOWN; + return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize, ZSTD_cpm_unknown); +} + +static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); +static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); + +static void ZSTD_overrideCParams( + ZSTD_compressionParameters* cParams, + const ZSTD_compressionParameters* overrides) +{ + if (overrides->windowLog) cParams->windowLog = overrides->windowLog; + if (overrides->hashLog) cParams->hashLog = overrides->hashLog; + if (overrides->chainLog) cParams->chainLog = overrides->chainLog; + if (overrides->searchLog) cParams->searchLog = overrides->searchLog; + if (overrides->minMatch) cParams->minMatch = overrides->minMatch; + if (overrides->targetLength) cParams->targetLength = overrides->targetLength; + if (overrides->strategy) cParams->strategy = overrides->strategy; +} + +ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( + const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) +{ + ZSTD_compressionParameters cParams; + if (srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN && CCtxParams->srcSizeHint > 0) { + srcSizeHint = CCtxParams->srcSizeHint; + } + cParams = ZSTD_getCParams_internal(CCtxParams->compressionLevel, srcSizeHint, dictSize, mode); + if (CCtxParams->ldmParams.enableLdm) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG; + ZSTD_overrideCParams(&cParams, &CCtxParams->cParams); + assert(!ZSTD_checkCParams(cParams)); + /* srcSizeHint == 0 means 0 */ + return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize, mode); +} + +static size_t +ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, + const U32 forCCtx) +{ + size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog); + size_t const hSize = ((size_t)1) << cParams->hashLog; + U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; + size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; + /* We don't use ZSTD_cwksp_alloc_size() here because the tables aren't + * surrounded by redzones in ASAN. */ + size_t const tableSpace = chainSize * sizeof(U32) + + hSize * sizeof(U32) + + h3Size * sizeof(U32); + size_t const optPotentialSpace = + ZSTD_cwksp_alloc_size((MaxML+1) * sizeof(U32)) + + ZSTD_cwksp_alloc_size((MaxLL+1) * sizeof(U32)) + + ZSTD_cwksp_alloc_size((MaxOff+1) * sizeof(U32)) + + ZSTD_cwksp_alloc_size((1<strategy >= ZSTD_btopt)) + ? optPotentialSpace + : 0; + DEBUGLOG(4, "chainSize: %u - hSize: %u - h3Size: %u", + (U32)chainSize, (U32)hSize, (U32)h3Size); + return tableSpace + optSpace; +} + +static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal( + const ZSTD_compressionParameters* cParams, + const ldmParams_t* ldmParams, + const int isStatic, + const size_t buffInSize, + const size_t buffOutSize, + const U64 pledgedSrcSize) +{ + size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << cParams->windowLog), pledgedSrcSize)); + size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); + U32 const divider = (cParams->minMatch==3) ? 3 : 4; + size_t const maxNbSeq = blockSize / divider; + size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize) + + ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(seqDef)) + + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE)); + size_t const entropySpace = ZSTD_cwksp_alloc_size(ENTROPY_WORKSPACE_SIZE); + size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t)); + size_t const matchStateSize = ZSTD_sizeof_matchState(cParams, /* forCCtx */ 1); + + size_t const ldmSpace = ZSTD_ldm_getTableSize(*ldmParams); + size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(*ldmParams, blockSize); + size_t const ldmSeqSpace = ldmParams->enableLdm ? + ZSTD_cwksp_alloc_size(maxNbLdmSeq * sizeof(rawSeq)) : 0; + + + size_t const bufferSpace = ZSTD_cwksp_alloc_size(buffInSize) + + ZSTD_cwksp_alloc_size(buffOutSize); + + size_t const cctxSpace = isStatic ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0; + + size_t const neededSpace = + cctxSpace + + entropySpace + + blockStateSpace + + ldmSpace + + ldmSeqSpace + + matchStateSize + + tokenSpace + + bufferSpace; + + DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace); + return neededSpace; +} + +size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) +{ + ZSTD_compressionParameters const cParams = + ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); + + RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); + /* estimateCCtxSize is for one-shot compression. So no buffers should + * be needed. However, we still allocate two 0-sized buffers, which can + * take space under ASAN. */ + return ZSTD_estimateCCtxSize_usingCCtxParams_internal( + &cParams, ¶ms->ldmParams, 1, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN); +} + +size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams) +{ + ZSTD_CCtx_params const params = ZSTD_makeCCtxParamsFromCParams(cParams); + return ZSTD_estimateCCtxSize_usingCCtxParams(¶ms); +} + +static size_t ZSTD_estimateCCtxSize_internal(int compressionLevel) +{ + int tier = 0; + size_t largestSize = 0; + static const unsigned long long srcSizeTiers[4] = {16 KB, 128 KB, 256 KB, ZSTD_CONTENTSIZE_UNKNOWN}; + for (; tier < 4; ++tier) { + /* Choose the set of cParams for a given level across all srcSizes that give the largest cctxSize */ + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeTiers[tier], 0, ZSTD_cpm_noAttachDict); + largestSize = MAX(ZSTD_estimateCCtxSize_usingCParams(cParams), largestSize); + } + return largestSize; +} + +size_t ZSTD_estimateCCtxSize(int compressionLevel) +{ + int level; + size_t memBudget = 0; + for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) { + /* Ensure monotonically increasing memory usage as compression level increases */ + size_t const newMB = ZSTD_estimateCCtxSize_internal(level); + if (newMB > memBudget) memBudget = newMB; + } + return memBudget; +} + +size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params) +{ + RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); + { ZSTD_compressionParameters const cParams = + ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); + size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); + size_t const inBuffSize = (params->inBufferMode == ZSTD_bm_buffered) + ? ((size_t)1 << cParams.windowLog) + blockSize + : 0; + size_t const outBuffSize = (params->outBufferMode == ZSTD_bm_buffered) + ? ZSTD_compressBound(blockSize) + 1 + : 0; + + return ZSTD_estimateCCtxSize_usingCCtxParams_internal( + &cParams, ¶ms->ldmParams, 1, inBuffSize, outBuffSize, + ZSTD_CONTENTSIZE_UNKNOWN); + } +} + +size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams) +{ + ZSTD_CCtx_params const params = ZSTD_makeCCtxParamsFromCParams(cParams); + return ZSTD_estimateCStreamSize_usingCCtxParams(¶ms); +} + +static size_t ZSTD_estimateCStreamSize_internal(int compressionLevel) +{ + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); + return ZSTD_estimateCStreamSize_usingCParams(cParams); +} + +size_t ZSTD_estimateCStreamSize(int compressionLevel) +{ + int level; + size_t memBudget = 0; + for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) { + size_t const newMB = ZSTD_estimateCStreamSize_internal(level); + if (newMB > memBudget) memBudget = newMB; + } + return memBudget; +} + +/* ZSTD_getFrameProgression(): + * tells how much data has been consumed (input) and produced (output) for current frame. + * able to count progression inside worker threads (non-blocking mode). + */ +ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx) +{ + { ZSTD_frameProgression fp; + size_t const buffered = (cctx->inBuff == NULL) ? 0 : + cctx->inBuffPos - cctx->inToCompress; + if (buffered) assert(cctx->inBuffPos >= cctx->inToCompress); + assert(buffered <= ZSTD_BLOCKSIZE_MAX); + fp.ingested = cctx->consumedSrcSize + buffered; + fp.consumed = cctx->consumedSrcSize; + fp.produced = cctx->producedCSize; + fp.flushed = cctx->producedCSize; /* simplified; some data might still be left within streaming output buffer */ + fp.currentJobID = 0; + fp.nbActiveWorkers = 0; + return fp; +} } + +/*! ZSTD_toFlushNow() + * Only useful for multithreading scenarios currently (nbWorkers >= 1). + */ +size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx) +{ + (void)cctx; + return 0; /* over-simplification; could also check if context is currently running in streaming mode, and in which case, report how many bytes are left to be flushed within output buffer */ +} + +static void ZSTD_assertEqualCParams(ZSTD_compressionParameters cParams1, + ZSTD_compressionParameters cParams2) +{ + (void)cParams1; + (void)cParams2; + assert(cParams1.windowLog == cParams2.windowLog); + assert(cParams1.chainLog == cParams2.chainLog); + assert(cParams1.hashLog == cParams2.hashLog); + assert(cParams1.searchLog == cParams2.searchLog); + assert(cParams1.minMatch == cParams2.minMatch); + assert(cParams1.targetLength == cParams2.targetLength); + assert(cParams1.strategy == cParams2.strategy); +} + +void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs) +{ + int i; + for (i = 0; i < ZSTD_REP_NUM; ++i) + bs->rep[i] = repStartValue[i]; + bs->entropy.huf.repeatMode = HUF_repeat_none; + bs->entropy.fse.offcode_repeatMode = FSE_repeat_none; + bs->entropy.fse.matchlength_repeatMode = FSE_repeat_none; + bs->entropy.fse.litlength_repeatMode = FSE_repeat_none; +} + +/*! ZSTD_invalidateMatchState() + * Invalidate all the matches in the match finder tables. + * Requires nextSrc and base to be set (can be NULL). + */ +static void ZSTD_invalidateMatchState(ZSTD_matchState_t* ms) +{ + ZSTD_window_clear(&ms->window); + + ms->nextToUpdate = ms->window.dictLimit; + ms->loadedDictEnd = 0; + ms->opt.litLengthSum = 0; /* force reset of btopt stats */ + ms->dictMatchState = NULL; +} + +/* + * Controls, for this matchState reset, whether the tables need to be cleared / + * prepared for the coming compression (ZSTDcrp_makeClean), or whether the + * tables can be left unclean (ZSTDcrp_leaveDirty), because we know that a + * subsequent operation will overwrite the table space anyways (e.g., copying + * the matchState contents in from a CDict). + */ +typedef enum { + ZSTDcrp_makeClean, + ZSTDcrp_leaveDirty +} ZSTD_compResetPolicy_e; + +/* + * Controls, for this matchState reset, whether indexing can continue where it + * left off (ZSTDirp_continue), or whether it needs to be restarted from zero + * (ZSTDirp_reset). + */ +typedef enum { + ZSTDirp_continue, + ZSTDirp_reset +} ZSTD_indexResetPolicy_e; + +typedef enum { + ZSTD_resetTarget_CDict, + ZSTD_resetTarget_CCtx +} ZSTD_resetTarget_e; + +static size_t +ZSTD_reset_matchState(ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, + const ZSTD_compressionParameters* cParams, + const ZSTD_compResetPolicy_e crp, + const ZSTD_indexResetPolicy_e forceResetIndex, + const ZSTD_resetTarget_e forWho) +{ + size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog); + size_t const hSize = ((size_t)1) << cParams->hashLog; + U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; + size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; + + DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset); + if (forceResetIndex == ZSTDirp_reset) { + ZSTD_window_init(&ms->window); + ZSTD_cwksp_mark_tables_dirty(ws); + } + + ms->hashLog3 = hashLog3; + + ZSTD_invalidateMatchState(ms); + + assert(!ZSTD_cwksp_reserve_failed(ws)); /* check that allocation hasn't already failed */ + + ZSTD_cwksp_clear_tables(ws); + + DEBUGLOG(5, "reserving table space"); + /* table Space */ + ms->hashTable = (U32*)ZSTD_cwksp_reserve_table(ws, hSize * sizeof(U32)); + ms->chainTable = (U32*)ZSTD_cwksp_reserve_table(ws, chainSize * sizeof(U32)); + ms->hashTable3 = (U32*)ZSTD_cwksp_reserve_table(ws, h3Size * sizeof(U32)); + RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, + "failed a workspace allocation in ZSTD_reset_matchState"); + + DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_leaveDirty); + if (crp!=ZSTDcrp_leaveDirty) { + /* reset tables only */ + ZSTD_cwksp_clean_tables(ws); + } + + /* opt parser space */ + if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) { + DEBUGLOG(4, "reserving optimal parser space"); + ms->opt.litFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (1<opt.litLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxLL+1) * sizeof(unsigned)); + ms->opt.matchLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxML+1) * sizeof(unsigned)); + ms->opt.offCodeFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxOff+1) * sizeof(unsigned)); + ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t)); + ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t)); + } + + ms->cParams = *cParams; + + RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, + "failed a workspace allocation in ZSTD_reset_matchState"); + + return 0; +} + +/* ZSTD_indexTooCloseToMax() : + * minor optimization : prefer memset() rather than reduceIndex() + * which is measurably slow in some circumstances (reported for Visual Studio). + * Works when re-using a context for a lot of smallish inputs : + * if all inputs are smaller than ZSTD_INDEXOVERFLOW_MARGIN, + * memset() will be triggered before reduceIndex(). + */ +#define ZSTD_INDEXOVERFLOW_MARGIN (16 MB) +static int ZSTD_indexTooCloseToMax(ZSTD_window_t w) +{ + return (size_t)(w.nextSrc - w.base) > (ZSTD_CURRENT_MAX - ZSTD_INDEXOVERFLOW_MARGIN); +} + +/*! ZSTD_resetCCtx_internal() : + note : `params` are assumed fully validated at this stage */ +static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, + ZSTD_CCtx_params params, + U64 const pledgedSrcSize, + ZSTD_compResetPolicy_e const crp, + ZSTD_buffered_policy_e const zbuff) +{ + ZSTD_cwksp* const ws = &zc->workspace; + DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u", + (U32)pledgedSrcSize, params.cParams.windowLog); + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + + zc->isFirstBlock = 1; + + if (params.ldmParams.enableLdm) { + /* Adjust long distance matching parameters */ + ZSTD_ldm_adjustParameters(¶ms.ldmParams, ¶ms.cParams); + assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog); + assert(params.ldmParams.hashRateLog < 32); + } + + { size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params.cParams.windowLog), pledgedSrcSize)); + size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); + U32 const divider = (params.cParams.minMatch==3) ? 3 : 4; + size_t const maxNbSeq = blockSize / divider; + size_t const buffOutSize = (zbuff == ZSTDb_buffered && params.outBufferMode == ZSTD_bm_buffered) + ? ZSTD_compressBound(blockSize) + 1 + : 0; + size_t const buffInSize = (zbuff == ZSTDb_buffered && params.inBufferMode == ZSTD_bm_buffered) + ? windowSize + blockSize + : 0; + size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params.ldmParams, blockSize); + + int const indexTooClose = ZSTD_indexTooCloseToMax(zc->blockState.matchState.window); + ZSTD_indexResetPolicy_e needsIndexReset = + (!indexTooClose && zc->initialized) ? ZSTDirp_continue : ZSTDirp_reset; + + size_t const neededSpace = + ZSTD_estimateCCtxSize_usingCCtxParams_internal( + ¶ms.cParams, ¶ms.ldmParams, zc->staticSize != 0, + buffInSize, buffOutSize, pledgedSrcSize); + FORWARD_IF_ERROR(neededSpace, "cctx size estimate failed!"); + + if (!zc->staticSize) ZSTD_cwksp_bump_oversized_duration(ws, 0); + + /* Check if workspace is large enough, alloc a new one if needed */ + { + int const workspaceTooSmall = ZSTD_cwksp_sizeof(ws) < neededSpace; + int const workspaceWasteful = ZSTD_cwksp_check_wasteful(ws, neededSpace); + + DEBUGLOG(4, "Need %zu B workspace", neededSpace); + DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize); + + if (workspaceTooSmall || workspaceWasteful) { + DEBUGLOG(4, "Resize workspaceSize from %zuKB to %zuKB", + ZSTD_cwksp_sizeof(ws) >> 10, + neededSpace >> 10); + + RETURN_ERROR_IF(zc->staticSize, memory_allocation, "static cctx : no resize"); + + needsIndexReset = ZSTDirp_reset; + + ZSTD_cwksp_free(ws, zc->customMem); + FORWARD_IF_ERROR(ZSTD_cwksp_create(ws, neededSpace, zc->customMem), ""); + + DEBUGLOG(5, "reserving object space"); + /* Statically sized space. + * entropyWorkspace never moves, + * though prev/next block swap places */ + assert(ZSTD_cwksp_check_available(ws, 2 * sizeof(ZSTD_compressedBlockState_t))); + zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); + RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock"); + zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); + RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate nextCBlock"); + zc->entropyWorkspace = (U32*) ZSTD_cwksp_reserve_object(ws, ENTROPY_WORKSPACE_SIZE); + RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate entropyWorkspace"); + } } + + ZSTD_cwksp_clear(ws); + + /* init params */ + zc->appliedParams = params; + zc->blockState.matchState.cParams = params.cParams; + zc->pledgedSrcSizePlusOne = pledgedSrcSize+1; + zc->consumedSrcSize = 0; + zc->producedCSize = 0; + if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN) + zc->appliedParams.fParams.contentSizeFlag = 0; + DEBUGLOG(4, "pledged content size : %u ; flag : %u", + (unsigned)pledgedSrcSize, zc->appliedParams.fParams.contentSizeFlag); + zc->blockSize = blockSize; + + xxh64_reset(&zc->xxhState, 0); + zc->stage = ZSTDcs_init; + zc->dictID = 0; + zc->dictContentSize = 0; + + ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock); + + /* ZSTD_wildcopy() is used to copy into the literals buffer, + * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes. + */ + zc->seqStore.litStart = ZSTD_cwksp_reserve_buffer(ws, blockSize + WILDCOPY_OVERLENGTH); + zc->seqStore.maxNbLit = blockSize; + + /* buffers */ + zc->bufferedPolicy = zbuff; + zc->inBuffSize = buffInSize; + zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffInSize); + zc->outBuffSize = buffOutSize; + zc->outBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffOutSize); + + /* ldm bucketOffsets table */ + if (params.ldmParams.enableLdm) { + /* TODO: avoid memset? */ + size_t const numBuckets = + ((size_t)1) << (params.ldmParams.hashLog - + params.ldmParams.bucketSizeLog); + zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, numBuckets); + ZSTD_memset(zc->ldmState.bucketOffsets, 0, numBuckets); + } + + /* sequences storage */ + ZSTD_referenceExternalSequences(zc, NULL, 0); + zc->seqStore.maxNbSeq = maxNbSeq; + zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(ws, maxNbSeq * sizeof(seqDef)); + + FORWARD_IF_ERROR(ZSTD_reset_matchState( + &zc->blockState.matchState, + ws, + ¶ms.cParams, + crp, + needsIndexReset, + ZSTD_resetTarget_CCtx), ""); + + /* ldm hash table */ + if (params.ldmParams.enableLdm) { + /* TODO: avoid memset? */ + size_t const ldmHSize = ((size_t)1) << params.ldmParams.hashLog; + zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t)); + ZSTD_memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); + zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq)); + zc->maxNbLdmSequences = maxNbLdmSeq; + + ZSTD_window_init(&zc->ldmState.window); + ZSTD_window_clear(&zc->ldmState.window); + zc->ldmState.loadedDictEnd = 0; + } + + /* Due to alignment, when reusing a workspace, we can actually consume + * up to 3 extra bytes for alignment. See the comments in zstd_cwksp.h + */ + assert(ZSTD_cwksp_used(ws) >= neededSpace && + ZSTD_cwksp_used(ws) <= neededSpace + 3); + + DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws)); + zc->initialized = 1; + + return 0; + } +} + +/* ZSTD_invalidateRepCodes() : + * ensures next compression will not use repcodes from previous block. + * Note : only works with regular variant; + * do not use with extDict variant ! */ +void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) { + int i; + for (i=0; iblockState.prevCBlock->rep[i] = 0; + assert(!ZSTD_window_hasExtDict(cctx->blockState.matchState.window)); +} + +/* These are the approximate sizes for each strategy past which copying the + * dictionary tables into the working context is faster than using them + * in-place. + */ +static const size_t attachDictSizeCutoffs[ZSTD_STRATEGY_MAX+1] = { + 8 KB, /* unused */ + 8 KB, /* ZSTD_fast */ + 16 KB, /* ZSTD_dfast */ + 32 KB, /* ZSTD_greedy */ + 32 KB, /* ZSTD_lazy */ + 32 KB, /* ZSTD_lazy2 */ + 32 KB, /* ZSTD_btlazy2 */ + 32 KB, /* ZSTD_btopt */ + 8 KB, /* ZSTD_btultra */ + 8 KB /* ZSTD_btultra2 */ +}; + +static int ZSTD_shouldAttachDict(const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + U64 pledgedSrcSize) +{ + size_t cutoff = attachDictSizeCutoffs[cdict->matchState.cParams.strategy]; + int const dedicatedDictSearch = cdict->matchState.dedicatedDictSearch; + return dedicatedDictSearch + || ( ( pledgedSrcSize <= cutoff + || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN + || params->attachDictPref == ZSTD_dictForceAttach ) + && params->attachDictPref != ZSTD_dictForceCopy + && !params->forceWindow ); /* dictMatchState isn't correctly + * handled in _enforceMaxDist */ +} + +static size_t +ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, + const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, + U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ + { + ZSTD_compressionParameters adjusted_cdict_cParams = cdict->matchState.cParams; + unsigned const windowLog = params.cParams.windowLog; + assert(windowLog != 0); + /* Resize working context table params for input only, since the dict + * has its own tables. */ + /* pledgedSrcSize == 0 means 0! */ + + if (cdict->matchState.dedicatedDictSearch) { + ZSTD_dedicatedDictSearch_revertCParams(&adjusted_cdict_cParams); + } + + params.cParams = ZSTD_adjustCParams_internal(adjusted_cdict_cParams, pledgedSrcSize, + cdict->dictContentSize, ZSTD_cpm_attachDict); + params.cParams.windowLog = windowLog; + FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, + ZSTDcrp_makeClean, zbuff), ""); + assert(cctx->appliedParams.cParams.strategy == adjusted_cdict_cParams.strategy); + } + + { const U32 cdictEnd = (U32)( cdict->matchState.window.nextSrc + - cdict->matchState.window.base); + const U32 cdictLen = cdictEnd - cdict->matchState.window.dictLimit; + if (cdictLen == 0) { + /* don't even attach dictionaries with no contents */ + DEBUGLOG(4, "skipping attaching empty dictionary"); + } else { + DEBUGLOG(4, "attaching dictionary into context"); + cctx->blockState.matchState.dictMatchState = &cdict->matchState; + + /* prep working match state so dict matches never have negative indices + * when they are translated to the working context's index space. */ + if (cctx->blockState.matchState.window.dictLimit < cdictEnd) { + cctx->blockState.matchState.window.nextSrc = + cctx->blockState.matchState.window.base + cdictEnd; + ZSTD_window_clear(&cctx->blockState.matchState.window); + } + /* loadedDictEnd is expressed within the referential of the active context */ + cctx->blockState.matchState.loadedDictEnd = cctx->blockState.matchState.window.dictLimit; + } } + + cctx->dictID = cdict->dictID; + cctx->dictContentSize = cdict->dictContentSize; + + /* copy block state */ + ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState)); + + return 0; +} + +static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, + const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, + U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ + const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams; + + assert(!cdict->matchState.dedicatedDictSearch); + + DEBUGLOG(4, "copying dictionary into context"); + + { unsigned const windowLog = params.cParams.windowLog; + assert(windowLog != 0); + /* Copy only compression parameters related to tables. */ + params.cParams = *cdict_cParams; + params.cParams.windowLog = windowLog; + FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, + ZSTDcrp_leaveDirty, zbuff), ""); + assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); + assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog); + assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog); + } + + ZSTD_cwksp_mark_tables_dirty(&cctx->workspace); + + /* copy tables */ + { size_t const chainSize = (cdict_cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cdict_cParams->chainLog); + size_t const hSize = (size_t)1 << cdict_cParams->hashLog; + + ZSTD_memcpy(cctx->blockState.matchState.hashTable, + cdict->matchState.hashTable, + hSize * sizeof(U32)); + ZSTD_memcpy(cctx->blockState.matchState.chainTable, + cdict->matchState.chainTable, + chainSize * sizeof(U32)); + } + + /* Zero the hashTable3, since the cdict never fills it */ + { int const h3log = cctx->blockState.matchState.hashLog3; + size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; + assert(cdict->matchState.hashLog3 == 0); + ZSTD_memset(cctx->blockState.matchState.hashTable3, 0, h3Size * sizeof(U32)); + } + + ZSTD_cwksp_mark_tables_clean(&cctx->workspace); + + /* copy dictionary offsets */ + { ZSTD_matchState_t const* srcMatchState = &cdict->matchState; + ZSTD_matchState_t* dstMatchState = &cctx->blockState.matchState; + dstMatchState->window = srcMatchState->window; + dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; + dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; + } + + cctx->dictID = cdict->dictID; + cctx->dictContentSize = cdict->dictContentSize; + + /* copy block state */ + ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState)); + + return 0; +} + +/* We have a choice between copying the dictionary context into the working + * context, or referencing the dictionary context from the working context + * in-place. We decide here which strategy to use. */ +static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ + + DEBUGLOG(4, "ZSTD_resetCCtx_usingCDict (pledgedSrcSize=%u)", + (unsigned)pledgedSrcSize); + + if (ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) { + return ZSTD_resetCCtx_byAttachingCDict( + cctx, cdict, *params, pledgedSrcSize, zbuff); + } else { + return ZSTD_resetCCtx_byCopyingCDict( + cctx, cdict, *params, pledgedSrcSize, zbuff); + } +} + +/*! ZSTD_copyCCtx_internal() : + * Duplicate an existing context `srcCCtx` into another one `dstCCtx`. + * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). + * The "context", in this case, refers to the hash and chain tables, + * entropy tables, and dictionary references. + * `windowLog` value is enforced if != 0, otherwise value is copied from srcCCtx. + * @return : 0, or an error code */ +static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, + const ZSTD_CCtx* srcCCtx, + ZSTD_frameParameters fParams, + U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ + DEBUGLOG(5, "ZSTD_copyCCtx_internal"); + RETURN_ERROR_IF(srcCCtx->stage!=ZSTDcs_init, stage_wrong, + "Can't copy a ctx that's not in init stage."); + + ZSTD_memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); + { ZSTD_CCtx_params params = dstCCtx->requestedParams; + /* Copy only compression parameters related to tables. */ + params.cParams = srcCCtx->appliedParams.cParams; + params.fParams = fParams; + ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize, + ZSTDcrp_leaveDirty, zbuff); + assert(dstCCtx->appliedParams.cParams.windowLog == srcCCtx->appliedParams.cParams.windowLog); + assert(dstCCtx->appliedParams.cParams.strategy == srcCCtx->appliedParams.cParams.strategy); + assert(dstCCtx->appliedParams.cParams.hashLog == srcCCtx->appliedParams.cParams.hashLog); + assert(dstCCtx->appliedParams.cParams.chainLog == srcCCtx->appliedParams.cParams.chainLog); + assert(dstCCtx->blockState.matchState.hashLog3 == srcCCtx->blockState.matchState.hashLog3); + } + + ZSTD_cwksp_mark_tables_dirty(&dstCCtx->workspace); + + /* copy tables */ + { size_t const chainSize = (srcCCtx->appliedParams.cParams.strategy == ZSTD_fast) ? 0 : ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog); + size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog; + int const h3log = srcCCtx->blockState.matchState.hashLog3; + size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; + + ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable, + srcCCtx->blockState.matchState.hashTable, + hSize * sizeof(U32)); + ZSTD_memcpy(dstCCtx->blockState.matchState.chainTable, + srcCCtx->blockState.matchState.chainTable, + chainSize * sizeof(U32)); + ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable3, + srcCCtx->blockState.matchState.hashTable3, + h3Size * sizeof(U32)); + } + + ZSTD_cwksp_mark_tables_clean(&dstCCtx->workspace); + + /* copy dictionary offsets */ + { + const ZSTD_matchState_t* srcMatchState = &srcCCtx->blockState.matchState; + ZSTD_matchState_t* dstMatchState = &dstCCtx->blockState.matchState; + dstMatchState->window = srcMatchState->window; + dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; + dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; + } + dstCCtx->dictID = srcCCtx->dictID; + dstCCtx->dictContentSize = srcCCtx->dictContentSize; + + /* copy block state */ + ZSTD_memcpy(dstCCtx->blockState.prevCBlock, srcCCtx->blockState.prevCBlock, sizeof(*srcCCtx->blockState.prevCBlock)); + + return 0; +} + +/*! ZSTD_copyCCtx() : + * Duplicate an existing context `srcCCtx` into another one `dstCCtx`. + * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). + * pledgedSrcSize==0 means "unknown". +* @return : 0, or an error code */ +size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize) +{ + ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; + ZSTD_buffered_policy_e const zbuff = srcCCtx->bufferedPolicy; + ZSTD_STATIC_ASSERT((U32)ZSTDb_buffered==1); + if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; + fParams.contentSizeFlag = (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN); + + return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx, + fParams, pledgedSrcSize, + zbuff); +} + + +#define ZSTD_ROWSIZE 16 +/*! ZSTD_reduceTable() : + * reduce table indexes by `reducerValue`, or squash to zero. + * PreserveMark preserves "unsorted mark" for btlazy2 strategy. + * It must be set to a clear 0/1 value, to remove branch during inlining. + * Presume table size is a multiple of ZSTD_ROWSIZE + * to help auto-vectorization */ +FORCE_INLINE_TEMPLATE void +ZSTD_reduceTable_internal (U32* const table, U32 const size, U32 const reducerValue, int const preserveMark) +{ + int const nbRows = (int)size / ZSTD_ROWSIZE; + int cellNb = 0; + int rowNb; + assert((size & (ZSTD_ROWSIZE-1)) == 0); /* multiple of ZSTD_ROWSIZE */ + assert(size < (1U<<31)); /* can be casted to int */ + + + for (rowNb=0 ; rowNb < nbRows ; rowNb++) { + int column; + for (column=0; columncParams.hashLog; + ZSTD_reduceTable(ms->hashTable, hSize, reducerValue); + } + + if (params->cParams.strategy != ZSTD_fast) { + U32 const chainSize = (U32)1 << params->cParams.chainLog; + if (params->cParams.strategy == ZSTD_btlazy2) + ZSTD_reduceTable_btlazy2(ms->chainTable, chainSize, reducerValue); + else + ZSTD_reduceTable(ms->chainTable, chainSize, reducerValue); + } + + if (ms->hashLog3) { + U32 const h3Size = (U32)1 << ms->hashLog3; + ZSTD_reduceTable(ms->hashTable3, h3Size, reducerValue); + } +} + + +/*-******************************************************* +* Block entropic compression +*********************************************************/ + +/* See doc/zstd_compression_format.md for detailed format description */ + +void ZSTD_seqToCodes(const seqStore_t* seqStorePtr) +{ + const seqDef* const sequences = seqStorePtr->sequencesStart; + BYTE* const llCodeTable = seqStorePtr->llCode; + BYTE* const ofCodeTable = seqStorePtr->ofCode; + BYTE* const mlCodeTable = seqStorePtr->mlCode; + U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + U32 u; + assert(nbSeq <= seqStorePtr->maxNbSeq); + for (u=0; ulongLengthID==1) + llCodeTable[seqStorePtr->longLengthPos] = MaxLL; + if (seqStorePtr->longLengthID==2) + mlCodeTable[seqStorePtr->longLengthPos] = MaxML; +} + +/* ZSTD_useTargetCBlockSize(): + * Returns if target compressed block size param is being used. + * If used, compression will do best effort to make a compressed block size to be around targetCBlockSize. + * Returns 1 if true, 0 otherwise. */ +static int ZSTD_useTargetCBlockSize(const ZSTD_CCtx_params* cctxParams) +{ + DEBUGLOG(5, "ZSTD_useTargetCBlockSize (targetCBlockSize=%zu)", cctxParams->targetCBlockSize); + return (cctxParams->targetCBlockSize != 0); +} + +/* ZSTD_entropyCompressSequences_internal(): + * actually compresses both literals and sequences */ +MEM_STATIC size_t +ZSTD_entropyCompressSequences_internal(seqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + void* entropyWorkspace, size_t entropyWkspSize, + const int bmi2) +{ + const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN; + ZSTD_strategy const strategy = cctxParams->cParams.strategy; + unsigned* count = (unsigned*)entropyWorkspace; + FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable; + FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable; + FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable; + U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */ + const seqDef* const sequences = seqStorePtr->sequencesStart; + const BYTE* const ofCodeTable = seqStorePtr->ofCode; + const BYTE* const llCodeTable = seqStorePtr->llCode; + const BYTE* const mlCodeTable = seqStorePtr->mlCode; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + size_t const nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + BYTE* seqHead; + BYTE* lastNCount = NULL; + + entropyWorkspace = count + (MaxSeq + 1); + entropyWkspSize -= (MaxSeq + 1) * sizeof(*count); + + DEBUGLOG(4, "ZSTD_entropyCompressSequences_internal (nbSeq=%zu)", nbSeq); + ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<= HUF_WORKSPACE_SIZE); + + /* Compress literals */ + { const BYTE* const literals = seqStorePtr->litStart; + size_t const litSize = (size_t)(seqStorePtr->lit - literals); + size_t const cSize = ZSTD_compressLiterals( + &prevEntropy->huf, &nextEntropy->huf, + cctxParams->cParams.strategy, + ZSTD_disableLiteralsCompression(cctxParams), + op, dstCapacity, + literals, litSize, + entropyWorkspace, entropyWkspSize, + bmi2); + FORWARD_IF_ERROR(cSize, "ZSTD_compressLiterals failed"); + assert(cSize <= dstCapacity); + op += cSize; + } + + /* Sequences Header */ + RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, + dstSize_tooSmall, "Can't fit seq hdr in output buf!"); + if (nbSeq < 128) { + *op++ = (BYTE)nbSeq; + } else if (nbSeq < LONGNBSEQ) { + op[0] = (BYTE)((nbSeq>>8) + 0x80); + op[1] = (BYTE)nbSeq; + op+=2; + } else { + op[0]=0xFF; + MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)); + op+=3; + } + assert(op <= oend); + if (nbSeq==0) { + /* Copy the old tables over as if we repeated them */ + ZSTD_memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse)); + return (size_t)(op - ostart); + } + + /* seqHead : flags for FSE encoding type */ + seqHead = op++; + assert(op <= oend); + + /* convert length/distances into codes */ + ZSTD_seqToCodes(seqStorePtr); + /* build CTable for Literal Lengths */ + { unsigned max = MaxLL; + size_t const mostFrequent = HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ + DEBUGLOG(5, "Building LL table"); + nextEntropy->fse.litlength_repeatMode = prevEntropy->fse.litlength_repeatMode; + LLtype = ZSTD_selectEncodingType(&nextEntropy->fse.litlength_repeatMode, + count, max, mostFrequent, nbSeq, + LLFSELog, prevEntropy->fse.litlengthCTable, + LL_defaultNorm, LL_defaultNormLog, + ZSTD_defaultAllowed, strategy); + assert(set_basic < set_compressed && set_rle < set_compressed); + assert(!(LLtype < set_compressed && nextEntropy->fse.litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype, + count, max, llCodeTable, nbSeq, + LL_defaultNorm, LL_defaultNormLog, MaxLL, + prevEntropy->fse.litlengthCTable, + sizeof(prevEntropy->fse.litlengthCTable), + entropyWorkspace, entropyWkspSize); + FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for LitLens failed"); + if (LLtype == set_compressed) + lastNCount = op; + op += countSize; + assert(op <= oend); + } } + /* build CTable for Offsets */ + { unsigned max = MaxOff; + size_t const mostFrequent = HIST_countFast_wksp( + count, &max, ofCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ + /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ + ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; + DEBUGLOG(5, "Building OF table"); + nextEntropy->fse.offcode_repeatMode = prevEntropy->fse.offcode_repeatMode; + Offtype = ZSTD_selectEncodingType(&nextEntropy->fse.offcode_repeatMode, + count, max, mostFrequent, nbSeq, + OffFSELog, prevEntropy->fse.offcodeCTable, + OF_defaultNorm, OF_defaultNormLog, + defaultPolicy, strategy); + assert(!(Offtype < set_compressed && nextEntropy->fse.offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype, + count, max, ofCodeTable, nbSeq, + OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, + prevEntropy->fse.offcodeCTable, + sizeof(prevEntropy->fse.offcodeCTable), + entropyWorkspace, entropyWkspSize); + FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for Offsets failed"); + if (Offtype == set_compressed) + lastNCount = op; + op += countSize; + assert(op <= oend); + } } + /* build CTable for MatchLengths */ + { unsigned max = MaxML; + size_t const mostFrequent = HIST_countFast_wksp( + count, &max, mlCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ + DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op)); + nextEntropy->fse.matchlength_repeatMode = prevEntropy->fse.matchlength_repeatMode; + MLtype = ZSTD_selectEncodingType(&nextEntropy->fse.matchlength_repeatMode, + count, max, mostFrequent, nbSeq, + MLFSELog, prevEntropy->fse.matchlengthCTable, + ML_defaultNorm, ML_defaultNormLog, + ZSTD_defaultAllowed, strategy); + assert(!(MLtype < set_compressed && nextEntropy->fse.matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype, + count, max, mlCodeTable, nbSeq, + ML_defaultNorm, ML_defaultNormLog, MaxML, + prevEntropy->fse.matchlengthCTable, + sizeof(prevEntropy->fse.matchlengthCTable), + entropyWorkspace, entropyWkspSize); + FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for MatchLengths failed"); + if (MLtype == set_compressed) + lastNCount = op; + op += countSize; + assert(op <= oend); + } } + + *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); + + { size_t const bitstreamSize = ZSTD_encodeSequences( + op, (size_t)(oend - op), + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, + longOffsets, bmi2); + FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed"); + op += bitstreamSize; + assert(op <= oend); + /* zstd versions <= 1.3.4 mistakenly report corruption when + * FSE_readNCount() receives a buffer < 4 bytes. + * Fixed by https://github.com/facebook/zstd/pull/1146. + * This can happen when the last set_compressed table present is 2 + * bytes and the bitstream is only one byte. + * In this exceedingly rare case, we will simply emit an uncompressed + * block, since it isn't worth optimizing. + */ + if (lastNCount && (op - lastNCount) < 4) { + /* NCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */ + assert(op - lastNCount == 3); + DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by " + "emitting an uncompressed block."); + return 0; + } + } + + DEBUGLOG(5, "compressed block size : %u", (unsigned)(op - ostart)); + return (size_t)(op - ostart); +} + +MEM_STATIC size_t +ZSTD_entropyCompressSequences(seqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + size_t srcSize, + void* entropyWorkspace, size_t entropyWkspSize, + int bmi2) +{ + size_t const cSize = ZSTD_entropyCompressSequences_internal( + seqStorePtr, prevEntropy, nextEntropy, cctxParams, + dst, dstCapacity, + entropyWorkspace, entropyWkspSize, bmi2); + if (cSize == 0) return 0; + /* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block. + * Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block. + */ + if ((cSize == ERROR(dstSize_tooSmall)) & (srcSize <= dstCapacity)) + return 0; /* block not compressed */ + FORWARD_IF_ERROR(cSize, "ZSTD_entropyCompressSequences_internal failed"); + + /* Check compressibility */ + { size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy); + if (cSize >= maxCSize) return 0; /* block not compressed */ + } + DEBUGLOG(4, "ZSTD_entropyCompressSequences() cSize: %zu\n", cSize); + return cSize; +} + +/* ZSTD_selectBlockCompressor() : + * Not static, but internal use only (used by long distance matcher) + * assumption : strat is a valid strategy */ +ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMode_e dictMode) +{ + static const ZSTD_blockCompressor blockCompressor[4][ZSTD_STRATEGY_MAX+1] = { + { ZSTD_compressBlock_fast /* default for 0 */, + ZSTD_compressBlock_fast, + ZSTD_compressBlock_doubleFast, + ZSTD_compressBlock_greedy, + ZSTD_compressBlock_lazy, + ZSTD_compressBlock_lazy2, + ZSTD_compressBlock_btlazy2, + ZSTD_compressBlock_btopt, + ZSTD_compressBlock_btultra, + ZSTD_compressBlock_btultra2 }, + { ZSTD_compressBlock_fast_extDict /* default for 0 */, + ZSTD_compressBlock_fast_extDict, + ZSTD_compressBlock_doubleFast_extDict, + ZSTD_compressBlock_greedy_extDict, + ZSTD_compressBlock_lazy_extDict, + ZSTD_compressBlock_lazy2_extDict, + ZSTD_compressBlock_btlazy2_extDict, + ZSTD_compressBlock_btopt_extDict, + ZSTD_compressBlock_btultra_extDict, + ZSTD_compressBlock_btultra_extDict }, + { ZSTD_compressBlock_fast_dictMatchState /* default for 0 */, + ZSTD_compressBlock_fast_dictMatchState, + ZSTD_compressBlock_doubleFast_dictMatchState, + ZSTD_compressBlock_greedy_dictMatchState, + ZSTD_compressBlock_lazy_dictMatchState, + ZSTD_compressBlock_lazy2_dictMatchState, + ZSTD_compressBlock_btlazy2_dictMatchState, + ZSTD_compressBlock_btopt_dictMatchState, + ZSTD_compressBlock_btultra_dictMatchState, + ZSTD_compressBlock_btultra_dictMatchState }, + { NULL /* default for 0 */, + NULL, + NULL, + ZSTD_compressBlock_greedy_dedicatedDictSearch, + ZSTD_compressBlock_lazy_dedicatedDictSearch, + ZSTD_compressBlock_lazy2_dedicatedDictSearch, + NULL, + NULL, + NULL, + NULL } + }; + ZSTD_blockCompressor selectedCompressor; + ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1); + + assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat)); + selectedCompressor = blockCompressor[(int)dictMode][(int)strat]; + assert(selectedCompressor != NULL); + return selectedCompressor; +} + +static void ZSTD_storeLastLiterals(seqStore_t* seqStorePtr, + const BYTE* anchor, size_t lastLLSize) +{ + ZSTD_memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; +} + +void ZSTD_resetSeqStore(seqStore_t* ssPtr) +{ + ssPtr->lit = ssPtr->litStart; + ssPtr->sequences = ssPtr->sequencesStart; + ssPtr->longLengthID = 0; +} + +typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e; + +static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) +{ + ZSTD_matchState_t* const ms = &zc->blockState.matchState; + DEBUGLOG(5, "ZSTD_buildSeqStore (srcSize=%zu)", srcSize); + assert(srcSize <= ZSTD_BLOCKSIZE_MAX); + /* Assert that we have correctly flushed the ctx params into the ms's copy */ + ZSTD_assertEqualCParams(zc->appliedParams.cParams, ms->cParams); + if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) { + if (zc->appliedParams.cParams.strategy >= ZSTD_btopt) { + ZSTD_ldm_skipRawSeqStoreBytes(&zc->externSeqStore, srcSize); + } else { + ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.minMatch); + } + return ZSTDbss_noCompress; /* don't even attempt compression below a certain srcSize */ + } + ZSTD_resetSeqStore(&(zc->seqStore)); + /* required for optimal parser to read stats from dictionary */ + ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy; + /* tell the optimal parser how we expect to compress literals */ + ms->opt.literalCompressionMode = zc->appliedParams.literalCompressionMode; + /* a gap between an attached dict and the current window is not safe, + * they must remain adjacent, + * and when that stops being the case, the dict must be unset */ + assert(ms->dictMatchState == NULL || ms->loadedDictEnd == ms->window.dictLimit); + + /* limited update after a very long match */ + { const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const U32 curr = (U32)(istart-base); + if (sizeof(ptrdiff_t)==8) assert(istart - base < (ptrdiff_t)(U32)(-1)); /* ensure no overflow */ + if (curr > ms->nextToUpdate + 384) + ms->nextToUpdate = curr - MIN(192, (U32)(curr - ms->nextToUpdate - 384)); + } + + /* select and store sequences */ + { ZSTD_dictMode_e const dictMode = ZSTD_matchState_dictMode(ms); + size_t lastLLSize; + { int i; + for (i = 0; i < ZSTD_REP_NUM; ++i) + zc->blockState.nextCBlock->rep[i] = zc->blockState.prevCBlock->rep[i]; + } + if (zc->externSeqStore.pos < zc->externSeqStore.size) { + assert(!zc->appliedParams.ldmParams.enableLdm); + /* Updates ldmSeqStore.pos */ + lastLLSize = + ZSTD_ldm_blockCompress(&zc->externSeqStore, + ms, &zc->seqStore, + zc->blockState.nextCBlock->rep, + src, srcSize); + assert(zc->externSeqStore.pos <= zc->externSeqStore.size); + } else if (zc->appliedParams.ldmParams.enableLdm) { + rawSeqStore_t ldmSeqStore = kNullRawSeqStore; + + ldmSeqStore.seq = zc->ldmSequences; + ldmSeqStore.capacity = zc->maxNbLdmSequences; + /* Updates ldmSeqStore.size */ + FORWARD_IF_ERROR(ZSTD_ldm_generateSequences(&zc->ldmState, &ldmSeqStore, + &zc->appliedParams.ldmParams, + src, srcSize), ""); + /* Updates ldmSeqStore.pos */ + lastLLSize = + ZSTD_ldm_blockCompress(&ldmSeqStore, + ms, &zc->seqStore, + zc->blockState.nextCBlock->rep, + src, srcSize); + assert(ldmSeqStore.pos == ldmSeqStore.size); + } else { /* not long range mode */ + ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, dictMode); + ms->ldmSeqStore = NULL; + lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize); + } + { const BYTE* const lastLiterals = (const BYTE*)src + srcSize - lastLLSize; + ZSTD_storeLastLiterals(&zc->seqStore, lastLiterals, lastLLSize); + } } + return ZSTDbss_compress; +} + +static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) +{ + const seqStore_t* seqStore = ZSTD_getSeqStore(zc); + const seqDef* seqStoreSeqs = seqStore->sequencesStart; + size_t seqStoreSeqSize = seqStore->sequences - seqStoreSeqs; + size_t seqStoreLiteralsSize = (size_t)(seqStore->lit - seqStore->litStart); + size_t literalsRead = 0; + size_t lastLLSize; + + ZSTD_Sequence* outSeqs = &zc->seqCollector.seqStart[zc->seqCollector.seqIndex]; + size_t i; + repcodes_t updatedRepcodes; + + assert(zc->seqCollector.seqIndex + 1 < zc->seqCollector.maxSequences); + /* Ensure we have enough space for last literals "sequence" */ + assert(zc->seqCollector.maxSequences >= seqStoreSeqSize + 1); + ZSTD_memcpy(updatedRepcodes.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t)); + for (i = 0; i < seqStoreSeqSize; ++i) { + U32 rawOffset = seqStoreSeqs[i].offset - ZSTD_REP_NUM; + outSeqs[i].litLength = seqStoreSeqs[i].litLength; + outSeqs[i].matchLength = seqStoreSeqs[i].matchLength + MINMATCH; + outSeqs[i].rep = 0; + + if (i == seqStore->longLengthPos) { + if (seqStore->longLengthID == 1) { + outSeqs[i].litLength += 0x10000; + } else if (seqStore->longLengthID == 2) { + outSeqs[i].matchLength += 0x10000; + } + } + + if (seqStoreSeqs[i].offset <= ZSTD_REP_NUM) { + /* Derive the correct offset corresponding to a repcode */ + outSeqs[i].rep = seqStoreSeqs[i].offset; + if (outSeqs[i].litLength != 0) { + rawOffset = updatedRepcodes.rep[outSeqs[i].rep - 1]; + } else { + if (outSeqs[i].rep == 3) { + rawOffset = updatedRepcodes.rep[0] - 1; + } else { + rawOffset = updatedRepcodes.rep[outSeqs[i].rep]; + } + } + } + outSeqs[i].offset = rawOffset; + /* seqStoreSeqs[i].offset == offCode+1, and ZSTD_updateRep() expects offCode + so we provide seqStoreSeqs[i].offset - 1 */ + updatedRepcodes = ZSTD_updateRep(updatedRepcodes.rep, + seqStoreSeqs[i].offset - 1, + seqStoreSeqs[i].litLength == 0); + literalsRead += outSeqs[i].litLength; + } + /* Insert last literals (if any exist) in the block as a sequence with ml == off == 0. + * If there are no last literals, then we'll emit (of: 0, ml: 0, ll: 0), which is a marker + * for the block boundary, according to the API. + */ + assert(seqStoreLiteralsSize >= literalsRead); + lastLLSize = seqStoreLiteralsSize - literalsRead; + outSeqs[i].litLength = (U32)lastLLSize; + outSeqs[i].matchLength = outSeqs[i].offset = outSeqs[i].rep = 0; + seqStoreSeqSize++; + zc->seqCollector.seqIndex += seqStoreSeqSize; +} + +size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, + size_t outSeqsSize, const void* src, size_t srcSize) +{ + const size_t dstCapacity = ZSTD_compressBound(srcSize); + void* dst = ZSTD_customMalloc(dstCapacity, ZSTD_defaultCMem); + SeqCollector seqCollector; + + RETURN_ERROR_IF(dst == NULL, memory_allocation, "NULL pointer!"); + + seqCollector.collectSequences = 1; + seqCollector.seqStart = outSeqs; + seqCollector.seqIndex = 0; + seqCollector.maxSequences = outSeqsSize; + zc->seqCollector = seqCollector; + + ZSTD_compress2(zc, dst, dstCapacity, src, srcSize); + ZSTD_customFree(dst, ZSTD_defaultCMem); + return zc->seqCollector.seqIndex; +} + +size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize) { + size_t in = 0; + size_t out = 0; + for (; in < seqsSize; ++in) { + if (sequences[in].offset == 0 && sequences[in].matchLength == 0) { + if (in != seqsSize - 1) { + sequences[in+1].litLength += sequences[in].litLength; + } + } else { + sequences[out] = sequences[in]; + ++out; + } + } + return out; +} + +/* Unrolled loop to read four size_ts of input at a time. Returns 1 if is RLE, 0 if not. */ +static int ZSTD_isRLE(const BYTE* src, size_t length) { + const BYTE* ip = src; + const BYTE value = ip[0]; + const size_t valueST = (size_t)((U64)value * 0x0101010101010101ULL); + const size_t unrollSize = sizeof(size_t) * 4; + const size_t unrollMask = unrollSize - 1; + const size_t prefixLength = length & unrollMask; + size_t i; + size_t u; + if (length == 1) return 1; + /* Check if prefix is RLE first before using unrolled loop */ + if (prefixLength && ZSTD_count(ip+1, ip, ip+prefixLength) != prefixLength-1) { + return 0; + } + for (i = prefixLength; i != length; i += unrollSize) { + for (u = 0; u < unrollSize; u += sizeof(size_t)) { + if (MEM_readST(ip + i + u) != valueST) { + return 0; + } + } + } + return 1; +} + +/* Returns true if the given block may be RLE. + * This is just a heuristic based on the compressibility. + * It may return both false positives and false negatives. + */ +static int ZSTD_maybeRLE(seqStore_t const* seqStore) +{ + size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); + size_t const nbLits = (size_t)(seqStore->lit - seqStore->litStart); + + return nbSeqs < 4 && nbLits < 10; +} + +static void ZSTD_confirmRepcodesAndEntropyTables(ZSTD_CCtx* zc) +{ + ZSTD_compressedBlockState_t* const tmp = zc->blockState.prevCBlock; + zc->blockState.prevCBlock = zc->blockState.nextCBlock; + zc->blockState.nextCBlock = tmp; +} + +static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, U32 frame) +{ + /* This the upper bound for the length of an rle block. + * This isn't the actual upper bound. Finding the real threshold + * needs further investigation. + */ + const U32 rleMaxLength = 25; + size_t cSize; + const BYTE* ip = (const BYTE*)src; + BYTE* op = (BYTE*)dst; + DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", + (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, + (unsigned)zc->blockState.matchState.nextToUpdate); + + { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); + FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); + if (bss == ZSTDbss_noCompress) { cSize = 0; goto out; } + } + + if (zc->seqCollector.collectSequences) { + ZSTD_copyBlockSequences(zc); + ZSTD_confirmRepcodesAndEntropyTables(zc); + return 0; + } + + /* encode sequences and literals */ + cSize = ZSTD_entropyCompressSequences(&zc->seqStore, + &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, + &zc->appliedParams, + dst, dstCapacity, + srcSize, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, + zc->bmi2); + + if (zc->seqCollector.collectSequences) { + ZSTD_copyBlockSequences(zc); + return 0; + } + + + if (frame && + /* We don't want to emit our first block as a RLE even if it qualifies because + * doing so will cause the decoder (cli only) to throw a "should consume all input error." + * This is only an issue for zstd <= v1.4.3 + */ + !zc->isFirstBlock && + cSize < rleMaxLength && + ZSTD_isRLE(ip, srcSize)) + { + cSize = 1; + op[0] = ip[0]; + } + +out: + if (!ZSTD_isError(cSize) && cSize > 1) { + ZSTD_confirmRepcodesAndEntropyTables(zc); + } + /* We check that dictionaries have offset codes available for the first + * block. After the first block, the offcode table might not have large + * enough codes to represent the offsets in the data. + */ + if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + + return cSize; +} + +static size_t ZSTD_compressBlock_targetCBlockSize_body(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const size_t bss, U32 lastBlock) +{ + DEBUGLOG(6, "Attempting ZSTD_compressSuperBlock()"); + if (bss == ZSTDbss_compress) { + if (/* We don't want to emit our first block as a RLE even if it qualifies because + * doing so will cause the decoder (cli only) to throw a "should consume all input error." + * This is only an issue for zstd <= v1.4.3 + */ + !zc->isFirstBlock && + ZSTD_maybeRLE(&zc->seqStore) && + ZSTD_isRLE((BYTE const*)src, srcSize)) + { + return ZSTD_rleCompressBlock(dst, dstCapacity, *(BYTE const*)src, srcSize, lastBlock); + } + /* Attempt superblock compression. + * + * Note that compressed size of ZSTD_compressSuperBlock() is not bound by the + * standard ZSTD_compressBound(). This is a problem, because even if we have + * space now, taking an extra byte now could cause us to run out of space later + * and violate ZSTD_compressBound(). + * + * Define blockBound(blockSize) = blockSize + ZSTD_blockHeaderSize. + * + * In order to respect ZSTD_compressBound() we must attempt to emit a raw + * uncompressed block in these cases: + * * cSize == 0: Return code for an uncompressed block. + * * cSize == dstSize_tooSmall: We may have expanded beyond blockBound(srcSize). + * ZSTD_noCompressBlock() will return dstSize_tooSmall if we are really out of + * output space. + * * cSize >= blockBound(srcSize): We have expanded the block too much so + * emit an uncompressed block. + */ + { + size_t const cSize = ZSTD_compressSuperBlock(zc, dst, dstCapacity, src, srcSize, lastBlock); + if (cSize != ERROR(dstSize_tooSmall)) { + size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy); + FORWARD_IF_ERROR(cSize, "ZSTD_compressSuperBlock failed"); + if (cSize != 0 && cSize < maxCSize + ZSTD_blockHeaderSize) { + ZSTD_confirmRepcodesAndEntropyTables(zc); + return cSize; + } + } + } + } + + DEBUGLOG(6, "Resorting to ZSTD_noCompressBlock()"); + /* Superblock compression failed, attempt to emit a single no compress block. + * The decoder will be able to stream this block since it is uncompressed. + */ + return ZSTD_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock); +} + +static size_t ZSTD_compressBlock_targetCBlockSize(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 lastBlock) +{ + size_t cSize = 0; + const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); + DEBUGLOG(5, "ZSTD_compressBlock_targetCBlockSize (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u, srcSize=%zu)", + (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, (unsigned)zc->blockState.matchState.nextToUpdate, srcSize); + FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); + + cSize = ZSTD_compressBlock_targetCBlockSize_body(zc, dst, dstCapacity, src, srcSize, bss, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_targetCBlockSize_body failed"); + + if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + + return cSize; +} + +static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, + ZSTD_CCtx_params const* params, + void const* ip, + void const* iend) +{ + if (ZSTD_window_needOverflowCorrection(ms->window, iend)) { + U32 const maxDist = (U32)1 << params->cParams.windowLog; + U32 const cycleLog = ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy); + U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, maxDist, ip); + ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30); + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30); + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); + ZSTD_cwksp_mark_tables_dirty(ws); + ZSTD_reduceIndex(ms, params, correction); + ZSTD_cwksp_mark_tables_clean(ws); + if (ms->nextToUpdate < correction) ms->nextToUpdate = 0; + else ms->nextToUpdate -= correction; + /* invalidate dictionaries on overflow correction */ + ms->loadedDictEnd = 0; + ms->dictMatchState = NULL; + } +} + +/*! ZSTD_compress_frameChunk() : +* Compress a chunk of data into one or multiple blocks. +* All blocks will be terminated, all input will be consumed. +* Function will issue an error if there is not enough `dstCapacity` to hold the compressed content. +* Frame is supposed already started (header already produced) +* @return : compressed size, or an error code +*/ +static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 lastFrameChunk) +{ + size_t blockSize = cctx->blockSize; + size_t remaining = srcSize; + const BYTE* ip = (const BYTE*)src; + BYTE* const ostart = (BYTE*)dst; + BYTE* op = ostart; + U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog; + + assert(cctx->appliedParams.cParams.windowLog <= ZSTD_WINDOWLOG_MAX); + + DEBUGLOG(4, "ZSTD_compress_frameChunk (blockSize=%u)", (unsigned)blockSize); + if (cctx->appliedParams.fParams.checksumFlag && srcSize) + xxh64_update(&cctx->xxhState, src, srcSize); + + while (remaining) { + ZSTD_matchState_t* const ms = &cctx->blockState.matchState; + U32 const lastBlock = lastFrameChunk & (blockSize >= remaining); + + RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE, + dstSize_tooSmall, + "not enough space to store compressed block"); + if (remaining < blockSize) blockSize = remaining; + + ZSTD_overflowCorrectIfNeeded( + ms, &cctx->workspace, &cctx->appliedParams, ip, ip + blockSize); + ZSTD_checkDictValidity(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); + + /* Ensure hash/chain table insertion resumes no sooner than lowlimit */ + if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit; + + { size_t cSize; + if (ZSTD_useTargetCBlockSize(&cctx->appliedParams)) { + cSize = ZSTD_compressBlock_targetCBlockSize(cctx, op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_targetCBlockSize failed"); + assert(cSize > 0); + assert(cSize <= blockSize + ZSTD_blockHeaderSize); + } else { + cSize = ZSTD_compressBlock_internal(cctx, + op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, + ip, blockSize, 1 /* frame */); + FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_internal failed"); + + if (cSize == 0) { /* block is not compressible */ + cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); + } else { + U32 const cBlockHeader = cSize == 1 ? + lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) : + lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); + MEM_writeLE24(op, cBlockHeader); + cSize += ZSTD_blockHeaderSize; + } + } + + + ip += blockSize; + assert(remaining >= blockSize); + remaining -= blockSize; + op += cSize; + assert(dstCapacity >= cSize); + dstCapacity -= cSize; + cctx->isFirstBlock = 0; + DEBUGLOG(5, "ZSTD_compress_frameChunk: adding a block of size %u", + (unsigned)cSize); + } } + + if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending; + return (size_t)(op-ostart); +} + + +static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, + const ZSTD_CCtx_params* params, U64 pledgedSrcSize, U32 dictID) +{ BYTE* const op = (BYTE*)dst; + U32 const dictIDSizeCodeLength = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ + U32 const dictIDSizeCode = params->fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength; /* 0-3 */ + U32 const checksumFlag = params->fParams.checksumFlag>0; + U32 const windowSize = (U32)1 << params->cParams.windowLog; + U32 const singleSegment = params->fParams.contentSizeFlag && (windowSize >= pledgedSrcSize); + BYTE const windowLogByte = (BYTE)((params->cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); + U32 const fcsCode = params->fParams.contentSizeFlag ? + (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : 0; /* 0-3 */ + BYTE const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) ); + size_t pos=0; + + assert(!(params->fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN)); + RETURN_ERROR_IF(dstCapacity < ZSTD_FRAMEHEADERSIZE_MAX, dstSize_tooSmall, + "dst buf is too small to fit worst-case frame header size."); + DEBUGLOG(4, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u", + !params->fParams.noDictIDFlag, (unsigned)dictID, (unsigned)dictIDSizeCode); + if (params->format == ZSTD_f_zstd1) { + MEM_writeLE32(dst, ZSTD_MAGICNUMBER); + pos = 4; + } + op[pos++] = frameHeaderDescriptionByte; + if (!singleSegment) op[pos++] = windowLogByte; + switch(dictIDSizeCode) + { + default: assert(0); /* impossible */ + case 0 : break; + case 1 : op[pos] = (BYTE)(dictID); pos++; break; + case 2 : MEM_writeLE16(op+pos, (U16)dictID); pos+=2; break; + case 3 : MEM_writeLE32(op+pos, dictID); pos+=4; break; + } + switch(fcsCode) + { + default: assert(0); /* impossible */ + case 0 : if (singleSegment) op[pos++] = (BYTE)(pledgedSrcSize); break; + case 1 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break; + case 2 : MEM_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break; + case 3 : MEM_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break; + } + return pos; +} + +/* ZSTD_writeSkippableFrame_advanced() : + * Writes out a skippable frame with the specified magic number variant (16 are supported), + * from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15, and the desired source data. + * + * Returns the total number of bytes written, or a ZSTD error code. + */ +size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, unsigned magicVariant) { + BYTE* op = (BYTE*)dst; + RETURN_ERROR_IF(dstCapacity < srcSize + ZSTD_SKIPPABLEHEADERSIZE /* Skippable frame overhead */, + dstSize_tooSmall, "Not enough room for skippable frame"); + RETURN_ERROR_IF(srcSize > (unsigned)0xFFFFFFFF, srcSize_wrong, "Src size too large for skippable frame"); + RETURN_ERROR_IF(magicVariant > 15, parameter_outOfBound, "Skippable frame magic number variant not supported"); + + MEM_writeLE32(op, (U32)(ZSTD_MAGIC_SKIPPABLE_START + magicVariant)); + MEM_writeLE32(op+4, (U32)srcSize); + ZSTD_memcpy(op+8, src, srcSize); + return srcSize + ZSTD_SKIPPABLEHEADERSIZE; +} + +/* ZSTD_writeLastEmptyBlock() : + * output an empty Block with end-of-frame mark to complete a frame + * @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h)) + * or an error code if `dstCapacity` is too small (stage != ZSTDcs_init, stage_wrong, + "wrong cctx stage"); + RETURN_ERROR_IF(cctx->appliedParams.ldmParams.enableLdm, + parameter_unsupported, + "incompatible with ldm"); + cctx->externSeqStore.seq = seq; + cctx->externSeqStore.size = nbSeq; + cctx->externSeqStore.capacity = nbSeq; + cctx->externSeqStore.pos = 0; + cctx->externSeqStore.posInSequence = 0; + return 0; +} + + +static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 frame, U32 lastFrameChunk) +{ + ZSTD_matchState_t* const ms = &cctx->blockState.matchState; + size_t fhSize = 0; + + DEBUGLOG(5, "ZSTD_compressContinue_internal, stage: %u, srcSize: %u", + cctx->stage, (unsigned)srcSize); + RETURN_ERROR_IF(cctx->stage==ZSTDcs_created, stage_wrong, + "missing init (ZSTD_compressBegin)"); + + if (frame && (cctx->stage==ZSTDcs_init)) { + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, + cctx->pledgedSrcSizePlusOne-1, cctx->dictID); + FORWARD_IF_ERROR(fhSize, "ZSTD_writeFrameHeader failed"); + assert(fhSize <= dstCapacity); + dstCapacity -= fhSize; + dst = (char*)dst + fhSize; + cctx->stage = ZSTDcs_ongoing; + } + + if (!srcSize) return fhSize; /* do not generate an empty block if no input */ + + if (!ZSTD_window_update(&ms->window, src, srcSize)) { + ms->nextToUpdate = ms->window.dictLimit; + } + if (cctx->appliedParams.ldmParams.enableLdm) { + ZSTD_window_update(&cctx->ldmState.window, src, srcSize); + } + + if (!frame) { + /* overflow check and correction for block mode */ + ZSTD_overflowCorrectIfNeeded( + ms, &cctx->workspace, &cctx->appliedParams, + src, (BYTE const*)src + srcSize); + } + + DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (unsigned)cctx->blockSize); + { size_t const cSize = frame ? + ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) : + ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize, 0 /* frame */); + FORWARD_IF_ERROR(cSize, "%s", frame ? "ZSTD_compress_frameChunk failed" : "ZSTD_compressBlock_internal failed"); + cctx->consumedSrcSize += srcSize; + cctx->producedCSize += (cSize + fhSize); + assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0)); + if (cctx->pledgedSrcSizePlusOne != 0) { /* control src size */ + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1); + RETURN_ERROR_IF( + cctx->consumedSrcSize+1 > cctx->pledgedSrcSizePlusOne, + srcSize_wrong, + "error : pledgedSrcSize = %u, while realSrcSize >= %u", + (unsigned)cctx->pledgedSrcSizePlusOne-1, + (unsigned)cctx->consumedSrcSize); + } + return cSize + fhSize; + } +} + +size_t ZSTD_compressContinue (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_compressContinue (srcSize=%u)", (unsigned)srcSize); + return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 0 /* last chunk */); +} + + +size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx) +{ + ZSTD_compressionParameters const cParams = cctx->appliedParams.cParams; + assert(!ZSTD_checkCParams(cParams)); + return MIN (ZSTD_BLOCKSIZE_MAX, (U32)1 << cParams.windowLog); +} + +size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_compressBlock: srcSize = %u", (unsigned)srcSize); + { size_t const blockSizeMax = ZSTD_getBlockSize(cctx); + RETURN_ERROR_IF(srcSize > blockSizeMax, srcSize_wrong, "input is larger than a block"); } + + return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */); +} + +/*! ZSTD_loadDictionaryContent() : + * @return : 0, or an error code + */ +static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, + ldmState_t* ls, + ZSTD_cwksp* ws, + ZSTD_CCtx_params const* params, + const void* src, size_t srcSize, + ZSTD_dictTableLoadMethod_e dtlm) +{ + const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; + + ZSTD_window_update(&ms->window, src, srcSize); + ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base); + + if (params->ldmParams.enableLdm && ls != NULL) { + ZSTD_window_update(&ls->window, src, srcSize); + ls->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ls->window.base); + } + + /* Assert that we the ms params match the params we're being given */ + ZSTD_assertEqualCParams(params->cParams, ms->cParams); + + if (srcSize <= HASH_READ_SIZE) return 0; + + while (iend - ip > HASH_READ_SIZE) { + size_t const remaining = (size_t)(iend - ip); + size_t const chunk = MIN(remaining, ZSTD_CHUNKSIZE_MAX); + const BYTE* const ichunk = ip + chunk; + + ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, ichunk); + + if (params->ldmParams.enableLdm && ls != NULL) + ZSTD_ldm_fillHashTable(ls, (const BYTE*)src, (const BYTE*)src + srcSize, ¶ms->ldmParams); + + switch(params->cParams.strategy) + { + case ZSTD_fast: + ZSTD_fillHashTable(ms, ichunk, dtlm); + break; + case ZSTD_dfast: + ZSTD_fillDoubleHashTable(ms, ichunk, dtlm); + break; + + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + if (chunk >= HASH_READ_SIZE && ms->dedicatedDictSearch) { + assert(chunk == remaining); /* must load everything in one go */ + ZSTD_dedicatedDictSearch_lazy_loadDictionary(ms, ichunk-HASH_READ_SIZE); + } else if (chunk >= HASH_READ_SIZE) { + ZSTD_insertAndFindFirstIndex(ms, ichunk-HASH_READ_SIZE); + } + break; + + case ZSTD_btlazy2: /* we want the dictionary table fully sorted */ + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: + if (chunk >= HASH_READ_SIZE) + ZSTD_updateTree(ms, ichunk-HASH_READ_SIZE, ichunk); + break; + + default: + assert(0); /* not possible : not a valid strategy id */ + } + + ip = ichunk; + } + + ms->nextToUpdate = (U32)(iend - ms->window.base); + return 0; +} + + +/* Dictionaries that assign zero probability to symbols that show up causes problems + * when FSE encoding. Mark dictionaries with zero probability symbols as FSE_repeat_check + * and only dictionaries with 100% valid symbols can be assumed valid. + */ +static FSE_repeat ZSTD_dictNCountRepeat(short* normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue) +{ + U32 s; + if (dictMaxSymbolValue < maxSymbolValue) { + return FSE_repeat_check; + } + for (s = 0; s <= maxSymbolValue; ++s) { + if (normalizedCounter[s] == 0) { + return FSE_repeat_check; + } + } + return FSE_repeat_valid; +} + +size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, + const void* const dict, size_t dictSize) +{ + short offcodeNCount[MaxOff+1]; + unsigned offcodeMaxValue = MaxOff; + const BYTE* dictPtr = (const BYTE*)dict; /* skip magic num and dict ID */ + const BYTE* const dictEnd = dictPtr + dictSize; + dictPtr += 8; + bs->entropy.huf.repeatMode = HUF_repeat_check; + + { unsigned maxSymbolValue = 255; + unsigned hasZeroWeights = 1; + size_t const hufHeaderSize = HUF_readCTable((HUF_CElt*)bs->entropy.huf.CTable, &maxSymbolValue, dictPtr, + dictEnd-dictPtr, &hasZeroWeights); + + /* We only set the loaded table as valid if it contains all non-zero + * weights. Otherwise, we set it to check */ + if (!hasZeroWeights) + bs->entropy.huf.repeatMode = HUF_repeat_valid; + + RETURN_ERROR_IF(HUF_isError(hufHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(maxSymbolValue < 255, dictionary_corrupted, ""); + dictPtr += hufHeaderSize; + } + + { unsigned offcodeLog; + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); + RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, ""); + /* fill all offset symbols to avoid garbage at end of table */ + RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( + bs->entropy.fse.offcodeCTable, + offcodeNCount, MaxOff, offcodeLog, + workspace, HUF_WORKSPACE_SIZE)), + dictionary_corrupted, ""); + /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */ + dictPtr += offcodeHeaderSize; + } + + { short matchlengthNCount[MaxML+1]; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; + size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); + RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, ""); + RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( + bs->entropy.fse.matchlengthCTable, + matchlengthNCount, matchlengthMaxValue, matchlengthLog, + workspace, HUF_WORKSPACE_SIZE)), + dictionary_corrupted, ""); + bs->entropy.fse.matchlength_repeatMode = ZSTD_dictNCountRepeat(matchlengthNCount, matchlengthMaxValue, MaxML); + dictPtr += matchlengthHeaderSize; + } + + { short litlengthNCount[MaxLL+1]; + unsigned litlengthMaxValue = MaxLL, litlengthLog; + size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); + RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, ""); + RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( + bs->entropy.fse.litlengthCTable, + litlengthNCount, litlengthMaxValue, litlengthLog, + workspace, HUF_WORKSPACE_SIZE)), + dictionary_corrupted, ""); + bs->entropy.fse.litlength_repeatMode = ZSTD_dictNCountRepeat(litlengthNCount, litlengthMaxValue, MaxLL); + dictPtr += litlengthHeaderSize; + } + + RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted, ""); + bs->rep[0] = MEM_readLE32(dictPtr+0); + bs->rep[1] = MEM_readLE32(dictPtr+4); + bs->rep[2] = MEM_readLE32(dictPtr+8); + dictPtr += 12; + + { size_t const dictContentSize = (size_t)(dictEnd - dictPtr); + U32 offcodeMax = MaxOff; + if (dictContentSize <= ((U32)-1) - 128 KB) { + U32 const maxOffset = (U32)dictContentSize + 128 KB; /* The maximum offset that must be supported */ + offcodeMax = ZSTD_highbit32(maxOffset); /* Calculate minimum offset code required to represent maxOffset */ + } + /* All offset values <= dictContentSize + 128 KB must be representable for a valid table */ + bs->entropy.fse.offcode_repeatMode = ZSTD_dictNCountRepeat(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff)); + + /* All repCodes must be <= dictContentSize and != 0 */ + { U32 u; + for (u=0; u<3; u++) { + RETURN_ERROR_IF(bs->rep[u] == 0, dictionary_corrupted, ""); + RETURN_ERROR_IF(bs->rep[u] > dictContentSize, dictionary_corrupted, ""); + } } } + + return dictPtr - (const BYTE*)dict; +} + +/* Dictionary format : + * See : + * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#dictionary-format + */ +/*! ZSTD_loadZstdDictionary() : + * @return : dictID, or an error code + * assumptions : magic number supposed already checked + * dictSize supposed >= 8 + */ +static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, + ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, + ZSTD_CCtx_params const* params, + const void* dict, size_t dictSize, + ZSTD_dictTableLoadMethod_e dtlm, + void* workspace) +{ + const BYTE* dictPtr = (const BYTE*)dict; + const BYTE* const dictEnd = dictPtr + dictSize; + size_t dictID; + size_t eSize; + + ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<= 8); + assert(MEM_readLE32(dictPtr) == ZSTD_MAGIC_DICTIONARY); + + dictID = params->fParams.noDictIDFlag ? 0 : MEM_readLE32(dictPtr + 4 /* skip magic number */ ); + eSize = ZSTD_loadCEntropy(bs, workspace, dict, dictSize); + FORWARD_IF_ERROR(eSize, "ZSTD_loadCEntropy failed"); + dictPtr += eSize; + + { + size_t const dictContentSize = (size_t)(dictEnd - dictPtr); + FORWARD_IF_ERROR(ZSTD_loadDictionaryContent( + ms, NULL, ws, params, dictPtr, dictContentSize, dtlm), ""); + } + return dictID; +} + +/* ZSTD_compress_insertDictionary() : +* @return : dictID, or an error code */ +static size_t +ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, + ZSTD_matchState_t* ms, + ldmState_t* ls, + ZSTD_cwksp* ws, + const ZSTD_CCtx_params* params, + const void* dict, size_t dictSize, + ZSTD_dictContentType_e dictContentType, + ZSTD_dictTableLoadMethod_e dtlm, + void* workspace) +{ + DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize); + if ((dict==NULL) || (dictSize<8)) { + RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, ""); + return 0; + } + + ZSTD_reset_compressedBlockState(bs); + + /* dict restricted modes */ + if (dictContentType == ZSTD_dct_rawContent) + return ZSTD_loadDictionaryContent(ms, ls, ws, params, dict, dictSize, dtlm); + + if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) { + if (dictContentType == ZSTD_dct_auto) { + DEBUGLOG(4, "raw content dictionary detected"); + return ZSTD_loadDictionaryContent( + ms, ls, ws, params, dict, dictSize, dtlm); + } + RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, ""); + assert(0); /* impossible */ + } + + /* dict as full zstd dictionary */ + return ZSTD_loadZstdDictionary( + bs, ms, ws, params, dict, dictSize, dtlm, workspace); +} + +#define ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF (128 KB) +#define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6ULL) + +/*! ZSTD_compressBegin_internal() : + * @return : 0, or an error code */ +static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_dictContentType_e dictContentType, + ZSTD_dictTableLoadMethod_e dtlm, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ + DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params->cParams.windowLog); + /* params are supposed to be fully validated at this point */ + assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); + assert(!((dict) && (cdict))); /* either dict or cdict, not both */ + if ( (cdict) + && (cdict->dictContentSize > 0) + && ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF + || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER + || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN + || cdict->compressionLevel == 0) + && (params->attachDictPref != ZSTD_dictForceLoad) ) { + return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); + } + + FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, *params, pledgedSrcSize, + ZSTDcrp_makeClean, zbuff) , ""); + { size_t const dictID = cdict ? + ZSTD_compress_insertDictionary( + cctx->blockState.prevCBlock, &cctx->blockState.matchState, + &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, cdict->dictContent, + cdict->dictContentSize, cdict->dictContentType, dtlm, + cctx->entropyWorkspace) + : ZSTD_compress_insertDictionary( + cctx->blockState.prevCBlock, &cctx->blockState.matchState, + &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, dict, dictSize, + dictContentType, dtlm, cctx->entropyWorkspace); + FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); + assert(dictID <= UINT_MAX); + cctx->dictID = (U32)dictID; + cctx->dictContentSize = cdict ? cdict->dictContentSize : dictSize; + } + return 0; +} + +size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_dictContentType_e dictContentType, + ZSTD_dictTableLoadMethod_e dtlm, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal: wlog=%u", params->cParams.windowLog); + /* compression parameters verification and optimization */ + FORWARD_IF_ERROR( ZSTD_checkCParams(params->cParams) , ""); + return ZSTD_compressBegin_internal(cctx, + dict, dictSize, dictContentType, dtlm, + cdict, + params, pledgedSrcSize, + ZSTDb_not_buffered); +} + +/*! ZSTD_compressBegin_advanced() : +* @return : 0, or an error code */ +size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize) +{ + ZSTD_CCtx_params cctxParams; + ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, ZSTD_NO_CLEVEL); + return ZSTD_compressBegin_advanced_internal(cctx, + dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, + NULL /*cdict*/, + &cctxParams, pledgedSrcSize); +} + +size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_CCtx_params cctxParams; + { + ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict); + ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel); + } + DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize); + return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, + &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered); +} + +size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel) +{ + return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel); +} + + +/*! ZSTD_writeEpilogue() : +* Ends a frame. +* @return : nb of bytes written into dst (or an error code) */ +static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* op = ostart; + size_t fhSize = 0; + + DEBUGLOG(4, "ZSTD_writeEpilogue"); + RETURN_ERROR_IF(cctx->stage == ZSTDcs_created, stage_wrong, "init missing"); + + /* special case : empty frame */ + if (cctx->stage == ZSTDcs_init) { + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, 0, 0); + FORWARD_IF_ERROR(fhSize, "ZSTD_writeFrameHeader failed"); + dstCapacity -= fhSize; + op += fhSize; + cctx->stage = ZSTDcs_ongoing; + } + + if (cctx->stage != ZSTDcs_ending) { + /* write one last empty block, make it the "last" block */ + U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1) + 0; + RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for epilogue"); + MEM_writeLE32(op, cBlockHeader24); + op += ZSTD_blockHeaderSize; + dstCapacity -= ZSTD_blockHeaderSize; + } + + if (cctx->appliedParams.fParams.checksumFlag) { + U32 const checksum = (U32) xxh64_digest(&cctx->xxhState); + RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for checksum"); + DEBUGLOG(4, "ZSTD_writeEpilogue: write checksum : %08X", (unsigned)checksum); + MEM_writeLE32(op, checksum); + op += 4; + } + + cctx->stage = ZSTDcs_created; /* return to "created but no init" status */ + return op-ostart; +} + +void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize) +{ + (void)cctx; + (void)extraCSize; +} + +size_t ZSTD_compressEnd (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t endResult; + size_t const cSize = ZSTD_compressContinue_internal(cctx, + dst, dstCapacity, src, srcSize, + 1 /* frame mode */, 1 /* last chunk */); + FORWARD_IF_ERROR(cSize, "ZSTD_compressContinue_internal failed"); + endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize); + FORWARD_IF_ERROR(endResult, "ZSTD_writeEpilogue failed"); + assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0)); + if (cctx->pledgedSrcSizePlusOne != 0) { /* control src size */ + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1); + DEBUGLOG(4, "end of frame : controlling src size"); + RETURN_ERROR_IF( + cctx->pledgedSrcSizePlusOne != cctx->consumedSrcSize+1, + srcSize_wrong, + "error : pledgedSrcSize = %u, while realSrcSize = %u", + (unsigned)cctx->pledgedSrcSizePlusOne-1, + (unsigned)cctx->consumedSrcSize); + } + ZSTD_CCtx_trace(cctx, endResult); + return cSize + endResult; +} + +size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params) +{ + ZSTD_CCtx_params cctxParams; + DEBUGLOG(4, "ZSTD_compress_advanced"); + FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), ""); + ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, ZSTD_NO_CLEVEL); + return ZSTD_compress_advanced_internal(cctx, + dst, dstCapacity, + src, srcSize, + dict, dictSize, + &cctxParams); +} + +/* Internal */ +size_t ZSTD_compress_advanced_internal( + ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + const ZSTD_CCtx_params* params) +{ + DEBUGLOG(4, "ZSTD_compress_advanced_internal (srcSize:%u)", (unsigned)srcSize); + FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, + dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, + params, srcSize, ZSTDb_not_buffered) , ""); + return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); +} + +size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + int compressionLevel) +{ + ZSTD_CCtx_params cctxParams; + { + ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, srcSize, dict ? dictSize : 0, ZSTD_cpm_noAttachDict); + assert(params.fParams.contentSizeFlag == 1); + ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT: compressionLevel); + } + DEBUGLOG(4, "ZSTD_compress_usingDict (srcSize=%u)", (unsigned)srcSize); + return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctxParams); +} + +size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel) +{ + DEBUGLOG(4, "ZSTD_compressCCtx (srcSize=%u)", (unsigned)srcSize); + assert(cctx != NULL); + return ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, srcSize, NULL, 0, compressionLevel); +} + +size_t ZSTD_compress(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel) +{ + size_t result; + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + RETURN_ERROR_IF(!cctx, memory_allocation, "ZSTD_createCCtx failed"); + result = ZSTD_compressCCtx(cctx, dst, dstCapacity, src, srcSize, compressionLevel); + ZSTD_freeCCtx(cctx); + return result; +} + + +/* ===== Dictionary API ===== */ + +/*! ZSTD_estimateCDictSize_advanced() : + * Estimate amount of memory that will be needed to create a dictionary with following arguments */ +size_t ZSTD_estimateCDictSize_advanced( + size_t dictSize, ZSTD_compressionParameters cParams, + ZSTD_dictLoadMethod_e dictLoadMethod) +{ + DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict)); + return ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 + : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void *)))); +} + +size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel) +{ + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + return ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); +} + +size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) +{ + if (cdict==NULL) return 0; /* support sizeof on NULL */ + DEBUGLOG(5, "sizeof(*cdict) : %u", (unsigned)sizeof(*cdict)); + /* cdict may be in the workspace */ + return (cdict->workspace.workspace == cdict ? 0 : sizeof(*cdict)) + + ZSTD_cwksp_sizeof(&cdict->workspace); +} + +static size_t ZSTD_initCDict_internal( + ZSTD_CDict* cdict, + const void* dictBuffer, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_CCtx_params params) +{ + DEBUGLOG(3, "ZSTD_initCDict_internal (dictContentType:%u)", (unsigned)dictContentType); + assert(!ZSTD_checkCParams(params.cParams)); + cdict->matchState.cParams = params.cParams; + cdict->matchState.dedicatedDictSearch = params.enableDedicatedDictSearch; + if (cdict->matchState.dedicatedDictSearch && dictSize > ZSTD_CHUNKSIZE_MAX) { + cdict->matchState.dedicatedDictSearch = 0; + } + if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) { + cdict->dictContent = dictBuffer; + } else { + void *internalBuffer = ZSTD_cwksp_reserve_object(&cdict->workspace, ZSTD_cwksp_align(dictSize, sizeof(void*))); + RETURN_ERROR_IF(!internalBuffer, memory_allocation, "NULL pointer!"); + cdict->dictContent = internalBuffer; + ZSTD_memcpy(internalBuffer, dictBuffer, dictSize); + } + cdict->dictContentSize = dictSize; + cdict->dictContentType = dictContentType; + + cdict->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cdict->workspace, HUF_WORKSPACE_SIZE); + + + /* Reset the state to no dictionary */ + ZSTD_reset_compressedBlockState(&cdict->cBlockState); + FORWARD_IF_ERROR(ZSTD_reset_matchState( + &cdict->matchState, + &cdict->workspace, + ¶ms.cParams, + ZSTDcrp_makeClean, + ZSTDirp_reset, + ZSTD_resetTarget_CDict), ""); + /* (Maybe) load the dictionary + * Skips loading the dictionary if it is < 8 bytes. + */ + { params.compressionLevel = ZSTD_CLEVEL_DEFAULT; + params.fParams.contentSizeFlag = 1; + { size_t const dictID = ZSTD_compress_insertDictionary( + &cdict->cBlockState, &cdict->matchState, NULL, &cdict->workspace, + ¶ms, cdict->dictContent, cdict->dictContentSize, + dictContentType, ZSTD_dtlm_full, cdict->entropyWorkspace); + FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); + assert(dictID <= (size_t)(U32)-1); + cdict->dictID = (U32)dictID; + } + } + + return 0; +} + +static ZSTD_CDict* ZSTD_createCDict_advanced_internal(size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_compressionParameters cParams, ZSTD_customMem customMem) +{ + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + + { size_t const workspaceSize = + ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 + : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))); + void* const workspace = ZSTD_customMalloc(workspaceSize, customMem); + ZSTD_cwksp ws; + ZSTD_CDict* cdict; + + if (!workspace) { + ZSTD_customFree(workspace, customMem); + return NULL; + } + + ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_dynamic_alloc); + + cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); + assert(cdict != NULL); + ZSTD_cwksp_move(&cdict->workspace, &ws); + cdict->customMem = customMem; + cdict->compressionLevel = ZSTD_NO_CLEVEL; /* signals advanced API usage */ + + return cdict; + } +} + +ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams, + ZSTD_customMem customMem) +{ + ZSTD_CCtx_params cctxParams; + ZSTD_memset(&cctxParams, 0, sizeof(cctxParams)); + ZSTD_CCtxParams_init(&cctxParams, 0); + cctxParams.cParams = cParams; + cctxParams.customMem = customMem; + return ZSTD_createCDict_advanced2( + dictBuffer, dictSize, + dictLoadMethod, dictContentType, + &cctxParams, customMem); +} + +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + const ZSTD_CCtx_params* originalCctxParams, + ZSTD_customMem customMem) +{ + ZSTD_CCtx_params cctxParams = *originalCctxParams; + ZSTD_compressionParameters cParams; + ZSTD_CDict* cdict; + + DEBUGLOG(3, "ZSTD_createCDict_advanced2, mode %u", (unsigned)dictContentType); + if (!customMem.customAlloc ^ !customMem.customFree) return NULL; + + if (cctxParams.enableDedicatedDictSearch) { + cParams = ZSTD_dedicatedDictSearch_getCParams( + cctxParams.compressionLevel, dictSize); + ZSTD_overrideCParams(&cParams, &cctxParams.cParams); + } else { + cParams = ZSTD_getCParamsFromCCtxParams( + &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + } + + if (!ZSTD_dedicatedDictSearch_isSupported(&cParams)) { + /* Fall back to non-DDSS params */ + cctxParams.enableDedicatedDictSearch = 0; + cParams = ZSTD_getCParamsFromCCtxParams( + &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + } + + cctxParams.cParams = cParams; + + cdict = ZSTD_createCDict_advanced_internal(dictSize, + dictLoadMethod, cctxParams.cParams, + customMem); + + if (ZSTD_isError( ZSTD_initCDict_internal(cdict, + dict, dictSize, + dictLoadMethod, dictContentType, + cctxParams) )) { + ZSTD_freeCDict(cdict); + return NULL; + } + + return cdict; +} + +ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_dlm_byCopy, ZSTD_dct_auto, + cParams, ZSTD_defaultCMem); + if (cdict) + cdict->compressionLevel = (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel; + return cdict; +} + +ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_dlm_byRef, ZSTD_dct_auto, + cParams, ZSTD_defaultCMem); + if (cdict) + cdict->compressionLevel = (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel; + return cdict; +} + +size_t ZSTD_freeCDict(ZSTD_CDict* cdict) +{ + if (cdict==NULL) return 0; /* support free on NULL */ + { ZSTD_customMem const cMem = cdict->customMem; + int cdictInWorkspace = ZSTD_cwksp_owns_buffer(&cdict->workspace, cdict); + ZSTD_cwksp_free(&cdict->workspace, cMem); + if (!cdictInWorkspace) { + ZSTD_customFree(cdict, cMem); + } + return 0; + } +} + +/*! ZSTD_initStaticCDict_advanced() : + * Generate a digested dictionary in provided memory area. + * workspace: The memory area to emplace the dictionary into. + * Provided pointer must 8-bytes aligned. + * It must outlive dictionary usage. + * workspaceSize: Use ZSTD_estimateCDictSize() + * to determine how large workspace must be. + * cParams : use ZSTD_getCParams() to transform a compression level + * into its relevants cParams. + * @return : pointer to ZSTD_CDict*, or NULL if error (size too small) + * Note : there is no corresponding "free" function. + * Since workspace was allocated externally, it must be freed externally. + */ +const ZSTD_CDict* ZSTD_initStaticCDict( + void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams) +{ + size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0); + size_t const neededSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 + : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))) + + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + + matchStateSize; + ZSTD_CDict* cdict; + ZSTD_CCtx_params params; + + if ((size_t)workspace & 7) return NULL; /* 8-aligned */ + + { + ZSTD_cwksp ws; + ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_static_alloc); + cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); + if (cdict == NULL) return NULL; + ZSTD_cwksp_move(&cdict->workspace, &ws); + } + + DEBUGLOG(4, "(workspaceSize < neededSize) : (%u < %u) => %u", + (unsigned)workspaceSize, (unsigned)neededSize, (unsigned)(workspaceSize < neededSize)); + if (workspaceSize < neededSize) return NULL; + + ZSTD_CCtxParams_init(¶ms, 0); + params.cParams = cParams; + + if (ZSTD_isError( ZSTD_initCDict_internal(cdict, + dict, dictSize, + dictLoadMethod, dictContentType, + params) )) + return NULL; + + return cdict; +} + +ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict) +{ + assert(cdict != NULL); + return cdict->matchState.cParams; +} + +/*! ZSTD_getDictID_fromCDict() : + * Provides the dictID of the dictionary loaded into `cdict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict) +{ + if (cdict==NULL) return 0; + return cdict->dictID; +} + + +/* ZSTD_compressBegin_usingCDict_advanced() : + * cdict must be != NULL */ +size_t ZSTD_compressBegin_usingCDict_advanced( + ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, + ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) +{ + ZSTD_CCtx_params cctxParams; + DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced"); + RETURN_ERROR_IF(cdict==NULL, dictionary_wrong, "NULL pointer!"); + /* Initialize the cctxParams from the cdict */ + { + ZSTD_parameters params; + params.fParams = fParams; + params.cParams = ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF + || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER + || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN + || cdict->compressionLevel == 0 ) ? + ZSTD_getCParamsFromCDict(cdict) + : ZSTD_getCParams(cdict->compressionLevel, + pledgedSrcSize, + cdict->dictContentSize); + ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, cdict->compressionLevel); + } + /* Increase window log to fit the entire dictionary and source if the + * source size is known. Limit the increase to 19, which is the + * window log for compression level 1 with the largest source size. + */ + if (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN) { + U32 const limitedSrcSize = (U32)MIN(pledgedSrcSize, 1U << 19); + U32 const limitedSrcLog = limitedSrcSize > 1 ? ZSTD_highbit32(limitedSrcSize - 1) + 1 : 1; + cctxParams.cParams.windowLog = MAX(cctxParams.cParams.windowLog, limitedSrcLog); + } + return ZSTD_compressBegin_internal(cctx, + NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, + cdict, + &cctxParams, pledgedSrcSize, + ZSTDb_not_buffered); +} + +/* ZSTD_compressBegin_usingCDict() : + * pledgedSrcSize=0 means "unknown" + * if pledgedSrcSize>0, it will enable contentSizeFlag */ +size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) +{ + ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; + DEBUGLOG(4, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag); + return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN); +} + +size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, ZSTD_frameParameters fParams) +{ + FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, srcSize), ""); /* will check if cdict != NULL */ + return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); +} + +/*! ZSTD_compress_usingCDict() : + * Compression using a digested Dictionary. + * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. + * Note that compression parameters are decided at CDict creation time + * while frame parameters are hardcoded */ +size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict) +{ + ZSTD_frameParameters const fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; + return ZSTD_compress_usingCDict_advanced(cctx, dst, dstCapacity, src, srcSize, cdict, fParams); +} + + + +/* ****************************************************************** +* Streaming +********************************************************************/ + +ZSTD_CStream* ZSTD_createCStream(void) +{ + DEBUGLOG(3, "ZSTD_createCStream"); + return ZSTD_createCStream_advanced(ZSTD_defaultCMem); +} + +ZSTD_CStream* ZSTD_initStaticCStream(void *workspace, size_t workspaceSize) +{ + return ZSTD_initStaticCCtx(workspace, workspaceSize); +} + +ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem) +{ /* CStream and CCtx are now same object */ + return ZSTD_createCCtx_advanced(customMem); +} + +size_t ZSTD_freeCStream(ZSTD_CStream* zcs) +{ + return ZSTD_freeCCtx(zcs); /* same object */ +} + + + +/*====== Initialization ======*/ + +size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX; } + +size_t ZSTD_CStreamOutSize(void) +{ + return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; +} + +static ZSTD_cParamMode_e ZSTD_getCParamMode(ZSTD_CDict const* cdict, ZSTD_CCtx_params const* params, U64 pledgedSrcSize) +{ + if (cdict != NULL && ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) + return ZSTD_cpm_attachDict; + else + return ZSTD_cpm_noAttachDict; +} + +/* ZSTD_resetCStream(): + * pledgedSrcSize == 0 means "unknown" */ +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pss) +{ + /* temporary : 0 interpreted as "unknown" during transition period. + * Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. + * 0 will be interpreted as "empty" in the future. + */ + U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; + DEBUGLOG(4, "ZSTD_resetCStream: pledgedSrcSize = %u", (unsigned)pledgedSrcSize); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); + return 0; +} + +/*! ZSTD_initCStream_internal() : + * Note : for lib/compress only. Used by zstdmt_compress.c. + * Assumption 1 : params are valid + * Assumption 2 : either dict, or cdict, is defined, not both */ +size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_initCStream_internal"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); + assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); + zcs->requestedParams = *params; + assert(!((dict) && (cdict))); /* either dict or cdict, not both */ + if (dict) { + FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , ""); + } else { + /* Dictionary is cleared if !cdict */ + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , ""); + } + return 0; +} + +/* ZSTD_initCStream_usingCDict_advanced() : + * same as ZSTD_initCStream_usingCDict(), with control over frame parameters */ +size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams, + unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_initCStream_usingCDict_advanced"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); + zcs->requestedParams.fParams = fParams; + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , ""); + return 0; +} + +/* note : cdict must outlive compression session */ +size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) +{ + DEBUGLOG(4, "ZSTD_initCStream_usingCDict"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , ""); + return 0; +} + + +/* ZSTD_initCStream_advanced() : + * pledgedSrcSize must be exact. + * if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. + * dict is loaded with default parameters ZSTD_dct_auto and ZSTD_dlm_byCopy. */ +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pss) +{ + /* for compatibility with older programs relying on this behavior. + * Users should now specify ZSTD_CONTENTSIZE_UNKNOWN. + * This line will be removed in the future. + */ + U64 const pledgedSrcSize = (pss==0 && params.fParams.contentSizeFlag==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; + DEBUGLOG(4, "ZSTD_initCStream_advanced"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); + FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) , ""); + ZSTD_CCtxParams_setZstdParams(&zcs->requestedParams, ¶ms); + FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , ""); + return 0; +} + +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) +{ + DEBUGLOG(4, "ZSTD_initCStream_usingDict"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , ""); + return 0; +} + +size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pss) +{ + /* temporary : 0 interpreted as "unknown" during transition period. + * Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. + * 0 will be interpreted as "empty" in the future. + */ + U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; + DEBUGLOG(4, "ZSTD_initCStream_srcSize"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); + return 0; +} + +size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) +{ + DEBUGLOG(4, "ZSTD_initCStream"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , ""); + return 0; +} + +/*====== Compression ======*/ + +static size_t ZSTD_nextInputSizeHint(const ZSTD_CCtx* cctx) +{ + size_t hintInSize = cctx->inBuffTarget - cctx->inBuffPos; + if (hintInSize==0) hintInSize = cctx->blockSize; + return hintInSize; +} + +/* ZSTD_compressStream_generic(): + * internal function for all *compressStream*() variants + * non-static, because can be called from zstdmt_compress.c + * @return : hint size for next input */ +static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective const flushMode) +{ + const char* const istart = (const char*)input->src; + const char* const iend = input->size != 0 ? istart + input->size : istart; + const char* ip = input->pos != 0 ? istart + input->pos : istart; + char* const ostart = (char*)output->dst; + char* const oend = output->size != 0 ? ostart + output->size : ostart; + char* op = output->pos != 0 ? ostart + output->pos : ostart; + U32 someMoreWork = 1; + + /* check expectations */ + DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%u", (unsigned)flushMode); + if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) { + assert(zcs->inBuff != NULL); + assert(zcs->inBuffSize > 0); + } + if (zcs->appliedParams.outBufferMode == ZSTD_bm_buffered) { + assert(zcs->outBuff != NULL); + assert(zcs->outBuffSize > 0); + } + assert(output->pos <= output->size); + assert(input->pos <= input->size); + assert((U32)flushMode <= (U32)ZSTD_e_end); + + while (someMoreWork) { + switch(zcs->streamStage) + { + case zcss_init: + RETURN_ERROR(init_missing, "call ZSTD_initCStream() first!"); + + case zcss_load: + if ( (flushMode == ZSTD_e_end) + && ( (size_t)(oend-op) >= ZSTD_compressBound(iend-ip) /* Enough output space */ + || zcs->appliedParams.outBufferMode == ZSTD_bm_stable) /* OR we are allowed to return dstSizeTooSmall */ + && (zcs->inBuffPos == 0) ) { + /* shortcut to compression pass directly into output buffer */ + size_t const cSize = ZSTD_compressEnd(zcs, + op, oend-op, ip, iend-ip); + DEBUGLOG(4, "ZSTD_compressEnd : cSize=%u", (unsigned)cSize); + FORWARD_IF_ERROR(cSize, "ZSTD_compressEnd failed"); + ip = iend; + op += cSize; + zcs->frameEnded = 1; + ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + someMoreWork = 0; break; + } + /* complete loading into inBuffer in buffered mode */ + if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) { + size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos; + size_t const loaded = ZSTD_limitCopy( + zcs->inBuff + zcs->inBuffPos, toLoad, + ip, iend-ip); + zcs->inBuffPos += loaded; + if (loaded != 0) + ip += loaded; + if ( (flushMode == ZSTD_e_continue) + && (zcs->inBuffPos < zcs->inBuffTarget) ) { + /* not enough input to fill full block : stop here */ + someMoreWork = 0; break; + } + if ( (flushMode == ZSTD_e_flush) + && (zcs->inBuffPos == zcs->inToCompress) ) { + /* empty */ + someMoreWork = 0; break; + } + } + /* compress current block (note : this stage cannot be stopped in the middle) */ + DEBUGLOG(5, "stream compression stage (flushMode==%u)", flushMode); + { int const inputBuffered = (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered); + void* cDst; + size_t cSize; + size_t oSize = oend-op; + size_t const iSize = inputBuffered + ? zcs->inBuffPos - zcs->inToCompress + : MIN((size_t)(iend - ip), zcs->blockSize); + if (oSize >= ZSTD_compressBound(iSize) || zcs->appliedParams.outBufferMode == ZSTD_bm_stable) + cDst = op; /* compress into output buffer, to skip flush stage */ + else + cDst = zcs->outBuff, oSize = zcs->outBuffSize; + if (inputBuffered) { + unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip==iend); + cSize = lastBlock ? + ZSTD_compressEnd(zcs, cDst, oSize, + zcs->inBuff + zcs->inToCompress, iSize) : + ZSTD_compressContinue(zcs, cDst, oSize, + zcs->inBuff + zcs->inToCompress, iSize); + FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed"); + zcs->frameEnded = lastBlock; + /* prepare next block */ + zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize; + if (zcs->inBuffTarget > zcs->inBuffSize) + zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; + DEBUGLOG(5, "inBuffTarget:%u / inBuffSize:%u", + (unsigned)zcs->inBuffTarget, (unsigned)zcs->inBuffSize); + if (!lastBlock) + assert(zcs->inBuffTarget <= zcs->inBuffSize); + zcs->inToCompress = zcs->inBuffPos; + } else { + unsigned const lastBlock = (ip + iSize == iend); + assert(flushMode == ZSTD_e_end /* Already validated */); + cSize = lastBlock ? + ZSTD_compressEnd(zcs, cDst, oSize, ip, iSize) : + ZSTD_compressContinue(zcs, cDst, oSize, ip, iSize); + /* Consume the input prior to error checking to mirror buffered mode. */ + if (iSize > 0) + ip += iSize; + FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed"); + zcs->frameEnded = lastBlock; + if (lastBlock) + assert(ip == iend); + } + if (cDst == op) { /* no need to flush */ + op += cSize; + if (zcs->frameEnded) { + DEBUGLOG(5, "Frame completed directly in outBuffer"); + someMoreWork = 0; + ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + } + break; + } + zcs->outBuffContentSize = cSize; + zcs->outBuffFlushedSize = 0; + zcs->streamStage = zcss_flush; /* pass-through to flush stage */ + } + /* fall-through */ + case zcss_flush: + DEBUGLOG(5, "flush stage"); + assert(zcs->appliedParams.outBufferMode == ZSTD_bm_buffered); + { size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; + size_t const flushed = ZSTD_limitCopy(op, (size_t)(oend-op), + zcs->outBuff + zcs->outBuffFlushedSize, toFlush); + DEBUGLOG(5, "toFlush: %u into %u ==> flushed: %u", + (unsigned)toFlush, (unsigned)(oend-op), (unsigned)flushed); + if (flushed) + op += flushed; + zcs->outBuffFlushedSize += flushed; + if (toFlush!=flushed) { + /* flush not fully completed, presumably because dst is too small */ + assert(op==oend); + someMoreWork = 0; + break; + } + zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; + if (zcs->frameEnded) { + DEBUGLOG(5, "Frame completed on flush"); + someMoreWork = 0; + ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + break; + } + zcs->streamStage = zcss_load; + break; + } + + default: /* impossible */ + assert(0); + } + } + + input->pos = ip - istart; + output->pos = op - ostart; + if (zcs->frameEnded) return 0; + return ZSTD_nextInputSizeHint(zcs); +} + +static size_t ZSTD_nextInputSizeHint_MTorST(const ZSTD_CCtx* cctx) +{ + return ZSTD_nextInputSizeHint(cctx); + +} + +size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + FORWARD_IF_ERROR( ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue) , ""); + return ZSTD_nextInputSizeHint_MTorST(zcs); +} + +/* After a compression call set the expected input/output buffer. + * This is validated at the start of the next compression call. + */ +static void ZSTD_setBufferExpectations(ZSTD_CCtx* cctx, ZSTD_outBuffer const* output, ZSTD_inBuffer const* input) +{ + if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { + cctx->expectedInBuffer = *input; + } + if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) { + cctx->expectedOutBufferSize = output->size - output->pos; + } +} + +/* Validate that the input/output buffers match the expectations set by + * ZSTD_setBufferExpectations. + */ +static size_t ZSTD_checkBufferStability(ZSTD_CCtx const* cctx, + ZSTD_outBuffer const* output, + ZSTD_inBuffer const* input, + ZSTD_EndDirective endOp) +{ + if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { + ZSTD_inBuffer const expect = cctx->expectedInBuffer; + if (expect.src != input->src || expect.pos != input->pos || expect.size != input->size) + RETURN_ERROR(srcBuffer_wrong, "ZSTD_c_stableInBuffer enabled but input differs!"); + if (endOp != ZSTD_e_end) + RETURN_ERROR(srcBuffer_wrong, "ZSTD_c_stableInBuffer can only be used with ZSTD_e_end!"); + } + if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) { + size_t const outBufferSize = output->size - output->pos; + if (cctx->expectedOutBufferSize != outBufferSize) + RETURN_ERROR(dstBuffer_wrong, "ZSTD_c_stableOutBuffer enabled but output size differs!"); + } + return 0; +} + +static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx, + ZSTD_EndDirective endOp, + size_t inSize) { + ZSTD_CCtx_params params = cctx->requestedParams; + ZSTD_prefixDict const prefixDict = cctx->prefixDict; + FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) , ""); /* Init the local dict if present. */ + ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */ + assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */ + if (cctx->cdict) + params.compressionLevel = cctx->cdict->compressionLevel; /* let cdict take priority in terms of compression level */ + DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage"); + if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = inSize + 1; /* auto-fix pledgedSrcSize */ + { + size_t const dictSize = prefixDict.dict + ? prefixDict.dictSize + : (cctx->cdict ? cctx->cdict->dictContentSize : 0); + ZSTD_cParamMode_e const mode = ZSTD_getCParamMode(cctx->cdict, ¶ms, cctx->pledgedSrcSizePlusOne - 1); + params.cParams = ZSTD_getCParamsFromCCtxParams( + ¶ms, cctx->pledgedSrcSizePlusOne-1, + dictSize, mode); + } + + if (ZSTD_CParams_shouldEnableLdm(¶ms.cParams)) { + /* Enable LDM by default for optimal parser and window size >= 128MB */ + DEBUGLOG(4, "LDM enabled by default (window size >= 128MB, strategy >= btopt)"); + params.ldmParams.enableLdm = 1; + } + + { U64 const pledgedSrcSize = cctx->pledgedSrcSizePlusOne - 1; + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, + prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType, ZSTD_dtlm_fast, + cctx->cdict, + ¶ms, pledgedSrcSize, + ZSTDb_buffered) , ""); + assert(cctx->appliedParams.nbWorkers == 0); + cctx->inToCompress = 0; + cctx->inBuffPos = 0; + if (cctx->appliedParams.inBufferMode == ZSTD_bm_buffered) { + /* for small input: avoid automatic flush on reaching end of block, since + * it would require to add a 3-bytes null block to end frame + */ + cctx->inBuffTarget = cctx->blockSize + (cctx->blockSize == pledgedSrcSize); + } else { + cctx->inBuffTarget = 0; + } + cctx->outBuffContentSize = cctx->outBuffFlushedSize = 0; + cctx->streamStage = zcss_load; + cctx->frameEnded = 0; + } + return 0; +} + +size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp) +{ + DEBUGLOG(5, "ZSTD_compressStream2, endOp=%u ", (unsigned)endOp); + /* check conditions */ + RETURN_ERROR_IF(output->pos > output->size, dstSize_tooSmall, "invalid output buffer"); + RETURN_ERROR_IF(input->pos > input->size, srcSize_wrong, "invalid input buffer"); + RETURN_ERROR_IF((U32)endOp > (U32)ZSTD_e_end, parameter_outOfBound, "invalid endDirective"); + assert(cctx != NULL); + + /* transparent initialization stage */ + if (cctx->streamStage == zcss_init) { + FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, endOp, input->size), "CompressStream2 initialization failed"); + ZSTD_setBufferExpectations(cctx, output, input); /* Set initial buffer expectations now that we've initialized */ + } + /* end of transparent initialization stage */ + + FORWARD_IF_ERROR(ZSTD_checkBufferStability(cctx, output, input, endOp), "invalid buffers"); + /* compression stage */ + FORWARD_IF_ERROR( ZSTD_compressStream_generic(cctx, output, input, endOp) , ""); + DEBUGLOG(5, "completed ZSTD_compressStream2"); + ZSTD_setBufferExpectations(cctx, output, input); + return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */ +} + +size_t ZSTD_compressStream2_simpleArgs ( + ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos, + ZSTD_EndDirective endOp) +{ + ZSTD_outBuffer output = { dst, dstCapacity, *dstPos }; + ZSTD_inBuffer input = { src, srcSize, *srcPos }; + /* ZSTD_compressStream2() will check validity of dstPos and srcPos */ + size_t const cErr = ZSTD_compressStream2(cctx, &output, &input, endOp); + *dstPos = output.pos; + *srcPos = input.pos; + return cErr; +} + +size_t ZSTD_compress2(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + ZSTD_bufferMode_e const originalInBufferMode = cctx->requestedParams.inBufferMode; + ZSTD_bufferMode_e const originalOutBufferMode = cctx->requestedParams.outBufferMode; + DEBUGLOG(4, "ZSTD_compress2 (srcSize=%u)", (unsigned)srcSize); + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); + /* Enable stable input/output buffers. */ + cctx->requestedParams.inBufferMode = ZSTD_bm_stable; + cctx->requestedParams.outBufferMode = ZSTD_bm_stable; + { size_t oPos = 0; + size_t iPos = 0; + size_t const result = ZSTD_compressStream2_simpleArgs(cctx, + dst, dstCapacity, &oPos, + src, srcSize, &iPos, + ZSTD_e_end); + /* Reset to the original values. */ + cctx->requestedParams.inBufferMode = originalInBufferMode; + cctx->requestedParams.outBufferMode = originalOutBufferMode; + FORWARD_IF_ERROR(result, "ZSTD_compressStream2_simpleArgs failed"); + if (result != 0) { /* compression not completed, due to lack of output space */ + assert(oPos == dstCapacity); + RETURN_ERROR(dstSize_tooSmall, ""); + } + assert(iPos == srcSize); /* all input is expected consumed */ + return oPos; + } +} + +typedef struct { + U32 idx; /* Index in array of ZSTD_Sequence */ + U32 posInSequence; /* Position within sequence at idx */ + size_t posInSrc; /* Number of bytes given by sequences provided so far */ +} ZSTD_sequencePosition; + +/* Returns a ZSTD error code if sequence is not valid */ +static size_t ZSTD_validateSequence(U32 offCode, U32 matchLength, + size_t posInSrc, U32 windowLog, size_t dictSize, U32 minMatch) { + size_t offsetBound; + U32 windowSize = 1 << windowLog; + /* posInSrc represents the amount of data the the decoder would decode up to this point. + * As long as the amount of data decoded is less than or equal to window size, offsets may be + * larger than the total length of output decoded in order to reference the dict, even larger than + * window size. After output surpasses windowSize, we're limited to windowSize offsets again. + */ + offsetBound = posInSrc > windowSize ? (size_t)windowSize : posInSrc + (size_t)dictSize; + RETURN_ERROR_IF(offCode > offsetBound + ZSTD_REP_MOVE, corruption_detected, "Offset too large!"); + RETURN_ERROR_IF(matchLength < minMatch, corruption_detected, "Matchlength too small"); + return 0; +} + +/* Returns an offset code, given a sequence's raw offset, the ongoing repcode array, and whether litLength == 0 */ +static U32 ZSTD_finalizeOffCode(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0) { + U32 offCode = rawOffset + ZSTD_REP_MOVE; + U32 repCode = 0; + + if (!ll0 && rawOffset == rep[0]) { + repCode = 1; + } else if (rawOffset == rep[1]) { + repCode = 2 - ll0; + } else if (rawOffset == rep[2]) { + repCode = 3 - ll0; + } else if (ll0 && rawOffset == rep[0] - 1) { + repCode = 3; + } + if (repCode) { + /* ZSTD_storeSeq expects a number in the range [0, 2] to represent a repcode */ + offCode = repCode - 1; + } + return offCode; +} + +/* Returns 0 on success, and a ZSTD_error otherwise. This function scans through an array of + * ZSTD_Sequence, storing the sequences it finds, until it reaches a block delimiter. + */ +static size_t ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize) { + U32 idx = seqPos->idx; + BYTE const* ip = (BYTE const*)(src); + const BYTE* const iend = ip + blockSize; + repcodes_t updatedRepcodes; + U32 dictSize; + U32 litLength; + U32 matchLength; + U32 ll0; + U32 offCode; + + if (cctx->cdict) { + dictSize = (U32)cctx->cdict->dictContentSize; + } else if (cctx->prefixDict.dict) { + dictSize = (U32)cctx->prefixDict.dictSize; + } else { + dictSize = 0; + } + ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t)); + for (; (inSeqs[idx].matchLength != 0 || inSeqs[idx].offset != 0) && idx < inSeqsSize; ++idx) { + litLength = inSeqs[idx].litLength; + matchLength = inSeqs[idx].matchLength; + ll0 = litLength == 0; + offCode = ZSTD_finalizeOffCode(inSeqs[idx].offset, updatedRepcodes.rep, ll0); + updatedRepcodes = ZSTD_updateRep(updatedRepcodes.rep, offCode, ll0); + + DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offCode, matchLength, litLength); + if (cctx->appliedParams.validateSequences) { + seqPos->posInSrc += litLength + matchLength; + FORWARD_IF_ERROR(ZSTD_validateSequence(offCode, matchLength, seqPos->posInSrc, + cctx->appliedParams.cParams.windowLog, dictSize, + cctx->appliedParams.cParams.minMatch), + "Sequence validation failed"); + } + RETURN_ERROR_IF(idx - seqPos->idx > cctx->seqStore.maxNbSeq, memory_allocation, + "Not enough memory allocated. Try adjusting ZSTD_c_minMatch."); + ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offCode, matchLength - MINMATCH); + ip += matchLength + litLength; + } + ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t)); + + if (inSeqs[idx].litLength) { + DEBUGLOG(6, "Storing last literals of size: %u", inSeqs[idx].litLength); + ZSTD_storeLastLiterals(&cctx->seqStore, ip, inSeqs[idx].litLength); + ip += inSeqs[idx].litLength; + seqPos->posInSrc += inSeqs[idx].litLength; + } + RETURN_ERROR_IF(ip != iend, corruption_detected, "Blocksize doesn't agree with block delimiter!"); + seqPos->idx = idx+1; + return 0; +} + +/* Returns the number of bytes to move the current read position back by. Only non-zero + * if we ended up splitting a sequence. Otherwise, it may return a ZSTD error if something + * went wrong. + * + * This function will attempt to scan through blockSize bytes represented by the sequences + * in inSeqs, storing any (partial) sequences. + * + * Occasionally, we may want to change the actual number of bytes we consumed from inSeqs to + * avoid splitting a match, or to avoid splitting a match such that it would produce a match + * smaller than MINMATCH. In this case, we return the number of bytes that we didn't read from this block. + */ +static size_t ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize) { + U32 idx = seqPos->idx; + U32 startPosInSequence = seqPos->posInSequence; + U32 endPosInSequence = seqPos->posInSequence + (U32)blockSize; + size_t dictSize; + BYTE const* ip = (BYTE const*)(src); + BYTE const* iend = ip + blockSize; /* May be adjusted if we decide to process fewer than blockSize bytes */ + repcodes_t updatedRepcodes; + U32 bytesAdjustment = 0; + U32 finalMatchSplit = 0; + U32 litLength; + U32 matchLength; + U32 rawOffset; + U32 offCode; + + if (cctx->cdict) { + dictSize = cctx->cdict->dictContentSize; + } else if (cctx->prefixDict.dict) { + dictSize = cctx->prefixDict.dictSize; + } else { + dictSize = 0; + } + DEBUGLOG(5, "ZSTD_copySequencesToSeqStore: idx: %u PIS: %u blockSize: %zu", idx, startPosInSequence, blockSize); + DEBUGLOG(5, "Start seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); + ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t)); + while (endPosInSequence && idx < inSeqsSize && !finalMatchSplit) { + const ZSTD_Sequence currSeq = inSeqs[idx]; + litLength = currSeq.litLength; + matchLength = currSeq.matchLength; + rawOffset = currSeq.offset; + + /* Modify the sequence depending on where endPosInSequence lies */ + if (endPosInSequence >= currSeq.litLength + currSeq.matchLength) { + if (startPosInSequence >= litLength) { + startPosInSequence -= litLength; + litLength = 0; + matchLength -= startPosInSequence; + } else { + litLength -= startPosInSequence; + } + /* Move to the next sequence */ + endPosInSequence -= currSeq.litLength + currSeq.matchLength; + startPosInSequence = 0; + idx++; + } else { + /* This is the final (partial) sequence we're adding from inSeqs, and endPosInSequence + does not reach the end of the match. So, we have to split the sequence */ + DEBUGLOG(6, "Require a split: diff: %u, idx: %u PIS: %u", + currSeq.litLength + currSeq.matchLength - endPosInSequence, idx, endPosInSequence); + if (endPosInSequence > litLength) { + U32 firstHalfMatchLength; + litLength = startPosInSequence >= litLength ? 0 : litLength - startPosInSequence; + firstHalfMatchLength = endPosInSequence - startPosInSequence - litLength; + if (matchLength > blockSize && firstHalfMatchLength >= cctx->appliedParams.cParams.minMatch) { + /* Only ever split the match if it is larger than the block size */ + U32 secondHalfMatchLength = currSeq.matchLength + currSeq.litLength - endPosInSequence; + if (secondHalfMatchLength < cctx->appliedParams.cParams.minMatch) { + /* Move the endPosInSequence backward so that it creates match of minMatch length */ + endPosInSequence -= cctx->appliedParams.cParams.minMatch - secondHalfMatchLength; + bytesAdjustment = cctx->appliedParams.cParams.minMatch - secondHalfMatchLength; + firstHalfMatchLength -= bytesAdjustment; + } + matchLength = firstHalfMatchLength; + /* Flag that we split the last match - after storing the sequence, exit the loop, + but keep the value of endPosInSequence */ + finalMatchSplit = 1; + } else { + /* Move the position in sequence backwards so that we don't split match, and break to store + * the last literals. We use the original currSeq.litLength as a marker for where endPosInSequence + * should go. We prefer to do this whenever it is not necessary to split the match, or if doing so + * would cause the first half of the match to be too small + */ + bytesAdjustment = endPosInSequence - currSeq.litLength; + endPosInSequence = currSeq.litLength; + break; + } + } else { + /* This sequence ends inside the literals, break to store the last literals */ + break; + } + } + /* Check if this offset can be represented with a repcode */ + { U32 ll0 = (litLength == 0); + offCode = ZSTD_finalizeOffCode(rawOffset, updatedRepcodes.rep, ll0); + updatedRepcodes = ZSTD_updateRep(updatedRepcodes.rep, offCode, ll0); + } + + if (cctx->appliedParams.validateSequences) { + seqPos->posInSrc += litLength + matchLength; + FORWARD_IF_ERROR(ZSTD_validateSequence(offCode, matchLength, seqPos->posInSrc, + cctx->appliedParams.cParams.windowLog, dictSize, + cctx->appliedParams.cParams.minMatch), + "Sequence validation failed"); + } + DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offCode, matchLength, litLength); + RETURN_ERROR_IF(idx - seqPos->idx > cctx->seqStore.maxNbSeq, memory_allocation, + "Not enough memory allocated. Try adjusting ZSTD_c_minMatch."); + ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offCode, matchLength - MINMATCH); + ip += matchLength + litLength; + } + DEBUGLOG(5, "Ending seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); + assert(idx == inSeqsSize || endPosInSequence <= inSeqs[idx].litLength + inSeqs[idx].matchLength); + seqPos->idx = idx; + seqPos->posInSequence = endPosInSequence; + ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t)); + + iend -= bytesAdjustment; + if (ip != iend) { + /* Store any last literals */ + U32 lastLLSize = (U32)(iend - ip); + assert(ip <= iend); + DEBUGLOG(6, "Storing last literals of size: %u", lastLLSize); + ZSTD_storeLastLiterals(&cctx->seqStore, ip, lastLLSize); + seqPos->posInSrc += lastLLSize; + } + + return bytesAdjustment; +} + +typedef size_t (*ZSTD_sequenceCopier) (ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize); +static ZSTD_sequenceCopier ZSTD_selectSequenceCopier(ZSTD_sequenceFormat_e mode) { + ZSTD_sequenceCopier sequenceCopier = NULL; + assert(ZSTD_cParam_withinBounds(ZSTD_c_blockDelimiters, mode)); + if (mode == ZSTD_sf_explicitBlockDelimiters) { + return ZSTD_copySequencesToSeqStoreExplicitBlockDelim; + } else if (mode == ZSTD_sf_noBlockDelimiters) { + return ZSTD_copySequencesToSeqStoreNoBlockDelim; + } + assert(sequenceCopier != NULL); + return sequenceCopier; +} + +/* Compress, block-by-block, all of the sequences given. + * + * Returns the cumulative size of all compressed blocks (including their headers), otherwise a ZSTD error. + */ +static size_t ZSTD_compressSequences_internal(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize) { + size_t cSize = 0; + U32 lastBlock; + size_t blockSize; + size_t compressedSeqsSize; + size_t remaining = srcSize; + ZSTD_sequencePosition seqPos = {0, 0, 0}; + + BYTE const* ip = (BYTE const*)src; + BYTE* op = (BYTE*)dst; + ZSTD_sequenceCopier sequenceCopier = ZSTD_selectSequenceCopier(cctx->appliedParams.blockDelimiters); + + DEBUGLOG(4, "ZSTD_compressSequences_internal srcSize: %zu, inSeqsSize: %zu", srcSize, inSeqsSize); + /* Special case: empty frame */ + if (remaining == 0) { + U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1); + RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "No room for empty frame block header"); + MEM_writeLE32(op, cBlockHeader24); + op += ZSTD_blockHeaderSize; + dstCapacity -= ZSTD_blockHeaderSize; + cSize += ZSTD_blockHeaderSize; + } + + while (remaining) { + size_t cBlockSize; + size_t additionalByteAdjustment; + lastBlock = remaining <= cctx->blockSize; + blockSize = lastBlock ? (U32)remaining : (U32)cctx->blockSize; + ZSTD_resetSeqStore(&cctx->seqStore); + DEBUGLOG(4, "Working on new block. Blocksize: %zu", blockSize); + + additionalByteAdjustment = sequenceCopier(cctx, &seqPos, inSeqs, inSeqsSize, ip, blockSize); + FORWARD_IF_ERROR(additionalByteAdjustment, "Bad sequence copy"); + blockSize -= additionalByteAdjustment; + + /* If blocks are too small, emit as a nocompress block */ + if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) { + cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed"); + DEBUGLOG(4, "Block too small, writing out nocompress block: cSize: %zu", cBlockSize); + cSize += cBlockSize; + ip += blockSize; + op += cBlockSize; + remaining -= blockSize; + dstCapacity -= cBlockSize; + continue; + } + + compressedSeqsSize = ZSTD_entropyCompressSequences(&cctx->seqStore, + &cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy, + &cctx->appliedParams, + op + ZSTD_blockHeaderSize /* Leave space for block header */, dstCapacity - ZSTD_blockHeaderSize, + blockSize, + cctx->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, + cctx->bmi2); + FORWARD_IF_ERROR(compressedSeqsSize, "Compressing sequences of block failed"); + DEBUGLOG(4, "Compressed sequences size: %zu", compressedSeqsSize); + + if (!cctx->isFirstBlock && + ZSTD_maybeRLE(&cctx->seqStore) && + ZSTD_isRLE((BYTE const*)src, srcSize)) { + /* We don't want to emit our first block as a RLE even if it qualifies because + * doing so will cause the decoder (cli only) to throw a "should consume all input error." + * This is only an issue for zstd <= v1.4.3 + */ + compressedSeqsSize = 1; + } + + if (compressedSeqsSize == 0) { + /* ZSTD_noCompressBlock writes the block header as well */ + cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed"); + DEBUGLOG(4, "Writing out nocompress block, size: %zu", cBlockSize); + } else if (compressedSeqsSize == 1) { + cBlockSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cBlockSize, "RLE compress block failed"); + DEBUGLOG(4, "Writing out RLE block, size: %zu", cBlockSize); + } else { + U32 cBlockHeader; + /* Error checking and repcodes update */ + ZSTD_confirmRepcodesAndEntropyTables(cctx); + if (cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + + /* Write block header into beginning of block*/ + cBlockHeader = lastBlock + (((U32)bt_compressed)<<1) + (U32)(compressedSeqsSize << 3); + MEM_writeLE24(op, cBlockHeader); + cBlockSize = ZSTD_blockHeaderSize + compressedSeqsSize; + DEBUGLOG(4, "Writing out compressed block, size: %zu", cBlockSize); + } + + cSize += cBlockSize; + DEBUGLOG(4, "cSize running total: %zu", cSize); + + if (lastBlock) { + break; + } else { + ip += blockSize; + op += cBlockSize; + remaining -= blockSize; + dstCapacity -= cBlockSize; + cctx->isFirstBlock = 0; + } + } + + return cSize; +} + +size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize) { + BYTE* op = (BYTE*)dst; + size_t cSize = 0; + size_t compressedBlocksSize = 0; + size_t frameHeaderSize = 0; + + /* Transparent initialization stage, same as compressStream2() */ + DEBUGLOG(3, "ZSTD_compressSequences()"); + assert(cctx != NULL); + FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_end, srcSize), "CCtx initialization failed"); + /* Begin writing output, starting with frame header */ + frameHeaderSize = ZSTD_writeFrameHeader(op, dstCapacity, &cctx->appliedParams, srcSize, cctx->dictID); + op += frameHeaderSize; + dstCapacity -= frameHeaderSize; + cSize += frameHeaderSize; + if (cctx->appliedParams.fParams.checksumFlag && srcSize) { + xxh64_update(&cctx->xxhState, src, srcSize); + } + /* cSize includes block header size and compressed sequences size */ + compressedBlocksSize = ZSTD_compressSequences_internal(cctx, + op, dstCapacity, + inSeqs, inSeqsSize, + src, srcSize); + FORWARD_IF_ERROR(compressedBlocksSize, "Compressing blocks failed!"); + cSize += compressedBlocksSize; + dstCapacity -= compressedBlocksSize; + + if (cctx->appliedParams.fParams.checksumFlag) { + U32 const checksum = (U32) xxh64_digest(&cctx->xxhState); + RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for checksum"); + DEBUGLOG(4, "Write checksum : %08X", (unsigned)checksum); + MEM_writeLE32((char*)dst + cSize, checksum); + cSize += 4; + } + + DEBUGLOG(3, "Final compressed size: %zu", cSize); + return cSize; +} + +/*====== Finalize ======*/ + +/*! ZSTD_flushStream() : + * @return : amount of data remaining to flush */ +size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) +{ + ZSTD_inBuffer input = { NULL, 0, 0 }; + return ZSTD_compressStream2(zcs, output, &input, ZSTD_e_flush); +} + + +size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) +{ + ZSTD_inBuffer input = { NULL, 0, 0 }; + size_t const remainingToFlush = ZSTD_compressStream2(zcs, output, &input, ZSTD_e_end); + FORWARD_IF_ERROR( remainingToFlush , "ZSTD_compressStream2 failed"); + if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush; /* minimal estimation */ + /* single thread mode : attempt to calculate remaining to flush more precisely */ + { size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE; + size_t const checksumSize = (size_t)(zcs->frameEnded ? 0 : zcs->appliedParams.fParams.checksumFlag * 4); + size_t const toFlush = remainingToFlush + lastBlockSize + checksumSize; + DEBUGLOG(4, "ZSTD_endStream : remaining to flush : %u", (unsigned)toFlush); + return toFlush; + } +} + + +/*-===== Pre-defined compression levels =====-*/ + +#define ZSTD_MAX_CLEVEL 22 +int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } +int ZSTD_minCLevel(void) { return (int)-ZSTD_TARGETLENGTH_MAX; } + +static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = { +{ /* "default" - for any srcSize > 256 KB */ + /* W, C, H, S, L, TL, strat */ + { 19, 12, 13, 1, 6, 1, ZSTD_fast }, /* base for negative levels */ + { 19, 13, 14, 1, 7, 0, ZSTD_fast }, /* level 1 */ + { 20, 15, 16, 1, 6, 0, ZSTD_fast }, /* level 2 */ + { 21, 16, 17, 1, 5, 0, ZSTD_dfast }, /* level 3 */ + { 21, 18, 18, 1, 5, 0, ZSTD_dfast }, /* level 4 */ + { 21, 18, 19, 2, 5, 2, ZSTD_greedy }, /* level 5 */ + { 21, 19, 19, 3, 5, 4, ZSTD_greedy }, /* level 6 */ + { 21, 19, 19, 3, 5, 8, ZSTD_lazy }, /* level 7 */ + { 21, 19, 19, 3, 5, 16, ZSTD_lazy2 }, /* level 8 */ + { 21, 19, 20, 4, 5, 16, ZSTD_lazy2 }, /* level 9 */ + { 22, 20, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */ + { 22, 21, 22, 4, 5, 16, ZSTD_lazy2 }, /* level 11 */ + { 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 12 */ + { 22, 21, 22, 5, 5, 32, ZSTD_btlazy2 }, /* level 13 */ + { 22, 22, 23, 5, 5, 32, ZSTD_btlazy2 }, /* level 14 */ + { 22, 23, 23, 6, 5, 32, ZSTD_btlazy2 }, /* level 15 */ + { 22, 22, 22, 5, 5, 48, ZSTD_btopt }, /* level 16 */ + { 23, 23, 22, 5, 4, 64, ZSTD_btopt }, /* level 17 */ + { 23, 23, 22, 6, 3, 64, ZSTD_btultra }, /* level 18 */ + { 23, 24, 22, 7, 3,256, ZSTD_btultra2}, /* level 19 */ + { 25, 25, 23, 7, 3,256, ZSTD_btultra2}, /* level 20 */ + { 26, 26, 24, 7, 3,512, ZSTD_btultra2}, /* level 21 */ + { 27, 27, 25, 9, 3,999, ZSTD_btultra2}, /* level 22 */ +}, +{ /* for srcSize <= 256 KB */ + /* W, C, H, S, L, T, strat */ + { 18, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ + { 18, 13, 14, 1, 6, 0, ZSTD_fast }, /* level 1 */ + { 18, 14, 14, 1, 5, 0, ZSTD_dfast }, /* level 2 */ + { 18, 16, 16, 1, 4, 0, ZSTD_dfast }, /* level 3 */ + { 18, 16, 17, 2, 5, 2, ZSTD_greedy }, /* level 4.*/ + { 18, 18, 18, 3, 5, 2, ZSTD_greedy }, /* level 5.*/ + { 18, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6.*/ + { 18, 18, 19, 4, 4, 4, ZSTD_lazy }, /* level 7 */ + { 18, 18, 19, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ + { 18, 18, 19, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ + { 18, 18, 19, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ + { 18, 18, 19, 5, 4, 12, ZSTD_btlazy2 }, /* level 11.*/ + { 18, 19, 19, 7, 4, 12, ZSTD_btlazy2 }, /* level 12.*/ + { 18, 18, 19, 4, 4, 16, ZSTD_btopt }, /* level 13 */ + { 18, 18, 19, 4, 3, 32, ZSTD_btopt }, /* level 14.*/ + { 18, 18, 19, 6, 3,128, ZSTD_btopt }, /* level 15.*/ + { 18, 19, 19, 6, 3,128, ZSTD_btultra }, /* level 16.*/ + { 18, 19, 19, 8, 3,256, ZSTD_btultra }, /* level 17.*/ + { 18, 19, 19, 6, 3,128, ZSTD_btultra2}, /* level 18.*/ + { 18, 19, 19, 8, 3,256, ZSTD_btultra2}, /* level 19.*/ + { 18, 19, 19, 10, 3,512, ZSTD_btultra2}, /* level 20.*/ + { 18, 19, 19, 12, 3,512, ZSTD_btultra2}, /* level 21.*/ + { 18, 19, 19, 13, 3,999, ZSTD_btultra2}, /* level 22.*/ +}, +{ /* for srcSize <= 128 KB */ + /* W, C, H, S, L, T, strat */ + { 17, 12, 12, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ + { 17, 12, 13, 1, 6, 0, ZSTD_fast }, /* level 1 */ + { 17, 13, 15, 1, 5, 0, ZSTD_fast }, /* level 2 */ + { 17, 15, 16, 2, 5, 0, ZSTD_dfast }, /* level 3 */ + { 17, 17, 17, 2, 4, 0, ZSTD_dfast }, /* level 4 */ + { 17, 16, 17, 3, 4, 2, ZSTD_greedy }, /* level 5 */ + { 17, 17, 17, 3, 4, 4, ZSTD_lazy }, /* level 6 */ + { 17, 17, 17, 3, 4, 8, ZSTD_lazy2 }, /* level 7 */ + { 17, 17, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ + { 17, 17, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ + { 17, 17, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ + { 17, 17, 17, 5, 4, 8, ZSTD_btlazy2 }, /* level 11 */ + { 17, 18, 17, 7, 4, 12, ZSTD_btlazy2 }, /* level 12 */ + { 17, 18, 17, 3, 4, 12, ZSTD_btopt }, /* level 13.*/ + { 17, 18, 17, 4, 3, 32, ZSTD_btopt }, /* level 14.*/ + { 17, 18, 17, 6, 3,256, ZSTD_btopt }, /* level 15.*/ + { 17, 18, 17, 6, 3,128, ZSTD_btultra }, /* level 16.*/ + { 17, 18, 17, 8, 3,256, ZSTD_btultra }, /* level 17.*/ + { 17, 18, 17, 10, 3,512, ZSTD_btultra }, /* level 18.*/ + { 17, 18, 17, 5, 3,256, ZSTD_btultra2}, /* level 19.*/ + { 17, 18, 17, 7, 3,512, ZSTD_btultra2}, /* level 20.*/ + { 17, 18, 17, 9, 3,512, ZSTD_btultra2}, /* level 21.*/ + { 17, 18, 17, 11, 3,999, ZSTD_btultra2}, /* level 22.*/ +}, +{ /* for srcSize <= 16 KB */ + /* W, C, H, S, L, T, strat */ + { 14, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ + { 14, 14, 15, 1, 5, 0, ZSTD_fast }, /* level 1 */ + { 14, 14, 15, 1, 4, 0, ZSTD_fast }, /* level 2 */ + { 14, 14, 15, 2, 4, 0, ZSTD_dfast }, /* level 3 */ + { 14, 14, 14, 4, 4, 2, ZSTD_greedy }, /* level 4 */ + { 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 5.*/ + { 14, 14, 14, 4, 4, 8, ZSTD_lazy2 }, /* level 6 */ + { 14, 14, 14, 6, 4, 8, ZSTD_lazy2 }, /* level 7 */ + { 14, 14, 14, 8, 4, 8, ZSTD_lazy2 }, /* level 8.*/ + { 14, 15, 14, 5, 4, 8, ZSTD_btlazy2 }, /* level 9.*/ + { 14, 15, 14, 9, 4, 8, ZSTD_btlazy2 }, /* level 10.*/ + { 14, 15, 14, 3, 4, 12, ZSTD_btopt }, /* level 11.*/ + { 14, 15, 14, 4, 3, 24, ZSTD_btopt }, /* level 12.*/ + { 14, 15, 14, 5, 3, 32, ZSTD_btultra }, /* level 13.*/ + { 14, 15, 15, 6, 3, 64, ZSTD_btultra }, /* level 14.*/ + { 14, 15, 15, 7, 3,256, ZSTD_btultra }, /* level 15.*/ + { 14, 15, 15, 5, 3, 48, ZSTD_btultra2}, /* level 16.*/ + { 14, 15, 15, 6, 3,128, ZSTD_btultra2}, /* level 17.*/ + { 14, 15, 15, 7, 3,256, ZSTD_btultra2}, /* level 18.*/ + { 14, 15, 15, 8, 3,256, ZSTD_btultra2}, /* level 19.*/ + { 14, 15, 15, 8, 3,512, ZSTD_btultra2}, /* level 20.*/ + { 14, 15, 15, 9, 3,512, ZSTD_btultra2}, /* level 21.*/ + { 14, 15, 15, 10, 3,999, ZSTD_btultra2}, /* level 22.*/ +}, +}; + +static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams(int const compressionLevel, size_t const dictSize) +{ + ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, 0, dictSize, ZSTD_cpm_createCDict); + switch (cParams.strategy) { + case ZSTD_fast: + case ZSTD_dfast: + break; + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + cParams.hashLog += ZSTD_LAZY_DDSS_BUCKET_LOG; + break; + case ZSTD_btlazy2: + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: + break; + } + return cParams; +} + +static int ZSTD_dedicatedDictSearch_isSupported( + ZSTD_compressionParameters const* cParams) +{ + return (cParams->strategy >= ZSTD_greedy) + && (cParams->strategy <= ZSTD_lazy2) + && (cParams->hashLog >= cParams->chainLog) + && (cParams->chainLog <= 24); +} + +/* + * Reverses the adjustment applied to cparams when enabling dedicated dict + * search. This is used to recover the params set to be used in the working + * context. (Otherwise, those tables would also grow.) + */ +static void ZSTD_dedicatedDictSearch_revertCParams( + ZSTD_compressionParameters* cParams) { + switch (cParams->strategy) { + case ZSTD_fast: + case ZSTD_dfast: + break; + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + cParams->hashLog -= ZSTD_LAZY_DDSS_BUCKET_LOG; + break; + case ZSTD_btlazy2: + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: + break; + } +} + +static U64 ZSTD_getCParamRowSize(U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) +{ + switch (mode) { + case ZSTD_cpm_unknown: + case ZSTD_cpm_noAttachDict: + case ZSTD_cpm_createCDict: + break; + case ZSTD_cpm_attachDict: + dictSize = 0; + break; + default: + assert(0); + break; + } + { int const unknown = srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN; + size_t const addedSize = unknown && dictSize > 0 ? 500 : 0; + return unknown && dictSize == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : srcSizeHint+dictSize+addedSize; + } +} + +/*! ZSTD_getCParams_internal() : + * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize. + * Note: srcSizeHint 0 means 0, use ZSTD_CONTENTSIZE_UNKNOWN for unknown. + * Use dictSize == 0 for unknown or unused. + * Note: `mode` controls how we treat the `dictSize`. See docs for `ZSTD_cParamMode_e`. */ +static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) +{ + U64 const rSize = ZSTD_getCParamRowSize(srcSizeHint, dictSize, mode); + U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); + int row; + DEBUGLOG(5, "ZSTD_getCParams_internal (cLevel=%i)", compressionLevel); + + /* row */ + if (compressionLevel == 0) row = ZSTD_CLEVEL_DEFAULT; /* 0 == default */ + else if (compressionLevel < 0) row = 0; /* entry 0 is baseline for fast mode */ + else if (compressionLevel > ZSTD_MAX_CLEVEL) row = ZSTD_MAX_CLEVEL; + else row = compressionLevel; + + { ZSTD_compressionParameters cp = ZSTD_defaultCParameters[tableID][row]; + /* acceleration factor */ + if (compressionLevel < 0) { + int const clampedCompressionLevel = MAX(ZSTD_minCLevel(), compressionLevel); + cp.targetLength = (unsigned)(-clampedCompressionLevel); + } + /* refine parameters based on srcSize & dictSize */ + return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize, mode); + } +} + +/*! ZSTD_getCParams() : + * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize. + * Size values are optional, provide 0 if not known or unused */ +ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) +{ + if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN; + return ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown); +} + +/*! ZSTD_getParams() : + * same idea as ZSTD_getCParams() + * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`). + * Fields of `ZSTD_frameParameters` are set to default values */ +static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) { + ZSTD_parameters params; + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize, mode); + DEBUGLOG(5, "ZSTD_getParams (cLevel=%i)", compressionLevel); + ZSTD_memset(¶ms, 0, sizeof(params)); + params.cParams = cParams; + params.fParams.contentSizeFlag = 1; + return params; +} + +/*! ZSTD_getParams() : + * same idea as ZSTD_getCParams() + * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`). + * Fields of `ZSTD_frameParameters` are set to default values */ +ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) { + if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN; + return ZSTD_getParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown); +} diff --git a/lib/zstd/compress/zstd_compress_internal.h b/lib/zstd/compress/zstd_compress_internal.h new file mode 100644 index 000000000000..0a3b81663eb5 --- /dev/null +++ b/lib/zstd/compress/zstd_compress_internal.h @@ -0,0 +1,1188 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* This header contains definitions + * that shall **only** be used by modules within lib/compress. + */ + +#ifndef ZSTD_COMPRESS_H +#define ZSTD_COMPRESS_H + +/*-************************************* +* Dependencies +***************************************/ +#include "../common/zstd_internal.h" +#include "zstd_cwksp.h" + + +/*-************************************* +* Constants +***************************************/ +#define kSearchStrength 8 +#define HASH_READ_SIZE 8 +#define ZSTD_DUBT_UNSORTED_MARK 1 /* For btlazy2 strategy, index ZSTD_DUBT_UNSORTED_MARK==1 means "unsorted". + It could be confused for a real successor at index "1", if sorted as larger than its predecessor. + It's not a big deal though : candidate will just be sorted again. + Additionally, candidate position 1 will be lost. + But candidate 1 cannot hide a large tree of candidates, so it's a minimal loss. + The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be mishandled after table re-use with a different strategy. + This constant is required by ZSTD_compressBlock_btlazy2() and ZSTD_reduceTable_internal() */ + + +/*-************************************* +* Context memory management +***************************************/ +typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e; +typedef enum { zcss_init=0, zcss_load, zcss_flush } ZSTD_cStreamStage; + +typedef struct ZSTD_prefixDict_s { + const void* dict; + size_t dictSize; + ZSTD_dictContentType_e dictContentType; +} ZSTD_prefixDict; + +typedef struct { + void* dictBuffer; + void const* dict; + size_t dictSize; + ZSTD_dictContentType_e dictContentType; + ZSTD_CDict* cdict; +} ZSTD_localDict; + +typedef struct { + HUF_CElt CTable[HUF_CTABLE_SIZE_U32(255)]; + HUF_repeat repeatMode; +} ZSTD_hufCTables_t; + +typedef struct { + FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)]; + FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)]; + FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)]; + FSE_repeat offcode_repeatMode; + FSE_repeat matchlength_repeatMode; + FSE_repeat litlength_repeatMode; +} ZSTD_fseCTables_t; + +typedef struct { + ZSTD_hufCTables_t huf; + ZSTD_fseCTables_t fse; +} ZSTD_entropyCTables_t; + +typedef struct { + U32 off; /* Offset code (offset + ZSTD_REP_MOVE) for the match */ + U32 len; /* Raw length of match */ +} ZSTD_match_t; + +typedef struct { + U32 offset; /* Offset of sequence */ + U32 litLength; /* Length of literals prior to match */ + U32 matchLength; /* Raw length of match */ +} rawSeq; + +typedef struct { + rawSeq* seq; /* The start of the sequences */ + size_t pos; /* The index in seq where reading stopped. pos <= size. */ + size_t posInSequence; /* The position within the sequence at seq[pos] where reading + stopped. posInSequence <= seq[pos].litLength + seq[pos].matchLength */ + size_t size; /* The number of sequences. <= capacity. */ + size_t capacity; /* The capacity starting from `seq` pointer */ +} rawSeqStore_t; + +UNUSED_ATTR static const rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0}; + +typedef struct { + int price; + U32 off; + U32 mlen; + U32 litlen; + U32 rep[ZSTD_REP_NUM]; +} ZSTD_optimal_t; + +typedef enum { zop_dynamic=0, zop_predef } ZSTD_OptPrice_e; + +typedef struct { + /* All tables are allocated inside cctx->workspace by ZSTD_resetCCtx_internal() */ + unsigned* litFreq; /* table of literals statistics, of size 256 */ + unsigned* litLengthFreq; /* table of litLength statistics, of size (MaxLL+1) */ + unsigned* matchLengthFreq; /* table of matchLength statistics, of size (MaxML+1) */ + unsigned* offCodeFreq; /* table of offCode statistics, of size (MaxOff+1) */ + ZSTD_match_t* matchTable; /* list of found matches, of size ZSTD_OPT_NUM+1 */ + ZSTD_optimal_t* priceTable; /* All positions tracked by optimal parser, of size ZSTD_OPT_NUM+1 */ + + U32 litSum; /* nb of literals */ + U32 litLengthSum; /* nb of litLength codes */ + U32 matchLengthSum; /* nb of matchLength codes */ + U32 offCodeSum; /* nb of offset codes */ + U32 litSumBasePrice; /* to compare to log2(litfreq) */ + U32 litLengthSumBasePrice; /* to compare to log2(llfreq) */ + U32 matchLengthSumBasePrice;/* to compare to log2(mlfreq) */ + U32 offCodeSumBasePrice; /* to compare to log2(offreq) */ + ZSTD_OptPrice_e priceType; /* prices can be determined dynamically, or follow a pre-defined cost structure */ + const ZSTD_entropyCTables_t* symbolCosts; /* pre-calculated dictionary statistics */ + ZSTD_literalCompressionMode_e literalCompressionMode; +} optState_t; + +typedef struct { + ZSTD_entropyCTables_t entropy; + U32 rep[ZSTD_REP_NUM]; +} ZSTD_compressedBlockState_t; + +typedef struct { + BYTE const* nextSrc; /* next block here to continue on current prefix */ + BYTE const* base; /* All regular indexes relative to this position */ + BYTE const* dictBase; /* extDict indexes relative to this position */ + U32 dictLimit; /* below that point, need extDict */ + U32 lowLimit; /* below that point, no more valid data */ +} ZSTD_window_t; + +typedef struct ZSTD_matchState_t ZSTD_matchState_t; +struct ZSTD_matchState_t { + ZSTD_window_t window; /* State for window round buffer management */ + U32 loadedDictEnd; /* index of end of dictionary, within context's referential. + * When loadedDictEnd != 0, a dictionary is in use, and still valid. + * This relies on a mechanism to set loadedDictEnd=0 when dictionary is no longer within distance. + * Such mechanism is provided within ZSTD_window_enforceMaxDist() and ZSTD_checkDictValidity(). + * When dict referential is copied into active context (i.e. not attached), + * loadedDictEnd == dictSize, since referential starts from zero. + */ + U32 nextToUpdate; /* index from which to continue table update */ + U32 hashLog3; /* dispatch table for matches of len==3 : larger == faster, more memory */ + U32* hashTable; + U32* hashTable3; + U32* chainTable; + int dedicatedDictSearch; /* Indicates whether this matchState is using the + * dedicated dictionary search structure. + */ + optState_t opt; /* optimal parser state */ + const ZSTD_matchState_t* dictMatchState; + ZSTD_compressionParameters cParams; + const rawSeqStore_t* ldmSeqStore; +}; + +typedef struct { + ZSTD_compressedBlockState_t* prevCBlock; + ZSTD_compressedBlockState_t* nextCBlock; + ZSTD_matchState_t matchState; +} ZSTD_blockState_t; + +typedef struct { + U32 offset; + U32 checksum; +} ldmEntry_t; + +typedef struct { + BYTE const* split; + U32 hash; + U32 checksum; + ldmEntry_t* bucket; +} ldmMatchCandidate_t; + +#define LDM_BATCH_SIZE 64 + +typedef struct { + ZSTD_window_t window; /* State for the window round buffer management */ + ldmEntry_t* hashTable; + U32 loadedDictEnd; + BYTE* bucketOffsets; /* Next position in bucket to insert entry */ + size_t splitIndices[LDM_BATCH_SIZE]; + ldmMatchCandidate_t matchCandidates[LDM_BATCH_SIZE]; +} ldmState_t; + +typedef struct { + U32 enableLdm; /* 1 if enable long distance matching */ + U32 hashLog; /* Log size of hashTable */ + U32 bucketSizeLog; /* Log bucket size for collision resolution, at most 8 */ + U32 minMatchLength; /* Minimum match length */ + U32 hashRateLog; /* Log number of entries to skip */ + U32 windowLog; /* Window log for the LDM */ +} ldmParams_t; + +typedef struct { + int collectSequences; + ZSTD_Sequence* seqStart; + size_t seqIndex; + size_t maxSequences; +} SeqCollector; + +struct ZSTD_CCtx_params_s { + ZSTD_format_e format; + ZSTD_compressionParameters cParams; + ZSTD_frameParameters fParams; + + int compressionLevel; + int forceWindow; /* force back-references to respect limit of + * 1< 63) ? ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; +} + +/* ZSTD_MLcode() : + * note : mlBase = matchLength - MINMATCH; + * because it's the format it's stored in seqStore->sequences */ +MEM_STATIC U32 ZSTD_MLcode(U32 mlBase) +{ + static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; + static const U32 ML_deltaCode = 36; + return (mlBase > 127) ? ZSTD_highbit32(mlBase) + ML_deltaCode : ML_Code[mlBase]; +} + +typedef struct repcodes_s { + U32 rep[3]; +} repcodes_t; + +MEM_STATIC repcodes_t ZSTD_updateRep(U32 const rep[3], U32 const offset, U32 const ll0) +{ + repcodes_t newReps; + if (offset >= ZSTD_REP_NUM) { /* full offset */ + newReps.rep[2] = rep[1]; + newReps.rep[1] = rep[0]; + newReps.rep[0] = offset - ZSTD_REP_MOVE; + } else { /* repcode */ + U32 const repCode = offset + ll0; + if (repCode > 0) { /* note : if repCode==0, no change */ + U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode]; + newReps.rep[2] = (repCode >= 2) ? rep[1] : rep[2]; + newReps.rep[1] = rep[0]; + newReps.rep[0] = currentOffset; + } else { /* repCode == 0 */ + ZSTD_memcpy(&newReps, rep, sizeof(newReps)); + } + } + return newReps; +} + +/* ZSTD_cParam_withinBounds: + * @return 1 if value is within cParam bounds, + * 0 otherwise */ +MEM_STATIC int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value) +{ + ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); + if (ZSTD_isError(bounds.error)) return 0; + if (value < bounds.lowerBound) return 0; + if (value > bounds.upperBound) return 0; + return 1; +} + +/* ZSTD_noCompressBlock() : + * Writes uncompressed block to dst buffer from given src. + * Returns the size of the block */ +MEM_STATIC size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock) +{ + U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(srcSize << 3); + RETURN_ERROR_IF(srcSize + ZSTD_blockHeaderSize > dstCapacity, + dstSize_tooSmall, "dst buf too small for uncompressed block"); + MEM_writeLE24(dst, cBlockHeader24); + ZSTD_memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize); + return ZSTD_blockHeaderSize + srcSize; +} + +MEM_STATIC size_t ZSTD_rleCompressBlock (void* dst, size_t dstCapacity, BYTE src, size_t srcSize, U32 lastBlock) +{ + BYTE* const op = (BYTE*)dst; + U32 const cBlockHeader = lastBlock + (((U32)bt_rle)<<1) + (U32)(srcSize << 3); + RETURN_ERROR_IF(dstCapacity < 4, dstSize_tooSmall, ""); + MEM_writeLE24(op, cBlockHeader); + op[3] = src; + return 4; +} + + +/* ZSTD_minGain() : + * minimum compression required + * to generate a compress block or a compressed literals section. + * note : use same formula for both situations */ +MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat) +{ + U32 const minlog = (strat>=ZSTD_btultra) ? (U32)(strat) - 1 : 6; + ZSTD_STATIC_ASSERT(ZSTD_btultra == 8); + assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat)); + return (srcSize >> minlog) + 2; +} + +MEM_STATIC int ZSTD_disableLiteralsCompression(const ZSTD_CCtx_params* cctxParams) +{ + switch (cctxParams->literalCompressionMode) { + case ZSTD_lcm_huffman: + return 0; + case ZSTD_lcm_uncompressed: + return 1; + default: + assert(0 /* impossible: pre-validated */); + /* fall-through */ + case ZSTD_lcm_auto: + return (cctxParams->cParams.strategy == ZSTD_fast) && (cctxParams->cParams.targetLength > 0); + } +} + +/*! ZSTD_safecopyLiterals() : + * memcpy() function that won't read beyond more than WILDCOPY_OVERLENGTH bytes past ilimit_w. + * Only called when the sequence ends past ilimit_w, so it only needs to be optimized for single + * large copies. + */ +static void ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const iend, BYTE const* ilimit_w) { + assert(iend > ilimit_w); + if (ip <= ilimit_w) { + ZSTD_wildcopy(op, ip, ilimit_w - ip, ZSTD_no_overlap); + op += ilimit_w - ip; + ip = ilimit_w; + } + while (ip < iend) *op++ = *ip++; +} + +/*! ZSTD_storeSeq() : + * Store a sequence (litlen, litPtr, offCode and mlBase) into seqStore_t. + * `offCode` : distance to match + ZSTD_REP_MOVE (values <= ZSTD_REP_MOVE are repCodes). + * `mlBase` : matchLength - MINMATCH + * Allowed to overread literals up to litLimit. +*/ +HINT_INLINE UNUSED_ATTR +void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, const BYTE* litLimit, U32 offCode, size_t mlBase) +{ + BYTE const* const litLimit_w = litLimit - WILDCOPY_OVERLENGTH; + BYTE const* const litEnd = literals + litLength; +#if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6) + static const BYTE* g_start = NULL; + if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */ + { U32 const pos = (U32)((const BYTE*)literals - g_start); + DEBUGLOG(6, "Cpos%7u :%3u literals, match%4u bytes at offCode%7u", + pos, (U32)litLength, (U32)mlBase+MINMATCH, (U32)offCode); + } +#endif + assert((size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart) < seqStorePtr->maxNbSeq); + /* copy Literals */ + assert(seqStorePtr->maxNbLit <= 128 KB); + assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit); + assert(literals + litLength <= litLimit); + if (litEnd <= litLimit_w) { + /* Common case we can use wildcopy. + * First copy 16 bytes, because literals are likely short. + */ + assert(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(seqStorePtr->lit, literals); + if (litLength > 16) { + ZSTD_wildcopy(seqStorePtr->lit+16, literals+16, (ptrdiff_t)litLength-16, ZSTD_no_overlap); + } + } else { + ZSTD_safecopyLiterals(seqStorePtr->lit, literals, litEnd, litLimit_w); + } + seqStorePtr->lit += litLength; + + /* literal Length */ + if (litLength>0xFFFF) { + assert(seqStorePtr->longLengthID == 0); /* there can only be a single long length */ + seqStorePtr->longLengthID = 1; + seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + } + seqStorePtr->sequences[0].litLength = (U16)litLength; + + /* match offset */ + seqStorePtr->sequences[0].offset = offCode + 1; + + /* match Length */ + if (mlBase>0xFFFF) { + assert(seqStorePtr->longLengthID == 0); /* there can only be a single long length */ + seqStorePtr->longLengthID = 2; + seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + } + seqStorePtr->sequences[0].matchLength = (U16)mlBase; + + seqStorePtr->sequences++; +} + + +/*-************************************* +* Match length counter +***************************************/ +static unsigned ZSTD_NbCommonBytes (size_t val) +{ + if (MEM_isLittleEndian()) { + if (MEM_64bits()) { +# if (__GNUC__ >= 4) + return (__builtin_ctzll((U64)val) >> 3); +# else + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, + 0, 3, 1, 3, 1, 4, 2, 7, + 0, 2, 3, 6, 1, 5, 3, 5, + 1, 3, 4, 4, 2, 5, 6, 7, + 7, 0, 1, 2, 3, 3, 4, 6, + 2, 6, 5, 5, 3, 4, 5, 6, + 7, 1, 2, 4, 6, 4, 4, 5, + 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; +# endif + } else { /* 32 bits */ +# if (__GNUC__ >= 3) + return (__builtin_ctz((U32)val) >> 3); +# else + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, + 3, 2, 2, 1, 3, 2, 0, 1, + 3, 3, 1, 2, 2, 2, 2, 0, + 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +# endif + } + } else { /* Big Endian CPU */ + if (MEM_64bits()) { +# if (__GNUC__ >= 4) + return (__builtin_clzll(val) >> 3); +# else + unsigned r; + const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */ + if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +# endif + } else { /* 32 bits */ +# if (__GNUC__ >= 3) + return (__builtin_clz((U32)val) >> 3); +# else + unsigned r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; +# endif + } } +} + + +MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit) +{ + const BYTE* const pStart = pIn; + const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t)-1); + + if (pIn < pInLoopLimit) { + { size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); + if (diff) return ZSTD_NbCommonBytes(diff); } + pIn+=sizeof(size_t); pMatch+=sizeof(size_t); + while (pIn < pInLoopLimit) { + size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); + if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; } + pIn += ZSTD_NbCommonBytes(diff); + return (size_t)(pIn - pStart); + } } + if (MEM_64bits() && (pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; } + if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; } + if ((pIn> (32-h) ; } +MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h); } /* only in zstd_opt.h */ + +static const U32 prime4bytes = 2654435761U; +static U32 ZSTD_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; } +static size_t ZSTD_hash4Ptr(const void* ptr, U32 h) { return ZSTD_hash4(MEM_read32(ptr), h); } + +static const U64 prime5bytes = 889523592379ULL; +static size_t ZSTD_hash5(U64 u, U32 h) { return (size_t)(((u << (64-40)) * prime5bytes) >> (64-h)) ; } +static size_t ZSTD_hash5Ptr(const void* p, U32 h) { return ZSTD_hash5(MEM_readLE64(p), h); } + +static const U64 prime6bytes = 227718039650203ULL; +static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; } +static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); } + +static const U64 prime7bytes = 58295818150454627ULL; +static size_t ZSTD_hash7(U64 u, U32 h) { return (size_t)(((u << (64-56)) * prime7bytes) >> (64-h)) ; } +static size_t ZSTD_hash7Ptr(const void* p, U32 h) { return ZSTD_hash7(MEM_readLE64(p), h); } + +static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; +static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; } +static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); } + +MEM_STATIC FORCE_INLINE_ATTR +size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls) +{ + switch(mls) + { + default: + case 4: return ZSTD_hash4Ptr(p, hBits); + case 5: return ZSTD_hash5Ptr(p, hBits); + case 6: return ZSTD_hash6Ptr(p, hBits); + case 7: return ZSTD_hash7Ptr(p, hBits); + case 8: return ZSTD_hash8Ptr(p, hBits); + } +} + +/* ZSTD_ipow() : + * Return base^exponent. + */ +static U64 ZSTD_ipow(U64 base, U64 exponent) +{ + U64 power = 1; + while (exponent) { + if (exponent & 1) power *= base; + exponent >>= 1; + base *= base; + } + return power; +} + +#define ZSTD_ROLL_HASH_CHAR_OFFSET 10 + +/* ZSTD_rollingHash_append() : + * Add the buffer to the hash value. + */ +static U64 ZSTD_rollingHash_append(U64 hash, void const* buf, size_t size) +{ + BYTE const* istart = (BYTE const*)buf; + size_t pos; + for (pos = 0; pos < size; ++pos) { + hash *= prime8bytes; + hash += istart[pos] + ZSTD_ROLL_HASH_CHAR_OFFSET; + } + return hash; +} + +/* ZSTD_rollingHash_compute() : + * Compute the rolling hash value of the buffer. + */ +MEM_STATIC U64 ZSTD_rollingHash_compute(void const* buf, size_t size) +{ + return ZSTD_rollingHash_append(0, buf, size); +} + +/* ZSTD_rollingHash_primePower() : + * Compute the primePower to be passed to ZSTD_rollingHash_rotate() for a hash + * over a window of length bytes. + */ +MEM_STATIC U64 ZSTD_rollingHash_primePower(U32 length) +{ + return ZSTD_ipow(prime8bytes, length - 1); +} + +/* ZSTD_rollingHash_rotate() : + * Rotate the rolling hash by one byte. + */ +MEM_STATIC U64 ZSTD_rollingHash_rotate(U64 hash, BYTE toRemove, BYTE toAdd, U64 primePower) +{ + hash -= (toRemove + ZSTD_ROLL_HASH_CHAR_OFFSET) * primePower; + hash *= prime8bytes; + hash += toAdd + ZSTD_ROLL_HASH_CHAR_OFFSET; + return hash; +} + +/*-************************************* +* Round buffer management +***************************************/ +#if (ZSTD_WINDOWLOG_MAX_64 > 31) +# error "ZSTD_WINDOWLOG_MAX is too large : would overflow ZSTD_CURRENT_MAX" +#endif +/* Max current allowed */ +#define ZSTD_CURRENT_MAX ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)) +/* Maximum chunk size before overflow correction needs to be called again */ +#define ZSTD_CHUNKSIZE_MAX \ + ( ((U32)-1) /* Maximum ending current index */ \ + - ZSTD_CURRENT_MAX) /* Maximum beginning lowLimit */ + +/* + * ZSTD_window_clear(): + * Clears the window containing the history by simply setting it to empty. + */ +MEM_STATIC void ZSTD_window_clear(ZSTD_window_t* window) +{ + size_t const endT = (size_t)(window->nextSrc - window->base); + U32 const end = (U32)endT; + + window->lowLimit = end; + window->dictLimit = end; +} + +/* + * ZSTD_window_hasExtDict(): + * Returns non-zero if the window has a non-empty extDict. + */ +MEM_STATIC U32 ZSTD_window_hasExtDict(ZSTD_window_t const window) +{ + return window.lowLimit < window.dictLimit; +} + +/* + * ZSTD_matchState_dictMode(): + * Inspects the provided matchState and figures out what dictMode should be + * passed to the compressor. + */ +MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_matchState_t *ms) +{ + return ZSTD_window_hasExtDict(ms->window) ? + ZSTD_extDict : + ms->dictMatchState != NULL ? + (ms->dictMatchState->dedicatedDictSearch ? ZSTD_dedicatedDictSearch : ZSTD_dictMatchState) : + ZSTD_noDict; +} + +/* + * ZSTD_window_needOverflowCorrection(): + * Returns non-zero if the indices are getting too large and need overflow + * protection. + */ +MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window, + void const* srcEnd) +{ + U32 const curr = (U32)((BYTE const*)srcEnd - window.base); + return curr > ZSTD_CURRENT_MAX; +} + +/* + * ZSTD_window_correctOverflow(): + * Reduces the indices to protect from index overflow. + * Returns the correction made to the indices, which must be applied to every + * stored index. + * + * The least significant cycleLog bits of the indices must remain the same, + * which may be 0. Every index up to maxDist in the past must be valid. + * NOTE: (maxDist & cycleMask) must be zero. + */ +MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog, + U32 maxDist, void const* src) +{ + /* preemptive overflow correction: + * 1. correction is large enough: + * lowLimit > (3<<29) ==> current > 3<<29 + 1< (3<<29 + 1< (3<<29) - (1< (3<<29) - (1<<30) (NOTE: chainLog <= 30) + * > 1<<29 + * + * 2. (ip+ZSTD_CHUNKSIZE_MAX - cctx->base) doesn't overflow: + * After correction, current is less than (1<base < 1<<32. + * 3. (cctx->lowLimit + 1< 3<<29 + 1<base); + U32 const currentCycle0 = curr & cycleMask; + /* Exclude zero so that newCurrent - maxDist >= 1. */ + U32 const currentCycle1 = currentCycle0 == 0 ? (1U << cycleLog) : currentCycle0; + U32 const newCurrent = currentCycle1 + maxDist; + U32 const correction = curr - newCurrent; + assert((maxDist & cycleMask) == 0); + assert(curr > newCurrent); + /* Loose bound, should be around 1<<29 (see above) */ + assert(correction > 1<<28); + + window->base += correction; + window->dictBase += correction; + if (window->lowLimit <= correction) window->lowLimit = 1; + else window->lowLimit -= correction; + if (window->dictLimit <= correction) window->dictLimit = 1; + else window->dictLimit -= correction; + + /* Ensure we can still reference the full window. */ + assert(newCurrent >= maxDist); + assert(newCurrent - maxDist >= 1); + /* Ensure that lowLimit and dictLimit didn't underflow. */ + assert(window->lowLimit <= newCurrent); + assert(window->dictLimit <= newCurrent); + + DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction, + window->lowLimit); + return correction; +} + +/* + * ZSTD_window_enforceMaxDist(): + * Updates lowLimit so that: + * (srcEnd - base) - lowLimit == maxDist + loadedDictEnd + * + * It ensures index is valid as long as index >= lowLimit. + * This must be called before a block compression call. + * + * loadedDictEnd is only defined if a dictionary is in use for current compression. + * As the name implies, loadedDictEnd represents the index at end of dictionary. + * The value lies within context's referential, it can be directly compared to blockEndIdx. + * + * If loadedDictEndPtr is NULL, no dictionary is in use, and we use loadedDictEnd == 0. + * If loadedDictEndPtr is not NULL, we set it to zero after updating lowLimit. + * This is because dictionaries are allowed to be referenced fully + * as long as the last byte of the dictionary is in the window. + * Once input has progressed beyond window size, dictionary cannot be referenced anymore. + * + * In normal dict mode, the dictionary lies between lowLimit and dictLimit. + * In dictMatchState mode, lowLimit and dictLimit are the same, + * and the dictionary is below them. + * forceWindow and dictMatchState are therefore incompatible. + */ +MEM_STATIC void +ZSTD_window_enforceMaxDist(ZSTD_window_t* window, + const void* blockEnd, + U32 maxDist, + U32* loadedDictEndPtr, + const ZSTD_matchState_t** dictMatchStatePtr) +{ + U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base); + U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0; + DEBUGLOG(5, "ZSTD_window_enforceMaxDist: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u", + (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd); + + /* - When there is no dictionary : loadedDictEnd == 0. + In which case, the test (blockEndIdx > maxDist) is merely to avoid + overflowing next operation `newLowLimit = blockEndIdx - maxDist`. + - When there is a standard dictionary : + Index referential is copied from the dictionary, + which means it starts from 0. + In which case, loadedDictEnd == dictSize, + and it makes sense to compare `blockEndIdx > maxDist + dictSize` + since `blockEndIdx` also starts from zero. + - When there is an attached dictionary : + loadedDictEnd is expressed within the referential of the context, + so it can be directly compared against blockEndIdx. + */ + if (blockEndIdx > maxDist + loadedDictEnd) { + U32 const newLowLimit = blockEndIdx - maxDist; + if (window->lowLimit < newLowLimit) window->lowLimit = newLowLimit; + if (window->dictLimit < window->lowLimit) { + DEBUGLOG(5, "Update dictLimit to match lowLimit, from %u to %u", + (unsigned)window->dictLimit, (unsigned)window->lowLimit); + window->dictLimit = window->lowLimit; + } + /* On reaching window size, dictionaries are invalidated */ + if (loadedDictEndPtr) *loadedDictEndPtr = 0; + if (dictMatchStatePtr) *dictMatchStatePtr = NULL; + } +} + +/* Similar to ZSTD_window_enforceMaxDist(), + * but only invalidates dictionary + * when input progresses beyond window size. + * assumption : loadedDictEndPtr and dictMatchStatePtr are valid (non NULL) + * loadedDictEnd uses same referential as window->base + * maxDist is the window size */ +MEM_STATIC void +ZSTD_checkDictValidity(const ZSTD_window_t* window, + const void* blockEnd, + U32 maxDist, + U32* loadedDictEndPtr, + const ZSTD_matchState_t** dictMatchStatePtr) +{ + assert(loadedDictEndPtr != NULL); + assert(dictMatchStatePtr != NULL); + { U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base); + U32 const loadedDictEnd = *loadedDictEndPtr; + DEBUGLOG(5, "ZSTD_checkDictValidity: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u", + (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd); + assert(blockEndIdx >= loadedDictEnd); + + if (blockEndIdx > loadedDictEnd + maxDist) { + /* On reaching window size, dictionaries are invalidated. + * For simplification, if window size is reached anywhere within next block, + * the dictionary is invalidated for the full block. + */ + DEBUGLOG(6, "invalidating dictionary for current block (distance > windowSize)"); + *loadedDictEndPtr = 0; + *dictMatchStatePtr = NULL; + } else { + if (*loadedDictEndPtr != 0) { + DEBUGLOG(6, "dictionary considered valid for current block"); + } } } +} + +MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) { + ZSTD_memset(window, 0, sizeof(*window)); + window->base = (BYTE const*)""; + window->dictBase = (BYTE const*)""; + window->dictLimit = 1; /* start from 1, so that 1st position is valid */ + window->lowLimit = 1; /* it ensures first and later CCtx usages compress the same */ + window->nextSrc = window->base + 1; /* see issue #1241 */ +} + +/* + * ZSTD_window_update(): + * Updates the window by appending [src, src + srcSize) to the window. + * If it is not contiguous, the current prefix becomes the extDict, and we + * forget about the extDict. Handles overlap of the prefix and extDict. + * Returns non-zero if the segment is contiguous. + */ +MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window, + void const* src, size_t srcSize) +{ + BYTE const* const ip = (BYTE const*)src; + U32 contiguous = 1; + DEBUGLOG(5, "ZSTD_window_update"); + if (srcSize == 0) + return contiguous; + assert(window->base != NULL); + assert(window->dictBase != NULL); + /* Check if blocks follow each other */ + if (src != window->nextSrc) { + /* not contiguous */ + size_t const distanceFromBase = (size_t)(window->nextSrc - window->base); + DEBUGLOG(5, "Non contiguous blocks, new segment starts at %u", window->dictLimit); + window->lowLimit = window->dictLimit; + assert(distanceFromBase == (size_t)(U32)distanceFromBase); /* should never overflow */ + window->dictLimit = (U32)distanceFromBase; + window->dictBase = window->base; + window->base = ip - distanceFromBase; + /* ms->nextToUpdate = window->dictLimit; */ + if (window->dictLimit - window->lowLimit < HASH_READ_SIZE) window->lowLimit = window->dictLimit; /* too small extDict */ + contiguous = 0; + } + window->nextSrc = ip + srcSize; + /* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */ + if ( (ip+srcSize > window->dictBase + window->lowLimit) + & (ip < window->dictBase + window->dictLimit)) { + ptrdiff_t const highInputIdx = (ip + srcSize) - window->dictBase; + U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)window->dictLimit) ? window->dictLimit : (U32)highInputIdx; + window->lowLimit = lowLimitMax; + DEBUGLOG(5, "Overlapping extDict and input : new lowLimit = %u", window->lowLimit); + } + return contiguous; +} + +/* + * Returns the lowest allowed match index. It may either be in the ext-dict or the prefix. + */ +MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog) +{ + U32 const maxDistance = 1U << windowLog; + U32 const lowestValid = ms->window.lowLimit; + U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; + U32 const isDictionary = (ms->loadedDictEnd != 0); + /* When using a dictionary the entire dictionary is valid if a single byte of the dictionary + * is within the window. We invalidate the dictionary (and set loadedDictEnd to 0) when it isn't + * valid for the entire block. So this check is sufficient to find the lowest valid match index. + */ + U32 const matchLowest = isDictionary ? lowestValid : withinWindow; + return matchLowest; +} + +/* + * Returns the lowest allowed match index in the prefix. + */ +MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog) +{ + U32 const maxDistance = 1U << windowLog; + U32 const lowestValid = ms->window.dictLimit; + U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; + U32 const isDictionary = (ms->loadedDictEnd != 0); + /* When computing the lowest prefix index we need to take the dictionary into account to handle + * the edge case where the dictionary and the source are contiguous in memory. + */ + U32 const matchLowest = isDictionary ? lowestValid : withinWindow; + return matchLowest; +} + + + +/* debug functions */ +#if (DEBUGLEVEL>=2) + +MEM_STATIC double ZSTD_fWeight(U32 rawStat) +{ + U32 const fp_accuracy = 8; + U32 const fp_multiplier = (1 << fp_accuracy); + U32 const newStat = rawStat + 1; + U32 const hb = ZSTD_highbit32(newStat); + U32 const BWeight = hb * fp_multiplier; + U32 const FWeight = (newStat << fp_accuracy) >> hb; + U32 const weight = BWeight + FWeight; + assert(hb + fp_accuracy < 31); + return (double)weight / fp_multiplier; +} + +/* display a table content, + * listing each element, its frequency, and its predicted bit cost */ +MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max) +{ + unsigned u, sum; + for (u=0, sum=0; u<=max; u++) sum += table[u]; + DEBUGLOG(2, "total nb elts: %u", sum); + for (u=0; u<=max; u++) { + DEBUGLOG(2, "%2u: %5u (%.2f)", + u, table[u], ZSTD_fWeight(sum) - ZSTD_fWeight(table[u]) ); + } +} + +#endif + + + +/* =============================================================== + * Shared internal declarations + * These prototypes may be called from sources not in lib/compress + * =============================================================== */ + +/* ZSTD_loadCEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * return : size of dictionary header (size of magic number + dict ID + entropy tables) + * assumptions : magic number supposed already checked + * and dictSize >= 8 */ +size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, + const void* const dict, size_t dictSize); + +void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs); + +/* ============================================================== + * Private declarations + * These prototypes shall only be called from within lib/compress + * ============================================================== */ + +/* ZSTD_getCParamsFromCCtxParams() : + * cParams are built depending on compressionLevel, src size hints, + * LDM and manually set compression parameters. + * Note: srcSizeHint == 0 means 0! + */ +ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( + const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); + +/*! ZSTD_initCStream_internal() : + * Private use only. Init streaming operation. + * expects params to be valid. + * must receive dict, or cdict, or none, but not both. + * @return : 0, or an error code */ +size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, unsigned long long pledgedSrcSize); + +void ZSTD_resetSeqStore(seqStore_t* ssPtr); + +/*! ZSTD_getCParamsFromCDict() : + * as the name implies */ +ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict); + +/* ZSTD_compressBegin_advanced_internal() : + * Private use only. To be called from zstdmt_compress.c. */ +size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_dictContentType_e dictContentType, + ZSTD_dictTableLoadMethod_e dtlm, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + unsigned long long pledgedSrcSize); + +/* ZSTD_compress_advanced_internal() : + * Private use only. To be called from zstdmt_compress.c. */ +size_t ZSTD_compress_advanced_internal(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + const ZSTD_CCtx_params* params); + + +/* ZSTD_writeLastEmptyBlock() : + * output an empty Block with end-of-frame mark to complete a frame + * @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h)) + * or an error code if `dstCapacity` is too small ( 1 */ +U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat); + +/* ZSTD_CCtx_trace() : + * Trace the end of a compression call. + */ +void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize); + +#endif /* ZSTD_COMPRESS_H */ diff --git a/lib/zstd/compress/zstd_compress_literals.c b/lib/zstd/compress/zstd_compress_literals.c new file mode 100644 index 000000000000..655bcda4d1f1 --- /dev/null +++ b/lib/zstd/compress/zstd_compress_literals.c @@ -0,0 +1,158 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + /*-************************************* + * Dependencies + ***************************************/ +#include "zstd_compress_literals.h" + +size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + BYTE* const ostart = (BYTE*)dst; + U32 const flSize = 1 + (srcSize>31) + (srcSize>4095); + + RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall, ""); + + switch(flSize) + { + case 1: /* 2 - 1 - 5 */ + ostart[0] = (BYTE)((U32)set_basic + (srcSize<<3)); + break; + case 2: /* 2 - 2 - 12 */ + MEM_writeLE16(ostart, (U16)((U32)set_basic + (1<<2) + (srcSize<<4))); + break; + case 3: /* 2 - 2 - 20 */ + MEM_writeLE32(ostart, (U32)((U32)set_basic + (3<<2) + (srcSize<<4))); + break; + default: /* not necessary : flSize is {1,2,3} */ + assert(0); + } + + ZSTD_memcpy(ostart + flSize, src, srcSize); + DEBUGLOG(5, "Raw literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize)); + return srcSize + flSize; +} + +size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + BYTE* const ostart = (BYTE*)dst; + U32 const flSize = 1 + (srcSize>31) + (srcSize>4095); + + (void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */ + + switch(flSize) + { + case 1: /* 2 - 1 - 5 */ + ostart[0] = (BYTE)((U32)set_rle + (srcSize<<3)); + break; + case 2: /* 2 - 2 - 12 */ + MEM_writeLE16(ostart, (U16)((U32)set_rle + (1<<2) + (srcSize<<4))); + break; + case 3: /* 2 - 2 - 20 */ + MEM_writeLE32(ostart, (U32)((U32)set_rle + (3<<2) + (srcSize<<4))); + break; + default: /* not necessary : flSize is {1,2,3} */ + assert(0); + } + + ostart[flSize] = *(const BYTE*)src; + DEBUGLOG(5, "RLE literals: %u -> %u", (U32)srcSize, (U32)flSize + 1); + return flSize+1; +} + +size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_strategy strategy, int disableLiteralCompression, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + void* entropyWorkspace, size_t entropyWorkspaceSize, + const int bmi2) +{ + size_t const minGain = ZSTD_minGain(srcSize, strategy); + size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB); + BYTE* const ostart = (BYTE*)dst; + U32 singleStream = srcSize < 256; + symbolEncodingType_e hType = set_compressed; + size_t cLitSize; + + DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i srcSize=%u)", + disableLiteralCompression, (U32)srcSize); + + /* Prepare nextEntropy assuming reusing the existing table */ + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + + if (disableLiteralCompression) + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + + /* small ? don't even attempt compression (speed opt) */ +# define COMPRESS_LITERALS_SIZE_MIN 63 + { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN; + if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + } + + RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression"); + { HUF_repeat repeat = prevHuf->repeatMode; + int const preferRepeat = strategy < ZSTD_lazy ? srcSize <= 1024 : 0; + if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; + cLitSize = singleStream ? + HUF_compress1X_repeat( + ostart+lhSize, dstCapacity-lhSize, src, srcSize, + HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize, + (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2) : + HUF_compress4X_repeat( + ostart+lhSize, dstCapacity-lhSize, src, srcSize, + HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize, + (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2); + if (repeat != HUF_repeat_none) { + /* reused the existing table */ + DEBUGLOG(5, "Reusing previous huffman table"); + hType = set_repeat; + } + } + + if ((cLitSize==0) | (cLitSize >= srcSize - minGain) | ERR_isError(cLitSize)) { + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + } + if (cLitSize==1) { + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); + } + + if (hType == set_compressed) { + /* using a newly constructed table */ + nextHuf->repeatMode = HUF_repeat_check; + } + + /* Build header */ + switch(lhSize) + { + case 3: /* 2 - 2 - 10 - 10 */ + { U32 const lhc = hType + ((!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14); + MEM_writeLE24(ostart, lhc); + break; + } + case 4: /* 2 - 2 - 14 - 14 */ + { U32 const lhc = hType + (2 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<18); + MEM_writeLE32(ostart, lhc); + break; + } + case 5: /* 2 - 2 - 18 - 18 */ + { U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22); + MEM_writeLE32(ostart, lhc); + ostart[4] = (BYTE)(cLitSize >> 10); + break; + } + default: /* not possible : lhSize is {3,4,5} */ + assert(0); + } + DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)srcSize, (U32)(lhSize+cLitSize)); + return lhSize+cLitSize; +} diff --git a/lib/zstd/compress/zstd_compress_literals.h b/lib/zstd/compress/zstd_compress_literals.h new file mode 100644 index 000000000000..9904c0cd30a0 --- /dev/null +++ b/lib/zstd/compress/zstd_compress_literals.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPRESS_LITERALS_H +#define ZSTD_COMPRESS_LITERALS_H + +#include "zstd_compress_internal.h" /* ZSTD_hufCTables_t, ZSTD_minGain() */ + + +size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_strategy strategy, int disableLiteralCompression, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + void* entropyWorkspace, size_t entropyWorkspaceSize, + const int bmi2); + +#endif /* ZSTD_COMPRESS_LITERALS_H */ diff --git a/lib/zstd/compress/zstd_compress_sequences.c b/lib/zstd/compress/zstd_compress_sequences.c new file mode 100644 index 000000000000..dcfcdc9cc5e8 --- /dev/null +++ b/lib/zstd/compress/zstd_compress_sequences.c @@ -0,0 +1,439 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + /*-************************************* + * Dependencies + ***************************************/ +#include "zstd_compress_sequences.h" + +/* + * -log2(x / 256) lookup table for x in [0, 256). + * If x == 0: Return 0 + * Else: Return floor(-log2(x / 256) * 256) + */ +static unsigned const kInverseProbabilityLog256[256] = { + 0, 2048, 1792, 1642, 1536, 1453, 1386, 1329, 1280, 1236, 1197, 1162, + 1130, 1100, 1073, 1047, 1024, 1001, 980, 960, 941, 923, 906, 889, + 874, 859, 844, 830, 817, 804, 791, 779, 768, 756, 745, 734, + 724, 714, 704, 694, 685, 676, 667, 658, 650, 642, 633, 626, + 618, 610, 603, 595, 588, 581, 574, 567, 561, 554, 548, 542, + 535, 529, 523, 517, 512, 506, 500, 495, 489, 484, 478, 473, + 468, 463, 458, 453, 448, 443, 438, 434, 429, 424, 420, 415, + 411, 407, 402, 398, 394, 390, 386, 382, 377, 373, 370, 366, + 362, 358, 354, 350, 347, 343, 339, 336, 332, 329, 325, 322, + 318, 315, 311, 308, 305, 302, 298, 295, 292, 289, 286, 282, + 279, 276, 273, 270, 267, 264, 261, 258, 256, 253, 250, 247, + 244, 241, 239, 236, 233, 230, 228, 225, 222, 220, 217, 215, + 212, 209, 207, 204, 202, 199, 197, 194, 192, 190, 187, 185, + 182, 180, 178, 175, 173, 171, 168, 166, 164, 162, 159, 157, + 155, 153, 151, 149, 146, 144, 142, 140, 138, 136, 134, 132, + 130, 128, 126, 123, 121, 119, 117, 115, 114, 112, 110, 108, + 106, 104, 102, 100, 98, 96, 94, 93, 91, 89, 87, 85, + 83, 82, 80, 78, 76, 74, 73, 71, 69, 67, 66, 64, + 62, 61, 59, 57, 55, 54, 52, 50, 49, 47, 46, 44, + 42, 41, 39, 37, 36, 34, 33, 31, 30, 28, 26, 25, + 23, 22, 20, 19, 17, 16, 14, 13, 11, 10, 8, 7, + 5, 4, 2, 1, +}; + +static unsigned ZSTD_getFSEMaxSymbolValue(FSE_CTable const* ctable) { + void const* ptr = ctable; + U16 const* u16ptr = (U16 const*)ptr; + U32 const maxSymbolValue = MEM_read16(u16ptr + 1); + return maxSymbolValue; +} + +/* + * Returns true if we should use ncount=-1 else we should + * use ncount=1 for low probability symbols instead. + */ +static unsigned ZSTD_useLowProbCount(size_t const nbSeq) +{ + /* Heuristic: This should cover most blocks <= 16K and + * start to fade out after 16K to about 32K depending on + * comprssibility. + */ + return nbSeq >= 2048; +} + +/* + * Returns the cost in bytes of encoding the normalized count header. + * Returns an error if any of the helper functions return an error. + */ +static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max, + size_t const nbSeq, unsigned const FSELog) +{ + BYTE wksp[FSE_NCOUNTBOUND]; + S16 norm[MaxSeq + 1]; + const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); + FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max, ZSTD_useLowProbCount(nbSeq)), ""); + return FSE_writeNCount(wksp, sizeof(wksp), norm, max, tableLog); +} + +/* + * Returns the cost in bits of encoding the distribution described by count + * using the entropy bound. + */ +static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t const total) +{ + unsigned cost = 0; + unsigned s; + for (s = 0; s <= max; ++s) { + unsigned norm = (unsigned)((256 * count[s]) / total); + if (count[s] != 0 && norm == 0) + norm = 1; + assert(count[s] < total); + cost += count[s] * kInverseProbabilityLog256[norm]; + } + return cost >> 8; +} + +/* + * Returns the cost in bits of encoding the distribution in count using ctable. + * Returns an error if ctable cannot represent all the symbols in count. + */ +size_t ZSTD_fseBitCost( + FSE_CTable const* ctable, + unsigned const* count, + unsigned const max) +{ + unsigned const kAccuracyLog = 8; + size_t cost = 0; + unsigned s; + FSE_CState_t cstate; + FSE_initCState(&cstate, ctable); + if (ZSTD_getFSEMaxSymbolValue(ctable) < max) { + DEBUGLOG(5, "Repeat FSE_CTable has maxSymbolValue %u < %u", + ZSTD_getFSEMaxSymbolValue(ctable), max); + return ERROR(GENERIC); + } + for (s = 0; s <= max; ++s) { + unsigned const tableLog = cstate.stateLog; + unsigned const badCost = (tableLog + 1) << kAccuracyLog; + unsigned const bitCost = FSE_bitCost(cstate.symbolTT, tableLog, s, kAccuracyLog); + if (count[s] == 0) + continue; + if (bitCost >= badCost) { + DEBUGLOG(5, "Repeat FSE_CTable has Prob[%u] == 0", s); + return ERROR(GENERIC); + } + cost += (size_t)count[s] * bitCost; + } + return cost >> kAccuracyLog; +} + +/* + * Returns the cost in bits of encoding the distribution in count using the + * table described by norm. The max symbol support by norm is assumed >= max. + * norm must be valid for every symbol with non-zero probability in count. + */ +size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog, + unsigned const* count, unsigned const max) +{ + unsigned const shift = 8 - accuracyLog; + size_t cost = 0; + unsigned s; + assert(accuracyLog <= 8); + for (s = 0; s <= max; ++s) { + unsigned const normAcc = (norm[s] != -1) ? (unsigned)norm[s] : 1; + unsigned const norm256 = normAcc << shift; + assert(norm256 > 0); + assert(norm256 < 256); + cost += count[s] * kInverseProbabilityLog256[norm256]; + } + return cost >> 8; +} + +symbolEncodingType_e +ZSTD_selectEncodingType( + FSE_repeat* repeatMode, unsigned const* count, unsigned const max, + size_t const mostFrequent, size_t nbSeq, unsigned const FSELog, + FSE_CTable const* prevCTable, + short const* defaultNorm, U32 defaultNormLog, + ZSTD_defaultPolicy_e const isDefaultAllowed, + ZSTD_strategy const strategy) +{ + ZSTD_STATIC_ASSERT(ZSTD_defaultDisallowed == 0 && ZSTD_defaultAllowed != 0); + if (mostFrequent == nbSeq) { + *repeatMode = FSE_repeat_none; + if (isDefaultAllowed && nbSeq <= 2) { + /* Prefer set_basic over set_rle when there are 2 or less symbols, + * since RLE uses 1 byte, but set_basic uses 5-6 bits per symbol. + * If basic encoding isn't possible, always choose RLE. + */ + DEBUGLOG(5, "Selected set_basic"); + return set_basic; + } + DEBUGLOG(5, "Selected set_rle"); + return set_rle; + } + if (strategy < ZSTD_lazy) { + if (isDefaultAllowed) { + size_t const staticFse_nbSeq_max = 1000; + size_t const mult = 10 - strategy; + size_t const baseLog = 3; + size_t const dynamicFse_nbSeq_min = (((size_t)1 << defaultNormLog) * mult) >> baseLog; /* 28-36 for offset, 56-72 for lengths */ + assert(defaultNormLog >= 5 && defaultNormLog <= 6); /* xx_DEFAULTNORMLOG */ + assert(mult <= 9 && mult >= 7); + if ( (*repeatMode == FSE_repeat_valid) + && (nbSeq < staticFse_nbSeq_max) ) { + DEBUGLOG(5, "Selected set_repeat"); + return set_repeat; + } + if ( (nbSeq < dynamicFse_nbSeq_min) + || (mostFrequent < (nbSeq >> (defaultNormLog-1))) ) { + DEBUGLOG(5, "Selected set_basic"); + /* The format allows default tables to be repeated, but it isn't useful. + * When using simple heuristics to select encoding type, we don't want + * to confuse these tables with dictionaries. When running more careful + * analysis, we don't need to waste time checking both repeating tables + * and default tables. + */ + *repeatMode = FSE_repeat_none; + return set_basic; + } + } + } else { + size_t const basicCost = isDefaultAllowed ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, count, max) : ERROR(GENERIC); + size_t const repeatCost = *repeatMode != FSE_repeat_none ? ZSTD_fseBitCost(prevCTable, count, max) : ERROR(GENERIC); + size_t const NCountCost = ZSTD_NCountCost(count, max, nbSeq, FSELog); + size_t const compressedCost = (NCountCost << 3) + ZSTD_entropyCost(count, max, nbSeq); + + if (isDefaultAllowed) { + assert(!ZSTD_isError(basicCost)); + assert(!(*repeatMode == FSE_repeat_valid && ZSTD_isError(repeatCost))); + } + assert(!ZSTD_isError(NCountCost)); + assert(compressedCost < ERROR(maxCode)); + DEBUGLOG(5, "Estimated bit costs: basic=%u\trepeat=%u\tcompressed=%u", + (unsigned)basicCost, (unsigned)repeatCost, (unsigned)compressedCost); + if (basicCost <= repeatCost && basicCost <= compressedCost) { + DEBUGLOG(5, "Selected set_basic"); + assert(isDefaultAllowed); + *repeatMode = FSE_repeat_none; + return set_basic; + } + if (repeatCost <= compressedCost) { + DEBUGLOG(5, "Selected set_repeat"); + assert(!ZSTD_isError(repeatCost)); + return set_repeat; + } + assert(compressedCost < basicCost && compressedCost < repeatCost); + } + DEBUGLOG(5, "Selected set_compressed"); + *repeatMode = FSE_repeat_check; + return set_compressed; +} + +typedef struct { + S16 norm[MaxSeq + 1]; + U32 wksp[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(MaxSeq, MaxFSELog)]; +} ZSTD_BuildCTableWksp; + +size_t +ZSTD_buildCTable(void* dst, size_t dstCapacity, + FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type, + unsigned* count, U32 max, + const BYTE* codeTable, size_t nbSeq, + const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, + const FSE_CTable* prevCTable, size_t prevCTableSize, + void* entropyWorkspace, size_t entropyWorkspaceSize) +{ + BYTE* op = (BYTE*)dst; + const BYTE* const oend = op + dstCapacity; + DEBUGLOG(6, "ZSTD_buildCTable (dstCapacity=%u)", (unsigned)dstCapacity); + + switch (type) { + case set_rle: + FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max), ""); + RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall, "not enough space"); + *op = codeTable[0]; + return 1; + case set_repeat: + ZSTD_memcpy(nextCTable, prevCTable, prevCTableSize); + return 0; + case set_basic: + FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize), ""); /* note : could be pre-calculated */ + return 0; + case set_compressed: { + ZSTD_BuildCTableWksp* wksp = (ZSTD_BuildCTableWksp*)entropyWorkspace; + size_t nbSeq_1 = nbSeq; + const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); + if (count[codeTable[nbSeq-1]] > 1) { + count[codeTable[nbSeq-1]]--; + nbSeq_1--; + } + assert(nbSeq_1 > 1); + assert(entropyWorkspaceSize >= sizeof(ZSTD_BuildCTableWksp)); + (void)entropyWorkspaceSize; + FORWARD_IF_ERROR(FSE_normalizeCount(wksp->norm, tableLog, count, nbSeq_1, max, ZSTD_useLowProbCount(nbSeq_1)), ""); + { size_t const NCountSize = FSE_writeNCount(op, oend - op, wksp->norm, max, tableLog); /* overflow protected */ + FORWARD_IF_ERROR(NCountSize, "FSE_writeNCount failed"); + FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, wksp->norm, max, tableLog, wksp->wksp, sizeof(wksp->wksp)), ""); + return NCountSize; + } + } + default: assert(0); RETURN_ERROR(GENERIC, "impossible to reach"); + } +} + +FORCE_INLINE_TEMPLATE size_t +ZSTD_encodeSequences_body( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets) +{ + BIT_CStream_t blockStream; + FSE_CState_t stateMatchLength; + FSE_CState_t stateOffsetBits; + FSE_CState_t stateLitLength; + + RETURN_ERROR_IF( + ERR_isError(BIT_initCStream(&blockStream, dst, dstCapacity)), + dstSize_tooSmall, "not enough space remaining"); + DEBUGLOG(6, "available space for bitstream : %i (dstCapacity=%u)", + (int)(blockStream.endPtr - blockStream.startPtr), + (unsigned)dstCapacity); + + /* first symbols */ + FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]); + FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq-1]); + FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]); + BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]); + if (MEM_32bits()) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[nbSeq-1].matchLength, ML_bits[mlCodeTable[nbSeq-1]]); + if (MEM_32bits()) BIT_flushBits(&blockStream); + if (longOffsets) { + U32 const ofBits = ofCodeTable[nbSeq-1]; + unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); + if (extraBits) { + BIT_addBits(&blockStream, sequences[nbSeq-1].offset, extraBits); + BIT_flushBits(&blockStream); + } + BIT_addBits(&blockStream, sequences[nbSeq-1].offset >> extraBits, + ofBits - extraBits); + } else { + BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]); + } + BIT_flushBits(&blockStream); + + { size_t n; + for (n=nbSeq-2 ; n= 64-7-(LLFSELog+MLFSELog+OffFSELog))) + BIT_flushBits(&blockStream); /* (7)*/ + BIT_addBits(&blockStream, sequences[n].litLength, llBits); + if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[n].matchLength, mlBits); + if (MEM_32bits() || (ofBits+mlBits+llBits > 56)) BIT_flushBits(&blockStream); + if (longOffsets) { + unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); + if (extraBits) { + BIT_addBits(&blockStream, sequences[n].offset, extraBits); + BIT_flushBits(&blockStream); /* (7)*/ + } + BIT_addBits(&blockStream, sequences[n].offset >> extraBits, + ofBits - extraBits); /* 31 */ + } else { + BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */ + } + BIT_flushBits(&blockStream); /* (7)*/ + DEBUGLOG(7, "remaining space : %i", (int)(blockStream.endPtr - blockStream.ptr)); + } } + + DEBUGLOG(6, "ZSTD_encodeSequences: flushing ML state with %u bits", stateMatchLength.stateLog); + FSE_flushCState(&blockStream, &stateMatchLength); + DEBUGLOG(6, "ZSTD_encodeSequences: flushing Off state with %u bits", stateOffsetBits.stateLog); + FSE_flushCState(&blockStream, &stateOffsetBits); + DEBUGLOG(6, "ZSTD_encodeSequences: flushing LL state with %u bits", stateLitLength.stateLog); + FSE_flushCState(&blockStream, &stateLitLength); + + { size_t const streamSize = BIT_closeCStream(&blockStream); + RETURN_ERROR_IF(streamSize==0, dstSize_tooSmall, "not enough space"); + return streamSize; + } +} + +static size_t +ZSTD_encodeSequences_default( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets) +{ + return ZSTD_encodeSequences_body(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); +} + + +#if DYNAMIC_BMI2 + +static TARGET_ATTRIBUTE("bmi2") size_t +ZSTD_encodeSequences_bmi2( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets) +{ + return ZSTD_encodeSequences_body(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); +} + +#endif + +size_t ZSTD_encodeSequences( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2) +{ + DEBUGLOG(5, "ZSTD_encodeSequences: dstCapacity = %u", (unsigned)dstCapacity); +#if DYNAMIC_BMI2 + if (bmi2) { + return ZSTD_encodeSequences_bmi2(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); + } +#endif + (void)bmi2; + return ZSTD_encodeSequences_default(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); +} diff --git a/lib/zstd/compress/zstd_compress_sequences.h b/lib/zstd/compress/zstd_compress_sequences.h new file mode 100644 index 000000000000..7991364c2f71 --- /dev/null +++ b/lib/zstd/compress/zstd_compress_sequences.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPRESS_SEQUENCES_H +#define ZSTD_COMPRESS_SEQUENCES_H + +#include "../common/fse.h" /* FSE_repeat, FSE_CTable */ +#include "../common/zstd_internal.h" /* symbolEncodingType_e, ZSTD_strategy */ + +typedef enum { + ZSTD_defaultDisallowed = 0, + ZSTD_defaultAllowed = 1 +} ZSTD_defaultPolicy_e; + +symbolEncodingType_e +ZSTD_selectEncodingType( + FSE_repeat* repeatMode, unsigned const* count, unsigned const max, + size_t const mostFrequent, size_t nbSeq, unsigned const FSELog, + FSE_CTable const* prevCTable, + short const* defaultNorm, U32 defaultNormLog, + ZSTD_defaultPolicy_e const isDefaultAllowed, + ZSTD_strategy const strategy); + +size_t +ZSTD_buildCTable(void* dst, size_t dstCapacity, + FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type, + unsigned* count, U32 max, + const BYTE* codeTable, size_t nbSeq, + const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, + const FSE_CTable* prevCTable, size_t prevCTableSize, + void* entropyWorkspace, size_t entropyWorkspaceSize); + +size_t ZSTD_encodeSequences( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2); + +size_t ZSTD_fseBitCost( + FSE_CTable const* ctable, + unsigned const* count, + unsigned const max); + +size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog, + unsigned const* count, unsigned const max); +#endif /* ZSTD_COMPRESS_SEQUENCES_H */ diff --git a/lib/zstd/compress/zstd_compress_superblock.c b/lib/zstd/compress/zstd_compress_superblock.c new file mode 100644 index 000000000000..ee03e0aedb03 --- /dev/null +++ b/lib/zstd/compress/zstd_compress_superblock.c @@ -0,0 +1,850 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + /*-************************************* + * Dependencies + ***************************************/ +#include "zstd_compress_superblock.h" + +#include "../common/zstd_internal.h" /* ZSTD_getSequenceLength */ +#include "hist.h" /* HIST_countFast_wksp */ +#include "zstd_compress_internal.h" +#include "zstd_compress_sequences.h" +#include "zstd_compress_literals.h" + +/*-************************************* +* Superblock entropy buffer structs +***************************************/ +/* ZSTD_hufCTablesMetadata_t : + * Stores Literals Block Type for a super-block in hType, and + * huffman tree description in hufDesBuffer. + * hufDesSize refers to the size of huffman tree description in bytes. + * This metadata is populated in ZSTD_buildSuperBlockEntropy_literal() */ +typedef struct { + symbolEncodingType_e hType; + BYTE hufDesBuffer[ZSTD_MAX_HUF_HEADER_SIZE]; + size_t hufDesSize; +} ZSTD_hufCTablesMetadata_t; + +/* ZSTD_fseCTablesMetadata_t : + * Stores symbol compression modes for a super-block in {ll, ol, ml}Type, and + * fse tables in fseTablesBuffer. + * fseTablesSize refers to the size of fse tables in bytes. + * This metadata is populated in ZSTD_buildSuperBlockEntropy_sequences() */ +typedef struct { + symbolEncodingType_e llType; + symbolEncodingType_e ofType; + symbolEncodingType_e mlType; + BYTE fseTablesBuffer[ZSTD_MAX_FSE_HEADERS_SIZE]; + size_t fseTablesSize; + size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_compressSubBlock_sequences() */ +} ZSTD_fseCTablesMetadata_t; + +typedef struct { + ZSTD_hufCTablesMetadata_t hufMetadata; + ZSTD_fseCTablesMetadata_t fseMetadata; +} ZSTD_entropyCTablesMetadata_t; + + +/* ZSTD_buildSuperBlockEntropy_literal() : + * Builds entropy for the super-block literals. + * Stores literals block type (raw, rle, compressed, repeat) and + * huffman description table to hufMetadata. + * @return : size of huffman description table or error code */ +static size_t ZSTD_buildSuperBlockEntropy_literal(void* const src, size_t srcSize, + const ZSTD_hufCTables_t* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_hufCTablesMetadata_t* hufMetadata, + const int disableLiteralsCompression, + void* workspace, size_t wkspSize) +{ + BYTE* const wkspStart = (BYTE*)workspace; + BYTE* const wkspEnd = wkspStart + wkspSize; + BYTE* const countWkspStart = wkspStart; + unsigned* const countWksp = (unsigned*)workspace; + const size_t countWkspSize = (HUF_SYMBOLVALUE_MAX + 1) * sizeof(unsigned); + BYTE* const nodeWksp = countWkspStart + countWkspSize; + const size_t nodeWkspSize = wkspEnd-nodeWksp; + unsigned maxSymbolValue = 255; + unsigned huffLog = HUF_TABLELOG_DEFAULT; + HUF_repeat repeat = prevHuf->repeatMode; + + DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy_literal (srcSize=%zu)", srcSize); + + /* Prepare nextEntropy assuming reusing the existing table */ + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + + if (disableLiteralsCompression) { + DEBUGLOG(5, "set_basic - disabled"); + hufMetadata->hType = set_basic; + return 0; + } + + /* small ? don't even attempt compression (speed opt) */ +# define COMPRESS_LITERALS_SIZE_MIN 63 + { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN; + if (srcSize <= minLitSize) { + DEBUGLOG(5, "set_basic - too small"); + hufMetadata->hType = set_basic; + return 0; + } + } + + /* Scan input and build symbol stats */ + { size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)src, srcSize, workspace, wkspSize); + FORWARD_IF_ERROR(largest, "HIST_count_wksp failed"); + if (largest == srcSize) { + DEBUGLOG(5, "set_rle"); + hufMetadata->hType = set_rle; + return 0; + } + if (largest <= (srcSize >> 7)+4) { + DEBUGLOG(5, "set_basic - no gain"); + hufMetadata->hType = set_basic; + return 0; + } + } + + /* Validate the previous Huffman table */ + if (repeat == HUF_repeat_check && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) { + repeat = HUF_repeat_none; + } + + /* Build Huffman Tree */ + ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable)); + huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); + { size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp, + maxSymbolValue, huffLog, + nodeWksp, nodeWkspSize); + FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp"); + huffLog = (U32)maxBits; + { /* Build and write the CTable */ + size_t const newCSize = HUF_estimateCompressedSize( + (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue); + size_t const hSize = HUF_writeCTable_wksp( + hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer), + (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog, + nodeWksp, nodeWkspSize); + /* Check against repeating the previous CTable */ + if (repeat != HUF_repeat_none) { + size_t const oldCSize = HUF_estimateCompressedSize( + (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue); + if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) { + DEBUGLOG(5, "set_repeat - smaller"); + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + hufMetadata->hType = set_repeat; + return 0; + } + } + if (newCSize + hSize >= srcSize) { + DEBUGLOG(5, "set_basic - no gains"); + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + hufMetadata->hType = set_basic; + return 0; + } + DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize); + hufMetadata->hType = set_compressed; + nextHuf->repeatMode = HUF_repeat_check; + return hSize; + } + } +} + +/* ZSTD_buildSuperBlockEntropy_sequences() : + * Builds entropy for the super-block sequences. + * Stores symbol compression modes and fse table to fseMetadata. + * @return : size of fse tables or error code */ +static size_t ZSTD_buildSuperBlockEntropy_sequences(seqStore_t* seqStorePtr, + const ZSTD_fseCTables_t* prevEntropy, + ZSTD_fseCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + ZSTD_fseCTablesMetadata_t* fseMetadata, + void* workspace, size_t wkspSize) +{ + BYTE* const wkspStart = (BYTE*)workspace; + BYTE* const wkspEnd = wkspStart + wkspSize; + BYTE* const countWkspStart = wkspStart; + unsigned* const countWksp = (unsigned*)workspace; + const size_t countWkspSize = (MaxSeq + 1) * sizeof(unsigned); + BYTE* const cTableWksp = countWkspStart + countWkspSize; + const size_t cTableWkspSize = wkspEnd-cTableWksp; + ZSTD_strategy const strategy = cctxParams->cParams.strategy; + FSE_CTable* CTable_LitLength = nextEntropy->litlengthCTable; + FSE_CTable* CTable_OffsetBits = nextEntropy->offcodeCTable; + FSE_CTable* CTable_MatchLength = nextEntropy->matchlengthCTable; + const BYTE* const ofCodeTable = seqStorePtr->ofCode; + const BYTE* const llCodeTable = seqStorePtr->llCode; + const BYTE* const mlCodeTable = seqStorePtr->mlCode; + size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart; + BYTE* const ostart = fseMetadata->fseTablesBuffer; + BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer); + BYTE* op = ostart; + + assert(cTableWkspSize >= (1 << MaxFSELog) * sizeof(FSE_FUNCTION_TYPE)); + DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy_sequences (nbSeq=%zu)", nbSeq); + ZSTD_memset(workspace, 0, wkspSize); + + fseMetadata->lastCountSize = 0; + /* convert length/distances into codes */ + ZSTD_seqToCodes(seqStorePtr); + /* build CTable for Literal Lengths */ + { U32 LLtype; + unsigned max = MaxLL; + size_t const mostFrequent = HIST_countFast_wksp(countWksp, &max, llCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ + DEBUGLOG(5, "Building LL table"); + nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode; + LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode, + countWksp, max, mostFrequent, nbSeq, + LLFSELog, prevEntropy->litlengthCTable, + LL_defaultNorm, LL_defaultNormLog, + ZSTD_defaultAllowed, strategy); + assert(set_basic < set_compressed && set_rle < set_compressed); + assert(!(LLtype < set_compressed && nextEntropy->litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype, + countWksp, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL, + prevEntropy->litlengthCTable, sizeof(prevEntropy->litlengthCTable), + cTableWksp, cTableWkspSize); + FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for LitLens failed"); + if (LLtype == set_compressed) + fseMetadata->lastCountSize = countSize; + op += countSize; + fseMetadata->llType = (symbolEncodingType_e) LLtype; + } } + /* build CTable for Offsets */ + { U32 Offtype; + unsigned max = MaxOff; + size_t const mostFrequent = HIST_countFast_wksp(countWksp, &max, ofCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ + /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ + ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; + DEBUGLOG(5, "Building OF table"); + nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode; + Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode, + countWksp, max, mostFrequent, nbSeq, + OffFSELog, prevEntropy->offcodeCTable, + OF_defaultNorm, OF_defaultNormLog, + defaultPolicy, strategy); + assert(!(Offtype < set_compressed && nextEntropy->offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype, + countWksp, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, + prevEntropy->offcodeCTable, sizeof(prevEntropy->offcodeCTable), + cTableWksp, cTableWkspSize); + FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for Offsets failed"); + if (Offtype == set_compressed) + fseMetadata->lastCountSize = countSize; + op += countSize; + fseMetadata->ofType = (symbolEncodingType_e) Offtype; + } } + /* build CTable for MatchLengths */ + { U32 MLtype; + unsigned max = MaxML; + size_t const mostFrequent = HIST_countFast_wksp(countWksp, &max, mlCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ + DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op)); + nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode; + MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode, + countWksp, max, mostFrequent, nbSeq, + MLFSELog, prevEntropy->matchlengthCTable, + ML_defaultNorm, ML_defaultNormLog, + ZSTD_defaultAllowed, strategy); + assert(!(MLtype < set_compressed && nextEntropy->matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype, + countWksp, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML, + prevEntropy->matchlengthCTable, sizeof(prevEntropy->matchlengthCTable), + cTableWksp, cTableWkspSize); + FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for MatchLengths failed"); + if (MLtype == set_compressed) + fseMetadata->lastCountSize = countSize; + op += countSize; + fseMetadata->mlType = (symbolEncodingType_e) MLtype; + } } + assert((size_t) (op-ostart) <= sizeof(fseMetadata->fseTablesBuffer)); + return op-ostart; +} + + +/* ZSTD_buildSuperBlockEntropy() : + * Builds entropy for the super-block. + * @return : 0 on success or error code */ +static size_t +ZSTD_buildSuperBlockEntropy(seqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize) +{ + size_t const litSize = seqStorePtr->lit - seqStorePtr->litStart; + DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy"); + entropyMetadata->hufMetadata.hufDesSize = + ZSTD_buildSuperBlockEntropy_literal(seqStorePtr->litStart, litSize, + &prevEntropy->huf, &nextEntropy->huf, + &entropyMetadata->hufMetadata, + ZSTD_disableLiteralsCompression(cctxParams), + workspace, wkspSize); + FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildSuperBlockEntropy_literal failed"); + entropyMetadata->fseMetadata.fseTablesSize = + ZSTD_buildSuperBlockEntropy_sequences(seqStorePtr, + &prevEntropy->fse, &nextEntropy->fse, + cctxParams, + &entropyMetadata->fseMetadata, + workspace, wkspSize); + FORWARD_IF_ERROR(entropyMetadata->fseMetadata.fseTablesSize, "ZSTD_buildSuperBlockEntropy_sequences failed"); + return 0; +} + +/* ZSTD_compressSubBlock_literal() : + * Compresses literals section for a sub-block. + * When we have to write the Huffman table we will sometimes choose a header + * size larger than necessary. This is because we have to pick the header size + * before we know the table size + compressed size, so we have a bound on the + * table size. If we guessed incorrectly, we fall back to uncompressed literals. + * + * We write the header when writeEntropy=1 and set entropyWritten=1 when we succeeded + * in writing the header, otherwise it is set to 0. + * + * hufMetadata->hType has literals block type info. + * If it is set_basic, all sub-blocks literals section will be Raw_Literals_Block. + * If it is set_rle, all sub-blocks literals section will be RLE_Literals_Block. + * If it is set_compressed, first sub-block's literals section will be Compressed_Literals_Block + * If it is set_compressed, first sub-block's literals section will be Treeless_Literals_Block + * and the following sub-blocks' literals sections will be Treeless_Literals_Block. + * @return : compressed size of literals section of a sub-block + * Or 0 if it unable to compress. + * Or error code */ +static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable, + const ZSTD_hufCTablesMetadata_t* hufMetadata, + const BYTE* literals, size_t litSize, + void* dst, size_t dstSize, + const int bmi2, int writeEntropy, int* entropyWritten) +{ + size_t const header = writeEntropy ? 200 : 0; + size_t const lhSize = 3 + (litSize >= (1 KB - header)) + (litSize >= (16 KB - header)); + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart + lhSize; + U32 const singleStream = lhSize == 3; + symbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat; + size_t cLitSize = 0; + + (void)bmi2; /* TODO bmi2... */ + + DEBUGLOG(5, "ZSTD_compressSubBlock_literal (litSize=%zu, lhSize=%zu, writeEntropy=%d)", litSize, lhSize, writeEntropy); + + *entropyWritten = 0; + if (litSize == 0 || hufMetadata->hType == set_basic) { + DEBUGLOG(5, "ZSTD_compressSubBlock_literal using raw literal"); + return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize); + } else if (hufMetadata->hType == set_rle) { + DEBUGLOG(5, "ZSTD_compressSubBlock_literal using rle literal"); + return ZSTD_compressRleLiteralsBlock(dst, dstSize, literals, litSize); + } + + assert(litSize > 0); + assert(hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat); + + if (writeEntropy && hufMetadata->hType == set_compressed) { + ZSTD_memcpy(op, hufMetadata->hufDesBuffer, hufMetadata->hufDesSize); + op += hufMetadata->hufDesSize; + cLitSize += hufMetadata->hufDesSize; + DEBUGLOG(5, "ZSTD_compressSubBlock_literal (hSize=%zu)", hufMetadata->hufDesSize); + } + + /* TODO bmi2 */ + { const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, oend-op, literals, litSize, hufTable) + : HUF_compress4X_usingCTable(op, oend-op, literals, litSize, hufTable); + op += cSize; + cLitSize += cSize; + if (cSize == 0 || ERR_isError(cSize)) { + DEBUGLOG(5, "Failed to write entropy tables %s", ZSTD_getErrorName(cSize)); + return 0; + } + /* If we expand and we aren't writing a header then emit uncompressed */ + if (!writeEntropy && cLitSize >= litSize) { + DEBUGLOG(5, "ZSTD_compressSubBlock_literal using raw literal because uncompressible"); + return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize); + } + /* If we are writing headers then allow expansion that doesn't change our header size. */ + if (lhSize < (size_t)(3 + (cLitSize >= 1 KB) + (cLitSize >= 16 KB))) { + assert(cLitSize > litSize); + DEBUGLOG(5, "Literals expanded beyond allowed header size"); + return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize); + } + DEBUGLOG(5, "ZSTD_compressSubBlock_literal (cSize=%zu)", cSize); + } + + /* Build header */ + switch(lhSize) + { + case 3: /* 2 - 2 - 10 - 10 */ + { U32 const lhc = hType + ((!singleStream) << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<14); + MEM_writeLE24(ostart, lhc); + break; + } + case 4: /* 2 - 2 - 14 - 14 */ + { U32 const lhc = hType + (2 << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<18); + MEM_writeLE32(ostart, lhc); + break; + } + case 5: /* 2 - 2 - 18 - 18 */ + { U32 const lhc = hType + (3 << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<22); + MEM_writeLE32(ostart, lhc); + ostart[4] = (BYTE)(cLitSize >> 10); + break; + } + default: /* not possible : lhSize is {3,4,5} */ + assert(0); + } + *entropyWritten = 1; + DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)litSize, (U32)(op-ostart)); + return op-ostart; +} + +static size_t ZSTD_seqDecompressedSize(seqStore_t const* seqStore, const seqDef* sequences, size_t nbSeq, size_t litSize, int lastSequence) { + const seqDef* const sstart = sequences; + const seqDef* const send = sequences + nbSeq; + const seqDef* sp = sstart; + size_t matchLengthSum = 0; + size_t litLengthSum = 0; + while (send-sp > 0) { + ZSTD_sequenceLength const seqLen = ZSTD_getSequenceLength(seqStore, sp); + litLengthSum += seqLen.litLength; + matchLengthSum += seqLen.matchLength; + sp++; + } + assert(litLengthSum <= litSize); + if (!lastSequence) { + assert(litLengthSum == litSize); + } + return matchLengthSum + litSize; +} + +/* ZSTD_compressSubBlock_sequences() : + * Compresses sequences section for a sub-block. + * fseMetadata->llType, fseMetadata->ofType, and fseMetadata->mlType have + * symbol compression modes for the super-block. + * The first successfully compressed block will have these in its header. + * We set entropyWritten=1 when we succeed in compressing the sequences. + * The following sub-blocks will always have repeat mode. + * @return : compressed size of sequences section of a sub-block + * Or 0 if it is unable to compress + * Or error code. */ +static size_t ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables, + const ZSTD_fseCTablesMetadata_t* fseMetadata, + const seqDef* sequences, size_t nbSeq, + const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + const int bmi2, int writeEntropy, int* entropyWritten) +{ + const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + BYTE* seqHead; + + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (nbSeq=%zu, writeEntropy=%d, longOffsets=%d)", nbSeq, writeEntropy, longOffsets); + + *entropyWritten = 0; + /* Sequences Header */ + RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, + dstSize_tooSmall, ""); + if (nbSeq < 0x7F) + *op++ = (BYTE)nbSeq; + else if (nbSeq < LONGNBSEQ) + op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; + else + op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; + if (nbSeq==0) { + return op - ostart; + } + + /* seqHead : flags for FSE encoding type */ + seqHead = op++; + + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (seqHeadSize=%u)", (unsigned)(op-ostart)); + + if (writeEntropy) { + const U32 LLtype = fseMetadata->llType; + const U32 Offtype = fseMetadata->ofType; + const U32 MLtype = fseMetadata->mlType; + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (fseTablesSize=%zu)", fseMetadata->fseTablesSize); + *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); + ZSTD_memcpy(op, fseMetadata->fseTablesBuffer, fseMetadata->fseTablesSize); + op += fseMetadata->fseTablesSize; + } else { + const U32 repeat = set_repeat; + *seqHead = (BYTE)((repeat<<6) + (repeat<<4) + (repeat<<2)); + } + + { size_t const bitstreamSize = ZSTD_encodeSequences( + op, oend - op, + fseTables->matchlengthCTable, mlCode, + fseTables->offcodeCTable, ofCode, + fseTables->litlengthCTable, llCode, + sequences, nbSeq, + longOffsets, bmi2); + FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed"); + op += bitstreamSize; + /* zstd versions <= 1.3.4 mistakenly report corruption when + * FSE_readNCount() receives a buffer < 4 bytes. + * Fixed by https://github.com/facebook/zstd/pull/1146. + * This can happen when the last set_compressed table present is 2 + * bytes and the bitstream is only one byte. + * In this exceedingly rare case, we will simply emit an uncompressed + * block, since it isn't worth optimizing. + */ +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + if (writeEntropy && fseMetadata->lastCountSize && fseMetadata->lastCountSize + bitstreamSize < 4) { + /* NCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */ + assert(fseMetadata->lastCountSize + bitstreamSize == 3); + DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by " + "emitting an uncompressed block."); + return 0; + } +#endif + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (bitstreamSize=%zu)", bitstreamSize); + } + + /* zstd versions <= 1.4.0 mistakenly report error when + * sequences section body size is less than 3 bytes. + * Fixed by https://github.com/facebook/zstd/pull/1664. + * This can happen when the previous sequences section block is compressed + * with rle mode and the current block's sequences section is compressed + * with repeat mode where sequences section body size can be 1 byte. + */ +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + if (op-seqHead < 4) { + DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.4.0 by emitting " + "an uncompressed block when sequences are < 4 bytes"); + return 0; + } +#endif + + *entropyWritten = 1; + return op - ostart; +} + +/* ZSTD_compressSubBlock() : + * Compresses a single sub-block. + * @return : compressed size of the sub-block + * Or 0 if it failed to compress. */ +static size_t ZSTD_compressSubBlock(const ZSTD_entropyCTables_t* entropy, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + const seqDef* sequences, size_t nbSeq, + const BYTE* literals, size_t litSize, + const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + const int bmi2, + int writeLitEntropy, int writeSeqEntropy, + int* litEntropyWritten, int* seqEntropyWritten, + U32 lastBlock) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart + ZSTD_blockHeaderSize; + DEBUGLOG(5, "ZSTD_compressSubBlock (litSize=%zu, nbSeq=%zu, writeLitEntropy=%d, writeSeqEntropy=%d, lastBlock=%d)", + litSize, nbSeq, writeLitEntropy, writeSeqEntropy, lastBlock); + { size_t cLitSize = ZSTD_compressSubBlock_literal((const HUF_CElt*)entropy->huf.CTable, + &entropyMetadata->hufMetadata, literals, litSize, + op, oend-op, bmi2, writeLitEntropy, litEntropyWritten); + FORWARD_IF_ERROR(cLitSize, "ZSTD_compressSubBlock_literal failed"); + if (cLitSize == 0) return 0; + op += cLitSize; + } + { size_t cSeqSize = ZSTD_compressSubBlock_sequences(&entropy->fse, + &entropyMetadata->fseMetadata, + sequences, nbSeq, + llCode, mlCode, ofCode, + cctxParams, + op, oend-op, + bmi2, writeSeqEntropy, seqEntropyWritten); + FORWARD_IF_ERROR(cSeqSize, "ZSTD_compressSubBlock_sequences failed"); + if (cSeqSize == 0) return 0; + op += cSeqSize; + } + /* Write block header */ + { size_t cSize = (op-ostart)-ZSTD_blockHeaderSize; + U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); + MEM_writeLE24(ostart, cBlockHeader24); + } + return op-ostart; +} + +static size_t ZSTD_estimateSubBlockSize_literal(const BYTE* literals, size_t litSize, + const ZSTD_hufCTables_t* huf, + const ZSTD_hufCTablesMetadata_t* hufMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) +{ + unsigned* const countWksp = (unsigned*)workspace; + unsigned maxSymbolValue = 255; + size_t literalSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */ + + if (hufMetadata->hType == set_basic) return litSize; + else if (hufMetadata->hType == set_rle) return 1; + else if (hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat) { + size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)literals, litSize, workspace, wkspSize); + if (ZSTD_isError(largest)) return litSize; + { size_t cLitSizeEstimate = HUF_estimateCompressedSize((const HUF_CElt*)huf->CTable, countWksp, maxSymbolValue); + if (writeEntropy) cLitSizeEstimate += hufMetadata->hufDesSize; + return cLitSizeEstimate + literalSectionHeaderSize; + } } + assert(0); /* impossible */ + return 0; +} + +static size_t ZSTD_estimateSubBlockSize_symbolType(symbolEncodingType_e type, + const BYTE* codeTable, unsigned maxCode, + size_t nbSeq, const FSE_CTable* fseCTable, + const U32* additionalBits, + short const* defaultNorm, U32 defaultNormLog, U32 defaultMax, + void* workspace, size_t wkspSize) +{ + unsigned* const countWksp = (unsigned*)workspace; + const BYTE* ctp = codeTable; + const BYTE* const ctStart = ctp; + const BYTE* const ctEnd = ctStart + nbSeq; + size_t cSymbolTypeSizeEstimateInBits = 0; + unsigned max = maxCode; + + HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */ + if (type == set_basic) { + /* We selected this encoding type, so it must be valid. */ + assert(max <= defaultMax); + cSymbolTypeSizeEstimateInBits = max <= defaultMax + ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max) + : ERROR(GENERIC); + } else if (type == set_rle) { + cSymbolTypeSizeEstimateInBits = 0; + } else if (type == set_compressed || type == set_repeat) { + cSymbolTypeSizeEstimateInBits = ZSTD_fseBitCost(fseCTable, countWksp, max); + } + if (ZSTD_isError(cSymbolTypeSizeEstimateInBits)) return nbSeq * 10; + while (ctp < ctEnd) { + if (additionalBits) cSymbolTypeSizeEstimateInBits += additionalBits[*ctp]; + else cSymbolTypeSizeEstimateInBits += *ctp; /* for offset, offset code is also the number of additional bits */ + ctp++; + } + return cSymbolTypeSizeEstimateInBits / 8; +} + +static size_t ZSTD_estimateSubBlockSize_sequences(const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_fseCTables_t* fseTables, + const ZSTD_fseCTablesMetadata_t* fseMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) +{ + size_t sequencesSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */ + size_t cSeqSizeEstimate = 0; + cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, MaxOff, + nbSeq, fseTables->offcodeCTable, NULL, + OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, + workspace, wkspSize); + cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->llType, llCodeTable, MaxLL, + nbSeq, fseTables->litlengthCTable, LL_bits, + LL_defaultNorm, LL_defaultNormLog, MaxLL, + workspace, wkspSize); + cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, MaxML, + nbSeq, fseTables->matchlengthCTable, ML_bits, + ML_defaultNorm, ML_defaultNormLog, MaxML, + workspace, wkspSize); + if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize; + return cSeqSizeEstimate + sequencesSectionHeaderSize; +} + +static size_t ZSTD_estimateSubBlockSize(const BYTE* literals, size_t litSize, + const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_entropyCTables_t* entropy, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize, + int writeLitEntropy, int writeSeqEntropy) { + size_t cSizeEstimate = 0; + cSizeEstimate += ZSTD_estimateSubBlockSize_literal(literals, litSize, + &entropy->huf, &entropyMetadata->hufMetadata, + workspace, wkspSize, writeLitEntropy); + cSizeEstimate += ZSTD_estimateSubBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable, + nbSeq, &entropy->fse, &entropyMetadata->fseMetadata, + workspace, wkspSize, writeSeqEntropy); + return cSizeEstimate + ZSTD_blockHeaderSize; +} + +static int ZSTD_needSequenceEntropyTables(ZSTD_fseCTablesMetadata_t const* fseMetadata) +{ + if (fseMetadata->llType == set_compressed || fseMetadata->llType == set_rle) + return 1; + if (fseMetadata->mlType == set_compressed || fseMetadata->mlType == set_rle) + return 1; + if (fseMetadata->ofType == set_compressed || fseMetadata->ofType == set_rle) + return 1; + return 0; +} + +/* ZSTD_compressSubBlock_multi() : + * Breaks super-block into multiple sub-blocks and compresses them. + * Entropy will be written to the first block. + * The following blocks will use repeat mode to compress. + * All sub-blocks are compressed blocks (no raw or rle blocks). + * @return : compressed size of the super block (which is multiple ZSTD blocks) + * Or 0 if it failed to compress. */ +static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr, + const ZSTD_compressedBlockState_t* prevCBlock, + ZSTD_compressedBlockState_t* nextCBlock, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const int bmi2, U32 lastBlock, + void* workspace, size_t wkspSize) +{ + const seqDef* const sstart = seqStorePtr->sequencesStart; + const seqDef* const send = seqStorePtr->sequences; + const seqDef* sp = sstart; + const BYTE* const lstart = seqStorePtr->litStart; + const BYTE* const lend = seqStorePtr->lit; + const BYTE* lp = lstart; + BYTE const* ip = (BYTE const*)src; + BYTE const* const iend = ip + srcSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + const BYTE* llCodePtr = seqStorePtr->llCode; + const BYTE* mlCodePtr = seqStorePtr->mlCode; + const BYTE* ofCodePtr = seqStorePtr->ofCode; + size_t targetCBlockSize = cctxParams->targetCBlockSize; + size_t litSize, seqCount; + int writeLitEntropy = entropyMetadata->hufMetadata.hType == set_compressed; + int writeSeqEntropy = 1; + int lastSequence = 0; + + DEBUGLOG(5, "ZSTD_compressSubBlock_multi (litSize=%u, nbSeq=%u)", + (unsigned)(lend-lp), (unsigned)(send-sstart)); + + litSize = 0; + seqCount = 0; + do { + size_t cBlockSizeEstimate = 0; + if (sstart == send) { + lastSequence = 1; + } else { + const seqDef* const sequence = sp + seqCount; + lastSequence = sequence == send - 1; + litSize += ZSTD_getSequenceLength(seqStorePtr, sequence).litLength; + seqCount++; + } + if (lastSequence) { + assert(lp <= lend); + assert(litSize <= (size_t)(lend - lp)); + litSize = (size_t)(lend - lp); + } + /* I think there is an optimization opportunity here. + * Calling ZSTD_estimateSubBlockSize for every sequence can be wasteful + * since it recalculates estimate from scratch. + * For example, it would recount literal distribution and symbol codes everytime. + */ + cBlockSizeEstimate = ZSTD_estimateSubBlockSize(lp, litSize, ofCodePtr, llCodePtr, mlCodePtr, seqCount, + &nextCBlock->entropy, entropyMetadata, + workspace, wkspSize, writeLitEntropy, writeSeqEntropy); + if (cBlockSizeEstimate > targetCBlockSize || lastSequence) { + int litEntropyWritten = 0; + int seqEntropyWritten = 0; + const size_t decompressedSize = ZSTD_seqDecompressedSize(seqStorePtr, sp, seqCount, litSize, lastSequence); + const size_t cSize = ZSTD_compressSubBlock(&nextCBlock->entropy, entropyMetadata, + sp, seqCount, + lp, litSize, + llCodePtr, mlCodePtr, ofCodePtr, + cctxParams, + op, oend-op, + bmi2, writeLitEntropy, writeSeqEntropy, + &litEntropyWritten, &seqEntropyWritten, + lastBlock && lastSequence); + FORWARD_IF_ERROR(cSize, "ZSTD_compressSubBlock failed"); + if (cSize > 0 && cSize < decompressedSize) { + DEBUGLOG(5, "Committed the sub-block"); + assert(ip + decompressedSize <= iend); + ip += decompressedSize; + sp += seqCount; + lp += litSize; + op += cSize; + llCodePtr += seqCount; + mlCodePtr += seqCount; + ofCodePtr += seqCount; + litSize = 0; + seqCount = 0; + /* Entropy only needs to be written once */ + if (litEntropyWritten) { + writeLitEntropy = 0; + } + if (seqEntropyWritten) { + writeSeqEntropy = 0; + } + } + } + } while (!lastSequence); + if (writeLitEntropy) { + DEBUGLOG(5, "ZSTD_compressSubBlock_multi has literal entropy tables unwritten"); + ZSTD_memcpy(&nextCBlock->entropy.huf, &prevCBlock->entropy.huf, sizeof(prevCBlock->entropy.huf)); + } + if (writeSeqEntropy && ZSTD_needSequenceEntropyTables(&entropyMetadata->fseMetadata)) { + /* If we haven't written our entropy tables, then we've violated our contract and + * must emit an uncompressed block. + */ + DEBUGLOG(5, "ZSTD_compressSubBlock_multi has sequence entropy tables unwritten"); + return 0; + } + if (ip < iend) { + size_t const cSize = ZSTD_noCompressBlock(op, oend - op, ip, iend - ip, lastBlock); + DEBUGLOG(5, "ZSTD_compressSubBlock_multi last sub-block uncompressed, %zu bytes", (size_t)(iend - ip)); + FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); + assert(cSize != 0); + op += cSize; + /* We have to regenerate the repcodes because we've skipped some sequences */ + if (sp < send) { + seqDef const* seq; + repcodes_t rep; + ZSTD_memcpy(&rep, prevCBlock->rep, sizeof(rep)); + for (seq = sstart; seq < sp; ++seq) { + rep = ZSTD_updateRep(rep.rep, seq->offset - 1, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0); + } + ZSTD_memcpy(nextCBlock->rep, &rep, sizeof(rep)); + } + } + DEBUGLOG(5, "ZSTD_compressSubBlock_multi compressed"); + return op-ostart; +} + +size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + void const* src, size_t srcSize, + unsigned lastBlock) { + ZSTD_entropyCTablesMetadata_t entropyMetadata; + + FORWARD_IF_ERROR(ZSTD_buildSuperBlockEntropy(&zc->seqStore, + &zc->blockState.prevCBlock->entropy, + &zc->blockState.nextCBlock->entropy, + &zc->appliedParams, + &entropyMetadata, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), ""); + + return ZSTD_compressSubBlock_multi(&zc->seqStore, + zc->blockState.prevCBlock, + zc->blockState.nextCBlock, + &entropyMetadata, + &zc->appliedParams, + dst, dstCapacity, + src, srcSize, + zc->bmi2, lastBlock, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */); +} diff --git a/lib/zstd/compress/zstd_compress_superblock.h b/lib/zstd/compress/zstd_compress_superblock.h new file mode 100644 index 000000000000..224ece79546e --- /dev/null +++ b/lib/zstd/compress/zstd_compress_superblock.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPRESS_ADVANCED_H +#define ZSTD_COMPRESS_ADVANCED_H + +/*-************************************* +* Dependencies +***************************************/ + +#include /* ZSTD_CCtx */ + +/*-************************************* +* Target Compressed Block Size +***************************************/ + +/* ZSTD_compressSuperBlock() : + * Used to compress a super block when targetCBlockSize is being used. + * The given block will be compressed into multiple sub blocks that are around targetCBlockSize. */ +size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + void const* src, size_t srcSize, + unsigned lastBlock); + +#endif /* ZSTD_COMPRESS_ADVANCED_H */ diff --git a/lib/zstd/compress/zstd_cwksp.h b/lib/zstd/compress/zstd_cwksp.h new file mode 100644 index 000000000000..98e359adf5d4 --- /dev/null +++ b/lib/zstd/compress/zstd_cwksp.h @@ -0,0 +1,482 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_CWKSP_H +#define ZSTD_CWKSP_H + +/*-************************************* +* Dependencies +***************************************/ +#include "../common/zstd_internal.h" + + +/*-************************************* +* Constants +***************************************/ + +/* Since the workspace is effectively its own little malloc implementation / + * arena, when we run under ASAN, we should similarly insert redzones between + * each internal element of the workspace, so ASAN will catch overruns that + * reach outside an object but that stay inside the workspace. + * + * This defines the size of that redzone. + */ +#ifndef ZSTD_CWKSP_ASAN_REDZONE_SIZE +#define ZSTD_CWKSP_ASAN_REDZONE_SIZE 128 +#endif + +/*-************************************* +* Structures +***************************************/ +typedef enum { + ZSTD_cwksp_alloc_objects, + ZSTD_cwksp_alloc_buffers, + ZSTD_cwksp_alloc_aligned +} ZSTD_cwksp_alloc_phase_e; + +/* + * Used to describe whether the workspace is statically allocated (and will not + * necessarily ever be freed), or if it's dynamically allocated and we can + * expect a well-formed caller to free this. + */ +typedef enum { + ZSTD_cwksp_dynamic_alloc, + ZSTD_cwksp_static_alloc +} ZSTD_cwksp_static_alloc_e; + +/* + * Zstd fits all its internal datastructures into a single continuous buffer, + * so that it only needs to perform a single OS allocation (or so that a buffer + * can be provided to it and it can perform no allocations at all). This buffer + * is called the workspace. + * + * Several optimizations complicate that process of allocating memory ranges + * from this workspace for each internal datastructure: + * + * - These different internal datastructures have different setup requirements: + * + * - The static objects need to be cleared once and can then be trivially + * reused for each compression. + * + * - Various buffers don't need to be initialized at all--they are always + * written into before they're read. + * + * - The matchstate tables have a unique requirement that they don't need + * their memory to be totally cleared, but they do need the memory to have + * some bound, i.e., a guarantee that all values in the memory they've been + * allocated is less than some maximum value (which is the starting value + * for the indices that they will then use for compression). When this + * guarantee is provided to them, they can use the memory without any setup + * work. When it can't, they have to clear the area. + * + * - These buffers also have different alignment requirements. + * + * - We would like to reuse the objects in the workspace for multiple + * compressions without having to perform any expensive reallocation or + * reinitialization work. + * + * - We would like to be able to efficiently reuse the workspace across + * multiple compressions **even when the compression parameters change** and + * we need to resize some of the objects (where possible). + * + * To attempt to manage this buffer, given these constraints, the ZSTD_cwksp + * abstraction was created. It works as follows: + * + * Workspace Layout: + * + * [ ... workspace ... ] + * [objects][tables ... ->] free space [<- ... aligned][<- ... buffers] + * + * The various objects that live in the workspace are divided into the + * following categories, and are allocated separately: + * + * - Static objects: this is optionally the enclosing ZSTD_CCtx or ZSTD_CDict, + * so that literally everything fits in a single buffer. Note: if present, + * this must be the first object in the workspace, since ZSTD_customFree{CCtx, + * CDict}() rely on a pointer comparison to see whether one or two frees are + * required. + * + * - Fixed size objects: these are fixed-size, fixed-count objects that are + * nonetheless "dynamically" allocated in the workspace so that we can + * control how they're initialized separately from the broader ZSTD_CCtx. + * Examples: + * - Entropy Workspace + * - 2 x ZSTD_compressedBlockState_t + * - CDict dictionary contents + * + * - Tables: these are any of several different datastructures (hash tables, + * chain tables, binary trees) that all respect a common format: they are + * uint32_t arrays, all of whose values are between 0 and (nextSrc - base). + * Their sizes depend on the cparams. + * + * - Aligned: these buffers are used for various purposes that require 4 byte + * alignment, but don't require any initialization before they're used. + * + * - Buffers: these buffers are used for various purposes that don't require + * any alignment or initialization before they're used. This means they can + * be moved around at no cost for a new compression. + * + * Allocating Memory: + * + * The various types of objects must be allocated in order, so they can be + * correctly packed into the workspace buffer. That order is: + * + * 1. Objects + * 2. Buffers + * 3. Aligned + * 4. Tables + * + * Attempts to reserve objects of different types out of order will fail. + */ +typedef struct { + void* workspace; + void* workspaceEnd; + + void* objectEnd; + void* tableEnd; + void* tableValidEnd; + void* allocStart; + + BYTE allocFailed; + int workspaceOversizedDuration; + ZSTD_cwksp_alloc_phase_e phase; + ZSTD_cwksp_static_alloc_e isStatic; +} ZSTD_cwksp; + +/*-************************************* +* Functions +***************************************/ + +MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws); + +MEM_STATIC void ZSTD_cwksp_assert_internal_consistency(ZSTD_cwksp* ws) { + (void)ws; + assert(ws->workspace <= ws->objectEnd); + assert(ws->objectEnd <= ws->tableEnd); + assert(ws->objectEnd <= ws->tableValidEnd); + assert(ws->tableEnd <= ws->allocStart); + assert(ws->tableValidEnd <= ws->allocStart); + assert(ws->allocStart <= ws->workspaceEnd); +} + +/* + * Align must be a power of 2. + */ +MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) { + size_t const mask = align - 1; + assert((align & mask) == 0); + return (size + mask) & ~mask; +} + +/* + * Use this to determine how much space in the workspace we will consume to + * allocate this object. (Normally it should be exactly the size of the object, + * but under special conditions, like ASAN, where we pad each object, it might + * be larger.) + * + * Since tables aren't currently redzoned, you don't need to call through this + * to figure out how much space you need for the matchState tables. Everything + * else is though. + */ +MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) { + if (size == 0) + return 0; + return size; +} + +MEM_STATIC void ZSTD_cwksp_internal_advance_phase( + ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase) { + assert(phase >= ws->phase); + if (phase > ws->phase) { + if (ws->phase < ZSTD_cwksp_alloc_buffers && + phase >= ZSTD_cwksp_alloc_buffers) { + ws->tableValidEnd = ws->objectEnd; + } + if (ws->phase < ZSTD_cwksp_alloc_aligned && + phase >= ZSTD_cwksp_alloc_aligned) { + /* If unaligned allocations down from a too-large top have left us + * unaligned, we need to realign our alloc ptr. Technically, this + * can consume space that is unaccounted for in the neededSpace + * calculation. However, I believe this can only happen when the + * workspace is too large, and specifically when it is too large + * by a larger margin than the space that will be consumed. */ + /* TODO: cleaner, compiler warning friendly way to do this??? */ + ws->allocStart = (BYTE*)ws->allocStart - ((size_t)ws->allocStart & (sizeof(U32)-1)); + if (ws->allocStart < ws->tableValidEnd) { + ws->tableValidEnd = ws->allocStart; + } + } + ws->phase = phase; + } +} + +/* + * Returns whether this object/buffer/etc was allocated in this workspace. + */ +MEM_STATIC int ZSTD_cwksp_owns_buffer(const ZSTD_cwksp* ws, const void* ptr) { + return (ptr != NULL) && (ws->workspace <= ptr) && (ptr <= ws->workspaceEnd); +} + +/* + * Internal function. Do not use directly. + */ +MEM_STATIC void* ZSTD_cwksp_reserve_internal( + ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) { + void* alloc; + void* bottom = ws->tableEnd; + ZSTD_cwksp_internal_advance_phase(ws, phase); + alloc = (BYTE *)ws->allocStart - bytes; + + if (bytes == 0) + return NULL; + + + DEBUGLOG(5, "cwksp: reserving %p %zd bytes, %zd bytes remaining", + alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes); + ZSTD_cwksp_assert_internal_consistency(ws); + assert(alloc >= bottom); + if (alloc < bottom) { + DEBUGLOG(4, "cwksp: alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + if (alloc < ws->tableValidEnd) { + ws->tableValidEnd = alloc; + } + ws->allocStart = alloc; + + + return alloc; +} + +/* + * Reserves and returns unaligned memory. + */ +MEM_STATIC BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) { + return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers); +} + +/* + * Reserves and returns memory sized on and aligned on sizeof(unsigned). + */ +MEM_STATIC void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) { + assert((bytes & (sizeof(U32)-1)) == 0); + return ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, sizeof(U32)), ZSTD_cwksp_alloc_aligned); +} + +/* + * Aligned on sizeof(unsigned). These buffers have the special property that + * their values remain constrained, allowing us to re-use them without + * memset()-ing them. + */ +MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { + const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned; + void* alloc = ws->tableEnd; + void* end = (BYTE *)alloc + bytes; + void* top = ws->allocStart; + + DEBUGLOG(5, "cwksp: reserving %p table %zd bytes, %zd bytes remaining", + alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes); + assert((bytes & (sizeof(U32)-1)) == 0); + ZSTD_cwksp_internal_advance_phase(ws, phase); + ZSTD_cwksp_assert_internal_consistency(ws); + assert(end <= top); + if (end > top) { + DEBUGLOG(4, "cwksp: table alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + ws->tableEnd = end; + + + return alloc; +} + +/* + * Aligned on sizeof(void*). + */ +MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { + size_t roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*)); + void* alloc = ws->objectEnd; + void* end = (BYTE*)alloc + roundedBytes; + + + DEBUGLOG(5, + "cwksp: reserving %p object %zd bytes (rounded to %zd), %zd bytes remaining", + alloc, bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes); + assert(((size_t)alloc & (sizeof(void*)-1)) == 0); + assert((bytes & (sizeof(void*)-1)) == 0); + ZSTD_cwksp_assert_internal_consistency(ws); + /* we must be in the first phase, no advance is possible */ + if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) { + DEBUGLOG(4, "cwksp: object alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + ws->objectEnd = end; + ws->tableEnd = end; + ws->tableValidEnd = end; + + + return alloc; +} + +MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_dirty"); + + + assert(ws->tableValidEnd >= ws->objectEnd); + assert(ws->tableValidEnd <= ws->allocStart); + ws->tableValidEnd = ws->objectEnd; + ZSTD_cwksp_assert_internal_consistency(ws); +} + +MEM_STATIC void ZSTD_cwksp_mark_tables_clean(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_clean"); + assert(ws->tableValidEnd >= ws->objectEnd); + assert(ws->tableValidEnd <= ws->allocStart); + if (ws->tableValidEnd < ws->tableEnd) { + ws->tableValidEnd = ws->tableEnd; + } + ZSTD_cwksp_assert_internal_consistency(ws); +} + +/* + * Zero the part of the allocated tables not already marked clean. + */ +MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: ZSTD_cwksp_clean_tables"); + assert(ws->tableValidEnd >= ws->objectEnd); + assert(ws->tableValidEnd <= ws->allocStart); + if (ws->tableValidEnd < ws->tableEnd) { + ZSTD_memset(ws->tableValidEnd, 0, (BYTE*)ws->tableEnd - (BYTE*)ws->tableValidEnd); + } + ZSTD_cwksp_mark_tables_clean(ws); +} + +/* + * Invalidates table allocations. + * All other allocations remain valid. + */ +MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: clearing tables!"); + + + ws->tableEnd = ws->objectEnd; + ZSTD_cwksp_assert_internal_consistency(ws); +} + +/* + * Invalidates all buffer, aligned, and table allocations. + * Object allocations remain valid. + */ +MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: clearing!"); + + + + ws->tableEnd = ws->objectEnd; + ws->allocStart = ws->workspaceEnd; + ws->allocFailed = 0; + if (ws->phase > ZSTD_cwksp_alloc_buffers) { + ws->phase = ZSTD_cwksp_alloc_buffers; + } + ZSTD_cwksp_assert_internal_consistency(ws); +} + +/* + * The provided workspace takes ownership of the buffer [start, start+size). + * Any existing values in the workspace are ignored (the previously managed + * buffer, if present, must be separately freed). + */ +MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size, ZSTD_cwksp_static_alloc_e isStatic) { + DEBUGLOG(4, "cwksp: init'ing workspace with %zd bytes", size); + assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ + ws->workspace = start; + ws->workspaceEnd = (BYTE*)start + size; + ws->objectEnd = ws->workspace; + ws->tableValidEnd = ws->objectEnd; + ws->phase = ZSTD_cwksp_alloc_objects; + ws->isStatic = isStatic; + ZSTD_cwksp_clear(ws); + ws->workspaceOversizedDuration = 0; + ZSTD_cwksp_assert_internal_consistency(ws); +} + +MEM_STATIC size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) { + void* workspace = ZSTD_customMalloc(size, customMem); + DEBUGLOG(4, "cwksp: creating new workspace with %zd bytes", size); + RETURN_ERROR_IF(workspace == NULL, memory_allocation, "NULL pointer!"); + ZSTD_cwksp_init(ws, workspace, size, ZSTD_cwksp_dynamic_alloc); + return 0; +} + +MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { + void *ptr = ws->workspace; + DEBUGLOG(4, "cwksp: freeing workspace"); + ZSTD_memset(ws, 0, sizeof(ZSTD_cwksp)); + ZSTD_customFree(ptr, customMem); +} + +/* + * Moves the management of a workspace from one cwksp to another. The src cwksp + * is left in an invalid state (src must be re-init()'ed before it's used again). + */ +MEM_STATIC void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) { + *dst = *src; + ZSTD_memset(src, 0, sizeof(ZSTD_cwksp)); +} + +MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace); +} + +MEM_STATIC size_t ZSTD_cwksp_used(const ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->tableEnd - (BYTE*)ws->workspace) + + (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->allocStart); +} + +MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { + return ws->allocFailed; +} + +/*-************************************* +* Functions Checking Free Space +***************************************/ + +MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd); +} + +MEM_STATIC int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_available_space(ws) >= additionalNeededSpace; +} + +MEM_STATIC int ZSTD_cwksp_check_too_large(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_check_available( + ws, additionalNeededSpace * ZSTD_WORKSPACETOOLARGE_FACTOR); +} + +MEM_STATIC int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_check_too_large(ws, additionalNeededSpace) + && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; +} + +MEM_STATIC void ZSTD_cwksp_bump_oversized_duration( + ZSTD_cwksp* ws, size_t additionalNeededSpace) { + if (ZSTD_cwksp_check_too_large(ws, additionalNeededSpace)) { + ws->workspaceOversizedDuration++; + } else { + ws->workspaceOversizedDuration = 0; + } +} + + +#endif /* ZSTD_CWKSP_H */ diff --git a/lib/zstd/compress/zstd_double_fast.c b/lib/zstd/compress/zstd_double_fast.c new file mode 100644 index 000000000000..b99172e9d2e4 --- /dev/null +++ b/lib/zstd/compress/zstd_double_fast.c @@ -0,0 +1,521 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_compress_internal.h" +#include "zstd_double_fast.h" + + +void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, + void const* end, ZSTD_dictTableLoadMethod_e dtlm) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashLarge = ms->hashTable; + U32 const hBitsL = cParams->hashLog; + U32 const mls = cParams->minMatch; + U32* const hashSmall = ms->chainTable; + U32 const hBitsS = cParams->chainLog; + const BYTE* const base = ms->window.base; + const BYTE* ip = base + ms->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const U32 fastHashFillStep = 3; + + /* Always insert every fastHashFillStep position into the hash tables. + * Insert the other positions into the large hash table if their entry + * is empty. + */ + for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) { + U32 const curr = (U32)(ip - base); + U32 i; + for (i = 0; i < fastHashFillStep; ++i) { + size_t const smHash = ZSTD_hashPtr(ip + i, hBitsS, mls); + size_t const lgHash = ZSTD_hashPtr(ip + i, hBitsL, 8); + if (i == 0) + hashSmall[smHash] = curr + i; + if (i == 0 || hashLarge[lgHash] == 0) + hashLarge[lgHash] = curr + i; + /* Only load extra positions for ZSTD_dtlm_full */ + if (dtlm == ZSTD_dtlm_fast) + break; + } } +} + + +FORCE_INLINE_TEMPLATE +size_t ZSTD_compressBlock_doubleFast_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, + U32 const mls /* template */, ZSTD_dictMode_e const dictMode) +{ + ZSTD_compressionParameters const* cParams = &ms->cParams; + U32* const hashLong = ms->hashTable; + const U32 hBitsL = cParams->hashLog; + U32* const hashSmall = ms->chainTable; + const U32 hBitsS = cParams->chainLog; + const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + /* presumes that, if there is a dictionary, it must be using Attach mode */ + const U32 prefixLowestIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog); + const BYTE* const prefixLowest = base + prefixLowestIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + U32 offset_1=rep[0], offset_2=rep[1]; + U32 offsetSaved = 0; + + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const ZSTD_compressionParameters* const dictCParams = + dictMode == ZSTD_dictMatchState ? + &dms->cParams : NULL; + const U32* const dictHashLong = dictMode == ZSTD_dictMatchState ? + dms->hashTable : NULL; + const U32* const dictHashSmall = dictMode == ZSTD_dictMatchState ? + dms->chainTable : NULL; + const U32 dictStartIndex = dictMode == ZSTD_dictMatchState ? + dms->window.dictLimit : 0; + const BYTE* const dictBase = dictMode == ZSTD_dictMatchState ? + dms->window.base : NULL; + const BYTE* const dictStart = dictMode == ZSTD_dictMatchState ? + dictBase + dictStartIndex : NULL; + const BYTE* const dictEnd = dictMode == ZSTD_dictMatchState ? + dms->window.nextSrc : NULL; + const U32 dictIndexDelta = dictMode == ZSTD_dictMatchState ? + prefixLowestIndex - (U32)(dictEnd - dictBase) : + 0; + const U32 dictHBitsL = dictMode == ZSTD_dictMatchState ? + dictCParams->hashLog : hBitsL; + const U32 dictHBitsS = dictMode == ZSTD_dictMatchState ? + dictCParams->chainLog : hBitsS; + const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictStart)); + + DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_generic"); + + assert(dictMode == ZSTD_noDict || dictMode == ZSTD_dictMatchState); + + /* if a dictionary is attached, it must be within window range */ + if (dictMode == ZSTD_dictMatchState) { + assert(ms->window.dictLimit + (1U << cParams->windowLog) >= endIndex); + } + + /* init */ + ip += (dictAndPrefixLength == 0); + if (dictMode == ZSTD_noDict) { + U32 const curr = (U32)(ip - base); + U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, cParams->windowLog); + U32 const maxRep = curr - windowLow; + if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; + if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; + } + if (dictMode == ZSTD_dictMatchState) { + /* dictMatchState repCode checks don't currently handle repCode == 0 + * disabling. */ + assert(offset_1 <= dictAndPrefixLength); + assert(offset_2 <= dictAndPrefixLength); + } + + /* Main Search Loop */ + while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ + size_t mLength; + U32 offset; + size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8); + size_t const h = ZSTD_hashPtr(ip, hBitsS, mls); + size_t const dictHL = ZSTD_hashPtr(ip, dictHBitsL, 8); + size_t const dictHS = ZSTD_hashPtr(ip, dictHBitsS, mls); + U32 const curr = (U32)(ip-base); + U32 const matchIndexL = hashLong[h2]; + U32 matchIndexS = hashSmall[h]; + const BYTE* matchLong = base + matchIndexL; + const BYTE* match = base + matchIndexS; + const U32 repIndex = curr + 1 - offset_1; + const BYTE* repMatch = (dictMode == ZSTD_dictMatchState + && repIndex < prefixLowestIndex) ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + hashLong[h2] = hashSmall[h] = curr; /* update hash tables */ + + /* check dictMatchState repcode */ + if (dictMode == ZSTD_dictMatchState + && ((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + ip++; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH); + goto _match_stored; + } + + /* check noDict repcode */ + if ( dictMode == ZSTD_noDict + && ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) { + mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; + ip++; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH); + goto _match_stored; + } + + if (matchIndexL > prefixLowestIndex) { + /* check prefix long match */ + if (MEM_read64(matchLong) == MEM_read64(ip)) { + mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8; + offset = (U32)(ip-matchLong); + while (((ip>anchor) & (matchLong>prefixLowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ + goto _match_found; + } + } else if (dictMode == ZSTD_dictMatchState) { + /* check dictMatchState long match */ + U32 const dictMatchIndexL = dictHashLong[dictHL]; + const BYTE* dictMatchL = dictBase + dictMatchIndexL; + assert(dictMatchL < dictEnd); + + if (dictMatchL > dictStart && MEM_read64(dictMatchL) == MEM_read64(ip)) { + mLength = ZSTD_count_2segments(ip+8, dictMatchL+8, iend, dictEnd, prefixLowest) + 8; + offset = (U32)(curr - dictMatchIndexL - dictIndexDelta); + while (((ip>anchor) & (dictMatchL>dictStart)) && (ip[-1] == dictMatchL[-1])) { ip--; dictMatchL--; mLength++; } /* catch up */ + goto _match_found; + } } + + if (matchIndexS > prefixLowestIndex) { + /* check prefix short match */ + if (MEM_read32(match) == MEM_read32(ip)) { + goto _search_next_long; + } + } else if (dictMode == ZSTD_dictMatchState) { + /* check dictMatchState short match */ + U32 const dictMatchIndexS = dictHashSmall[dictHS]; + match = dictBase + dictMatchIndexS; + matchIndexS = dictMatchIndexS + dictIndexDelta; + + if (match > dictStart && MEM_read32(match) == MEM_read32(ip)) { + goto _search_next_long; + } } + + ip += ((ip-anchor) >> kSearchStrength) + 1; +#if defined(__aarch64__) + PREFETCH_L1(ip+256); +#endif + continue; + +_search_next_long: + + { size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8); + size_t const dictHLNext = ZSTD_hashPtr(ip+1, dictHBitsL, 8); + U32 const matchIndexL3 = hashLong[hl3]; + const BYTE* matchL3 = base + matchIndexL3; + hashLong[hl3] = curr + 1; + + /* check prefix long +1 match */ + if (matchIndexL3 > prefixLowestIndex) { + if (MEM_read64(matchL3) == MEM_read64(ip+1)) { + mLength = ZSTD_count(ip+9, matchL3+8, iend) + 8; + ip++; + offset = (U32)(ip-matchL3); + while (((ip>anchor) & (matchL3>prefixLowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */ + goto _match_found; + } + } else if (dictMode == ZSTD_dictMatchState) { + /* check dict long +1 match */ + U32 const dictMatchIndexL3 = dictHashLong[dictHLNext]; + const BYTE* dictMatchL3 = dictBase + dictMatchIndexL3; + assert(dictMatchL3 < dictEnd); + if (dictMatchL3 > dictStart && MEM_read64(dictMatchL3) == MEM_read64(ip+1)) { + mLength = ZSTD_count_2segments(ip+1+8, dictMatchL3+8, iend, dictEnd, prefixLowest) + 8; + ip++; + offset = (U32)(curr + 1 - dictMatchIndexL3 - dictIndexDelta); + while (((ip>anchor) & (dictMatchL3>dictStart)) && (ip[-1] == dictMatchL3[-1])) { ip--; dictMatchL3--; mLength++; } /* catch up */ + goto _match_found; + } } } + + /* if no long +1 match, explore the short match we found */ + if (dictMode == ZSTD_dictMatchState && matchIndexS < prefixLowestIndex) { + mLength = ZSTD_count_2segments(ip+4, match+4, iend, dictEnd, prefixLowest) + 4; + offset = (U32)(curr - matchIndexS); + while (((ip>anchor) & (match>dictStart)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } else { + mLength = ZSTD_count(ip+4, match+4, iend) + 4; + offset = (U32)(ip - match); + while (((ip>anchor) & (match>prefixLowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } + + /* fall-through */ + +_match_found: + offset_2 = offset_1; + offset_1 = offset; + + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + +_match_stored: + /* match found */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Complementary insertion */ + /* done after iLimit test, as candidates could be > iend-8 */ + { U32 const indexToInsert = curr+2; + hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert; + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert; + hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base); + } + + /* check immediate repcode */ + if (dictMode == ZSTD_dictMatchState) { + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = dictMode == ZSTD_dictMatchState + && repIndex2 < prefixLowestIndex ? + dictBase + repIndex2 - dictIndexDelta : + base + repIndex2; + if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */) + && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4; + U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, repLength2-MINMATCH); + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; + ip += repLength2; + anchor = ip; + continue; + } + break; + } } + + if (dictMode == ZSTD_noDict) { + while ( (ip <= ilimit) + && ( (offset_2>0) + & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) { + /* store sequence */ + size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; + U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */ + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base); + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base); + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, rLength-MINMATCH); + ip += rLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } } } + } /* while (ip < ilimit) */ + + /* save reps for next block */ + rep[0] = offset_1 ? offset_1 : offsetSaved; + rep[1] = offset_2 ? offset_2 : offsetSaved; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + + +size_t ZSTD_compressBlock_doubleFast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + const U32 mls = ms->cParams.minMatch; + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_noDict); + case 5 : + return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_noDict); + case 6 : + return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_noDict); + case 7 : + return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_noDict); + } +} + + +size_t ZSTD_compressBlock_doubleFast_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + const U32 mls = ms->cParams.minMatch; + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_dictMatchState); + case 5 : + return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_dictMatchState); + case 6 : + return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_dictMatchState); + case 7 : + return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_dictMatchState); + } +} + + +static size_t ZSTD_compressBlock_doubleFast_extDict_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, + U32 const mls /* template */) +{ + ZSTD_compressionParameters const* cParams = &ms->cParams; + U32* const hashLong = ms->hashTable; + U32 const hBitsL = cParams->hashLog; + U32* const hashSmall = ms->chainTable; + U32 const hBitsS = cParams->chainLog; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ms->window.base; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog); + const U32 dictStartIndex = lowLimit; + const U32 dictLimit = ms->window.dictLimit; + const U32 prefixStartIndex = (dictLimit > lowLimit) ? dictLimit : lowLimit; + const BYTE* const prefixStart = base + prefixStartIndex; + const BYTE* const dictBase = ms->window.dictBase; + const BYTE* const dictStart = dictBase + dictStartIndex; + const BYTE* const dictEnd = dictBase + prefixStartIndex; + U32 offset_1=rep[0], offset_2=rep[1]; + + DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_extDict_generic (srcSize=%zu)", srcSize); + + /* if extDict is invalidated due to maxDistance, switch to "regular" variant */ + if (prefixStartIndex == dictStartIndex) + return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, mls, ZSTD_noDict); + + /* Search Loop */ + while (ip < ilimit) { /* < instead of <=, because (ip+1) */ + const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls); + const U32 matchIndex = hashSmall[hSmall]; + const BYTE* const matchBase = matchIndex < prefixStartIndex ? dictBase : base; + const BYTE* match = matchBase + matchIndex; + + const size_t hLong = ZSTD_hashPtr(ip, hBitsL, 8); + const U32 matchLongIndex = hashLong[hLong]; + const BYTE* const matchLongBase = matchLongIndex < prefixStartIndex ? dictBase : base; + const BYTE* matchLong = matchLongBase + matchLongIndex; + + const U32 curr = (U32)(ip-base); + const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */ + const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + size_t mLength; + hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */ + + if ((((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex doesn't overlap dict + prefix */ + & (repIndex > dictStartIndex)) + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; + ip++; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH); + } else { + if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { + const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend; + const BYTE* const lowMatchPtr = matchLongIndex < prefixStartIndex ? dictStart : prefixStart; + U32 offset; + mLength = ZSTD_count_2segments(ip+8, matchLong+8, iend, matchEnd, prefixStart) + 8; + offset = curr - matchLongIndex; + while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + + } else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) { + size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); + U32 const matchIndex3 = hashLong[h3]; + const BYTE* const match3Base = matchIndex3 < prefixStartIndex ? dictBase : base; + const BYTE* match3 = match3Base + matchIndex3; + U32 offset; + hashLong[h3] = curr + 1; + if ( (matchIndex3 > dictStartIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) { + const BYTE* const matchEnd = matchIndex3 < prefixStartIndex ? dictEnd : iend; + const BYTE* const lowMatchPtr = matchIndex3 < prefixStartIndex ? dictStart : prefixStart; + mLength = ZSTD_count_2segments(ip+9, match3+8, iend, matchEnd, prefixStart) + 8; + ip++; + offset = curr+1 - matchIndex3; + while (((ip>anchor) & (match3>lowMatchPtr)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */ + } else { + const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend; + const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart; + mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4; + offset = curr - matchIndex; + while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + + } else { + ip += ((ip-anchor) >> kSearchStrength) + 1; + continue; + } } + + /* move to next sequence start */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Complementary insertion */ + /* done after iLimit test, as candidates could be > iend-8 */ + { U32 const indexToInsert = curr+2; + hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert; + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert; + hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base); + } + + /* check immediate repcode */ + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; + if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) /* intentional overflow : ensure repIndex2 doesn't overlap dict + prefix */ + & (repIndex2 > dictStartIndex)) + && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; + U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, repLength2-MINMATCH); + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; + ip += repLength2; + anchor = ip; + continue; + } + break; + } } } + + /* save reps for next block */ + rep[0] = offset_1; + rep[1] = offset_2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + + +size_t ZSTD_compressBlock_doubleFast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + U32 const mls = ms->cParams.minMatch; + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 4); + case 5 : + return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 5); + case 6 : + return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 6); + case 7 : + return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 7); + } +} diff --git a/lib/zstd/compress/zstd_double_fast.h b/lib/zstd/compress/zstd_double_fast.h new file mode 100644 index 000000000000..6822bde65a1d --- /dev/null +++ b/lib/zstd/compress/zstd_double_fast.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_DOUBLE_FAST_H +#define ZSTD_DOUBLE_FAST_H + + +#include "../common/mem.h" /* U32 */ +#include "zstd_compress_internal.h" /* ZSTD_CCtx, size_t */ + +void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, + void const* end, ZSTD_dictTableLoadMethod_e dtlm); +size_t ZSTD_compressBlock_doubleFast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_doubleFast_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_doubleFast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + + + +#endif /* ZSTD_DOUBLE_FAST_H */ diff --git a/lib/zstd/compress/zstd_fast.c b/lib/zstd/compress/zstd_fast.c new file mode 100644 index 000000000000..96b7d48e2868 --- /dev/null +++ b/lib/zstd/compress/zstd_fast.c @@ -0,0 +1,496 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_compress_internal.h" /* ZSTD_hashPtr, ZSTD_count, ZSTD_storeSeq */ +#include "zstd_fast.h" + + +void ZSTD_fillHashTable(ZSTD_matchState_t* ms, + const void* const end, + ZSTD_dictTableLoadMethod_e dtlm) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hBits = cParams->hashLog; + U32 const mls = cParams->minMatch; + const BYTE* const base = ms->window.base; + const BYTE* ip = base + ms->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const U32 fastHashFillStep = 3; + + /* Always insert every fastHashFillStep position into the hash table. + * Insert the other positions if their hash entry is empty. + */ + for ( ; ip + fastHashFillStep < iend + 2; ip += fastHashFillStep) { + U32 const curr = (U32)(ip - base); + size_t const hash0 = ZSTD_hashPtr(ip, hBits, mls); + hashTable[hash0] = curr; + if (dtlm == ZSTD_dtlm_fast) continue; + /* Only load extra positions for ZSTD_dtlm_full */ + { U32 p; + for (p = 1; p < fastHashFillStep; ++p) { + size_t const hash = ZSTD_hashPtr(ip + p, hBits, mls); + if (hashTable[hash] == 0) { /* not yet filled */ + hashTable[hash] = curr + p; + } } } } +} + + +FORCE_INLINE_TEMPLATE size_t +ZSTD_compressBlock_fast_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, + U32 const mls) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hlog = cParams->hashLog; + /* support stepSize of 0 */ + size_t const stepSize = cParams->targetLength + !(cParams->targetLength) + 1; + const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + /* We check ip0 (ip + 0) and ip1 (ip + 1) each loop */ + const BYTE* ip0 = istart; + const BYTE* ip1; + const BYTE* anchor = istart; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + const U32 prefixStartIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog); + const BYTE* const prefixStart = base + prefixStartIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + U32 offset_1=rep[0], offset_2=rep[1]; + U32 offsetSaved = 0; + + /* init */ + DEBUGLOG(5, "ZSTD_compressBlock_fast_generic"); + ip0 += (ip0 == prefixStart); + ip1 = ip0 + 1; + { U32 const curr = (U32)(ip0 - base); + U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, cParams->windowLog); + U32 const maxRep = curr - windowLow; + if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; + if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; + } + + /* Main Search Loop */ +#ifdef __INTEL_COMPILER + /* From intel 'The vector pragma indicates that the loop should be + * vectorized if it is legal to do so'. Can be used together with + * #pragma ivdep (but have opted to exclude that because intel + * warns against using it).*/ + #pragma vector always +#endif + while (ip1 < ilimit) { /* < instead of <=, because check at ip0+2 */ + size_t mLength; + BYTE const* ip2 = ip0 + 2; + size_t const h0 = ZSTD_hashPtr(ip0, hlog, mls); + U32 const val0 = MEM_read32(ip0); + size_t const h1 = ZSTD_hashPtr(ip1, hlog, mls); + U32 const val1 = MEM_read32(ip1); + U32 const current0 = (U32)(ip0-base); + U32 const current1 = (U32)(ip1-base); + U32 const matchIndex0 = hashTable[h0]; + U32 const matchIndex1 = hashTable[h1]; + BYTE const* repMatch = ip2 - offset_1; + const BYTE* match0 = base + matchIndex0; + const BYTE* match1 = base + matchIndex1; + U32 offcode; + +#if defined(__aarch64__) + PREFETCH_L1(ip0+256); +#endif + + hashTable[h0] = current0; /* update hash table */ + hashTable[h1] = current1; /* update hash table */ + + assert(ip0 + 1 == ip1); + + if ((offset_1 > 0) & (MEM_read32(repMatch) == MEM_read32(ip2))) { + mLength = (ip2[-1] == repMatch[-1]) ? 1 : 0; + ip0 = ip2 - mLength; + match0 = repMatch - mLength; + mLength += 4; + offcode = 0; + goto _match; + } + if ((matchIndex0 > prefixStartIndex) && MEM_read32(match0) == val0) { + /* found a regular match */ + goto _offset; + } + if ((matchIndex1 > prefixStartIndex) && MEM_read32(match1) == val1) { + /* found a regular match after one literal */ + ip0 = ip1; + match0 = match1; + goto _offset; + } + { size_t const step = ((size_t)(ip0-anchor) >> (kSearchStrength - 1)) + stepSize; + assert(step >= 2); + ip0 += step; + ip1 += step; + continue; + } +_offset: /* Requires: ip0, match0 */ + /* Compute the offset code */ + offset_2 = offset_1; + offset_1 = (U32)(ip0-match0); + offcode = offset_1 + ZSTD_REP_MOVE; + mLength = 4; + /* Count the backwards match length */ + while (((ip0>anchor) & (match0>prefixStart)) + && (ip0[-1] == match0[-1])) { ip0--; match0--; mLength++; } /* catch up */ + +_match: /* Requires: ip0, match0, offcode */ + /* Count the forward length */ + mLength += ZSTD_count(ip0+mLength, match0+mLength, iend); + ZSTD_storeSeq(seqStore, (size_t)(ip0-anchor), anchor, iend, offcode, mLength-MINMATCH); + /* match found */ + ip0 += mLength; + anchor = ip0; + + if (ip0 <= ilimit) { + /* Fill Table */ + assert(base+current0+2 > istart); /* check base overflow */ + hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */ + hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); + + if (offset_2 > 0) { /* offset_2==0 means offset_2 is invalidated */ + while ( (ip0 <= ilimit) && (MEM_read32(ip0) == MEM_read32(ip0 - offset_2)) ) { + /* store sequence */ + size_t const rLength = ZSTD_count(ip0+4, ip0+4-offset_2, iend) + 4; + { U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */ + hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base); + ip0 += rLength; + ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, iend, 0 /*offCode*/, rLength-MINMATCH); + anchor = ip0; + continue; /* faster when present (confirmed on gcc-8) ... (?) */ + } } } + ip1 = ip0 + 1; + } + + /* save reps for next block */ + rep[0] = offset_1 ? offset_1 : offsetSaved; + rep[1] = offset_2 ? offset_2 : offsetSaved; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + + +size_t ZSTD_compressBlock_fast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + U32 const mls = ms->cParams.minMatch; + assert(ms->dictMatchState == NULL); + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 4); + case 5 : + return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 5); + case 6 : + return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 6); + case 7 : + return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 7); + } +} + +FORCE_INLINE_TEMPLATE +size_t ZSTD_compressBlock_fast_dictMatchState_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, U32 const mls) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hlog = cParams->hashLog; + /* support stepSize of 0 */ + U32 const stepSize = cParams->targetLength + !(cParams->targetLength); + const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 prefixStartIndex = ms->window.dictLimit; + const BYTE* const prefixStart = base + prefixStartIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + U32 offset_1=rep[0], offset_2=rep[1]; + U32 offsetSaved = 0; + + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const ZSTD_compressionParameters* const dictCParams = &dms->cParams ; + const U32* const dictHashTable = dms->hashTable; + const U32 dictStartIndex = dms->window.dictLimit; + const BYTE* const dictBase = dms->window.base; + const BYTE* const dictStart = dictBase + dictStartIndex; + const BYTE* const dictEnd = dms->window.nextSrc; + const U32 dictIndexDelta = prefixStartIndex - (U32)(dictEnd - dictBase); + const U32 dictAndPrefixLength = (U32)(ip - prefixStart + dictEnd - dictStart); + const U32 dictHLog = dictCParams->hashLog; + + /* if a dictionary is still attached, it necessarily means that + * it is within window size. So we just check it. */ + const U32 maxDistance = 1U << cParams->windowLog; + const U32 endIndex = (U32)((size_t)(ip - base) + srcSize); + assert(endIndex - prefixStartIndex <= maxDistance); + (void)maxDistance; (void)endIndex; /* these variables are not used when assert() is disabled */ + + /* ensure there will be no underflow + * when translating a dict index into a local index */ + assert(prefixStartIndex >= (U32)(dictEnd - dictBase)); + + /* init */ + DEBUGLOG(5, "ZSTD_compressBlock_fast_dictMatchState_generic"); + ip += (dictAndPrefixLength == 0); + /* dictMatchState repCode checks don't currently handle repCode == 0 + * disabling. */ + assert(offset_1 <= dictAndPrefixLength); + assert(offset_2 <= dictAndPrefixLength); + + /* Main Search Loop */ + while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ + size_t mLength; + size_t const h = ZSTD_hashPtr(ip, hlog, mls); + U32 const curr = (U32)(ip-base); + U32 const matchIndex = hashTable[h]; + const BYTE* match = base + matchIndex; + const U32 repIndex = curr + 1 - offset_1; + const BYTE* repMatch = (repIndex < prefixStartIndex) ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + hashTable[h] = curr; /* update hash table */ + + if ( ((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex isn't overlapping dict + prefix */ + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; + ip++; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH); + } else if ( (matchIndex <= prefixStartIndex) ) { + size_t const dictHash = ZSTD_hashPtr(ip, dictHLog, mls); + U32 const dictMatchIndex = dictHashTable[dictHash]; + const BYTE* dictMatch = dictBase + dictMatchIndex; + if (dictMatchIndex <= dictStartIndex || + MEM_read32(dictMatch) != MEM_read32(ip)) { + assert(stepSize >= 1); + ip += ((ip-anchor) >> kSearchStrength) + stepSize; + continue; + } else { + /* found a dict match */ + U32 const offset = (U32)(curr-dictMatchIndex-dictIndexDelta); + mLength = ZSTD_count_2segments(ip+4, dictMatch+4, iend, dictEnd, prefixStart) + 4; + while (((ip>anchor) & (dictMatch>dictStart)) + && (ip[-1] == dictMatch[-1])) { + ip--; dictMatch--; mLength++; + } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + } + } else if (MEM_read32(match) != MEM_read32(ip)) { + /* it's not a match, and we're not going to check the dictionary */ + assert(stepSize >= 1); + ip += ((ip-anchor) >> kSearchStrength) + stepSize; + continue; + } else { + /* found a regular match */ + U32 const offset = (U32)(ip-match); + mLength = ZSTD_count(ip+4, match+4, iend) + 4; + while (((ip>anchor) & (match>prefixStart)) + && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + } + + /* match found */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Fill Table */ + assert(base+curr+2 > istart); /* check base overflow */ + hashTable[ZSTD_hashPtr(base+curr+2, hlog, mls)] = curr+2; /* here because curr+2 could be > iend-8 */ + hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base); + + /* check immediate repcode */ + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? + dictBase - dictIndexDelta + repIndex2 : + base + repIndex2; + if ( ((U32)((prefixStartIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */) + && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; + U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, repLength2-MINMATCH); + hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2; + ip += repLength2; + anchor = ip; + continue; + } + break; + } + } + } + + /* save reps for next block */ + rep[0] = offset_1 ? offset_1 : offsetSaved; + rep[1] = offset_2 ? offset_2 : offsetSaved; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + +size_t ZSTD_compressBlock_fast_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + U32 const mls = ms->cParams.minMatch; + assert(ms->dictMatchState != NULL); + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 4); + case 5 : + return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 5); + case 6 : + return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 6); + case 7 : + return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 7); + } +} + + +static size_t ZSTD_compressBlock_fast_extDict_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, U32 const mls) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hlog = cParams->hashLog; + /* support stepSize of 0 */ + U32 const stepSize = cParams->targetLength + !(cParams->targetLength); + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog); + const U32 dictStartIndex = lowLimit; + const BYTE* const dictStart = dictBase + dictStartIndex; + const U32 dictLimit = ms->window.dictLimit; + const U32 prefixStartIndex = dictLimit < lowLimit ? lowLimit : dictLimit; + const BYTE* const prefixStart = base + prefixStartIndex; + const BYTE* const dictEnd = dictBase + prefixStartIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + U32 offset_1=rep[0], offset_2=rep[1]; + + DEBUGLOG(5, "ZSTD_compressBlock_fast_extDict_generic (offset_1=%u)", offset_1); + + /* switch to "regular" variant if extDict is invalidated due to maxDistance */ + if (prefixStartIndex == dictStartIndex) + return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls); + + /* Search Loop */ + while (ip < ilimit) { /* < instead of <=, because (ip+1) */ + const size_t h = ZSTD_hashPtr(ip, hlog, mls); + const U32 matchIndex = hashTable[h]; + const BYTE* const matchBase = matchIndex < prefixStartIndex ? dictBase : base; + const BYTE* match = matchBase + matchIndex; + const U32 curr = (U32)(ip-base); + const U32 repIndex = curr + 1 - offset_1; + const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + hashTable[h] = curr; /* update hash table */ + DEBUGLOG(7, "offset_1 = %u , curr = %u", offset_1, curr); + assert(offset_1 <= curr +1); /* check repIndex */ + + if ( (((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > dictStartIndex)) + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + size_t const rLength = ZSTD_count_2segments(ip+1 +4, repMatch +4, iend, repMatchEnd, prefixStart) + 4; + ip++; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, rLength-MINMATCH); + ip += rLength; + anchor = ip; + } else { + if ( (matchIndex < dictStartIndex) || + (MEM_read32(match) != MEM_read32(ip)) ) { + assert(stepSize >= 1); + ip += ((ip-anchor) >> kSearchStrength) + stepSize; + continue; + } + { const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend; + const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart; + U32 const offset = curr - matchIndex; + size_t mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4; + while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + offset_2 = offset_1; offset_1 = offset; /* update offset history */ + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ip += mLength; + anchor = ip; + } } + + if (ip <= ilimit) { + /* Fill Table */ + hashTable[ZSTD_hashPtr(base+curr+2, hlog, mls)] = curr+2; + hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base); + /* check immediate repcode */ + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* const repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; + if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) & (repIndex2 > dictStartIndex)) /* intentional overflow */ + && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; + { U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; } /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, iend, 0 /*offcode*/, repLength2-MINMATCH); + hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2; + ip += repLength2; + anchor = ip; + continue; + } + break; + } } } + + /* save reps for next block */ + rep[0] = offset_1; + rep[1] = offset_2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + + +size_t ZSTD_compressBlock_fast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + U32 const mls = ms->cParams.minMatch; + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 4); + case 5 : + return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 5); + case 6 : + return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 6); + case 7 : + return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 7); + } +} diff --git a/lib/zstd/compress/zstd_fast.h b/lib/zstd/compress/zstd_fast.h new file mode 100644 index 000000000000..fddc2f532d21 --- /dev/null +++ b/lib/zstd/compress/zstd_fast.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_FAST_H +#define ZSTD_FAST_H + + +#include "../common/mem.h" /* U32 */ +#include "zstd_compress_internal.h" + +void ZSTD_fillHashTable(ZSTD_matchState_t* ms, + void const* end, ZSTD_dictTableLoadMethod_e dtlm); +size_t ZSTD_compressBlock_fast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_fast_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_fast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + + +#endif /* ZSTD_FAST_H */ diff --git a/lib/zstd/compress/zstd_lazy.c b/lib/zstd/compress/zstd_lazy.c new file mode 100644 index 000000000000..87cfa353a693 --- /dev/null +++ b/lib/zstd/compress/zstd_lazy.c @@ -0,0 +1,1412 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_compress_internal.h" +#include "zstd_lazy.h" + + +/*-************************************* +* Binary Tree search +***************************************/ + +static void +ZSTD_updateDUBT(ZSTD_matchState_t* ms, + const BYTE* ip, const BYTE* iend, + U32 mls) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hashLog = cParams->hashLog; + + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + + const BYTE* const base = ms->window.base; + U32 const target = (U32)(ip - base); + U32 idx = ms->nextToUpdate; + + if (idx != target) + DEBUGLOG(7, "ZSTD_updateDUBT, from %u to %u (dictLimit:%u)", + idx, target, ms->window.dictLimit); + assert(ip + 8 <= iend); /* condition for ZSTD_hashPtr */ + (void)iend; + + assert(idx >= ms->window.dictLimit); /* condition for valid base+idx */ + for ( ; idx < target ; idx++) { + size_t const h = ZSTD_hashPtr(base + idx, hashLog, mls); /* assumption : ip + 8 <= iend */ + U32 const matchIndex = hashTable[h]; + + U32* const nextCandidatePtr = bt + 2*(idx&btMask); + U32* const sortMarkPtr = nextCandidatePtr + 1; + + DEBUGLOG(8, "ZSTD_updateDUBT: insert %u", idx); + hashTable[h] = idx; /* Update Hash Table */ + *nextCandidatePtr = matchIndex; /* update BT like a chain */ + *sortMarkPtr = ZSTD_DUBT_UNSORTED_MARK; + } + ms->nextToUpdate = target; +} + + +/* ZSTD_insertDUBT1() : + * sort one already inserted but unsorted position + * assumption : curr >= btlow == (curr - btmask) + * doesn't fail */ +static void +ZSTD_insertDUBT1(ZSTD_matchState_t* ms, + U32 curr, const BYTE* inputEnd, + U32 nbCompares, U32 btLow, + const ZSTD_dictMode_e dictMode) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const ip = (curr>=dictLimit) ? base + curr : dictBase + curr; + const BYTE* const iend = (curr>=dictLimit) ? inputEnd : dictBase + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* match; + U32* smallerPtr = bt + 2*(curr&btMask); + U32* largerPtr = smallerPtr + 1; + U32 matchIndex = *smallerPtr; /* this candidate is unsorted : next sorted candidate is reached through *smallerPtr, while *largerPtr contains previous unsorted candidate (which is already saved and can be overwritten) */ + U32 dummy32; /* to be nullified at the end */ + U32 const windowValid = ms->window.lowLimit; + U32 const maxDistance = 1U << cParams->windowLog; + U32 const windowLow = (curr - windowValid > maxDistance) ? curr - maxDistance : windowValid; + + + DEBUGLOG(8, "ZSTD_insertDUBT1(%u) (dictLimit=%u, lowLimit=%u)", + curr, dictLimit, windowLow); + assert(curr >= btLow); + assert(ip < iend); /* condition for ZSTD_count */ + + while (nbCompares-- && (matchIndex > windowLow)) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + assert(matchIndex < curr); + /* note : all candidates are now supposed sorted, + * but it's still possible to have nextPtr[1] == ZSTD_DUBT_UNSORTED_MARK + * when a real index has the same value as ZSTD_DUBT_UNSORTED_MARK */ + + if ( (dictMode != ZSTD_extDict) + || (matchIndex+matchLength >= dictLimit) /* both in current segment*/ + || (curr < dictLimit) /* both in extDict */) { + const BYTE* const mBase = ( (dictMode != ZSTD_extDict) + || (matchIndex+matchLength >= dictLimit)) ? + base : dictBase; + assert( (matchIndex+matchLength >= dictLimit) /* might be wrong if extDict is incorrectly set to 0 */ + || (curr < dictLimit) ); + match = mBase + matchIndex; + matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* preparation for next read of match[matchLength] */ + } + + DEBUGLOG(8, "ZSTD_insertDUBT1: comparing %u with %u : found %u common bytes ", + curr, matchIndex, (U32)matchLength); + + if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */ + break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt tree */ + } + + if (match[matchLength] < ip[matchLength]) { /* necessarily within buffer */ + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop searching */ + DEBUGLOG(8, "ZSTD_insertDUBT1: %u (>btLow=%u) is smaller : next => %u", + matchIndex, btLow, nextPtr[1]); + smallerPtr = nextPtr+1; /* new "candidate" => larger than match, which was smaller than target */ + matchIndex = nextPtr[1]; /* new matchIndex, larger than previous and closer to current */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop searching */ + DEBUGLOG(8, "ZSTD_insertDUBT1: %u (>btLow=%u) is larger => %u", + matchIndex, btLow, nextPtr[0]); + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; +} + + +static size_t +ZSTD_DUBT_findBetterDictMatch ( + ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iend, + size_t* offsetPtr, + size_t bestLength, + U32 nbCompares, + U32 const mls, + const ZSTD_dictMode_e dictMode) +{ + const ZSTD_matchState_t * const dms = ms->dictMatchState; + const ZSTD_compressionParameters* const dmsCParams = &dms->cParams; + const U32 * const dictHashTable = dms->hashTable; + U32 const hashLog = dmsCParams->hashLog; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32 dictMatchIndex = dictHashTable[h]; + + const BYTE* const base = ms->window.base; + const BYTE* const prefixStart = base + ms->window.dictLimit; + U32 const curr = (U32)(ip-base); + const BYTE* const dictBase = dms->window.base; + const BYTE* const dictEnd = dms->window.nextSrc; + U32 const dictHighLimit = (U32)(dms->window.nextSrc - dms->window.base); + U32 const dictLowLimit = dms->window.lowLimit; + U32 const dictIndexDelta = ms->window.lowLimit - dictHighLimit; + + U32* const dictBt = dms->chainTable; + U32 const btLog = dmsCParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + U32 const btLow = (btMask >= dictHighLimit - dictLowLimit) ? dictLowLimit : dictHighLimit - btMask; + + size_t commonLengthSmaller=0, commonLengthLarger=0; + + (void)dictMode; + assert(dictMode == ZSTD_dictMatchState); + + while (nbCompares-- && (dictMatchIndex > dictLowLimit)) { + U32* const nextPtr = dictBt + 2*(dictMatchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match = dictBase + dictMatchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (dictMatchIndex+matchLength >= dictHighLimit) + match = base + dictMatchIndex + dictIndexDelta; /* to prepare for next usage of match[matchLength] */ + + if (matchLength > bestLength) { + U32 matchIndex = dictMatchIndex + dictIndexDelta; + if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) { + DEBUGLOG(9, "ZSTD_DUBT_findBetterDictMatch(%u) : found better match length %u -> %u and offsetCode %u -> %u (dictMatchIndex %u, matchIndex %u)", + curr, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr, ZSTD_REP_MOVE + curr - matchIndex, dictMatchIndex, matchIndex); + bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + curr - matchIndex; + } + if (ip+matchLength == iend) { /* reached end of input : ip[matchLength] is not valid, no way to know if it's larger or smaller than match */ + break; /* drop, to guarantee consistency (miss a little bit of compression) */ + } + } + + if (match[matchLength] < ip[matchLength]) { + if (dictMatchIndex <= btLow) { break; } /* beyond tree size, stop the search */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + dictMatchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + if (dictMatchIndex <= btLow) { break; } /* beyond tree size, stop the search */ + commonLengthLarger = matchLength; + dictMatchIndex = nextPtr[0]; + } + } + + if (bestLength >= MINMATCH) { + U32 const mIndex = curr - ((U32)*offsetPtr - ZSTD_REP_MOVE); (void)mIndex; + DEBUGLOG(8, "ZSTD_DUBT_findBetterDictMatch(%u) : found match of length %u and offsetCode %u (pos %u)", + curr, (U32)bestLength, (U32)*offsetPtr, mIndex); + } + return bestLength; + +} + + +static size_t +ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iend, + size_t* offsetPtr, + U32 const mls, + const ZSTD_dictMode_e dictMode) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hashLog = cParams->hashLog; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32 matchIndex = hashTable[h]; + + const BYTE* const base = ms->window.base; + U32 const curr = (U32)(ip-base); + U32 const windowLow = ZSTD_getLowestMatchIndex(ms, curr, cParams->windowLog); + + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + U32 const btLow = (btMask >= curr) ? 0 : curr - btMask; + U32 const unsortLimit = MAX(btLow, windowLow); + + U32* nextCandidate = bt + 2*(matchIndex&btMask); + U32* unsortedMark = bt + 2*(matchIndex&btMask) + 1; + U32 nbCompares = 1U << cParams->searchLog; + U32 nbCandidates = nbCompares; + U32 previousCandidate = 0; + + DEBUGLOG(7, "ZSTD_DUBT_findBestMatch (%u) ", curr); + assert(ip <= iend-8); /* required for h calculation */ + assert(dictMode != ZSTD_dedicatedDictSearch); + + /* reach end of unsorted candidates list */ + while ( (matchIndex > unsortLimit) + && (*unsortedMark == ZSTD_DUBT_UNSORTED_MARK) + && (nbCandidates > 1) ) { + DEBUGLOG(8, "ZSTD_DUBT_findBestMatch: candidate %u is unsorted", + matchIndex); + *unsortedMark = previousCandidate; /* the unsortedMark becomes a reversed chain, to move up back to original position */ + previousCandidate = matchIndex; + matchIndex = *nextCandidate; + nextCandidate = bt + 2*(matchIndex&btMask); + unsortedMark = bt + 2*(matchIndex&btMask) + 1; + nbCandidates --; + } + + /* nullify last candidate if it's still unsorted + * simplification, detrimental to compression ratio, beneficial for speed */ + if ( (matchIndex > unsortLimit) + && (*unsortedMark==ZSTD_DUBT_UNSORTED_MARK) ) { + DEBUGLOG(7, "ZSTD_DUBT_findBestMatch: nullify last unsorted candidate %u", + matchIndex); + *nextCandidate = *unsortedMark = 0; + } + + /* batch sort stacked candidates */ + matchIndex = previousCandidate; + while (matchIndex) { /* will end on matchIndex == 0 */ + U32* const nextCandidateIdxPtr = bt + 2*(matchIndex&btMask) + 1; + U32 const nextCandidateIdx = *nextCandidateIdxPtr; + ZSTD_insertDUBT1(ms, matchIndex, iend, + nbCandidates, unsortLimit, dictMode); + matchIndex = nextCandidateIdx; + nbCandidates++; + } + + /* find longest match */ + { size_t commonLengthSmaller = 0, commonLengthLarger = 0; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + U32* smallerPtr = bt + 2*(curr&btMask); + U32* largerPtr = bt + 2*(curr&btMask) + 1; + U32 matchEndIdx = curr + 8 + 1; + U32 dummy32; /* to be nullified at the end */ + size_t bestLength = 0; + + matchIndex = hashTable[h]; + hashTable[h] = curr; /* Update Hash Table */ + + while (nbCompares-- && (matchIndex > windowLow)) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match; + + if ((dictMode != ZSTD_extDict) || (matchIndex+matchLength >= dictLimit)) { + match = base + matchIndex; + matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ + } + + if (matchLength > bestLength) { + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) + bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + curr - matchIndex; + if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */ + if (dictMode == ZSTD_dictMatchState) { + nbCompares = 0; /* in addition to avoiding checking any + * further in this loop, make sure we + * skip checking in the dictionary. */ + } + break; /* drop, to guarantee consistency (miss a little bit of compression) */ + } + } + + if (match[matchLength] < ip[matchLength]) { + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; + + if (dictMode == ZSTD_dictMatchState && nbCompares) { + bestLength = ZSTD_DUBT_findBetterDictMatch( + ms, ip, iend, + offsetPtr, bestLength, nbCompares, + mls, dictMode); + } + + assert(matchEndIdx > curr+8); /* ensure nextToUpdate is increased */ + ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */ + if (bestLength >= MINMATCH) { + U32 const mIndex = curr - ((U32)*offsetPtr - ZSTD_REP_MOVE); (void)mIndex; + DEBUGLOG(8, "ZSTD_DUBT_findBestMatch(%u) : found match of length %u and offsetCode %u (pos %u)", + curr, (U32)bestLength, (U32)*offsetPtr, mIndex); + } + return bestLength; + } +} + + +/* ZSTD_BtFindBestMatch() : Tree updater, providing best match */ +FORCE_INLINE_TEMPLATE size_t +ZSTD_BtFindBestMatch( ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 mls /* template */, + const ZSTD_dictMode_e dictMode) +{ + DEBUGLOG(7, "ZSTD_BtFindBestMatch"); + if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */ + ZSTD_updateDUBT(ms, ip, iLimit, mls); + return ZSTD_DUBT_findBestMatch(ms, ip, iLimit, offsetPtr, mls, dictMode); +} + + +static size_t +ZSTD_BtFindBestMatch_selectMLS ( ZSTD_matchState_t* ms, + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr) +{ + switch(ms->cParams.minMatch) + { + default : /* includes case 3 */ + case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict); + case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_noDict); + case 7 : + case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_noDict); + } +} + + +static size_t ZSTD_BtFindBestMatch_dictMatchState_selectMLS ( + ZSTD_matchState_t* ms, + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr) +{ + switch(ms->cParams.minMatch) + { + default : /* includes case 3 */ + case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState); + case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState); + case 7 : + case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState); + } +} + + +static size_t ZSTD_BtFindBestMatch_extDict_selectMLS ( + ZSTD_matchState_t* ms, + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr) +{ + switch(ms->cParams.minMatch) + { + default : /* includes case 3 */ + case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict); + case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_extDict); + case 7 : + case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_extDict); + } +} + + + +/* ********************************* +* Hash Chain +***********************************/ +#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & (mask)] + +/* Update chains up to ip (excluded) + Assumption : always within prefix (i.e. not within extDict) */ +FORCE_INLINE_TEMPLATE U32 ZSTD_insertAndFindFirstIndex_internal( + ZSTD_matchState_t* ms, + const ZSTD_compressionParameters* const cParams, + const BYTE* ip, U32 const mls) +{ + U32* const hashTable = ms->hashTable; + const U32 hashLog = cParams->hashLog; + U32* const chainTable = ms->chainTable; + const U32 chainMask = (1 << cParams->chainLog) - 1; + const BYTE* const base = ms->window.base; + const U32 target = (U32)(ip - base); + U32 idx = ms->nextToUpdate; + + while(idx < target) { /* catch up */ + size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls); + NEXT_IN_CHAIN(idx, chainMask) = hashTable[h]; + hashTable[h] = idx; + idx++; + } + + ms->nextToUpdate = target; + return hashTable[ZSTD_hashPtr(ip, hashLog, mls)]; +} + +U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip) { + const ZSTD_compressionParameters* const cParams = &ms->cParams; + return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch); +} + +void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip) +{ + const BYTE* const base = ms->window.base; + U32 const target = (U32)(ip - base); + U32* const hashTable = ms->hashTable; + U32* const chainTable = ms->chainTable; + U32 const chainSize = 1 << ms->cParams.chainLog; + U32 idx = ms->nextToUpdate; + U32 const minChain = chainSize < target ? target - chainSize : idx; + U32 const bucketSize = 1 << ZSTD_LAZY_DDSS_BUCKET_LOG; + U32 const cacheSize = bucketSize - 1; + U32 const chainAttempts = (1 << ms->cParams.searchLog) - cacheSize; + U32 const chainLimit = chainAttempts > 255 ? 255 : chainAttempts; + + /* We know the hashtable is oversized by a factor of `bucketSize`. + * We are going to temporarily pretend `bucketSize == 1`, keeping only a + * single entry. We will use the rest of the space to construct a temporary + * chaintable. + */ + U32 const hashLog = ms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG; + U32* const tmpHashTable = hashTable; + U32* const tmpChainTable = hashTable + ((size_t)1 << hashLog); + U32 const tmpChainSize = ((1 << ZSTD_LAZY_DDSS_BUCKET_LOG) - 1) << hashLog; + U32 const tmpMinChain = tmpChainSize < target ? target - tmpChainSize : idx; + + U32 hashIdx; + + assert(ms->cParams.chainLog <= 24); + assert(ms->cParams.hashLog >= ms->cParams.chainLog); + assert(idx != 0); + assert(tmpMinChain <= minChain); + + /* fill conventional hash table and conventional chain table */ + for ( ; idx < target; idx++) { + U32 const h = (U32)ZSTD_hashPtr(base + idx, hashLog, ms->cParams.minMatch); + if (idx >= tmpMinChain) { + tmpChainTable[idx - tmpMinChain] = hashTable[h]; + } + tmpHashTable[h] = idx; + } + + /* sort chains into ddss chain table */ + { + U32 chainPos = 0; + for (hashIdx = 0; hashIdx < (1U << hashLog); hashIdx++) { + U32 count; + U32 countBeyondMinChain = 0; + U32 i = tmpHashTable[hashIdx]; + for (count = 0; i >= tmpMinChain && count < cacheSize; count++) { + /* skip through the chain to the first position that won't be + * in the hash cache bucket */ + if (i < minChain) { + countBeyondMinChain++; + } + i = tmpChainTable[i - tmpMinChain]; + } + if (count == cacheSize) { + for (count = 0; count < chainLimit;) { + if (i < minChain) { + if (!i || countBeyondMinChain++ > cacheSize) { + /* only allow pulling `cacheSize` number of entries + * into the cache or chainTable beyond `minChain`, + * to replace the entries pulled out of the + * chainTable into the cache. This lets us reach + * back further without increasing the total number + * of entries in the chainTable, guaranteeing the + * DDSS chain table will fit into the space + * allocated for the regular one. */ + break; + } + } + chainTable[chainPos++] = i; + count++; + if (i < tmpMinChain) { + break; + } + i = tmpChainTable[i - tmpMinChain]; + } + } else { + count = 0; + } + if (count) { + tmpHashTable[hashIdx] = ((chainPos - count) << 8) + count; + } else { + tmpHashTable[hashIdx] = 0; + } + } + assert(chainPos <= chainSize); /* I believe this is guaranteed... */ + } + + /* move chain pointers into the last entry of each hash bucket */ + for (hashIdx = (1 << hashLog); hashIdx; ) { + U32 const bucketIdx = --hashIdx << ZSTD_LAZY_DDSS_BUCKET_LOG; + U32 const chainPackedPointer = tmpHashTable[hashIdx]; + U32 i; + for (i = 0; i < cacheSize; i++) { + hashTable[bucketIdx + i] = 0; + } + hashTable[bucketIdx + bucketSize - 1] = chainPackedPointer; + } + + /* fill the buckets of the hash table */ + for (idx = ms->nextToUpdate; idx < target; idx++) { + U32 const h = (U32)ZSTD_hashPtr(base + idx, hashLog, ms->cParams.minMatch) + << ZSTD_LAZY_DDSS_BUCKET_LOG; + U32 i; + /* Shift hash cache down 1. */ + for (i = cacheSize - 1; i; i--) + hashTable[h + i] = hashTable[h + i - 1]; + hashTable[h] = idx; + } + + ms->nextToUpdate = target; +} + + +/* inlining is important to hardwire a hot branch (template emulation) */ +FORCE_INLINE_TEMPLATE +size_t ZSTD_HcFindBestMatch_generic ( + ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 mls, const ZSTD_dictMode_e dictMode) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const chainTable = ms->chainTable; + const U32 chainSize = (1 << cParams->chainLog); + const U32 chainMask = chainSize-1; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const U32 curr = (U32)(ip-base); + const U32 maxDistance = 1U << cParams->windowLog; + const U32 lowestValid = ms->window.lowLimit; + const U32 withinMaxDistance = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; + const U32 isDictionary = (ms->loadedDictEnd != 0); + const U32 lowLimit = isDictionary ? lowestValid : withinMaxDistance; + const U32 minChain = curr > chainSize ? curr - chainSize : 0; + U32 nbAttempts = 1U << cParams->searchLog; + size_t ml=4-1; + + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const U32 ddsHashLog = dictMode == ZSTD_dedicatedDictSearch + ? dms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG : 0; + const size_t ddsIdx = dictMode == ZSTD_dedicatedDictSearch + ? ZSTD_hashPtr(ip, ddsHashLog, mls) << ZSTD_LAZY_DDSS_BUCKET_LOG : 0; + + U32 matchIndex; + + if (dictMode == ZSTD_dedicatedDictSearch) { + const U32* entry = &dms->hashTable[ddsIdx]; + PREFETCH_L1(entry); + } + + /* HC4 match finder */ + matchIndex = ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, mls); + + for ( ; (matchIndex>=lowLimit) & (nbAttempts>0) ; nbAttempts--) { + size_t currentMl=0; + if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) { + const BYTE* const match = base + matchIndex; + assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */ + if (match[ml] == ip[ml]) /* potentially better */ + currentMl = ZSTD_count(ip, match, iLimit); + } else { + const BYTE* const match = dictBase + matchIndex; + assert(match+4 <= dictEnd); + if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dictEnd, prefixStart) + 4; + } + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = curr - matchIndex + ZSTD_REP_MOVE; + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } + + if (matchIndex <= minChain) break; + matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask); + } + + if (dictMode == ZSTD_dedicatedDictSearch) { + const U32 ddsLowestIndex = dms->window.dictLimit; + const BYTE* const ddsBase = dms->window.base; + const BYTE* const ddsEnd = dms->window.nextSrc; + const U32 ddsSize = (U32)(ddsEnd - ddsBase); + const U32 ddsIndexDelta = dictLimit - ddsSize; + const U32 bucketSize = (1 << ZSTD_LAZY_DDSS_BUCKET_LOG); + const U32 bucketLimit = nbAttempts < bucketSize - 1 ? nbAttempts : bucketSize - 1; + U32 ddsAttempt; + + for (ddsAttempt = 0; ddsAttempt < bucketSize - 1; ddsAttempt++) { + PREFETCH_L1(ddsBase + dms->hashTable[ddsIdx + ddsAttempt]); + } + + { + U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1]; + U32 const chainIndex = chainPackedPointer >> 8; + + PREFETCH_L1(&dms->chainTable[chainIndex]); + } + + for (ddsAttempt = 0; ddsAttempt < bucketLimit; ddsAttempt++) { + size_t currentMl=0; + const BYTE* match; + matchIndex = dms->hashTable[ddsIdx + ddsAttempt]; + match = ddsBase + matchIndex; + + if (!matchIndex) { + return ml; + } + + /* guaranteed by table construction */ + (void)ddsLowestIndex; + assert(matchIndex >= ddsLowestIndex); + assert(match+4 <= ddsEnd); + if (MEM_read32(match) == MEM_read32(ip)) { + /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4; + } + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = curr - (matchIndex + ddsIndexDelta) + ZSTD_REP_MOVE; + if (ip+currentMl == iLimit) { + /* best possible, avoids read overflow on next attempt */ + return ml; + } + } + } + + { + U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1]; + U32 chainIndex = chainPackedPointer >> 8; + U32 const chainLength = chainPackedPointer & 0xFF; + U32 const chainAttempts = nbAttempts - ddsAttempt; + U32 const chainLimit = chainAttempts > chainLength ? chainLength : chainAttempts; + U32 chainAttempt; + + for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++) { + PREFETCH_L1(ddsBase + dms->chainTable[chainIndex + chainAttempt]); + } + + for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++, chainIndex++) { + size_t currentMl=0; + const BYTE* match; + matchIndex = dms->chainTable[chainIndex]; + match = ddsBase + matchIndex; + + /* guaranteed by table construction */ + assert(matchIndex >= ddsLowestIndex); + assert(match+4 <= ddsEnd); + if (MEM_read32(match) == MEM_read32(ip)) { + /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4; + } + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = curr - (matchIndex + ddsIndexDelta) + ZSTD_REP_MOVE; + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } + } + } + } else if (dictMode == ZSTD_dictMatchState) { + const U32* const dmsChainTable = dms->chainTable; + const U32 dmsChainSize = (1 << dms->cParams.chainLog); + const U32 dmsChainMask = dmsChainSize - 1; + const U32 dmsLowestIndex = dms->window.dictLimit; + const BYTE* const dmsBase = dms->window.base; + const BYTE* const dmsEnd = dms->window.nextSrc; + const U32 dmsSize = (U32)(dmsEnd - dmsBase); + const U32 dmsIndexDelta = dictLimit - dmsSize; + const U32 dmsMinChain = dmsSize > dmsChainSize ? dmsSize - dmsChainSize : 0; + + matchIndex = dms->hashTable[ZSTD_hashPtr(ip, dms->cParams.hashLog, mls)]; + + for ( ; (matchIndex>=dmsLowestIndex) & (nbAttempts>0) ; nbAttempts--) { + size_t currentMl=0; + const BYTE* const match = dmsBase + matchIndex; + assert(match+4 <= dmsEnd); + if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dmsEnd, prefixStart) + 4; + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = curr - (matchIndex + dmsIndexDelta) + ZSTD_REP_MOVE; + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } + + if (matchIndex <= dmsMinChain) break; + + matchIndex = dmsChainTable[matchIndex & dmsChainMask]; + } + } + + return ml; +} + + +FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_selectMLS ( + ZSTD_matchState_t* ms, + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr) +{ + switch(ms->cParams.minMatch) + { + default : /* includes case 3 */ + case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict); + case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_noDict); + case 7 : + case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_noDict); + } +} + + +static size_t ZSTD_HcFindBestMatch_dictMatchState_selectMLS ( + ZSTD_matchState_t* ms, + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr) +{ + switch(ms->cParams.minMatch) + { + default : /* includes case 3 */ + case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState); + case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState); + case 7 : + case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState); + } +} + + +static size_t ZSTD_HcFindBestMatch_dedicatedDictSearch_selectMLS ( + ZSTD_matchState_t* ms, + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr) +{ + switch(ms->cParams.minMatch) + { + default : /* includes case 3 */ + case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dedicatedDictSearch); + case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_dedicatedDictSearch); + case 7 : + case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_dedicatedDictSearch); + } +} + + +FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS ( + ZSTD_matchState_t* ms, + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr) +{ + switch(ms->cParams.minMatch) + { + default : /* includes case 3 */ + case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict); + case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_extDict); + case 7 : + case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_extDict); + } +} + + +/* ******************************* +* Common parser - lazy strategy +*********************************/ +typedef enum { search_hashChain, search_binaryTree } searchMethod_e; + +FORCE_INLINE_TEMPLATE size_t +ZSTD_compressBlock_lazy_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, + const searchMethod_e searchMethod, const U32 depth, + ZSTD_dictMode_e const dictMode) +{ + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ms->window.base; + const U32 prefixLowestIndex = ms->window.dictLimit; + const BYTE* const prefixLowest = base + prefixLowestIndex; + + typedef size_t (*searchMax_f)( + ZSTD_matchState_t* ms, + const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); + + /* + * This table is indexed first by the four ZSTD_dictMode_e values, and then + * by the two searchMethod_e values. NULLs are placed for configurations + * that should never occur (extDict modes go to the other implementation + * below and there is no DDSS for binary tree search yet). + */ + const searchMax_f searchFuncs[4][2] = { + { + ZSTD_HcFindBestMatch_selectMLS, + ZSTD_BtFindBestMatch_selectMLS + }, + { + NULL, + NULL + }, + { + ZSTD_HcFindBestMatch_dictMatchState_selectMLS, + ZSTD_BtFindBestMatch_dictMatchState_selectMLS + }, + { + ZSTD_HcFindBestMatch_dedicatedDictSearch_selectMLS, + NULL + } + }; + + searchMax_f const searchMax = searchFuncs[dictMode][searchMethod == search_binaryTree]; + U32 offset_1 = rep[0], offset_2 = rep[1], savedOffset=0; + + const int isDMS = dictMode == ZSTD_dictMatchState; + const int isDDS = dictMode == ZSTD_dedicatedDictSearch; + const int isDxS = isDMS || isDDS; + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const U32 dictLowestIndex = isDxS ? dms->window.dictLimit : 0; + const BYTE* const dictBase = isDxS ? dms->window.base : NULL; + const BYTE* const dictLowest = isDxS ? dictBase + dictLowestIndex : NULL; + const BYTE* const dictEnd = isDxS ? dms->window.nextSrc : NULL; + const U32 dictIndexDelta = isDxS ? + prefixLowestIndex - (U32)(dictEnd - dictBase) : + 0; + const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictLowest)); + + assert(searchMax != NULL); + + DEBUGLOG(5, "ZSTD_compressBlock_lazy_generic (dictMode=%u)", (U32)dictMode); + + /* init */ + ip += (dictAndPrefixLength == 0); + if (dictMode == ZSTD_noDict) { + U32 const curr = (U32)(ip - base); + U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, ms->cParams.windowLog); + U32 const maxRep = curr - windowLow; + if (offset_2 > maxRep) savedOffset = offset_2, offset_2 = 0; + if (offset_1 > maxRep) savedOffset = offset_1, offset_1 = 0; + } + if (isDxS) { + /* dictMatchState repCode checks don't currently handle repCode == 0 + * disabling. */ + assert(offset_1 <= dictAndPrefixLength); + assert(offset_2 <= dictAndPrefixLength); + } + + /* Match Loop */ +#if defined(__x86_64__) + /* I've measured random a 5% speed loss on levels 5 & 6 (greedy) when the + * code alignment is perturbed. To fix the instability align the loop on 32-bytes. + */ + __asm__(".p2align 5"); +#endif + while (ip < ilimit) { + size_t matchLength=0; + size_t offset=0; + const BYTE* start=ip+1; + + /* check repCode */ + if (isDxS) { + const U32 repIndex = (U32)(ip - base) + 1 - offset_1; + const BYTE* repMatch = ((dictMode == ZSTD_dictMatchState || dictMode == ZSTD_dedicatedDictSearch) + && repIndex < prefixLowestIndex) ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + if (depth==0) goto _storeSequence; + } + } + if ( dictMode == ZSTD_noDict + && ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) { + matchLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; + if (depth==0) goto _storeSequence; + } + + /* first search (depth 0) */ + { size_t offsetFound = 999999999; + size_t const ml2 = searchMax(ms, ip, iend, &offsetFound); + if (ml2 > matchLength) + matchLength = ml2, start = ip, offset=offsetFound; + } + + if (matchLength < 4) { + ip += ((ip-anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */ + continue; + } + + /* let's try to find a better solution */ + if (depth>=1) + while (ip0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { + size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4; + int const gain2 = (int)(mlRep * 3); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offset = 0, start = ip; + } + if (isDxS) { + const U32 repIndex = (U32)(ip - base) - offset_1; + const BYTE* repMatch = repIndex < prefixLowestIndex ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + && (MEM_read32(repMatch) == MEM_read32(ip)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + int const gain2 = (int)(mlRep * 3); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offset = 0, start = ip; + } + } + { size_t offset2=999999999; + size_t const ml2 = searchMax(ms, ip, iend, &offset2); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offset = offset2, start = ip; + continue; /* search a better one */ + } } + + /* let's find an even better one */ + if ((depth==2) && (ip0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { + size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4; + int const gain2 = (int)(mlRep * 4); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offset = 0, start = ip; + } + if (isDxS) { + const U32 repIndex = (U32)(ip - base) - offset_1; + const BYTE* repMatch = repIndex < prefixLowestIndex ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + && (MEM_read32(repMatch) == MEM_read32(ip)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + int const gain2 = (int)(mlRep * 4); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offset = 0, start = ip; + } + } + { size_t offset2=999999999; + size_t const ml2 = searchMax(ms, ip, iend, &offset2); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offset = offset2, start = ip; + continue; + } } } + break; /* nothing found : store previous solution */ + } + + /* NOTE: + * start[-offset+ZSTD_REP_MOVE-1] is undefined behavior. + * (-offset+ZSTD_REP_MOVE-1) is unsigned, and is added to start, which + * overflows the pointer, which is undefined behavior. + */ + /* catch up */ + if (offset) { + if (dictMode == ZSTD_noDict) { + while ( ((start > anchor) & (start - (offset-ZSTD_REP_MOVE) > prefixLowest)) + && (start[-1] == (start-(offset-ZSTD_REP_MOVE))[-1]) ) /* only search for offset within prefix */ + { start--; matchLength++; } + } + if (isDxS) { + U32 const matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE)); + const BYTE* match = (matchIndex < prefixLowestIndex) ? dictBase + matchIndex - dictIndexDelta : base + matchIndex; + const BYTE* const mStart = (matchIndex < prefixLowestIndex) ? dictLowest : prefixLowest; + while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ + } + offset_2 = offset_1; offset_1 = (U32)(offset - ZSTD_REP_MOVE); + } + /* store sequence */ +_storeSequence: + { size_t const litLength = start - anchor; + ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offset, matchLength-MINMATCH); + anchor = ip = start + matchLength; + } + + /* check immediate repcode */ + if (isDxS) { + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex = current2 - offset_2; + const BYTE* repMatch = repIndex < prefixLowestIndex ? + dictBase - dictIndexDelta + repIndex : + base + repIndex; + if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex) >= 3 /* intentional overflow */) + && (MEM_read32(repMatch) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex < prefixLowestIndex ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd2, prefixLowest) + 4; + offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, matchLength-MINMATCH); + ip += matchLength; + anchor = ip; + continue; + } + break; + } + } + + if (dictMode == ZSTD_noDict) { + while ( ((ip <= ilimit) & (offset_2>0)) + && (MEM_read32(ip) == MEM_read32(ip - offset_2)) ) { + /* store sequence */ + matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; + offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap repcodes */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, matchLength-MINMATCH); + ip += matchLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } } } + + /* Save reps for next block */ + rep[0] = offset_1 ? offset_1 : savedOffset; + rep[1] = offset_2 ? offset_2 : savedOffset; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + + +size_t ZSTD_compressBlock_btlazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_lazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_lazy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_greedy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_btlazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_lazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_lazy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_greedy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dictMatchState); +} + + +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dedicatedDictSearch); +} + +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dedicatedDictSearch); +} + +size_t ZSTD_compressBlock_greedy_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dedicatedDictSearch); +} + + +FORCE_INLINE_TEMPLATE +size_t ZSTD_compressBlock_lazy_extDict_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, + const searchMethod_e searchMethod, const U32 depth) +{ + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ms->window.base; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictBase = ms->window.dictBase; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const dictStart = dictBase + ms->window.lowLimit; + const U32 windowLog = ms->cParams.windowLog; + + typedef size_t (*searchMax_f)( + ZSTD_matchState_t* ms, + const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); + searchMax_f searchMax = searchMethod==search_binaryTree ? ZSTD_BtFindBestMatch_extDict_selectMLS : ZSTD_HcFindBestMatch_extDict_selectMLS; + + U32 offset_1 = rep[0], offset_2 = rep[1]; + + DEBUGLOG(5, "ZSTD_compressBlock_lazy_extDict_generic"); + + /* init */ + ip += (ip == prefixStart); + + /* Match Loop */ +#if defined(__x86_64__) + /* I've measured random a 5% speed loss on levels 5 & 6 (greedy) when the + * code alignment is perturbed. To fix the instability align the loop on 32-bytes. + */ + __asm__(".p2align 5"); +#endif + while (ip < ilimit) { + size_t matchLength=0; + size_t offset=0; + const BYTE* start=ip+1; + U32 curr = (U32)(ip-base); + + /* check repCode */ + { const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr+1, windowLog); + const U32 repIndex = (U32)(curr+1 - offset_1); + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > windowLow)) /* intentional overflow */ + if (MEM_read32(ip+1) == MEM_read32(repMatch)) { + /* repcode detected we should take it */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repEnd, prefixStart) + 4; + if (depth==0) goto _storeSequence; + } } + + /* first search (depth 0) */ + { size_t offsetFound = 999999999; + size_t const ml2 = searchMax(ms, ip, iend, &offsetFound); + if (ml2 > matchLength) + matchLength = ml2, start = ip, offset=offsetFound; + } + + if (matchLength < 4) { + ip += ((ip-anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */ + continue; + } + + /* let's try to find a better solution */ + if (depth>=1) + while (ip= 3) & (repIndex > windowLow)) /* intentional overflow */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; + int const gain2 = (int)(repLength * 3); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1); + if ((repLength >= 4) && (gain2 > gain1)) + matchLength = repLength, offset = 0, start = ip; + } } + + /* search match, depth 1 */ + { size_t offset2=999999999; + size_t const ml2 = searchMax(ms, ip, iend, &offset2); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offset = offset2, start = ip; + continue; /* search a better one */ + } } + + /* let's find an even better one */ + if ((depth==2) && (ip= 3) & (repIndex > windowLow)) /* intentional overflow */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; + int const gain2 = (int)(repLength * 4); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1); + if ((repLength >= 4) && (gain2 > gain1)) + matchLength = repLength, offset = 0, start = ip; + } } + + /* search match, depth 2 */ + { size_t offset2=999999999; + size_t const ml2 = searchMax(ms, ip, iend, &offset2); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offset = offset2, start = ip; + continue; + } } } + break; /* nothing found : store previous solution */ + } + + /* catch up */ + if (offset) { + U32 const matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE)); + const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex; + const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart; + while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ + offset_2 = offset_1; offset_1 = (U32)(offset - ZSTD_REP_MOVE); + } + + /* store sequence */ +_storeSequence: + { size_t const litLength = start - anchor; + ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offset, matchLength-MINMATCH); + anchor = ip = start + matchLength; + } + + /* check immediate repcode */ + while (ip <= ilimit) { + const U32 repCurrent = (U32)(ip-base); + const U32 windowLow = ZSTD_getLowestMatchIndex(ms, repCurrent, windowLog); + const U32 repIndex = repCurrent - offset_2; + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > windowLow)) /* intentional overflow */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected we should take it */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; + offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap offset history */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, matchLength-MINMATCH); + ip += matchLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } + break; + } } + + /* Save reps for next block */ + rep[0] = offset_1; + rep[1] = offset_2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + + +size_t ZSTD_compressBlock_greedy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0); +} + +size_t ZSTD_compressBlock_lazy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) + +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1); +} + +size_t ZSTD_compressBlock_lazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) + +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2); +} + +size_t ZSTD_compressBlock_btlazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) + +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2); +} diff --git a/lib/zstd/compress/zstd_lazy.h b/lib/zstd/compress/zstd_lazy.h new file mode 100644 index 000000000000..2fc5a6182134 --- /dev/null +++ b/lib/zstd/compress/zstd_lazy.h @@ -0,0 +1,81 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_LAZY_H +#define ZSTD_LAZY_H + + +#include "zstd_compress_internal.h" + +/* + * Dedicated Dictionary Search Structure bucket log. In the + * ZSTD_dedicatedDictSearch mode, the hashTable has + * 2 ** ZSTD_LAZY_DDSS_BUCKET_LOG entries in each bucket, rather than just + * one. + */ +#define ZSTD_LAZY_DDSS_BUCKET_LOG 2 + +U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip); + +void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip); + +void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue); /*! used in ZSTD_reduceIndex(). preemptively increase value of ZSTD_DUBT_UNSORTED_MARK */ + +size_t ZSTD_compressBlock_btlazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +size_t ZSTD_compressBlock_btlazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +size_t ZSTD_compressBlock_greedy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btlazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + + +#endif /* ZSTD_LAZY_H */ diff --git a/lib/zstd/compress/zstd_ldm.c b/lib/zstd/compress/zstd_ldm.c new file mode 100644 index 000000000000..8ef7e88a5add --- /dev/null +++ b/lib/zstd/compress/zstd_ldm.c @@ -0,0 +1,686 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_ldm.h" + +#include "../common/debug.h" +#include +#include "zstd_fast.h" /* ZSTD_fillHashTable() */ +#include "zstd_double_fast.h" /* ZSTD_fillDoubleHashTable() */ +#include "zstd_ldm_geartab.h" + +#define LDM_BUCKET_SIZE_LOG 3 +#define LDM_MIN_MATCH_LENGTH 64 +#define LDM_HASH_RLOG 7 + +typedef struct { + U64 rolling; + U64 stopMask; +} ldmRollingHashState_t; + +/* ZSTD_ldm_gear_init(): + * + * Initializes the rolling hash state such that it will honor the + * settings in params. */ +static void ZSTD_ldm_gear_init(ldmRollingHashState_t* state, ldmParams_t const* params) +{ + unsigned maxBitsInMask = MIN(params->minMatchLength, 64); + unsigned hashRateLog = params->hashRateLog; + + state->rolling = ~(U32)0; + + /* The choice of the splitting criterion is subject to two conditions: + * 1. it has to trigger on average every 2^(hashRateLog) bytes; + * 2. ideally, it has to depend on a window of minMatchLength bytes. + * + * In the gear hash algorithm, bit n depends on the last n bytes; + * so in order to obtain a good quality splitting criterion it is + * preferable to use bits with high weight. + * + * To match condition 1 we use a mask with hashRateLog bits set + * and, because of the previous remark, we make sure these bits + * have the highest possible weight while still respecting + * condition 2. + */ + if (hashRateLog > 0 && hashRateLog <= maxBitsInMask) { + state->stopMask = (((U64)1 << hashRateLog) - 1) << (maxBitsInMask - hashRateLog); + } else { + /* In this degenerate case we simply honor the hash rate. */ + state->stopMask = ((U64)1 << hashRateLog) - 1; + } +} + +/* ZSTD_ldm_gear_feed(): + * + * Registers in the splits array all the split points found in the first + * size bytes following the data pointer. This function terminates when + * either all the data has been processed or LDM_BATCH_SIZE splits are + * present in the splits array. + * + * Precondition: The splits array must not be full. + * Returns: The number of bytes processed. */ +static size_t ZSTD_ldm_gear_feed(ldmRollingHashState_t* state, + BYTE const* data, size_t size, + size_t* splits, unsigned* numSplits) +{ + size_t n; + U64 hash, mask; + + hash = state->rolling; + mask = state->stopMask; + n = 0; + +#define GEAR_ITER_ONCE() do { \ + hash = (hash << 1) + ZSTD_ldm_gearTab[data[n] & 0xff]; \ + n += 1; \ + if (UNLIKELY((hash & mask) == 0)) { \ + splits[*numSplits] = n; \ + *numSplits += 1; \ + if (*numSplits == LDM_BATCH_SIZE) \ + goto done; \ + } \ + } while (0) + + while (n + 3 < size) { + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + } + while (n < size) { + GEAR_ITER_ONCE(); + } + +#undef GEAR_ITER_ONCE + +done: + state->rolling = hash; + return n; +} + +void ZSTD_ldm_adjustParameters(ldmParams_t* params, + ZSTD_compressionParameters const* cParams) +{ + params->windowLog = cParams->windowLog; + ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX); + DEBUGLOG(4, "ZSTD_ldm_adjustParameters"); + if (!params->bucketSizeLog) params->bucketSizeLog = LDM_BUCKET_SIZE_LOG; + if (!params->minMatchLength) params->minMatchLength = LDM_MIN_MATCH_LENGTH; + if (params->hashLog == 0) { + params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - LDM_HASH_RLOG); + assert(params->hashLog <= ZSTD_HASHLOG_MAX); + } + if (params->hashRateLog == 0) { + params->hashRateLog = params->windowLog < params->hashLog + ? 0 + : params->windowLog - params->hashLog; + } + params->bucketSizeLog = MIN(params->bucketSizeLog, params->hashLog); +} + +size_t ZSTD_ldm_getTableSize(ldmParams_t params) +{ + size_t const ldmHSize = ((size_t)1) << params.hashLog; + size_t const ldmBucketSizeLog = MIN(params.bucketSizeLog, params.hashLog); + size_t const ldmBucketSize = ((size_t)1) << (params.hashLog - ldmBucketSizeLog); + size_t const totalSize = ZSTD_cwksp_alloc_size(ldmBucketSize) + + ZSTD_cwksp_alloc_size(ldmHSize * sizeof(ldmEntry_t)); + return params.enableLdm ? totalSize : 0; +} + +size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize) +{ + return params.enableLdm ? (maxChunkSize / params.minMatchLength) : 0; +} + +/* ZSTD_ldm_getBucket() : + * Returns a pointer to the start of the bucket associated with hash. */ +static ldmEntry_t* ZSTD_ldm_getBucket( + ldmState_t* ldmState, size_t hash, ldmParams_t const ldmParams) +{ + return ldmState->hashTable + (hash << ldmParams.bucketSizeLog); +} + +/* ZSTD_ldm_insertEntry() : + * Insert the entry with corresponding hash into the hash table */ +static void ZSTD_ldm_insertEntry(ldmState_t* ldmState, + size_t const hash, const ldmEntry_t entry, + ldmParams_t const ldmParams) +{ + BYTE* const pOffset = ldmState->bucketOffsets + hash; + unsigned const offset = *pOffset; + + *(ZSTD_ldm_getBucket(ldmState, hash, ldmParams) + offset) = entry; + *pOffset = (BYTE)((offset + 1) & ((1u << ldmParams.bucketSizeLog) - 1)); + +} + +/* ZSTD_ldm_countBackwardsMatch() : + * Returns the number of bytes that match backwards before pIn and pMatch. + * + * We count only bytes where pMatch >= pBase and pIn >= pAnchor. */ +static size_t ZSTD_ldm_countBackwardsMatch( + const BYTE* pIn, const BYTE* pAnchor, + const BYTE* pMatch, const BYTE* pMatchBase) +{ + size_t matchLength = 0; + while (pIn > pAnchor && pMatch > pMatchBase && pIn[-1] == pMatch[-1]) { + pIn--; + pMatch--; + matchLength++; + } + return matchLength; +} + +/* ZSTD_ldm_countBackwardsMatch_2segments() : + * Returns the number of bytes that match backwards from pMatch, + * even with the backwards match spanning 2 different segments. + * + * On reaching `pMatchBase`, start counting from mEnd */ +static size_t ZSTD_ldm_countBackwardsMatch_2segments( + const BYTE* pIn, const BYTE* pAnchor, + const BYTE* pMatch, const BYTE* pMatchBase, + const BYTE* pExtDictStart, const BYTE* pExtDictEnd) +{ + size_t matchLength = ZSTD_ldm_countBackwardsMatch(pIn, pAnchor, pMatch, pMatchBase); + if (pMatch - matchLength != pMatchBase || pMatchBase == pExtDictStart) { + /* If backwards match is entirely in the extDict or prefix, immediately return */ + return matchLength; + } + DEBUGLOG(7, "ZSTD_ldm_countBackwardsMatch_2segments: found 2-parts backwards match (length in prefix==%zu)", matchLength); + matchLength += ZSTD_ldm_countBackwardsMatch(pIn - matchLength, pAnchor, pExtDictEnd, pExtDictStart); + DEBUGLOG(7, "final backwards match length = %zu", matchLength); + return matchLength; +} + +/* ZSTD_ldm_fillFastTables() : + * + * Fills the relevant tables for the ZSTD_fast and ZSTD_dfast strategies. + * This is similar to ZSTD_loadDictionaryContent. + * + * The tables for the other strategies are filled within their + * block compressors. */ +static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms, + void const* end) +{ + const BYTE* const iend = (const BYTE*)end; + + switch(ms->cParams.strategy) + { + case ZSTD_fast: + ZSTD_fillHashTable(ms, iend, ZSTD_dtlm_fast); + break; + + case ZSTD_dfast: + ZSTD_fillDoubleHashTable(ms, iend, ZSTD_dtlm_fast); + break; + + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + case ZSTD_btlazy2: + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: + break; + default: + assert(0); /* not possible : not a valid strategy id */ + } + + return 0; +} + +void ZSTD_ldm_fillHashTable( + ldmState_t* ldmState, const BYTE* ip, + const BYTE* iend, ldmParams_t const* params) +{ + U32 const minMatchLength = params->minMatchLength; + U32 const hBits = params->hashLog - params->bucketSizeLog; + BYTE const* const base = ldmState->window.base; + BYTE const* const istart = ip; + ldmRollingHashState_t hashState; + size_t* const splits = ldmState->splitIndices; + unsigned numSplits; + + DEBUGLOG(5, "ZSTD_ldm_fillHashTable"); + + ZSTD_ldm_gear_init(&hashState, params); + while (ip < iend) { + size_t hashed; + unsigned n; + + numSplits = 0; + hashed = ZSTD_ldm_gear_feed(&hashState, ip, iend - ip, splits, &numSplits); + + for (n = 0; n < numSplits; n++) { + if (ip + splits[n] >= istart + minMatchLength) { + BYTE const* const split = ip + splits[n] - minMatchLength; + U64 const xxhash = xxh64(split, minMatchLength, 0); + U32 const hash = (U32)(xxhash & (((U32)1 << hBits) - 1)); + ldmEntry_t entry; + + entry.offset = (U32)(split - base); + entry.checksum = (U32)(xxhash >> 32); + ZSTD_ldm_insertEntry(ldmState, hash, entry, *params); + } + } + + ip += hashed; + } +} + + +/* ZSTD_ldm_limitTableUpdate() : + * + * Sets cctx->nextToUpdate to a position corresponding closer to anchor + * if it is far way + * (after a long match, only update tables a limited amount). */ +static void ZSTD_ldm_limitTableUpdate(ZSTD_matchState_t* ms, const BYTE* anchor) +{ + U32 const curr = (U32)(anchor - ms->window.base); + if (curr > ms->nextToUpdate + 1024) { + ms->nextToUpdate = + curr - MIN(512, curr - ms->nextToUpdate - 1024); + } +} + +static size_t ZSTD_ldm_generateSequences_internal( + ldmState_t* ldmState, rawSeqStore_t* rawSeqStore, + ldmParams_t const* params, void const* src, size_t srcSize) +{ + /* LDM parameters */ + int const extDict = ZSTD_window_hasExtDict(ldmState->window); + U32 const minMatchLength = params->minMatchLength; + U32 const entsPerBucket = 1U << params->bucketSizeLog; + U32 const hBits = params->hashLog - params->bucketSizeLog; + /* Prefix and extDict parameters */ + U32 const dictLimit = ldmState->window.dictLimit; + U32 const lowestIndex = extDict ? ldmState->window.lowLimit : dictLimit; + BYTE const* const base = ldmState->window.base; + BYTE const* const dictBase = extDict ? ldmState->window.dictBase : NULL; + BYTE const* const dictStart = extDict ? dictBase + lowestIndex : NULL; + BYTE const* const dictEnd = extDict ? dictBase + dictLimit : NULL; + BYTE const* const lowPrefixPtr = base + dictLimit; + /* Input bounds */ + BYTE const* const istart = (BYTE const*)src; + BYTE const* const iend = istart + srcSize; + BYTE const* const ilimit = iend - HASH_READ_SIZE; + /* Input positions */ + BYTE const* anchor = istart; + BYTE const* ip = istart; + /* Rolling hash state */ + ldmRollingHashState_t hashState; + /* Arrays for staged-processing */ + size_t* const splits = ldmState->splitIndices; + ldmMatchCandidate_t* const candidates = ldmState->matchCandidates; + unsigned numSplits; + + if (srcSize < minMatchLength) + return iend - anchor; + + /* Initialize the rolling hash state with the first minMatchLength bytes */ + ZSTD_ldm_gear_init(&hashState, params); + { + size_t n = 0; + + while (n < minMatchLength) { + numSplits = 0; + n += ZSTD_ldm_gear_feed(&hashState, ip + n, minMatchLength - n, + splits, &numSplits); + } + ip += minMatchLength; + } + + while (ip < ilimit) { + size_t hashed; + unsigned n; + + numSplits = 0; + hashed = ZSTD_ldm_gear_feed(&hashState, ip, ilimit - ip, + splits, &numSplits); + + for (n = 0; n < numSplits; n++) { + BYTE const* const split = ip + splits[n] - minMatchLength; + U64 const xxhash = xxh64(split, minMatchLength, 0); + U32 const hash = (U32)(xxhash & (((U32)1 << hBits) - 1)); + + candidates[n].split = split; + candidates[n].hash = hash; + candidates[n].checksum = (U32)(xxhash >> 32); + candidates[n].bucket = ZSTD_ldm_getBucket(ldmState, hash, *params); + PREFETCH_L1(candidates[n].bucket); + } + + for (n = 0; n < numSplits; n++) { + size_t forwardMatchLength = 0, backwardMatchLength = 0, + bestMatchLength = 0, mLength; + BYTE const* const split = candidates[n].split; + U32 const checksum = candidates[n].checksum; + U32 const hash = candidates[n].hash; + ldmEntry_t* const bucket = candidates[n].bucket; + ldmEntry_t const* cur; + ldmEntry_t const* bestEntry = NULL; + ldmEntry_t newEntry; + + newEntry.offset = (U32)(split - base); + newEntry.checksum = checksum; + + /* If a split point would generate a sequence overlapping with + * the previous one, we merely register it in the hash table and + * move on */ + if (split < anchor) { + ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params); + continue; + } + + for (cur = bucket; cur < bucket + entsPerBucket; cur++) { + size_t curForwardMatchLength, curBackwardMatchLength, + curTotalMatchLength; + if (cur->checksum != checksum || cur->offset <= lowestIndex) { + continue; + } + if (extDict) { + BYTE const* const curMatchBase = + cur->offset < dictLimit ? dictBase : base; + BYTE const* const pMatch = curMatchBase + cur->offset; + BYTE const* const matchEnd = + cur->offset < dictLimit ? dictEnd : iend; + BYTE const* const lowMatchPtr = + cur->offset < dictLimit ? dictStart : lowPrefixPtr; + curForwardMatchLength = + ZSTD_count_2segments(split, pMatch, iend, matchEnd, lowPrefixPtr); + if (curForwardMatchLength < minMatchLength) { + continue; + } + curBackwardMatchLength = ZSTD_ldm_countBackwardsMatch_2segments( + split, anchor, pMatch, lowMatchPtr, dictStart, dictEnd); + } else { /* !extDict */ + BYTE const* const pMatch = base + cur->offset; + curForwardMatchLength = ZSTD_count(split, pMatch, iend); + if (curForwardMatchLength < minMatchLength) { + continue; + } + curBackwardMatchLength = + ZSTD_ldm_countBackwardsMatch(split, anchor, pMatch, lowPrefixPtr); + } + curTotalMatchLength = curForwardMatchLength + curBackwardMatchLength; + + if (curTotalMatchLength > bestMatchLength) { + bestMatchLength = curTotalMatchLength; + forwardMatchLength = curForwardMatchLength; + backwardMatchLength = curBackwardMatchLength; + bestEntry = cur; + } + } + + /* No match found -- insert an entry into the hash table + * and process the next candidate match */ + if (bestEntry == NULL) { + ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params); + continue; + } + + /* Match found */ + mLength = forwardMatchLength + backwardMatchLength; + { + U32 const offset = (U32)(split - base) - bestEntry->offset; + rawSeq* const seq = rawSeqStore->seq + rawSeqStore->size; + + /* Out of sequence storage */ + if (rawSeqStore->size == rawSeqStore->capacity) + return ERROR(dstSize_tooSmall); + seq->litLength = (U32)(split - backwardMatchLength - anchor); + seq->matchLength = (U32)mLength; + seq->offset = offset; + rawSeqStore->size++; + } + + /* Insert the current entry into the hash table --- it must be + * done after the previous block to avoid clobbering bestEntry */ + ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params); + + anchor = split + forwardMatchLength; + } + + ip += hashed; + } + + return iend - anchor; +} + +/*! ZSTD_ldm_reduceTable() : + * reduce table indexes by `reducerValue` */ +static void ZSTD_ldm_reduceTable(ldmEntry_t* const table, U32 const size, + U32 const reducerValue) +{ + U32 u; + for (u = 0; u < size; u++) { + if (table[u].offset < reducerValue) table[u].offset = 0; + else table[u].offset -= reducerValue; + } +} + +size_t ZSTD_ldm_generateSequences( + ldmState_t* ldmState, rawSeqStore_t* sequences, + ldmParams_t const* params, void const* src, size_t srcSize) +{ + U32 const maxDist = 1U << params->windowLog; + BYTE const* const istart = (BYTE const*)src; + BYTE const* const iend = istart + srcSize; + size_t const kMaxChunkSize = 1 << 20; + size_t const nbChunks = (srcSize / kMaxChunkSize) + ((srcSize % kMaxChunkSize) != 0); + size_t chunk; + size_t leftoverSize = 0; + + assert(ZSTD_CHUNKSIZE_MAX >= kMaxChunkSize); + /* Check that ZSTD_window_update() has been called for this chunk prior + * to passing it to this function. + */ + assert(ldmState->window.nextSrc >= (BYTE const*)src + srcSize); + /* The input could be very large (in zstdmt), so it must be broken up into + * chunks to enforce the maximum distance and handle overflow correction. + */ + assert(sequences->pos <= sequences->size); + assert(sequences->size <= sequences->capacity); + for (chunk = 0; chunk < nbChunks && sequences->size < sequences->capacity; ++chunk) { + BYTE const* const chunkStart = istart + chunk * kMaxChunkSize; + size_t const remaining = (size_t)(iend - chunkStart); + BYTE const *const chunkEnd = + (remaining < kMaxChunkSize) ? iend : chunkStart + kMaxChunkSize; + size_t const chunkSize = chunkEnd - chunkStart; + size_t newLeftoverSize; + size_t const prevSize = sequences->size; + + assert(chunkStart < iend); + /* 1. Perform overflow correction if necessary. */ + if (ZSTD_window_needOverflowCorrection(ldmState->window, chunkEnd)) { + U32 const ldmHSize = 1U << params->hashLog; + U32 const correction = ZSTD_window_correctOverflow( + &ldmState->window, /* cycleLog */ 0, maxDist, chunkStart); + ZSTD_ldm_reduceTable(ldmState->hashTable, ldmHSize, correction); + /* invalidate dictionaries on overflow correction */ + ldmState->loadedDictEnd = 0; + } + /* 2. We enforce the maximum offset allowed. + * + * kMaxChunkSize should be small enough that we don't lose too much of + * the window through early invalidation. + * TODO: * Test the chunk size. + * * Try invalidation after the sequence generation and test the + * the offset against maxDist directly. + * + * NOTE: Because of dictionaries + sequence splitting we MUST make sure + * that any offset used is valid at the END of the sequence, since it may + * be split into two sequences. This condition holds when using + * ZSTD_window_enforceMaxDist(), but if we move to checking offsets + * against maxDist directly, we'll have to carefully handle that case. + */ + ZSTD_window_enforceMaxDist(&ldmState->window, chunkEnd, maxDist, &ldmState->loadedDictEnd, NULL); + /* 3. Generate the sequences for the chunk, and get newLeftoverSize. */ + newLeftoverSize = ZSTD_ldm_generateSequences_internal( + ldmState, sequences, params, chunkStart, chunkSize); + if (ZSTD_isError(newLeftoverSize)) + return newLeftoverSize; + /* 4. We add the leftover literals from previous iterations to the first + * newly generated sequence, or add the `newLeftoverSize` if none are + * generated. + */ + /* Prepend the leftover literals from the last call */ + if (prevSize < sequences->size) { + sequences->seq[prevSize].litLength += (U32)leftoverSize; + leftoverSize = newLeftoverSize; + } else { + assert(newLeftoverSize == chunkSize); + leftoverSize += chunkSize; + } + } + return 0; +} + +void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch) { + while (srcSize > 0 && rawSeqStore->pos < rawSeqStore->size) { + rawSeq* seq = rawSeqStore->seq + rawSeqStore->pos; + if (srcSize <= seq->litLength) { + /* Skip past srcSize literals */ + seq->litLength -= (U32)srcSize; + return; + } + srcSize -= seq->litLength; + seq->litLength = 0; + if (srcSize < seq->matchLength) { + /* Skip past the first srcSize of the match */ + seq->matchLength -= (U32)srcSize; + if (seq->matchLength < minMatch) { + /* The match is too short, omit it */ + if (rawSeqStore->pos + 1 < rawSeqStore->size) { + seq[1].litLength += seq[0].matchLength; + } + rawSeqStore->pos++; + } + return; + } + srcSize -= seq->matchLength; + seq->matchLength = 0; + rawSeqStore->pos++; + } +} + +/* + * If the sequence length is longer than remaining then the sequence is split + * between this block and the next. + * + * Returns the current sequence to handle, or if the rest of the block should + * be literals, it returns a sequence with offset == 0. + */ +static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore, + U32 const remaining, U32 const minMatch) +{ + rawSeq sequence = rawSeqStore->seq[rawSeqStore->pos]; + assert(sequence.offset > 0); + /* Likely: No partial sequence */ + if (remaining >= sequence.litLength + sequence.matchLength) { + rawSeqStore->pos++; + return sequence; + } + /* Cut the sequence short (offset == 0 ==> rest is literals). */ + if (remaining <= sequence.litLength) { + sequence.offset = 0; + } else if (remaining < sequence.litLength + sequence.matchLength) { + sequence.matchLength = remaining - sequence.litLength; + if (sequence.matchLength < minMatch) { + sequence.offset = 0; + } + } + /* Skip past `remaining` bytes for the future sequences. */ + ZSTD_ldm_skipSequences(rawSeqStore, remaining, minMatch); + return sequence; +} + +void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) { + U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes); + while (currPos && rawSeqStore->pos < rawSeqStore->size) { + rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos]; + if (currPos >= currSeq.litLength + currSeq.matchLength) { + currPos -= currSeq.litLength + currSeq.matchLength; + rawSeqStore->pos++; + } else { + rawSeqStore->posInSequence = currPos; + break; + } + } + if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) { + rawSeqStore->posInSequence = 0; + } +} + +size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + unsigned const minMatch = cParams->minMatch; + ZSTD_blockCompressor const blockCompressor = + ZSTD_selectBlockCompressor(cParams->strategy, ZSTD_matchState_dictMode(ms)); + /* Input bounds */ + BYTE const* const istart = (BYTE const*)src; + BYTE const* const iend = istart + srcSize; + /* Input positions */ + BYTE const* ip = istart; + + DEBUGLOG(5, "ZSTD_ldm_blockCompress: srcSize=%zu", srcSize); + /* If using opt parser, use LDMs only as candidates rather than always accepting them */ + if (cParams->strategy >= ZSTD_btopt) { + size_t lastLLSize; + ms->ldmSeqStore = rawSeqStore; + lastLLSize = blockCompressor(ms, seqStore, rep, src, srcSize); + ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore, srcSize); + return lastLLSize; + } + + assert(rawSeqStore->pos <= rawSeqStore->size); + assert(rawSeqStore->size <= rawSeqStore->capacity); + /* Loop through each sequence and apply the block compressor to the literals */ + while (rawSeqStore->pos < rawSeqStore->size && ip < iend) { + /* maybeSplitSequence updates rawSeqStore->pos */ + rawSeq const sequence = maybeSplitSequence(rawSeqStore, + (U32)(iend - ip), minMatch); + int i; + /* End signal */ + if (sequence.offset == 0) + break; + + assert(ip + sequence.litLength + sequence.matchLength <= iend); + + /* Fill tables for block compressor */ + ZSTD_ldm_limitTableUpdate(ms, ip); + ZSTD_ldm_fillFastTables(ms, ip); + /* Run the block compressor */ + DEBUGLOG(5, "pos %u : calling block compressor on segment of size %u", (unsigned)(ip-istart), sequence.litLength); + { + size_t const newLitLength = + blockCompressor(ms, seqStore, rep, ip, sequence.litLength); + ip += sequence.litLength; + /* Update the repcodes */ + for (i = ZSTD_REP_NUM - 1; i > 0; i--) + rep[i] = rep[i-1]; + rep[0] = sequence.offset; + /* Store the sequence */ + ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength, iend, + sequence.offset + ZSTD_REP_MOVE, + sequence.matchLength - MINMATCH); + ip += sequence.matchLength; + } + } + /* Fill the tables for the block compressor */ + ZSTD_ldm_limitTableUpdate(ms, ip); + ZSTD_ldm_fillFastTables(ms, ip); + /* Compress the last literals */ + return blockCompressor(ms, seqStore, rep, ip, iend - ip); +} diff --git a/lib/zstd/compress/zstd_ldm.h b/lib/zstd/compress/zstd_ldm.h new file mode 100644 index 000000000000..25b25270b72e --- /dev/null +++ b/lib/zstd/compress/zstd_ldm.h @@ -0,0 +1,110 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_LDM_H +#define ZSTD_LDM_H + + +#include "zstd_compress_internal.h" /* ldmParams_t, U32 */ +#include /* ZSTD_CCtx, size_t */ + +/*-************************************* +* Long distance matching +***************************************/ + +#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_LIMIT_DEFAULT + +void ZSTD_ldm_fillHashTable( + ldmState_t* state, const BYTE* ip, + const BYTE* iend, ldmParams_t const* params); + +/* + * ZSTD_ldm_generateSequences(): + * + * Generates the sequences using the long distance match finder. + * Generates long range matching sequences in `sequences`, which parse a prefix + * of the source. `sequences` must be large enough to store every sequence, + * which can be checked with `ZSTD_ldm_getMaxNbSeq()`. + * @returns 0 or an error code. + * + * NOTE: The user must have called ZSTD_window_update() for all of the input + * they have, even if they pass it to ZSTD_ldm_generateSequences() in chunks. + * NOTE: This function returns an error if it runs out of space to store + * sequences. + */ +size_t ZSTD_ldm_generateSequences( + ldmState_t* ldms, rawSeqStore_t* sequences, + ldmParams_t const* params, void const* src, size_t srcSize); + +/* + * ZSTD_ldm_blockCompress(): + * + * Compresses a block using the predefined sequences, along with a secondary + * block compressor. The literals section of every sequence is passed to the + * secondary block compressor, and those sequences are interspersed with the + * predefined sequences. Returns the length of the last literals. + * Updates `rawSeqStore.pos` to indicate how many sequences have been consumed. + * `rawSeqStore.seq` may also be updated to split the last sequence between two + * blocks. + * @return The length of the last literals. + * + * NOTE: The source must be at most the maximum block size, but the predefined + * sequences can be any size, and may be longer than the block. In the case that + * they are longer than the block, the last sequences may need to be split into + * two. We handle that case correctly, and update `rawSeqStore` appropriately. + * NOTE: This function does not return any errors. + */ +size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +/* + * ZSTD_ldm_skipSequences(): + * + * Skip past `srcSize` bytes worth of sequences in `rawSeqStore`. + * Avoids emitting matches less than `minMatch` bytes. + * Must be called for data that is not passed to ZSTD_ldm_blockCompress(). + */ +void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, + U32 const minMatch); + +/* ZSTD_ldm_skipRawSeqStoreBytes(): + * Moves forward in rawSeqStore by nbBytes, updating fields 'pos' and 'posInSequence'. + * Not to be used in conjunction with ZSTD_ldm_skipSequences(). + * Must be called for data with is not passed to ZSTD_ldm_blockCompress(). + */ +void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes); + +/* ZSTD_ldm_getTableSize() : + * Estimate the space needed for long distance matching tables or 0 if LDM is + * disabled. + */ +size_t ZSTD_ldm_getTableSize(ldmParams_t params); + +/* ZSTD_ldm_getSeqSpace() : + * Return an upper bound on the number of sequences that can be produced by + * the long distance matcher, or 0 if LDM is disabled. + */ +size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize); + +/* ZSTD_ldm_adjustParameters() : + * If the params->hashRateLog is not set, set it to its default value based on + * windowLog and params->hashLog. + * + * Ensures that params->bucketSizeLog is <= params->hashLog (setting it to + * params->hashLog if it is not). + * + * Ensures that the minMatchLength >= targetLength during optimal parsing. + */ +void ZSTD_ldm_adjustParameters(ldmParams_t* params, + ZSTD_compressionParameters const* cParams); + + +#endif /* ZSTD_FAST_H */ diff --git a/lib/zstd/compress/zstd_ldm_geartab.h b/lib/zstd/compress/zstd_ldm_geartab.h new file mode 100644 index 000000000000..e5c24d856b0a --- /dev/null +++ b/lib/zstd/compress/zstd_ldm_geartab.h @@ -0,0 +1,103 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_LDM_GEARTAB_H +#define ZSTD_LDM_GEARTAB_H + +static U64 ZSTD_ldm_gearTab[256] = { + 0xf5b8f72c5f77775c, 0x84935f266b7ac412, 0xb647ada9ca730ccc, + 0xb065bb4b114fb1de, 0x34584e7e8c3a9fd0, 0x4e97e17c6ae26b05, + 0x3a03d743bc99a604, 0xcecd042422c4044f, 0x76de76c58524259e, + 0x9c8528f65badeaca, 0x86563706e2097529, 0x2902475fa375d889, + 0xafb32a9739a5ebe6, 0xce2714da3883e639, 0x21eaf821722e69e, + 0x37b628620b628, 0x49a8d455d88caf5, 0x8556d711e6958140, + 0x4f7ae74fc605c1f, 0x829f0c3468bd3a20, 0x4ffdc885c625179e, + 0x8473de048a3daf1b, 0x51008822b05646b2, 0x69d75d12b2d1cc5f, + 0x8c9d4a19159154bc, 0xc3cc10f4abbd4003, 0xd06ddc1cecb97391, + 0xbe48e6e7ed80302e, 0x3481db31cee03547, 0xacc3f67cdaa1d210, + 0x65cb771d8c7f96cc, 0x8eb27177055723dd, 0xc789950d44cd94be, + 0x934feadc3700b12b, 0x5e485f11edbdf182, 0x1e2e2a46fd64767a, + 0x2969ca71d82efa7c, 0x9d46e9935ebbba2e, 0xe056b67e05e6822b, + 0x94d73f55739d03a0, 0xcd7010bdb69b5a03, 0x455ef9fcd79b82f4, + 0x869cb54a8749c161, 0x38d1a4fa6185d225, 0xb475166f94bbe9bb, + 0xa4143548720959f1, 0x7aed4780ba6b26ba, 0xd0ce264439e02312, + 0x84366d746078d508, 0xa8ce973c72ed17be, 0x21c323a29a430b01, + 0x9962d617e3af80ee, 0xab0ce91d9c8cf75b, 0x530e8ee6d19a4dbc, + 0x2ef68c0cf53f5d72, 0xc03a681640a85506, 0x496e4e9f9c310967, + 0x78580472b59b14a0, 0x273824c23b388577, 0x66bf923ad45cb553, + 0x47ae1a5a2492ba86, 0x35e304569e229659, 0x4765182a46870b6f, + 0x6cbab625e9099412, 0xddac9a2e598522c1, 0x7172086e666624f2, + 0xdf5003ca503b7837, 0x88c0c1db78563d09, 0x58d51865acfc289d, + 0x177671aec65224f1, 0xfb79d8a241e967d7, 0x2be1e101cad9a49a, + 0x6625682f6e29186b, 0x399553457ac06e50, 0x35dffb4c23abb74, + 0x429db2591f54aade, 0xc52802a8037d1009, 0x6acb27381f0b25f3, + 0xf45e2551ee4f823b, 0x8b0ea2d99580c2f7, 0x3bed519cbcb4e1e1, + 0xff452823dbb010a, 0x9d42ed614f3dd267, 0x5b9313c06257c57b, + 0xa114b8008b5e1442, 0xc1fe311c11c13d4b, 0x66e8763ea34c5568, + 0x8b982af1c262f05d, 0xee8876faaa75fbb7, 0x8a62a4d0d172bb2a, + 0xc13d94a3b7449a97, 0x6dbbba9dc15d037c, 0xc786101f1d92e0f1, + 0xd78681a907a0b79b, 0xf61aaf2962c9abb9, 0x2cfd16fcd3cb7ad9, + 0x868c5b6744624d21, 0x25e650899c74ddd7, 0xba042af4a7c37463, + 0x4eb1a539465a3eca, 0xbe09dbf03b05d5ca, 0x774e5a362b5472ba, + 0x47a1221229d183cd, 0x504b0ca18ef5a2df, 0xdffbdfbde2456eb9, + 0x46cd2b2fbee34634, 0xf2aef8fe819d98c3, 0x357f5276d4599d61, + 0x24a5483879c453e3, 0x88026889192b4b9, 0x28da96671782dbec, + 0x4ef37c40588e9aaa, 0x8837b90651bc9fb3, 0xc164f741d3f0e5d6, + 0xbc135a0a704b70ba, 0x69cd868f7622ada, 0xbc37ba89e0b9c0ab, + 0x47c14a01323552f6, 0x4f00794bacee98bb, 0x7107de7d637a69d5, + 0x88af793bb6f2255e, 0xf3c6466b8799b598, 0xc288c616aa7f3b59, + 0x81ca63cf42fca3fd, 0x88d85ace36a2674b, 0xd056bd3792389e7, + 0xe55c396c4e9dd32d, 0xbefb504571e6c0a6, 0x96ab32115e91e8cc, + 0xbf8acb18de8f38d1, 0x66dae58801672606, 0x833b6017872317fb, + 0xb87c16f2d1c92864, 0xdb766a74e58b669c, 0x89659f85c61417be, + 0xc8daad856011ea0c, 0x76a4b565b6fe7eae, 0xa469d085f6237312, + 0xaaf0365683a3e96c, 0x4dbb746f8424f7b8, 0x638755af4e4acc1, + 0x3d7807f5bde64486, 0x17be6d8f5bbb7639, 0x903f0cd44dc35dc, + 0x67b672eafdf1196c, 0xa676ff93ed4c82f1, 0x521d1004c5053d9d, + 0x37ba9ad09ccc9202, 0x84e54d297aacfb51, 0xa0b4b776a143445, + 0x820d471e20b348e, 0x1874383cb83d46dc, 0x97edeec7a1efe11c, + 0xb330e50b1bdc42aa, 0x1dd91955ce70e032, 0xa514cdb88f2939d5, + 0x2791233fd90db9d3, 0x7b670a4cc50f7a9b, 0x77c07d2a05c6dfa5, + 0xe3778b6646d0a6fa, 0xb39c8eda47b56749, 0x933ed448addbef28, + 0xaf846af6ab7d0bf4, 0xe5af208eb666e49, 0x5e6622f73534cd6a, + 0x297daeca42ef5b6e, 0x862daef3d35539a6, 0xe68722498f8e1ea9, + 0x981c53093dc0d572, 0xfa09b0bfbf86fbf5, 0x30b1e96166219f15, + 0x70e7d466bdc4fb83, 0x5a66736e35f2a8e9, 0xcddb59d2b7c1baef, + 0xd6c7d247d26d8996, 0xea4e39eac8de1ba3, 0x539c8bb19fa3aff2, + 0x9f90e4c5fd508d8, 0xa34e5956fbaf3385, 0x2e2f8e151d3ef375, + 0x173691e9b83faec1, 0xb85a8d56bf016379, 0x8382381267408ae3, + 0xb90f901bbdc0096d, 0x7c6ad32933bcec65, 0x76bb5e2f2c8ad595, + 0x390f851a6cf46d28, 0xc3e6064da1c2da72, 0xc52a0c101cfa5389, + 0xd78eaf84a3fbc530, 0x3781b9e2288b997e, 0x73c2f6dea83d05c4, + 0x4228e364c5b5ed7, 0x9d7a3edf0da43911, 0x8edcfeda24686756, + 0x5e7667a7b7a9b3a1, 0x4c4f389fa143791d, 0xb08bc1023da7cddc, + 0x7ab4be3ae529b1cc, 0x754e6132dbe74ff9, 0x71635442a839df45, + 0x2f6fb1643fbe52de, 0x961e0a42cf7a8177, 0xf3b45d83d89ef2ea, + 0xee3de4cf4a6e3e9b, 0xcd6848542c3295e7, 0xe4cee1664c78662f, + 0x9947548b474c68c4, 0x25d73777a5ed8b0b, 0xc915b1d636b7fc, + 0x21c2ba75d9b0d2da, 0x5f6b5dcf608a64a1, 0xdcf333255ff9570c, + 0x633b922418ced4ee, 0xc136dde0b004b34a, 0x58cc83b05d4b2f5a, + 0x5eb424dda28e42d2, 0x62df47369739cd98, 0xb4e0b42485e4ce17, + 0x16e1f0c1f9a8d1e7, 0x8ec3916707560ebf, 0x62ba6e2df2cc9db3, + 0xcbf9f4ff77d83a16, 0x78d9d7d07d2bbcc4, 0xef554ce1e02c41f4, + 0x8d7581127eccf94d, 0xa9b53336cb3c8a05, 0x38c42c0bf45c4f91, + 0x640893cdf4488863, 0x80ec34bc575ea568, 0x39f324f5b48eaa40, + 0xe9d9ed1f8eff527f, 0x9224fc058cc5a214, 0xbaba00b04cfe7741, + 0x309a9f120fcf52af, 0xa558f3ec65626212, 0x424bec8b7adabe2f, + 0x41622513a6aea433, 0xb88da2d5324ca798, 0xd287733b245528a4, + 0x9a44697e6d68aec3, 0x7b1093be2f49bb28, 0x50bbec632e3d8aad, + 0x6cd90723e1ea8283, 0x897b9e7431b02bf3, 0x219efdcb338a7047, + 0x3b0311f0a27c0656, 0xdb17bf91c0db96e7, 0x8cd4fd6b4e85a5b2, + 0xfab071054ba6409d, 0x40d6fe831fa9dfd9, 0xaf358debad7d791e, + 0xeb8d0e25a65e3e58, 0xbbcbd3df14e08580, 0xcf751f27ecdab2b, + 0x2b4da14f2613d8f4 +}; + +#endif /* ZSTD_LDM_GEARTAB_H */ diff --git a/lib/zstd/compress/zstd_opt.c b/lib/zstd/compress/zstd_opt.c new file mode 100644 index 000000000000..956ee6b2ae61 --- /dev/null +++ b/lib/zstd/compress/zstd_opt.c @@ -0,0 +1,1345 @@ +/* + * Copyright (c) Przemyslaw Skibinski, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_compress_internal.h" +#include "hist.h" +#include "zstd_opt.h" + + +#define ZSTD_LITFREQ_ADD 2 /* scaling factor for litFreq, so that frequencies adapt faster to new stats */ +#define ZSTD_FREQ_DIV 4 /* log factor when using previous stats to init next stats */ +#define ZSTD_MAX_PRICE (1<<30) + +#define ZSTD_PREDEF_THRESHOLD 1024 /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */ + + +/*-************************************* +* Price functions for optimal parser +***************************************/ + +#if 0 /* approximation at bit level */ +# define BITCOST_ACCURACY 0 +# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) +# define WEIGHT(stat) ((void)opt, ZSTD_bitWeight(stat)) +#elif 0 /* fractional bit accuracy */ +# define BITCOST_ACCURACY 8 +# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) +# define WEIGHT(stat,opt) ((void)opt, ZSTD_fracWeight(stat)) +#else /* opt==approx, ultra==accurate */ +# define BITCOST_ACCURACY 8 +# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) +# define WEIGHT(stat,opt) (opt ? ZSTD_fracWeight(stat) : ZSTD_bitWeight(stat)) +#endif + +MEM_STATIC U32 ZSTD_bitWeight(U32 stat) +{ + return (ZSTD_highbit32(stat+1) * BITCOST_MULTIPLIER); +} + +MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat) +{ + U32 const stat = rawStat + 1; + U32 const hb = ZSTD_highbit32(stat); + U32 const BWeight = hb * BITCOST_MULTIPLIER; + U32 const FWeight = (stat << BITCOST_ACCURACY) >> hb; + U32 const weight = BWeight + FWeight; + assert(hb + BITCOST_ACCURACY < 31); + return weight; +} + +#if (DEBUGLEVEL>=2) +/* debugging function, + * @return price in bytes as fractional value + * for debug messages only */ +MEM_STATIC double ZSTD_fCost(U32 price) +{ + return (double)price / (BITCOST_MULTIPLIER*8); +} +#endif + +static int ZSTD_compressedLiterals(optState_t const* const optPtr) +{ + return optPtr->literalCompressionMode != ZSTD_lcm_uncompressed; +} + +static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel) +{ + if (ZSTD_compressedLiterals(optPtr)) + optPtr->litSumBasePrice = WEIGHT(optPtr->litSum, optLevel); + optPtr->litLengthSumBasePrice = WEIGHT(optPtr->litLengthSum, optLevel); + optPtr->matchLengthSumBasePrice = WEIGHT(optPtr->matchLengthSum, optLevel); + optPtr->offCodeSumBasePrice = WEIGHT(optPtr->offCodeSum, optLevel); +} + + +/* ZSTD_downscaleStat() : + * reduce all elements in table by a factor 2^(ZSTD_FREQ_DIV+malus) + * return the resulting sum of elements */ +static U32 ZSTD_downscaleStat(unsigned* table, U32 lastEltIndex, int malus) +{ + U32 s, sum=0; + DEBUGLOG(5, "ZSTD_downscaleStat (nbElts=%u)", (unsigned)lastEltIndex+1); + assert(ZSTD_FREQ_DIV+malus > 0 && ZSTD_FREQ_DIV+malus < 31); + for (s=0; s> (ZSTD_FREQ_DIV+malus)); + sum += table[s]; + } + return sum; +} + +/* ZSTD_rescaleFreqs() : + * if first block (detected by optPtr->litLengthSum == 0) : init statistics + * take hints from dictionary if there is one + * or init from zero, using src for literals stats, or flat 1 for match symbols + * otherwise downscale existing stats, to be used as seed for next block. + */ +static void +ZSTD_rescaleFreqs(optState_t* const optPtr, + const BYTE* const src, size_t const srcSize, + int const optLevel) +{ + int const compressedLiterals = ZSTD_compressedLiterals(optPtr); + DEBUGLOG(5, "ZSTD_rescaleFreqs (srcSize=%u)", (unsigned)srcSize); + optPtr->priceType = zop_dynamic; + + if (optPtr->litLengthSum == 0) { /* first block : init */ + if (srcSize <= ZSTD_PREDEF_THRESHOLD) { /* heuristic */ + DEBUGLOG(5, "(srcSize <= ZSTD_PREDEF_THRESHOLD) => zop_predef"); + optPtr->priceType = zop_predef; + } + + assert(optPtr->symbolCosts != NULL); + if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) { + /* huffman table presumed generated by dictionary */ + optPtr->priceType = zop_dynamic; + + if (compressedLiterals) { + unsigned lit; + assert(optPtr->litFreq != NULL); + optPtr->litSum = 0; + for (lit=0; lit<=MaxLit; lit++) { + U32 const scaleLog = 11; /* scale to 2K */ + U32 const bitCost = HUF_getNbBits(optPtr->symbolCosts->huf.CTable, lit); + assert(bitCost <= scaleLog); + optPtr->litFreq[lit] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; + optPtr->litSum += optPtr->litFreq[lit]; + } } + + { unsigned ll; + FSE_CState_t llstate; + FSE_initCState(&llstate, optPtr->symbolCosts->fse.litlengthCTable); + optPtr->litLengthSum = 0; + for (ll=0; ll<=MaxLL; ll++) { + U32 const scaleLog = 10; /* scale to 1K */ + U32 const bitCost = FSE_getMaxNbBits(llstate.symbolTT, ll); + assert(bitCost < scaleLog); + optPtr->litLengthFreq[ll] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; + optPtr->litLengthSum += optPtr->litLengthFreq[ll]; + } } + + { unsigned ml; + FSE_CState_t mlstate; + FSE_initCState(&mlstate, optPtr->symbolCosts->fse.matchlengthCTable); + optPtr->matchLengthSum = 0; + for (ml=0; ml<=MaxML; ml++) { + U32 const scaleLog = 10; + U32 const bitCost = FSE_getMaxNbBits(mlstate.symbolTT, ml); + assert(bitCost < scaleLog); + optPtr->matchLengthFreq[ml] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; + optPtr->matchLengthSum += optPtr->matchLengthFreq[ml]; + } } + + { unsigned of; + FSE_CState_t ofstate; + FSE_initCState(&ofstate, optPtr->symbolCosts->fse.offcodeCTable); + optPtr->offCodeSum = 0; + for (of=0; of<=MaxOff; of++) { + U32 const scaleLog = 10; + U32 const bitCost = FSE_getMaxNbBits(ofstate.symbolTT, of); + assert(bitCost < scaleLog); + optPtr->offCodeFreq[of] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; + optPtr->offCodeSum += optPtr->offCodeFreq[of]; + } } + + } else { /* not a dictionary */ + + assert(optPtr->litFreq != NULL); + if (compressedLiterals) { + unsigned lit = MaxLit; + HIST_count_simple(optPtr->litFreq, &lit, src, srcSize); /* use raw first block to init statistics */ + optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1); + } + + { unsigned ll; + for (ll=0; ll<=MaxLL; ll++) + optPtr->litLengthFreq[ll] = 1; + } + optPtr->litLengthSum = MaxLL+1; + + { unsigned ml; + for (ml=0; ml<=MaxML; ml++) + optPtr->matchLengthFreq[ml] = 1; + } + optPtr->matchLengthSum = MaxML+1; + + { unsigned of; + for (of=0; of<=MaxOff; of++) + optPtr->offCodeFreq[of] = 1; + } + optPtr->offCodeSum = MaxOff+1; + + } + + } else { /* new block : re-use previous statistics, scaled down */ + + if (compressedLiterals) + optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1); + optPtr->litLengthSum = ZSTD_downscaleStat(optPtr->litLengthFreq, MaxLL, 0); + optPtr->matchLengthSum = ZSTD_downscaleStat(optPtr->matchLengthFreq, MaxML, 0); + optPtr->offCodeSum = ZSTD_downscaleStat(optPtr->offCodeFreq, MaxOff, 0); + } + + ZSTD_setBasePrices(optPtr, optLevel); +} + +/* ZSTD_rawLiteralsCost() : + * price of literals (only) in specified segment (which length can be 0). + * does not include price of literalLength symbol */ +static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength, + const optState_t* const optPtr, + int optLevel) +{ + if (litLength == 0) return 0; + + if (!ZSTD_compressedLiterals(optPtr)) + return (litLength << 3) * BITCOST_MULTIPLIER; /* Uncompressed - 8 bytes per literal. */ + + if (optPtr->priceType == zop_predef) + return (litLength*6) * BITCOST_MULTIPLIER; /* 6 bit per literal - no statistic used */ + + /* dynamic statistics */ + { U32 price = litLength * optPtr->litSumBasePrice; + U32 u; + for (u=0; u < litLength; u++) { + assert(WEIGHT(optPtr->litFreq[literals[u]], optLevel) <= optPtr->litSumBasePrice); /* literal cost should never be negative */ + price -= WEIGHT(optPtr->litFreq[literals[u]], optLevel); + } + return price; + } +} + +/* ZSTD_litLengthPrice() : + * cost of literalLength symbol */ +static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optPtr, int optLevel) +{ + if (optPtr->priceType == zop_predef) return WEIGHT(litLength, optLevel); + + /* dynamic statistics */ + { U32 const llCode = ZSTD_LLcode(litLength); + return (LL_bits[llCode] * BITCOST_MULTIPLIER) + + optPtr->litLengthSumBasePrice + - WEIGHT(optPtr->litLengthFreq[llCode], optLevel); + } +} + +/* ZSTD_getMatchPrice() : + * Provides the cost of the match part (offset + matchLength) of a sequence + * Must be combined with ZSTD_fullLiteralsCost() to get the full cost of a sequence. + * optLevel: when <2, favors small offset for decompression speed (improved cache efficiency) */ +FORCE_INLINE_TEMPLATE U32 +ZSTD_getMatchPrice(U32 const offset, + U32 const matchLength, + const optState_t* const optPtr, + int const optLevel) +{ + U32 price; + U32 const offCode = ZSTD_highbit32(offset+1); + U32 const mlBase = matchLength - MINMATCH; + assert(matchLength >= MINMATCH); + + if (optPtr->priceType == zop_predef) /* fixed scheme, do not use statistics */ + return WEIGHT(mlBase, optLevel) + ((16 + offCode) * BITCOST_MULTIPLIER); + + /* dynamic statistics */ + price = (offCode * BITCOST_MULTIPLIER) + (optPtr->offCodeSumBasePrice - WEIGHT(optPtr->offCodeFreq[offCode], optLevel)); + if ((optLevel<2) /*static*/ && offCode >= 20) + price += (offCode-19)*2 * BITCOST_MULTIPLIER; /* handicap for long distance offsets, favor decompression speed */ + + /* match Length */ + { U32 const mlCode = ZSTD_MLcode(mlBase); + price += (ML_bits[mlCode] * BITCOST_MULTIPLIER) + (optPtr->matchLengthSumBasePrice - WEIGHT(optPtr->matchLengthFreq[mlCode], optLevel)); + } + + price += BITCOST_MULTIPLIER / 5; /* heuristic : make matches a bit more costly to favor less sequences -> faster decompression speed */ + + DEBUGLOG(8, "ZSTD_getMatchPrice(ml:%u) = %u", matchLength, price); + return price; +} + +/* ZSTD_updateStats() : + * assumption : literals + litLengtn <= iend */ +static void ZSTD_updateStats(optState_t* const optPtr, + U32 litLength, const BYTE* literals, + U32 offsetCode, U32 matchLength) +{ + /* literals */ + if (ZSTD_compressedLiterals(optPtr)) { + U32 u; + for (u=0; u < litLength; u++) + optPtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD; + optPtr->litSum += litLength*ZSTD_LITFREQ_ADD; + } + + /* literal Length */ + { U32 const llCode = ZSTD_LLcode(litLength); + optPtr->litLengthFreq[llCode]++; + optPtr->litLengthSum++; + } + + /* match offset code (0-2=>repCode; 3+=>offset+2) */ + { U32 const offCode = ZSTD_highbit32(offsetCode+1); + assert(offCode <= MaxOff); + optPtr->offCodeFreq[offCode]++; + optPtr->offCodeSum++; + } + + /* match Length */ + { U32 const mlBase = matchLength - MINMATCH; + U32 const mlCode = ZSTD_MLcode(mlBase); + optPtr->matchLengthFreq[mlCode]++; + optPtr->matchLengthSum++; + } +} + + +/* ZSTD_readMINMATCH() : + * function safe only for comparisons + * assumption : memPtr must be at least 4 bytes before end of buffer */ +MEM_STATIC U32 ZSTD_readMINMATCH(const void* memPtr, U32 length) +{ + switch (length) + { + default : + case 4 : return MEM_read32(memPtr); + case 3 : if (MEM_isLittleEndian()) + return MEM_read32(memPtr)<<8; + else + return MEM_read32(memPtr)>>8; + } +} + + +/* Update hashTable3 up to ip (excluded) + Assumption : always within prefix (i.e. not within extDict) */ +static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_matchState_t* ms, + U32* nextToUpdate3, + const BYTE* const ip) +{ + U32* const hashTable3 = ms->hashTable3; + U32 const hashLog3 = ms->hashLog3; + const BYTE* const base = ms->window.base; + U32 idx = *nextToUpdate3; + U32 const target = (U32)(ip - base); + size_t const hash3 = ZSTD_hash3Ptr(ip, hashLog3); + assert(hashLog3 > 0); + + while(idx < target) { + hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx; + idx++; + } + + *nextToUpdate3 = target; + return hashTable3[hash3]; +} + + +/*-************************************* +* Binary Tree search +***************************************/ +/* ZSTD_insertBt1() : add one or multiple positions to tree. + * ip : assumed <= iend-8 . + * @return : nb of positions added */ +static U32 ZSTD_insertBt1( + ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iend, + U32 const mls, const int extDict) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hashLog = cParams->hashLog; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + U32 matchIndex = hashTable[h]; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* match; + const U32 curr = (U32)(ip-base); + const U32 btLow = btMask >= curr ? 0 : curr - btMask; + U32* smallerPtr = bt + 2*(curr&btMask); + U32* largerPtr = smallerPtr + 1; + U32 dummy32; /* to be nullified at the end */ + U32 const windowLow = ms->window.lowLimit; + U32 matchEndIdx = curr+8+1; + size_t bestLength = 8; + U32 nbCompares = 1U << cParams->searchLog; +#ifdef ZSTD_C_PREDICT + U32 predictedSmall = *(bt + 2*((curr-1)&btMask) + 0); + U32 predictedLarge = *(bt + 2*((curr-1)&btMask) + 1); + predictedSmall += (predictedSmall>0); + predictedLarge += (predictedLarge>0); +#endif /* ZSTD_C_PREDICT */ + + DEBUGLOG(8, "ZSTD_insertBt1 (%u)", curr); + + assert(ip <= iend-8); /* required for h calculation */ + hashTable[h] = curr; /* Update Hash Table */ + + assert(windowLow > 0); + while (nbCompares-- && (matchIndex >= windowLow)) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + assert(matchIndex < curr); + +#ifdef ZSTD_C_PREDICT /* note : can create issues when hlog small <= 11 */ + const U32* predictPtr = bt + 2*((matchIndex-1) & btMask); /* written this way, as bt is a roll buffer */ + if (matchIndex == predictedSmall) { + /* no need to check length, result known */ + *smallerPtr = matchIndex; + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + predictedSmall = predictPtr[1] + (predictPtr[1]>0); + continue; + } + if (matchIndex == predictedLarge) { + *largerPtr = matchIndex; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + predictedLarge = predictPtr[0] + (predictPtr[0]>0); + continue; + } +#endif + + if (!extDict || (matchIndex+matchLength >= dictLimit)) { + assert(matchIndex+matchLength >= dictLimit); /* might be wrong if actually extDict */ + match = base + matchIndex; + matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ + } + + if (matchLength > bestLength) { + bestLength = matchLength; + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + } + + if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */ + break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt tree */ + } + + if (match[matchLength] < ip[matchLength]) { /* necessarily within buffer */ + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop searching */ + smallerPtr = nextPtr+1; /* new "candidate" => larger than match, which was smaller than target */ + matchIndex = nextPtr[1]; /* new matchIndex, larger than previous and closer to current */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop searching */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; + { U32 positions = 0; + if (bestLength > 384) positions = MIN(192, (U32)(bestLength - 384)); /* speed optimization */ + assert(matchEndIdx > curr + 8); + return MAX(positions, matchEndIdx - (curr + 8)); + } +} + +FORCE_INLINE_TEMPLATE +void ZSTD_updateTree_internal( + ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iend, + const U32 mls, const ZSTD_dictMode_e dictMode) +{ + const BYTE* const base = ms->window.base; + U32 const target = (U32)(ip - base); + U32 idx = ms->nextToUpdate; + DEBUGLOG(6, "ZSTD_updateTree_internal, from %u to %u (dictMode:%u)", + idx, target, dictMode); + + while(idx < target) { + U32 const forward = ZSTD_insertBt1(ms, base+idx, iend, mls, dictMode == ZSTD_extDict); + assert(idx < (U32)(idx + forward)); + idx += forward; + } + assert((size_t)(ip - base) <= (size_t)(U32)(-1)); + assert((size_t)(iend - base) <= (size_t)(U32)(-1)); + ms->nextToUpdate = target; +} + +void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend) { + ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.minMatch, ZSTD_noDict); +} + +FORCE_INLINE_TEMPLATE +U32 ZSTD_insertBtAndGetAllMatches ( + ZSTD_match_t* matches, /* store result (found matches) in this table (presumed large enough) */ + ZSTD_matchState_t* ms, + U32* nextToUpdate3, + const BYTE* const ip, const BYTE* const iLimit, const ZSTD_dictMode_e dictMode, + const U32 rep[ZSTD_REP_NUM], + U32 const ll0, /* tells if associated literal length is 0 or not. This value must be 0 or 1 */ + const U32 lengthToBeat, + U32 const mls /* template */) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); + const BYTE* const base = ms->window.base; + U32 const curr = (U32)(ip-base); + U32 const hashLog = cParams->hashLog; + U32 const minMatch = (mls==3) ? 3 : 4; + U32* const hashTable = ms->hashTable; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32 matchIndex = hashTable[h]; + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask= (1U << btLog) - 1; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const dictBase = ms->window.dictBase; + U32 const dictLimit = ms->window.dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + U32 const btLow = (btMask >= curr) ? 0 : curr - btMask; + U32 const windowLow = ZSTD_getLowestMatchIndex(ms, curr, cParams->windowLog); + U32 const matchLow = windowLow ? windowLow : 1; + U32* smallerPtr = bt + 2*(curr&btMask); + U32* largerPtr = bt + 2*(curr&btMask) + 1; + U32 matchEndIdx = curr+8+1; /* farthest referenced position of any match => detects repetitive patterns */ + U32 dummy32; /* to be nullified at the end */ + U32 mnum = 0; + U32 nbCompares = 1U << cParams->searchLog; + + const ZSTD_matchState_t* dms = dictMode == ZSTD_dictMatchState ? ms->dictMatchState : NULL; + const ZSTD_compressionParameters* const dmsCParams = + dictMode == ZSTD_dictMatchState ? &dms->cParams : NULL; + const BYTE* const dmsBase = dictMode == ZSTD_dictMatchState ? dms->window.base : NULL; + const BYTE* const dmsEnd = dictMode == ZSTD_dictMatchState ? dms->window.nextSrc : NULL; + U32 const dmsHighLimit = dictMode == ZSTD_dictMatchState ? (U32)(dmsEnd - dmsBase) : 0; + U32 const dmsLowLimit = dictMode == ZSTD_dictMatchState ? dms->window.lowLimit : 0; + U32 const dmsIndexDelta = dictMode == ZSTD_dictMatchState ? windowLow - dmsHighLimit : 0; + U32 const dmsHashLog = dictMode == ZSTD_dictMatchState ? dmsCParams->hashLog : hashLog; + U32 const dmsBtLog = dictMode == ZSTD_dictMatchState ? dmsCParams->chainLog - 1 : btLog; + U32 const dmsBtMask = dictMode == ZSTD_dictMatchState ? (1U << dmsBtLog) - 1 : 0; + U32 const dmsBtLow = dictMode == ZSTD_dictMatchState && dmsBtMask < dmsHighLimit - dmsLowLimit ? dmsHighLimit - dmsBtMask : dmsLowLimit; + + size_t bestLength = lengthToBeat-1; + DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches: current=%u", curr); + + /* check repCode */ + assert(ll0 <= 1); /* necessarily 1 or 0 */ + { U32 const lastR = ZSTD_REP_NUM + ll0; + U32 repCode; + for (repCode = ll0; repCode < lastR; repCode++) { + U32 const repOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode]; + U32 const repIndex = curr - repOffset; + U32 repLen = 0; + assert(curr >= dictLimit); + if (repOffset-1 /* intentional overflow, discards 0 and -1 */ < curr-dictLimit) { /* equivalent to `curr > repIndex >= dictLimit` */ + /* We must validate the repcode offset because when we're using a dictionary the + * valid offset range shrinks when the dictionary goes out of bounds. + */ + if ((repIndex >= windowLow) & (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repOffset, minMatch))) { + repLen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repOffset, iLimit) + minMatch; + } + } else { /* repIndex < dictLimit || repIndex >= curr */ + const BYTE* const repMatch = dictMode == ZSTD_dictMatchState ? + dmsBase + repIndex - dmsIndexDelta : + dictBase + repIndex; + assert(curr >= windowLow); + if ( dictMode == ZSTD_extDict + && ( ((repOffset-1) /*intentional overflow*/ < curr - windowLow) /* equivalent to `curr > repIndex >= windowLow` */ + & (((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */) + && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { + repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dictEnd, prefixStart) + minMatch; + } + if (dictMode == ZSTD_dictMatchState + && ( ((repOffset-1) /*intentional overflow*/ < curr - (dmsLowLimit + dmsIndexDelta)) /* equivalent to `curr > repIndex >= dmsLowLimit` */ + & ((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */ + && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { + repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dmsEnd, prefixStart) + minMatch; + } } + /* save longer solution */ + if (repLen > bestLength) { + DEBUGLOG(8, "found repCode %u (ll0:%u, offset:%u) of length %u", + repCode, ll0, repOffset, repLen); + bestLength = repLen; + matches[mnum].off = repCode - ll0; + matches[mnum].len = (U32)repLen; + mnum++; + if ( (repLen > sufficient_len) + | (ip+repLen == iLimit) ) { /* best possible */ + return mnum; + } } } } + + /* HC3 match finder */ + if ((mls == 3) /*static*/ && (bestLength < mls)) { + U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, nextToUpdate3, ip); + if ((matchIndex3 >= matchLow) + & (curr - matchIndex3 < (1<<18)) /*heuristic : longer distance likely too expensive*/ ) { + size_t mlen; + if ((dictMode == ZSTD_noDict) /*static*/ || (dictMode == ZSTD_dictMatchState) /*static*/ || (matchIndex3 >= dictLimit)) { + const BYTE* const match = base + matchIndex3; + mlen = ZSTD_count(ip, match, iLimit); + } else { + const BYTE* const match = dictBase + matchIndex3; + mlen = ZSTD_count_2segments(ip, match, iLimit, dictEnd, prefixStart); + } + + /* save best solution */ + if (mlen >= mls /* == 3 > bestLength */) { + DEBUGLOG(8, "found small match with hlog3, of length %u", + (U32)mlen); + bestLength = mlen; + assert(curr > matchIndex3); + assert(mnum==0); /* no prior solution */ + matches[0].off = (curr - matchIndex3) + ZSTD_REP_MOVE; + matches[0].len = (U32)mlen; + mnum = 1; + if ( (mlen > sufficient_len) | + (ip+mlen == iLimit) ) { /* best possible length */ + ms->nextToUpdate = curr+1; /* skip insertion */ + return 1; + } } } + /* no dictMatchState lookup: dicts don't have a populated HC3 table */ + } + + hashTable[h] = curr; /* Update Hash Table */ + + while (nbCompares-- && (matchIndex >= matchLow)) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + const BYTE* match; + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + assert(curr > matchIndex); + + if ((dictMode == ZSTD_noDict) || (dictMode == ZSTD_dictMatchState) || (matchIndex+matchLength >= dictLimit)) { + assert(matchIndex+matchLength >= dictLimit); /* ensure the condition is correct when !extDict */ + match = base + matchIndex; + if (matchIndex >= dictLimit) assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */ + matchLength += ZSTD_count(ip+matchLength, match+matchLength, iLimit); + } else { + match = dictBase + matchIndex; + assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */ + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* prepare for match[matchLength] read */ + } + + if (matchLength > bestLength) { + DEBUGLOG(8, "found match of length %u at distance %u (offCode=%u)", + (U32)matchLength, curr - matchIndex, curr - matchIndex + ZSTD_REP_MOVE); + assert(matchEndIdx > matchIndex); + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + bestLength = matchLength; + matches[mnum].off = (curr - matchIndex) + ZSTD_REP_MOVE; + matches[mnum].len = (U32)matchLength; + mnum++; + if ( (matchLength > ZSTD_OPT_NUM) + | (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */) { + if (dictMode == ZSTD_dictMatchState) nbCompares = 0; /* break should also skip searching dms */ + break; /* drop, to preserve bt consistency (miss a little bit of compression) */ + } + } + + if (match[matchLength] < ip[matchLength]) { + /* match smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new candidate => larger than match, which was smaller than current */ + matchIndex = nextPtr[1]; /* new matchIndex, larger than previous, closer to current */ + } else { + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; + + if (dictMode == ZSTD_dictMatchState && nbCompares) { + size_t const dmsH = ZSTD_hashPtr(ip, dmsHashLog, mls); + U32 dictMatchIndex = dms->hashTable[dmsH]; + const U32* const dmsBt = dms->chainTable; + commonLengthSmaller = commonLengthLarger = 0; + while (nbCompares-- && (dictMatchIndex > dmsLowLimit)) { + const U32* const nextPtr = dmsBt + 2*(dictMatchIndex & dmsBtMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match = dmsBase + dictMatchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dmsEnd, prefixStart); + if (dictMatchIndex+matchLength >= dmsHighLimit) + match = base + dictMatchIndex + dmsIndexDelta; /* to prepare for next usage of match[matchLength] */ + + if (matchLength > bestLength) { + matchIndex = dictMatchIndex + dmsIndexDelta; + DEBUGLOG(8, "found dms match of length %u at distance %u (offCode=%u)", + (U32)matchLength, curr - matchIndex, curr - matchIndex + ZSTD_REP_MOVE); + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + bestLength = matchLength; + matches[mnum].off = (curr - matchIndex) + ZSTD_REP_MOVE; + matches[mnum].len = (U32)matchLength; + mnum++; + if ( (matchLength > ZSTD_OPT_NUM) + | (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */) { + break; /* drop, to guarantee consistency (miss a little bit of compression) */ + } + } + + if (dictMatchIndex <= dmsBtLow) { break; } /* beyond tree size, stop the search */ + if (match[matchLength] < ip[matchLength]) { + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + dictMatchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + commonLengthLarger = matchLength; + dictMatchIndex = nextPtr[0]; + } + } + } + + assert(matchEndIdx > curr+8); + ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */ + return mnum; +} + + +FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( + ZSTD_match_t* matches, /* store result (match found, increasing size) in this table */ + ZSTD_matchState_t* ms, + U32* nextToUpdate3, + const BYTE* ip, const BYTE* const iHighLimit, const ZSTD_dictMode_e dictMode, + const U32 rep[ZSTD_REP_NUM], + U32 const ll0, + U32 const lengthToBeat) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32 const matchLengthSearch = cParams->minMatch; + DEBUGLOG(8, "ZSTD_BtGetAllMatches"); + if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */ + ZSTD_updateTree_internal(ms, ip, iHighLimit, matchLengthSearch, dictMode); + switch(matchLengthSearch) + { + case 3 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 3); + default : + case 4 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 4); + case 5 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 5); + case 7 : + case 6 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 6); + } +} + +/* *********************** +* LDM helper functions * +*************************/ + +/* Struct containing info needed to make decision about ldm inclusion */ +typedef struct { + rawSeqStore_t seqStore; /* External match candidates store for this block */ + U32 startPosInBlock; /* Start position of the current match candidate */ + U32 endPosInBlock; /* End position of the current match candidate */ + U32 offset; /* Offset of the match candidate */ +} ZSTD_optLdm_t; + +/* ZSTD_optLdm_skipRawSeqStoreBytes(): + * Moves forward in rawSeqStore by nbBytes, which will update the fields 'pos' and 'posInSequence'. + */ +static void ZSTD_optLdm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) { + U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes); + while (currPos && rawSeqStore->pos < rawSeqStore->size) { + rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos]; + if (currPos >= currSeq.litLength + currSeq.matchLength) { + currPos -= currSeq.litLength + currSeq.matchLength; + rawSeqStore->pos++; + } else { + rawSeqStore->posInSequence = currPos; + break; + } + } + if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) { + rawSeqStore->posInSequence = 0; + } +} + +/* ZSTD_opt_getNextMatchAndUpdateSeqStore(): + * Calculates the beginning and end of the next match in the current block. + * Updates 'pos' and 'posInSequence' of the ldmSeqStore. + */ +static void ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 currPosInBlock, + U32 blockBytesRemaining) { + rawSeq currSeq; + U32 currBlockEndPos; + U32 literalsBytesRemaining; + U32 matchBytesRemaining; + + /* Setting match end position to MAX to ensure we never use an LDM during this block */ + if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) { + optLdm->startPosInBlock = UINT_MAX; + optLdm->endPosInBlock = UINT_MAX; + return; + } + /* Calculate appropriate bytes left in matchLength and litLength after adjusting + based on ldmSeqStore->posInSequence */ + currSeq = optLdm->seqStore.seq[optLdm->seqStore.pos]; + assert(optLdm->seqStore.posInSequence <= currSeq.litLength + currSeq.matchLength); + currBlockEndPos = currPosInBlock + blockBytesRemaining; + literalsBytesRemaining = (optLdm->seqStore.posInSequence < currSeq.litLength) ? + currSeq.litLength - (U32)optLdm->seqStore.posInSequence : + 0; + matchBytesRemaining = (literalsBytesRemaining == 0) ? + currSeq.matchLength - ((U32)optLdm->seqStore.posInSequence - currSeq.litLength) : + currSeq.matchLength; + + /* If there are more literal bytes than bytes remaining in block, no ldm is possible */ + if (literalsBytesRemaining >= blockBytesRemaining) { + optLdm->startPosInBlock = UINT_MAX; + optLdm->endPosInBlock = UINT_MAX; + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, blockBytesRemaining); + return; + } + + /* Matches may be < MINMATCH by this process. In that case, we will reject them + when we are deciding whether or not to add the ldm */ + optLdm->startPosInBlock = currPosInBlock + literalsBytesRemaining; + optLdm->endPosInBlock = optLdm->startPosInBlock + matchBytesRemaining; + optLdm->offset = currSeq.offset; + + if (optLdm->endPosInBlock > currBlockEndPos) { + /* Match ends after the block ends, we can't use the whole match */ + optLdm->endPosInBlock = currBlockEndPos; + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, currBlockEndPos - currPosInBlock); + } else { + /* Consume nb of bytes equal to size of sequence left */ + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, literalsBytesRemaining + matchBytesRemaining); + } +} + +/* ZSTD_optLdm_maybeAddMatch(): + * Adds a match if it's long enough, based on it's 'matchStartPosInBlock' + * and 'matchEndPosInBlock', into 'matches'. Maintains the correct ordering of 'matches' + */ +static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches, + ZSTD_optLdm_t* optLdm, U32 currPosInBlock) { + U32 posDiff = currPosInBlock - optLdm->startPosInBlock; + /* Note: ZSTD_match_t actually contains offCode and matchLength (before subtracting MINMATCH) */ + U32 candidateMatchLength = optLdm->endPosInBlock - optLdm->startPosInBlock - posDiff; + U32 candidateOffCode = optLdm->offset + ZSTD_REP_MOVE; + + /* Ensure that current block position is not outside of the match */ + if (currPosInBlock < optLdm->startPosInBlock + || currPosInBlock >= optLdm->endPosInBlock + || candidateMatchLength < MINMATCH) { + return; + } + + if (*nbMatches == 0 || ((candidateMatchLength > matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM)) { + DEBUGLOG(6, "ZSTD_optLdm_maybeAddMatch(): Adding ldm candidate match (offCode: %u matchLength %u) at block position=%u", + candidateOffCode, candidateMatchLength, currPosInBlock); + matches[*nbMatches].len = candidateMatchLength; + matches[*nbMatches].off = candidateOffCode; + (*nbMatches)++; + } +} + +/* ZSTD_optLdm_processMatchCandidate(): + * Wrapper function to update ldm seq store and call ldm functions as necessary. + */ +static void ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm, ZSTD_match_t* matches, U32* nbMatches, + U32 currPosInBlock, U32 remainingBytes) { + if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) { + return; + } + + if (currPosInBlock >= optLdm->endPosInBlock) { + if (currPosInBlock > optLdm->endPosInBlock) { + /* The position at which ZSTD_optLdm_processMatchCandidate() is called is not necessarily + * at the end of a match from the ldm seq store, and will often be some bytes + * over beyond matchEndPosInBlock. As such, we need to correct for these "overshoots" + */ + U32 posOvershoot = currPosInBlock - optLdm->endPosInBlock; + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, posOvershoot); + } + ZSTD_opt_getNextMatchAndUpdateSeqStore(optLdm, currPosInBlock, remainingBytes); + } + ZSTD_optLdm_maybeAddMatch(matches, nbMatches, optLdm, currPosInBlock); +} + +/*-******************************* +* Optimal parser +*********************************/ + + +static U32 ZSTD_totalLen(ZSTD_optimal_t sol) +{ + return sol.litlen + sol.mlen; +} + +#if 0 /* debug */ + +static void +listStats(const U32* table, int lastEltID) +{ + int const nbElts = lastEltID + 1; + int enb; + for (enb=0; enb < nbElts; enb++) { + (void)table; + /* RAWLOG(2, "%3i:%3i, ", enb, table[enb]); */ + RAWLOG(2, "%4i,", table[enb]); + } + RAWLOG(2, " \n"); +} + +#endif + +FORCE_INLINE_TEMPLATE size_t +ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, + seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, + const int optLevel, + const ZSTD_dictMode_e dictMode) +{ + optState_t* const optStatePtr = &ms->opt; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ms->window.base; + const BYTE* const prefixStart = base + ms->window.dictLimit; + const ZSTD_compressionParameters* const cParams = &ms->cParams; + + U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); + U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4; + U32 nextToUpdate3 = ms->nextToUpdate; + + ZSTD_optimal_t* const opt = optStatePtr->priceTable; + ZSTD_match_t* const matches = optStatePtr->matchTable; + ZSTD_optimal_t lastSequence; + ZSTD_optLdm_t optLdm; + + optLdm.seqStore = ms->ldmSeqStore ? *ms->ldmSeqStore : kNullRawSeqStore; + optLdm.endPosInBlock = optLdm.startPosInBlock = optLdm.offset = 0; + ZSTD_opt_getNextMatchAndUpdateSeqStore(&optLdm, (U32)(ip-istart), (U32)(iend-ip)); + + /* init */ + DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", + (U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate); + assert(optLevel <= 2); + ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel); + ip += (ip==prefixStart); + + /* Match Loop */ + while (ip < ilimit) { + U32 cur, last_pos = 0; + + /* find first match */ + { U32 const litlen = (U32)(ip - anchor); + U32 const ll0 = !litlen; + U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch); + ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches, + (U32)(ip-istart), (U32)(iend - ip)); + if (!nbMatches) { ip++; continue; } + + /* initialize opt[0] */ + { U32 i ; for (i=0; i immediate encoding */ + { U32 const maxML = matches[nbMatches-1].len; + U32 const maxOffset = matches[nbMatches-1].off; + DEBUGLOG(6, "found %u matches of maxLength=%u and maxOffCode=%u at cPos=%u => start new series", + nbMatches, maxML, maxOffset, (U32)(ip-prefixStart)); + + if (maxML > sufficient_len) { + lastSequence.litlen = litlen; + lastSequence.mlen = maxML; + lastSequence.off = maxOffset; + DEBUGLOG(6, "large match (%u>%u), immediate encoding", + maxML, sufficient_len); + cur = 0; + last_pos = ZSTD_totalLen(lastSequence); + goto _shortestPath; + } } + + /* set prices for first matches starting position == 0 */ + { U32 const literalsPrice = opt[0].price + ZSTD_litLengthPrice(0, optStatePtr, optLevel); + U32 pos; + U32 matchNb; + for (pos = 1; pos < minMatch; pos++) { + opt[pos].price = ZSTD_MAX_PRICE; /* mlen, litlen and price will be fixed during forward scanning */ + } + for (matchNb = 0; matchNb < nbMatches; matchNb++) { + U32 const offset = matches[matchNb].off; + U32 const end = matches[matchNb].len; + for ( ; pos <= end ; pos++ ) { + U32 const matchPrice = ZSTD_getMatchPrice(offset, pos, optStatePtr, optLevel); + U32 const sequencePrice = literalsPrice + matchPrice; + DEBUGLOG(7, "rPos:%u => set initial price : %.2f", + pos, ZSTD_fCost(sequencePrice)); + opt[pos].mlen = pos; + opt[pos].off = offset; + opt[pos].litlen = litlen; + opt[pos].price = sequencePrice; + } } + last_pos = pos-1; + } + } + + /* check further positions */ + for (cur = 1; cur <= last_pos; cur++) { + const BYTE* const inr = ip + cur; + assert(cur < ZSTD_OPT_NUM); + DEBUGLOG(7, "cPos:%zi==rPos:%u", inr-istart, cur) + + /* Fix current position with one literal if cheaper */ + { U32 const litlen = (opt[cur-1].mlen == 0) ? opt[cur-1].litlen + 1 : 1; + int const price = opt[cur-1].price + + ZSTD_rawLiteralsCost(ip+cur-1, 1, optStatePtr, optLevel) + + ZSTD_litLengthPrice(litlen, optStatePtr, optLevel) + - ZSTD_litLengthPrice(litlen-1, optStatePtr, optLevel); + assert(price < 1000000000); /* overflow check */ + if (price <= opt[cur].price) { + DEBUGLOG(7, "cPos:%zi==rPos:%u : better price (%.2f<=%.2f) using literal (ll==%u) (hist:%u,%u,%u)", + inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), litlen, + opt[cur-1].rep[0], opt[cur-1].rep[1], opt[cur-1].rep[2]); + opt[cur].mlen = 0; + opt[cur].off = 0; + opt[cur].litlen = litlen; + opt[cur].price = price; + } else { + DEBUGLOG(7, "cPos:%zi==rPos:%u : literal would cost more (%.2f>%.2f) (hist:%u,%u,%u)", + inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), + opt[cur].rep[0], opt[cur].rep[1], opt[cur].rep[2]); + } + } + + /* Set the repcodes of the current position. We must do it here + * because we rely on the repcodes of the 2nd to last sequence being + * correct to set the next chunks repcodes during the backward + * traversal. + */ + ZSTD_STATIC_ASSERT(sizeof(opt[cur].rep) == sizeof(repcodes_t)); + assert(cur >= opt[cur].mlen); + if (opt[cur].mlen != 0) { + U32 const prev = cur - opt[cur].mlen; + repcodes_t newReps = ZSTD_updateRep(opt[prev].rep, opt[cur].off, opt[cur].litlen==0); + ZSTD_memcpy(opt[cur].rep, &newReps, sizeof(repcodes_t)); + } else { + ZSTD_memcpy(opt[cur].rep, opt[cur - 1].rep, sizeof(repcodes_t)); + } + + /* last match must start at a minimum distance of 8 from oend */ + if (inr > ilimit) continue; + + if (cur == last_pos) break; + + if ( (optLevel==0) /*static_test*/ + && (opt[cur+1].price <= opt[cur].price + (BITCOST_MULTIPLIER/2)) ) { + DEBUGLOG(7, "move to next rPos:%u : price is <=", cur+1); + continue; /* skip unpromising positions; about ~+6% speed, -0.01 ratio */ + } + + { U32 const ll0 = (opt[cur].mlen != 0); + U32 const litlen = (opt[cur].mlen == 0) ? opt[cur].litlen : 0; + U32 const previousPrice = opt[cur].price; + U32 const basePrice = previousPrice + ZSTD_litLengthPrice(0, optStatePtr, optLevel); + U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch); + U32 matchNb; + + ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches, + (U32)(inr-istart), (U32)(iend-inr)); + + if (!nbMatches) { + DEBUGLOG(7, "rPos:%u : no match found", cur); + continue; + } + + { U32 const maxML = matches[nbMatches-1].len; + DEBUGLOG(7, "cPos:%zi==rPos:%u, found %u matches, of maxLength=%u", + inr-istart, cur, nbMatches, maxML); + + if ( (maxML > sufficient_len) + || (cur + maxML >= ZSTD_OPT_NUM) ) { + lastSequence.mlen = maxML; + lastSequence.off = matches[nbMatches-1].off; + lastSequence.litlen = litlen; + cur -= (opt[cur].mlen==0) ? opt[cur].litlen : 0; /* last sequence is actually only literals, fix cur to last match - note : may underflow, in which case, it's first sequence, and it's okay */ + last_pos = cur + ZSTD_totalLen(lastSequence); + if (cur > ZSTD_OPT_NUM) cur = 0; /* underflow => first match */ + goto _shortestPath; + } } + + /* set prices using matches found at position == cur */ + for (matchNb = 0; matchNb < nbMatches; matchNb++) { + U32 const offset = matches[matchNb].off; + U32 const lastML = matches[matchNb].len; + U32 const startML = (matchNb>0) ? matches[matchNb-1].len+1 : minMatch; + U32 mlen; + + DEBUGLOG(7, "testing match %u => offCode=%4u, mlen=%2u, llen=%2u", + matchNb, matches[matchNb].off, lastML, litlen); + + for (mlen = lastML; mlen >= startML; mlen--) { /* scan downward */ + U32 const pos = cur + mlen; + int const price = basePrice + ZSTD_getMatchPrice(offset, mlen, optStatePtr, optLevel); + + if ((pos > last_pos) || (price < opt[pos].price)) { + DEBUGLOG(7, "rPos:%u (ml=%2u) => new better price (%.2f<%.2f)", + pos, mlen, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price)); + while (last_pos < pos) { opt[last_pos+1].price = ZSTD_MAX_PRICE; last_pos++; } /* fill empty positions */ + opt[pos].mlen = mlen; + opt[pos].off = offset; + opt[pos].litlen = litlen; + opt[pos].price = price; + } else { + DEBUGLOG(7, "rPos:%u (ml=%2u) => new price is worse (%.2f>=%.2f)", + pos, mlen, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price)); + if (optLevel==0) break; /* early update abort; gets ~+10% speed for about -0.01 ratio loss */ + } + } } } + } /* for (cur = 1; cur <= last_pos; cur++) */ + + lastSequence = opt[last_pos]; + cur = last_pos > ZSTD_totalLen(lastSequence) ? last_pos - ZSTD_totalLen(lastSequence) : 0; /* single sequence, and it starts before `ip` */ + assert(cur < ZSTD_OPT_NUM); /* control overflow*/ + +_shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ + assert(opt[0].mlen == 0); + + /* Set the next chunk's repcodes based on the repcodes of the beginning + * of the last match, and the last sequence. This avoids us having to + * update them while traversing the sequences. + */ + if (lastSequence.mlen != 0) { + repcodes_t reps = ZSTD_updateRep(opt[cur].rep, lastSequence.off, lastSequence.litlen==0); + ZSTD_memcpy(rep, &reps, sizeof(reps)); + } else { + ZSTD_memcpy(rep, opt[cur].rep, sizeof(repcodes_t)); + } + + { U32 const storeEnd = cur + 1; + U32 storeStart = storeEnd; + U32 seqPos = cur; + + DEBUGLOG(6, "start reverse traversal (last_pos:%u, cur:%u)", + last_pos, cur); (void)last_pos; + assert(storeEnd < ZSTD_OPT_NUM); + DEBUGLOG(6, "last sequence copied into pos=%u (llen=%u,mlen=%u,ofc=%u)", + storeEnd, lastSequence.litlen, lastSequence.mlen, lastSequence.off); + opt[storeEnd] = lastSequence; + while (seqPos > 0) { + U32 const backDist = ZSTD_totalLen(opt[seqPos]); + storeStart--; + DEBUGLOG(6, "sequence from rPos=%u copied into pos=%u (llen=%u,mlen=%u,ofc=%u)", + seqPos, storeStart, opt[seqPos].litlen, opt[seqPos].mlen, opt[seqPos].off); + opt[storeStart] = opt[seqPos]; + seqPos = (seqPos > backDist) ? seqPos - backDist : 0; + } + + /* save sequences */ + DEBUGLOG(6, "sending selected sequences into seqStore") + { U32 storePos; + for (storePos=storeStart; storePos <= storeEnd; storePos++) { + U32 const llen = opt[storePos].litlen; + U32 const mlen = opt[storePos].mlen; + U32 const offCode = opt[storePos].off; + U32 const advance = llen + mlen; + DEBUGLOG(6, "considering seq starting at %zi, llen=%u, mlen=%u", + anchor - istart, (unsigned)llen, (unsigned)mlen); + + if (mlen==0) { /* only literals => must be last "sequence", actually starting a new stream of sequences */ + assert(storePos == storeEnd); /* must be last sequence */ + ip = anchor + llen; /* last "sequence" is a bunch of literals => don't progress anchor */ + continue; /* will finish */ + } + + assert(anchor + llen <= iend); + ZSTD_updateStats(optStatePtr, llen, anchor, offCode, mlen); + ZSTD_storeSeq(seqStore, llen, anchor, iend, offCode, mlen-MINMATCH); + anchor += advance; + ip = anchor; + } } + ZSTD_setBasePrices(optStatePtr, optLevel); + } + } /* while (ip < ilimit) */ + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + + +size_t ZSTD_compressBlock_btopt( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_compressBlock_btopt"); + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /*optLevel*/, ZSTD_noDict); +} + + +/* used in 2-pass strategy */ +static U32 ZSTD_upscaleStat(unsigned* table, U32 lastEltIndex, int bonus) +{ + U32 s, sum=0; + assert(ZSTD_FREQ_DIV+bonus >= 0); + for (s=0; slitSum = ZSTD_upscaleStat(optPtr->litFreq, MaxLit, 0); + optPtr->litLengthSum = ZSTD_upscaleStat(optPtr->litLengthFreq, MaxLL, 0); + optPtr->matchLengthSum = ZSTD_upscaleStat(optPtr->matchLengthFreq, MaxML, 0); + optPtr->offCodeSum = ZSTD_upscaleStat(optPtr->offCodeFreq, MaxOff, 0); +} + +/* ZSTD_initStats_ultra(): + * make a first compression pass, just to seed stats with more accurate starting values. + * only works on first block, with no dictionary and no ldm. + * this function cannot error, hence its contract must be respected. + */ +static void +ZSTD_initStats_ultra(ZSTD_matchState_t* ms, + seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + U32 tmpRep[ZSTD_REP_NUM]; /* updated rep codes will sink here */ + ZSTD_memcpy(tmpRep, rep, sizeof(tmpRep)); + + DEBUGLOG(4, "ZSTD_initStats_ultra (srcSize=%zu)", srcSize); + assert(ms->opt.litLengthSum == 0); /* first block */ + assert(seqStore->sequences == seqStore->sequencesStart); /* no ldm */ + assert(ms->window.dictLimit == ms->window.lowLimit); /* no dictionary */ + assert(ms->window.dictLimit - ms->nextToUpdate <= 1); /* no prefix (note: intentional overflow, defined as 2-complement) */ + + ZSTD_compressBlock_opt_generic(ms, seqStore, tmpRep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); /* generate stats into ms->opt*/ + + /* invalidate first scan from history */ + ZSTD_resetSeqStore(seqStore); + ms->window.base -= srcSize; + ms->window.dictLimit += (U32)srcSize; + ms->window.lowLimit = ms->window.dictLimit; + ms->nextToUpdate = ms->window.dictLimit; + + /* re-inforce weight of collected statistics */ + ZSTD_upscaleStats(&ms->opt); +} + +size_t ZSTD_compressBlock_btultra( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_compressBlock_btultra (srcSize=%zu)", srcSize); + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_btultra2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + U32 const curr = (U32)((const BYTE*)src - ms->window.base); + DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize); + + /* 2-pass strategy: + * this strategy makes a first pass over first block to collect statistics + * and seed next round's statistics with it. + * After 1st pass, function forgets everything, and starts a new block. + * Consequently, this can only work if no data has been previously loaded in tables, + * aka, no dictionary, no prefix, no ldm preprocessing. + * The compression ratio gain is generally small (~0.5% on first block), + * the cost is 2x cpu time on first block. */ + assert(srcSize <= ZSTD_BLOCKSIZE_MAX); + if ( (ms->opt.litLengthSum==0) /* first block */ + && (seqStore->sequences == seqStore->sequencesStart) /* no ldm */ + && (ms->window.dictLimit == ms->window.lowLimit) /* no dictionary */ + && (curr == ms->window.dictLimit) /* start of frame, nothing already loaded nor skipped */ + && (srcSize > ZSTD_PREDEF_THRESHOLD) + ) { + ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize); + } + + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_btopt_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /*optLevel*/, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_btultra_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_btopt_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /*optLevel*/, ZSTD_extDict); +} + +size_t ZSTD_compressBlock_btultra_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_extDict); +} + +/* note : no btultra2 variant for extDict nor dictMatchState, + * because btultra2 is not meant to work with dictionaries + * and is only specific for the first block (no prefix) */ diff --git a/lib/zstd/compress/zstd_opt.h b/lib/zstd/compress/zstd_opt.h new file mode 100644 index 000000000000..22b862858ba7 --- /dev/null +++ b/lib/zstd/compress/zstd_opt.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_OPT_H +#define ZSTD_OPT_H + + +#include "zstd_compress_internal.h" + +/* used in ZSTD_loadDictionaryContent() */ +void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend); + +size_t ZSTD_compressBlock_btopt( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + + +size_t ZSTD_compressBlock_btopt_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +size_t ZSTD_compressBlock_btopt_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + + /* note : no btultra2 variant for extDict nor dictMatchState, + * because btultra2 is not meant to work with dictionaries + * and is only specific for the first block (no prefix) */ + + +#endif /* ZSTD_OPT_H */ diff --git a/lib/zstd/decompress.c b/lib/zstd/decompress.c deleted file mode 100644 index 66cd487a326a..000000000000 --- a/lib/zstd/decompress.c +++ /dev/null @@ -1,2531 +0,0 @@ -/** - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of https://github.com/facebook/zstd. - * An additional grant of patent rights can be found in the PATENTS file in the - * same directory. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - */ - -/* *************************************************************** -* Tuning parameters -*****************************************************************/ -/*! -* MAXWINDOWSIZE_DEFAULT : -* maximum window size accepted by DStream, by default. -* Frames requiring more memory will be rejected. -*/ -#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT -#define ZSTD_MAXWINDOWSIZE_DEFAULT ((1 << ZSTD_WINDOWLOG_MAX) + 1) /* defined within zstd.h */ -#endif - -/*-******************************************************* -* Dependencies -*********************************************************/ -#include "fse.h" -#include "huf.h" -#include "mem.h" /* low level memory routines */ -#include "zstd_internal.h" -#include -#include -#include /* memcpy, memmove, memset */ - -#define ZSTD_PREFETCH(ptr) __builtin_prefetch(ptr, 0, 0) - -/*-************************************* -* Macros -***************************************/ -#define ZSTD_isError ERR_isError /* for inlining */ -#define FSE_isError ERR_isError -#define HUF_isError ERR_isError - -/*_******************************************************* -* Memory operations -**********************************************************/ -static void ZSTD_copy4(void *dst, const void *src) { memcpy(dst, src, 4); } - -/*-************************************************************* -* Context management -***************************************************************/ -typedef enum { - ZSTDds_getFrameHeaderSize, - ZSTDds_decodeFrameHeader, - ZSTDds_decodeBlockHeader, - ZSTDds_decompressBlock, - ZSTDds_decompressLastBlock, - ZSTDds_checkChecksum, - ZSTDds_decodeSkippableHeader, - ZSTDds_skipFrame -} ZSTD_dStage; - -typedef struct { - FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)]; - FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)]; - FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)]; - HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */ - U64 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32 / 2]; - U32 rep[ZSTD_REP_NUM]; -} ZSTD_entropyTables_t; - -struct ZSTD_DCtx_s { - const FSE_DTable *LLTptr; - const FSE_DTable *MLTptr; - const FSE_DTable *OFTptr; - const HUF_DTable *HUFptr; - ZSTD_entropyTables_t entropy; - const void *previousDstEnd; /* detect continuity */ - const void *base; /* start of curr segment */ - const void *vBase; /* virtual start of previous segment if it was just before curr one */ - const void *dictEnd; /* end of previous segment */ - size_t expected; - ZSTD_frameParams fParams; - blockType_e bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ - ZSTD_dStage stage; - U32 litEntropy; - U32 fseEntropy; - struct xxh64_state xxhState; - size_t headerSize; - U32 dictID; - const BYTE *litPtr; - ZSTD_customMem customMem; - size_t litSize; - size_t rleSize; - BYTE litBuffer[ZSTD_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH]; - BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; -}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ - -size_t ZSTD_DCtxWorkspaceBound(void) { return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_DCtx)); } - -size_t ZSTD_decompressBegin(ZSTD_DCtx *dctx) -{ - dctx->expected = ZSTD_frameHeaderSize_prefix; - dctx->stage = ZSTDds_getFrameHeaderSize; - dctx->previousDstEnd = NULL; - dctx->base = NULL; - dctx->vBase = NULL; - dctx->dictEnd = NULL; - dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ - dctx->litEntropy = dctx->fseEntropy = 0; - dctx->dictID = 0; - ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue)); - memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ - dctx->LLTptr = dctx->entropy.LLTable; - dctx->MLTptr = dctx->entropy.MLTable; - dctx->OFTptr = dctx->entropy.OFTable; - dctx->HUFptr = dctx->entropy.hufTable; - return 0; -} - -ZSTD_DCtx *ZSTD_createDCtx_advanced(ZSTD_customMem customMem) -{ - ZSTD_DCtx *dctx; - - if (!customMem.customAlloc || !customMem.customFree) - return NULL; - - dctx = (ZSTD_DCtx *)ZSTD_malloc(sizeof(ZSTD_DCtx), customMem); - if (!dctx) - return NULL; - memcpy(&dctx->customMem, &customMem, sizeof(customMem)); - ZSTD_decompressBegin(dctx); - return dctx; -} - -ZSTD_DCtx *ZSTD_initDCtx(void *workspace, size_t workspaceSize) -{ - ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize); - return ZSTD_createDCtx_advanced(stackMem); -} - -size_t ZSTD_freeDCtx(ZSTD_DCtx *dctx) -{ - if (dctx == NULL) - return 0; /* support free on NULL */ - ZSTD_free(dctx, dctx->customMem); - return 0; /* reserved as a potential error code in the future */ -} - -void ZSTD_copyDCtx(ZSTD_DCtx *dstDCtx, const ZSTD_DCtx *srcDCtx) -{ - size_t const workSpaceSize = (ZSTD_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH) + ZSTD_frameHeaderSize_max; - memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize); /* no need to copy workspace */ -} - -static void ZSTD_refDDict(ZSTD_DCtx *dstDCtx, const ZSTD_DDict *ddict); - -/*-************************************************************* -* Decompression section -***************************************************************/ - -/*! ZSTD_isFrame() : - * Tells if the content of `buffer` starts with a valid Frame Identifier. - * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. - * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. - * Note 3 : Skippable Frame Identifiers are considered valid. */ -unsigned ZSTD_isFrame(const void *buffer, size_t size) -{ - if (size < 4) - return 0; - { - U32 const magic = ZSTD_readLE32(buffer); - if (magic == ZSTD_MAGICNUMBER) - return 1; - if ((magic & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) - return 1; - } - return 0; -} - -/** ZSTD_frameHeaderSize() : -* srcSize must be >= ZSTD_frameHeaderSize_prefix. -* @return : size of the Frame Header */ -static size_t ZSTD_frameHeaderSize(const void *src, size_t srcSize) -{ - if (srcSize < ZSTD_frameHeaderSize_prefix) - return ERROR(srcSize_wrong); - { - BYTE const fhd = ((const BYTE *)src)[4]; - U32 const dictID = fhd & 3; - U32 const singleSegment = (fhd >> 5) & 1; - U32 const fcsId = fhd >> 6; - return ZSTD_frameHeaderSize_prefix + !singleSegment + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] + (singleSegment && !fcsId); - } -} - -/** ZSTD_getFrameParams() : -* decode Frame Header, or require larger `srcSize`. -* @return : 0, `fparamsPtr` is correctly filled, -* >0, `srcSize` is too small, result is expected `srcSize`, -* or an error code, which can be tested using ZSTD_isError() */ -size_t ZSTD_getFrameParams(ZSTD_frameParams *fparamsPtr, const void *src, size_t srcSize) -{ - const BYTE *ip = (const BYTE *)src; - - if (srcSize < ZSTD_frameHeaderSize_prefix) - return ZSTD_frameHeaderSize_prefix; - if (ZSTD_readLE32(src) != ZSTD_MAGICNUMBER) { - if ((ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { - if (srcSize < ZSTD_skippableHeaderSize) - return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */ - memset(fparamsPtr, 0, sizeof(*fparamsPtr)); - fparamsPtr->frameContentSize = ZSTD_readLE32((const char *)src + 4); - fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */ - return 0; - } - return ERROR(prefix_unknown); - } - - /* ensure there is enough `srcSize` to fully read/decode frame header */ - { - size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize); - if (srcSize < fhsize) - return fhsize; - } - - { - BYTE const fhdByte = ip[4]; - size_t pos = 5; - U32 const dictIDSizeCode = fhdByte & 3; - U32 const checksumFlag = (fhdByte >> 2) & 1; - U32 const singleSegment = (fhdByte >> 5) & 1; - U32 const fcsID = fhdByte >> 6; - U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; - U32 windowSize = 0; - U32 dictID = 0; - U64 frameContentSize = 0; - if ((fhdByte & 0x08) != 0) - return ERROR(frameParameter_unsupported); /* reserved bits, which must be zero */ - if (!singleSegment) { - BYTE const wlByte = ip[pos++]; - U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; - if (windowLog > ZSTD_WINDOWLOG_MAX) - return ERROR(frameParameter_windowTooLarge); /* avoids issue with 1 << windowLog */ - windowSize = (1U << windowLog); - windowSize += (windowSize >> 3) * (wlByte & 7); - } - - switch (dictIDSizeCode) { - default: /* impossible */ - case 0: break; - case 1: - dictID = ip[pos]; - pos++; - break; - case 2: - dictID = ZSTD_readLE16(ip + pos); - pos += 2; - break; - case 3: - dictID = ZSTD_readLE32(ip + pos); - pos += 4; - break; - } - switch (fcsID) { - default: /* impossible */ - case 0: - if (singleSegment) - frameContentSize = ip[pos]; - break; - case 1: frameContentSize = ZSTD_readLE16(ip + pos) + 256; break; - case 2: frameContentSize = ZSTD_readLE32(ip + pos); break; - case 3: frameContentSize = ZSTD_readLE64(ip + pos); break; - } - if (!windowSize) - windowSize = (U32)frameContentSize; - if (windowSize > windowSizeMax) - return ERROR(frameParameter_windowTooLarge); - fparamsPtr->frameContentSize = frameContentSize; - fparamsPtr->windowSize = windowSize; - fparamsPtr->dictID = dictID; - fparamsPtr->checksumFlag = checksumFlag; - } - return 0; -} - -/** ZSTD_getFrameContentSize() : -* compatible with legacy mode -* @return : decompressed size of the single frame pointed to be `src` if known, otherwise -* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined -* - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */ -unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) -{ - { - ZSTD_frameParams fParams; - if (ZSTD_getFrameParams(&fParams, src, srcSize) != 0) - return ZSTD_CONTENTSIZE_ERROR; - if (fParams.windowSize == 0) { - /* Either skippable or empty frame, size == 0 either way */ - return 0; - } else if (fParams.frameContentSize != 0) { - return fParams.frameContentSize; - } else { - return ZSTD_CONTENTSIZE_UNKNOWN; - } - } -} - -/** ZSTD_findDecompressedSize() : - * compatible with legacy mode - * `srcSize` must be the exact length of some number of ZSTD compressed and/or - * skippable frames - * @return : decompressed size of the frames contained */ -unsigned long long ZSTD_findDecompressedSize(const void *src, size_t srcSize) -{ - { - unsigned long long totalDstSize = 0; - while (srcSize >= ZSTD_frameHeaderSize_prefix) { - const U32 magicNumber = ZSTD_readLE32(src); - - if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { - size_t skippableSize; - if (srcSize < ZSTD_skippableHeaderSize) - return ERROR(srcSize_wrong); - skippableSize = ZSTD_readLE32((const BYTE *)src + 4) + ZSTD_skippableHeaderSize; - if (srcSize < skippableSize) { - return ZSTD_CONTENTSIZE_ERROR; - } - - src = (const BYTE *)src + skippableSize; - srcSize -= skippableSize; - continue; - } - - { - unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); - if (ret >= ZSTD_CONTENTSIZE_ERROR) - return ret; - - /* check for overflow */ - if (totalDstSize + ret < totalDstSize) - return ZSTD_CONTENTSIZE_ERROR; - totalDstSize += ret; - } - { - size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize); - if (ZSTD_isError(frameSrcSize)) { - return ZSTD_CONTENTSIZE_ERROR; - } - - src = (const BYTE *)src + frameSrcSize; - srcSize -= frameSrcSize; - } - } - - if (srcSize) { - return ZSTD_CONTENTSIZE_ERROR; - } - - return totalDstSize; - } -} - -/** ZSTD_decodeFrameHeader() : -* `headerSize` must be the size provided by ZSTD_frameHeaderSize(). -* @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ -static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx *dctx, const void *src, size_t headerSize) -{ - size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, headerSize); - if (ZSTD_isError(result)) - return result; /* invalid header */ - if (result > 0) - return ERROR(srcSize_wrong); /* headerSize too small */ - if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) - return ERROR(dictionary_wrong); - if (dctx->fParams.checksumFlag) - xxh64_reset(&dctx->xxhState, 0); - return 0; -} - -typedef struct { - blockType_e blockType; - U32 lastBlock; - U32 origSize; -} blockProperties_t; - -/*! ZSTD_getcBlockSize() : -* Provides the size of compressed block from block header `src` */ -size_t ZSTD_getcBlockSize(const void *src, size_t srcSize, blockProperties_t *bpPtr) -{ - if (srcSize < ZSTD_blockHeaderSize) - return ERROR(srcSize_wrong); - { - U32 const cBlockHeader = ZSTD_readLE24(src); - U32 const cSize = cBlockHeader >> 3; - bpPtr->lastBlock = cBlockHeader & 1; - bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); - bpPtr->origSize = cSize; /* only useful for RLE */ - if (bpPtr->blockType == bt_rle) - return 1; - if (bpPtr->blockType == bt_reserved) - return ERROR(corruption_detected); - return cSize; - } -} - -static size_t ZSTD_copyRawBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - if (srcSize > dstCapacity) - return ERROR(dstSize_tooSmall); - memcpy(dst, src, srcSize); - return srcSize; -} - -static size_t ZSTD_setRleBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize, size_t regenSize) -{ - if (srcSize != 1) - return ERROR(srcSize_wrong); - if (regenSize > dstCapacity) - return ERROR(dstSize_tooSmall); - memset(dst, *(const BYTE *)src, regenSize); - return regenSize; -} - -/*! ZSTD_decodeLiteralsBlock() : - @return : nb of bytes read from src (< srcSize ) */ -size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx *dctx, const void *src, size_t srcSize) /* note : srcSize < BLOCKSIZE */ -{ - if (srcSize < MIN_CBLOCK_SIZE) - return ERROR(corruption_detected); - - { - const BYTE *const istart = (const BYTE *)src; - symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); - - switch (litEncType) { - case set_repeat: - if (dctx->litEntropy == 0) - return ERROR(dictionary_corrupted); - fallthrough; - case set_compressed: - if (srcSize < 5) - return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */ - { - size_t lhSize, litSize, litCSize; - U32 singleStream = 0; - U32 const lhlCode = (istart[0] >> 2) & 3; - U32 const lhc = ZSTD_readLE32(istart); - switch (lhlCode) { - case 0: - case 1: - default: /* note : default is impossible, since lhlCode into [0..3] */ - /* 2 - 2 - 10 - 10 */ - singleStream = !lhlCode; - lhSize = 3; - litSize = (lhc >> 4) & 0x3FF; - litCSize = (lhc >> 14) & 0x3FF; - break; - case 2: - /* 2 - 2 - 14 - 14 */ - lhSize = 4; - litSize = (lhc >> 4) & 0x3FFF; - litCSize = lhc >> 18; - break; - case 3: - /* 2 - 2 - 18 - 18 */ - lhSize = 5; - litSize = (lhc >> 4) & 0x3FFFF; - litCSize = (lhc >> 22) + (istart[4] << 10); - break; - } - if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) - return ERROR(corruption_detected); - if (litCSize + lhSize > srcSize) - return ERROR(corruption_detected); - - if (HUF_isError( - (litEncType == set_repeat) - ? (singleStream ? HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart + lhSize, litCSize, dctx->HUFptr) - : HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart + lhSize, litCSize, dctx->HUFptr)) - : (singleStream - ? HUF_decompress1X2_DCtx_wksp(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart + lhSize, litCSize, - dctx->entropy.workspace, sizeof(dctx->entropy.workspace)) - : HUF_decompress4X_hufOnly_wksp(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart + lhSize, litCSize, - dctx->entropy.workspace, sizeof(dctx->entropy.workspace))))) - return ERROR(corruption_detected); - - dctx->litPtr = dctx->litBuffer; - dctx->litSize = litSize; - dctx->litEntropy = 1; - if (litEncType == set_compressed) - dctx->HUFptr = dctx->entropy.hufTable; - memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); - return litCSize + lhSize; - } - - case set_basic: { - size_t litSize, lhSize; - U32 const lhlCode = ((istart[0]) >> 2) & 3; - switch (lhlCode) { - case 0: - case 2: - default: /* note : default is impossible, since lhlCode into [0..3] */ - lhSize = 1; - litSize = istart[0] >> 3; - break; - case 1: - lhSize = 2; - litSize = ZSTD_readLE16(istart) >> 4; - break; - case 3: - lhSize = 3; - litSize = ZSTD_readLE24(istart) >> 4; - break; - } - - if (lhSize + litSize + WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ - if (litSize + lhSize > srcSize) - return ERROR(corruption_detected); - memcpy(dctx->litBuffer, istart + lhSize, litSize); - dctx->litPtr = dctx->litBuffer; - dctx->litSize = litSize; - memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); - return lhSize + litSize; - } - /* direct reference into compressed stream */ - dctx->litPtr = istart + lhSize; - dctx->litSize = litSize; - return lhSize + litSize; - } - - case set_rle: { - U32 const lhlCode = ((istart[0]) >> 2) & 3; - size_t litSize, lhSize; - switch (lhlCode) { - case 0: - case 2: - default: /* note : default is impossible, since lhlCode into [0..3] */ - lhSize = 1; - litSize = istart[0] >> 3; - break; - case 1: - lhSize = 2; - litSize = ZSTD_readLE16(istart) >> 4; - break; - case 3: - lhSize = 3; - litSize = ZSTD_readLE24(istart) >> 4; - if (srcSize < 4) - return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */ - break; - } - if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) - return ERROR(corruption_detected); - memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH); - dctx->litPtr = dctx->litBuffer; - dctx->litSize = litSize; - return lhSize + 1; - } - default: - return ERROR(corruption_detected); /* impossible */ - } - } -} - -typedef union { - FSE_decode_t realData; - U32 alignedBy4; -} FSE_decode_t4; - -static const FSE_decode_t4 LL_defaultDTable[(1 << LL_DEFAULTNORMLOG) + 1] = { - {{LL_DEFAULTNORMLOG, 1, 1}}, /* header : tableLog, fastMode, fastMode */ - {{0, 0, 4}}, /* 0 : base, symbol, bits */ - {{16, 0, 4}}, - {{32, 1, 5}}, - {{0, 3, 5}}, - {{0, 4, 5}}, - {{0, 6, 5}}, - {{0, 7, 5}}, - {{0, 9, 5}}, - {{0, 10, 5}}, - {{0, 12, 5}}, - {{0, 14, 6}}, - {{0, 16, 5}}, - {{0, 18, 5}}, - {{0, 19, 5}}, - {{0, 21, 5}}, - {{0, 22, 5}}, - {{0, 24, 5}}, - {{32, 25, 5}}, - {{0, 26, 5}}, - {{0, 27, 6}}, - {{0, 29, 6}}, - {{0, 31, 6}}, - {{32, 0, 4}}, - {{0, 1, 4}}, - {{0, 2, 5}}, - {{32, 4, 5}}, - {{0, 5, 5}}, - {{32, 7, 5}}, - {{0, 8, 5}}, - {{32, 10, 5}}, - {{0, 11, 5}}, - {{0, 13, 6}}, - {{32, 16, 5}}, - {{0, 17, 5}}, - {{32, 19, 5}}, - {{0, 20, 5}}, - {{32, 22, 5}}, - {{0, 23, 5}}, - {{0, 25, 4}}, - {{16, 25, 4}}, - {{32, 26, 5}}, - {{0, 28, 6}}, - {{0, 30, 6}}, - {{48, 0, 4}}, - {{16, 1, 4}}, - {{32, 2, 5}}, - {{32, 3, 5}}, - {{32, 5, 5}}, - {{32, 6, 5}}, - {{32, 8, 5}}, - {{32, 9, 5}}, - {{32, 11, 5}}, - {{32, 12, 5}}, - {{0, 15, 6}}, - {{32, 17, 5}}, - {{32, 18, 5}}, - {{32, 20, 5}}, - {{32, 21, 5}}, - {{32, 23, 5}}, - {{32, 24, 5}}, - {{0, 35, 6}}, - {{0, 34, 6}}, - {{0, 33, 6}}, - {{0, 32, 6}}, -}; /* LL_defaultDTable */ - -static const FSE_decode_t4 ML_defaultDTable[(1 << ML_DEFAULTNORMLOG) + 1] = { - {{ML_DEFAULTNORMLOG, 1, 1}}, /* header : tableLog, fastMode, fastMode */ - {{0, 0, 6}}, /* 0 : base, symbol, bits */ - {{0, 1, 4}}, - {{32, 2, 5}}, - {{0, 3, 5}}, - {{0, 5, 5}}, - {{0, 6, 5}}, - {{0, 8, 5}}, - {{0, 10, 6}}, - {{0, 13, 6}}, - {{0, 16, 6}}, - {{0, 19, 6}}, - {{0, 22, 6}}, - {{0, 25, 6}}, - {{0, 28, 6}}, - {{0, 31, 6}}, - {{0, 33, 6}}, - {{0, 35, 6}}, - {{0, 37, 6}}, - {{0, 39, 6}}, - {{0, 41, 6}}, - {{0, 43, 6}}, - {{0, 45, 6}}, - {{16, 1, 4}}, - {{0, 2, 4}}, - {{32, 3, 5}}, - {{0, 4, 5}}, - {{32, 6, 5}}, - {{0, 7, 5}}, - {{0, 9, 6}}, - {{0, 12, 6}}, - {{0, 15, 6}}, - {{0, 18, 6}}, - {{0, 21, 6}}, - {{0, 24, 6}}, - {{0, 27, 6}}, - {{0, 30, 6}}, - {{0, 32, 6}}, - {{0, 34, 6}}, - {{0, 36, 6}}, - {{0, 38, 6}}, - {{0, 40, 6}}, - {{0, 42, 6}}, - {{0, 44, 6}}, - {{32, 1, 4}}, - {{48, 1, 4}}, - {{16, 2, 4}}, - {{32, 4, 5}}, - {{32, 5, 5}}, - {{32, 7, 5}}, - {{32, 8, 5}}, - {{0, 11, 6}}, - {{0, 14, 6}}, - {{0, 17, 6}}, - {{0, 20, 6}}, - {{0, 23, 6}}, - {{0, 26, 6}}, - {{0, 29, 6}}, - {{0, 52, 6}}, - {{0, 51, 6}}, - {{0, 50, 6}}, - {{0, 49, 6}}, - {{0, 48, 6}}, - {{0, 47, 6}}, - {{0, 46, 6}}, -}; /* ML_defaultDTable */ - -static const FSE_decode_t4 OF_defaultDTable[(1 << OF_DEFAULTNORMLOG) + 1] = { - {{OF_DEFAULTNORMLOG, 1, 1}}, /* header : tableLog, fastMode, fastMode */ - {{0, 0, 5}}, /* 0 : base, symbol, bits */ - {{0, 6, 4}}, - {{0, 9, 5}}, - {{0, 15, 5}}, - {{0, 21, 5}}, - {{0, 3, 5}}, - {{0, 7, 4}}, - {{0, 12, 5}}, - {{0, 18, 5}}, - {{0, 23, 5}}, - {{0, 5, 5}}, - {{0, 8, 4}}, - {{0, 14, 5}}, - {{0, 20, 5}}, - {{0, 2, 5}}, - {{16, 7, 4}}, - {{0, 11, 5}}, - {{0, 17, 5}}, - {{0, 22, 5}}, - {{0, 4, 5}}, - {{16, 8, 4}}, - {{0, 13, 5}}, - {{0, 19, 5}}, - {{0, 1, 5}}, - {{16, 6, 4}}, - {{0, 10, 5}}, - {{0, 16, 5}}, - {{0, 28, 5}}, - {{0, 27, 5}}, - {{0, 26, 5}}, - {{0, 25, 5}}, - {{0, 24, 5}}, -}; /* OF_defaultDTable */ - -/*! ZSTD_buildSeqTable() : - @return : nb bytes read from src, - or an error code if it fails, testable with ZSTD_isError() -*/ -static size_t ZSTD_buildSeqTable(FSE_DTable *DTableSpace, const FSE_DTable **DTablePtr, symbolEncodingType_e type, U32 max, U32 maxLog, const void *src, - size_t srcSize, const FSE_decode_t4 *defaultTable, U32 flagRepeatTable, void *workspace, size_t workspaceSize) -{ - const void *const tmpPtr = defaultTable; /* bypass strict aliasing */ - switch (type) { - case set_rle: - if (!srcSize) - return ERROR(srcSize_wrong); - if ((*(const BYTE *)src) > max) - return ERROR(corruption_detected); - FSE_buildDTable_rle(DTableSpace, *(const BYTE *)src); - *DTablePtr = DTableSpace; - return 1; - case set_basic: *DTablePtr = (const FSE_DTable *)tmpPtr; return 0; - case set_repeat: - if (!flagRepeatTable) - return ERROR(corruption_detected); - return 0; - default: /* impossible */ - case set_compressed: { - U32 tableLog; - S16 *norm = (S16 *)workspace; - size_t const spaceUsed32 = ALIGN(sizeof(S16) * (MaxSeq + 1), sizeof(U32)) >> 2; - - if ((spaceUsed32 << 2) > workspaceSize) - return ERROR(GENERIC); - workspace = (U32 *)workspace + spaceUsed32; - workspaceSize -= (spaceUsed32 << 2); - { - size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize); - if (FSE_isError(headerSize)) - return ERROR(corruption_detected); - if (tableLog > maxLog) - return ERROR(corruption_detected); - FSE_buildDTable_wksp(DTableSpace, norm, max, tableLog, workspace, workspaceSize); - *DTablePtr = DTableSpace; - return headerSize; - } - } - } -} - -size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx *dctx, int *nbSeqPtr, const void *src, size_t srcSize) -{ - const BYTE *const istart = (const BYTE *const)src; - const BYTE *const iend = istart + srcSize; - const BYTE *ip = istart; - - /* check */ - if (srcSize < MIN_SEQUENCES_SIZE) - return ERROR(srcSize_wrong); - - /* SeqHead */ - { - int nbSeq = *ip++; - if (!nbSeq) { - *nbSeqPtr = 0; - return 1; - } - if (nbSeq > 0x7F) { - if (nbSeq == 0xFF) { - if (ip + 2 > iend) - return ERROR(srcSize_wrong); - nbSeq = ZSTD_readLE16(ip) + LONGNBSEQ, ip += 2; - } else { - if (ip >= iend) - return ERROR(srcSize_wrong); - nbSeq = ((nbSeq - 0x80) << 8) + *ip++; - } - } - *nbSeqPtr = nbSeq; - } - - /* FSE table descriptors */ - if (ip + 4 > iend) - return ERROR(srcSize_wrong); /* minimum possible size */ - { - symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); - symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); - symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); - ip++; - - /* Build DTables */ - { - size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr, LLtype, MaxLL, LLFSELog, ip, iend - ip, - LL_defaultDTable, dctx->fseEntropy, dctx->entropy.workspace, sizeof(dctx->entropy.workspace)); - if (ZSTD_isError(llhSize)) - return ERROR(corruption_detected); - ip += llhSize; - } - { - size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr, OFtype, MaxOff, OffFSELog, ip, iend - ip, - OF_defaultDTable, dctx->fseEntropy, dctx->entropy.workspace, sizeof(dctx->entropy.workspace)); - if (ZSTD_isError(ofhSize)) - return ERROR(corruption_detected); - ip += ofhSize; - } - { - size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr, MLtype, MaxML, MLFSELog, ip, iend - ip, - ML_defaultDTable, dctx->fseEntropy, dctx->entropy.workspace, sizeof(dctx->entropy.workspace)); - if (ZSTD_isError(mlhSize)) - return ERROR(corruption_detected); - ip += mlhSize; - } - } - - return ip - istart; -} - -typedef struct { - size_t litLength; - size_t matchLength; - size_t offset; - const BYTE *match; -} seq_t; - -typedef struct { - BIT_DStream_t DStream; - FSE_DState_t stateLL; - FSE_DState_t stateOffb; - FSE_DState_t stateML; - size_t prevOffset[ZSTD_REP_NUM]; - const BYTE *base; - size_t pos; - uPtrDiff gotoDict; -} seqState_t; - -FORCE_NOINLINE -size_t ZSTD_execSequenceLast7(BYTE *op, BYTE *const oend, seq_t sequence, const BYTE **litPtr, const BYTE *const litLimit, const BYTE *const base, - const BYTE *const vBase, const BYTE *const dictEnd) -{ - BYTE *const oLitEnd = op + sequence.litLength; - size_t const sequenceLength = sequence.litLength + sequence.matchLength; - BYTE *const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ - BYTE *const oend_w = oend - WILDCOPY_OVERLENGTH; - const BYTE *const iLitEnd = *litPtr + sequence.litLength; - const BYTE *match = oLitEnd - sequence.offset; - - /* check */ - if (oMatchEnd > oend) - return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ - if (iLitEnd > litLimit) - return ERROR(corruption_detected); /* over-read beyond lit buffer */ - if (oLitEnd <= oend_w) - return ERROR(GENERIC); /* Precondition */ - - /* copy literals */ - if (op < oend_w) { - ZSTD_wildcopy(op, *litPtr, oend_w - op); - *litPtr += oend_w - op; - op = oend_w; - } - while (op < oLitEnd) - *op++ = *(*litPtr)++; - - /* copy Match */ - if (sequence.offset > (size_t)(oLitEnd - base)) { - /* offset beyond prefix */ - if (sequence.offset > (size_t)(oLitEnd - vBase)) - return ERROR(corruption_detected); - match = dictEnd - (base - match); - if (match + sequence.matchLength <= dictEnd) { - memmove(oLitEnd, match, sequence.matchLength); - return sequenceLength; - } - /* span extDict & currPrefixSegment */ - { - size_t const length1 = dictEnd - match; - memmove(oLitEnd, match, length1); - op = oLitEnd + length1; - sequence.matchLength -= length1; - match = base; - } - } - while (op < oMatchEnd) - *op++ = *match++; - return sequenceLength; -} - -static seq_t ZSTD_decodeSequence(seqState_t *seqState) -{ - seq_t seq; - - U32 const llCode = FSE_peekSymbol(&seqState->stateLL); - U32 const mlCode = FSE_peekSymbol(&seqState->stateML); - U32 const ofCode = FSE_peekSymbol(&seqState->stateOffb); /* <= maxOff, by table construction */ - - U32 const llBits = LL_bits[llCode]; - U32 const mlBits = ML_bits[mlCode]; - U32 const ofBits = ofCode; - U32 const totalBits = llBits + mlBits + ofBits; - - static const U32 LL_base[MaxLL + 1] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, - 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000}; - - static const U32 ML_base[MaxML + 1] = {3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, - 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 39, 41, - 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, 0x1003, 0x2003, 0x4003, 0x8003, 0x10003}; - - static const U32 OF_base[MaxOff + 1] = {0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, 0xFD, 0x1FD, - 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, - 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD}; - - /* sequence */ - { - size_t offset; - if (!ofCode) - offset = 0; - else { - offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ - if (ZSTD_32bits()) - BIT_reloadDStream(&seqState->DStream); - } - - if (ofCode <= 1) { - offset += (llCode == 0); - if (offset) { - size_t temp = (offset == 3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; - temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ - if (offset != 1) - seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset = temp; - } else { - offset = seqState->prevOffset[0]; - } - } else { - seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset; - } - seq.offset = offset; - } - - seq.matchLength = ML_base[mlCode] + ((mlCode > 31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */ - if (ZSTD_32bits() && (mlBits + llBits > 24)) - BIT_reloadDStream(&seqState->DStream); - - seq.litLength = LL_base[llCode] + ((llCode > 15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */ - if (ZSTD_32bits() || (totalBits > 64 - 7 - (LLFSELog + MLFSELog + OffFSELog))) - BIT_reloadDStream(&seqState->DStream); - - /* ANS state update */ - FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ - FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ - if (ZSTD_32bits()) - BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ - FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ - - seq.match = NULL; - - return seq; -} - -FORCE_INLINE -size_t ZSTD_execSequence(BYTE *op, BYTE *const oend, seq_t sequence, const BYTE **litPtr, const BYTE *const litLimit, const BYTE *const base, - const BYTE *const vBase, const BYTE *const dictEnd) -{ - BYTE *const oLitEnd = op + sequence.litLength; - size_t const sequenceLength = sequence.litLength + sequence.matchLength; - BYTE *const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ - BYTE *const oend_w = oend - WILDCOPY_OVERLENGTH; - const BYTE *const iLitEnd = *litPtr + sequence.litLength; - const BYTE *match = oLitEnd - sequence.offset; - - /* check */ - if (oMatchEnd > oend) - return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ - if (iLitEnd > litLimit) - return ERROR(corruption_detected); /* over-read beyond lit buffer */ - if (oLitEnd > oend_w) - return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd); - - /* copy Literals */ - ZSTD_copy8(op, *litPtr); - if (sequence.litLength > 8) - ZSTD_wildcopy(op + 8, (*litPtr) + 8, - sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ - op = oLitEnd; - *litPtr = iLitEnd; /* update for next sequence */ - - /* copy Match */ - if (sequence.offset > (size_t)(oLitEnd - base)) { - /* offset beyond prefix */ - if (sequence.offset > (size_t)(oLitEnd - vBase)) - return ERROR(corruption_detected); - match = dictEnd + (match - base); - if (match + sequence.matchLength <= dictEnd) { - memmove(oLitEnd, match, sequence.matchLength); - return sequenceLength; - } - /* span extDict & currPrefixSegment */ - { - size_t const length1 = dictEnd - match; - memmove(oLitEnd, match, length1); - op = oLitEnd + length1; - sequence.matchLength -= length1; - match = base; - if (op > oend_w || sequence.matchLength < MINMATCH) { - U32 i; - for (i = 0; i < sequence.matchLength; ++i) - op[i] = match[i]; - return sequenceLength; - } - } - } - /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */ - - /* match within prefix */ - if (sequence.offset < 8) { - /* close range match, overlap */ - static const U32 dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */ - static const int dec64table[] = {8, 8, 8, 7, 8, 9, 10, 11}; /* subtracted */ - int const sub2 = dec64table[sequence.offset]; - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += dec32table[sequence.offset]; - ZSTD_copy4(op + 4, match); - match -= sub2; - } else { - ZSTD_copy8(op, match); - } - op += 8; - match += 8; - - if (oMatchEnd > oend - (16 - MINMATCH)) { - if (op < oend_w) { - ZSTD_wildcopy(op, match, oend_w - op); - match += oend_w - op; - op = oend_w; - } - while (op < oMatchEnd) - *op++ = *match++; - } else { - ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8); /* works even if matchLength < 8 */ - } - return sequenceLength; -} - -static size_t ZSTD_decompressSequences(ZSTD_DCtx *dctx, void *dst, size_t maxDstSize, const void *seqStart, size_t seqSize) -{ - const BYTE *ip = (const BYTE *)seqStart; - const BYTE *const iend = ip + seqSize; - BYTE *const ostart = (BYTE * const)dst; - BYTE *const oend = ostart + maxDstSize; - BYTE *op = ostart; - const BYTE *litPtr = dctx->litPtr; - const BYTE *const litEnd = litPtr + dctx->litSize; - const BYTE *const base = (const BYTE *)(dctx->base); - const BYTE *const vBase = (const BYTE *)(dctx->vBase); - const BYTE *const dictEnd = (const BYTE *)(dctx->dictEnd); - int nbSeq; - - /* Build Decoding Tables */ - { - size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize); - if (ZSTD_isError(seqHSize)) - return seqHSize; - ip += seqHSize; - } - - /* Regen sequences */ - if (nbSeq) { - seqState_t seqState; - dctx->fseEntropy = 1; - { - U32 i; - for (i = 0; i < ZSTD_REP_NUM; i++) - seqState.prevOffset[i] = dctx->entropy.rep[i]; - } - CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend - ip), corruption_detected); - FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); - FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); - FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); - - for (; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq;) { - nbSeq--; - { - seq_t const sequence = ZSTD_decodeSequence(&seqState); - size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd); - if (ZSTD_isError(oneSeqSize)) - return oneSeqSize; - op += oneSeqSize; - } - } - - /* check if reached exact end */ - if (nbSeq) - return ERROR(corruption_detected); - /* save reps for next block */ - { - U32 i; - for (i = 0; i < ZSTD_REP_NUM; i++) - dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); - } - } - - /* last literal segment */ - { - size_t const lastLLSize = litEnd - litPtr; - if (lastLLSize > (size_t)(oend - op)) - return ERROR(dstSize_tooSmall); - memcpy(op, litPtr, lastLLSize); - op += lastLLSize; - } - - return op - ostart; -} - -FORCE_INLINE seq_t ZSTD_decodeSequenceLong_generic(seqState_t *seqState, int const longOffsets) -{ - seq_t seq; - - U32 const llCode = FSE_peekSymbol(&seqState->stateLL); - U32 const mlCode = FSE_peekSymbol(&seqState->stateML); - U32 const ofCode = FSE_peekSymbol(&seqState->stateOffb); /* <= maxOff, by table construction */ - - U32 const llBits = LL_bits[llCode]; - U32 const mlBits = ML_bits[mlCode]; - U32 const ofBits = ofCode; - U32 const totalBits = llBits + mlBits + ofBits; - - static const U32 LL_base[MaxLL + 1] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, - 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000}; - - static const U32 ML_base[MaxML + 1] = {3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, - 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 39, 41, - 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, 0x1003, 0x2003, 0x4003, 0x8003, 0x10003}; - - static const U32 OF_base[MaxOff + 1] = {0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, 0xFD, 0x1FD, - 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, - 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD}; - - /* sequence */ - { - size_t offset; - if (!ofCode) - offset = 0; - else { - if (longOffsets) { - int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN); - offset = OF_base[ofCode] + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); - if (ZSTD_32bits() || extraBits) - BIT_reloadDStream(&seqState->DStream); - if (extraBits) - offset += BIT_readBitsFast(&seqState->DStream, extraBits); - } else { - offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ - if (ZSTD_32bits()) - BIT_reloadDStream(&seqState->DStream); - } - } - - if (ofCode <= 1) { - offset += (llCode == 0); - if (offset) { - size_t temp = (offset == 3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; - temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ - if (offset != 1) - seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset = temp; - } else { - offset = seqState->prevOffset[0]; - } - } else { - seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset; - } - seq.offset = offset; - } - - seq.matchLength = ML_base[mlCode] + ((mlCode > 31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */ - if (ZSTD_32bits() && (mlBits + llBits > 24)) - BIT_reloadDStream(&seqState->DStream); - - seq.litLength = LL_base[llCode] + ((llCode > 15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */ - if (ZSTD_32bits() || (totalBits > 64 - 7 - (LLFSELog + MLFSELog + OffFSELog))) - BIT_reloadDStream(&seqState->DStream); - - { - size_t const pos = seqState->pos + seq.litLength; - seq.match = seqState->base + pos - seq.offset; /* single memory segment */ - if (seq.offset > pos) - seq.match += seqState->gotoDict; /* separate memory segment */ - seqState->pos = pos + seq.matchLength; - } - - /* ANS state update */ - FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ - FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ - if (ZSTD_32bits()) - BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ - FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ - - return seq; -} - -static seq_t ZSTD_decodeSequenceLong(seqState_t *seqState, unsigned const windowSize) -{ - if (ZSTD_highbit32(windowSize) > STREAM_ACCUMULATOR_MIN) { - return ZSTD_decodeSequenceLong_generic(seqState, 1); - } else { - return ZSTD_decodeSequenceLong_generic(seqState, 0); - } -} - -FORCE_INLINE -size_t ZSTD_execSequenceLong(BYTE *op, BYTE *const oend, seq_t sequence, const BYTE **litPtr, const BYTE *const litLimit, const BYTE *const base, - const BYTE *const vBase, const BYTE *const dictEnd) -{ - BYTE *const oLitEnd = op + sequence.litLength; - size_t const sequenceLength = sequence.litLength + sequence.matchLength; - BYTE *const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ - BYTE *const oend_w = oend - WILDCOPY_OVERLENGTH; - const BYTE *const iLitEnd = *litPtr + sequence.litLength; - const BYTE *match = sequence.match; - - /* check */ - if (oMatchEnd > oend) - return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ - if (iLitEnd > litLimit) - return ERROR(corruption_detected); /* over-read beyond lit buffer */ - if (oLitEnd > oend_w) - return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd); - - /* copy Literals */ - ZSTD_copy8(op, *litPtr); - if (sequence.litLength > 8) - ZSTD_wildcopy(op + 8, (*litPtr) + 8, - sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ - op = oLitEnd; - *litPtr = iLitEnd; /* update for next sequence */ - - /* copy Match */ - if (sequence.offset > (size_t)(oLitEnd - base)) { - /* offset beyond prefix */ - if (sequence.offset > (size_t)(oLitEnd - vBase)) - return ERROR(corruption_detected); - if (match + sequence.matchLength <= dictEnd) { - memmove(oLitEnd, match, sequence.matchLength); - return sequenceLength; - } - /* span extDict & currPrefixSegment */ - { - size_t const length1 = dictEnd - match; - memmove(oLitEnd, match, length1); - op = oLitEnd + length1; - sequence.matchLength -= length1; - match = base; - if (op > oend_w || sequence.matchLength < MINMATCH) { - U32 i; - for (i = 0; i < sequence.matchLength; ++i) - op[i] = match[i]; - return sequenceLength; - } - } - } - /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */ - - /* match within prefix */ - if (sequence.offset < 8) { - /* close range match, overlap */ - static const U32 dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */ - static const int dec64table[] = {8, 8, 8, 7, 8, 9, 10, 11}; /* subtracted */ - int const sub2 = dec64table[sequence.offset]; - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += dec32table[sequence.offset]; - ZSTD_copy4(op + 4, match); - match -= sub2; - } else { - ZSTD_copy8(op, match); - } - op += 8; - match += 8; - - if (oMatchEnd > oend - (16 - MINMATCH)) { - if (op < oend_w) { - ZSTD_wildcopy(op, match, oend_w - op); - match += oend_w - op; - op = oend_w; - } - while (op < oMatchEnd) - *op++ = *match++; - } else { - ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8); /* works even if matchLength < 8 */ - } - return sequenceLength; -} - -static size_t ZSTD_decompressSequencesLong(ZSTD_DCtx *dctx, void *dst, size_t maxDstSize, const void *seqStart, size_t seqSize) -{ - const BYTE *ip = (const BYTE *)seqStart; - const BYTE *const iend = ip + seqSize; - BYTE *const ostart = (BYTE * const)dst; - BYTE *const oend = ostart + maxDstSize; - BYTE *op = ostart; - const BYTE *litPtr = dctx->litPtr; - const BYTE *const litEnd = litPtr + dctx->litSize; - const BYTE *const base = (const BYTE *)(dctx->base); - const BYTE *const vBase = (const BYTE *)(dctx->vBase); - const BYTE *const dictEnd = (const BYTE *)(dctx->dictEnd); - unsigned const windowSize = dctx->fParams.windowSize; - int nbSeq; - - /* Build Decoding Tables */ - { - size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize); - if (ZSTD_isError(seqHSize)) - return seqHSize; - ip += seqHSize; - } - - /* Regen sequences */ - if (nbSeq) { -#define STORED_SEQS 4 -#define STOSEQ_MASK (STORED_SEQS - 1) -#define ADVANCED_SEQS 4 - seq_t *sequences = (seq_t *)dctx->entropy.workspace; - int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS); - seqState_t seqState; - int seqNb; - ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.workspace) >= sizeof(seq_t) * STORED_SEQS); - dctx->fseEntropy = 1; - { - U32 i; - for (i = 0; i < ZSTD_REP_NUM; i++) - seqState.prevOffset[i] = dctx->entropy.rep[i]; - } - seqState.base = base; - seqState.pos = (size_t)(op - base); - seqState.gotoDict = (uPtrDiff)dictEnd - (uPtrDiff)base; /* cast to avoid undefined behaviour */ - CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend - ip), corruption_detected); - FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); - FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); - FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); - - /* prepare in advance */ - for (seqNb = 0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && seqNb < seqAdvance; seqNb++) { - sequences[seqNb] = ZSTD_decodeSequenceLong(&seqState, windowSize); - } - if (seqNb < seqAdvance) - return ERROR(corruption_detected); - - /* decode and decompress */ - for (; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && seqNb < nbSeq; seqNb++) { - seq_t const sequence = ZSTD_decodeSequenceLong(&seqState, windowSize); - size_t const oneSeqSize = - ZSTD_execSequenceLong(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd); - if (ZSTD_isError(oneSeqSize)) - return oneSeqSize; - ZSTD_PREFETCH(sequence.match); - sequences[seqNb & STOSEQ_MASK] = sequence; - op += oneSeqSize; - } - if (seqNb < nbSeq) - return ERROR(corruption_detected); - - /* finish queue */ - seqNb -= seqAdvance; - for (; seqNb < nbSeq; seqNb++) { - size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[seqNb & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd); - if (ZSTD_isError(oneSeqSize)) - return oneSeqSize; - op += oneSeqSize; - } - - /* save reps for next block */ - { - U32 i; - for (i = 0; i < ZSTD_REP_NUM; i++) - dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); - } - } - - /* last literal segment */ - { - size_t const lastLLSize = litEnd - litPtr; - if (lastLLSize > (size_t)(oend - op)) - return ERROR(dstSize_tooSmall); - memcpy(op, litPtr, lastLLSize); - op += lastLLSize; - } - - return op - ostart; -} - -static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ /* blockType == blockCompressed */ - const BYTE *ip = (const BYTE *)src; - - if (srcSize >= ZSTD_BLOCKSIZE_ABSOLUTEMAX) - return ERROR(srcSize_wrong); - - /* Decode literals section */ - { - size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize); - if (ZSTD_isError(litCSize)) - return litCSize; - ip += litCSize; - srcSize -= litCSize; - } - if (sizeof(size_t) > 4) /* do not enable prefetching on 32-bits x86, as it's performance detrimental */ - /* likely because of register pressure */ - /* if that's the correct cause, then 32-bits ARM should be affected differently */ - /* it would be good to test this on ARM real hardware, to see if prefetch version improves speed */ - if (dctx->fParams.windowSize > (1 << 23)) - return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize); - return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize); -} - -static void ZSTD_checkContinuity(ZSTD_DCtx *dctx, const void *dst) -{ - if (dst != dctx->previousDstEnd) { /* not contiguous */ - dctx->dictEnd = dctx->previousDstEnd; - dctx->vBase = (const char *)dst - ((const char *)(dctx->previousDstEnd) - (const char *)(dctx->base)); - dctx->base = dst; - dctx->previousDstEnd = dst; - } -} - -size_t ZSTD_decompressBlock(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - size_t dSize; - ZSTD_checkContinuity(dctx, dst); - dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); - dctx->previousDstEnd = (char *)dst + dSize; - return dSize; -} - -/** ZSTD_insertBlock() : - insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ -size_t ZSTD_insertBlock(ZSTD_DCtx *dctx, const void *blockStart, size_t blockSize) -{ - ZSTD_checkContinuity(dctx, blockStart); - dctx->previousDstEnd = (const char *)blockStart + blockSize; - return blockSize; -} - -size_t ZSTD_generateNxBytes(void *dst, size_t dstCapacity, BYTE byte, size_t length) -{ - if (length > dstCapacity) - return ERROR(dstSize_tooSmall); - memset(dst, byte, length); - return length; -} - -/** ZSTD_findFrameCompressedSize() : - * compatible with legacy mode - * `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame - * `srcSize` must be at least as large as the frame contained - * @return : the compressed size of the frame starting at `src` */ -size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) -{ - if (srcSize >= ZSTD_skippableHeaderSize && (ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { - return ZSTD_skippableHeaderSize + ZSTD_readLE32((const BYTE *)src + 4); - } else { - const BYTE *ip = (const BYTE *)src; - const BYTE *const ipstart = ip; - size_t remainingSize = srcSize; - ZSTD_frameParams fParams; - - size_t const headerSize = ZSTD_frameHeaderSize(ip, remainingSize); - if (ZSTD_isError(headerSize)) - return headerSize; - - /* Frame Header */ - { - size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize); - if (ZSTD_isError(ret)) - return ret; - if (ret > 0) - return ERROR(srcSize_wrong); - } - - ip += headerSize; - remainingSize -= headerSize; - - /* Loop on each block */ - while (1) { - blockProperties_t blockProperties; - size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); - if (ZSTD_isError(cBlockSize)) - return cBlockSize; - - if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) - return ERROR(srcSize_wrong); - - ip += ZSTD_blockHeaderSize + cBlockSize; - remainingSize -= ZSTD_blockHeaderSize + cBlockSize; - - if (blockProperties.lastBlock) - break; - } - - if (fParams.checksumFlag) { /* Frame content checksum */ - if (remainingSize < 4) - return ERROR(srcSize_wrong); - ip += 4; - remainingSize -= 4; - } - - return ip - ipstart; - } -} - -/*! ZSTD_decompressFrame() : -* @dctx must be properly initialized */ -static size_t ZSTD_decompressFrame(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void **srcPtr, size_t *srcSizePtr) -{ - const BYTE *ip = (const BYTE *)(*srcPtr); - BYTE *const ostart = (BYTE * const)dst; - BYTE *const oend = ostart + dstCapacity; - BYTE *op = ostart; - size_t remainingSize = *srcSizePtr; - - /* check */ - if (remainingSize < ZSTD_frameHeaderSize_min + ZSTD_blockHeaderSize) - return ERROR(srcSize_wrong); - - /* Frame Header */ - { - size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_frameHeaderSize_prefix); - if (ZSTD_isError(frameHeaderSize)) - return frameHeaderSize; - if (remainingSize < frameHeaderSize + ZSTD_blockHeaderSize) - return ERROR(srcSize_wrong); - CHECK_F(ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize)); - ip += frameHeaderSize; - remainingSize -= frameHeaderSize; - } - - /* Loop on each block */ - while (1) { - size_t decodedSize; - blockProperties_t blockProperties; - size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); - if (ZSTD_isError(cBlockSize)) - return cBlockSize; - - ip += ZSTD_blockHeaderSize; - remainingSize -= ZSTD_blockHeaderSize; - if (cBlockSize > remainingSize) - return ERROR(srcSize_wrong); - - switch (blockProperties.blockType) { - case bt_compressed: decodedSize = ZSTD_decompressBlock_internal(dctx, op, oend - op, ip, cBlockSize); break; - case bt_raw: decodedSize = ZSTD_copyRawBlock(op, oend - op, ip, cBlockSize); break; - case bt_rle: decodedSize = ZSTD_generateNxBytes(op, oend - op, *ip, blockProperties.origSize); break; - case bt_reserved: - default: return ERROR(corruption_detected); - } - - if (ZSTD_isError(decodedSize)) - return decodedSize; - if (dctx->fParams.checksumFlag) - xxh64_update(&dctx->xxhState, op, decodedSize); - op += decodedSize; - ip += cBlockSize; - remainingSize -= cBlockSize; - if (blockProperties.lastBlock) - break; - } - - if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */ - U32 const checkCalc = (U32)xxh64_digest(&dctx->xxhState); - U32 checkRead; - if (remainingSize < 4) - return ERROR(checksum_wrong); - checkRead = ZSTD_readLE32(ip); - if (checkRead != checkCalc) - return ERROR(checksum_wrong); - ip += 4; - remainingSize -= 4; - } - - /* Allow caller to get size read */ - *srcPtr = ip; - *srcSizePtr = remainingSize; - return op - ostart; -} - -static const void *ZSTD_DDictDictContent(const ZSTD_DDict *ddict); -static size_t ZSTD_DDictDictSize(const ZSTD_DDict *ddict); - -static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize, - const ZSTD_DDict *ddict) -{ - void *const dststart = dst; - - if (ddict) { - if (dict) { - /* programmer error, these two cases should be mutually exclusive */ - return ERROR(GENERIC); - } - - dict = ZSTD_DDictDictContent(ddict); - dictSize = ZSTD_DDictDictSize(ddict); - } - - while (srcSize >= ZSTD_frameHeaderSize_prefix) { - U32 magicNumber; - - magicNumber = ZSTD_readLE32(src); - if (magicNumber != ZSTD_MAGICNUMBER) { - if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { - size_t skippableSize; - if (srcSize < ZSTD_skippableHeaderSize) - return ERROR(srcSize_wrong); - skippableSize = ZSTD_readLE32((const BYTE *)src + 4) + ZSTD_skippableHeaderSize; - if (srcSize < skippableSize) { - return ERROR(srcSize_wrong); - } - - src = (const BYTE *)src + skippableSize; - srcSize -= skippableSize; - continue; - } else { - return ERROR(prefix_unknown); - } - } - - if (ddict) { - /* we were called from ZSTD_decompress_usingDDict */ - ZSTD_refDDict(dctx, ddict); - } else { - /* this will initialize correctly with no dict if dict == NULL, so - * use this in all cases but ddict */ - CHECK_F(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize)); - } - ZSTD_checkContinuity(dctx, dst); - - { - const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity, &src, &srcSize); - if (ZSTD_isError(res)) - return res; - /* don't need to bounds check this, ZSTD_decompressFrame will have - * already */ - dst = (BYTE *)dst + res; - dstCapacity -= res; - } - } - - if (srcSize) - return ERROR(srcSize_wrong); /* input not entirely consumed */ - - return (BYTE *)dst - (BYTE *)dststart; -} - -size_t ZSTD_decompress_usingDict(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize) -{ - return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL); -} - -size_t ZSTD_decompressDCtx(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - return ZSTD_decompress_usingDict(dctx, dst, dstCapacity, src, srcSize, NULL, 0); -} - -/*-************************************** -* Advanced Streaming Decompression API -* Bufferless and synchronous -****************************************/ -size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx *dctx) { return dctx->expected; } - -ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx *dctx) -{ - switch (dctx->stage) { - default: /* should not happen */ - case ZSTDds_getFrameHeaderSize: - case ZSTDds_decodeFrameHeader: return ZSTDnit_frameHeader; - case ZSTDds_decodeBlockHeader: return ZSTDnit_blockHeader; - case ZSTDds_decompressBlock: return ZSTDnit_block; - case ZSTDds_decompressLastBlock: return ZSTDnit_lastBlock; - case ZSTDds_checkChecksum: return ZSTDnit_checksum; - case ZSTDds_decodeSkippableHeader: - case ZSTDds_skipFrame: return ZSTDnit_skippableFrame; - } -} - -int ZSTD_isSkipFrame(ZSTD_DCtx *dctx) { return dctx->stage == ZSTDds_skipFrame; } /* for zbuff */ - -/** ZSTD_decompressContinue() : -* @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity) -* or an error code, which can be tested using ZSTD_isError() */ -size_t ZSTD_decompressContinue(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - /* Sanity check */ - if (srcSize != dctx->expected) - return ERROR(srcSize_wrong); - if (dstCapacity) - ZSTD_checkContinuity(dctx, dst); - - switch (dctx->stage) { - case ZSTDds_getFrameHeaderSize: - if (srcSize != ZSTD_frameHeaderSize_prefix) - return ERROR(srcSize_wrong); /* impossible */ - if ((ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ - memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix); - dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_prefix; /* magic number + skippable frame length */ - dctx->stage = ZSTDds_decodeSkippableHeader; - return 0; - } - dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_prefix); - if (ZSTD_isError(dctx->headerSize)) - return dctx->headerSize; - memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix); - if (dctx->headerSize > ZSTD_frameHeaderSize_prefix) { - dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_prefix; - dctx->stage = ZSTDds_decodeFrameHeader; - return 0; - } - dctx->expected = 0; /* not necessary to copy more */ - fallthrough; - - case ZSTDds_decodeFrameHeader: - memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected); - CHECK_F(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize)); - dctx->expected = ZSTD_blockHeaderSize; - dctx->stage = ZSTDds_decodeBlockHeader; - return 0; - - case ZSTDds_decodeBlockHeader: { - blockProperties_t bp; - size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); - if (ZSTD_isError(cBlockSize)) - return cBlockSize; - dctx->expected = cBlockSize; - dctx->bType = bp.blockType; - dctx->rleSize = bp.origSize; - if (cBlockSize) { - dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock; - return 0; - } - /* empty block */ - if (bp.lastBlock) { - if (dctx->fParams.checksumFlag) { - dctx->expected = 4; - dctx->stage = ZSTDds_checkChecksum; - } else { - dctx->expected = 0; /* end of frame */ - dctx->stage = ZSTDds_getFrameHeaderSize; - } - } else { - dctx->expected = 3; /* go directly to next header */ - dctx->stage = ZSTDds_decodeBlockHeader; - } - return 0; - } - case ZSTDds_decompressLastBlock: - case ZSTDds_decompressBlock: { - size_t rSize; - switch (dctx->bType) { - case bt_compressed: rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); break; - case bt_raw: rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); break; - case bt_rle: rSize = ZSTD_setRleBlock(dst, dstCapacity, src, srcSize, dctx->rleSize); break; - case bt_reserved: /* should never happen */ - default: return ERROR(corruption_detected); - } - if (ZSTD_isError(rSize)) - return rSize; - if (dctx->fParams.checksumFlag) - xxh64_update(&dctx->xxhState, dst, rSize); - - if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */ - if (dctx->fParams.checksumFlag) { /* another round for frame checksum */ - dctx->expected = 4; - dctx->stage = ZSTDds_checkChecksum; - } else { - dctx->expected = 0; /* ends here */ - dctx->stage = ZSTDds_getFrameHeaderSize; - } - } else { - dctx->stage = ZSTDds_decodeBlockHeader; - dctx->expected = ZSTD_blockHeaderSize; - dctx->previousDstEnd = (char *)dst + rSize; - } - return rSize; - } - case ZSTDds_checkChecksum: { - U32 const h32 = (U32)xxh64_digest(&dctx->xxhState); - U32 const check32 = ZSTD_readLE32(src); /* srcSize == 4, guaranteed by dctx->expected */ - if (check32 != h32) - return ERROR(checksum_wrong); - dctx->expected = 0; - dctx->stage = ZSTDds_getFrameHeaderSize; - return 0; - } - case ZSTDds_decodeSkippableHeader: { - memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected); - dctx->expected = ZSTD_readLE32(dctx->headerBuffer + 4); - dctx->stage = ZSTDds_skipFrame; - return 0; - } - case ZSTDds_skipFrame: { - dctx->expected = 0; - dctx->stage = ZSTDds_getFrameHeaderSize; - return 0; - } - default: - return ERROR(GENERIC); /* impossible */ - } -} - -static size_t ZSTD_refDictContent(ZSTD_DCtx *dctx, const void *dict, size_t dictSize) -{ - dctx->dictEnd = dctx->previousDstEnd; - dctx->vBase = (const char *)dict - ((const char *)(dctx->previousDstEnd) - (const char *)(dctx->base)); - dctx->base = dict; - dctx->previousDstEnd = (const char *)dict + dictSize; - return 0; -} - -/* ZSTD_loadEntropy() : - * dict : must point at beginning of a valid zstd dictionary - * @return : size of entropy tables read */ -static size_t ZSTD_loadEntropy(ZSTD_entropyTables_t *entropy, const void *const dict, size_t const dictSize) -{ - const BYTE *dictPtr = (const BYTE *)dict; - const BYTE *const dictEnd = dictPtr + dictSize; - - if (dictSize <= 8) - return ERROR(dictionary_corrupted); - dictPtr += 8; /* skip header = magic + dictID */ - - { - size_t const hSize = HUF_readDTableX4_wksp(entropy->hufTable, dictPtr, dictEnd - dictPtr, entropy->workspace, sizeof(entropy->workspace)); - if (HUF_isError(hSize)) - return ERROR(dictionary_corrupted); - dictPtr += hSize; - } - - { - short offcodeNCount[MaxOff + 1]; - U32 offcodeMaxValue = MaxOff, offcodeLog; - size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd - dictPtr); - if (FSE_isError(offcodeHeaderSize)) - return ERROR(dictionary_corrupted); - if (offcodeLog > OffFSELog) - return ERROR(dictionary_corrupted); - CHECK_E(FSE_buildDTable_wksp(entropy->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog, entropy->workspace, sizeof(entropy->workspace)), dictionary_corrupted); - dictPtr += offcodeHeaderSize; - } - - { - short matchlengthNCount[MaxML + 1]; - unsigned matchlengthMaxValue = MaxML, matchlengthLog; - size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd - dictPtr); - if (FSE_isError(matchlengthHeaderSize)) - return ERROR(dictionary_corrupted); - if (matchlengthLog > MLFSELog) - return ERROR(dictionary_corrupted); - CHECK_E(FSE_buildDTable_wksp(entropy->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, entropy->workspace, sizeof(entropy->workspace)), dictionary_corrupted); - dictPtr += matchlengthHeaderSize; - } - - { - short litlengthNCount[MaxLL + 1]; - unsigned litlengthMaxValue = MaxLL, litlengthLog; - size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd - dictPtr); - if (FSE_isError(litlengthHeaderSize)) - return ERROR(dictionary_corrupted); - if (litlengthLog > LLFSELog) - return ERROR(dictionary_corrupted); - CHECK_E(FSE_buildDTable_wksp(entropy->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog, entropy->workspace, sizeof(entropy->workspace)), dictionary_corrupted); - dictPtr += litlengthHeaderSize; - } - - if (dictPtr + 12 > dictEnd) - return ERROR(dictionary_corrupted); - { - int i; - size_t const dictContentSize = (size_t)(dictEnd - (dictPtr + 12)); - for (i = 0; i < 3; i++) { - U32 const rep = ZSTD_readLE32(dictPtr); - dictPtr += 4; - if (rep == 0 || rep >= dictContentSize) - return ERROR(dictionary_corrupted); - entropy->rep[i] = rep; - } - } - - return dictPtr - (const BYTE *)dict; -} - -static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx *dctx, const void *dict, size_t dictSize) -{ - if (dictSize < 8) - return ZSTD_refDictContent(dctx, dict, dictSize); - { - U32 const magic = ZSTD_readLE32(dict); - if (magic != ZSTD_DICT_MAGIC) { - return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */ - } - } - dctx->dictID = ZSTD_readLE32((const char *)dict + 4); - - /* load entropy tables */ - { - size_t const eSize = ZSTD_loadEntropy(&dctx->entropy, dict, dictSize); - if (ZSTD_isError(eSize)) - return ERROR(dictionary_corrupted); - dict = (const char *)dict + eSize; - dictSize -= eSize; - } - dctx->litEntropy = dctx->fseEntropy = 1; - - /* reference dictionary content */ - return ZSTD_refDictContent(dctx, dict, dictSize); -} - -size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx *dctx, const void *dict, size_t dictSize) -{ - CHECK_F(ZSTD_decompressBegin(dctx)); - if (dict && dictSize) - CHECK_E(ZSTD_decompress_insertDictionary(dctx, dict, dictSize), dictionary_corrupted); - return 0; -} - -/* ====== ZSTD_DDict ====== */ - -struct ZSTD_DDict_s { - void *dictBuffer; - const void *dictContent; - size_t dictSize; - ZSTD_entropyTables_t entropy; - U32 dictID; - U32 entropyPresent; - ZSTD_customMem cMem; -}; /* typedef'd to ZSTD_DDict within "zstd.h" */ - -size_t ZSTD_DDictWorkspaceBound(void) { return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_DDict)); } - -static const void *ZSTD_DDictDictContent(const ZSTD_DDict *ddict) { return ddict->dictContent; } - -static size_t ZSTD_DDictDictSize(const ZSTD_DDict *ddict) { return ddict->dictSize; } - -static void ZSTD_refDDict(ZSTD_DCtx *dstDCtx, const ZSTD_DDict *ddict) -{ - ZSTD_decompressBegin(dstDCtx); /* init */ - if (ddict) { /* support refDDict on NULL */ - dstDCtx->dictID = ddict->dictID; - dstDCtx->base = ddict->dictContent; - dstDCtx->vBase = ddict->dictContent; - dstDCtx->dictEnd = (const BYTE *)ddict->dictContent + ddict->dictSize; - dstDCtx->previousDstEnd = dstDCtx->dictEnd; - if (ddict->entropyPresent) { - dstDCtx->litEntropy = 1; - dstDCtx->fseEntropy = 1; - dstDCtx->LLTptr = ddict->entropy.LLTable; - dstDCtx->MLTptr = ddict->entropy.MLTable; - dstDCtx->OFTptr = ddict->entropy.OFTable; - dstDCtx->HUFptr = ddict->entropy.hufTable; - dstDCtx->entropy.rep[0] = ddict->entropy.rep[0]; - dstDCtx->entropy.rep[1] = ddict->entropy.rep[1]; - dstDCtx->entropy.rep[2] = ddict->entropy.rep[2]; - } else { - dstDCtx->litEntropy = 0; - dstDCtx->fseEntropy = 0; - } - } -} - -static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict *ddict) -{ - ddict->dictID = 0; - ddict->entropyPresent = 0; - if (ddict->dictSize < 8) - return 0; - { - U32 const magic = ZSTD_readLE32(ddict->dictContent); - if (magic != ZSTD_DICT_MAGIC) - return 0; /* pure content mode */ - } - ddict->dictID = ZSTD_readLE32((const char *)ddict->dictContent + 4); - - /* load entropy tables */ - CHECK_E(ZSTD_loadEntropy(&ddict->entropy, ddict->dictContent, ddict->dictSize), dictionary_corrupted); - ddict->entropyPresent = 1; - return 0; -} - -static ZSTD_DDict *ZSTD_createDDict_advanced(const void *dict, size_t dictSize, unsigned byReference, ZSTD_customMem customMem) -{ - if (!customMem.customAlloc || !customMem.customFree) - return NULL; - - { - ZSTD_DDict *const ddict = (ZSTD_DDict *)ZSTD_malloc(sizeof(ZSTD_DDict), customMem); - if (!ddict) - return NULL; - ddict->cMem = customMem; - - if ((byReference) || (!dict) || (!dictSize)) { - ddict->dictBuffer = NULL; - ddict->dictContent = dict; - } else { - void *const internalBuffer = ZSTD_malloc(dictSize, customMem); - if (!internalBuffer) { - ZSTD_freeDDict(ddict); - return NULL; - } - memcpy(internalBuffer, dict, dictSize); - ddict->dictBuffer = internalBuffer; - ddict->dictContent = internalBuffer; - } - ddict->dictSize = dictSize; - ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ - /* parse dictionary content */ - { - size_t const errorCode = ZSTD_loadEntropy_inDDict(ddict); - if (ZSTD_isError(errorCode)) { - ZSTD_freeDDict(ddict); - return NULL; - } - } - - return ddict; - } -} - -/*! ZSTD_initDDict() : -* Create a digested dictionary, to start decompression without startup delay. -* `dict` content is copied inside DDict. -* Consequently, `dict` can be released after `ZSTD_DDict` creation */ -ZSTD_DDict *ZSTD_initDDict(const void *dict, size_t dictSize, void *workspace, size_t workspaceSize) -{ - ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize); - return ZSTD_createDDict_advanced(dict, dictSize, 1, stackMem); -} - -size_t ZSTD_freeDDict(ZSTD_DDict *ddict) -{ - if (ddict == NULL) - return 0; /* support free on NULL */ - { - ZSTD_customMem const cMem = ddict->cMem; - ZSTD_free(ddict->dictBuffer, cMem); - ZSTD_free(ddict, cMem); - return 0; - } -} - -/*! ZSTD_getDictID_fromDict() : - * Provides the dictID stored within dictionary. - * if @return == 0, the dictionary is not conformant with Zstandard specification. - * It can still be loaded, but as a content-only dictionary. */ -unsigned ZSTD_getDictID_fromDict(const void *dict, size_t dictSize) -{ - if (dictSize < 8) - return 0; - if (ZSTD_readLE32(dict) != ZSTD_DICT_MAGIC) - return 0; - return ZSTD_readLE32((const char *)dict + 4); -} - -/*! ZSTD_getDictID_fromDDict() : - * Provides the dictID of the dictionary loaded into `ddict`. - * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. - * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ -unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict *ddict) -{ - if (ddict == NULL) - return 0; - return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize); -} - -/*! ZSTD_getDictID_fromFrame() : - * Provides the dictID required to decompressed the frame stored within `src`. - * If @return == 0, the dictID could not be decoded. - * This could for one of the following reasons : - * - The frame does not require a dictionary to be decoded (most common case). - * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information. - * Note : this use case also happens when using a non-conformant dictionary. - * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). - * - This is not a Zstandard frame. - * When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */ -unsigned ZSTD_getDictID_fromFrame(const void *src, size_t srcSize) -{ - ZSTD_frameParams zfp = {0, 0, 0, 0}; - size_t const hError = ZSTD_getFrameParams(&zfp, src, srcSize); - if (ZSTD_isError(hError)) - return 0; - return zfp.dictID; -} - -/*! ZSTD_decompress_usingDDict() : -* Decompression using a pre-digested Dictionary -* Use dictionary without significant overhead. */ -size_t ZSTD_decompress_usingDDict(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const ZSTD_DDict *ddict) -{ - /* pass content and size in case legacy frames are encountered */ - return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, NULL, 0, ddict); -} - -/*===================================== -* Streaming decompression -*====================================*/ - -typedef enum { zdss_init, zdss_loadHeader, zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage; - -/* *** Resource management *** */ -struct ZSTD_DStream_s { - ZSTD_DCtx *dctx; - ZSTD_DDict *ddictLocal; - const ZSTD_DDict *ddict; - ZSTD_frameParams fParams; - ZSTD_dStreamStage stage; - char *inBuff; - size_t inBuffSize; - size_t inPos; - size_t maxWindowSize; - char *outBuff; - size_t outBuffSize; - size_t outStart; - size_t outEnd; - size_t blockSize; - BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; /* tmp buffer to store frame header */ - size_t lhSize; - ZSTD_customMem customMem; - void *legacyContext; - U32 previousLegacyVersion; - U32 legacyVersion; - U32 hostageByte; -}; /* typedef'd to ZSTD_DStream within "zstd.h" */ - -size_t ZSTD_DStreamWorkspaceBound(size_t maxWindowSize) -{ - size_t const blockSize = MIN(maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); - size_t const inBuffSize = blockSize; - size_t const outBuffSize = maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2; - return ZSTD_DCtxWorkspaceBound() + ZSTD_ALIGN(sizeof(ZSTD_DStream)) + ZSTD_ALIGN(inBuffSize) + ZSTD_ALIGN(outBuffSize); -} - -static ZSTD_DStream *ZSTD_createDStream_advanced(ZSTD_customMem customMem) -{ - ZSTD_DStream *zds; - - if (!customMem.customAlloc || !customMem.customFree) - return NULL; - - zds = (ZSTD_DStream *)ZSTD_malloc(sizeof(ZSTD_DStream), customMem); - if (zds == NULL) - return NULL; - memset(zds, 0, sizeof(ZSTD_DStream)); - memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem)); - zds->dctx = ZSTD_createDCtx_advanced(customMem); - if (zds->dctx == NULL) { - ZSTD_freeDStream(zds); - return NULL; - } - zds->stage = zdss_init; - zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; - return zds; -} - -ZSTD_DStream *ZSTD_initDStream(size_t maxWindowSize, void *workspace, size_t workspaceSize) -{ - ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize); - ZSTD_DStream *zds = ZSTD_createDStream_advanced(stackMem); - if (!zds) { - return NULL; - } - - zds->maxWindowSize = maxWindowSize; - zds->stage = zdss_loadHeader; - zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; - ZSTD_freeDDict(zds->ddictLocal); - zds->ddictLocal = NULL; - zds->ddict = zds->ddictLocal; - zds->legacyVersion = 0; - zds->hostageByte = 0; - - { - size_t const blockSize = MIN(zds->maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); - size_t const neededOutSize = zds->maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2; - - zds->inBuff = (char *)ZSTD_malloc(blockSize, zds->customMem); - zds->inBuffSize = blockSize; - zds->outBuff = (char *)ZSTD_malloc(neededOutSize, zds->customMem); - zds->outBuffSize = neededOutSize; - if (zds->inBuff == NULL || zds->outBuff == NULL) { - ZSTD_freeDStream(zds); - return NULL; - } - } - return zds; -} - -ZSTD_DStream *ZSTD_initDStream_usingDDict(size_t maxWindowSize, const ZSTD_DDict *ddict, void *workspace, size_t workspaceSize) -{ - ZSTD_DStream *zds = ZSTD_initDStream(maxWindowSize, workspace, workspaceSize); - if (zds) { - zds->ddict = ddict; - } - return zds; -} - -size_t ZSTD_freeDStream(ZSTD_DStream *zds) -{ - if (zds == NULL) - return 0; /* support free on null */ - { - ZSTD_customMem const cMem = zds->customMem; - ZSTD_freeDCtx(zds->dctx); - zds->dctx = NULL; - ZSTD_freeDDict(zds->ddictLocal); - zds->ddictLocal = NULL; - ZSTD_free(zds->inBuff, cMem); - zds->inBuff = NULL; - ZSTD_free(zds->outBuff, cMem); - zds->outBuff = NULL; - ZSTD_free(zds, cMem); - return 0; - } -} - -/* *** Initialization *** */ - -size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize; } -size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } - -size_t ZSTD_resetDStream(ZSTD_DStream *zds) -{ - zds->stage = zdss_loadHeader; - zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; - zds->legacyVersion = 0; - zds->hostageByte = 0; - return ZSTD_frameHeaderSize_prefix; -} - -/* ***** Decompression ***** */ - -ZSTD_STATIC size_t ZSTD_limitCopy(void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - size_t const length = MIN(dstCapacity, srcSize); - memcpy(dst, src, length); - return length; -} - -size_t ZSTD_decompressStream(ZSTD_DStream *zds, ZSTD_outBuffer *output, ZSTD_inBuffer *input) -{ - const char *const istart = (const char *)(input->src) + input->pos; - const char *const iend = (const char *)(input->src) + input->size; - const char *ip = istart; - char *const ostart = (char *)(output->dst) + output->pos; - char *const oend = (char *)(output->dst) + output->size; - char *op = ostart; - U32 someMoreWork = 1; - - while (someMoreWork) { - switch (zds->stage) { - case zdss_init: - ZSTD_resetDStream(zds); /* transparent reset on starting decoding a new frame */ - fallthrough; - - case zdss_loadHeader: { - size_t const hSize = ZSTD_getFrameParams(&zds->fParams, zds->headerBuffer, zds->lhSize); - if (ZSTD_isError(hSize)) - return hSize; - if (hSize != 0) { /* need more input */ - size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */ - if (toLoad > (size_t)(iend - ip)) { /* not enough input to load full header */ - memcpy(zds->headerBuffer + zds->lhSize, ip, iend - ip); - zds->lhSize += iend - ip; - input->pos = input->size; - return (MAX(ZSTD_frameHeaderSize_min, hSize) - zds->lhSize) + - ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ - } - memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); - zds->lhSize = hSize; - ip += toLoad; - break; - } - - /* check for single-pass mode opportunity */ - if (zds->fParams.frameContentSize && zds->fParams.windowSize /* skippable frame if == 0 */ - && (U64)(size_t)(oend - op) >= zds->fParams.frameContentSize) { - size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend - istart); - if (cSize <= (size_t)(iend - istart)) { - size_t const decompressedSize = ZSTD_decompress_usingDDict(zds->dctx, op, oend - op, istart, cSize, zds->ddict); - if (ZSTD_isError(decompressedSize)) - return decompressedSize; - ip = istart + cSize; - op += decompressedSize; - zds->dctx->expected = 0; - zds->stage = zdss_init; - someMoreWork = 0; - break; - } - } - - /* Consume header */ - ZSTD_refDDict(zds->dctx, zds->ddict); - { - size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */ - CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size)); - { - size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); - CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer + h1Size, h2Size)); - } - } - - zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); - if (zds->fParams.windowSize > zds->maxWindowSize) - return ERROR(frameParameter_windowTooLarge); - - /* Buffers are preallocated, but double check */ - { - size_t const blockSize = MIN(zds->maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); - size_t const neededOutSize = zds->maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2; - if (zds->inBuffSize < blockSize) { - return ERROR(GENERIC); - } - if (zds->outBuffSize < neededOutSize) { - return ERROR(GENERIC); - } - zds->blockSize = blockSize; - } - zds->stage = zdss_read; - } - fallthrough; - - case zdss_read: { - size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx); - if (neededInSize == 0) { /* end of frame */ - zds->stage = zdss_init; - someMoreWork = 0; - break; - } - if ((size_t)(iend - ip) >= neededInSize) { /* decode directly from src */ - const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx); - size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, zds->outBuff + zds->outStart, - (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart), ip, neededInSize); - if (ZSTD_isError(decodedSize)) - return decodedSize; - ip += neededInSize; - if (!decodedSize && !isSkipFrame) - break; /* this was just a header */ - zds->outEnd = zds->outStart + decodedSize; - zds->stage = zdss_flush; - break; - } - if (ip == iend) { - someMoreWork = 0; - break; - } /* no more input */ - zds->stage = zdss_load; - /* pass-through */ - } - fallthrough; - - case zdss_load: { - size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx); - size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */ - size_t loadedSize; - if (toLoad > zds->inBuffSize - zds->inPos) - return ERROR(corruption_detected); /* should never happen */ - loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend - ip); - ip += loadedSize; - zds->inPos += loadedSize; - if (loadedSize < toLoad) { - someMoreWork = 0; - break; - } /* not enough input, wait for more */ - - /* decode loaded input */ - { - const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx); - size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart, - zds->inBuff, neededInSize); - if (ZSTD_isError(decodedSize)) - return decodedSize; - zds->inPos = 0; /* input is consumed */ - if (!decodedSize && !isSkipFrame) { - zds->stage = zdss_read; - break; - } /* this was just a header */ - zds->outEnd = zds->outStart + decodedSize; - zds->stage = zdss_flush; - /* pass-through */ - } - } - fallthrough; - - case zdss_flush: { - size_t const toFlushSize = zds->outEnd - zds->outStart; - size_t const flushedSize = ZSTD_limitCopy(op, oend - op, zds->outBuff + zds->outStart, toFlushSize); - op += flushedSize; - zds->outStart += flushedSize; - if (flushedSize == toFlushSize) { /* flush completed */ - zds->stage = zdss_read; - if (zds->outStart + zds->blockSize > zds->outBuffSize) - zds->outStart = zds->outEnd = 0; - break; - } - /* cannot complete flush */ - someMoreWork = 0; - break; - } - default: - return ERROR(GENERIC); /* impossible */ - } - } - - /* result */ - input->pos += (size_t)(ip - istart); - output->pos += (size_t)(op - ostart); - { - size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->dctx); - if (!nextSrcSizeHint) { /* frame fully decoded */ - if (zds->outEnd == zds->outStart) { /* output fully flushed */ - if (zds->hostageByte) { - if (input->pos >= input->size) { - zds->stage = zdss_read; - return 1; - } /* can't release hostage (not present) */ - input->pos++; /* release hostage */ - } - return 0; - } - if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */ - input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */ - zds->hostageByte = 1; - } - return 1; - } - nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->dctx) == ZSTDnit_block); /* preload header of next block */ - if (zds->inPos > nextSrcSizeHint) - return ERROR(GENERIC); /* should never happen */ - nextSrcSizeHint -= zds->inPos; /* already loaded*/ - return nextSrcSizeHint; - } -} - -EXPORT_SYMBOL(ZSTD_DCtxWorkspaceBound); -EXPORT_SYMBOL(ZSTD_initDCtx); -EXPORT_SYMBOL(ZSTD_decompressDCtx); -EXPORT_SYMBOL(ZSTD_decompress_usingDict); - -EXPORT_SYMBOL(ZSTD_DDictWorkspaceBound); -EXPORT_SYMBOL(ZSTD_initDDict); -EXPORT_SYMBOL(ZSTD_decompress_usingDDict); - -EXPORT_SYMBOL(ZSTD_DStreamWorkspaceBound); -EXPORT_SYMBOL(ZSTD_initDStream); -EXPORT_SYMBOL(ZSTD_initDStream_usingDDict); -EXPORT_SYMBOL(ZSTD_resetDStream); -EXPORT_SYMBOL(ZSTD_decompressStream); -EXPORT_SYMBOL(ZSTD_DStreamInSize); -EXPORT_SYMBOL(ZSTD_DStreamOutSize); - -EXPORT_SYMBOL(ZSTD_findFrameCompressedSize); -EXPORT_SYMBOL(ZSTD_getFrameContentSize); -EXPORT_SYMBOL(ZSTD_findDecompressedSize); - -EXPORT_SYMBOL(ZSTD_isFrame); -EXPORT_SYMBOL(ZSTD_getDictID_fromDict); -EXPORT_SYMBOL(ZSTD_getDictID_fromDDict); -EXPORT_SYMBOL(ZSTD_getDictID_fromFrame); - -EXPORT_SYMBOL(ZSTD_getFrameParams); -EXPORT_SYMBOL(ZSTD_decompressBegin); -EXPORT_SYMBOL(ZSTD_decompressBegin_usingDict); -EXPORT_SYMBOL(ZSTD_copyDCtx); -EXPORT_SYMBOL(ZSTD_nextSrcSizeToDecompress); -EXPORT_SYMBOL(ZSTD_decompressContinue); -EXPORT_SYMBOL(ZSTD_nextInputType); - -EXPORT_SYMBOL(ZSTD_decompressBlock); -EXPORT_SYMBOL(ZSTD_insertBlock); - -MODULE_LICENSE("Dual BSD/GPL"); -MODULE_DESCRIPTION("Zstd Decompressor"); diff --git a/lib/zstd/decompress/huf_decompress.c b/lib/zstd/decompress/huf_decompress.c new file mode 100644 index 000000000000..05570ed5f8be --- /dev/null +++ b/lib/zstd/decompress/huf_decompress.c @@ -0,0 +1,1206 @@ +/* ****************************************************************** + * huff0 huffman decoder, + * part of Finite State Entropy library + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* ************************************************************** +* Dependencies +****************************************************************/ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */ +#include "../common/compiler.h" +#include "../common/bitstream.h" /* BIT_* */ +#include "../common/fse.h" /* to compress headers */ +#define HUF_STATIC_LINKING_ONLY +#include "../common/huf.h" +#include "../common/error_private.h" + +/* ************************************************************** +* Macros +****************************************************************/ + +/* These two optional macros force the use one way or another of the two + * Huffman decompression implementations. You can't force in both directions + * at the same time. + */ +#if defined(HUF_FORCE_DECOMPRESS_X1) && \ + defined(HUF_FORCE_DECOMPRESS_X2) +#error "Cannot force the use of the X1 and X2 decoders at the same time!" +#endif + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_isError ERR_isError + + +/* ************************************************************** +* Byte alignment for workSpace management +****************************************************************/ +#define HUF_ALIGN(x, a) HUF_ALIGN_MASK((x), (a) - 1) +#define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask)) + + +/* ************************************************************** +* BMI2 Variant Wrappers +****************************************************************/ +#if DYNAMIC_BMI2 + +#define HUF_DGEN(fn) \ + \ + static size_t fn##_default( \ + void* dst, size_t dstSize, \ + const void* cSrc, size_t cSrcSize, \ + const HUF_DTable* DTable) \ + { \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + \ + static TARGET_ATTRIBUTE("bmi2") size_t fn##_bmi2( \ + void* dst, size_t dstSize, \ + const void* cSrc, size_t cSrcSize, \ + const HUF_DTable* DTable) \ + { \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + \ + static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ + size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \ + { \ + if (bmi2) { \ + return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \ + } + +#else + +#define HUF_DGEN(fn) \ + static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ + size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \ + { \ + (void)bmi2; \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } + +#endif + + +/*-***************************/ +/* generic DTableDesc */ +/*-***************************/ +typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc; + +static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) +{ + DTableDesc dtd; + ZSTD_memcpy(&dtd, table, sizeof(dtd)); + return dtd; +} + + +#ifndef HUF_FORCE_DECOMPRESS_X2 + +/*-***************************/ +/* single-symbol decoding */ +/*-***************************/ +typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX1; /* single-symbol decoding */ + +/* + * Packs 4 HUF_DEltX1 structs into a U64. This is used to lay down 4 entries at + * a time. + */ +static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) { + U64 D4; + if (MEM_isLittleEndian()) { + D4 = symbol + (nbBits << 8); + } else { + D4 = (symbol << 8) + nbBits; + } + D4 *= 0x0001000100010001ULL; + return D4; +} + +typedef struct { + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; + U32 rankStart[HUF_TABLELOG_ABSOLUTEMAX + 1]; + U32 statsWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; + BYTE symbols[HUF_SYMBOLVALUE_MAX + 1]; + BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; +} HUF_ReadDTableX1_Workspace; + + +size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize) +{ + return HUF_readDTableX1_wksp_bmi2(DTable, src, srcSize, workSpace, wkspSize, /* bmi2 */ 0); +} + +size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2) +{ + U32 tableLog = 0; + U32 nbSymbols = 0; + size_t iSize; + void* const dtPtr = DTable + 1; + HUF_DEltX1* const dt = (HUF_DEltX1*)dtPtr; + HUF_ReadDTableX1_Workspace* wksp = (HUF_ReadDTableX1_Workspace*)workSpace; + + DEBUG_STATIC_ASSERT(HUF_DECOMPRESS_WORKSPACE_SIZE >= sizeof(*wksp)); + if (sizeof(*wksp) > wkspSize) return ERROR(tableLog_tooLarge); + + DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); + /* ZSTD_memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), bmi2); + if (HUF_isError(iSize)) return iSize; + + /* Table header */ + { DTableDesc dtd = HUF_getDTableDesc(DTable); + if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */ + dtd.tableType = 0; + dtd.tableLog = (BYTE)tableLog; + ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); + } + + /* Compute symbols and rankStart given rankVal: + * + * rankVal already contains the number of values of each weight. + * + * symbols contains the symbols ordered by weight. First are the rankVal[0] + * weight 0 symbols, followed by the rankVal[1] weight 1 symbols, and so on. + * symbols[0] is filled (but unused) to avoid a branch. + * + * rankStart contains the offset where each rank belongs in the DTable. + * rankStart[0] is not filled because there are no entries in the table for + * weight 0. + */ + { + int n; + int nextRankStart = 0; + int const unroll = 4; + int const nLimit = (int)nbSymbols - unroll + 1; + for (n=0; n<(int)tableLog+1; n++) { + U32 const curr = nextRankStart; + nextRankStart += wksp->rankVal[n]; + wksp->rankStart[n] = curr; + } + for (n=0; n < nLimit; n += unroll) { + int u; + for (u=0; u < unroll; ++u) { + size_t const w = wksp->huffWeight[n+u]; + wksp->symbols[wksp->rankStart[w]++] = (BYTE)(n+u); + } + } + for (; n < (int)nbSymbols; ++n) { + size_t const w = wksp->huffWeight[n]; + wksp->symbols[wksp->rankStart[w]++] = (BYTE)n; + } + } + + /* fill DTable + * We fill all entries of each weight in order. + * That way length is a constant for each iteration of the outter loop. + * We can switch based on the length to a different inner loop which is + * optimized for that particular case. + */ + { + U32 w; + int symbol=wksp->rankVal[0]; + int rankStart=0; + for (w=1; wrankVal[w]; + int const length = (1 << w) >> 1; + int uStart = rankStart; + BYTE const nbBits = (BYTE)(tableLog + 1 - w); + int s; + int u; + switch (length) { + case 1: + for (s=0; ssymbols[symbol + s]; + D.nbBits = nbBits; + dt[uStart] = D; + uStart += 1; + } + break; + case 2: + for (s=0; ssymbols[symbol + s]; + D.nbBits = nbBits; + dt[uStart+0] = D; + dt[uStart+1] = D; + uStart += 2; + } + break; + case 4: + for (s=0; ssymbols[symbol + s], nbBits); + MEM_write64(dt + uStart, D4); + uStart += 4; + } + break; + case 8: + for (s=0; ssymbols[symbol + s], nbBits); + MEM_write64(dt + uStart, D4); + MEM_write64(dt + uStart + 4, D4); + uStart += 8; + } + break; + default: + for (s=0; ssymbols[symbol + s], nbBits); + for (u=0; u < length; u += 16) { + MEM_write64(dt + uStart + u + 0, D4); + MEM_write64(dt + uStart + u + 4, D4); + MEM_write64(dt + uStart + u + 8, D4); + MEM_write64(dt + uStart + u + 12, D4); + } + assert(u == length); + uStart += length; + } + break; + } + symbol += symbolCount; + rankStart += symbolCount * length; + } + } + return iSize; +} + +FORCE_INLINE_TEMPLATE BYTE +HUF_decodeSymbolX1(BIT_DStream_t* Dstream, const HUF_DEltX1* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ + BYTE const c = dt[val].byte; + BIT_skipBits(Dstream, dt[val].nbBits); + return c; +} + +#define HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) \ + *ptr++ = HUF_decodeSymbolX1(DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX1_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) + +#define HUF_DECODE_SYMBOLX1_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) + +HINT_INLINE size_t +HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX1* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 4 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-3)) { + HUF_DECODE_SYMBOLX1_2(p, bitDPtr); + HUF_DECODE_SYMBOLX1_1(p, bitDPtr); + HUF_DECODE_SYMBOLX1_2(p, bitDPtr); + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + } + + /* [0-3] symbols remaining */ + if (MEM_32bits()) + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd)) + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + + /* no more data to retrieve from bitstream, no need to reload */ + while (p < pEnd) + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + + return pEnd-pStart; +} + +FORCE_INLINE_TEMPLATE size_t +HUF_decompress1X1_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + dstSize; + const void* dtPtr = DTable + 1; + const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; + BIT_DStream_t bitD; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); + + HUF_decodeStreamX1(op, &bitD, oend, dt, dtLog); + + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + return dstSize; +} + +FORCE_INLINE_TEMPLATE size_t +HUF_decompress4X1_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + /* Check */ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* const olimit = oend - 3; + const void* const dtPtr = DTable + 1; + const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + const size_t segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + U32 endSignal = 1; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); + CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); + CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); + CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); + + /* up to 16 symbols per loop (4 symbols per stream) in 64-bit mode */ + for ( ; (endSignal) & (op4 < olimit) ; ) { + HUF_DECODE_SYMBOLX1_2(op1, &bitD1); + HUF_DECODE_SYMBOLX1_2(op2, &bitD2); + HUF_DECODE_SYMBOLX1_2(op3, &bitD3); + HUF_DECODE_SYMBOLX1_2(op4, &bitD4); + HUF_DECODE_SYMBOLX1_1(op1, &bitD1); + HUF_DECODE_SYMBOLX1_1(op2, &bitD2); + HUF_DECODE_SYMBOLX1_1(op3, &bitD3); + HUF_DECODE_SYMBOLX1_1(op4, &bitD4); + HUF_DECODE_SYMBOLX1_2(op1, &bitD1); + HUF_DECODE_SYMBOLX1_2(op2, &bitD2); + HUF_DECODE_SYMBOLX1_2(op3, &bitD3); + HUF_DECODE_SYMBOLX1_2(op4, &bitD4); + HUF_DECODE_SYMBOLX1_0(op1, &bitD1); + HUF_DECODE_SYMBOLX1_0(op2, &bitD2); + HUF_DECODE_SYMBOLX1_0(op3, &bitD3); + HUF_DECODE_SYMBOLX1_0(op4, &bitD4); + endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; + } + + /* check corruption */ + /* note : should not be necessary : op# advance in lock step, and we control op4. + * but curiously, binary generated by gcc 7.2 & 7.3 with -mbmi2 runs faster when >=1 test is present */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 supposed already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX1(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX1(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX1(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX1(op4, &bitD4, oend, dt, dtLog); + + /* check */ + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } + + /* decoded size */ + return dstSize; + } +} + + +typedef size_t (*HUF_decompress_usingDTable_t)(void *dst, size_t dstSize, + const void *cSrc, + size_t cSrcSize, + const HUF_DTable *DTable); + +HUF_DGEN(HUF_decompress1X1_usingDTable_internal) +HUF_DGEN(HUF_decompress4X1_usingDTable_internal) + + + +size_t HUF_decompress1X1_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 0) return ERROR(GENERIC); + return HUF_decompress1X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +} + +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX1_wksp(DCtx, cSrc, cSrcSize, workSpace, wkspSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0); +} + + +size_t HUF_decompress4X1_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 0) return ERROR(GENERIC); + return HUF_decompress4X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +} + +static size_t HUF_decompress4X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int bmi2) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); +} + +size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize) +{ + return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, 0); +} + + +#endif /* HUF_FORCE_DECOMPRESS_X2 */ + + +#ifndef HUF_FORCE_DECOMPRESS_X1 + +/* *************************/ +/* double-symbols decoding */ +/* *************************/ + +typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX2; /* double-symbols decoding */ +typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; +typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1]; +typedef rankValCol_t rankVal_t[HUF_TABLELOG_MAX]; + + +/* HUF_fillDTableX2Level2() : + * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */ +static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 sizeLog, const U32 consumed, + const U32* rankValOrigin, const int minWeight, + const sortedSymbol_t* sortedSymbols, const U32 sortedListSize, + U32 nbBitsBaseline, U16 baseSeq, U32* wksp, size_t wkspSize) +{ + HUF_DEltX2 DElt; + U32* rankVal = wksp; + + assert(wkspSize >= HUF_TABLELOG_MAX + 1); + (void)wkspSize; + /* get pre-calculated rankVal */ + ZSTD_memcpy(rankVal, rankValOrigin, sizeof(U32) * (HUF_TABLELOG_MAX + 1)); + + /* fill skipped values */ + if (minWeight>1) { + U32 i, skipSize = rankVal[minWeight]; + MEM_writeLE16(&(DElt.sequence), baseSeq); + DElt.nbBits = (BYTE)(consumed); + DElt.length = 1; + for (i = 0; i < skipSize; i++) + DTable[i] = DElt; + } + + /* fill DTable */ + { U32 s; for (s=0; s= 1 */ + + rankVal[weight] += length; + } } +} + + +static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog, + const sortedSymbol_t* sortedList, const U32 sortedListSize, + const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight, + const U32 nbBitsBaseline, U32* wksp, size_t wkspSize) +{ + U32* rankVal = wksp; + const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ + const U32 minBits = nbBitsBaseline - maxWeight; + U32 s; + + assert(wkspSize >= HUF_TABLELOG_MAX + 1); + wksp += HUF_TABLELOG_MAX + 1; + wkspSize -= HUF_TABLELOG_MAX + 1; + + ZSTD_memcpy(rankVal, rankValOrigin, sizeof(U32) * (HUF_TABLELOG_MAX + 1)); + + /* fill DTable */ + for (s=0; s= minBits) { /* enough room for a second symbol */ + U32 sortedRank; + int minWeight = nbBits + scaleLog; + if (minWeight < 1) minWeight = 1; + sortedRank = rankStart[minWeight]; + HUF_fillDTableX2Level2(DTable+start, targetLog-nbBits, nbBits, + rankValOrigin[nbBits], minWeight, + sortedList+sortedRank, sortedListSize-sortedRank, + nbBitsBaseline, symbol, wksp, wkspSize); + } else { + HUF_DEltX2 DElt; + MEM_writeLE16(&(DElt.sequence), symbol); + DElt.nbBits = (BYTE)(nbBits); + DElt.length = 1; + { U32 const end = start + length; + U32 u; + for (u = start; u < end; u++) DTable[u] = DElt; + } } + rankVal[weight] += length; + } +} + +typedef struct { + rankValCol_t rankVal[HUF_TABLELOG_MAX]; + U32 rankStats[HUF_TABLELOG_MAX + 1]; + U32 rankStart0[HUF_TABLELOG_MAX + 2]; + sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1]; + BYTE weightList[HUF_SYMBOLVALUE_MAX + 1]; + U32 calleeWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; +} HUF_ReadDTableX2_Workspace; + +size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize) +{ + U32 tableLog, maxW, sizeOfSort, nbSymbols; + DTableDesc dtd = HUF_getDTableDesc(DTable); + U32 const maxTableLog = dtd.maxTableLog; + size_t iSize; + void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ + HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; + U32 *rankStart; + + HUF_ReadDTableX2_Workspace* const wksp = (HUF_ReadDTableX2_Workspace*)workSpace; + + if (sizeof(*wksp) > wkspSize) return ERROR(GENERIC); + + rankStart = wksp->rankStart0 + 1; + ZSTD_memset(wksp->rankStats, 0, sizeof(wksp->rankStats)); + ZSTD_memset(wksp->rankStart0, 0, sizeof(wksp->rankStart0)); + + DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */ + if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + /* ZSTD_memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), /* bmi2 */ 0); + if (HUF_isError(iSize)) return iSize; + + /* check result */ + if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ + + /* find maxWeight */ + for (maxW = tableLog; wksp->rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ + + /* Get start index of each weight */ + { U32 w, nextRankStart = 0; + for (w=1; wrankStats[w]; + rankStart[w] = curr; + } + rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ + sizeOfSort = nextRankStart; + } + + /* sort symbols by weight */ + { U32 s; + for (s=0; sweightList[s]; + U32 const r = rankStart[w]++; + wksp->sortedSymbol[r].symbol = (BYTE)s; + wksp->sortedSymbol[r].weight = (BYTE)w; + } + rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ + } + + /* Build rankVal */ + { U32* const rankVal0 = wksp->rankVal[0]; + { int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */ + U32 nextRankVal = 0; + U32 w; + for (w=1; wrankStats[w] << (w+rescale); + rankVal0[w] = curr; + } } + { U32 const minBits = tableLog+1 - maxW; + U32 consumed; + for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) { + U32* const rankValPtr = wksp->rankVal[consumed]; + U32 w; + for (w = 1; w < maxW+1; w++) { + rankValPtr[w] = rankVal0[w] >> consumed; + } } } } + + HUF_fillDTableX2(dt, maxTableLog, + wksp->sortedSymbol, sizeOfSort, + wksp->rankStart0, wksp->rankVal, maxW, + tableLog+1, + wksp->calleeWksp, sizeof(wksp->calleeWksp) / sizeof(U32)); + + dtd.tableLog = (BYTE)maxTableLog; + dtd.tableType = 1; + ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); + return iSize; +} + + +FORCE_INLINE_TEMPLATE U32 +HUF_decodeSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + ZSTD_memcpy(op, dt+val, 2); + BIT_skipBits(DStream, dt[val].nbBits); + return dt[val].length; +} + +FORCE_INLINE_TEMPLATE U32 +HUF_decodeLastSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + ZSTD_memcpy(op, dt+val, 1); + if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits); + else { + if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { + BIT_skipBits(DStream, dt[val].nbBits); + if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) + /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ + DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); + } } + return 1; +} + +#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) + +HINT_INLINE size_t +HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, + const HUF_DEltX2* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 8 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) { + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_1(p, bitDPtr); + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + + /* closer to end : up to 2 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2)) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + + while (p <= pEnd-2) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ + + if (p < pEnd) + p += HUF_decodeLastSymbolX2(p, bitDPtr, dt, dtLog); + + return p-pStart; +} + +FORCE_INLINE_TEMPLATE size_t +HUF_decompress1X2_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BIT_DStream_t bitD; + + /* Init */ + CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); + + /* decode */ + { BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + HUF_decodeStreamX2(ostart, &bitD, oend, dt, dtd.tableLog); + } + + /* check */ + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + /* decoded size */ + return dstSize; +} + +FORCE_INLINE_TEMPLATE size_t +HUF_decompress4X2_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* const olimit = oend - (sizeof(size_t)-1); + const void* const dtPtr = DTable+1; + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + size_t const segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal = 1; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); + CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); + CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); + CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); + + /* 16-32 symbols per loop (4-8 symbols per stream) */ + for ( ; (endSignal) & (op4 < olimit); ) { +#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__)) + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; +#else + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal = (U32)LIKELY( + (BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished)); +#endif + } + + /* check corruption */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); + + /* check */ + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } + + /* decoded size */ + return dstSize; + } +} + +HUF_DGEN(HUF_decompress1X2_usingDTable_internal) +HUF_DGEN(HUF_decompress4X2_usingDTable_internal) + +size_t HUF_decompress1X2_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 1) return ERROR(GENERIC); + return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +} + +size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, + workSpace, wkspSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0); +} + + +size_t HUF_decompress4X2_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 1) return ERROR(GENERIC); + return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +} + +static size_t HUF_decompress4X2_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int bmi2) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize, + workSpace, wkspSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); +} + +size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize) +{ + return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, /* bmi2 */ 0); +} + + +#endif /* HUF_FORCE_DECOMPRESS_X1 */ + + +/* ***********************************/ +/* Universal decompression selectors */ +/* ***********************************/ + +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#else + return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) : + HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#endif +} + +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#else + return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) : + HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#endif +} + + +#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) +typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; +static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = +{ + /* single, double, quad */ + {{0,0}, {1,1}, {2,2}}, /* Q==0 : impossible */ + {{0,0}, {1,1}, {2,2}}, /* Q==1 : impossible */ + {{ 38,130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */ + {{ 448,128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */ + {{ 556,128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */ + {{ 714,128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */ + {{ 883,128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */ + {{ 897,128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */ + {{ 926,128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */ + {{ 947,128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */ + {{1107,128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */ + {{1177,128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */ + {{1242,128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */ + {{1349,128}, {2644,106}, {5260,106}}, /* Q ==13 : 81-87% */ + {{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */ + {{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */ +}; +#endif + +/* HUF_selectDecoder() : + * Tells which decoder is likely to decode faster, + * based on a set of pre-computed metrics. + * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 . + * Assumption : 0 < dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize) +{ + assert(dstSize > 0); + assert(dstSize <= 128*1024); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dstSize; + (void)cSrcSize; + return 0; +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dstSize; + (void)cSrcSize; + return 1; +#else + /* decoder timing evaluation */ + { U32 const Q = (cSrcSize >= dstSize) ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */ + U32 const D256 = (U32)(dstSize >> 8); + U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); + U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); + DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, to reduce cache eviction */ + return DTime1 < DTime0; + } +#endif +} + + +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, + size_t dstSize, const void* cSrc, + size_t cSrcSize, void* workSpace, + size_t wkspSize) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize == 0) return ERROR(corruption_detected); + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); +#else + return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize): + HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); +#endif + } +} + +size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize); +#else + return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize): + HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize); +#endif + } +} + + +size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#else + return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) : + HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#endif +} + +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); +} +#endif + +size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#else + return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) : + HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#endif +} + +size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize == 0) return ERROR(corruption_detected); + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); +#else + return algoNb ? HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2) : + HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); +#endif + } +} + diff --git a/lib/zstd/decompress/zstd_ddict.c b/lib/zstd/decompress/zstd_ddict.c new file mode 100644 index 000000000000..dbbc7919de53 --- /dev/null +++ b/lib/zstd/decompress/zstd_ddict.c @@ -0,0 +1,241 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* zstd_ddict.c : + * concentrates all logic that needs to know the internals of ZSTD_DDict object */ + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ +#include "../common/cpu.h" /* bmi2 */ +#include "../common/mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "../common/fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "../common/huf.h" +#include "zstd_decompress_internal.h" +#include "zstd_ddict.h" + + + + +/*-******************************************************* +* Types +*********************************************************/ +struct ZSTD_DDict_s { + void* dictBuffer; + const void* dictContent; + size_t dictSize; + ZSTD_entropyDTables_t entropy; + U32 dictID; + U32 entropyPresent; + ZSTD_customMem cMem; +}; /* typedef'd to ZSTD_DDict within "zstd.h" */ + +const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict) +{ + assert(ddict != NULL); + return ddict->dictContent; +} + +size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict) +{ + assert(ddict != NULL); + return ddict->dictSize; +} + +void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + DEBUGLOG(4, "ZSTD_copyDDictParameters"); + assert(dctx != NULL); + assert(ddict != NULL); + dctx->dictID = ddict->dictID; + dctx->prefixStart = ddict->dictContent; + dctx->virtualStart = ddict->dictContent; + dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize; + dctx->previousDstEnd = dctx->dictEnd; +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + dctx->dictContentBeginForFuzzing = dctx->prefixStart; + dctx->dictContentEndForFuzzing = dctx->previousDstEnd; +#endif + if (ddict->entropyPresent) { + dctx->litEntropy = 1; + dctx->fseEntropy = 1; + dctx->LLTptr = ddict->entropy.LLTable; + dctx->MLTptr = ddict->entropy.MLTable; + dctx->OFTptr = ddict->entropy.OFTable; + dctx->HUFptr = ddict->entropy.hufTable; + dctx->entropy.rep[0] = ddict->entropy.rep[0]; + dctx->entropy.rep[1] = ddict->entropy.rep[1]; + dctx->entropy.rep[2] = ddict->entropy.rep[2]; + } else { + dctx->litEntropy = 0; + dctx->fseEntropy = 0; + } +} + + +static size_t +ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict, + ZSTD_dictContentType_e dictContentType) +{ + ddict->dictID = 0; + ddict->entropyPresent = 0; + if (dictContentType == ZSTD_dct_rawContent) return 0; + + if (ddict->dictSize < 8) { + if (dictContentType == ZSTD_dct_fullDict) + return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ + return 0; /* pure content mode */ + } + { U32 const magic = MEM_readLE32(ddict->dictContent); + if (magic != ZSTD_MAGIC_DICTIONARY) { + if (dictContentType == ZSTD_dct_fullDict) + return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ + return 0; /* pure content mode */ + } + } + ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE); + + /* load entropy tables */ + RETURN_ERROR_IF(ZSTD_isError(ZSTD_loadDEntropy( + &ddict->entropy, ddict->dictContent, ddict->dictSize)), + dictionary_corrupted, ""); + ddict->entropyPresent = 1; + return 0; +} + + +static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) { + ddict->dictBuffer = NULL; + ddict->dictContent = dict; + if (!dict) dictSize = 0; + } else { + void* const internalBuffer = ZSTD_customMalloc(dictSize, ddict->cMem); + ddict->dictBuffer = internalBuffer; + ddict->dictContent = internalBuffer; + if (!internalBuffer) return ERROR(memory_allocation); + ZSTD_memcpy(internalBuffer, dict, dictSize); + } + ddict->dictSize = dictSize; + ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ + + /* parse dictionary content */ + FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , ""); + + return 0; +} + +ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_customMem customMem) +{ + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + + { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_customMalloc(sizeof(ZSTD_DDict), customMem); + if (ddict == NULL) return NULL; + ddict->cMem = customMem; + { size_t const initResult = ZSTD_initDDict_internal(ddict, + dict, dictSize, + dictLoadMethod, dictContentType); + if (ZSTD_isError(initResult)) { + ZSTD_freeDDict(ddict); + return NULL; + } } + return ddict; + } +} + +/*! ZSTD_createDDict() : +* Create a digested dictionary, to start decompression without startup delay. +* `dict` content is copied inside DDict. +* Consequently, `dict` can be released after `ZSTD_DDict` creation */ +ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator); +} + +/*! ZSTD_createDDict_byReference() : + * Create a digested dictionary, to start decompression without startup delay. + * Dictionary content is simply referenced, it will be accessed during decompression. + * Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */ +ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator); +} + + +const ZSTD_DDict* ZSTD_initStaticDDict( + void* sBuffer, size_t sBufferSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + size_t const neededSpace = sizeof(ZSTD_DDict) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); + ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer; + assert(sBuffer != NULL); + assert(dict != NULL); + if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */ + if (sBufferSize < neededSpace) return NULL; + if (dictLoadMethod == ZSTD_dlm_byCopy) { + ZSTD_memcpy(ddict+1, dict, dictSize); /* local copy */ + dict = ddict+1; + } + if (ZSTD_isError( ZSTD_initDDict_internal(ddict, + dict, dictSize, + ZSTD_dlm_byRef, dictContentType) )) + return NULL; + return ddict; +} + + +size_t ZSTD_freeDDict(ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support free on NULL */ + { ZSTD_customMem const cMem = ddict->cMem; + ZSTD_customFree(ddict->dictBuffer, cMem); + ZSTD_customFree(ddict, cMem); + return 0; + } +} + +/*! ZSTD_estimateDDictSize() : + * Estimate amount of memory that will be needed to create a dictionary for decompression. + * Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */ +size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod) +{ + return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); +} + +size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support sizeof on NULL */ + return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ; +} + +/*! ZSTD_getDictID_fromDDict() : + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; + return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize); +} diff --git a/lib/zstd/decompress/zstd_ddict.h b/lib/zstd/decompress/zstd_ddict.h new file mode 100644 index 000000000000..8c1a79d666f8 --- /dev/null +++ b/lib/zstd/decompress/zstd_ddict.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +#ifndef ZSTD_DDICT_H +#define ZSTD_DDICT_H + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include "../common/zstd_deps.h" /* size_t */ +#include /* ZSTD_DDict, and several public functions */ + + +/*-******************************************************* + * Interface + *********************************************************/ + +/* note: several prototypes are already published in `zstd.h` : + * ZSTD_createDDict() + * ZSTD_createDDict_byReference() + * ZSTD_createDDict_advanced() + * ZSTD_freeDDict() + * ZSTD_initStaticDDict() + * ZSTD_sizeof_DDict() + * ZSTD_estimateDDictSize() + * ZSTD_getDictID_fromDict() + */ + +const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict); +size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict); + +void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + + + +#endif /* ZSTD_DDICT_H */ diff --git a/lib/zstd/decompress/zstd_decompress.c b/lib/zstd/decompress/zstd_decompress.c new file mode 100644 index 000000000000..099f708acd16 --- /dev/null +++ b/lib/zstd/decompress/zstd_decompress.c @@ -0,0 +1,2075 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* *************************************************************** +* Tuning parameters +*****************************************************************/ +/*! + * HEAPMODE : + * Select how default decompression function ZSTD_decompress() allocates its context, + * on stack (0), or into heap (1, default; requires malloc()). + * Note that functions with explicit context such as ZSTD_decompressDCtx() are unaffected. + */ +#ifndef ZSTD_HEAPMODE +# define ZSTD_HEAPMODE 1 +#endif + +/*! +* LEGACY_SUPPORT : +* if set to 1+, ZSTD_decompress() can decode older formats (v0.1+) +*/ + +/*! + * MAXWINDOWSIZE_DEFAULT : + * maximum window size accepted by DStream __by default__. + * Frames requiring more memory will be rejected. + * It's possible to set a different limit using ZSTD_DCtx_setMaxWindowSize(). + */ +#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT +# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + 1) +#endif + +/*! + * NO_FORWARD_PROGRESS_MAX : + * maximum allowed nb of calls to ZSTD_decompressStream() + * without any forward progress + * (defined as: no byte read from input, and no byte flushed to output) + * before triggering an error. + */ +#ifndef ZSTD_NO_FORWARD_PROGRESS_MAX +# define ZSTD_NO_FORWARD_PROGRESS_MAX 16 +#endif + + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ +#include "../common/cpu.h" /* bmi2 */ +#include "../common/mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "../common/fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "../common/huf.h" +#include /* xxh64_reset, xxh64_update, xxh64_digest, XXH64 */ +#include "../common/zstd_internal.h" /* blockProperties_t */ +#include "zstd_decompress_internal.h" /* ZSTD_DCtx */ +#include "zstd_ddict.h" /* ZSTD_DDictDictContent */ +#include "zstd_decompress_block.h" /* ZSTD_decompressBlock_internal */ + + + + +/* *********************************** + * Multiple DDicts Hashset internals * + *************************************/ + +#define DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT 4 +#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3 /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float. + * Currently, that means a 0.75 load factor. + * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded + * the load factor of the ddict hash set. + */ + +#define DDICT_HASHSET_TABLE_BASE_SIZE 64 +#define DDICT_HASHSET_RESIZE_FACTOR 2 + +/* Hash function to determine starting position of dict insertion within the table + * Returns an index between [0, hashSet->ddictPtrTableSize] + */ +static size_t ZSTD_DDictHashSet_getIndex(const ZSTD_DDictHashSet* hashSet, U32 dictID) { + const U64 hash = xxh64(&dictID, sizeof(U32), 0); + /* DDict ptr table size is a multiple of 2, use size - 1 as mask to get index within [0, hashSet->ddictPtrTableSize) */ + return hash & (hashSet->ddictPtrTableSize - 1); +} + +/* Adds DDict to a hashset without resizing it. + * If inserting a DDict with a dictID that already exists in the set, replaces the one in the set. + * Returns 0 if successful, or a zstd error code if something went wrong. + */ +static size_t ZSTD_DDictHashSet_emplaceDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict) { + const U32 dictID = ZSTD_getDictID_fromDDict(ddict); + size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID); + const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1; + RETURN_ERROR_IF(hashSet->ddictPtrCount == hashSet->ddictPtrTableSize, GENERIC, "Hash set is full!"); + DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx); + while (hashSet->ddictPtrTable[idx] != NULL) { + /* Replace existing ddict if inserting ddict with same dictID */ + if (ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]) == dictID) { + DEBUGLOG(4, "DictID already exists, replacing rather than adding"); + hashSet->ddictPtrTable[idx] = ddict; + return 0; + } + idx &= idxRangeMask; + idx++; + } + DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx); + hashSet->ddictPtrTable[idx] = ddict; + hashSet->ddictPtrCount++; + return 0; +} + +/* Expands hash table by factor of DDICT_HASHSET_RESIZE_FACTOR and + * rehashes all values, allocates new table, frees old table. + * Returns 0 on success, otherwise a zstd error code. + */ +static size_t ZSTD_DDictHashSet_expand(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) { + size_t newTableSize = hashSet->ddictPtrTableSize * DDICT_HASHSET_RESIZE_FACTOR; + const ZSTD_DDict** newTable = (const ZSTD_DDict**)ZSTD_customCalloc(sizeof(ZSTD_DDict*) * newTableSize, customMem); + const ZSTD_DDict** oldTable = hashSet->ddictPtrTable; + size_t oldTableSize = hashSet->ddictPtrTableSize; + size_t i; + + DEBUGLOG(4, "Expanding DDict hash table! Old size: %zu new size: %zu", oldTableSize, newTableSize); + RETURN_ERROR_IF(!newTable, memory_allocation, "Expanded hashset allocation failed!"); + hashSet->ddictPtrTable = newTable; + hashSet->ddictPtrTableSize = newTableSize; + hashSet->ddictPtrCount = 0; + for (i = 0; i < oldTableSize; ++i) { + if (oldTable[i] != NULL) { + FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, oldTable[i]), ""); + } + } + ZSTD_customFree((void*)oldTable, customMem); + DEBUGLOG(4, "Finished re-hash"); + return 0; +} + +/* Fetches a DDict with the given dictID + * Returns the ZSTD_DDict* with the requested dictID. If it doesn't exist, then returns NULL. + */ +static const ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet, U32 dictID) { + size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID); + const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1; + DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx); + for (;;) { + size_t currDictID = ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]); + if (currDictID == dictID || currDictID == 0) { + /* currDictID == 0 implies a NULL ddict entry */ + break; + } else { + idx &= idxRangeMask; /* Goes to start of table when we reach the end */ + idx++; + } + } + DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx); + return hashSet->ddictPtrTable[idx]; +} + +/* Allocates space for and returns a ddict hash set + * The hash set's ZSTD_DDict* table has all values automatically set to NULL to begin with. + * Returns NULL if allocation failed. + */ +static ZSTD_DDictHashSet* ZSTD_createDDictHashSet(ZSTD_customMem customMem) { + ZSTD_DDictHashSet* ret = (ZSTD_DDictHashSet*)ZSTD_customMalloc(sizeof(ZSTD_DDictHashSet), customMem); + DEBUGLOG(4, "Allocating new hash set"); + ret->ddictPtrTable = (const ZSTD_DDict**)ZSTD_customCalloc(DDICT_HASHSET_TABLE_BASE_SIZE * sizeof(ZSTD_DDict*), customMem); + ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE; + ret->ddictPtrCount = 0; + if (!ret || !ret->ddictPtrTable) { + return NULL; + } + return ret; +} + +/* Frees the table of ZSTD_DDict* within a hashset, then frees the hashset itself. + * Note: The ZSTD_DDict* within the table are NOT freed. + */ +static void ZSTD_freeDDictHashSet(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) { + DEBUGLOG(4, "Freeing ddict hash set"); + if (hashSet && hashSet->ddictPtrTable) { + ZSTD_customFree((void*)hashSet->ddictPtrTable, customMem); + } + if (hashSet) { + ZSTD_customFree(hashSet, customMem); + } +} + +/* Public function: Adds a DDict into the ZSTD_DDictHashSet, possibly triggering a resize of the hash set. + * Returns 0 on success, or a ZSTD error. + */ +static size_t ZSTD_DDictHashSet_addDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict, ZSTD_customMem customMem) { + DEBUGLOG(4, "Adding dict ID: %u to hashset with - Count: %zu Tablesize: %zu", ZSTD_getDictID_fromDDict(ddict), hashSet->ddictPtrCount, hashSet->ddictPtrTableSize); + if (hashSet->ddictPtrCount * DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT / hashSet->ddictPtrTableSize * DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT != 0) { + FORWARD_IF_ERROR(ZSTD_DDictHashSet_expand(hashSet, customMem), ""); + } + FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, ddict), ""); + return 0; +} + +/*-************************************************************* +* Context management +***************************************************************/ +size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) +{ + if (dctx==NULL) return 0; /* support sizeof NULL */ + return sizeof(*dctx) + + ZSTD_sizeof_DDict(dctx->ddictLocal) + + dctx->inBuffSize + dctx->outBuffSize; +} + +size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); } + + +static size_t ZSTD_startingInputLength(ZSTD_format_e format) +{ + size_t const startingInputLength = ZSTD_FRAMEHEADERSIZE_PREFIX(format); + /* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */ + assert( (format == ZSTD_f_zstd1) || (format == ZSTD_f_zstd1_magicless) ); + return startingInputLength; +} + +static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx) +{ + assert(dctx->streamStage == zdss_init); + dctx->format = ZSTD_f_zstd1; + dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; + dctx->outBufferMode = ZSTD_bm_buffered; + dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum; + dctx->refMultipleDDicts = ZSTD_rmd_refSingleDDict; +} + +static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) +{ + dctx->staticSize = 0; + dctx->ddict = NULL; + dctx->ddictLocal = NULL; + dctx->dictEnd = NULL; + dctx->ddictIsCold = 0; + dctx->dictUses = ZSTD_dont_use; + dctx->inBuff = NULL; + dctx->inBuffSize = 0; + dctx->outBuffSize = 0; + dctx->streamStage = zdss_init; + dctx->legacyContext = NULL; + dctx->previousLegacyVersion = 0; + dctx->noForwardProgress = 0; + dctx->oversizedDuration = 0; + dctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); + dctx->ddictSet = NULL; + ZSTD_DCtx_resetParameters(dctx); +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + dctx->dictContentEndForFuzzing = NULL; +#endif +} + +ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize) +{ + ZSTD_DCtx* const dctx = (ZSTD_DCtx*) workspace; + + if ((size_t)workspace & 7) return NULL; /* 8-aligned */ + if (workspaceSize < sizeof(ZSTD_DCtx)) return NULL; /* minimum size */ + + ZSTD_initDCtx_internal(dctx); + dctx->staticSize = workspaceSize; + dctx->inBuff = (char*)(dctx+1); + return dctx; +} + +ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) +{ + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + + { ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_customMalloc(sizeof(*dctx), customMem); + if (!dctx) return NULL; + dctx->customMem = customMem; + ZSTD_initDCtx_internal(dctx); + return dctx; + } +} + +ZSTD_DCtx* ZSTD_createDCtx(void) +{ + DEBUGLOG(3, "ZSTD_createDCtx"); + return ZSTD_createDCtx_advanced(ZSTD_defaultCMem); +} + +static void ZSTD_clearDict(ZSTD_DCtx* dctx) +{ + ZSTD_freeDDict(dctx->ddictLocal); + dctx->ddictLocal = NULL; + dctx->ddict = NULL; + dctx->dictUses = ZSTD_dont_use; +} + +size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) +{ + if (dctx==NULL) return 0; /* support free on NULL */ + RETURN_ERROR_IF(dctx->staticSize, memory_allocation, "not compatible with static DCtx"); + { ZSTD_customMem const cMem = dctx->customMem; + ZSTD_clearDict(dctx); + ZSTD_customFree(dctx->inBuff, cMem); + dctx->inBuff = NULL; + if (dctx->ddictSet) { + ZSTD_freeDDictHashSet(dctx->ddictSet, cMem); + dctx->ddictSet = NULL; + } + ZSTD_customFree(dctx, cMem); + return 0; + } +} + +/* no longer useful */ +void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) +{ + size_t const toCopy = (size_t)((char*)(&dstDCtx->inBuff) - (char*)dstDCtx); + ZSTD_memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */ +} + +/* Given a dctx with a digested frame params, re-selects the correct ZSTD_DDict based on + * the requested dict ID from the frame. If there exists a reference to the correct ZSTD_DDict, then + * accordingly sets the ddict to be used to decompress the frame. + * + * If no DDict is found, then no action is taken, and the ZSTD_DCtx::ddict remains as-is. + * + * ZSTD_d_refMultipleDDicts must be enabled for this function to be called. + */ +static void ZSTD_DCtx_selectFrameDDict(ZSTD_DCtx* dctx) { + assert(dctx->refMultipleDDicts && dctx->ddictSet); + DEBUGLOG(4, "Adjusting DDict based on requested dict ID from frame"); + if (dctx->ddict) { + const ZSTD_DDict* frameDDict = ZSTD_DDictHashSet_getDDict(dctx->ddictSet, dctx->fParams.dictID); + if (frameDDict) { + DEBUGLOG(4, "DDict found!"); + ZSTD_clearDict(dctx); + dctx->dictID = dctx->fParams.dictID; + dctx->ddict = frameDDict; + dctx->dictUses = ZSTD_use_indefinitely; + } + } +} + + +/*-************************************************************* + * Frame header decoding + ***************************************************************/ + +/*! ZSTD_isFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. + * Note 3 : Skippable Frame Identifiers are considered valid. */ +unsigned ZSTD_isFrame(const void* buffer, size_t size) +{ + if (size < ZSTD_FRAMEIDSIZE) return 0; + { U32 const magic = MEM_readLE32(buffer); + if (magic == ZSTD_MAGICNUMBER) return 1; + if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1; + } + return 0; +} + +/* ZSTD_frameHeaderSize_internal() : + * srcSize must be large enough to reach header size fields. + * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless. + * @return : size of the Frame Header + * or an error code, which can be tested with ZSTD_isError() */ +static size_t ZSTD_frameHeaderSize_internal(const void* src, size_t srcSize, ZSTD_format_e format) +{ + size_t const minInputSize = ZSTD_startingInputLength(format); + RETURN_ERROR_IF(srcSize < minInputSize, srcSize_wrong, ""); + + { BYTE const fhd = ((const BYTE*)src)[minInputSize-1]; + U32 const dictID= fhd & 3; + U32 const singleSegment = (fhd >> 5) & 1; + U32 const fcsId = fhd >> 6; + return minInputSize + !singleSegment + + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] + + (singleSegment && !fcsId); + } +} + +/* ZSTD_frameHeaderSize() : + * srcSize must be >= ZSTD_frameHeaderSize_prefix. + * @return : size of the Frame Header, + * or an error code (if srcSize is too small) */ +size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) +{ + return ZSTD_frameHeaderSize_internal(src, srcSize, ZSTD_f_zstd1); +} + + +/* ZSTD_getFrameHeader_advanced() : + * decode Frame Header, or require larger `srcSize`. + * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, + * or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format) +{ + const BYTE* ip = (const BYTE*)src; + size_t const minInputSize = ZSTD_startingInputLength(format); + + ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */ + if (srcSize < minInputSize) return minInputSize; + RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter"); + + if ( (format != ZSTD_f_zstd1_magicless) + && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) { + if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + /* skippable frame */ + if (srcSize < ZSTD_SKIPPABLEHEADERSIZE) + return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */ + ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); + zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE); + zfhPtr->frameType = ZSTD_skippableFrame; + return 0; + } + RETURN_ERROR(prefix_unknown, ""); + } + + /* ensure there is enough `srcSize` to fully read/decode frame header */ + { size_t const fhsize = ZSTD_frameHeaderSize_internal(src, srcSize, format); + if (srcSize < fhsize) return fhsize; + zfhPtr->headerSize = (U32)fhsize; + } + + { BYTE const fhdByte = ip[minInputSize-1]; + size_t pos = minInputSize; + U32 const dictIDSizeCode = fhdByte&3; + U32 const checksumFlag = (fhdByte>>2)&1; + U32 const singleSegment = (fhdByte>>5)&1; + U32 const fcsID = fhdByte>>6; + U64 windowSize = 0; + U32 dictID = 0; + U64 frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN; + RETURN_ERROR_IF((fhdByte & 0x08) != 0, frameParameter_unsupported, + "reserved bits, must be zero"); + + if (!singleSegment) { + BYTE const wlByte = ip[pos++]; + U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; + RETURN_ERROR_IF(windowLog > ZSTD_WINDOWLOG_MAX, frameParameter_windowTooLarge, ""); + windowSize = (1ULL << windowLog); + windowSize += (windowSize >> 3) * (wlByte&7); + } + switch(dictIDSizeCode) + { + default: assert(0); /* impossible */ + case 0 : break; + case 1 : dictID = ip[pos]; pos++; break; + case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break; + case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break; + } + switch(fcsID) + { + default: assert(0); /* impossible */ + case 0 : if (singleSegment) frameContentSize = ip[pos]; break; + case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break; + case 2 : frameContentSize = MEM_readLE32(ip+pos); break; + case 3 : frameContentSize = MEM_readLE64(ip+pos); break; + } + if (singleSegment) windowSize = frameContentSize; + + zfhPtr->frameType = ZSTD_frame; + zfhPtr->frameContentSize = frameContentSize; + zfhPtr->windowSize = windowSize; + zfhPtr->blockSizeMax = (unsigned) MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + zfhPtr->dictID = dictID; + zfhPtr->checksumFlag = checksumFlag; + } + return 0; +} + +/* ZSTD_getFrameHeader() : + * decode Frame Header, or require larger `srcSize`. + * note : this function does not consume input, it only reads it. + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, + * or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize) +{ + return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1); +} + + +/* ZSTD_getFrameContentSize() : + * compatible with legacy mode + * @return : decompressed size of the single frame pointed to be `src` if known, otherwise + * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined + * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */ +unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) +{ + { ZSTD_frameHeader zfh; + if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0) + return ZSTD_CONTENTSIZE_ERROR; + if (zfh.frameType == ZSTD_skippableFrame) { + return 0; + } else { + return zfh.frameContentSize; + } } +} + +static size_t readSkippableFrameSize(void const* src, size_t srcSize) +{ + size_t const skippableHeaderSize = ZSTD_SKIPPABLEHEADERSIZE; + U32 sizeU32; + + RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, ""); + + sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE); + RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32, + frameParameter_unsupported, ""); + { + size_t const skippableSize = skippableHeaderSize + sizeU32; + RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong, ""); + return skippableSize; + } +} + +/* ZSTD_findDecompressedSize() : + * compatible with legacy mode + * `srcSize` must be the exact length of some number of ZSTD compressed and/or + * skippable frames + * @return : decompressed size of the frames contained */ +unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) +{ + unsigned long long totalDstSize = 0; + + while (srcSize >= ZSTD_startingInputLength(ZSTD_f_zstd1)) { + U32 const magicNumber = MEM_readLE32(src); + + if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + size_t const skippableSize = readSkippableFrameSize(src, srcSize); + if (ZSTD_isError(skippableSize)) { + return ZSTD_CONTENTSIZE_ERROR; + } + assert(skippableSize <= srcSize); + + src = (const BYTE *)src + skippableSize; + srcSize -= skippableSize; + continue; + } + + { unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); + if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret; + + /* check for overflow */ + if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR; + totalDstSize += ret; + } + { size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize); + if (ZSTD_isError(frameSrcSize)) { + return ZSTD_CONTENTSIZE_ERROR; + } + + src = (const BYTE *)src + frameSrcSize; + srcSize -= frameSrcSize; + } + } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */ + + if (srcSize) return ZSTD_CONTENTSIZE_ERROR; + + return totalDstSize; +} + +/* ZSTD_getDecompressedSize() : + * compatible with legacy mode + * @return : decompressed size if known, 0 otherwise + note : 0 can mean any of the following : + - frame content is empty + - decompressed size field is not present in frame header + - frame header unknown / not supported + - frame header not complete (`srcSize` too small) */ +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize) +{ + unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_ERROR < ZSTD_CONTENTSIZE_UNKNOWN); + return (ret >= ZSTD_CONTENTSIZE_ERROR) ? 0 : ret; +} + + +/* ZSTD_decodeFrameHeader() : + * `headerSize` must be the size provided by ZSTD_frameHeaderSize(). + * If multiple DDict references are enabled, also will choose the correct DDict to use. + * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ +static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize) +{ + size_t const result = ZSTD_getFrameHeader_advanced(&(dctx->fParams), src, headerSize, dctx->format); + if (ZSTD_isError(result)) return result; /* invalid header */ + RETURN_ERROR_IF(result>0, srcSize_wrong, "headerSize too small"); + + /* Reference DDict requested by frame if dctx references multiple ddicts */ + if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts && dctx->ddictSet) { + ZSTD_DCtx_selectFrameDDict(dctx); + } + +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + /* Skip the dictID check in fuzzing mode, because it makes the search + * harder. + */ + RETURN_ERROR_IF(dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID), + dictionary_wrong, ""); +#endif + dctx->validateChecksum = (dctx->fParams.checksumFlag && !dctx->forceIgnoreChecksum) ? 1 : 0; + if (dctx->validateChecksum) xxh64_reset(&dctx->xxhState, 0); + dctx->processedCSize += headerSize; + return 0; +} + +static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret) +{ + ZSTD_frameSizeInfo frameSizeInfo; + frameSizeInfo.compressedSize = ret; + frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR; + return frameSizeInfo; +} + +static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize) +{ + ZSTD_frameSizeInfo frameSizeInfo; + ZSTD_memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo)); + + + if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE) + && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize); + assert(ZSTD_isError(frameSizeInfo.compressedSize) || + frameSizeInfo.compressedSize <= srcSize); + return frameSizeInfo; + } else { + const BYTE* ip = (const BYTE*)src; + const BYTE* const ipstart = ip; + size_t remainingSize = srcSize; + size_t nbBlocks = 0; + ZSTD_frameHeader zfh; + + /* Extract Frame Header */ + { size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize); + if (ZSTD_isError(ret)) + return ZSTD_errorFrameSizeInfo(ret); + if (ret > 0) + return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); + } + + ip += zfh.headerSize; + remainingSize -= zfh.headerSize; + + /* Iterate over each block */ + while (1) { + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) + return ZSTD_errorFrameSizeInfo(cBlockSize); + + if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) + return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); + + ip += ZSTD_blockHeaderSize + cBlockSize; + remainingSize -= ZSTD_blockHeaderSize + cBlockSize; + nbBlocks++; + + if (blockProperties.lastBlock) break; + } + + /* Final frame content checksum */ + if (zfh.checksumFlag) { + if (remainingSize < 4) + return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); + ip += 4; + } + + frameSizeInfo.compressedSize = (size_t)(ip - ipstart); + frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) + ? zfh.frameContentSize + : nbBlocks * zfh.blockSizeMax; + return frameSizeInfo; + } +} + +/* ZSTD_findFrameCompressedSize() : + * compatible with legacy mode + * `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame + * `srcSize` must be at least as large as the frame contained + * @return : the compressed size of the frame starting at `src` */ +size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) +{ + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); + return frameSizeInfo.compressedSize; +} + +/* ZSTD_decompressBound() : + * compatible with legacy mode + * `src` must point to the start of a ZSTD frame or a skippeable frame + * `srcSize` must be at least as large as the frame contained + * @return : the maximum decompressed size of the compressed source + */ +unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize) +{ + unsigned long long bound = 0; + /* Iterate over each frame */ + while (srcSize > 0) { + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); + size_t const compressedSize = frameSizeInfo.compressedSize; + unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; + if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) + return ZSTD_CONTENTSIZE_ERROR; + assert(srcSize >= compressedSize); + src = (const BYTE*)src + compressedSize; + srcSize -= compressedSize; + bound += decompressedBound; + } + return bound; +} + + +/*-************************************************************* + * Frame decoding + ***************************************************************/ + +/* ZSTD_insertBlock() : + * insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ +size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize) +{ + DEBUGLOG(5, "ZSTD_insertBlock: %u bytes", (unsigned)blockSize); + ZSTD_checkContinuity(dctx, blockStart, blockSize); + dctx->previousDstEnd = (const char*)blockStart + blockSize; + return blockSize; +} + + +static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_copyRawBlock"); + RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall, ""); + if (dst == NULL) { + if (srcSize == 0) return 0; + RETURN_ERROR(dstBuffer_null, ""); + } + ZSTD_memcpy(dst, src, srcSize); + return srcSize; +} + +static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, + BYTE b, + size_t regenSize) +{ + RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall, ""); + if (dst == NULL) { + if (regenSize == 0) return 0; + RETURN_ERROR(dstBuffer_null, ""); + } + ZSTD_memset(dst, b, regenSize); + return regenSize; +} + +static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, unsigned streaming) +{ + (void)dctx; + (void)uncompressedSize; + (void)compressedSize; + (void)streaming; +} + + +/*! ZSTD_decompressFrame() : + * @dctx must be properly initialized + * will update *srcPtr and *srcSizePtr, + * to make *srcPtr progress by one frame. */ +static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void** srcPtr, size_t *srcSizePtr) +{ + const BYTE* const istart = (const BYTE*)(*srcPtr); + const BYTE* ip = istart; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = dstCapacity != 0 ? ostart + dstCapacity : ostart; + BYTE* op = ostart; + size_t remainingSrcSize = *srcSizePtr; + + DEBUGLOG(4, "ZSTD_decompressFrame (srcSize:%i)", (int)*srcSizePtr); + + /* check */ + RETURN_ERROR_IF( + remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN(dctx->format)+ZSTD_blockHeaderSize, + srcSize_wrong, ""); + + /* Frame Header */ + { size_t const frameHeaderSize = ZSTD_frameHeaderSize_internal( + ip, ZSTD_FRAMEHEADERSIZE_PREFIX(dctx->format), dctx->format); + if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; + RETURN_ERROR_IF(remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize, + srcSize_wrong, ""); + FORWARD_IF_ERROR( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) , ""); + ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize; + } + + /* Loop on each block */ + while (1) { + size_t decodedSize; + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + + ip += ZSTD_blockHeaderSize; + remainingSrcSize -= ZSTD_blockHeaderSize; + RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong, ""); + + switch(blockProperties.blockType) + { + case bt_compressed: + decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oend-op), ip, cBlockSize, /* frame */ 1); + break; + case bt_raw : + decodedSize = ZSTD_copyRawBlock(op, (size_t)(oend-op), ip, cBlockSize); + break; + case bt_rle : + decodedSize = ZSTD_setRleBlock(op, (size_t)(oend-op), *ip, blockProperties.origSize); + break; + case bt_reserved : + default: + RETURN_ERROR(corruption_detected, "invalid block type"); + } + + if (ZSTD_isError(decodedSize)) return decodedSize; + if (dctx->validateChecksum) + xxh64_update(&dctx->xxhState, op, decodedSize); + if (decodedSize != 0) + op += decodedSize; + assert(ip != NULL); + ip += cBlockSize; + remainingSrcSize -= cBlockSize; + if (blockProperties.lastBlock) break; + } + + if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) { + RETURN_ERROR_IF((U64)(op-ostart) != dctx->fParams.frameContentSize, + corruption_detected, ""); + } + if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */ + RETURN_ERROR_IF(remainingSrcSize<4, checksum_wrong, ""); + if (!dctx->forceIgnoreChecksum) { + U32 const checkCalc = (U32)xxh64_digest(&dctx->xxhState); + U32 checkRead; + checkRead = MEM_readLE32(ip); + RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong, ""); + } + ip += 4; + remainingSrcSize -= 4; + } + ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0); + /* Allow caller to get size read */ + *srcPtr = ip; + *srcSizePtr = remainingSrcSize; + return (size_t)(op-ostart); +} + +static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + const ZSTD_DDict* ddict) +{ + void* const dststart = dst; + int moreThan1Frame = 0; + + DEBUGLOG(5, "ZSTD_decompressMultiFrame"); + assert(dict==NULL || ddict==NULL); /* either dict or ddict set, not both */ + + if (ddict) { + dict = ZSTD_DDict_dictContent(ddict); + dictSize = ZSTD_DDict_dictSize(ddict); + } + + while (srcSize >= ZSTD_startingInputLength(dctx->format)) { + + + { U32 const magicNumber = MEM_readLE32(src); + DEBUGLOG(4, "reading magic number %08X (expecting %08X)", + (unsigned)magicNumber, ZSTD_MAGICNUMBER); + if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + size_t const skippableSize = readSkippableFrameSize(src, srcSize); + FORWARD_IF_ERROR(skippableSize, "readSkippableFrameSize failed"); + assert(skippableSize <= srcSize); + + src = (const BYTE *)src + skippableSize; + srcSize -= skippableSize; + continue; + } } + + if (ddict) { + /* we were called from ZSTD_decompress_usingDDict */ + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(dctx, ddict), ""); + } else { + /* this will initialize correctly with no dict if dict == NULL, so + * use this in all cases but ddict */ + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize), ""); + } + ZSTD_checkContinuity(dctx, dst, dstCapacity); + + { const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity, + &src, &srcSize); + RETURN_ERROR_IF( + (ZSTD_getErrorCode(res) == ZSTD_error_prefix_unknown) + && (moreThan1Frame==1), + srcSize_wrong, + "At least one frame successfully completed, " + "but following bytes are garbage: " + "it's more likely to be a srcSize error, " + "specifying more input bytes than size of frame(s). " + "Note: one could be unlucky, it might be a corruption error instead, " + "happening right at the place where we expect zstd magic bytes. " + "But this is _much_ less likely than a srcSize field error."); + if (ZSTD_isError(res)) return res; + assert(res <= dstCapacity); + if (res != 0) + dst = (BYTE*)dst + res; + dstCapacity -= res; + } + moreThan1Frame = 1; + } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */ + + RETURN_ERROR_IF(srcSize, srcSize_wrong, "input not entirely consumed"); + + return (size_t)((BYTE*)dst - (BYTE*)dststart); +} + +size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize) +{ + return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL); +} + + +static ZSTD_DDict const* ZSTD_getDDict(ZSTD_DCtx* dctx) +{ + switch (dctx->dictUses) { + default: + assert(0 /* Impossible */); + /* fall-through */ + case ZSTD_dont_use: + ZSTD_clearDict(dctx); + return NULL; + case ZSTD_use_indefinitely: + return dctx->ddict; + case ZSTD_use_once: + dctx->dictUses = ZSTD_dont_use; + return dctx->ddict; + } +} + +size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + return ZSTD_decompress_usingDDict(dctx, dst, dstCapacity, src, srcSize, ZSTD_getDDict(dctx)); +} + + +size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ +#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1) + size_t regenSize; + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + RETURN_ERROR_IF(dctx==NULL, memory_allocation, "NULL pointer!"); + regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize); + ZSTD_freeDCtx(dctx); + return regenSize; +#else /* stack mode */ + ZSTD_DCtx dctx; + ZSTD_initDCtx_internal(&dctx); + return ZSTD_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize); +#endif +} + + +/*-************************************** +* Advanced Streaming Decompression API +* Bufferless and synchronous +****************************************/ +size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; } + +/* + * Similar to ZSTD_nextSrcSizeToDecompress(), but when when a block input can be streamed, + * we allow taking a partial block as the input. Currently only raw uncompressed blocks can + * be streamed. + * + * For blocks that can be streamed, this allows us to reduce the latency until we produce + * output, and avoid copying the input. + * + * @param inputSize - The total amount of input that the caller currently has. + */ +static size_t ZSTD_nextSrcSizeToDecompressWithInputSize(ZSTD_DCtx* dctx, size_t inputSize) { + if (!(dctx->stage == ZSTDds_decompressBlock || dctx->stage == ZSTDds_decompressLastBlock)) + return dctx->expected; + if (dctx->bType != bt_raw) + return dctx->expected; + return MIN(MAX(inputSize, 1), dctx->expected); +} + +ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { + switch(dctx->stage) + { + default: /* should not happen */ + assert(0); + case ZSTDds_getFrameHeaderSize: + case ZSTDds_decodeFrameHeader: + return ZSTDnit_frameHeader; + case ZSTDds_decodeBlockHeader: + return ZSTDnit_blockHeader; + case ZSTDds_decompressBlock: + return ZSTDnit_block; + case ZSTDds_decompressLastBlock: + return ZSTDnit_lastBlock; + case ZSTDds_checkChecksum: + return ZSTDnit_checksum; + case ZSTDds_decodeSkippableHeader: + case ZSTDds_skipFrame: + return ZSTDnit_skippableFrame; + } +} + +static int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; } + +/* ZSTD_decompressContinue() : + * srcSize : must be the exact nb of bytes expected (see ZSTD_nextSrcSizeToDecompress()) + * @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity) + * or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_decompressContinue (srcSize:%u)", (unsigned)srcSize); + /* Sanity check */ + RETURN_ERROR_IF(srcSize != ZSTD_nextSrcSizeToDecompressWithInputSize(dctx, srcSize), srcSize_wrong, "not allowed"); + ZSTD_checkContinuity(dctx, dst, dstCapacity); + + dctx->processedCSize += srcSize; + + switch (dctx->stage) + { + case ZSTDds_getFrameHeaderSize : + assert(src != NULL); + if (dctx->format == ZSTD_f_zstd1) { /* allows header */ + assert(srcSize >= ZSTD_FRAMEIDSIZE); /* to read skippable magic number */ + if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + ZSTD_memcpy(dctx->headerBuffer, src, srcSize); + dctx->expected = ZSTD_SKIPPABLEHEADERSIZE - srcSize; /* remaining to load to get full skippable frame header */ + dctx->stage = ZSTDds_decodeSkippableHeader; + return 0; + } } + dctx->headerSize = ZSTD_frameHeaderSize_internal(src, srcSize, dctx->format); + if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; + ZSTD_memcpy(dctx->headerBuffer, src, srcSize); + dctx->expected = dctx->headerSize - srcSize; + dctx->stage = ZSTDds_decodeFrameHeader; + return 0; + + case ZSTDds_decodeFrameHeader: + assert(src != NULL); + ZSTD_memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize); + FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize), ""); + dctx->expected = ZSTD_blockHeaderSize; + dctx->stage = ZSTDds_decodeBlockHeader; + return 0; + + case ZSTDds_decodeBlockHeader: + { blockProperties_t bp; + size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + RETURN_ERROR_IF(cBlockSize > dctx->fParams.blockSizeMax, corruption_detected, "Block Size Exceeds Maximum"); + dctx->expected = cBlockSize; + dctx->bType = bp.blockType; + dctx->rleSize = bp.origSize; + if (cBlockSize) { + dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock; + return 0; + } + /* empty block */ + if (bp.lastBlock) { + if (dctx->fParams.checksumFlag) { + dctx->expected = 4; + dctx->stage = ZSTDds_checkChecksum; + } else { + dctx->expected = 0; /* end of frame */ + dctx->stage = ZSTDds_getFrameHeaderSize; + } + } else { + dctx->expected = ZSTD_blockHeaderSize; /* jump to next header */ + dctx->stage = ZSTDds_decodeBlockHeader; + } + return 0; + } + + case ZSTDds_decompressLastBlock: + case ZSTDds_decompressBlock: + DEBUGLOG(5, "ZSTD_decompressContinue: case ZSTDds_decompressBlock"); + { size_t rSize; + switch(dctx->bType) + { + case bt_compressed: + DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed"); + rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1); + dctx->expected = 0; /* Streaming not supported */ + break; + case bt_raw : + assert(srcSize <= dctx->expected); + rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); + FORWARD_IF_ERROR(rSize, "ZSTD_copyRawBlock failed"); + assert(rSize == srcSize); + dctx->expected -= rSize; + break; + case bt_rle : + rSize = ZSTD_setRleBlock(dst, dstCapacity, *(const BYTE*)src, dctx->rleSize); + dctx->expected = 0; /* Streaming not supported */ + break; + case bt_reserved : /* should never happen */ + default: + RETURN_ERROR(corruption_detected, "invalid block type"); + } + FORWARD_IF_ERROR(rSize, ""); + RETURN_ERROR_IF(rSize > dctx->fParams.blockSizeMax, corruption_detected, "Decompressed Block Size Exceeds Maximum"); + DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize); + dctx->decodedSize += rSize; + if (dctx->validateChecksum) xxh64_update(&dctx->xxhState, dst, rSize); + dctx->previousDstEnd = (char*)dst + rSize; + + /* Stay on the same stage until we are finished streaming the block. */ + if (dctx->expected > 0) { + return rSize; + } + + if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */ + DEBUGLOG(4, "ZSTD_decompressContinue: decoded size from frame : %u", (unsigned)dctx->decodedSize); + RETURN_ERROR_IF( + dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN + && dctx->decodedSize != dctx->fParams.frameContentSize, + corruption_detected, ""); + if (dctx->fParams.checksumFlag) { /* another round for frame checksum */ + dctx->expected = 4; + dctx->stage = ZSTDds_checkChecksum; + } else { + ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1); + dctx->expected = 0; /* ends here */ + dctx->stage = ZSTDds_getFrameHeaderSize; + } + } else { + dctx->stage = ZSTDds_decodeBlockHeader; + dctx->expected = ZSTD_blockHeaderSize; + } + return rSize; + } + + case ZSTDds_checkChecksum: + assert(srcSize == 4); /* guaranteed by dctx->expected */ + { + if (dctx->validateChecksum) { + U32 const h32 = (U32)xxh64_digest(&dctx->xxhState); + U32 const check32 = MEM_readLE32(src); + DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32); + RETURN_ERROR_IF(check32 != h32, checksum_wrong, ""); + } + ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1); + dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + } + + case ZSTDds_decodeSkippableHeader: + assert(src != NULL); + assert(srcSize <= ZSTD_SKIPPABLEHEADERSIZE); + ZSTD_memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */ + dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */ + dctx->stage = ZSTDds_skipFrame; + return 0; + + case ZSTDds_skipFrame: + dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + + default: + assert(0); /* impossible */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */ + } +} + + +static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + dctx->dictEnd = dctx->previousDstEnd; + dctx->virtualStart = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); + dctx->prefixStart = dict; + dctx->previousDstEnd = (const char*)dict + dictSize; +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + dctx->dictContentBeginForFuzzing = dctx->prefixStart; + dctx->dictContentEndForFuzzing = dctx->previousDstEnd; +#endif + return 0; +} + +/*! ZSTD_loadDEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * @return : size of entropy tables read */ +size_t +ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, + const void* const dict, size_t const dictSize) +{ + const BYTE* dictPtr = (const BYTE*)dict; + const BYTE* const dictEnd = dictPtr + dictSize; + + RETURN_ERROR_IF(dictSize <= 8, dictionary_corrupted, "dict is too small"); + assert(MEM_readLE32(dict) == ZSTD_MAGIC_DICTIONARY); /* dict must be valid */ + dictPtr += 8; /* skip header = magic + dictID */ + + ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, OFTable) == offsetof(ZSTD_entropyDTables_t, LLTable) + sizeof(entropy->LLTable)); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, MLTable) == offsetof(ZSTD_entropyDTables_t, OFTable) + sizeof(entropy->OFTable)); + ZSTD_STATIC_ASSERT(sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable) >= HUF_DECOMPRESS_WORKSPACE_SIZE); + { void* const workspace = &entropy->LLTable; /* use fse tables as temporary workspace; implies fse tables are grouped together */ + size_t const workspaceSize = sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable); +#ifdef HUF_FORCE_DECOMPRESS_X1 + /* in minimal huffman, we always use X1 variants */ + size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable, + dictPtr, dictEnd - dictPtr, + workspace, workspaceSize); +#else + size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable, + dictPtr, (size_t)(dictEnd - dictPtr), + workspace, workspaceSize); +#endif + RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, ""); + dictPtr += hSize; + } + + { short offcodeNCount[MaxOff+1]; + unsigned offcodeMaxValue = MaxOff, offcodeLog; + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, (size_t)(dictEnd-dictPtr)); + RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(offcodeMaxValue > MaxOff, dictionary_corrupted, ""); + RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, ""); + ZSTD_buildFSETable( entropy->OFTable, + offcodeNCount, offcodeMaxValue, + OF_base, OF_bits, + offcodeLog, + entropy->workspace, sizeof(entropy->workspace), + /* bmi2 */0); + dictPtr += offcodeHeaderSize; + } + + { short matchlengthNCount[MaxML+1]; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; + size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, (size_t)(dictEnd-dictPtr)); + RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(matchlengthMaxValue > MaxML, dictionary_corrupted, ""); + RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, ""); + ZSTD_buildFSETable( entropy->MLTable, + matchlengthNCount, matchlengthMaxValue, + ML_base, ML_bits, + matchlengthLog, + entropy->workspace, sizeof(entropy->workspace), + /* bmi2 */ 0); + dictPtr += matchlengthHeaderSize; + } + + { short litlengthNCount[MaxLL+1]; + unsigned litlengthMaxValue = MaxLL, litlengthLog; + size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, (size_t)(dictEnd-dictPtr)); + RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(litlengthMaxValue > MaxLL, dictionary_corrupted, ""); + RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, ""); + ZSTD_buildFSETable( entropy->LLTable, + litlengthNCount, litlengthMaxValue, + LL_base, LL_bits, + litlengthLog, + entropy->workspace, sizeof(entropy->workspace), + /* bmi2 */ 0); + dictPtr += litlengthHeaderSize; + } + + RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted, ""); + { int i; + size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12)); + for (i=0; i<3; i++) { + U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4; + RETURN_ERROR_IF(rep==0 || rep > dictContentSize, + dictionary_corrupted, ""); + entropy->rep[i] = rep; + } } + + return (size_t)(dictPtr - (const BYTE*)dict); +} + +static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize); + { U32 const magic = MEM_readLE32(dict); + if (magic != ZSTD_MAGIC_DICTIONARY) { + return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */ + } } + dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); + + /* load entropy tables */ + { size_t const eSize = ZSTD_loadDEntropy(&dctx->entropy, dict, dictSize); + RETURN_ERROR_IF(ZSTD_isError(eSize), dictionary_corrupted, ""); + dict = (const char*)dict + eSize; + dictSize -= eSize; + } + dctx->litEntropy = dctx->fseEntropy = 1; + + /* reference dictionary content */ + return ZSTD_refDictContent(dctx, dict, dictSize); +} + +size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) +{ + assert(dctx != NULL); + dctx->expected = ZSTD_startingInputLength(dctx->format); /* dctx->format must be properly set */ + dctx->stage = ZSTDds_getFrameHeaderSize; + dctx->processedCSize = 0; + dctx->decodedSize = 0; + dctx->previousDstEnd = NULL; + dctx->prefixStart = NULL; + dctx->virtualStart = NULL; + dctx->dictEnd = NULL; + dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ + dctx->litEntropy = dctx->fseEntropy = 0; + dctx->dictID = 0; + dctx->bType = bt_reserved; + ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue)); + ZSTD_memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ + dctx->LLTptr = dctx->entropy.LLTable; + dctx->MLTptr = dctx->entropy.MLTable; + dctx->OFTptr = dctx->entropy.OFTable; + dctx->HUFptr = dctx->entropy.hufTable; + return 0; +} + +size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , ""); + if (dict && dictSize) + RETURN_ERROR_IF( + ZSTD_isError(ZSTD_decompress_insertDictionary(dctx, dict, dictSize)), + dictionary_corrupted, ""); + return 0; +} + + +/* ====== ZSTD_DDict ====== */ + +size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + DEBUGLOG(4, "ZSTD_decompressBegin_usingDDict"); + assert(dctx != NULL); + if (ddict) { + const char* const dictStart = (const char*)ZSTD_DDict_dictContent(ddict); + size_t const dictSize = ZSTD_DDict_dictSize(ddict); + const void* const dictEnd = dictStart + dictSize; + dctx->ddictIsCold = (dctx->dictEnd != dictEnd); + DEBUGLOG(4, "DDict is %s", + dctx->ddictIsCold ? "~cold~" : "hot!"); + } + FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , ""); + if (ddict) { /* NULL ddict is equivalent to no dictionary */ + ZSTD_copyDDictParameters(dctx, ddict); + } + return 0; +} + +/*! ZSTD_getDictID_fromDict() : + * Provides the dictID stored within dictionary. + * if @return == 0, the dictionary is not conformant with Zstandard specification. + * It can still be loaded, but as a content-only dictionary. */ +unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) +{ + if (dictSize < 8) return 0; + if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0; + return MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); +} + +/*! ZSTD_getDictID_fromFrame() : + * Provides the dictID required to decompress frame stored within `src`. + * If @return == 0, the dictID could not be decoded. + * This could for one of the following reasons : + * - The frame does not require a dictionary (most common case). + * - The frame was built with dictID intentionally removed. + * Needed dictionary is a hidden information. + * Note : this use case also happens when using a non-conformant dictionary. + * - `srcSize` is too small, and as a result, frame header could not be decoded. + * Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`. + * - This is not a Zstandard frame. + * When identifying the exact failure cause, it's possible to use + * ZSTD_getFrameHeader(), which will provide a more precise error code. */ +unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize) +{ + ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0 }; + size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize); + if (ZSTD_isError(hError)) return 0; + return zfp.dictID; +} + + +/*! ZSTD_decompress_usingDDict() : +* Decompression using a pre-digested Dictionary +* Use dictionary without significant overhead. */ +size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict) +{ + /* pass content and size in case legacy frames are encountered */ + return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, + NULL, 0, + ddict); +} + + +/*===================================== +* Streaming decompression +*====================================*/ + +ZSTD_DStream* ZSTD_createDStream(void) +{ + DEBUGLOG(3, "ZSTD_createDStream"); + return ZSTD_createDStream_advanced(ZSTD_defaultCMem); +} + +ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize) +{ + return ZSTD_initStaticDCtx(workspace, workspaceSize); +} + +ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) +{ + return ZSTD_createDCtx_advanced(customMem); +} + +size_t ZSTD_freeDStream(ZSTD_DStream* zds) +{ + return ZSTD_freeDCtx(zds); +} + + +/* *** Initialization *** */ + +size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; } +size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; } + +size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + ZSTD_clearDict(dctx); + if (dict && dictSize != 0) { + dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem); + RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation, "NULL pointer!"); + dctx->ddict = dctx->ddictLocal; + dctx->dictUses = ZSTD_use_indefinitely; + } + return 0; +} + +size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto); +} + +size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); +} + +size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType) +{ + FORWARD_IF_ERROR(ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType), ""); + dctx->dictUses = ZSTD_use_once; + return 0; +} + +size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize) +{ + return ZSTD_DCtx_refPrefix_advanced(dctx, prefix, prefixSize, ZSTD_dct_rawContent); +} + + +/* ZSTD_initDStream_usingDict() : + * return : expected size, aka ZSTD_startingInputLength(). + * this function cannot fail */ +size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize) +{ + DEBUGLOG(4, "ZSTD_initDStream_usingDict"); + FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) , ""); + return ZSTD_startingInputLength(zds->format); +} + +/* note : this variant can't fail */ +size_t ZSTD_initDStream(ZSTD_DStream* zds) +{ + DEBUGLOG(4, "ZSTD_initDStream"); + return ZSTD_initDStream_usingDDict(zds, NULL); +} + +/* ZSTD_initDStream_usingDDict() : + * ddict will just be referenced, and must outlive decompression session + * this function cannot fail */ +size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) +{ + FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , ""); + return ZSTD_startingInputLength(dctx->format); +} + +/* ZSTD_resetDStream() : + * return : expected size, aka ZSTD_startingInputLength(). + * this function cannot fail */ +size_t ZSTD_resetDStream(ZSTD_DStream* dctx) +{ + FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), ""); + return ZSTD_startingInputLength(dctx->format); +} + + +size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + ZSTD_clearDict(dctx); + if (ddict) { + dctx->ddict = ddict; + dctx->dictUses = ZSTD_use_indefinitely; + if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts) { + if (dctx->ddictSet == NULL) { + dctx->ddictSet = ZSTD_createDDictHashSet(dctx->customMem); + if (!dctx->ddictSet) { + RETURN_ERROR(memory_allocation, "Failed to allocate memory for hash set!"); + } + } + assert(!dctx->staticSize); /* Impossible: ddictSet cannot have been allocated if static dctx */ + FORWARD_IF_ERROR(ZSTD_DDictHashSet_addDDict(dctx->ddictSet, ddict, dctx->customMem), ""); + } + } + return 0; +} + +/* ZSTD_DCtx_setMaxWindowSize() : + * note : no direct equivalence in ZSTD_DCtx_setParameter, + * since this version sets windowSize, and the other sets windowLog */ +size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize) +{ + ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax); + size_t const min = (size_t)1 << bounds.lowerBound; + size_t const max = (size_t)1 << bounds.upperBound; + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + RETURN_ERROR_IF(maxWindowSize < min, parameter_outOfBound, ""); + RETURN_ERROR_IF(maxWindowSize > max, parameter_outOfBound, ""); + dctx->maxWindowSize = maxWindowSize; + return 0; +} + +size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format) +{ + return ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (int)format); +} + +ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam) +{ + ZSTD_bounds bounds = { 0, 0, 0 }; + switch(dParam) { + case ZSTD_d_windowLogMax: + bounds.lowerBound = ZSTD_WINDOWLOG_ABSOLUTEMIN; + bounds.upperBound = ZSTD_WINDOWLOG_MAX; + return bounds; + case ZSTD_d_format: + bounds.lowerBound = (int)ZSTD_f_zstd1; + bounds.upperBound = (int)ZSTD_f_zstd1_magicless; + ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless); + return bounds; + case ZSTD_d_stableOutBuffer: + bounds.lowerBound = (int)ZSTD_bm_buffered; + bounds.upperBound = (int)ZSTD_bm_stable; + return bounds; + case ZSTD_d_forceIgnoreChecksum: + bounds.lowerBound = (int)ZSTD_d_validateChecksum; + bounds.upperBound = (int)ZSTD_d_ignoreChecksum; + return bounds; + case ZSTD_d_refMultipleDDicts: + bounds.lowerBound = (int)ZSTD_rmd_refSingleDDict; + bounds.upperBound = (int)ZSTD_rmd_refMultipleDDicts; + return bounds; + default:; + } + bounds.error = ERROR(parameter_unsupported); + return bounds; +} + +/* ZSTD_dParam_withinBounds: + * @return 1 if value is within dParam bounds, + * 0 otherwise */ +static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value) +{ + ZSTD_bounds const bounds = ZSTD_dParam_getBounds(dParam); + if (ZSTD_isError(bounds.error)) return 0; + if (value < bounds.lowerBound) return 0; + if (value > bounds.upperBound) return 0; + return 1; +} + +#define CHECK_DBOUNDS(p,v) { \ + RETURN_ERROR_IF(!ZSTD_dParam_withinBounds(p, v), parameter_outOfBound, ""); \ +} + +size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value) +{ + switch (param) { + case ZSTD_d_windowLogMax: + *value = (int)ZSTD_highbit32((U32)dctx->maxWindowSize); + return 0; + case ZSTD_d_format: + *value = (int)dctx->format; + return 0; + case ZSTD_d_stableOutBuffer: + *value = (int)dctx->outBufferMode; + return 0; + case ZSTD_d_forceIgnoreChecksum: + *value = (int)dctx->forceIgnoreChecksum; + return 0; + case ZSTD_d_refMultipleDDicts: + *value = (int)dctx->refMultipleDDicts; + return 0; + default:; + } + RETURN_ERROR(parameter_unsupported, ""); +} + +size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value) +{ + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + switch(dParam) { + case ZSTD_d_windowLogMax: + if (value == 0) value = ZSTD_WINDOWLOG_LIMIT_DEFAULT; + CHECK_DBOUNDS(ZSTD_d_windowLogMax, value); + dctx->maxWindowSize = ((size_t)1) << value; + return 0; + case ZSTD_d_format: + CHECK_DBOUNDS(ZSTD_d_format, value); + dctx->format = (ZSTD_format_e)value; + return 0; + case ZSTD_d_stableOutBuffer: + CHECK_DBOUNDS(ZSTD_d_stableOutBuffer, value); + dctx->outBufferMode = (ZSTD_bufferMode_e)value; + return 0; + case ZSTD_d_forceIgnoreChecksum: + CHECK_DBOUNDS(ZSTD_d_forceIgnoreChecksum, value); + dctx->forceIgnoreChecksum = (ZSTD_forceIgnoreChecksum_e)value; + return 0; + case ZSTD_d_refMultipleDDicts: + CHECK_DBOUNDS(ZSTD_d_refMultipleDDicts, value); + if (dctx->staticSize != 0) { + RETURN_ERROR(parameter_unsupported, "Static dctx does not support multiple DDicts!"); + } + dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value; + return 0; + default:; + } + RETURN_ERROR(parameter_unsupported, ""); +} + +size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset) +{ + if ( (reset == ZSTD_reset_session_only) + || (reset == ZSTD_reset_session_and_parameters) ) { + dctx->streamStage = zdss_init; + dctx->noForwardProgress = 0; + } + if ( (reset == ZSTD_reset_parameters) + || (reset == ZSTD_reset_session_and_parameters) ) { + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + ZSTD_clearDict(dctx); + ZSTD_DCtx_resetParameters(dctx); + } + return 0; +} + + +size_t ZSTD_sizeof_DStream(const ZSTD_DStream* dctx) +{ + return ZSTD_sizeof_DCtx(dctx); +} + +size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize) +{ + size_t const blockSize = (size_t) MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + unsigned long long const neededRBSize = windowSize + blockSize + (WILDCOPY_OVERLENGTH * 2); + unsigned long long const neededSize = MIN(frameContentSize, neededRBSize); + size_t const minRBSize = (size_t) neededSize; + RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize, + frameParameter_windowTooLarge, ""); + return minRBSize; +} + +size_t ZSTD_estimateDStreamSize(size_t windowSize) +{ + size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + size_t const inBuffSize = blockSize; /* no block can be larger */ + size_t const outBuffSize = ZSTD_decodingBufferSize_min(windowSize, ZSTD_CONTENTSIZE_UNKNOWN); + return ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize; +} + +size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize) +{ + U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable, but requires an additional parameter (or a dctx) */ + ZSTD_frameHeader zfh; + size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize); + if (ZSTD_isError(err)) return err; + RETURN_ERROR_IF(err>0, srcSize_wrong, ""); + RETURN_ERROR_IF(zfh.windowSize > windowSizeMax, + frameParameter_windowTooLarge, ""); + return ZSTD_estimateDStreamSize((size_t)zfh.windowSize); +} + + +/* ***** Decompression ***** */ + +static int ZSTD_DCtx_isOverflow(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) +{ + return (zds->inBuffSize + zds->outBuffSize) >= (neededInBuffSize + neededOutBuffSize) * ZSTD_WORKSPACETOOLARGE_FACTOR; +} + +static void ZSTD_DCtx_updateOversizedDuration(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) +{ + if (ZSTD_DCtx_isOverflow(zds, neededInBuffSize, neededOutBuffSize)) + zds->oversizedDuration++; + else + zds->oversizedDuration = 0; +} + +static int ZSTD_DCtx_isOversizedTooLong(ZSTD_DStream* zds) +{ + return zds->oversizedDuration >= ZSTD_WORKSPACETOOLARGE_MAXDURATION; +} + +/* Checks that the output buffer hasn't changed if ZSTD_obm_stable is used. */ +static size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const* output) +{ + ZSTD_outBuffer const expect = zds->expectedOutBuffer; + /* No requirement when ZSTD_obm_stable is not enabled. */ + if (zds->outBufferMode != ZSTD_bm_stable) + return 0; + /* Any buffer is allowed in zdss_init, this must be the same for every other call until + * the context is reset. + */ + if (zds->streamStage == zdss_init) + return 0; + /* The buffer must match our expectation exactly. */ + if (expect.dst == output->dst && expect.pos == output->pos && expect.size == output->size) + return 0; + RETURN_ERROR(dstBuffer_wrong, "ZSTD_d_stableOutBuffer enabled but output differs!"); +} + +/* Calls ZSTD_decompressContinue() with the right parameters for ZSTD_decompressStream() + * and updates the stage and the output buffer state. This call is extracted so it can be + * used both when reading directly from the ZSTD_inBuffer, and in buffered input mode. + * NOTE: You must break after calling this function since the streamStage is modified. + */ +static size_t ZSTD_decompressContinueStream( + ZSTD_DStream* zds, char** op, char* oend, + void const* src, size_t srcSize) { + int const isSkipFrame = ZSTD_isSkipFrame(zds); + if (zds->outBufferMode == ZSTD_bm_buffered) { + size_t const dstSize = isSkipFrame ? 0 : zds->outBuffSize - zds->outStart; + size_t const decodedSize = ZSTD_decompressContinue(zds, + zds->outBuff + zds->outStart, dstSize, src, srcSize); + FORWARD_IF_ERROR(decodedSize, ""); + if (!decodedSize && !isSkipFrame) { + zds->streamStage = zdss_read; + } else { + zds->outEnd = zds->outStart + decodedSize; + zds->streamStage = zdss_flush; + } + } else { + /* Write directly into the output buffer */ + size_t const dstSize = isSkipFrame ? 0 : (size_t)(oend - *op); + size_t const decodedSize = ZSTD_decompressContinue(zds, *op, dstSize, src, srcSize); + FORWARD_IF_ERROR(decodedSize, ""); + *op += decodedSize; + /* Flushing is not needed. */ + zds->streamStage = zdss_read; + assert(*op <= oend); + assert(zds->outBufferMode == ZSTD_bm_stable); + } + return 0; +} + +size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + const char* const src = (const char*)input->src; + const char* const istart = input->pos != 0 ? src + input->pos : src; + const char* const iend = input->size != 0 ? src + input->size : src; + const char* ip = istart; + char* const dst = (char*)output->dst; + char* const ostart = output->pos != 0 ? dst + output->pos : dst; + char* const oend = output->size != 0 ? dst + output->size : dst; + char* op = ostart; + U32 someMoreWork = 1; + + DEBUGLOG(5, "ZSTD_decompressStream"); + RETURN_ERROR_IF( + input->pos > input->size, + srcSize_wrong, + "forbidden. in: pos: %u vs size: %u", + (U32)input->pos, (U32)input->size); + RETURN_ERROR_IF( + output->pos > output->size, + dstSize_tooSmall, + "forbidden. out: pos: %u vs size: %u", + (U32)output->pos, (U32)output->size); + DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos)); + FORWARD_IF_ERROR(ZSTD_checkOutBuffer(zds, output), ""); + + while (someMoreWork) { + switch(zds->streamStage) + { + case zdss_init : + DEBUGLOG(5, "stage zdss_init => transparent reset "); + zds->streamStage = zdss_loadHeader; + zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; + zds->legacyVersion = 0; + zds->hostageByte = 0; + zds->expectedOutBuffer = *output; + /* fall-through */ + + case zdss_loadHeader : + DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip)); + { size_t const hSize = ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format); + if (zds->refMultipleDDicts && zds->ddictSet) { + ZSTD_DCtx_selectFrameDDict(zds); + } + DEBUGLOG(5, "header size : %u", (U32)hSize); + if (ZSTD_isError(hSize)) { + return hSize; /* error */ + } + if (hSize != 0) { /* need more input */ + size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */ + size_t const remainingInput = (size_t)(iend-ip); + assert(iend >= ip); + if (toLoad > remainingInput) { /* not enough input to load full header */ + if (remainingInput > 0) { + ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, remainingInput); + zds->lhSize += remainingInput; + } + input->pos = input->size; + return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ + } + assert(ip != NULL); + ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; + break; + } } + + /* check for single-pass mode opportunity */ + if (zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN + && zds->fParams.frameType != ZSTD_skippableFrame + && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) { + size_t const cSize = ZSTD_findFrameCompressedSize(istart, (size_t)(iend-istart)); + if (cSize <= (size_t)(iend-istart)) { + /* shortcut : using single-pass mode */ + size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, (size_t)(oend-op), istart, cSize, ZSTD_getDDict(zds)); + if (ZSTD_isError(decompressedSize)) return decompressedSize; + DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()") + ip = istart + cSize; + op += decompressedSize; + zds->expected = 0; + zds->streamStage = zdss_init; + someMoreWork = 0; + break; + } } + + /* Check output buffer is large enough for ZSTD_odm_stable. */ + if (zds->outBufferMode == ZSTD_bm_stable + && zds->fParams.frameType != ZSTD_skippableFrame + && zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN + && (U64)(size_t)(oend-op) < zds->fParams.frameContentSize) { + RETURN_ERROR(dstSize_tooSmall, "ZSTD_obm_stable passed but ZSTD_outBuffer is too small"); + } + + /* Consume header (see ZSTDds_decodeFrameHeader) */ + DEBUGLOG(4, "Consume header"); + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds)), ""); + + if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE); + zds->stage = ZSTDds_skipFrame; + } else { + FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize), ""); + zds->expected = ZSTD_blockHeaderSize; + zds->stage = ZSTDds_decodeBlockHeader; + } + + /* control buffer memory usage */ + DEBUGLOG(4, "Control max memory usage (%u KB <= max %u KB)", + (U32)(zds->fParams.windowSize >>10), + (U32)(zds->maxWindowSize >> 10) ); + zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); + RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize, + frameParameter_windowTooLarge, ""); + + /* Adapt buffer sizes to frame header instructions */ + { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */); + size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_bm_buffered + ? ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize) + : 0; + + ZSTD_DCtx_updateOversizedDuration(zds, neededInBuffSize, neededOutBuffSize); + + { int const tooSmall = (zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize); + int const tooLarge = ZSTD_DCtx_isOversizedTooLong(zds); + + if (tooSmall || tooLarge) { + size_t const bufferSize = neededInBuffSize + neededOutBuffSize; + DEBUGLOG(4, "inBuff : from %u to %u", + (U32)zds->inBuffSize, (U32)neededInBuffSize); + DEBUGLOG(4, "outBuff : from %u to %u", + (U32)zds->outBuffSize, (U32)neededOutBuffSize); + if (zds->staticSize) { /* static DCtx */ + DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize); + assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */ + RETURN_ERROR_IF( + bufferSize > zds->staticSize - sizeof(ZSTD_DCtx), + memory_allocation, ""); + } else { + ZSTD_customFree(zds->inBuff, zds->customMem); + zds->inBuffSize = 0; + zds->outBuffSize = 0; + zds->inBuff = (char*)ZSTD_customMalloc(bufferSize, zds->customMem); + RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation, ""); + } + zds->inBuffSize = neededInBuffSize; + zds->outBuff = zds->inBuff + zds->inBuffSize; + zds->outBuffSize = neededOutBuffSize; + } } } + zds->streamStage = zdss_read; + /* fall-through */ + + case zdss_read: + DEBUGLOG(5, "stage zdss_read"); + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip)); + DEBUGLOG(5, "neededInSize = %u", (U32)neededInSize); + if (neededInSize==0) { /* end of frame */ + zds->streamStage = zdss_init; + someMoreWork = 0; + break; + } + if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ + FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, ip, neededInSize), ""); + ip += neededInSize; + /* Function modifies the stage so we must break */ + break; + } } + if (ip==iend) { someMoreWork = 0; break; } /* no more input */ + zds->streamStage = zdss_load; + /* fall-through */ + + case zdss_load: + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds); + size_t const toLoad = neededInSize - zds->inPos; + int const isSkipFrame = ZSTD_isSkipFrame(zds); + size_t loadedSize; + /* At this point we shouldn't be decompressing a block that we can stream. */ + assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, iend - ip)); + if (isSkipFrame) { + loadedSize = MIN(toLoad, (size_t)(iend-ip)); + } else { + RETURN_ERROR_IF(toLoad > zds->inBuffSize - zds->inPos, + corruption_detected, + "should never happen"); + loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, (size_t)(iend-ip)); + } + ip += loadedSize; + zds->inPos += loadedSize; + if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ + + /* decode loaded input */ + zds->inPos = 0; /* input is consumed */ + FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, zds->inBuff, neededInSize), ""); + /* Function modifies the stage so we must break */ + break; + } + case zdss_flush: + { size_t const toFlushSize = zds->outEnd - zds->outStart; + size_t const flushedSize = ZSTD_limitCopy(op, (size_t)(oend-op), zds->outBuff + zds->outStart, toFlushSize); + op += flushedSize; + zds->outStart += flushedSize; + if (flushedSize == toFlushSize) { /* flush completed */ + zds->streamStage = zdss_read; + if ( (zds->outBuffSize < zds->fParams.frameContentSize) + && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) { + DEBUGLOG(5, "restart filling outBuff from beginning (left:%i, needed:%u)", + (int)(zds->outBuffSize - zds->outStart), + (U32)zds->fParams.blockSizeMax); + zds->outStart = zds->outEnd = 0; + } + break; + } } + /* cannot complete flush */ + someMoreWork = 0; + break; + + default: + assert(0); /* impossible */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */ + } } + + /* result */ + input->pos = (size_t)(ip - (const char*)(input->src)); + output->pos = (size_t)(op - (char*)(output->dst)); + + /* Update the expected output buffer for ZSTD_obm_stable. */ + zds->expectedOutBuffer = *output; + + if ((ip==istart) && (op==ostart)) { /* no forward progress */ + zds->noForwardProgress ++; + if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) { + RETURN_ERROR_IF(op==oend, dstSize_tooSmall, ""); + RETURN_ERROR_IF(ip==iend, srcSize_wrong, ""); + assert(0); + } + } else { + zds->noForwardProgress = 0; + } + { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds); + if (!nextSrcSizeHint) { /* frame fully decoded */ + if (zds->outEnd == zds->outStart) { /* output fully flushed */ + if (zds->hostageByte) { + if (input->pos >= input->size) { + /* can't release hostage (not present) */ + zds->streamStage = zdss_read; + return 1; + } + input->pos++; /* release hostage */ + } /* zds->hostageByte */ + return 0; + } /* zds->outEnd == zds->outStart */ + if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */ + input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */ + zds->hostageByte=1; + } + return 1; + } /* nextSrcSizeHint==0 */ + nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds) == ZSTDnit_block); /* preload header of next block */ + assert(zds->inPos <= nextSrcSizeHint); + nextSrcSizeHint -= zds->inPos; /* part already loaded*/ + return nextSrcSizeHint; + } +} + +size_t ZSTD_decompressStream_simpleArgs ( + ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos) +{ + ZSTD_outBuffer output = { dst, dstCapacity, *dstPos }; + ZSTD_inBuffer input = { src, srcSize, *srcPos }; + /* ZSTD_compress_generic() will check validity of dstPos and srcPos */ + size_t const cErr = ZSTD_decompressStream(dctx, &output, &input); + *dstPos = output.pos; + *srcPos = input.pos; + return cErr; +} diff --git a/lib/zstd/decompress/zstd_decompress_block.c b/lib/zstd/decompress/zstd_decompress_block.c new file mode 100644 index 000000000000..cd6eba55a21c --- /dev/null +++ b/lib/zstd/decompress/zstd_decompress_block.c @@ -0,0 +1,1540 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* zstd_decompress_block : + * this module takes care of decompressing _compressed_ block */ + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ +#include "../common/compiler.h" /* prefetch */ +#include "../common/cpu.h" /* bmi2 */ +#include "../common/mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "../common/fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "../common/huf.h" +#include "../common/zstd_internal.h" +#include "zstd_decompress_internal.h" /* ZSTD_DCtx */ +#include "zstd_ddict.h" /* ZSTD_DDictDictContent */ +#include "zstd_decompress_block.h" + +/*_******************************************************* +* Macros +**********************************************************/ + +/* These two optional macros force the use one way or another of the two + * ZSTD_decompressSequences implementations. You can't force in both directions + * at the same time. + */ +#if defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) +#error "Cannot force the use of the short and the long ZSTD_decompressSequences variants!" +#endif + + +/*_******************************************************* +* Memory operations +**********************************************************/ +static void ZSTD_copy4(void* dst, const void* src) { ZSTD_memcpy(dst, src, 4); } + + +/*-************************************************************* + * Block decoding + ***************************************************************/ + +/*! ZSTD_getcBlockSize() : + * Provides the size of compressed block from block header `src` */ +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, + blockProperties_t* bpPtr) +{ + RETURN_ERROR_IF(srcSize < ZSTD_blockHeaderSize, srcSize_wrong, ""); + + { U32 const cBlockHeader = MEM_readLE24(src); + U32 const cSize = cBlockHeader >> 3; + bpPtr->lastBlock = cBlockHeader & 1; + bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); + bpPtr->origSize = cSize; /* only useful for RLE */ + if (bpPtr->blockType == bt_rle) return 1; + RETURN_ERROR_IF(bpPtr->blockType == bt_reserved, corruption_detected, ""); + return cSize; + } +} + + +/* Hidden declaration for fullbench */ +size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, + const void* src, size_t srcSize); +/*! ZSTD_decodeLiteralsBlock() : + * @return : nb of bytes read from src (< srcSize ) + * note : symbol not declared but exposed for fullbench */ +size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, + const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */ +{ + DEBUGLOG(5, "ZSTD_decodeLiteralsBlock"); + RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, ""); + + { const BYTE* const istart = (const BYTE*) src; + symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); + + switch(litEncType) + { + case set_repeat: + DEBUGLOG(5, "set_repeat flag : re-using stats from previous compressed literals block"); + RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted, ""); + /* fall-through */ + + case set_compressed: + RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3"); + { size_t lhSize, litSize, litCSize; + U32 singleStream=0; + U32 const lhlCode = (istart[0] >> 2) & 3; + U32 const lhc = MEM_readLE32(istart); + size_t hufSuccess; + switch(lhlCode) + { + case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ + /* 2 - 2 - 10 - 10 */ + singleStream = !lhlCode; + lhSize = 3; + litSize = (lhc >> 4) & 0x3FF; + litCSize = (lhc >> 14) & 0x3FF; + break; + case 2: + /* 2 - 2 - 14 - 14 */ + lhSize = 4; + litSize = (lhc >> 4) & 0x3FFF; + litCSize = lhc >> 18; + break; + case 3: + /* 2 - 2 - 18 - 18 */ + lhSize = 5; + litSize = (lhc >> 4) & 0x3FFFF; + litCSize = (lhc >> 22) + ((size_t)istart[4] << 10); + break; + } + RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); + RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, ""); + + /* prefetch huffman table if cold */ + if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) { + PREFETCH_AREA(dctx->HUFptr, sizeof(dctx->entropy.hufTable)); + } + + if (litEncType==set_repeat) { + if (singleStream) { + hufSuccess = HUF_decompress1X_usingDTable_bmi2( + dctx->litBuffer, litSize, istart+lhSize, litCSize, + dctx->HUFptr, dctx->bmi2); + } else { + hufSuccess = HUF_decompress4X_usingDTable_bmi2( + dctx->litBuffer, litSize, istart+lhSize, litCSize, + dctx->HUFptr, dctx->bmi2); + } + } else { + if (singleStream) { +#if defined(HUF_FORCE_DECOMPRESS_X2) + hufSuccess = HUF_decompress1X_DCtx_wksp( + dctx->entropy.hufTable, dctx->litBuffer, litSize, + istart+lhSize, litCSize, dctx->workspace, + sizeof(dctx->workspace)); +#else + hufSuccess = HUF_decompress1X1_DCtx_wksp_bmi2( + dctx->entropy.hufTable, dctx->litBuffer, litSize, + istart+lhSize, litCSize, dctx->workspace, + sizeof(dctx->workspace), dctx->bmi2); +#endif + } else { + hufSuccess = HUF_decompress4X_hufOnly_wksp_bmi2( + dctx->entropy.hufTable, dctx->litBuffer, litSize, + istart+lhSize, litCSize, dctx->workspace, + sizeof(dctx->workspace), dctx->bmi2); + } + } + + RETURN_ERROR_IF(HUF_isError(hufSuccess), corruption_detected, ""); + + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + dctx->litEntropy = 1; + if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable; + ZSTD_memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); + return litCSize + lhSize; + } + + case set_basic: + { size_t litSize, lhSize; + U32 const lhlCode = ((istart[0]) >> 2) & 3; + switch(lhlCode) + { + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + litSize = MEM_readLE24(istart) >> 4; + break; + } + + if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ + RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected, ""); + ZSTD_memcpy(dctx->litBuffer, istart+lhSize, litSize); + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + ZSTD_memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); + return lhSize+litSize; + } + /* direct reference into compressed stream */ + dctx->litPtr = istart+lhSize; + dctx->litSize = litSize; + return lhSize+litSize; + } + + case set_rle: + { U32 const lhlCode = ((istart[0]) >> 2) & 3; + size_t litSize, lhSize; + switch(lhlCode) + { + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + litSize = MEM_readLE24(istart) >> 4; + RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4"); + break; + } + RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); + ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH); + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + return lhSize+1; + } + default: + RETURN_ERROR(corruption_detected, "impossible"); + } + } +} + +/* Default FSE distribution tables. + * These are pre-calculated FSE decoding tables using default distributions as defined in specification : + * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#default-distributions + * They were generated programmatically with following method : + * - start from default distributions, present in /lib/common/zstd_internal.h + * - generate tables normally, using ZSTD_buildFSETable() + * - printout the content of tables + * - pretify output, report below, test with fuzzer to ensure it's correct */ + +/* Default FSE distribution table for Literal Lengths */ +static const ZSTD_seqSymbol LL_defaultDTable[(1<tableLog = 0; + DTableH->fastMode = 0; + + cell->nbBits = 0; + cell->nextState = 0; + assert(nbAddBits < 255); + cell->nbAdditionalBits = (BYTE)nbAddBits; + cell->baseValue = baseValue; +} + + +/* ZSTD_buildFSETable() : + * generate FSE decoding table for one symbol (ll, ml or off) + * cannot fail if input is valid => + * all inputs are presumed validated at this stage */ +FORCE_INLINE_TEMPLATE +void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, + const short* normalizedCounter, unsigned maxSymbolValue, + const U32* baseValue, const U32* nbAdditionalBits, + unsigned tableLog, void* wksp, size_t wkspSize) +{ + ZSTD_seqSymbol* const tableDecode = dt+1; + U32 const maxSV1 = maxSymbolValue + 1; + U32 const tableSize = 1 << tableLog; + + U16* symbolNext = (U16*)wksp; + BYTE* spread = (BYTE*)(symbolNext + MaxSeq + 1); + U32 highThreshold = tableSize - 1; + + + /* Sanity Checks */ + assert(maxSymbolValue <= MaxSeq); + assert(tableLog <= MaxFSELog); + assert(wkspSize >= ZSTD_BUILD_FSE_TABLE_WKSP_SIZE); + (void)wkspSize; + /* Init, lay down lowprob symbols */ + { ZSTD_seqSymbol_header DTableH; + DTableH.tableLog = tableLog; + DTableH.fastMode = 1; + { S16 const largeLimit= (S16)(1 << (tableLog-1)); + U32 s; + for (s=0; s= largeLimit) DTableH.fastMode=0; + assert(normalizedCounter[s]>=0); + symbolNext[s] = (U16)normalizedCounter[s]; + } } } + ZSTD_memcpy(dt, &DTableH, sizeof(DTableH)); + } + + /* Spread symbols */ + assert(tableSize <= 512); + /* Specialized symbol spreading for the case when there are + * no low probability (-1 count) symbols. When compressing + * small blocks we avoid low probability symbols to hit this + * case, since header decoding speed matters more. + */ + if (highThreshold == tableSize - 1) { + size_t const tableMask = tableSize-1; + size_t const step = FSE_TABLESTEP(tableSize); + /* First lay down the symbols in order. + * We use a uint64_t to lay down 8 bytes at a time. This reduces branch + * misses since small blocks generally have small table logs, so nearly + * all symbols have counts <= 8. We ensure we have 8 bytes at the end of + * our buffer to handle the over-write. + */ + { + U64 const add = 0x0101010101010101ull; + size_t pos = 0; + U64 sv = 0; + U32 s; + for (s=0; s highThreshold) position = (position + step) & tableMask; /* lowprob area */ + } } + assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { + U32 u; + for (u=0; u max, corruption_detected, ""); + { U32 const symbol = *(const BYTE*)src; + U32 const baseline = baseValue[symbol]; + U32 const nbBits = nbAdditionalBits[symbol]; + ZSTD_buildSeqTable_rle(DTableSpace, baseline, nbBits); + } + *DTablePtr = DTableSpace; + return 1; + case set_basic : + *DTablePtr = defaultTable; + return 0; + case set_repeat: + RETURN_ERROR_IF(!flagRepeatTable, corruption_detected, ""); + /* prefetch FSE table if used */ + if (ddictIsCold && (nbSeq > 24 /* heuristic */)) { + const void* const pStart = *DTablePtr; + size_t const pSize = sizeof(ZSTD_seqSymbol) * (SEQSYMBOL_TABLE_SIZE(maxLog)); + PREFETCH_AREA(pStart, pSize); + } + return 0; + case set_compressed : + { unsigned tableLog; + S16 norm[MaxSeq+1]; + size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize); + RETURN_ERROR_IF(FSE_isError(headerSize), corruption_detected, ""); + RETURN_ERROR_IF(tableLog > maxLog, corruption_detected, ""); + ZSTD_buildFSETable(DTableSpace, norm, max, baseValue, nbAdditionalBits, tableLog, wksp, wkspSize, bmi2); + *DTablePtr = DTableSpace; + return headerSize; + } + default : + assert(0); + RETURN_ERROR(GENERIC, "impossible"); + } +} + +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, + const void* src, size_t srcSize) +{ + const BYTE* const istart = (const BYTE*)src; + const BYTE* const iend = istart + srcSize; + const BYTE* ip = istart; + int nbSeq; + DEBUGLOG(5, "ZSTD_decodeSeqHeaders"); + + /* check */ + RETURN_ERROR_IF(srcSize < MIN_SEQUENCES_SIZE, srcSize_wrong, ""); + + /* SeqHead */ + nbSeq = *ip++; + if (!nbSeq) { + *nbSeqPtr=0; + RETURN_ERROR_IF(srcSize != 1, srcSize_wrong, ""); + return 1; + } + if (nbSeq > 0x7F) { + if (nbSeq == 0xFF) { + RETURN_ERROR_IF(ip+2 > iend, srcSize_wrong, ""); + nbSeq = MEM_readLE16(ip) + LONGNBSEQ; + ip+=2; + } else { + RETURN_ERROR_IF(ip >= iend, srcSize_wrong, ""); + nbSeq = ((nbSeq-0x80)<<8) + *ip++; + } + } + *nbSeqPtr = nbSeq; + + /* FSE table descriptors */ + RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong, ""); /* minimum possible size: 1 byte for symbol encoding types */ + { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); + symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); + symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); + ip++; + + /* Build DTables */ + { size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr, + LLtype, MaxLL, LLFSELog, + ip, iend-ip, + LL_base, LL_bits, + LL_defaultDTable, dctx->fseEntropy, + dctx->ddictIsCold, nbSeq, + dctx->workspace, sizeof(dctx->workspace), + dctx->bmi2); + RETURN_ERROR_IF(ZSTD_isError(llhSize), corruption_detected, "ZSTD_buildSeqTable failed"); + ip += llhSize; + } + + { size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr, + OFtype, MaxOff, OffFSELog, + ip, iend-ip, + OF_base, OF_bits, + OF_defaultDTable, dctx->fseEntropy, + dctx->ddictIsCold, nbSeq, + dctx->workspace, sizeof(dctx->workspace), + dctx->bmi2); + RETURN_ERROR_IF(ZSTD_isError(ofhSize), corruption_detected, "ZSTD_buildSeqTable failed"); + ip += ofhSize; + } + + { size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr, + MLtype, MaxML, MLFSELog, + ip, iend-ip, + ML_base, ML_bits, + ML_defaultDTable, dctx->fseEntropy, + dctx->ddictIsCold, nbSeq, + dctx->workspace, sizeof(dctx->workspace), + dctx->bmi2); + RETURN_ERROR_IF(ZSTD_isError(mlhSize), corruption_detected, "ZSTD_buildSeqTable failed"); + ip += mlhSize; + } + } + + return ip-istart; +} + + +typedef struct { + size_t litLength; + size_t matchLength; + size_t offset; + const BYTE* match; +} seq_t; + +typedef struct { + size_t state; + const ZSTD_seqSymbol* table; +} ZSTD_fseState; + +typedef struct { + BIT_DStream_t DStream; + ZSTD_fseState stateLL; + ZSTD_fseState stateOffb; + ZSTD_fseState stateML; + size_t prevOffset[ZSTD_REP_NUM]; + const BYTE* prefixStart; + const BYTE* dictEnd; + size_t pos; +} seqState_t; + +/*! ZSTD_overlapCopy8() : + * Copies 8 bytes from ip to op and updates op and ip where ip <= op. + * If the offset is < 8 then the offset is spread to at least 8 bytes. + * + * Precondition: *ip <= *op + * Postcondition: *op - *op >= 8 + */ +HINT_INLINE void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) { + assert(*ip <= *op); + if (offset < 8) { + /* close range match, overlap */ + static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ + static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ + int const sub2 = dec64table[offset]; + (*op)[0] = (*ip)[0]; + (*op)[1] = (*ip)[1]; + (*op)[2] = (*ip)[2]; + (*op)[3] = (*ip)[3]; + *ip += dec32table[offset]; + ZSTD_copy4(*op+4, *ip); + *ip -= sub2; + } else { + ZSTD_copy8(*op, *ip); + } + *ip += 8; + *op += 8; + assert(*op - *ip >= 8); +} + +/*! ZSTD_safecopy() : + * Specialized version of memcpy() that is allowed to READ up to WILDCOPY_OVERLENGTH past the input buffer + * and write up to 16 bytes past oend_w (op >= oend_w is allowed). + * This function is only called in the uncommon case where the sequence is near the end of the block. It + * should be fast for a single long sequence, but can be slow for several short sequences. + * + * @param ovtype controls the overlap detection + * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart. + * - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart. + * The src buffer must be before the dst buffer. + */ +static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) { + ptrdiff_t const diff = op - ip; + BYTE* const oend = op + length; + + assert((ovtype == ZSTD_no_overlap && (diff <= -8 || diff >= 8 || op >= oend_w)) || + (ovtype == ZSTD_overlap_src_before_dst && diff >= 0)); + + if (length < 8) { + /* Handle short lengths. */ + while (op < oend) *op++ = *ip++; + return; + } + if (ovtype == ZSTD_overlap_src_before_dst) { + /* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */ + assert(length >= 8); + ZSTD_overlapCopy8(&op, &ip, diff); + assert(op - ip >= 8); + assert(op <= oend); + } + + if (oend <= oend_w) { + /* No risk of overwrite. */ + ZSTD_wildcopy(op, ip, length, ovtype); + return; + } + if (op <= oend_w) { + /* Wildcopy until we get close to the end. */ + assert(oend > oend_w); + ZSTD_wildcopy(op, ip, oend_w - op, ovtype); + ip += oend_w - op; + op = oend_w; + } + /* Handle the leftovers. */ + while (op < oend) *op++ = *ip++; +} + +/* ZSTD_execSequenceEnd(): + * This version handles cases that are near the end of the output buffer. It requires + * more careful checks to make sure there is no overflow. By separating out these hard + * and unlikely cases, we can speed up the common cases. + * + * NOTE: This function needs to be fast for a single long sequence, but doesn't need + * to be optimized for many small sequences, since those fall into ZSTD_execSequence(). + */ +FORCE_NOINLINE +size_t ZSTD_execSequenceEnd(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; + + /* bounds checks : careful of address space overflow in 32-bit mode */ + RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer"); + RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer"); + assert(op < op + sequenceLength); + assert(oLitEnd < op + sequenceLength); + + /* copy literals */ + ZSTD_safecopy(op, oend_w, *litPtr, sequence.litLength, ZSTD_no_overlap); + op = oLitEnd; + *litPtr = iLitEnd; + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix */ + RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); + match = dictEnd - (prefixStart-match); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } } + ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); + return sequenceLength; +} + +HINT_INLINE +size_t ZSTD_execSequence(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; /* risk : address space underflow on oend=NULL */ + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + assert(op != NULL /* Precondition */); + assert(oend_w < oend /* No underflow */); + /* Handle edge cases in a slow path: + * - Read beyond end of literals + * - Match end is within WILDCOPY_OVERLIMIT of oend + * - 32-bit mode and the match length overflows + */ + if (UNLIKELY( + iLitEnd > litLimit || + oMatchEnd > oend_w || + (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) + return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + + /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ + assert(op <= oLitEnd /* No overflow */); + assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */); + assert(oMatchEnd <= oend /* No underflow */); + assert(iLitEnd <= litLimit /* Literal length is in bounds */); + assert(oLitEnd <= oend_w /* Can wildcopy literals */); + assert(oMatchEnd <= oend_w /* Can wildcopy matches */); + + /* Copy Literals: + * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. + * We likely don't need the full 32-byte wildcopy. + */ + assert(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(op, (*litPtr)); + if (UNLIKELY(sequence.litLength > 16)) { + ZSTD_wildcopy(op+16, (*litPtr)+16, sequence.litLength-16, ZSTD_no_overlap); + } + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* Copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix -> go into extDict */ + RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); + match = dictEnd + (match - prefixStart); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } } + /* Match within prefix of 1 or more bytes */ + assert(op <= oMatchEnd); + assert(oMatchEnd <= oend_w); + assert(match >= prefixStart); + assert(sequence.matchLength >= 1); + + /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy + * without overlap checking. + */ + if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) { + /* We bet on a full wildcopy for matches, since we expect matches to be + * longer than literals (in general). In silesia, ~10% of matches are longer + * than 16 bytes. + */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap); + return sequenceLength; + } + assert(sequence.offset < WILDCOPY_VECLEN); + + /* Copy 8 bytes and spread the offset to be >= 8. */ + ZSTD_overlapCopy8(&op, &match, sequence.offset); + + /* If the match length is > 8 bytes, then continue with the wildcopy. */ + if (sequence.matchLength > 8) { + assert(op < oMatchEnd); + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); + } + return sequenceLength; +} + +static void +ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqSymbol* dt) +{ + const void* ptr = dt; + const ZSTD_seqSymbol_header* const DTableH = (const ZSTD_seqSymbol_header*)ptr; + DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + DEBUGLOG(6, "ZSTD_initFseState : val=%u using %u bits", + (U32)DStatePtr->state, DTableH->tableLog); + BIT_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +FORCE_INLINE_TEMPLATE void +ZSTD_updateFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD) +{ + ZSTD_seqSymbol const DInfo = DStatePtr->table[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = DInfo.nextState + lowBits; +} + +FORCE_INLINE_TEMPLATE void +ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, ZSTD_seqSymbol const DInfo) +{ + U32 const nbBits = DInfo.nbBits; + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = DInfo.nextState + lowBits; +} + +/* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum + * offset bits. But we can only read at most (STREAM_ACCUMULATOR_MIN_32 - 1) + * bits before reloading. This value is the maximum number of bytes we read + * after reloading when we are decoding long offsets. + */ +#define LONG_OFFSETS_MAX_EXTRA_BITS_32 \ + (ZSTD_WINDOWLOG_MAX_32 > STREAM_ACCUMULATOR_MIN_32 \ + ? ZSTD_WINDOWLOG_MAX_32 - STREAM_ACCUMULATOR_MIN_32 \ + : 0) + +typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e; +typedef enum { ZSTD_p_noPrefetch=0, ZSTD_p_prefetch=1 } ZSTD_prefetch_e; + +FORCE_INLINE_TEMPLATE seq_t +ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, const ZSTD_prefetch_e prefetch) +{ + seq_t seq; + ZSTD_seqSymbol const llDInfo = seqState->stateLL.table[seqState->stateLL.state]; + ZSTD_seqSymbol const mlDInfo = seqState->stateML.table[seqState->stateML.state]; + ZSTD_seqSymbol const ofDInfo = seqState->stateOffb.table[seqState->stateOffb.state]; + U32 const llBase = llDInfo.baseValue; + U32 const mlBase = mlDInfo.baseValue; + U32 const ofBase = ofDInfo.baseValue; + BYTE const llBits = llDInfo.nbAdditionalBits; + BYTE const mlBits = mlDInfo.nbAdditionalBits; + BYTE const ofBits = ofDInfo.nbAdditionalBits; + BYTE const totalBits = llBits+mlBits+ofBits; + + /* sequence */ + { size_t offset; + if (ofBits > 1) { + ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); + ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); + assert(ofBits <= MaxOff); + if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) { + U32 const extraBits = ofBits - MIN(ofBits, 32 - seqState->DStream.bitsConsumed); + offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); + BIT_reloadDStream(&seqState->DStream); + if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits); + assert(extraBits <= LONG_OFFSETS_MAX_EXTRA_BITS_32); /* to avoid another reload */ + } else { + offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); + } + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; + } else { + U32 const ll0 = (llBase == 0); + if (LIKELY((ofBits == 0))) { + if (LIKELY(!ll0)) + offset = seqState->prevOffset[0]; + else { + offset = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; + } + } else { + offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1); + { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ + if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset = temp; + } } } + seq.offset = offset; + } + + seq.matchLength = mlBase; + if (mlBits > 0) + seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/); + + if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) + BIT_reloadDStream(&seqState->DStream); + if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) + BIT_reloadDStream(&seqState->DStream); + /* Ensure there are enough bits to read the rest of data in 64-bit mode. */ + ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); + + seq.litLength = llBase; + if (llBits > 0) + seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/); + + if (MEM_32bits()) + BIT_reloadDStream(&seqState->DStream); + + DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u", + (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + + if (prefetch == ZSTD_p_prefetch) { + size_t const pos = seqState->pos + seq.litLength; + const BYTE* const matchBase = (seq.offset > pos) ? seqState->dictEnd : seqState->prefixStart; + seq.match = matchBase + pos - seq.offset; /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted. + * No consequence though : no memory access will occur, offset is only used for prefetching */ + seqState->pos = pos + seq.matchLength; + } + + /* ANS state update + * gcc-9.0.0 does 2.5% worse with ZSTD_updateFseStateWithDInfo(). + * clang-9.2.0 does 7% worse with ZSTD_updateFseState(). + * Naturally it seems like ZSTD_updateFseStateWithDInfo() should be the + * better option, so it is the default for other compilers. But, if you + * measure that it is worse, please put up a pull request. + */ + { +#if !defined(__clang__) + const int kUseUpdateFseState = 1; +#else + const int kUseUpdateFseState = 0; +#endif + if (kUseUpdateFseState) { + ZSTD_updateFseState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ + ZSTD_updateFseState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ + ZSTD_updateFseState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ + } else { + ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llDInfo); /* <= 9 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlDInfo); /* <= 9 bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofDInfo); /* <= 8 bits */ + } + } + + return seq; +} + +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION +MEM_STATIC int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefixStart, BYTE const* oLitEnd) +{ + size_t const windowSize = dctx->fParams.windowSize; + /* No dictionary used. */ + if (dctx->dictContentEndForFuzzing == NULL) return 0; + /* Dictionary is our prefix. */ + if (prefixStart == dctx->dictContentBeginForFuzzing) return 1; + /* Dictionary is not our ext-dict. */ + if (dctx->dictEnd != dctx->dictContentEndForFuzzing) return 0; + /* Dictionary is not within our window size. */ + if ((size_t)(oLitEnd - prefixStart) >= windowSize) return 0; + /* Dictionary is active. */ + return 1; +} + +MEM_STATIC void ZSTD_assertValidSequence( + ZSTD_DCtx const* dctx, + BYTE const* op, BYTE const* oend, + seq_t const seq, + BYTE const* prefixStart, BYTE const* virtualStart) +{ +#if DEBUGLEVEL >= 1 + size_t const windowSize = dctx->fParams.windowSize; + size_t const sequenceSize = seq.litLength + seq.matchLength; + BYTE const* const oLitEnd = op + seq.litLength; + DEBUGLOG(6, "Checking sequence: litL=%u matchL=%u offset=%u", + (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + assert(op <= oend); + assert((size_t)(oend - op) >= sequenceSize); + assert(sequenceSize <= ZSTD_BLOCKSIZE_MAX); + if (ZSTD_dictionaryIsActive(dctx, prefixStart, oLitEnd)) { + size_t const dictSize = (size_t)((char const*)dctx->dictContentEndForFuzzing - (char const*)dctx->dictContentBeginForFuzzing); + /* Offset must be within the dictionary. */ + assert(seq.offset <= (size_t)(oLitEnd - virtualStart)); + assert(seq.offset <= windowSize + dictSize); + } else { + /* Offset must be within our window. */ + assert(seq.offset <= windowSize); + } +#else + (void)dctx, (void)op, (void)oend, (void)seq, (void)prefixStart, (void)virtualStart; +#endif +} +#endif + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG +FORCE_INLINE_TEMPLATE size_t +DONT_VECTORIZE +ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + maxDstSize; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); + const BYTE* const vBase = (const BYTE*) (dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + DEBUGLOG(5, "ZSTD_decompressSequences_body"); + (void)frame; + + /* Regen sequences */ + if (nbSeq) { + seqState_t seqState; + size_t error = 0; + dctx->fseEntropy = 1; + { U32 i; for (i=0; ientropy.rep[i]; } + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)), + corruption_detected, ""); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + assert(dst != NULL); + + ZSTD_STATIC_ASSERT( + BIT_DStream_unfinished < BIT_DStream_completed && + BIT_DStream_endOfBuffer < BIT_DStream_completed && + BIT_DStream_completed < BIT_DStream_overflow); + +#if defined(__x86_64__) + /* Align the decompression loop to 32 + 16 bytes. + * + * zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression + * speed swings based on the alignment of the decompression loop. This + * performance swing is caused by parts of the decompression loop falling + * out of the DSB. The entire decompression loop should fit in the DSB, + * when it can't we get much worse performance. You can measure if you've + * hit the good case or the bad case with this perf command for some + * compressed file test.zst: + * + * perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \ + * -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst + * + * If you see most cycles served out of the MITE you've hit the bad case. + * If you see most cycles served out of the DSB you've hit the good case. + * If it is pretty even then you may be in an okay case. + * + * I've been able to reproduce this issue on the following CPUs: + * - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9 + * Use Instruments->Counters to get DSB/MITE cycles. + * I never got performance swings, but I was able to + * go from the good case of mostly DSB to half of the + * cycles served from MITE. + * - Coffeelake: Intel i9-9900k + * + * I haven't been able to reproduce the instability or DSB misses on any + * of the following CPUS: + * - Haswell + * - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH + * - Skylake + * + * If you are seeing performance stability this script can help test. + * It tests on 4 commits in zstd where I saw performance change. + * + * https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4 + */ + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 4"); +#endif + for ( ; ; ) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, ZSTD_p_noPrefetch); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + BIT_reloadDStream(&(seqState.DStream)); + op += oneSeqSize; + /* gcc and clang both don't like early returns in this loop. + * Instead break and check for an error at the end of the loop. + */ + if (UNLIKELY(ZSTD_isError(oneSeqSize))) { + error = oneSeqSize; + break; + } + if (UNLIKELY(!--nbSeq)) break; + } + + /* check if reached exact end */ + DEBUGLOG(5, "ZSTD_decompressSequences_body: after decode loop, remaining nbSeq : %i", nbSeq); + if (ZSTD_isError(error)) return error; + RETURN_ERROR_IF(nbSeq, corruption_detected, ""); + RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected, ""); + /* save reps for next block */ + { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + { size_t const lastLLSize = litEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } + } + + return op-ostart; +} + +static size_t +ZSTD_decompressSequences_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT +FORCE_INLINE_TEMPLATE size_t +ZSTD_decompressSequencesLong_body( + ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + maxDstSize; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); + const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + (void)frame; + + /* Regen sequences */ + if (nbSeq) { +#define STORED_SEQS 4 +#define STORED_SEQS_MASK (STORED_SEQS-1) +#define ADVANCED_SEQS 4 + seq_t sequences[STORED_SEQS]; + int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS); + seqState_t seqState; + int seqNb; + dctx->fseEntropy = 1; + { int i; for (i=0; ientropy.rep[i]; } + seqState.prefixStart = prefixStart; + seqState.pos = (size_t)(op-prefixStart); + seqState.dictEnd = dictEnd; + assert(dst != NULL); + assert(iend >= ip); + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)), + corruption_detected, ""); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + + /* prepare in advance */ + for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && (seqNbentropy.rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + { size_t const lastLLSize = litEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } + } + + return op-ostart; +} + +static size_t +ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + + + +#if DYNAMIC_BMI2 + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG +static TARGET_ATTRIBUTE("bmi2") size_t +DONT_VECTORIZE +ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT +static TARGET_ATTRIBUTE("bmi2") size_t +ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + +#endif /* DYNAMIC_BMI2 */ + +typedef size_t (*ZSTD_decompressSequences_t)( + ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame); + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG +static size_t +ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + DEBUGLOG(5, "ZSTD_decompressSequences"); +#if DYNAMIC_BMI2 + if (dctx->bmi2) { + return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + } +#endif + return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT +/* ZSTD_decompressSequencesLong() : + * decompression function triggered when a minimum share of offsets is considered "long", + * aka out of cache. + * note : "long" definition seems overloaded here, sometimes meaning "wider than bitstream register", and sometimes meaning "farther than memory cache distance". + * This function will try to mitigate main memory latency through the use of prefetching */ +static size_t +ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + DEBUGLOG(5, "ZSTD_decompressSequencesLong"); +#if DYNAMIC_BMI2 + if (dctx->bmi2) { + return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + } +#endif + return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + + + +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) +/* ZSTD_getLongOffsetsShare() : + * condition : offTable must be valid + * @return : "share" of long offsets (arbitrarily defined as > (1<<23)) + * compared to maximum possible of (1< 22) total += 1; + } + + assert(tableLog <= OffFSELog); + total <<= (OffFSELog - tableLog); /* scale to OffFSELog */ + + return total; +} +#endif + +size_t +ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, const int frame) +{ /* blockType == blockCompressed */ + const BYTE* ip = (const BYTE*)src; + /* isLongOffset must be true if there are long offsets. + * Offsets are long if they are larger than 2^STREAM_ACCUMULATOR_MIN. + * We don't expect that to be the case in 64-bit mode. + * In block mode, window size is not known, so we have to be conservative. + * (note: but it could be evaluated from current-lowLimit) + */ + ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame || (dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN)))); + DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize); + + RETURN_ERROR_IF(srcSize >= ZSTD_BLOCKSIZE_MAX, srcSize_wrong, ""); + + /* Decode literals section */ + { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize); + DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : %u", (U32)litCSize); + if (ZSTD_isError(litCSize)) return litCSize; + ip += litCSize; + srcSize -= litCSize; + } + + /* Build Decoding Tables */ + { + /* These macros control at build-time which decompressor implementation + * we use. If neither is defined, we do some inspection and dispatch at + * runtime. + */ +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) + int usePrefetchDecoder = dctx->ddictIsCold; +#endif + int nbSeq; + size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize); + if (ZSTD_isError(seqHSize)) return seqHSize; + ip += seqHSize; + srcSize -= seqHSize; + + RETURN_ERROR_IF(dst == NULL && nbSeq > 0, dstSize_tooSmall, "NULL not handled"); + +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) + if ( !usePrefetchDecoder + && (!frame || (dctx->fParams.windowSize > (1<<24))) + && (nbSeq>ADVANCED_SEQS) ) { /* could probably use a larger nbSeq limit */ + U32 const shareLongOffsets = ZSTD_getLongOffsetsShare(dctx->OFTptr); + U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */ + usePrefetchDecoder = (shareLongOffsets >= minShare); + } +#endif + + dctx->ddictIsCold = 0; + +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) + if (usePrefetchDecoder) +#endif +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT + return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); +#endif + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG + /* else */ + return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); +#endif + } +} + + +void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize) +{ + if (dst != dctx->previousDstEnd && dstSize > 0) { /* not contiguous */ + dctx->dictEnd = dctx->previousDstEnd; + dctx->virtualStart = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); + dctx->prefixStart = dst; + dctx->previousDstEnd = dst; + } +} + + +size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t dSize; + ZSTD_checkContinuity(dctx, dst, dstCapacity); + dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 0); + dctx->previousDstEnd = (char*)dst + dSize; + return dSize; +} diff --git a/lib/zstd/decompress/zstd_decompress_block.h b/lib/zstd/decompress/zstd_decompress_block.h new file mode 100644 index 000000000000..e7f5f6689459 --- /dev/null +++ b/lib/zstd/decompress/zstd_decompress_block.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +#ifndef ZSTD_DEC_BLOCK_H +#define ZSTD_DEC_BLOCK_H + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include "../common/zstd_deps.h" /* size_t */ +#include /* DCtx, and some public functions */ +#include "../common/zstd_internal.h" /* blockProperties_t, and some public functions */ +#include "zstd_decompress_internal.h" /* ZSTD_seqSymbol */ + + +/* === Prototypes === */ + +/* note: prototypes already published within `zstd.h` : + * ZSTD_decompressBlock() + */ + +/* note: prototypes already published within `zstd_internal.h` : + * ZSTD_getcBlockSize() + * ZSTD_decodeSeqHeaders() + */ + + +/* ZSTD_decompressBlock_internal() : + * decompress block, starting at `src`, + * into destination buffer `dst`. + * @return : decompressed block size, + * or an error code (which can be tested using ZSTD_isError()) + */ +size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, const int frame); + +/* ZSTD_buildFSETable() : + * generate FSE decoding table for one symbol (ll, ml or off) + * this function must be called with valid parameters only + * (dt is large enough, normalizedCounter distribution total is a power of 2, max is within range, etc.) + * in which case it cannot fail. + * The workspace must be 4-byte aligned and at least ZSTD_BUILD_FSE_TABLE_WKSP_SIZE bytes, which is + * defined in zstd_decompress_internal.h. + * Internal use only. + */ +void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, + const short* normalizedCounter, unsigned maxSymbolValue, + const U32* baseValue, const U32* nbAdditionalBits, + unsigned tableLog, void* wksp, size_t wkspSize, + int bmi2); + + +#endif /* ZSTD_DEC_BLOCK_H */ diff --git a/lib/zstd/decompress/zstd_decompress_internal.h b/lib/zstd/decompress/zstd_decompress_internal.h new file mode 100644 index 000000000000..4b9052f68755 --- /dev/null +++ b/lib/zstd/decompress/zstd_decompress_internal.h @@ -0,0 +1,202 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* zstd_decompress_internal: + * objects and definitions shared within lib/decompress modules */ + + #ifndef ZSTD_DECOMPRESS_INTERNAL_H + #define ZSTD_DECOMPRESS_INTERNAL_H + + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include "../common/mem.h" /* BYTE, U16, U32 */ +#include "../common/zstd_internal.h" /* ZSTD_seqSymbol */ + + + +/*-******************************************************* + * Constants + *********************************************************/ +static UNUSED_ATTR const U32 LL_base[MaxLL+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, + 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, + 0x2000, 0x4000, 0x8000, 0x10000 }; + +static UNUSED_ATTR const U32 OF_base[MaxOff+1] = { + 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, + 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, + 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, + 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD }; + +static UNUSED_ATTR const U32 OF_bits[MaxOff+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31 }; + +static UNUSED_ATTR const U32 ML_base[MaxML+1] = { + 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, + 35, 37, 39, 41, 43, 47, 51, 59, + 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, + 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; + + +/*-******************************************************* + * Decompression types + *********************************************************/ + typedef struct { + U32 fastMode; + U32 tableLog; + } ZSTD_seqSymbol_header; + + typedef struct { + U16 nextState; + BYTE nbAdditionalBits; + BYTE nbBits; + U32 baseValue; + } ZSTD_seqSymbol; + + #define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log))) + +#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE (sizeof(S16) * (MaxSeq + 1) + (1u << MaxFSELog) + sizeof(U64)) +#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 ((ZSTD_BUILD_FSE_TABLE_WKSP_SIZE + sizeof(U32) - 1) / sizeof(U32)) + +typedef struct { + ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */ + ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */ + ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */ + HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */ + U32 rep[ZSTD_REP_NUM]; + U32 workspace[ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32]; +} ZSTD_entropyDTables_t; + +typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, + ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, + ZSTDds_decompressLastBlock, ZSTDds_checkChecksum, + ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage; + +typedef enum { zdss_init=0, zdss_loadHeader, + zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage; + +typedef enum { + ZSTD_use_indefinitely = -1, /* Use the dictionary indefinitely */ + ZSTD_dont_use = 0, /* Do not use the dictionary (if one exists free it) */ + ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */ +} ZSTD_dictUses_e; + +/* Hashset for storing references to multiple ZSTD_DDict within ZSTD_DCtx */ +typedef struct { + const ZSTD_DDict** ddictPtrTable; + size_t ddictPtrTableSize; + size_t ddictPtrCount; +} ZSTD_DDictHashSet; + +struct ZSTD_DCtx_s +{ + const ZSTD_seqSymbol* LLTptr; + const ZSTD_seqSymbol* MLTptr; + const ZSTD_seqSymbol* OFTptr; + const HUF_DTable* HUFptr; + ZSTD_entropyDTables_t entropy; + U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */ + const void* previousDstEnd; /* detect continuity */ + const void* prefixStart; /* start of current segment */ + const void* virtualStart; /* virtual start of previous segment if it was just before current one */ + const void* dictEnd; /* end of previous segment */ + size_t expected; + ZSTD_frameHeader fParams; + U64 processedCSize; + U64 decodedSize; + blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */ + ZSTD_dStage stage; + U32 litEntropy; + U32 fseEntropy; + struct xxh64_state xxhState; + size_t headerSize; + ZSTD_format_e format; + ZSTD_forceIgnoreChecksum_e forceIgnoreChecksum; /* User specified: if == 1, will ignore checksums in compressed frame. Default == 0 */ + U32 validateChecksum; /* if == 1, will validate checksum. Is == 1 if (fParams.checksumFlag == 1) and (forceIgnoreChecksum == 0). */ + const BYTE* litPtr; + ZSTD_customMem customMem; + size_t litSize; + size_t rleSize; + size_t staticSize; + int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */ + + /* dictionary */ + ZSTD_DDict* ddictLocal; + const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */ + U32 dictID; + int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */ + ZSTD_dictUses_e dictUses; + ZSTD_DDictHashSet* ddictSet; /* Hash set for multiple ddicts */ + ZSTD_refMultipleDDicts_e refMultipleDDicts; /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */ + + /* streaming */ + ZSTD_dStreamStage streamStage; + char* inBuff; + size_t inBuffSize; + size_t inPos; + size_t maxWindowSize; + char* outBuff; + size_t outBuffSize; + size_t outStart; + size_t outEnd; + size_t lhSize; + void* legacyContext; + U32 previousLegacyVersion; + U32 legacyVersion; + U32 hostageByte; + int noForwardProgress; + ZSTD_bufferMode_e outBufferMode; + ZSTD_outBuffer expectedOutBuffer; + + /* workspace */ + BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH]; + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + + size_t oversizedDuration; + +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + void const* dictContentBeginForFuzzing; + void const* dictContentEndForFuzzing; +#endif + + /* Tracing */ +}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ + + +/*-******************************************************* + * Shared internal functions + *********************************************************/ + +/*! ZSTD_loadDEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * @return : size of dictionary header (size of magic number + dict ID + entropy tables) */ +size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, + const void* const dict, size_t const dictSize); + +/*! ZSTD_checkContinuity() : + * check if next `dst` follows previous position, where decompression ended. + * If yes, do nothing (continue on current segment). + * If not, classify previous segment as "external dictionary", and start a new segment. + * This function cannot fail. */ +void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize); + + +#endif /* ZSTD_DECOMPRESS_INTERNAL_H */ diff --git a/lib/zstd/decompress_sources.h b/lib/zstd/decompress_sources.h new file mode 100644 index 000000000000..f35bef03eb22 --- /dev/null +++ b/lib/zstd/decompress_sources.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* + * This file includes every .c file needed for decompression. + * It is used by lib/decompress_unzstd.c to include the decompression + * source into the translation-unit, so it can be used for kernel + * decompression. + */ + +#include "common/debug.c" +#include "common/entropy_common.c" +#include "common/error_private.c" +#include "common/fse_decompress.c" +#include "common/zstd_common.c" +#include "decompress/huf_decompress.c" +#include "decompress/zstd_ddict.c" +#include "decompress/zstd_decompress.c" +#include "decompress/zstd_decompress_block.c" +#include "zstd_decompress_module.c" diff --git a/lib/zstd/entropy_common.c b/lib/zstd/entropy_common.c deleted file mode 100644 index 2b0a643c32c4..000000000000 --- a/lib/zstd/entropy_common.c +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Common functions of New Generation Entropy library - * Copyright (C) 2016, Yann Collet. - * - * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - * - * You can contact the author at : - * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy - */ - -/* ************************************* -* Dependencies -***************************************/ -#include "error_private.h" /* ERR_*, ERROR */ -#include "fse.h" -#include "huf.h" -#include "mem.h" - -/*=== Version ===*/ -unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; } - -/*=== Error Management ===*/ -unsigned FSE_isError(size_t code) { return ERR_isError(code); } - -unsigned HUF_isError(size_t code) { return ERR_isError(code); } - -/*-************************************************************** -* FSE NCount encoding-decoding -****************************************************************/ -size_t FSE_readNCount(short *normalizedCounter, unsigned *maxSVPtr, unsigned *tableLogPtr, const void *headerBuffer, size_t hbSize) -{ - const BYTE *const istart = (const BYTE *)headerBuffer; - const BYTE *const iend = istart + hbSize; - const BYTE *ip = istart; - int nbBits; - int remaining; - int threshold; - U32 bitStream; - int bitCount; - unsigned charnum = 0; - int previous0 = 0; - - if (hbSize < 4) - return ERROR(srcSize_wrong); - bitStream = ZSTD_readLE32(ip); - nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ - if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) - return ERROR(tableLog_tooLarge); - bitStream >>= 4; - bitCount = 4; - *tableLogPtr = nbBits; - remaining = (1 << nbBits) + 1; - threshold = 1 << nbBits; - nbBits++; - - while ((remaining > 1) & (charnum <= *maxSVPtr)) { - if (previous0) { - unsigned n0 = charnum; - while ((bitStream & 0xFFFF) == 0xFFFF) { - n0 += 24; - if (ip < iend - 5) { - ip += 2; - bitStream = ZSTD_readLE32(ip) >> bitCount; - } else { - bitStream >>= 16; - bitCount += 16; - } - } - while ((bitStream & 3) == 3) { - n0 += 3; - bitStream >>= 2; - bitCount += 2; - } - n0 += bitStream & 3; - bitCount += 2; - if (n0 > *maxSVPtr) - return ERROR(maxSymbolValue_tooSmall); - while (charnum < n0) - normalizedCounter[charnum++] = 0; - if ((ip <= iend - 7) || (ip + (bitCount >> 3) <= iend - 4)) { - ip += bitCount >> 3; - bitCount &= 7; - bitStream = ZSTD_readLE32(ip) >> bitCount; - } else { - bitStream >>= 2; - } - } - { - int const max = (2 * threshold - 1) - remaining; - int count; - - if ((bitStream & (threshold - 1)) < (U32)max) { - count = bitStream & (threshold - 1); - bitCount += nbBits - 1; - } else { - count = bitStream & (2 * threshold - 1); - if (count >= threshold) - count -= max; - bitCount += nbBits; - } - - count--; /* extra accuracy */ - remaining -= count < 0 ? -count : count; /* -1 means +1 */ - normalizedCounter[charnum++] = (short)count; - previous0 = !count; - while (remaining < threshold) { - nbBits--; - threshold >>= 1; - } - - if ((ip <= iend - 7) || (ip + (bitCount >> 3) <= iend - 4)) { - ip += bitCount >> 3; - bitCount &= 7; - } else { - bitCount -= (int)(8 * (iend - 4 - ip)); - ip = iend - 4; - } - bitStream = ZSTD_readLE32(ip) >> (bitCount & 31); - } - } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */ - if (remaining != 1) - return ERROR(corruption_detected); - if (bitCount > 32) - return ERROR(corruption_detected); - *maxSVPtr = charnum - 1; - - ip += (bitCount + 7) >> 3; - return ip - istart; -} - -/*! HUF_readStats() : - Read compact Huffman tree, saved by HUF_writeCTable(). - `huffWeight` is destination buffer. - `rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32. - @return : size read from `src` , or an error Code . - Note : Needed by HUF_readCTable() and HUF_readDTableX?() . -*/ -size_t HUF_readStats_wksp(BYTE *huffWeight, size_t hwSize, U32 *rankStats, U32 *nbSymbolsPtr, U32 *tableLogPtr, const void *src, size_t srcSize, void *workspace, size_t workspaceSize) -{ - U32 weightTotal; - const BYTE *ip = (const BYTE *)src; - size_t iSize; - size_t oSize; - - if (!srcSize) - return ERROR(srcSize_wrong); - iSize = ip[0]; - /* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */ - - if (iSize >= 128) { /* special header */ - oSize = iSize - 127; - iSize = ((oSize + 1) / 2); - if (iSize + 1 > srcSize) - return ERROR(srcSize_wrong); - if (oSize >= hwSize) - return ERROR(corruption_detected); - ip += 1; - { - U32 n; - for (n = 0; n < oSize; n += 2) { - huffWeight[n] = ip[n / 2] >> 4; - huffWeight[n + 1] = ip[n / 2] & 15; - } - } - } else { /* header compressed with FSE (normal case) */ - if (iSize + 1 > srcSize) - return ERROR(srcSize_wrong); - oSize = FSE_decompress_wksp(huffWeight, hwSize - 1, ip + 1, iSize, 6, workspace, workspaceSize); /* max (hwSize-1) values decoded, as last one is implied */ - if (FSE_isError(oSize)) - return oSize; - } - - /* collect weight stats */ - memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32)); - weightTotal = 0; - { - U32 n; - for (n = 0; n < oSize; n++) { - if (huffWeight[n] >= HUF_TABLELOG_MAX) - return ERROR(corruption_detected); - rankStats[huffWeight[n]]++; - weightTotal += (1 << huffWeight[n]) >> 1; - } - } - if (weightTotal == 0) - return ERROR(corruption_detected); - - /* get last non-null symbol weight (implied, total must be 2^n) */ - { - U32 const tableLog = BIT_highbit32(weightTotal) + 1; - if (tableLog > HUF_TABLELOG_MAX) - return ERROR(corruption_detected); - *tableLogPtr = tableLog; - /* determine last weight */ - { - U32 const total = 1 << tableLog; - U32 const rest = total - weightTotal; - U32 const verif = 1 << BIT_highbit32(rest); - U32 const lastWeight = BIT_highbit32(rest) + 1; - if (verif != rest) - return ERROR(corruption_detected); /* last value must be a clean power of 2 */ - huffWeight[oSize] = (BYTE)lastWeight; - rankStats[lastWeight]++; - } - } - - /* check tree construction validity */ - if ((rankStats[1] < 2) || (rankStats[1] & 1)) - return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ - - /* results */ - *nbSymbolsPtr = (U32)(oSize + 1); - return iSize + 1; -} diff --git a/lib/zstd/error_private.h b/lib/zstd/error_private.h deleted file mode 100644 index 1a60b31f706c..000000000000 --- a/lib/zstd/error_private.h +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of https://github.com/facebook/zstd. - * An additional grant of patent rights can be found in the PATENTS file in the - * same directory. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - */ - -/* Note : this module is expected to remain private, do not expose it */ - -#ifndef ERROR_H_MODULE -#define ERROR_H_MODULE - -/* **************************************** -* Dependencies -******************************************/ -#include /* size_t */ -#include /* enum list */ - -/* **************************************** -* Compiler-specific -******************************************/ -#define ERR_STATIC static __attribute__((unused)) - -/*-**************************************** -* Customization (error_public.h) -******************************************/ -typedef ZSTD_ErrorCode ERR_enum; -#define PREFIX(name) ZSTD_error_##name - -/*-**************************************** -* Error codes handling -******************************************/ -#define ERROR(name) ((size_t)-PREFIX(name)) - -ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } - -ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) -{ - if (!ERR_isError(code)) - return (ERR_enum)0; - return (ERR_enum)(0 - code); -} - -#endif /* ERROR_H_MODULE */ diff --git a/lib/zstd/fse.h b/lib/zstd/fse.h deleted file mode 100644 index 7460ab04b191..000000000000 --- a/lib/zstd/fse.h +++ /dev/null @@ -1,575 +0,0 @@ -/* - * FSE : Finite State Entropy codec - * Public Prototypes declaration - * Copyright (C) 2013-2016, Yann Collet. - * - * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - * - * You can contact the author at : - * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy - */ -#ifndef FSE_H -#define FSE_H - -/*-***************************************** -* Dependencies -******************************************/ -#include /* size_t, ptrdiff_t */ - -/*-***************************************** -* FSE_PUBLIC_API : control library symbols visibility -******************************************/ -#define FSE_PUBLIC_API - -/*------ Version ------*/ -#define FSE_VERSION_MAJOR 0 -#define FSE_VERSION_MINOR 9 -#define FSE_VERSION_RELEASE 0 - -#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE -#define FSE_QUOTE(str) #str -#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str) -#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION) - -#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR * 100 * 100 + FSE_VERSION_MINOR * 100 + FSE_VERSION_RELEASE) -FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */ - -/*-***************************************** -* Tool functions -******************************************/ -FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */ - -/* Error Management */ -FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */ - -/*-***************************************** -* FSE detailed API -******************************************/ -/*! -FSE_compress() does the following: -1. count symbol occurrence from source[] into table count[] -2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog) -3. save normalized counters to memory buffer using writeNCount() -4. build encoding table 'CTable' from normalized counters -5. encode the data stream using encoding table 'CTable' - -FSE_decompress() does the following: -1. read normalized counters with readNCount() -2. build decoding table 'DTable' from normalized counters -3. decode the data stream using decoding table 'DTable' - -The following API allows targeting specific sub-functions for advanced tasks. -For example, it's possible to compress several blocks using the same 'CTable', -or to save and provide normalized distribution using external method. -*/ - -/* *** COMPRESSION *** */ -/*! FSE_optimalTableLog(): - dynamically downsize 'tableLog' when conditions are met. - It saves CPU time, by using smaller tables, while preserving or even improving compression ratio. - @return : recommended tableLog (necessarily <= 'maxTableLog') */ -FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); - -/*! FSE_normalizeCount(): - normalize counts so that sum(count[]) == Power_of_2 (2^tableLog) - 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). - @return : tableLog, - or an errorCode, which can be tested using FSE_isError() */ -FSE_PUBLIC_API size_t FSE_normalizeCount(short *normalizedCounter, unsigned tableLog, const unsigned *count, size_t srcSize, unsigned maxSymbolValue); - -/*! FSE_NCountWriteBound(): - Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'. - Typically useful for allocation purpose. */ -FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog); - -/*! FSE_writeNCount(): - Compactly save 'normalizedCounter' into 'buffer'. - @return : size of the compressed table, - or an errorCode, which can be tested using FSE_isError(). */ -FSE_PUBLIC_API size_t FSE_writeNCount(void *buffer, size_t bufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); - -/*! Constructor and Destructor of FSE_CTable. - Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ -typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ - -/*! FSE_compress_usingCTable(): - Compress `src` using `ct` into `dst` which must be already allocated. - @return : size of compressed data (<= `dstCapacity`), - or 0 if compressed data could not fit into `dst`, - or an errorCode, which can be tested using FSE_isError() */ -FSE_PUBLIC_API size_t FSE_compress_usingCTable(void *dst, size_t dstCapacity, const void *src, size_t srcSize, const FSE_CTable *ct); - -/*! -Tutorial : ----------- -The first step is to count all symbols. FSE_count() does this job very fast. -Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells. -'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0] -maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value) -FSE_count() will return the number of occurrence of the most frequent symbol. -This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility. -If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). - -The next step is to normalize the frequencies. -FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'. -It also guarantees a minimum of 1 to any Symbol with frequency >= 1. -You can use 'tableLog'==0 to mean "use default tableLog value". -If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(), -which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default"). - -The result of FSE_normalizeCount() will be saved into a table, -called 'normalizedCounter', which is a table of signed short. -'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells. -The return value is tableLog if everything proceeded as expected. -It is 0 if there is a single symbol within distribution. -If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()). - -'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount(). -'buffer' must be already allocated. -For guaranteed success, buffer size must be at least FSE_headerBound(). -The result of the function is the number of bytes written into 'buffer'. -If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small). - -'normalizedCounter' can then be used to create the compression table 'CTable'. -The space required by 'CTable' must be already allocated, using FSE_createCTable(). -You can then use FSE_buildCTable() to fill 'CTable'. -If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()). - -'CTable' can then be used to compress 'src', with FSE_compress_usingCTable(). -Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize' -The function returns the size of compressed data (without header), necessarily <= `dstCapacity`. -If it returns '0', compressed data could not fit into 'dst'. -If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). -*/ - -/* *** DECOMPRESSION *** */ - -/*! FSE_readNCount(): - Read compactly saved 'normalizedCounter' from 'rBuffer'. - @return : size read from 'rBuffer', - or an errorCode, which can be tested using FSE_isError(). - maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ -FSE_PUBLIC_API size_t FSE_readNCount(short *normalizedCounter, unsigned *maxSymbolValuePtr, unsigned *tableLogPtr, const void *rBuffer, size_t rBuffSize); - -/*! Constructor and Destructor of FSE_DTable. - Note that its size depends on 'tableLog' */ -typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ - -/*! FSE_buildDTable(): - Builds 'dt', which must be already allocated, using FSE_createDTable(). - return : 0, or an errorCode, which can be tested using FSE_isError() */ -FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable *dt, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workspace, size_t workspaceSize); - -/*! FSE_decompress_usingDTable(): - Decompress compressed source `cSrc` of size `cSrcSize` using `dt` - into `dst` which must be already allocated. - @return : size of regenerated data (necessarily <= `dstCapacity`), - or an errorCode, which can be tested using FSE_isError() */ -FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt); - -/*! -Tutorial : ----------- -(Note : these functions only decompress FSE-compressed blocks. - If block is uncompressed, use memcpy() instead - If block is a single repeated byte, use memset() instead ) - -The first step is to obtain the normalized frequencies of symbols. -This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount(). -'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short. -In practice, that means it's necessary to know 'maxSymbolValue' beforehand, -or size the table to handle worst case situations (typically 256). -FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'. -The result of FSE_readNCount() is the number of bytes read from 'rBuffer'. -Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that. -If there is an error, the function will return an error code, which can be tested using FSE_isError(). - -The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'. -This is performed by the function FSE_buildDTable(). -The space required by 'FSE_DTable' must be already allocated using FSE_createDTable(). -If there is an error, the function will return an error code, which can be tested using FSE_isError(). - -`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable(). -`cSrcSize` must be strictly correct, otherwise decompression will fail. -FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`). -If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small) -*/ - -/* *** Dependency *** */ -#include "bitstream.h" - -/* ***************************************** -* Static allocation -*******************************************/ -/* FSE buffer bounds */ -#define FSE_NCOUNTBOUND 512 -#define FSE_BLOCKBOUND(size) (size + (size >> 7)) -#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ - -/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */ -#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1 << (maxTableLog - 1)) + ((maxSymbolValue + 1) * 2)) -#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1 << maxTableLog)) - -/* ***************************************** -* FSE advanced API -*******************************************/ -/* FSE_count_wksp() : - * Same as FSE_count(), but using an externally provided scratch buffer. - * `workSpace` size must be table of >= `1024` unsigned - */ -size_t FSE_count_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned *workSpace); - -/* FSE_countFast_wksp() : - * Same as FSE_countFast(), but using an externally provided scratch buffer. - * `workSpace` must be a table of minimum `1024` unsigned - */ -size_t FSE_countFast_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *src, size_t srcSize, unsigned *workSpace); - -/*! FSE_count_simple - * Same as FSE_countFast(), but does not use any additional memory (not even on stack). - * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` (presuming it's also the size of `count`). -*/ -size_t FSE_count_simple(unsigned *count, unsigned *maxSymbolValuePtr, const void *src, size_t srcSize); - -unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); -/**< same as FSE_optimalTableLog(), which used `minus==2` */ - -size_t FSE_buildCTable_raw(FSE_CTable *ct, unsigned nbBits); -/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */ - -size_t FSE_buildCTable_rle(FSE_CTable *ct, unsigned char symbolValue); -/**< build a fake FSE_CTable, designed to compress always the same symbolValue */ - -/* FSE_buildCTable_wksp() : - * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). - * `wkspSize` must be >= `(1<= BIT_DStream_completed - -When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. -Checking if DStream has reached its end is performed by : - BIT_endOfDStream(&DStream); -Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible. - FSE_endOfDState(&DState); -*/ - -/* ***************************************** -* FSE unsafe API -*******************************************/ -static unsigned char FSE_decodeSymbolFast(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD); -/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ - -/* ***************************************** -* Implementation of inlined functions -*******************************************/ -typedef struct { - int deltaFindState; - U32 deltaNbBits; -} FSE_symbolCompressionTransform; /* total 8 bytes */ - -ZSTD_STATIC void FSE_initCState(FSE_CState_t *statePtr, const FSE_CTable *ct) -{ - const void *ptr = ct; - const U16 *u16ptr = (const U16 *)ptr; - const U32 tableLog = ZSTD_read16(ptr); - statePtr->value = (ptrdiff_t)1 << tableLog; - statePtr->stateTable = u16ptr + 2; - statePtr->symbolTT = ((const U32 *)ct + 1 + (tableLog ? (1 << (tableLog - 1)) : 1)); - statePtr->stateLog = tableLog; -} - -/*! FSE_initCState2() : -* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read) -* uses the smallest state value possible, saving the cost of this symbol */ -ZSTD_STATIC void FSE_initCState2(FSE_CState_t *statePtr, const FSE_CTable *ct, U32 symbol) -{ - FSE_initCState(statePtr, ct); - { - const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform *)(statePtr->symbolTT))[symbol]; - const U16 *stateTable = (const U16 *)(statePtr->stateTable); - U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1 << 15)) >> 16); - statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits; - statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; - } -} - -ZSTD_STATIC void FSE_encodeSymbol(BIT_CStream_t *bitC, FSE_CState_t *statePtr, U32 symbol) -{ - const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform *)(statePtr->symbolTT))[symbol]; - const U16 *const stateTable = (const U16 *)(statePtr->stateTable); - U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); - BIT_addBits(bitC, statePtr->value, nbBitsOut); - statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; -} - -ZSTD_STATIC void FSE_flushCState(BIT_CStream_t *bitC, const FSE_CState_t *statePtr) -{ - BIT_addBits(bitC, statePtr->value, statePtr->stateLog); - BIT_flushBits(bitC); -} - -/* ====== Decompression ====== */ - -typedef struct { - U16 tableLog; - U16 fastMode; -} FSE_DTableHeader; /* sizeof U32 */ - -typedef struct { - unsigned short newState; - unsigned char symbol; - unsigned char nbBits; -} FSE_decode_t; /* size == U32 */ - -ZSTD_STATIC void FSE_initDState(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD, const FSE_DTable *dt) -{ - const void *ptr = dt; - const FSE_DTableHeader *const DTableH = (const FSE_DTableHeader *)ptr; - DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); - BIT_reloadDStream(bitD); - DStatePtr->table = dt + 1; -} - -ZSTD_STATIC BYTE FSE_peekSymbol(const FSE_DState_t *DStatePtr) -{ - FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state]; - return DInfo.symbol; -} - -ZSTD_STATIC void FSE_updateState(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD) -{ - FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state]; - U32 const nbBits = DInfo.nbBits; - size_t const lowBits = BIT_readBits(bitD, nbBits); - DStatePtr->state = DInfo.newState + lowBits; -} - -ZSTD_STATIC BYTE FSE_decodeSymbol(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD) -{ - FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state]; - U32 const nbBits = DInfo.nbBits; - BYTE const symbol = DInfo.symbol; - size_t const lowBits = BIT_readBits(bitD, nbBits); - - DStatePtr->state = DInfo.newState + lowBits; - return symbol; -} - -/*! FSE_decodeSymbolFast() : - unsafe, only works if no symbol has a probability > 50% */ -ZSTD_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD) -{ - FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state]; - U32 const nbBits = DInfo.nbBits; - BYTE const symbol = DInfo.symbol; - size_t const lowBits = BIT_readBitsFast(bitD, nbBits); - - DStatePtr->state = DInfo.newState + lowBits; - return symbol; -} - -ZSTD_STATIC unsigned FSE_endOfDState(const FSE_DState_t *DStatePtr) { return DStatePtr->state == 0; } - -/* ************************************************************** -* Tuning parameters -****************************************************************/ -/*!MEMORY_USAGE : -* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) -* Increasing memory usage improves compression ratio -* Reduced memory usage can improve speed, due to cache effect -* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ -#ifndef FSE_MAX_MEMORY_USAGE -#define FSE_MAX_MEMORY_USAGE 14 -#endif -#ifndef FSE_DEFAULT_MEMORY_USAGE -#define FSE_DEFAULT_MEMORY_USAGE 13 -#endif - -/*!FSE_MAX_SYMBOL_VALUE : -* Maximum symbol value authorized. -* Required for proper stack allocation */ -#ifndef FSE_MAX_SYMBOL_VALUE -#define FSE_MAX_SYMBOL_VALUE 255 -#endif - -/* ************************************************************** -* template functions type & suffix -****************************************************************/ -#define FSE_FUNCTION_TYPE BYTE -#define FSE_FUNCTION_EXTENSION -#define FSE_DECODE_TYPE FSE_decode_t - -/* *************************************************************** -* Constants -*****************************************************************/ -#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE - 2) -#define FSE_MAX_TABLESIZE (1U << FSE_MAX_TABLELOG) -#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE - 1) -#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE - 2) -#define FSE_MIN_TABLELOG 5 - -#define FSE_TABLELOG_ABSOLUTE_MAX 15 -#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX -#error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" -#endif - -#define FSE_TABLESTEP(tableSize) ((tableSize >> 1) + (tableSize >> 3) + 3) - -#endif /* FSE_H */ diff --git a/lib/zstd/fse_compress.c b/lib/zstd/fse_compress.c deleted file mode 100644 index ef3d1741d532..000000000000 --- a/lib/zstd/fse_compress.c +++ /dev/null @@ -1,795 +0,0 @@ -/* - * FSE : Finite State Entropy encoder - * Copyright (C) 2013-2015, Yann Collet. - * - * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - * - * You can contact the author at : - * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy - */ - -/* ************************************************************** -* Compiler specifics -****************************************************************/ -#define FORCE_INLINE static __always_inline - -/* ************************************************************** -* Includes -****************************************************************/ -#include "bitstream.h" -#include "fse.h" -#include -#include -#include -#include /* memcpy, memset */ - -/* ************************************************************** -* Error Management -****************************************************************/ -#define FSE_STATIC_ASSERT(c) \ - { \ - enum { FSE_static_assert = 1 / (int)(!!(c)) }; \ - } /* use only *after* variable declarations */ - -/* ************************************************************** -* Templates -****************************************************************/ -/* - designed to be included - for type-specific functions (template emulation in C) - Objective is to write these functions only once, for improved maintenance -*/ - -/* safety checks */ -#ifndef FSE_FUNCTION_EXTENSION -#error "FSE_FUNCTION_EXTENSION must be defined" -#endif -#ifndef FSE_FUNCTION_TYPE -#error "FSE_FUNCTION_TYPE must be defined" -#endif - -/* Function names */ -#define FSE_CAT(X, Y) X##Y -#define FSE_FUNCTION_NAME(X, Y) FSE_CAT(X, Y) -#define FSE_TYPE_NAME(X, Y) FSE_CAT(X, Y) - -/* Function templates */ - -/* FSE_buildCTable_wksp() : - * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). - * wkspSize should be sized to handle worst case situation, which is `1<> 1 : 1); - FSE_symbolCompressionTransform *const symbolTT = (FSE_symbolCompressionTransform *)(FSCT); - U32 const step = FSE_TABLESTEP(tableSize); - U32 highThreshold = tableSize - 1; - - U32 *cumul; - FSE_FUNCTION_TYPE *tableSymbol; - size_t spaceUsed32 = 0; - - cumul = (U32 *)workspace + spaceUsed32; - spaceUsed32 += FSE_MAX_SYMBOL_VALUE + 2; - tableSymbol = (FSE_FUNCTION_TYPE *)((U32 *)workspace + spaceUsed32); - spaceUsed32 += ALIGN(sizeof(FSE_FUNCTION_TYPE) * ((size_t)1 << tableLog), sizeof(U32)) >> 2; - - if ((spaceUsed32 << 2) > workspaceSize) - return ERROR(tableLog_tooLarge); - workspace = (U32 *)workspace + spaceUsed32; - workspaceSize -= (spaceUsed32 << 2); - - /* CTable header */ - tableU16[-2] = (U16)tableLog; - tableU16[-1] = (U16)maxSymbolValue; - - /* For explanations on how to distribute symbol values over the table : - * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ - - /* symbol start positions */ - { - U32 u; - cumul[0] = 0; - for (u = 1; u <= maxSymbolValue + 1; u++) { - if (normalizedCounter[u - 1] == -1) { /* Low proba symbol */ - cumul[u] = cumul[u - 1] + 1; - tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u - 1); - } else { - cumul[u] = cumul[u - 1] + normalizedCounter[u - 1]; - } - } - cumul[maxSymbolValue + 1] = tableSize + 1; - } - - /* Spread symbols */ - { - U32 position = 0; - U32 symbol; - for (symbol = 0; symbol <= maxSymbolValue; symbol++) { - int nbOccurences; - for (nbOccurences = 0; nbOccurences < normalizedCounter[symbol]; nbOccurences++) { - tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol; - position = (position + step) & tableMask; - while (position > highThreshold) - position = (position + step) & tableMask; /* Low proba area */ - } - } - - if (position != 0) - return ERROR(GENERIC); /* Must have gone through all positions */ - } - - /* Build table */ - { - U32 u; - for (u = 0; u < tableSize; u++) { - FSE_FUNCTION_TYPE s = tableSymbol[u]; /* note : static analyzer may not understand tableSymbol is properly initialized */ - tableU16[cumul[s]++] = (U16)(tableSize + u); /* TableU16 : sorted by symbol order; gives next state value */ - } - } - - /* Build Symbol Transformation Table */ - { - unsigned total = 0; - unsigned s; - for (s = 0; s <= maxSymbolValue; s++) { - switch (normalizedCounter[s]) { - case 0: break; - - case -1: - case 1: - symbolTT[s].deltaNbBits = (tableLog << 16) - (1 << tableLog); - symbolTT[s].deltaFindState = total - 1; - total++; - break; - default: { - U32 const maxBitsOut = tableLog - BIT_highbit32(normalizedCounter[s] - 1); - U32 const minStatePlus = normalizedCounter[s] << maxBitsOut; - symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus; - symbolTT[s].deltaFindState = total - normalizedCounter[s]; - total += normalizedCounter[s]; - } - } - } - } - - return 0; -} - -/*-************************************************************** -* FSE NCount encoding-decoding -****************************************************************/ -size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog) -{ - size_t const maxHeaderSize = (((maxSymbolValue + 1) * tableLog) >> 3) + 3; - return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ -} - -static size_t FSE_writeNCount_generic(void *header, size_t headerBufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, - unsigned writeIsSafe) -{ - BYTE *const ostart = (BYTE *)header; - BYTE *out = ostart; - BYTE *const oend = ostart + headerBufferSize; - int nbBits; - const int tableSize = 1 << tableLog; - int remaining; - int threshold; - U32 bitStream; - int bitCount; - unsigned charnum = 0; - int previous0 = 0; - - bitStream = 0; - bitCount = 0; - /* Table Size */ - bitStream += (tableLog - FSE_MIN_TABLELOG) << bitCount; - bitCount += 4; - - /* Init */ - remaining = tableSize + 1; /* +1 for extra accuracy */ - threshold = tableSize; - nbBits = tableLog + 1; - - while (remaining > 1) { /* stops at 1 */ - if (previous0) { - unsigned start = charnum; - while (!normalizedCounter[charnum]) - charnum++; - while (charnum >= start + 24) { - start += 24; - bitStream += 0xFFFFU << bitCount; - if ((!writeIsSafe) && (out > oend - 2)) - return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE)bitStream; - out[1] = (BYTE)(bitStream >> 8); - out += 2; - bitStream >>= 16; - } - while (charnum >= start + 3) { - start += 3; - bitStream += 3 << bitCount; - bitCount += 2; - } - bitStream += (charnum - start) << bitCount; - bitCount += 2; - if (bitCount > 16) { - if ((!writeIsSafe) && (out > oend - 2)) - return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE)bitStream; - out[1] = (BYTE)(bitStream >> 8); - out += 2; - bitStream >>= 16; - bitCount -= 16; - } - } - { - int count = normalizedCounter[charnum++]; - int const max = (2 * threshold - 1) - remaining; - remaining -= count < 0 ? -count : count; - count++; /* +1 for extra accuracy */ - if (count >= threshold) - count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ - bitStream += count << bitCount; - bitCount += nbBits; - bitCount -= (count < max); - previous0 = (count == 1); - if (remaining < 1) - return ERROR(GENERIC); - while (remaining < threshold) - nbBits--, threshold >>= 1; - } - if (bitCount > 16) { - if ((!writeIsSafe) && (out > oend - 2)) - return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE)bitStream; - out[1] = (BYTE)(bitStream >> 8); - out += 2; - bitStream >>= 16; - bitCount -= 16; - } - } - - /* flush remaining bitStream */ - if ((!writeIsSafe) && (out > oend - 2)) - return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE)bitStream; - out[1] = (BYTE)(bitStream >> 8); - out += (bitCount + 7) / 8; - - if (charnum > maxSymbolValue + 1) - return ERROR(GENERIC); - - return (out - ostart); -} - -size_t FSE_writeNCount(void *buffer, size_t bufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) -{ - if (tableLog > FSE_MAX_TABLELOG) - return ERROR(tableLog_tooLarge); /* Unsupported */ - if (tableLog < FSE_MIN_TABLELOG) - return ERROR(GENERIC); /* Unsupported */ - - if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) - return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); - - return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1); -} - -/*-************************************************************** -* Counting histogram -****************************************************************/ -/*! FSE_count_simple - This function counts byte values within `src`, and store the histogram into table `count`. - It doesn't use any additional memory. - But this function is unsafe : it doesn't check that all values within `src` can fit into `count`. - For this reason, prefer using a table `count` with 256 elements. - @return : count of most numerous element -*/ -size_t FSE_count_simple(unsigned *count, unsigned *maxSymbolValuePtr, const void *src, size_t srcSize) -{ - const BYTE *ip = (const BYTE *)src; - const BYTE *const end = ip + srcSize; - unsigned maxSymbolValue = *maxSymbolValuePtr; - unsigned max = 0; - - memset(count, 0, (maxSymbolValue + 1) * sizeof(*count)); - if (srcSize == 0) { - *maxSymbolValuePtr = 0; - return 0; - } - - while (ip < end) - count[*ip++]++; - - while (!count[maxSymbolValue]) - maxSymbolValue--; - *maxSymbolValuePtr = maxSymbolValue; - - { - U32 s; - for (s = 0; s <= maxSymbolValue; s++) - if (count[s] > max) - max = count[s]; - } - - return (size_t)max; -} - -/* FSE_count_parallel_wksp() : - * Same as FSE_count_parallel(), but using an externally provided scratch buffer. - * `workSpace` size must be a minimum of `1024 * sizeof(unsigned)`` */ -static size_t FSE_count_parallel_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned checkMax, - unsigned *const workSpace) -{ - const BYTE *ip = (const BYTE *)source; - const BYTE *const iend = ip + sourceSize; - unsigned maxSymbolValue = *maxSymbolValuePtr; - unsigned max = 0; - U32 *const Counting1 = workSpace; - U32 *const Counting2 = Counting1 + 256; - U32 *const Counting3 = Counting2 + 256; - U32 *const Counting4 = Counting3 + 256; - - memset(Counting1, 0, 4 * 256 * sizeof(unsigned)); - - /* safety checks */ - if (!sourceSize) { - memset(count, 0, maxSymbolValue + 1); - *maxSymbolValuePtr = 0; - return 0; - } - if (!maxSymbolValue) - maxSymbolValue = 255; /* 0 == default */ - - /* by stripes of 16 bytes */ - { - U32 cached = ZSTD_read32(ip); - ip += 4; - while (ip < iend - 15) { - U32 c = cached; - cached = ZSTD_read32(ip); - ip += 4; - Counting1[(BYTE)c]++; - Counting2[(BYTE)(c >> 8)]++; - Counting3[(BYTE)(c >> 16)]++; - Counting4[c >> 24]++; - c = cached; - cached = ZSTD_read32(ip); - ip += 4; - Counting1[(BYTE)c]++; - Counting2[(BYTE)(c >> 8)]++; - Counting3[(BYTE)(c >> 16)]++; - Counting4[c >> 24]++; - c = cached; - cached = ZSTD_read32(ip); - ip += 4; - Counting1[(BYTE)c]++; - Counting2[(BYTE)(c >> 8)]++; - Counting3[(BYTE)(c >> 16)]++; - Counting4[c >> 24]++; - c = cached; - cached = ZSTD_read32(ip); - ip += 4; - Counting1[(BYTE)c]++; - Counting2[(BYTE)(c >> 8)]++; - Counting3[(BYTE)(c >> 16)]++; - Counting4[c >> 24]++; - } - ip -= 4; - } - - /* finish last symbols */ - while (ip < iend) - Counting1[*ip++]++; - - if (checkMax) { /* verify stats will fit into destination table */ - U32 s; - for (s = 255; s > maxSymbolValue; s--) { - Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s]; - if (Counting1[s]) - return ERROR(maxSymbolValue_tooSmall); - } - } - - { - U32 s; - for (s = 0; s <= maxSymbolValue; s++) { - count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s]; - if (count[s] > max) - max = count[s]; - } - } - - while (!count[maxSymbolValue]) - maxSymbolValue--; - *maxSymbolValuePtr = maxSymbolValue; - return (size_t)max; -} - -/* FSE_countFast_wksp() : - * Same as FSE_countFast(), but using an externally provided scratch buffer. - * `workSpace` size must be table of >= `1024` unsigned */ -size_t FSE_countFast_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned *workSpace) -{ - if (sourceSize < 1500) - return FSE_count_simple(count, maxSymbolValuePtr, source, sourceSize); - return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 0, workSpace); -} - -/* FSE_count_wksp() : - * Same as FSE_count(), but using an externally provided scratch buffer. - * `workSpace` size must be table of >= `1024` unsigned */ -size_t FSE_count_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned *workSpace) -{ - if (*maxSymbolValuePtr < 255) - return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 1, workSpace); - *maxSymbolValuePtr = 255; - return FSE_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace); -} - -/*-************************************************************** -* FSE Compression Code -****************************************************************/ -/*! FSE_sizeof_CTable() : - FSE_CTable is a variable size structure which contains : - `U16 tableLog;` - `U16 maxSymbolValue;` - `U16 nextStateNumber[1 << tableLog];` // This size is variable - `FSE_symbolCompressionTransform symbolTT[maxSymbolValue+1];` // This size is variable -Allocation is manual (C standard does not support variable-size structures). -*/ -size_t FSE_sizeof_CTable(unsigned maxSymbolValue, unsigned tableLog) -{ - if (tableLog > FSE_MAX_TABLELOG) - return ERROR(tableLog_tooLarge); - return FSE_CTABLE_SIZE_U32(tableLog, maxSymbolValue) * sizeof(U32); -} - -/* provides the minimum logSize to safely represent a distribution */ -static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) -{ - U32 minBitsSrc = BIT_highbit32((U32)(srcSize - 1)) + 1; - U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2; - U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; - return minBits; -} - -unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus) -{ - U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus; - U32 tableLog = maxTableLog; - U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); - if (tableLog == 0) - tableLog = FSE_DEFAULT_TABLELOG; - if (maxBitsSrc < tableLog) - tableLog = maxBitsSrc; /* Accuracy can be reduced */ - if (minBits > tableLog) - tableLog = minBits; /* Need a minimum to safely represent all symbol values */ - if (tableLog < FSE_MIN_TABLELOG) - tableLog = FSE_MIN_TABLELOG; - if (tableLog > FSE_MAX_TABLELOG) - tableLog = FSE_MAX_TABLELOG; - return tableLog; -} - -unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) -{ - return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2); -} - -/* Secondary normalization method. - To be used when primary method fails. */ - -static size_t FSE_normalizeM2(short *norm, U32 tableLog, const unsigned *count, size_t total, U32 maxSymbolValue) -{ - short const NOT_YET_ASSIGNED = -2; - U32 s; - U32 distributed = 0; - U32 ToDistribute; - - /* Init */ - U32 const lowThreshold = (U32)(total >> tableLog); - U32 lowOne = (U32)((total * 3) >> (tableLog + 1)); - - for (s = 0; s <= maxSymbolValue; s++) { - if (count[s] == 0) { - norm[s] = 0; - continue; - } - if (count[s] <= lowThreshold) { - norm[s] = -1; - distributed++; - total -= count[s]; - continue; - } - if (count[s] <= lowOne) { - norm[s] = 1; - distributed++; - total -= count[s]; - continue; - } - - norm[s] = NOT_YET_ASSIGNED; - } - ToDistribute = (1 << tableLog) - distributed; - - if ((total / ToDistribute) > lowOne) { - /* risk of rounding to zero */ - lowOne = (U32)((total * 3) / (ToDistribute * 2)); - for (s = 0; s <= maxSymbolValue; s++) { - if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) { - norm[s] = 1; - distributed++; - total -= count[s]; - continue; - } - } - ToDistribute = (1 << tableLog) - distributed; - } - - if (distributed == maxSymbolValue + 1) { - /* all values are pretty poor; - probably incompressible data (should have already been detected); - find max, then give all remaining points to max */ - U32 maxV = 0, maxC = 0; - for (s = 0; s <= maxSymbolValue; s++) - if (count[s] > maxC) - maxV = s, maxC = count[s]; - norm[maxV] += (short)ToDistribute; - return 0; - } - - if (total == 0) { - /* all of the symbols were low enough for the lowOne or lowThreshold */ - for (s = 0; ToDistribute > 0; s = (s + 1) % (maxSymbolValue + 1)) - if (norm[s] > 0) - ToDistribute--, norm[s]++; - return 0; - } - - { - U64 const vStepLog = 62 - tableLog; - U64 const mid = (1ULL << (vStepLog - 1)) - 1; - U64 const rStep = div_u64((((U64)1 << vStepLog) * ToDistribute) + mid, (U32)total); /* scale on remaining */ - U64 tmpTotal = mid; - for (s = 0; s <= maxSymbolValue; s++) { - if (norm[s] == NOT_YET_ASSIGNED) { - U64 const end = tmpTotal + (count[s] * rStep); - U32 const sStart = (U32)(tmpTotal >> vStepLog); - U32 const sEnd = (U32)(end >> vStepLog); - U32 const weight = sEnd - sStart; - if (weight < 1) - return ERROR(GENERIC); - norm[s] = (short)weight; - tmpTotal = end; - } - } - } - - return 0; -} - -size_t FSE_normalizeCount(short *normalizedCounter, unsigned tableLog, const unsigned *count, size_t total, unsigned maxSymbolValue) -{ - /* Sanity checks */ - if (tableLog == 0) - tableLog = FSE_DEFAULT_TABLELOG; - if (tableLog < FSE_MIN_TABLELOG) - return ERROR(GENERIC); /* Unsupported size */ - if (tableLog > FSE_MAX_TABLELOG) - return ERROR(tableLog_tooLarge); /* Unsupported size */ - if (tableLog < FSE_minTableLog(total, maxSymbolValue)) - return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */ - - { - U32 const rtbTable[] = {0, 473195, 504333, 520860, 550000, 700000, 750000, 830000}; - U64 const scale = 62 - tableLog; - U64 const step = div_u64((U64)1 << 62, (U32)total); /* <== here, one division ! */ - U64 const vStep = 1ULL << (scale - 20); - int stillToDistribute = 1 << tableLog; - unsigned s; - unsigned largest = 0; - short largestP = 0; - U32 lowThreshold = (U32)(total >> tableLog); - - for (s = 0; s <= maxSymbolValue; s++) { - if (count[s] == total) - return 0; /* rle special case */ - if (count[s] == 0) { - normalizedCounter[s] = 0; - continue; - } - if (count[s] <= lowThreshold) { - normalizedCounter[s] = -1; - stillToDistribute--; - } else { - short proba = (short)((count[s] * step) >> scale); - if (proba < 8) { - U64 restToBeat = vStep * rtbTable[proba]; - proba += (count[s] * step) - ((U64)proba << scale) > restToBeat; - } - if (proba > largestP) - largestP = proba, largest = s; - normalizedCounter[s] = proba; - stillToDistribute -= proba; - } - } - if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) { - /* corner case, need another normalization method */ - size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue); - if (FSE_isError(errorCode)) - return errorCode; - } else - normalizedCounter[largest] += (short)stillToDistribute; - } - - return tableLog; -} - -/* fake FSE_CTable, for raw (uncompressed) input */ -size_t FSE_buildCTable_raw(FSE_CTable *ct, unsigned nbBits) -{ - const unsigned tableSize = 1 << nbBits; - const unsigned tableMask = tableSize - 1; - const unsigned maxSymbolValue = tableMask; - void *const ptr = ct; - U16 *const tableU16 = ((U16 *)ptr) + 2; - void *const FSCT = ((U32 *)ptr) + 1 /* header */ + (tableSize >> 1); /* assumption : tableLog >= 1 */ - FSE_symbolCompressionTransform *const symbolTT = (FSE_symbolCompressionTransform *)(FSCT); - unsigned s; - - /* Sanity checks */ - if (nbBits < 1) - return ERROR(GENERIC); /* min size */ - - /* header */ - tableU16[-2] = (U16)nbBits; - tableU16[-1] = (U16)maxSymbolValue; - - /* Build table */ - for (s = 0; s < tableSize; s++) - tableU16[s] = (U16)(tableSize + s); - - /* Build Symbol Transformation Table */ - { - const U32 deltaNbBits = (nbBits << 16) - (1 << nbBits); - for (s = 0; s <= maxSymbolValue; s++) { - symbolTT[s].deltaNbBits = deltaNbBits; - symbolTT[s].deltaFindState = s - 1; - } - } - - return 0; -} - -/* fake FSE_CTable, for rle input (always same symbol) */ -size_t FSE_buildCTable_rle(FSE_CTable *ct, BYTE symbolValue) -{ - void *ptr = ct; - U16 *tableU16 = ((U16 *)ptr) + 2; - void *FSCTptr = (U32 *)ptr + 2; - FSE_symbolCompressionTransform *symbolTT = (FSE_symbolCompressionTransform *)FSCTptr; - - /* header */ - tableU16[-2] = (U16)0; - tableU16[-1] = (U16)symbolValue; - - /* Build table */ - tableU16[0] = 0; - tableU16[1] = 0; /* just in case */ - - /* Build Symbol Transformation Table */ - symbolTT[symbolValue].deltaNbBits = 0; - symbolTT[symbolValue].deltaFindState = 0; - - return 0; -} - -static size_t FSE_compress_usingCTable_generic(void *dst, size_t dstSize, const void *src, size_t srcSize, const FSE_CTable *ct, const unsigned fast) -{ - const BYTE *const istart = (const BYTE *)src; - const BYTE *const iend = istart + srcSize; - const BYTE *ip = iend; - - BIT_CStream_t bitC; - FSE_CState_t CState1, CState2; - - /* init */ - if (srcSize <= 2) - return 0; - { - size_t const initError = BIT_initCStream(&bitC, dst, dstSize); - if (FSE_isError(initError)) - return 0; /* not enough space available to write a bitstream */ - } - -#define FSE_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s)) - - if (srcSize & 1) { - FSE_initCState2(&CState1, ct, *--ip); - FSE_initCState2(&CState2, ct, *--ip); - FSE_encodeSymbol(&bitC, &CState1, *--ip); - FSE_FLUSHBITS(&bitC); - } else { - FSE_initCState2(&CState2, ct, *--ip); - FSE_initCState2(&CState1, ct, *--ip); - } - - /* join to mod 4 */ - srcSize -= 2; - if ((sizeof(bitC.bitContainer) * 8 > FSE_MAX_TABLELOG * 4 + 7) && (srcSize & 2)) { /* test bit 2 */ - FSE_encodeSymbol(&bitC, &CState2, *--ip); - FSE_encodeSymbol(&bitC, &CState1, *--ip); - FSE_FLUSHBITS(&bitC); - } - - /* 2 or 4 encoding per loop */ - while (ip > istart) { - - FSE_encodeSymbol(&bitC, &CState2, *--ip); - - if (sizeof(bitC.bitContainer) * 8 < FSE_MAX_TABLELOG * 2 + 7) /* this test must be static */ - FSE_FLUSHBITS(&bitC); - - FSE_encodeSymbol(&bitC, &CState1, *--ip); - - if (sizeof(bitC.bitContainer) * 8 > FSE_MAX_TABLELOG * 4 + 7) { /* this test must be static */ - FSE_encodeSymbol(&bitC, &CState2, *--ip); - FSE_encodeSymbol(&bitC, &CState1, *--ip); - } - - FSE_FLUSHBITS(&bitC); - } - - FSE_flushCState(&bitC, &CState2); - FSE_flushCState(&bitC, &CState1); - return BIT_closeCStream(&bitC); -} - -size_t FSE_compress_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const FSE_CTable *ct) -{ - unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize)); - - if (fast) - return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1); - else - return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0); -} - -size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); } diff --git a/lib/zstd/fse_decompress.c b/lib/zstd/fse_decompress.c deleted file mode 100644 index 0b353530fb3f..000000000000 --- a/lib/zstd/fse_decompress.c +++ /dev/null @@ -1,325 +0,0 @@ -/* - * FSE : Finite State Entropy decoder - * Copyright (C) 2013-2015, Yann Collet. - * - * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - * - * You can contact the author at : - * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy - */ - -/* ************************************************************** -* Compiler specifics -****************************************************************/ -#define FORCE_INLINE static __always_inline - -/* ************************************************************** -* Includes -****************************************************************/ -#include "bitstream.h" -#include "fse.h" -#include "zstd_internal.h" -#include -#include -#include /* memcpy, memset */ - -/* ************************************************************** -* Error Management -****************************************************************/ -#define FSE_isError ERR_isError -#define FSE_STATIC_ASSERT(c) \ - { \ - enum { FSE_static_assert = 1 / (int)(!!(c)) }; \ - } /* use only *after* variable declarations */ - -/* ************************************************************** -* Templates -****************************************************************/ -/* - designed to be included - for type-specific functions (template emulation in C) - Objective is to write these functions only once, for improved maintenance -*/ - -/* safety checks */ -#ifndef FSE_FUNCTION_EXTENSION -#error "FSE_FUNCTION_EXTENSION must be defined" -#endif -#ifndef FSE_FUNCTION_TYPE -#error "FSE_FUNCTION_TYPE must be defined" -#endif - -/* Function names */ -#define FSE_CAT(X, Y) X##Y -#define FSE_FUNCTION_NAME(X, Y) FSE_CAT(X, Y) -#define FSE_TYPE_NAME(X, Y) FSE_CAT(X, Y) - -/* Function templates */ - -size_t FSE_buildDTable_wksp(FSE_DTable *dt, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workspace, size_t workspaceSize) -{ - void *const tdPtr = dt + 1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ - FSE_DECODE_TYPE *const tableDecode = (FSE_DECODE_TYPE *)(tdPtr); - U16 *symbolNext = (U16 *)workspace; - - U32 const maxSV1 = maxSymbolValue + 1; - U32 const tableSize = 1 << tableLog; - U32 highThreshold = tableSize - 1; - - /* Sanity Checks */ - if (workspaceSize < sizeof(U16) * (FSE_MAX_SYMBOL_VALUE + 1)) - return ERROR(tableLog_tooLarge); - if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) - return ERROR(maxSymbolValue_tooLarge); - if (tableLog > FSE_MAX_TABLELOG) - return ERROR(tableLog_tooLarge); - - /* Init, lay down lowprob symbols */ - { - FSE_DTableHeader DTableH; - DTableH.tableLog = (U16)tableLog; - DTableH.fastMode = 1; - { - S16 const largeLimit = (S16)(1 << (tableLog - 1)); - U32 s; - for (s = 0; s < maxSV1; s++) { - if (normalizedCounter[s] == -1) { - tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s; - symbolNext[s] = 1; - } else { - if (normalizedCounter[s] >= largeLimit) - DTableH.fastMode = 0; - symbolNext[s] = normalizedCounter[s]; - } - } - } - memcpy(dt, &DTableH, sizeof(DTableH)); - } - - /* Spread symbols */ - { - U32 const tableMask = tableSize - 1; - U32 const step = FSE_TABLESTEP(tableSize); - U32 s, position = 0; - for (s = 0; s < maxSV1; s++) { - int i; - for (i = 0; i < normalizedCounter[s]; i++) { - tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s; - position = (position + step) & tableMask; - while (position > highThreshold) - position = (position + step) & tableMask; /* lowprob area */ - } - } - if (position != 0) - return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ - } - - /* Build Decoding table */ - { - U32 u; - for (u = 0; u < tableSize; u++) { - FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol); - U16 nextState = symbolNext[symbol]++; - tableDecode[u].nbBits = (BYTE)(tableLog - BIT_highbit32((U32)nextState)); - tableDecode[u].newState = (U16)((nextState << tableDecode[u].nbBits) - tableSize); - } - } - - return 0; -} - -/*-******************************************************* -* Decompression (Byte symbols) -*********************************************************/ -size_t FSE_buildDTable_rle(FSE_DTable *dt, BYTE symbolValue) -{ - void *ptr = dt; - FSE_DTableHeader *const DTableH = (FSE_DTableHeader *)ptr; - void *dPtr = dt + 1; - FSE_decode_t *const cell = (FSE_decode_t *)dPtr; - - DTableH->tableLog = 0; - DTableH->fastMode = 0; - - cell->newState = 0; - cell->symbol = symbolValue; - cell->nbBits = 0; - - return 0; -} - -size_t FSE_buildDTable_raw(FSE_DTable *dt, unsigned nbBits) -{ - void *ptr = dt; - FSE_DTableHeader *const DTableH = (FSE_DTableHeader *)ptr; - void *dPtr = dt + 1; - FSE_decode_t *const dinfo = (FSE_decode_t *)dPtr; - const unsigned tableSize = 1 << nbBits; - const unsigned tableMask = tableSize - 1; - const unsigned maxSV1 = tableMask + 1; - unsigned s; - - /* Sanity checks */ - if (nbBits < 1) - return ERROR(GENERIC); /* min size */ - - /* Build Decoding Table */ - DTableH->tableLog = (U16)nbBits; - DTableH->fastMode = 1; - for (s = 0; s < maxSV1; s++) { - dinfo[s].newState = 0; - dinfo[s].symbol = (BYTE)s; - dinfo[s].nbBits = (BYTE)nbBits; - } - - return 0; -} - -FORCE_INLINE size_t FSE_decompress_usingDTable_generic(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt, - const unsigned fast) -{ - BYTE *const ostart = (BYTE *)dst; - BYTE *op = ostart; - BYTE *const omax = op + maxDstSize; - BYTE *const olimit = omax - 3; - - BIT_DStream_t bitD; - FSE_DState_t state1; - FSE_DState_t state2; - - /* Init */ - CHECK_F(BIT_initDStream(&bitD, cSrc, cSrcSize)); - - FSE_initDState(&state1, &bitD, dt); - FSE_initDState(&state2, &bitD, dt); - -#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD) - - /* 4 symbols per loop */ - for (; (BIT_reloadDStream(&bitD) == BIT_DStream_unfinished) & (op < olimit); op += 4) { - op[0] = FSE_GETSYMBOL(&state1); - - if (FSE_MAX_TABLELOG * 2 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */ - BIT_reloadDStream(&bitD); - - op[1] = FSE_GETSYMBOL(&state2); - - if (FSE_MAX_TABLELOG * 4 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */ - { - if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { - op += 2; - break; - } - } - - op[2] = FSE_GETSYMBOL(&state1); - - if (FSE_MAX_TABLELOG * 2 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */ - BIT_reloadDStream(&bitD); - - op[3] = FSE_GETSYMBOL(&state2); - } - - /* tail */ - /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ - while (1) { - if (op > (omax - 2)) - return ERROR(dstSize_tooSmall); - *op++ = FSE_GETSYMBOL(&state1); - if (BIT_reloadDStream(&bitD) == BIT_DStream_overflow) { - *op++ = FSE_GETSYMBOL(&state2); - break; - } - - if (op > (omax - 2)) - return ERROR(dstSize_tooSmall); - *op++ = FSE_GETSYMBOL(&state2); - if (BIT_reloadDStream(&bitD) == BIT_DStream_overflow) { - *op++ = FSE_GETSYMBOL(&state1); - break; - } - } - - return op - ostart; -} - -size_t FSE_decompress_usingDTable(void *dst, size_t originalSize, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt) -{ - const void *ptr = dt; - const FSE_DTableHeader *DTableH = (const FSE_DTableHeader *)ptr; - const U32 fastMode = DTableH->fastMode; - - /* select fast mode (static) */ - if (fastMode) - return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); - return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); -} - -size_t FSE_decompress_wksp(void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, unsigned maxLog, void *workspace, size_t workspaceSize) -{ - const BYTE *const istart = (const BYTE *)cSrc; - const BYTE *ip = istart; - unsigned tableLog; - unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; - size_t NCountLength; - - FSE_DTable *dt; - short *counting; - size_t spaceUsed32 = 0; - - FSE_STATIC_ASSERT(sizeof(FSE_DTable) == sizeof(U32)); - - dt = (FSE_DTable *)((U32 *)workspace + spaceUsed32); - spaceUsed32 += FSE_DTABLE_SIZE_U32(maxLog); - counting = (short *)((U32 *)workspace + spaceUsed32); - spaceUsed32 += ALIGN(sizeof(short) * (FSE_MAX_SYMBOL_VALUE + 1), sizeof(U32)) >> 2; - - if ((spaceUsed32 << 2) > workspaceSize) - return ERROR(tableLog_tooLarge); - workspace = (U32 *)workspace + spaceUsed32; - workspaceSize -= (spaceUsed32 << 2); - - /* normal FSE decoding mode */ - NCountLength = FSE_readNCount(counting, &maxSymbolValue, &tableLog, istart, cSrcSize); - if (FSE_isError(NCountLength)) - return NCountLength; - // if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size; supposed to be already checked in NCountLength, only remaining - // case : NCountLength==cSrcSize */ - if (tableLog > maxLog) - return ERROR(tableLog_tooLarge); - ip += NCountLength; - cSrcSize -= NCountLength; - - CHECK_F(FSE_buildDTable_wksp(dt, counting, maxSymbolValue, tableLog, workspace, workspaceSize)); - - return FSE_decompress_usingDTable(dst, dstCapacity, ip, cSrcSize, dt); /* always return, even if it is an error code */ -} diff --git a/lib/zstd/huf.h b/lib/zstd/huf.h deleted file mode 100644 index 2143da28d952..000000000000 --- a/lib/zstd/huf.h +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Huffman coder, part of New Generation Entropy library - * header file - * Copyright (C) 2013-2016, Yann Collet. - * - * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - * - * You can contact the author at : - * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy - */ -#ifndef HUF_H_298734234 -#define HUF_H_298734234 - -/* *** Dependencies *** */ -#include /* size_t */ - -/* *** Tool functions *** */ -#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ -size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ - -/* Error Management */ -unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ - -/* *** Advanced function *** */ - -/** HUF_compress4X_wksp() : -* Same as HUF_compress2(), but uses externally allocated `workSpace`, which must be a table of >= 1024 unsigned */ -size_t HUF_compress4X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace, - size_t wkspSize); /**< `workSpace` must be a table of at least HUF_COMPRESS_WORKSPACE_SIZE_U32 unsigned */ - -/* *** Dependencies *** */ -#include "mem.h" /* U32 */ - -/* *** Constants *** */ -#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ -#define HUF_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */ -#define HUF_SYMBOLVALUE_MAX 255 - -#define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ -#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) -#error "HUF_TABLELOG_MAX is too large !" -#endif - -/* **************************************** -* Static allocation -******************************************/ -/* HUF buffer bounds */ -#define HUF_CTABLEBOUND 129 -#define HUF_BLOCKBOUND(size) (size + (size >> 8) + 8) /* only true if incompressible pre-filtered with fast heuristic */ -#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ - -/* static allocation of HUF's Compression Table */ -#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ - U32 name##hb[maxSymbolValue + 1]; \ - void *name##hv = &(name##hb); \ - HUF_CElt *name = (HUF_CElt *)(name##hv) /* no final ; */ - -/* static allocation of HUF's DTable */ -typedef U32 HUF_DTable; -#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1 << (maxTableLog))) -#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = {((U32)((maxTableLog)-1) * 0x01000001)} -#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = {((U32)(maxTableLog)*0x01000001)} - -/* The workspace must have alignment at least 4 and be at least this large */ -#define HUF_COMPRESS_WORKSPACE_SIZE (6 << 10) -#define HUF_COMPRESS_WORKSPACE_SIZE_U32 (HUF_COMPRESS_WORKSPACE_SIZE / sizeof(U32)) - -/* The workspace must have alignment at least 4 and be at least this large */ -#define HUF_DECOMPRESS_WORKSPACE_SIZE (3 << 10) -#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32)) - -/* **************************************** -* Advanced decompression functions -******************************************/ -size_t HUF_decompress4X_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize); /**< decodes RLE and uncompressed */ -size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, - size_t workspaceSize); /**< considers RLE and uncompressed as errors */ -size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, - size_t workspaceSize); /**< single-symbol decoder */ -size_t HUF_decompress4X4_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, - size_t workspaceSize); /**< double-symbols decoder */ - -/* **************************************** -* HUF detailed API -******************************************/ -/*! -HUF_compress() does the following: -1. count symbol occurrence from source[] into table count[] using FSE_count() -2. (optional) refine tableLog using HUF_optimalTableLog() -3. build Huffman table from count using HUF_buildCTable() -4. save Huffman table to memory buffer using HUF_writeCTable_wksp() -5. encode the data stream using HUF_compress4X_usingCTable() - -The following API allows targeting specific sub-functions for advanced tasks. -For example, it's possible to compress several blocks using the same 'CTable', -or to save and regenerate 'CTable' using external methods. -*/ -/* FSE_count() : find it within "fse.h" */ -unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); -typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */ -size_t HUF_writeCTable_wksp(void *dst, size_t maxDstSize, const HUF_CElt *CTable, unsigned maxSymbolValue, unsigned huffLog, void *workspace, size_t workspaceSize); -size_t HUF_compress4X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable); - -typedef enum { - HUF_repeat_none, /**< Cannot use the previous table */ - HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, - 4}X_repeat */ - HUF_repeat_valid /**< Can use the previous table and it is asumed to be valid */ -} HUF_repeat; -/** HUF_compress4X_repeat() : -* Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. -* If it uses hufTable it does not modify hufTable or repeat. -* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. -* If preferRepeat then the old table will always be used if valid. */ -size_t HUF_compress4X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace, - size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat, - int preferRepeat); /**< `workSpace` must be a table of at least HUF_COMPRESS_WORKSPACE_SIZE_U32 unsigned */ - -/** HUF_buildCTable_wksp() : - * Same as HUF_buildCTable(), but using externally allocated scratch buffer. - * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned. - */ -size_t HUF_buildCTable_wksp(HUF_CElt *tree, const U32 *count, U32 maxSymbolValue, U32 maxNbBits, void *workSpace, size_t wkspSize); - -/*! HUF_readStats() : - Read compact Huffman tree, saved by HUF_writeCTable(). - `huffWeight` is destination buffer. - @return : size read from `src` , or an error Code . - Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */ -size_t HUF_readStats_wksp(BYTE *huffWeight, size_t hwSize, U32 *rankStats, U32 *nbSymbolsPtr, U32 *tableLogPtr, const void *src, size_t srcSize, - void *workspace, size_t workspaceSize); - -/** HUF_readCTable() : -* Loading a CTable saved with HUF_writeCTable() */ -size_t HUF_readCTable_wksp(HUF_CElt *CTable, unsigned maxSymbolValue, const void *src, size_t srcSize, void *workspace, size_t workspaceSize); - -/* -HUF_decompress() does the following: -1. select the decompression algorithm (X2, X4) based on pre-computed heuristics -2. build Huffman table from save, using HUF_readDTableXn() -3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable -*/ - -/** HUF_selectDecoder() : -* Tells which decoder is likely to decode faster, -* based on a set of pre-determined metrics. -* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 . -* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ -U32 HUF_selectDecoder(size_t dstSize, size_t cSrcSize); - -size_t HUF_readDTableX2_wksp(HUF_DTable *DTable, const void *src, size_t srcSize, void *workspace, size_t workspaceSize); -size_t HUF_readDTableX4_wksp(HUF_DTable *DTable, const void *src, size_t srcSize, void *workspace, size_t workspaceSize); - -size_t HUF_decompress4X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable); -size_t HUF_decompress4X2_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable); -size_t HUF_decompress4X4_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable); - -/* single stream variants */ - -size_t HUF_compress1X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace, - size_t wkspSize); /**< `workSpace` must be a table of at least HUF_COMPRESS_WORKSPACE_SIZE_U32 unsigned */ -size_t HUF_compress1X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable); -/** HUF_compress1X_repeat() : -* Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. -* If it uses hufTable it does not modify hufTable or repeat. -* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. -* If preferRepeat then the old table will always be used if valid. */ -size_t HUF_compress1X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace, - size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat, - int preferRepeat); /**< `workSpace` must be a table of at least HUF_COMPRESS_WORKSPACE_SIZE_U32 unsigned */ - -size_t HUF_decompress1X_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize); -size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, - size_t workspaceSize); /**< single-symbol decoder */ -size_t HUF_decompress1X4_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, - size_t workspaceSize); /**< double-symbols decoder */ - -size_t HUF_decompress1X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, - const HUF_DTable *DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */ -size_t HUF_decompress1X2_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable); -size_t HUF_decompress1X4_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable); - -#endif /* HUF_H_298734234 */ diff --git a/lib/zstd/huf_compress.c b/lib/zstd/huf_compress.c deleted file mode 100644 index fd32838c185f..000000000000 --- a/lib/zstd/huf_compress.c +++ /dev/null @@ -1,773 +0,0 @@ -/* - * Huffman encoder, part of New Generation Entropy library - * Copyright (C) 2013-2016, Yann Collet. - * - * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - * - * You can contact the author at : - * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy - */ - -/* ************************************************************** -* Includes -****************************************************************/ -#include "bitstream.h" -#include "fse.h" /* header compression */ -#include "huf.h" -#include -#include /* memcpy, memset */ - -/* ************************************************************** -* Error Management -****************************************************************/ -#define HUF_STATIC_ASSERT(c) \ - { \ - enum { HUF_static_assert = 1 / (int)(!!(c)) }; \ - } /* use only *after* variable declarations */ -#define CHECK_V_F(e, f) \ - size_t const e = f; \ - if (ERR_isError(e)) \ - return f -#define CHECK_F(f) \ - { \ - CHECK_V_F(_var_err__, f); \ - } - -/* ************************************************************** -* Utils -****************************************************************/ -unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) -{ - return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); -} - -/* ******************************************************* -* HUF : Huffman block compression -*********************************************************/ -/* HUF_compressWeights() : - * Same as FSE_compress(), but dedicated to huff0's weights compression. - * The use case needs much less stack memory. - * Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX. - */ -#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6 -size_t HUF_compressWeights_wksp(void *dst, size_t dstSize, const void *weightTable, size_t wtSize, void *workspace, size_t workspaceSize) -{ - BYTE *const ostart = (BYTE *)dst; - BYTE *op = ostart; - BYTE *const oend = ostart + dstSize; - - U32 maxSymbolValue = HUF_TABLELOG_MAX; - U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER; - - FSE_CTable *CTable; - U32 *count; - S16 *norm; - size_t spaceUsed32 = 0; - - HUF_STATIC_ASSERT(sizeof(FSE_CTable) == sizeof(U32)); - - CTable = (FSE_CTable *)((U32 *)workspace + spaceUsed32); - spaceUsed32 += FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX); - count = (U32 *)workspace + spaceUsed32; - spaceUsed32 += HUF_TABLELOG_MAX + 1; - norm = (S16 *)((U32 *)workspace + spaceUsed32); - spaceUsed32 += ALIGN(sizeof(S16) * (HUF_TABLELOG_MAX + 1), sizeof(U32)) >> 2; - - if ((spaceUsed32 << 2) > workspaceSize) - return ERROR(tableLog_tooLarge); - workspace = (U32 *)workspace + spaceUsed32; - workspaceSize -= (spaceUsed32 << 2); - - /* init conditions */ - if (wtSize <= 1) - return 0; /* Not compressible */ - - /* Scan input and build symbol stats */ - { - CHECK_V_F(maxCount, FSE_count_simple(count, &maxSymbolValue, weightTable, wtSize)); - if (maxCount == wtSize) - return 1; /* only a single symbol in src : rle */ - if (maxCount == 1) - return 0; /* each symbol present maximum once => not compressible */ - } - - tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue); - CHECK_F(FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue)); - - /* Write table description header */ - { - CHECK_V_F(hSize, FSE_writeNCount(op, oend - op, norm, maxSymbolValue, tableLog)); - op += hSize; - } - - /* Compress */ - CHECK_F(FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, workspace, workspaceSize)); - { - CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable)); - if (cSize == 0) - return 0; /* not enough space for compressed data */ - op += cSize; - } - - return op - ostart; -} - -struct HUF_CElt_s { - U16 val; - BYTE nbBits; -}; /* typedef'd to HUF_CElt within "huf.h" */ - -/*! HUF_writeCTable_wksp() : - `CTable` : Huffman tree to save, using huf representation. - @return : size of saved CTable */ -size_t HUF_writeCTable_wksp(void *dst, size_t maxDstSize, const HUF_CElt *CTable, U32 maxSymbolValue, U32 huffLog, void *workspace, size_t workspaceSize) -{ - BYTE *op = (BYTE *)dst; - U32 n; - - BYTE *bitsToWeight; - BYTE *huffWeight; - size_t spaceUsed32 = 0; - - bitsToWeight = (BYTE *)((U32 *)workspace + spaceUsed32); - spaceUsed32 += ALIGN(HUF_TABLELOG_MAX + 1, sizeof(U32)) >> 2; - huffWeight = (BYTE *)((U32 *)workspace + spaceUsed32); - spaceUsed32 += ALIGN(HUF_SYMBOLVALUE_MAX, sizeof(U32)) >> 2; - - if ((spaceUsed32 << 2) > workspaceSize) - return ERROR(tableLog_tooLarge); - workspace = (U32 *)workspace + spaceUsed32; - workspaceSize -= (spaceUsed32 << 2); - - /* check conditions */ - if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) - return ERROR(maxSymbolValue_tooLarge); - - /* convert to weight */ - bitsToWeight[0] = 0; - for (n = 1; n < huffLog + 1; n++) - bitsToWeight[n] = (BYTE)(huffLog + 1 - n); - for (n = 0; n < maxSymbolValue; n++) - huffWeight[n] = bitsToWeight[CTable[n].nbBits]; - - /* attempt weights compression by FSE */ - { - CHECK_V_F(hSize, HUF_compressWeights_wksp(op + 1, maxDstSize - 1, huffWeight, maxSymbolValue, workspace, workspaceSize)); - if ((hSize > 1) & (hSize < maxSymbolValue / 2)) { /* FSE compressed */ - op[0] = (BYTE)hSize; - return hSize + 1; - } - } - - /* write raw values as 4-bits (max : 15) */ - if (maxSymbolValue > (256 - 128)) - return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */ - if (((maxSymbolValue + 1) / 2) + 1 > maxDstSize) - return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */ - op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue - 1)); - huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */ - for (n = 0; n < maxSymbolValue; n += 2) - op[(n / 2) + 1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n + 1]); - return ((maxSymbolValue + 1) / 2) + 1; -} - -size_t HUF_readCTable_wksp(HUF_CElt *CTable, U32 maxSymbolValue, const void *src, size_t srcSize, void *workspace, size_t workspaceSize) -{ - U32 *rankVal; - BYTE *huffWeight; - U32 tableLog = 0; - U32 nbSymbols = 0; - size_t readSize; - size_t spaceUsed32 = 0; - - rankVal = (U32 *)workspace + spaceUsed32; - spaceUsed32 += HUF_TABLELOG_ABSOLUTEMAX + 1; - huffWeight = (BYTE *)((U32 *)workspace + spaceUsed32); - spaceUsed32 += ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2; - - if ((spaceUsed32 << 2) > workspaceSize) - return ERROR(tableLog_tooLarge); - workspace = (U32 *)workspace + spaceUsed32; - workspaceSize -= (spaceUsed32 << 2); - - /* get symbol weights */ - readSize = HUF_readStats_wksp(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize, workspace, workspaceSize); - if (ERR_isError(readSize)) - return readSize; - - /* check result */ - if (tableLog > HUF_TABLELOG_MAX) - return ERROR(tableLog_tooLarge); - if (nbSymbols > maxSymbolValue + 1) - return ERROR(maxSymbolValue_tooSmall); - - /* Prepare base value per rank */ - { - U32 n, nextRankStart = 0; - for (n = 1; n <= tableLog; n++) { - U32 curr = nextRankStart; - nextRankStart += (rankVal[n] << (n - 1)); - rankVal[n] = curr; - } - } - - /* fill nbBits */ - { - U32 n; - for (n = 0; n < nbSymbols; n++) { - const U32 w = huffWeight[n]; - CTable[n].nbBits = (BYTE)(tableLog + 1 - w); - } - } - - /* fill val */ - { - U16 nbPerRank[HUF_TABLELOG_MAX + 2] = {0}; /* support w=0=>n=tableLog+1 */ - U16 valPerRank[HUF_TABLELOG_MAX + 2] = {0}; - { - U32 n; - for (n = 0; n < nbSymbols; n++) - nbPerRank[CTable[n].nbBits]++; - } - /* determine stating value per rank */ - valPerRank[tableLog + 1] = 0; /* for w==0 */ - { - U16 min = 0; - U32 n; - for (n = tableLog; n > 0; n--) { /* start at n=tablelog <-> w=1 */ - valPerRank[n] = min; /* get starting value within each rank */ - min += nbPerRank[n]; - min >>= 1; - } - } - /* assign value within rank, symbol order */ - { - U32 n; - for (n = 0; n <= maxSymbolValue; n++) - CTable[n].val = valPerRank[CTable[n].nbBits]++; - } - } - - return readSize; -} - -typedef struct nodeElt_s { - U32 count; - U16 parent; - BYTE byte; - BYTE nbBits; -} nodeElt; - -static U32 HUF_setMaxHeight(nodeElt *huffNode, U32 lastNonNull, U32 maxNbBits) -{ - const U32 largestBits = huffNode[lastNonNull].nbBits; - if (largestBits <= maxNbBits) - return largestBits; /* early exit : no elt > maxNbBits */ - - /* there are several too large elements (at least >= 2) */ - { - int totalCost = 0; - const U32 baseCost = 1 << (largestBits - maxNbBits); - U32 n = lastNonNull; - - while (huffNode[n].nbBits > maxNbBits) { - totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); - huffNode[n].nbBits = (BYTE)maxNbBits; - n--; - } /* n stops at huffNode[n].nbBits <= maxNbBits */ - while (huffNode[n].nbBits == maxNbBits) - n--; /* n end at index of smallest symbol using < maxNbBits */ - - /* renorm totalCost */ - totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */ - - /* repay normalized cost */ - { - U32 const noSymbol = 0xF0F0F0F0; - U32 rankLast[HUF_TABLELOG_MAX + 2]; - int pos; - - /* Get pos of last (smallest) symbol per rank */ - memset(rankLast, 0xF0, sizeof(rankLast)); - { - U32 currNbBits = maxNbBits; - for (pos = n; pos >= 0; pos--) { - if (huffNode[pos].nbBits >= currNbBits) - continue; - currNbBits = huffNode[pos].nbBits; /* < maxNbBits */ - rankLast[maxNbBits - currNbBits] = pos; - } - } - - while (totalCost > 0) { - U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1; - for (; nBitsToDecrease > 1; nBitsToDecrease--) { - U32 highPos = rankLast[nBitsToDecrease]; - U32 lowPos = rankLast[nBitsToDecrease - 1]; - if (highPos == noSymbol) - continue; - if (lowPos == noSymbol) - break; - { - U32 const highTotal = huffNode[highPos].count; - U32 const lowTotal = 2 * huffNode[lowPos].count; - if (highTotal <= lowTotal) - break; - } - } - /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ - /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ - while ((nBitsToDecrease <= HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) - nBitsToDecrease++; - totalCost -= 1 << (nBitsToDecrease - 1); - if (rankLast[nBitsToDecrease - 1] == noSymbol) - rankLast[nBitsToDecrease - 1] = rankLast[nBitsToDecrease]; /* this rank is no longer empty */ - huffNode[rankLast[nBitsToDecrease]].nbBits++; - if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */ - rankLast[nBitsToDecrease] = noSymbol; - else { - rankLast[nBitsToDecrease]--; - if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits - nBitsToDecrease) - rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ - } - } /* while (totalCost > 0) */ - - while (totalCost < 0) { /* Sometimes, cost correction overshoot */ - if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 - (using maxNbBits) */ - while (huffNode[n].nbBits == maxNbBits) - n--; - huffNode[n + 1].nbBits--; - rankLast[1] = n + 1; - totalCost++; - continue; - } - huffNode[rankLast[1] + 1].nbBits--; - rankLast[1]++; - totalCost++; - } - } - } /* there are several too large elements (at least >= 2) */ - - return maxNbBits; -} - -typedef struct { - U32 base; - U32 curr; -} rankPos; - -static void HUF_sort(nodeElt *huffNode, const U32 *count, U32 maxSymbolValue) -{ - rankPos rank[32]; - U32 n; - - memset(rank, 0, sizeof(rank)); - for (n = 0; n <= maxSymbolValue; n++) { - U32 r = BIT_highbit32(count[n] + 1); - rank[r].base++; - } - for (n = 30; n > 0; n--) - rank[n - 1].base += rank[n].base; - for (n = 0; n < 32; n++) - rank[n].curr = rank[n].base; - for (n = 0; n <= maxSymbolValue; n++) { - U32 const c = count[n]; - U32 const r = BIT_highbit32(c + 1) + 1; - U32 pos = rank[r].curr++; - while ((pos > rank[r].base) && (c > huffNode[pos - 1].count)) - huffNode[pos] = huffNode[pos - 1], pos--; - huffNode[pos].count = c; - huffNode[pos].byte = (BYTE)n; - } -} - -/** HUF_buildCTable_wksp() : - * Same as HUF_buildCTable(), but using externally allocated scratch buffer. - * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned. - */ -#define STARTNODE (HUF_SYMBOLVALUE_MAX + 1) -typedef nodeElt huffNodeTable[2 * HUF_SYMBOLVALUE_MAX + 1 + 1]; -size_t HUF_buildCTable_wksp(HUF_CElt *tree, const U32 *count, U32 maxSymbolValue, U32 maxNbBits, void *workSpace, size_t wkspSize) -{ - nodeElt *const huffNode0 = (nodeElt *)workSpace; - nodeElt *const huffNode = huffNode0 + 1; - U32 n, nonNullRank; - int lowS, lowN; - U16 nodeNb = STARTNODE; - U32 nodeRoot; - - /* safety checks */ - if (wkspSize < sizeof(huffNodeTable)) - return ERROR(GENERIC); /* workSpace is not large enough */ - if (maxNbBits == 0) - maxNbBits = HUF_TABLELOG_DEFAULT; - if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) - return ERROR(GENERIC); - memset(huffNode0, 0, sizeof(huffNodeTable)); - - /* sort, decreasing order */ - HUF_sort(huffNode, count, maxSymbolValue); - - /* init for parents */ - nonNullRank = maxSymbolValue; - while (huffNode[nonNullRank].count == 0) - nonNullRank--; - lowS = nonNullRank; - nodeRoot = nodeNb + lowS - 1; - lowN = nodeNb; - huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS - 1].count; - huffNode[lowS].parent = huffNode[lowS - 1].parent = nodeNb; - nodeNb++; - lowS -= 2; - for (n = nodeNb; n <= nodeRoot; n++) - huffNode[n].count = (U32)(1U << 30); - huffNode0[0].count = (U32)(1U << 31); /* fake entry, strong barrier */ - - /* create parents */ - while (nodeNb <= nodeRoot) { - U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; - U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; - huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count; - huffNode[n1].parent = huffNode[n2].parent = nodeNb; - nodeNb++; - } - - /* distribute weights (unlimited tree height) */ - huffNode[nodeRoot].nbBits = 0; - for (n = nodeRoot - 1; n >= STARTNODE; n--) - huffNode[n].nbBits = huffNode[huffNode[n].parent].nbBits + 1; - for (n = 0; n <= nonNullRank; n++) - huffNode[n].nbBits = huffNode[huffNode[n].parent].nbBits + 1; - - /* enforce maxTableLog */ - maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits); - - /* fill result into tree (val, nbBits) */ - { - U16 nbPerRank[HUF_TABLELOG_MAX + 1] = {0}; - U16 valPerRank[HUF_TABLELOG_MAX + 1] = {0}; - if (maxNbBits > HUF_TABLELOG_MAX) - return ERROR(GENERIC); /* check fit into table */ - for (n = 0; n <= nonNullRank; n++) - nbPerRank[huffNode[n].nbBits]++; - /* determine stating value per rank */ - { - U16 min = 0; - for (n = maxNbBits; n > 0; n--) { - valPerRank[n] = min; /* get starting value within each rank */ - min += nbPerRank[n]; - min >>= 1; - } - } - for (n = 0; n <= maxSymbolValue; n++) - tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */ - for (n = 0; n <= maxSymbolValue; n++) - tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */ - } - - return maxNbBits; -} - -static size_t HUF_estimateCompressedSize(HUF_CElt *CTable, const unsigned *count, unsigned maxSymbolValue) -{ - size_t nbBits = 0; - int s; - for (s = 0; s <= (int)maxSymbolValue; ++s) { - nbBits += CTable[s].nbBits * count[s]; - } - return nbBits >> 3; -} - -static int HUF_validateCTable(const HUF_CElt *CTable, const unsigned *count, unsigned maxSymbolValue) -{ - int bad = 0; - int s; - for (s = 0; s <= (int)maxSymbolValue; ++s) { - bad |= (count[s] != 0) & (CTable[s].nbBits == 0); - } - return !bad; -} - -static void HUF_encodeSymbol(BIT_CStream_t *bitCPtr, U32 symbol, const HUF_CElt *CTable) -{ - BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits); -} - -size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); } - -#define HUF_FLUSHBITS(s) BIT_flushBits(s) - -#define HUF_FLUSHBITS_1(stream) \ - if (sizeof((stream)->bitContainer) * 8 < HUF_TABLELOG_MAX * 2 + 7) \ - HUF_FLUSHBITS(stream) - -#define HUF_FLUSHBITS_2(stream) \ - if (sizeof((stream)->bitContainer) * 8 < HUF_TABLELOG_MAX * 4 + 7) \ - HUF_FLUSHBITS(stream) - -size_t HUF_compress1X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable) -{ - const BYTE *ip = (const BYTE *)src; - BYTE *const ostart = (BYTE *)dst; - BYTE *const oend = ostart + dstSize; - BYTE *op = ostart; - size_t n; - BIT_CStream_t bitC; - - /* init */ - if (dstSize < 8) - return 0; /* not enough space to compress */ - { - size_t const initErr = BIT_initCStream(&bitC, op, oend - op); - if (HUF_isError(initErr)) - return 0; - } - - n = srcSize & ~3; /* join to mod 4 */ - switch (srcSize & 3) { - case 3: HUF_encodeSymbol(&bitC, ip[n + 2], CTable); HUF_FLUSHBITS_2(&bitC); - fallthrough; - case 2: HUF_encodeSymbol(&bitC, ip[n + 1], CTable); HUF_FLUSHBITS_1(&bitC); - fallthrough; - case 1: HUF_encodeSymbol(&bitC, ip[n + 0], CTable); HUF_FLUSHBITS(&bitC); - fallthrough; - case 0: - default:; - } - - for (; n > 0; n -= 4) { /* note : n&3==0 at this stage */ - HUF_encodeSymbol(&bitC, ip[n - 1], CTable); - HUF_FLUSHBITS_1(&bitC); - HUF_encodeSymbol(&bitC, ip[n - 2], CTable); - HUF_FLUSHBITS_2(&bitC); - HUF_encodeSymbol(&bitC, ip[n - 3], CTable); - HUF_FLUSHBITS_1(&bitC); - HUF_encodeSymbol(&bitC, ip[n - 4], CTable); - HUF_FLUSHBITS(&bitC); - } - - return BIT_closeCStream(&bitC); -} - -size_t HUF_compress4X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable) -{ - size_t const segmentSize = (srcSize + 3) / 4; /* first 3 segments */ - const BYTE *ip = (const BYTE *)src; - const BYTE *const iend = ip + srcSize; - BYTE *const ostart = (BYTE *)dst; - BYTE *const oend = ostart + dstSize; - BYTE *op = ostart; - - if (dstSize < 6 + 1 + 1 + 1 + 8) - return 0; /* minimum space to compress successfully */ - if (srcSize < 12) - return 0; /* no saving possible : too small input */ - op += 6; /* jumpTable */ - - { - CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, segmentSize, CTable)); - if (cSize == 0) - return 0; - ZSTD_writeLE16(ostart, (U16)cSize); - op += cSize; - } - - ip += segmentSize; - { - CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, segmentSize, CTable)); - if (cSize == 0) - return 0; - ZSTD_writeLE16(ostart + 2, (U16)cSize); - op += cSize; - } - - ip += segmentSize; - { - CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, segmentSize, CTable)); - if (cSize == 0) - return 0; - ZSTD_writeLE16(ostart + 4, (U16)cSize); - op += cSize; - } - - ip += segmentSize; - { - CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, iend - ip, CTable)); - if (cSize == 0) - return 0; - op += cSize; - } - - return op - ostart; -} - -static size_t HUF_compressCTable_internal(BYTE *const ostart, BYTE *op, BYTE *const oend, const void *src, size_t srcSize, unsigned singleStream, - const HUF_CElt *CTable) -{ - size_t const cSize = - singleStream ? HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) : HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable); - if (HUF_isError(cSize)) { - return cSize; - } - if (cSize == 0) { - return 0; - } /* uncompressible */ - op += cSize; - /* check compressibility */ - if ((size_t)(op - ostart) >= srcSize - 1) { - return 0; - } - return op - ostart; -} - -/* `workSpace` must a table of at least 1024 unsigned */ -static size_t HUF_compress_internal(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, - unsigned singleStream, void *workSpace, size_t wkspSize, HUF_CElt *oldHufTable, HUF_repeat *repeat, int preferRepeat) -{ - BYTE *const ostart = (BYTE *)dst; - BYTE *const oend = ostart + dstSize; - BYTE *op = ostart; - - U32 *count; - size_t const countSize = sizeof(U32) * (HUF_SYMBOLVALUE_MAX + 1); - HUF_CElt *CTable; - size_t const CTableSize = sizeof(HUF_CElt) * (HUF_SYMBOLVALUE_MAX + 1); - - /* checks & inits */ - if (wkspSize < sizeof(huffNodeTable) + countSize + CTableSize) - return ERROR(GENERIC); - if (!srcSize) - return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */ - if (!dstSize) - return 0; /* cannot fit within dst budget */ - if (srcSize > HUF_BLOCKSIZE_MAX) - return ERROR(srcSize_wrong); /* curr block size limit */ - if (huffLog > HUF_TABLELOG_MAX) - return ERROR(tableLog_tooLarge); - if (!maxSymbolValue) - maxSymbolValue = HUF_SYMBOLVALUE_MAX; - if (!huffLog) - huffLog = HUF_TABLELOG_DEFAULT; - - count = (U32 *)workSpace; - workSpace = (BYTE *)workSpace + countSize; - wkspSize -= countSize; - CTable = (HUF_CElt *)workSpace; - workSpace = (BYTE *)workSpace + CTableSize; - wkspSize -= CTableSize; - - /* Heuristic : If we don't need to check the validity of the old table use the old table for small inputs */ - if (preferRepeat && repeat && *repeat == HUF_repeat_valid) { - return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable); - } - - /* Scan input and build symbol stats */ - { - CHECK_V_F(largest, FSE_count_wksp(count, &maxSymbolValue, (const BYTE *)src, srcSize, (U32 *)workSpace)); - if (largest == srcSize) { - *ostart = ((const BYTE *)src)[0]; - return 1; - } /* single symbol, rle */ - if (largest <= (srcSize >> 7) + 1) - return 0; /* Fast heuristic : not compressible enough */ - } - - /* Check validity of previous table */ - if (repeat && *repeat == HUF_repeat_check && !HUF_validateCTable(oldHufTable, count, maxSymbolValue)) { - *repeat = HUF_repeat_none; - } - /* Heuristic : use existing table for small inputs */ - if (preferRepeat && repeat && *repeat != HUF_repeat_none) { - return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable); - } - - /* Build Huffman Tree */ - huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); - { - CHECK_V_F(maxBits, HUF_buildCTable_wksp(CTable, count, maxSymbolValue, huffLog, workSpace, wkspSize)); - huffLog = (U32)maxBits; - /* Zero the unused symbols so we can check it for validity */ - memset(CTable + maxSymbolValue + 1, 0, CTableSize - (maxSymbolValue + 1) * sizeof(HUF_CElt)); - } - - /* Write table description header */ - { - CHECK_V_F(hSize, HUF_writeCTable_wksp(op, dstSize, CTable, maxSymbolValue, huffLog, workSpace, wkspSize)); - /* Check if using the previous table will be beneficial */ - if (repeat && *repeat != HUF_repeat_none) { - size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, count, maxSymbolValue); - size_t const newSize = HUF_estimateCompressedSize(CTable, count, maxSymbolValue); - if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) { - return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable); - } - } - /* Use the new table */ - if (hSize + 12ul >= srcSize) { - return 0; - } - op += hSize; - if (repeat) { - *repeat = HUF_repeat_none; - } - if (oldHufTable) { - memcpy(oldHufTable, CTable, CTableSize); - } /* Save the new table */ - } - return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, CTable); -} - -size_t HUF_compress1X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace, - size_t wkspSize) -{ - return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, NULL, NULL, 0); -} - -size_t HUF_compress1X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace, - size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat, int preferRepeat) -{ - return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, hufTable, repeat, - preferRepeat); -} - -size_t HUF_compress4X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace, - size_t wkspSize) -{ - return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, NULL, NULL, 0); -} - -size_t HUF_compress4X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace, - size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat, int preferRepeat) -{ - return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, hufTable, repeat, - preferRepeat); -} diff --git a/lib/zstd/huf_decompress.c b/lib/zstd/huf_decompress.c deleted file mode 100644 index 6526482047dc..000000000000 --- a/lib/zstd/huf_decompress.c +++ /dev/null @@ -1,960 +0,0 @@ -/* - * Huffman decoder, part of New Generation Entropy library - * Copyright (C) 2013-2016, Yann Collet. - * - * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - * - * You can contact the author at : - * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy - */ - -/* ************************************************************** -* Compiler specifics -****************************************************************/ -#define FORCE_INLINE static __always_inline - -/* ************************************************************** -* Dependencies -****************************************************************/ -#include "bitstream.h" /* BIT_* */ -#include "fse.h" /* header compression */ -#include "huf.h" -#include -#include -#include /* memcpy, memset */ - -/* ************************************************************** -* Error Management -****************************************************************/ -#define HUF_STATIC_ASSERT(c) \ - { \ - enum { HUF_static_assert = 1 / (int)(!!(c)) }; \ - } /* use only *after* variable declarations */ - -/*-***************************/ -/* generic DTableDesc */ -/*-***************************/ - -typedef struct { - BYTE maxTableLog; - BYTE tableType; - BYTE tableLog; - BYTE reserved; -} DTableDesc; - -static DTableDesc HUF_getDTableDesc(const HUF_DTable *table) -{ - DTableDesc dtd; - memcpy(&dtd, table, sizeof(dtd)); - return dtd; -} - -/*-***************************/ -/* single-symbol decoding */ -/*-***************************/ - -typedef struct { - BYTE byte; - BYTE nbBits; -} HUF_DEltX2; /* single-symbol decoding */ - -size_t HUF_readDTableX2_wksp(HUF_DTable *DTable, const void *src, size_t srcSize, void *workspace, size_t workspaceSize) -{ - U32 tableLog = 0; - U32 nbSymbols = 0; - size_t iSize; - void *const dtPtr = DTable + 1; - HUF_DEltX2 *const dt = (HUF_DEltX2 *)dtPtr; - - U32 *rankVal; - BYTE *huffWeight; - size_t spaceUsed32 = 0; - - rankVal = (U32 *)workspace + spaceUsed32; - spaceUsed32 += HUF_TABLELOG_ABSOLUTEMAX + 1; - huffWeight = (BYTE *)((U32 *)workspace + spaceUsed32); - spaceUsed32 += ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2; - - if ((spaceUsed32 << 2) > workspaceSize) - return ERROR(tableLog_tooLarge); - workspace = (U32 *)workspace + spaceUsed32; - workspaceSize -= (spaceUsed32 << 2); - - HUF_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); - /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ - - iSize = HUF_readStats_wksp(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize, workspace, workspaceSize); - if (HUF_isError(iSize)) - return iSize; - - /* Table header */ - { - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (tableLog > (U32)(dtd.maxTableLog + 1)) - return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */ - dtd.tableType = 0; - dtd.tableLog = (BYTE)tableLog; - memcpy(DTable, &dtd, sizeof(dtd)); - } - - /* Calculate starting value for each rank */ - { - U32 n, nextRankStart = 0; - for (n = 1; n < tableLog + 1; n++) { - U32 const curr = nextRankStart; - nextRankStart += (rankVal[n] << (n - 1)); - rankVal[n] = curr; - } - } - - /* fill DTable */ - { - U32 n; - for (n = 0; n < nbSymbols; n++) { - U32 const w = huffWeight[n]; - U32 const length = (1 << w) >> 1; - U32 u; - HUF_DEltX2 D; - D.byte = (BYTE)n; - D.nbBits = (BYTE)(tableLog + 1 - w); - for (u = rankVal[w]; u < rankVal[w] + length; u++) - dt[u] = D; - rankVal[w] += length; - } - } - - return iSize; -} - -static BYTE HUF_decodeSymbolX2(BIT_DStream_t *Dstream, const HUF_DEltX2 *dt, const U32 dtLog) -{ - size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ - BYTE const c = dt[val].byte; - BIT_skipBits(Dstream, dt[val].nbBits); - return c; -} - -#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) *ptr++ = HUF_decodeSymbolX2(DStreamPtr, dt, dtLog) - -#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ - if (ZSTD_64bits() || (HUF_TABLELOG_MAX <= 12)) \ - HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) - -#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ - if (ZSTD_64bits()) \ - HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) - -FORCE_INLINE size_t HUF_decodeStreamX2(BYTE *p, BIT_DStream_t *const bitDPtr, BYTE *const pEnd, const HUF_DEltX2 *const dt, const U32 dtLog) -{ - BYTE *const pStart = p; - - /* up to 4 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd - 4)) { - HUF_DECODE_SYMBOLX2_2(p, bitDPtr); - HUF_DECODE_SYMBOLX2_1(p, bitDPtr); - HUF_DECODE_SYMBOLX2_2(p, bitDPtr); - HUF_DECODE_SYMBOLX2_0(p, bitDPtr); - } - - /* closer to the end */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd)) - HUF_DECODE_SYMBOLX2_0(p, bitDPtr); - - /* no more data to retrieve from bitstream, hence no need to reload */ - while (p < pEnd) - HUF_DECODE_SYMBOLX2_0(p, bitDPtr); - - return pEnd - pStart; -} - -static size_t HUF_decompress1X2_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -{ - BYTE *op = (BYTE *)dst; - BYTE *const oend = op + dstSize; - const void *dtPtr = DTable + 1; - const HUF_DEltX2 *const dt = (const HUF_DEltX2 *)dtPtr; - BIT_DStream_t bitD; - DTableDesc const dtd = HUF_getDTableDesc(DTable); - U32 const dtLog = dtd.tableLog; - - { - size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); - if (HUF_isError(errorCode)) - return errorCode; - } - - HUF_decodeStreamX2(op, &bitD, oend, dt, dtLog); - - /* check */ - if (!BIT_endOfDStream(&bitD)) - return ERROR(corruption_detected); - - return dstSize; -} - -size_t HUF_decompress1X2_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 0) - return ERROR(GENERIC); - return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -} - -size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable *DCtx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize) -{ - const BYTE *ip = (const BYTE *)cSrc; - - size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, workspace, workspaceSize); - if (HUF_isError(hSize)) - return hSize; - if (hSize >= cSrcSize) - return ERROR(srcSize_wrong); - ip += hSize; - cSrcSize -= hSize; - - return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx); -} - -static size_t HUF_decompress4X2_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -{ - /* Check */ - if (cSrcSize < 10) - return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ - - { - const BYTE *const istart = (const BYTE *)cSrc; - BYTE *const ostart = (BYTE *)dst; - BYTE *const oend = ostart + dstSize; - const void *const dtPtr = DTable + 1; - const HUF_DEltX2 *const dt = (const HUF_DEltX2 *)dtPtr; - - /* Init */ - BIT_DStream_t bitD1; - BIT_DStream_t bitD2; - BIT_DStream_t bitD3; - BIT_DStream_t bitD4; - size_t const length1 = ZSTD_readLE16(istart); - size_t const length2 = ZSTD_readLE16(istart + 2); - size_t const length3 = ZSTD_readLE16(istart + 4); - size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); - const BYTE *const istart1 = istart + 6; /* jumpTable */ - const BYTE *const istart2 = istart1 + length1; - const BYTE *const istart3 = istart2 + length2; - const BYTE *const istart4 = istart3 + length3; - const size_t segmentSize = (dstSize + 3) / 4; - BYTE *const opStart2 = ostart + segmentSize; - BYTE *const opStart3 = opStart2 + segmentSize; - BYTE *const opStart4 = opStart3 + segmentSize; - BYTE *op1 = ostart; - BYTE *op2 = opStart2; - BYTE *op3 = opStart3; - BYTE *op4 = opStart4; - U32 endSignal; - DTableDesc const dtd = HUF_getDTableDesc(DTable); - U32 const dtLog = dtd.tableLog; - - if (length4 > cSrcSize) - return ERROR(corruption_detected); /* overflow */ - { - size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); - if (HUF_isError(errorCode)) - return errorCode; - } - { - size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); - if (HUF_isError(errorCode)) - return errorCode; - } - { - size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); - if (HUF_isError(errorCode)) - return errorCode; - } - { - size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); - if (HUF_isError(errorCode)) - return errorCode; - } - - /* 16-32 symbols per loop (4-8 symbols per stream) */ - endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - for (; (endSignal == BIT_DStream_unfinished) && (op4 < (oend - 7));) { - HUF_DECODE_SYMBOLX2_2(op1, &bitD1); - HUF_DECODE_SYMBOLX2_2(op2, &bitD2); - HUF_DECODE_SYMBOLX2_2(op3, &bitD3); - HUF_DECODE_SYMBOLX2_2(op4, &bitD4); - HUF_DECODE_SYMBOLX2_1(op1, &bitD1); - HUF_DECODE_SYMBOLX2_1(op2, &bitD2); - HUF_DECODE_SYMBOLX2_1(op3, &bitD3); - HUF_DECODE_SYMBOLX2_1(op4, &bitD4); - HUF_DECODE_SYMBOLX2_2(op1, &bitD1); - HUF_DECODE_SYMBOLX2_2(op2, &bitD2); - HUF_DECODE_SYMBOLX2_2(op3, &bitD3); - HUF_DECODE_SYMBOLX2_2(op4, &bitD4); - HUF_DECODE_SYMBOLX2_0(op1, &bitD1); - HUF_DECODE_SYMBOLX2_0(op2, &bitD2); - HUF_DECODE_SYMBOLX2_0(op3, &bitD3); - HUF_DECODE_SYMBOLX2_0(op4, &bitD4); - endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - } - - /* check corruption */ - if (op1 > opStart2) - return ERROR(corruption_detected); - if (op2 > opStart3) - return ERROR(corruption_detected); - if (op3 > opStart4) - return ERROR(corruption_detected); - /* note : op4 supposed already verified within main loop */ - - /* finish bitStreams one by one */ - HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); - HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); - HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); - HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); - - /* check */ - endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); - if (!endSignal) - return ERROR(corruption_detected); - - /* decoded size */ - return dstSize; - } -} - -size_t HUF_decompress4X2_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 0) - return ERROR(GENERIC); - return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -} - -size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize) -{ - const BYTE *ip = (const BYTE *)cSrc; - - size_t const hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize, workspace, workspaceSize); - if (HUF_isError(hSize)) - return hSize; - if (hSize >= cSrcSize) - return ERROR(srcSize_wrong); - ip += hSize; - cSrcSize -= hSize; - - return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx); -} - -/* *************************/ -/* double-symbols decoding */ -/* *************************/ -typedef struct { - U16 sequence; - BYTE nbBits; - BYTE length; -} HUF_DEltX4; /* double-symbols decoding */ - -typedef struct { - BYTE symbol; - BYTE weight; -} sortedSymbol_t; - -/* HUF_fillDTableX4Level2() : - * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */ -static void HUF_fillDTableX4Level2(HUF_DEltX4 *DTable, U32 sizeLog, const U32 consumed, const U32 *rankValOrigin, const int minWeight, - const sortedSymbol_t *sortedSymbols, const U32 sortedListSize, U32 nbBitsBaseline, U16 baseSeq) -{ - HUF_DEltX4 DElt; - U32 rankVal[HUF_TABLELOG_MAX + 1]; - - /* get pre-calculated rankVal */ - memcpy(rankVal, rankValOrigin, sizeof(rankVal)); - - /* fill skipped values */ - if (minWeight > 1) { - U32 i, skipSize = rankVal[minWeight]; - ZSTD_writeLE16(&(DElt.sequence), baseSeq); - DElt.nbBits = (BYTE)(consumed); - DElt.length = 1; - for (i = 0; i < skipSize; i++) - DTable[i] = DElt; - } - - /* fill DTable */ - { - U32 s; - for (s = 0; s < sortedListSize; s++) { /* note : sortedSymbols already skipped */ - const U32 symbol = sortedSymbols[s].symbol; - const U32 weight = sortedSymbols[s].weight; - const U32 nbBits = nbBitsBaseline - weight; - const U32 length = 1 << (sizeLog - nbBits); - const U32 start = rankVal[weight]; - U32 i = start; - const U32 end = start + length; - - ZSTD_writeLE16(&(DElt.sequence), (U16)(baseSeq + (symbol << 8))); - DElt.nbBits = (BYTE)(nbBits + consumed); - DElt.length = 2; - do { - DTable[i++] = DElt; - } while (i < end); /* since length >= 1 */ - - rankVal[weight] += length; - } - } -} - -typedef U32 rankVal_t[HUF_TABLELOG_MAX][HUF_TABLELOG_MAX + 1]; -typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1]; - -static void HUF_fillDTableX4(HUF_DEltX4 *DTable, const U32 targetLog, const sortedSymbol_t *sortedList, const U32 sortedListSize, const U32 *rankStart, - rankVal_t rankValOrigin, const U32 maxWeight, const U32 nbBitsBaseline) -{ - U32 rankVal[HUF_TABLELOG_MAX + 1]; - const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ - const U32 minBits = nbBitsBaseline - maxWeight; - U32 s; - - memcpy(rankVal, rankValOrigin, sizeof(rankVal)); - - /* fill DTable */ - for (s = 0; s < sortedListSize; s++) { - const U16 symbol = sortedList[s].symbol; - const U32 weight = sortedList[s].weight; - const U32 nbBits = nbBitsBaseline - weight; - const U32 start = rankVal[weight]; - const U32 length = 1 << (targetLog - nbBits); - - if (targetLog - nbBits >= minBits) { /* enough room for a second symbol */ - U32 sortedRank; - int minWeight = nbBits + scaleLog; - if (minWeight < 1) - minWeight = 1; - sortedRank = rankStart[minWeight]; - HUF_fillDTableX4Level2(DTable + start, targetLog - nbBits, nbBits, rankValOrigin[nbBits], minWeight, sortedList + sortedRank, - sortedListSize - sortedRank, nbBitsBaseline, symbol); - } else { - HUF_DEltX4 DElt; - ZSTD_writeLE16(&(DElt.sequence), symbol); - DElt.nbBits = (BYTE)(nbBits); - DElt.length = 1; - { - U32 const end = start + length; - U32 u; - for (u = start; u < end; u++) - DTable[u] = DElt; - } - } - rankVal[weight] += length; - } -} - -size_t HUF_readDTableX4_wksp(HUF_DTable *DTable, const void *src, size_t srcSize, void *workspace, size_t workspaceSize) -{ - U32 tableLog, maxW, sizeOfSort, nbSymbols; - DTableDesc dtd = HUF_getDTableDesc(DTable); - U32 const maxTableLog = dtd.maxTableLog; - size_t iSize; - void *dtPtr = DTable + 1; /* force compiler to avoid strict-aliasing */ - HUF_DEltX4 *const dt = (HUF_DEltX4 *)dtPtr; - U32 *rankStart; - - rankValCol_t *rankVal; - U32 *rankStats; - U32 *rankStart0; - sortedSymbol_t *sortedSymbol; - BYTE *weightList; - size_t spaceUsed32 = 0; - - HUF_STATIC_ASSERT((sizeof(rankValCol_t) & 3) == 0); - - rankVal = (rankValCol_t *)((U32 *)workspace + spaceUsed32); - spaceUsed32 += (sizeof(rankValCol_t) * HUF_TABLELOG_MAX) >> 2; - rankStats = (U32 *)workspace + spaceUsed32; - spaceUsed32 += HUF_TABLELOG_MAX + 1; - rankStart0 = (U32 *)workspace + spaceUsed32; - spaceUsed32 += HUF_TABLELOG_MAX + 2; - sortedSymbol = (sortedSymbol_t *)((U32 *)workspace + spaceUsed32); - spaceUsed32 += ALIGN(sizeof(sortedSymbol_t) * (HUF_SYMBOLVALUE_MAX + 1), sizeof(U32)) >> 2; - weightList = (BYTE *)((U32 *)workspace + spaceUsed32); - spaceUsed32 += ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2; - - if ((spaceUsed32 << 2) > workspaceSize) - return ERROR(tableLog_tooLarge); - workspace = (U32 *)workspace + spaceUsed32; - workspaceSize -= (spaceUsed32 << 2); - - rankStart = rankStart0 + 1; - memset(rankStats, 0, sizeof(U32) * (2 * HUF_TABLELOG_MAX + 2 + 1)); - - HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */ - if (maxTableLog > HUF_TABLELOG_MAX) - return ERROR(tableLog_tooLarge); - /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ - - iSize = HUF_readStats_wksp(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize, workspace, workspaceSize); - if (HUF_isError(iSize)) - return iSize; - - /* check result */ - if (tableLog > maxTableLog) - return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ - - /* find maxWeight */ - for (maxW = tableLog; rankStats[maxW] == 0; maxW--) { - } /* necessarily finds a solution before 0 */ - - /* Get start index of each weight */ - { - U32 w, nextRankStart = 0; - for (w = 1; w < maxW + 1; w++) { - U32 curr = nextRankStart; - nextRankStart += rankStats[w]; - rankStart[w] = curr; - } - rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ - sizeOfSort = nextRankStart; - } - - /* sort symbols by weight */ - { - U32 s; - for (s = 0; s < nbSymbols; s++) { - U32 const w = weightList[s]; - U32 const r = rankStart[w]++; - sortedSymbol[r].symbol = (BYTE)s; - sortedSymbol[r].weight = (BYTE)w; - } - rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ - } - - /* Build rankVal */ - { - U32 *const rankVal0 = rankVal[0]; - { - int const rescale = (maxTableLog - tableLog) - 1; /* tableLog <= maxTableLog */ - U32 nextRankVal = 0; - U32 w; - for (w = 1; w < maxW + 1; w++) { - U32 curr = nextRankVal; - nextRankVal += rankStats[w] << (w + rescale); - rankVal0[w] = curr; - } - } - { - U32 const minBits = tableLog + 1 - maxW; - U32 consumed; - for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) { - U32 *const rankValPtr = rankVal[consumed]; - U32 w; - for (w = 1; w < maxW + 1; w++) { - rankValPtr[w] = rankVal0[w] >> consumed; - } - } - } - } - - HUF_fillDTableX4(dt, maxTableLog, sortedSymbol, sizeOfSort, rankStart0, rankVal, maxW, tableLog + 1); - - dtd.tableLog = (BYTE)maxTableLog; - dtd.tableType = 1; - memcpy(DTable, &dtd, sizeof(dtd)); - return iSize; -} - -static U32 HUF_decodeSymbolX4(void *op, BIT_DStream_t *DStream, const HUF_DEltX4 *dt, const U32 dtLog) -{ - size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ - memcpy(op, dt + val, 2); - BIT_skipBits(DStream, dt[val].nbBits); - return dt[val].length; -} - -static U32 HUF_decodeLastSymbolX4(void *op, BIT_DStream_t *DStream, const HUF_DEltX4 *dt, const U32 dtLog) -{ - size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ - memcpy(op, dt + val, 1); - if (dt[val].length == 1) - BIT_skipBits(DStream, dt[val].nbBits); - else { - if (DStream->bitsConsumed < (sizeof(DStream->bitContainer) * 8)) { - BIT_skipBits(DStream, dt[val].nbBits); - if (DStream->bitsConsumed > (sizeof(DStream->bitContainer) * 8)) - /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ - DStream->bitsConsumed = (sizeof(DStream->bitContainer) * 8); - } - } - return 1; -} - -#define HUF_DECODE_SYMBOLX4_0(ptr, DStreamPtr) ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) - -#define HUF_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \ - if (ZSTD_64bits() || (HUF_TABLELOG_MAX <= 12)) \ - ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) - -#define HUF_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \ - if (ZSTD_64bits()) \ - ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) - -FORCE_INLINE size_t HUF_decodeStreamX4(BYTE *p, BIT_DStream_t *bitDPtr, BYTE *const pEnd, const HUF_DEltX4 *const dt, const U32 dtLog) -{ - BYTE *const pStart = p; - - /* up to 8 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd - (sizeof(bitDPtr->bitContainer) - 1))) { - HUF_DECODE_SYMBOLX4_2(p, bitDPtr); - HUF_DECODE_SYMBOLX4_1(p, bitDPtr); - HUF_DECODE_SYMBOLX4_2(p, bitDPtr); - HUF_DECODE_SYMBOLX4_0(p, bitDPtr); - } - - /* closer to end : up to 2 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd - 2)) - HUF_DECODE_SYMBOLX4_0(p, bitDPtr); - - while (p <= pEnd - 2) - HUF_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ - - if (p < pEnd) - p += HUF_decodeLastSymbolX4(p, bitDPtr, dt, dtLog); - - return p - pStart; -} - -static size_t HUF_decompress1X4_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -{ - BIT_DStream_t bitD; - - /* Init */ - { - size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); - if (HUF_isError(errorCode)) - return errorCode; - } - - /* decode */ - { - BYTE *const ostart = (BYTE *)dst; - BYTE *const oend = ostart + dstSize; - const void *const dtPtr = DTable + 1; /* force compiler to not use strict-aliasing */ - const HUF_DEltX4 *const dt = (const HUF_DEltX4 *)dtPtr; - DTableDesc const dtd = HUF_getDTableDesc(DTable); - HUF_decodeStreamX4(ostart, &bitD, oend, dt, dtd.tableLog); - } - - /* check */ - if (!BIT_endOfDStream(&bitD)) - return ERROR(corruption_detected); - - /* decoded size */ - return dstSize; -} - -size_t HUF_decompress1X4_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 1) - return ERROR(GENERIC); - return HUF_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -} - -size_t HUF_decompress1X4_DCtx_wksp(HUF_DTable *DCtx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize) -{ - const BYTE *ip = (const BYTE *)cSrc; - - size_t const hSize = HUF_readDTableX4_wksp(DCtx, cSrc, cSrcSize, workspace, workspaceSize); - if (HUF_isError(hSize)) - return hSize; - if (hSize >= cSrcSize) - return ERROR(srcSize_wrong); - ip += hSize; - cSrcSize -= hSize; - - return HUF_decompress1X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx); -} - -static size_t HUF_decompress4X4_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -{ - if (cSrcSize < 10) - return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ - - { - const BYTE *const istart = (const BYTE *)cSrc; - BYTE *const ostart = (BYTE *)dst; - BYTE *const oend = ostart + dstSize; - const void *const dtPtr = DTable + 1; - const HUF_DEltX4 *const dt = (const HUF_DEltX4 *)dtPtr; - - /* Init */ - BIT_DStream_t bitD1; - BIT_DStream_t bitD2; - BIT_DStream_t bitD3; - BIT_DStream_t bitD4; - size_t const length1 = ZSTD_readLE16(istart); - size_t const length2 = ZSTD_readLE16(istart + 2); - size_t const length3 = ZSTD_readLE16(istart + 4); - size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); - const BYTE *const istart1 = istart + 6; /* jumpTable */ - const BYTE *const istart2 = istart1 + length1; - const BYTE *const istart3 = istart2 + length2; - const BYTE *const istart4 = istart3 + length3; - size_t const segmentSize = (dstSize + 3) / 4; - BYTE *const opStart2 = ostart + segmentSize; - BYTE *const opStart3 = opStart2 + segmentSize; - BYTE *const opStart4 = opStart3 + segmentSize; - BYTE *op1 = ostart; - BYTE *op2 = opStart2; - BYTE *op3 = opStart3; - BYTE *op4 = opStart4; - U32 endSignal; - DTableDesc const dtd = HUF_getDTableDesc(DTable); - U32 const dtLog = dtd.tableLog; - - if (length4 > cSrcSize) - return ERROR(corruption_detected); /* overflow */ - { - size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); - if (HUF_isError(errorCode)) - return errorCode; - } - { - size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); - if (HUF_isError(errorCode)) - return errorCode; - } - { - size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); - if (HUF_isError(errorCode)) - return errorCode; - } - { - size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); - if (HUF_isError(errorCode)) - return errorCode; - } - - /* 16-32 symbols per loop (4-8 symbols per stream) */ - endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - for (; (endSignal == BIT_DStream_unfinished) & (op4 < (oend - (sizeof(bitD4.bitContainer) - 1)));) { - HUF_DECODE_SYMBOLX4_2(op1, &bitD1); - HUF_DECODE_SYMBOLX4_2(op2, &bitD2); - HUF_DECODE_SYMBOLX4_2(op3, &bitD3); - HUF_DECODE_SYMBOLX4_2(op4, &bitD4); - HUF_DECODE_SYMBOLX4_1(op1, &bitD1); - HUF_DECODE_SYMBOLX4_1(op2, &bitD2); - HUF_DECODE_SYMBOLX4_1(op3, &bitD3); - HUF_DECODE_SYMBOLX4_1(op4, &bitD4); - HUF_DECODE_SYMBOLX4_2(op1, &bitD1); - HUF_DECODE_SYMBOLX4_2(op2, &bitD2); - HUF_DECODE_SYMBOLX4_2(op3, &bitD3); - HUF_DECODE_SYMBOLX4_2(op4, &bitD4); - HUF_DECODE_SYMBOLX4_0(op1, &bitD1); - HUF_DECODE_SYMBOLX4_0(op2, &bitD2); - HUF_DECODE_SYMBOLX4_0(op3, &bitD3); - HUF_DECODE_SYMBOLX4_0(op4, &bitD4); - - endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - } - - /* check corruption */ - if (op1 > opStart2) - return ERROR(corruption_detected); - if (op2 > opStart3) - return ERROR(corruption_detected); - if (op3 > opStart4) - return ERROR(corruption_detected); - /* note : op4 already verified within main loop */ - - /* finish bitStreams one by one */ - HUF_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog); - HUF_decodeStreamX4(op2, &bitD2, opStart3, dt, dtLog); - HUF_decodeStreamX4(op3, &bitD3, opStart4, dt, dtLog); - HUF_decodeStreamX4(op4, &bitD4, oend, dt, dtLog); - - /* check */ - { - U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); - if (!endCheck) - return ERROR(corruption_detected); - } - - /* decoded size */ - return dstSize; - } -} - -size_t HUF_decompress4X4_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 1) - return ERROR(GENERIC); - return HUF_decompress4X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -} - -size_t HUF_decompress4X4_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize) -{ - const BYTE *ip = (const BYTE *)cSrc; - - size_t hSize = HUF_readDTableX4_wksp(dctx, cSrc, cSrcSize, workspace, workspaceSize); - if (HUF_isError(hSize)) - return hSize; - if (hSize >= cSrcSize) - return ERROR(srcSize_wrong); - ip += hSize; - cSrcSize -= hSize; - - return HUF_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx); -} - -/* ********************************/ -/* Generic decompression selector */ -/* ********************************/ - -size_t HUF_decompress1X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -{ - DTableDesc const dtd = HUF_getDTableDesc(DTable); - return dtd.tableType ? HUF_decompress1X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) - : HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); -} - -size_t HUF_decompress4X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -{ - DTableDesc const dtd = HUF_getDTableDesc(DTable); - return dtd.tableType ? HUF_decompress4X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) - : HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); -} - -typedef struct { - U32 tableTime; - U32 decode256Time; -} algo_time_t; -static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = { - /* single, double, quad */ - {{0, 0}, {1, 1}, {2, 2}}, /* Q==0 : impossible */ - {{0, 0}, {1, 1}, {2, 2}}, /* Q==1 : impossible */ - {{38, 130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */ - {{448, 128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */ - {{556, 128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */ - {{714, 128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */ - {{883, 128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */ - {{897, 128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */ - {{926, 128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */ - {{947, 128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */ - {{1107, 128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */ - {{1177, 128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */ - {{1242, 128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */ - {{1349, 128}, {2644, 106}, {5260, 106}}, /* Q ==13 : 81-87% */ - {{1455, 128}, {2422, 124}, {4174, 124}}, /* Q ==14 : 87-93% */ - {{722, 128}, {1891, 145}, {1936, 146}}, /* Q ==15 : 93-99% */ -}; - -/** HUF_selectDecoder() : -* Tells which decoder is likely to decode faster, -* based on a set of pre-determined metrics. -* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 . -* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ -U32 HUF_selectDecoder(size_t dstSize, size_t cSrcSize) -{ - /* decoder timing evaluation */ - U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */ - U32 const D256 = (U32)(dstSize >> 8); - U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); - U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); - DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, for cache eviction */ - - return DTime1 < DTime0; -} - -typedef size_t (*decompressionAlgo)(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize); - -size_t HUF_decompress4X_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize) -{ - /* validation checks */ - if (dstSize == 0) - return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) - return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { - memcpy(dst, cSrc, dstSize); - return dstSize; - } /* not compressed */ - if (cSrcSize == 1) { - memset(dst, *(const BYTE *)cSrc, dstSize); - return dstSize; - } /* RLE */ - - { - U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); - return algoNb ? HUF_decompress4X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize) - : HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize); - } -} - -size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize) -{ - /* validation checks */ - if (dstSize == 0) - return ERROR(dstSize_tooSmall); - if ((cSrcSize >= dstSize) || (cSrcSize <= 1)) - return ERROR(corruption_detected); /* invalid */ - - { - U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); - return algoNb ? HUF_decompress4X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize) - : HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize); - } -} - -size_t HUF_decompress1X_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize) -{ - /* validation checks */ - if (dstSize == 0) - return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) - return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { - memcpy(dst, cSrc, dstSize); - return dstSize; - } /* not compressed */ - if (cSrcSize == 1) { - memset(dst, *(const BYTE *)cSrc, dstSize); - return dstSize; - } /* RLE */ - - { - U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); - return algoNb ? HUF_decompress1X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize) - : HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize); - } -} diff --git a/lib/zstd/mem.h b/lib/zstd/mem.h deleted file mode 100644 index 93d7a2c377fe..000000000000 --- a/lib/zstd/mem.h +++ /dev/null @@ -1,151 +0,0 @@ -/** - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of https://github.com/facebook/zstd. - * An additional grant of patent rights can be found in the PATENTS file in the - * same directory. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - */ - -#ifndef MEM_H_MODULE -#define MEM_H_MODULE - -/*-**************************************** -* Dependencies -******************************************/ -#include -#include /* memcpy */ -#include /* size_t, ptrdiff_t */ - -/*-**************************************** -* Compiler specifics -******************************************/ -#define ZSTD_STATIC static inline - -/*-************************************************************** -* Basic Types -*****************************************************************/ -typedef uint8_t BYTE; -typedef uint16_t U16; -typedef int16_t S16; -typedef uint32_t U32; -typedef int32_t S32; -typedef uint64_t U64; -typedef int64_t S64; -typedef ptrdiff_t iPtrDiff; -typedef uintptr_t uPtrDiff; - -/*-************************************************************** -* Memory I/O -*****************************************************************/ -ZSTD_STATIC unsigned ZSTD_32bits(void) { return sizeof(size_t) == 4; } -ZSTD_STATIC unsigned ZSTD_64bits(void) { return sizeof(size_t) == 8; } - -#if defined(__LITTLE_ENDIAN) -#define ZSTD_LITTLE_ENDIAN 1 -#else -#define ZSTD_LITTLE_ENDIAN 0 -#endif - -ZSTD_STATIC unsigned ZSTD_isLittleEndian(void) { return ZSTD_LITTLE_ENDIAN; } - -ZSTD_STATIC U16 ZSTD_read16(const void *memPtr) { return get_unaligned((const U16 *)memPtr); } - -ZSTD_STATIC U32 ZSTD_read32(const void *memPtr) { return get_unaligned((const U32 *)memPtr); } - -ZSTD_STATIC U64 ZSTD_read64(const void *memPtr) { return get_unaligned((const U64 *)memPtr); } - -ZSTD_STATIC size_t ZSTD_readST(const void *memPtr) { return get_unaligned((const size_t *)memPtr); } - -ZSTD_STATIC void ZSTD_write16(void *memPtr, U16 value) { put_unaligned(value, (U16 *)memPtr); } - -ZSTD_STATIC void ZSTD_write32(void *memPtr, U32 value) { put_unaligned(value, (U32 *)memPtr); } - -ZSTD_STATIC void ZSTD_write64(void *memPtr, U64 value) { put_unaligned(value, (U64 *)memPtr); } - -/*=== Little endian r/w ===*/ - -ZSTD_STATIC U16 ZSTD_readLE16(const void *memPtr) { return get_unaligned_le16(memPtr); } - -ZSTD_STATIC void ZSTD_writeLE16(void *memPtr, U16 val) { put_unaligned_le16(val, memPtr); } - -ZSTD_STATIC U32 ZSTD_readLE24(const void *memPtr) { return ZSTD_readLE16(memPtr) + (((const BYTE *)memPtr)[2] << 16); } - -ZSTD_STATIC void ZSTD_writeLE24(void *memPtr, U32 val) -{ - ZSTD_writeLE16(memPtr, (U16)val); - ((BYTE *)memPtr)[2] = (BYTE)(val >> 16); -} - -ZSTD_STATIC U32 ZSTD_readLE32(const void *memPtr) { return get_unaligned_le32(memPtr); } - -ZSTD_STATIC void ZSTD_writeLE32(void *memPtr, U32 val32) { put_unaligned_le32(val32, memPtr); } - -ZSTD_STATIC U64 ZSTD_readLE64(const void *memPtr) { return get_unaligned_le64(memPtr); } - -ZSTD_STATIC void ZSTD_writeLE64(void *memPtr, U64 val64) { put_unaligned_le64(val64, memPtr); } - -ZSTD_STATIC size_t ZSTD_readLEST(const void *memPtr) -{ - if (ZSTD_32bits()) - return (size_t)ZSTD_readLE32(memPtr); - else - return (size_t)ZSTD_readLE64(memPtr); -} - -ZSTD_STATIC void ZSTD_writeLEST(void *memPtr, size_t val) -{ - if (ZSTD_32bits()) - ZSTD_writeLE32(memPtr, (U32)val); - else - ZSTD_writeLE64(memPtr, (U64)val); -} - -/*=== Big endian r/w ===*/ - -ZSTD_STATIC U32 ZSTD_readBE32(const void *memPtr) { return get_unaligned_be32(memPtr); } - -ZSTD_STATIC void ZSTD_writeBE32(void *memPtr, U32 val32) { put_unaligned_be32(val32, memPtr); } - -ZSTD_STATIC U64 ZSTD_readBE64(const void *memPtr) { return get_unaligned_be64(memPtr); } - -ZSTD_STATIC void ZSTD_writeBE64(void *memPtr, U64 val64) { put_unaligned_be64(val64, memPtr); } - -ZSTD_STATIC size_t ZSTD_readBEST(const void *memPtr) -{ - if (ZSTD_32bits()) - return (size_t)ZSTD_readBE32(memPtr); - else - return (size_t)ZSTD_readBE64(memPtr); -} - -ZSTD_STATIC void ZSTD_writeBEST(void *memPtr, size_t val) -{ - if (ZSTD_32bits()) - ZSTD_writeBE32(memPtr, (U32)val); - else - ZSTD_writeBE64(memPtr, (U64)val); -} - -/* function safe only for comparisons */ -ZSTD_STATIC U32 ZSTD_readMINMATCH(const void *memPtr, U32 length) -{ - switch (length) { - default: - case 4: return ZSTD_read32(memPtr); - case 3: - if (ZSTD_isLittleEndian()) - return ZSTD_read32(memPtr) << 8; - else - return ZSTD_read32(memPtr) >> 8; - } -} - -#endif /* MEM_H_MODULE */ diff --git a/lib/zstd/zstd_common.c b/lib/zstd/zstd_common.c deleted file mode 100644 index a282624ee155..000000000000 --- a/lib/zstd/zstd_common.c +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of https://github.com/facebook/zstd. - * An additional grant of patent rights can be found in the PATENTS file in the - * same directory. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - */ - -/*-************************************* -* Dependencies -***************************************/ -#include "error_private.h" -#include "zstd_internal.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode, ZSTD_getErrorString, ZSTD_versionNumber */ -#include - -/*=************************************************************** -* Custom allocator -****************************************************************/ - -#define stack_push(stack, size) \ - ({ \ - void *const ptr = ZSTD_PTR_ALIGN((stack)->ptr); \ - (stack)->ptr = (char *)ptr + (size); \ - (stack)->ptr <= (stack)->end ? ptr : NULL; \ - }) - -ZSTD_customMem ZSTD_initStack(void *workspace, size_t workspaceSize) -{ - ZSTD_customMem stackMem = {ZSTD_stackAlloc, ZSTD_stackFree, workspace}; - ZSTD_stack *stack = (ZSTD_stack *)workspace; - /* Verify preconditions */ - if (!workspace || workspaceSize < sizeof(ZSTD_stack) || workspace != ZSTD_PTR_ALIGN(workspace)) { - ZSTD_customMem error = {NULL, NULL, NULL}; - return error; - } - /* Initialize the stack */ - stack->ptr = workspace; - stack->end = (char *)workspace + workspaceSize; - stack_push(stack, sizeof(ZSTD_stack)); - return stackMem; -} - -void *ZSTD_stackAllocAll(void *opaque, size_t *size) -{ - ZSTD_stack *stack = (ZSTD_stack *)opaque; - *size = (BYTE const *)stack->end - (BYTE *)ZSTD_PTR_ALIGN(stack->ptr); - return stack_push(stack, *size); -} - -void *ZSTD_stackAlloc(void *opaque, size_t size) -{ - ZSTD_stack *stack = (ZSTD_stack *)opaque; - return stack_push(stack, size); -} -void ZSTD_stackFree(void *opaque, void *address) -{ - (void)opaque; - (void)address; -} - -void *ZSTD_malloc(size_t size, ZSTD_customMem customMem) { return customMem.customAlloc(customMem.opaque, size); } - -void ZSTD_free(void *ptr, ZSTD_customMem customMem) -{ - if (ptr != NULL) - customMem.customFree(customMem.opaque, ptr); -} diff --git a/lib/zstd/zstd_compress_module.c b/lib/zstd/zstd_compress_module.c new file mode 100644 index 000000000000..37d08ff43e6e --- /dev/null +++ b/lib/zstd/zstd_compress_module.c @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include +#include +#include +#include + +#include "common/zstd_deps.h" +#include "common/zstd_internal.h" + +int zstd_min_clevel(void) +{ + return ZSTD_minCLevel(); +} +EXPORT_SYMBOL(zstd_min_clevel); + +int zstd_max_clevel(void) +{ + return ZSTD_maxCLevel(); +} +EXPORT_SYMBOL(zstd_max_clevel); + +size_t zstd_compress_bound(size_t src_size) +{ + return ZSTD_compressBound(src_size); +} +EXPORT_SYMBOL(zstd_compress_bound); + +zstd_parameters zstd_get_params(int level, + unsigned long long estimated_src_size) +{ + return ZSTD_getParams(level, estimated_src_size, 0); +} +EXPORT_SYMBOL(zstd_get_params); + +size_t zstd_cctx_workspace_bound(const zstd_compression_parameters *cparams) +{ + return ZSTD_estimateCCtxSize_usingCParams(*cparams); +} +EXPORT_SYMBOL(zstd_cctx_workspace_bound); + +zstd_cctx *zstd_init_cctx(void *workspace, size_t workspace_size) +{ + if (workspace == NULL) + return NULL; + return ZSTD_initStaticCCtx(workspace, workspace_size); +} +EXPORT_SYMBOL(zstd_init_cctx); + +size_t zstd_compress_cctx(zstd_cctx *cctx, void *dst, size_t dst_capacity, + const void *src, size_t src_size, const zstd_parameters *parameters) +{ + return ZSTD_compress_advanced(cctx, dst, dst_capacity, src, src_size, NULL, 0, *parameters); +} +EXPORT_SYMBOL(zstd_compress_cctx); + +size_t zstd_cstream_workspace_bound(const zstd_compression_parameters *cparams) +{ + return ZSTD_estimateCStreamSize_usingCParams(*cparams); +} +EXPORT_SYMBOL(zstd_cstream_workspace_bound); + +zstd_cstream *zstd_init_cstream(const zstd_parameters *parameters, + unsigned long long pledged_src_size, void *workspace, size_t workspace_size) +{ + zstd_cstream *cstream; + size_t ret; + + if (workspace == NULL) + return NULL; + + cstream = ZSTD_initStaticCStream(workspace, workspace_size); + if (cstream == NULL) + return NULL; + + /* 0 means unknown in linux zstd API but means 0 in new zstd API */ + if (pledged_src_size == 0) + pledged_src_size = ZSTD_CONTENTSIZE_UNKNOWN; + + ret = ZSTD_initCStream_advanced(cstream, NULL, 0, *parameters, pledged_src_size); + if (ZSTD_isError(ret)) + return NULL; + + return cstream; +} +EXPORT_SYMBOL(zstd_init_cstream); + +size_t zstd_reset_cstream(zstd_cstream *cstream, + unsigned long long pledged_src_size) +{ + return ZSTD_resetCStream(cstream, pledged_src_size); +} +EXPORT_SYMBOL(zstd_reset_cstream); + +size_t zstd_compress_stream(zstd_cstream *cstream, zstd_out_buffer *output, + zstd_in_buffer *input) +{ + return ZSTD_compressStream(cstream, output, input); +} +EXPORT_SYMBOL(zstd_compress_stream); + +size_t zstd_flush_stream(zstd_cstream *cstream, zstd_out_buffer *output) +{ + return ZSTD_flushStream(cstream, output); +} +EXPORT_SYMBOL(zstd_flush_stream); + +size_t zstd_end_stream(zstd_cstream *cstream, zstd_out_buffer *output) +{ + return ZSTD_endStream(cstream, output); +} +EXPORT_SYMBOL(zstd_end_stream); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_DESCRIPTION("Zstd Compressor"); diff --git a/lib/zstd/zstd_decompress_module.c b/lib/zstd/zstd_decompress_module.c new file mode 100644 index 000000000000..15005cdb9eca --- /dev/null +++ b/lib/zstd/zstd_decompress_module.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include +#include +#include +#include + +#include "common/zstd_deps.h" + +/* Common symbols. zstd_compress must depend on zstd_decompress. */ + +unsigned int zstd_is_error(size_t code) +{ + return ZSTD_isError(code); +} +EXPORT_SYMBOL(zstd_is_error); + +zstd_error_code zstd_get_error_code(size_t code) +{ + return ZSTD_getErrorCode(code); +} +EXPORT_SYMBOL(zstd_get_error_code); + +const char *zstd_get_error_name(size_t code) +{ + return ZSTD_getErrorName(code); +} +EXPORT_SYMBOL(zstd_get_error_name); + +/* Decompression symbols. */ + +size_t zstd_dctx_workspace_bound(void) +{ + return ZSTD_estimateDCtxSize(); +} +EXPORT_SYMBOL(zstd_dctx_workspace_bound); + +zstd_dctx *zstd_init_dctx(void *workspace, size_t workspace_size) +{ + if (workspace == NULL) + return NULL; + return ZSTD_initStaticDCtx(workspace, workspace_size); +} +EXPORT_SYMBOL(zstd_init_dctx); + +size_t zstd_decompress_dctx(zstd_dctx *dctx, void *dst, size_t dst_capacity, + const void *src, size_t src_size) +{ + return ZSTD_decompressDCtx(dctx, dst, dst_capacity, src, src_size); +} +EXPORT_SYMBOL(zstd_decompress_dctx); + +size_t zstd_dstream_workspace_bound(size_t max_window_size) +{ + return ZSTD_estimateDStreamSize(max_window_size); +} +EXPORT_SYMBOL(zstd_dstream_workspace_bound); + +zstd_dstream *zstd_init_dstream(size_t max_window_size, void *workspace, + size_t workspace_size) +{ + if (workspace == NULL) + return NULL; + (void)max_window_size; + return ZSTD_initStaticDStream(workspace, workspace_size); +} +EXPORT_SYMBOL(zstd_init_dstream); + +size_t zstd_reset_dstream(zstd_dstream *dstream) +{ + return ZSTD_resetDStream(dstream); +} +EXPORT_SYMBOL(zstd_reset_dstream); + +size_t zstd_decompress_stream(zstd_dstream *dstream, zstd_out_buffer *output, + zstd_in_buffer *input) +{ + return ZSTD_decompressStream(dstream, output, input); +} +EXPORT_SYMBOL(zstd_decompress_stream); + +size_t zstd_find_frame_compressed_size(const void *src, size_t src_size) +{ + return ZSTD_findFrameCompressedSize(src, src_size); +} +EXPORT_SYMBOL(zstd_find_frame_compressed_size); + +size_t zstd_get_frame_header(zstd_frame_header *header, const void *src, + size_t src_size) +{ + return ZSTD_getFrameHeader(header, src, src_size); +} +EXPORT_SYMBOL(zstd_get_frame_header); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_DESCRIPTION("Zstd Decompressor"); diff --git a/lib/zstd/zstd_internal.h b/lib/zstd/zstd_internal.h deleted file mode 100644 index dac753397f86..000000000000 --- a/lib/zstd/zstd_internal.h +++ /dev/null @@ -1,273 +0,0 @@ -/** - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of https://github.com/facebook/zstd. - * An additional grant of patent rights can be found in the PATENTS file in the - * same directory. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - */ - -#ifndef ZSTD_CCOMMON_H_MODULE -#define ZSTD_CCOMMON_H_MODULE - -/*-******************************************************* -* Compiler specifics -*********************************************************/ -#define FORCE_INLINE static __always_inline -#define FORCE_NOINLINE static noinline - -/*-************************************* -* Dependencies -***************************************/ -#include "error_private.h" -#include "mem.h" -#include -#include -#include -#include - -/*-************************************* -* shared macros -***************************************/ -#define MIN(a, b) ((a) < (b) ? (a) : (b)) -#define MAX(a, b) ((a) > (b) ? (a) : (b)) -#define CHECK_F(f) \ - { \ - size_t const errcod = f; \ - if (ERR_isError(errcod)) \ - return errcod; \ - } /* check and Forward error code */ -#define CHECK_E(f, e) \ - { \ - size_t const errcod = f; \ - if (ERR_isError(errcod)) \ - return ERROR(e); \ - } /* check and send Error code */ -#define ZSTD_STATIC_ASSERT(c) \ - { \ - enum { ZSTD_static_assert = 1 / (int)(!!(c)) }; \ - } - -/*-************************************* -* Common constants -***************************************/ -#define ZSTD_OPT_NUM (1 << 12) -#define ZSTD_DICT_MAGIC 0xEC30A437 /* v0.7+ */ - -#define ZSTD_REP_NUM 3 /* number of repcodes */ -#define ZSTD_REP_CHECK (ZSTD_REP_NUM) /* number of repcodes to check by the optimal parser */ -#define ZSTD_REP_MOVE (ZSTD_REP_NUM - 1) -#define ZSTD_REP_MOVE_OPT (ZSTD_REP_NUM) -static const U32 repStartValue[ZSTD_REP_NUM] = {1, 4, 8}; - -#define KB *(1 << 10) -#define MB *(1 << 20) -#define GB *(1U << 30) - -#define BIT7 128 -#define BIT6 64 -#define BIT5 32 -#define BIT4 16 -#define BIT1 2 -#define BIT0 1 - -#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 -static const size_t ZSTD_fcs_fieldSize[4] = {0, 2, 4, 8}; -static const size_t ZSTD_did_fieldSize[4] = {0, 1, 2, 4}; - -#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ -static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; -typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; - -#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ -#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */ - -#define HufLog 12 -typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e; - -#define LONGNBSEQ 0x7F00 - -#define MINMATCH 3 -#define EQUAL_READ32 4 - -#define Litbits 8 -#define MaxLit ((1 << Litbits) - 1) -#define MaxML 52 -#define MaxLL 35 -#define MaxOff 28 -#define MaxSeq MAX(MaxLL, MaxML) /* Assumption : MaxOff < MaxLL,MaxML */ -#define MLFSELog 9 -#define LLFSELog 9 -#define OffFSELog 8 - -static const U32 LL_bits[MaxLL + 1] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; -static const S16 LL_defaultNorm[MaxLL + 1] = {4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1, -1, -1, -1, -1}; -#define LL_DEFAULTNORMLOG 6 /* for static allocation */ -static const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG; - -static const U32 ML_bits[MaxML + 1] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; -static const S16 ML_defaultNorm[MaxML + 1] = {1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, -1, -1}; -#define ML_DEFAULTNORMLOG 6 /* for static allocation */ -static const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG; - -static const S16 OF_defaultNorm[MaxOff + 1] = {1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1}; -#define OF_DEFAULTNORMLOG 5 /* for static allocation */ -static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG; - -/*-******************************************* -* Shared functions to include for inlining -*********************************************/ -ZSTD_STATIC void ZSTD_copy8(void *dst, const void *src) { - /* - * zstd relies heavily on gcc being able to analyze and inline this - * memcpy() call, since it is called in a tight loop. Preboot mode - * is compiled in freestanding mode, which stops gcc from analyzing - * memcpy(). Use __builtin_memcpy() to tell gcc to analyze this as a - * regular memcpy(). - */ - __builtin_memcpy(dst, src, 8); -} -/*! ZSTD_wildcopy() : -* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */ -#define WILDCOPY_OVERLENGTH 8 -ZSTD_STATIC void ZSTD_wildcopy(void *dst, const void *src, ptrdiff_t length) -{ - const BYTE* ip = (const BYTE*)src; - BYTE* op = (BYTE*)dst; - BYTE* const oend = op + length; -#if defined(GCC_VERSION) && GCC_VERSION >= 70000 && GCC_VERSION < 70200 - /* - * Work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388. - * Avoid the bad case where the loop only runs once by handling the - * special case separately. This doesn't trigger the bug because it - * doesn't involve pointer/integer overflow. - */ - if (length <= 8) - return ZSTD_copy8(dst, src); -#endif - do { - ZSTD_copy8(op, ip); - op += 8; - ip += 8; - } while (op < oend); -} - -/*-******************************************* -* Private interfaces -*********************************************/ -typedef struct ZSTD_stats_s ZSTD_stats_t; - -typedef struct { - U32 off; - U32 len; -} ZSTD_match_t; - -typedef struct { - U32 price; - U32 off; - U32 mlen; - U32 litlen; - U32 rep[ZSTD_REP_NUM]; -} ZSTD_optimal_t; - -typedef struct seqDef_s { - U32 offset; - U16 litLength; - U16 matchLength; -} seqDef; - -typedef struct { - seqDef *sequencesStart; - seqDef *sequences; - BYTE *litStart; - BYTE *lit; - BYTE *llCode; - BYTE *mlCode; - BYTE *ofCode; - U32 longLengthID; /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */ - U32 longLengthPos; - /* opt */ - ZSTD_optimal_t *priceTable; - ZSTD_match_t *matchTable; - U32 *matchLengthFreq; - U32 *litLengthFreq; - U32 *litFreq; - U32 *offCodeFreq; - U32 matchLengthSum; - U32 matchSum; - U32 litLengthSum; - U32 litSum; - U32 offCodeSum; - U32 log2matchLengthSum; - U32 log2matchSum; - U32 log2litLengthSum; - U32 log2litSum; - U32 log2offCodeSum; - U32 factor; - U32 staticPrices; - U32 cachedPrice; - U32 cachedLitLength; - const BYTE *cachedLiterals; -} seqStore_t; - -const seqStore_t *ZSTD_getSeqStore(const ZSTD_CCtx *ctx); -void ZSTD_seqToCodes(const seqStore_t *seqStorePtr); -int ZSTD_isSkipFrame(ZSTD_DCtx *dctx); - -/*= Custom memory allocation functions */ -typedef void *(*ZSTD_allocFunction)(void *opaque, size_t size); -typedef void (*ZSTD_freeFunction)(void *opaque, void *address); -typedef struct { - ZSTD_allocFunction customAlloc; - ZSTD_freeFunction customFree; - void *opaque; -} ZSTD_customMem; - -void *ZSTD_malloc(size_t size, ZSTD_customMem customMem); -void ZSTD_free(void *ptr, ZSTD_customMem customMem); - -/*====== stack allocation ======*/ - -typedef struct { - void *ptr; - const void *end; -} ZSTD_stack; - -#define ZSTD_ALIGN(x) ALIGN(x, sizeof(size_t)) -#define ZSTD_PTR_ALIGN(p) PTR_ALIGN(p, sizeof(size_t)) - -ZSTD_customMem ZSTD_initStack(void *workspace, size_t workspaceSize); - -void *ZSTD_stackAllocAll(void *opaque, size_t *size); -void *ZSTD_stackAlloc(void *opaque, size_t size); -void ZSTD_stackFree(void *opaque, void *address); - -/*====== common function ======*/ - -ZSTD_STATIC U32 ZSTD_highbit32(U32 val) { return 31 - __builtin_clz(val); } - -/* hidden functions */ - -/* ZSTD_invalidateRepCodes() : - * ensures next compression will not use repcodes from previous block. - * Note : only works with regular variant; - * do not use with extDict variant ! */ -void ZSTD_invalidateRepCodes(ZSTD_CCtx *cctx); - -size_t ZSTD_freeCCtx(ZSTD_CCtx *cctx); -size_t ZSTD_freeDCtx(ZSTD_DCtx *dctx); -size_t ZSTD_freeCDict(ZSTD_CDict *cdict); -size_t ZSTD_freeDDict(ZSTD_DDict *cdict); -size_t ZSTD_freeCStream(ZSTD_CStream *zcs); -size_t ZSTD_freeDStream(ZSTD_DStream *zds); - -#endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/lib/zstd/zstd_opt.h b/lib/zstd/zstd_opt.h deleted file mode 100644 index 55e1b4cba808..000000000000 --- a/lib/zstd/zstd_opt.h +++ /dev/null @@ -1,1014 +0,0 @@ -/** - * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of https://github.com/facebook/zstd. - * An additional grant of patent rights can be found in the PATENTS file in the - * same directory. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - */ - -/* Note : this file is intended to be included within zstd_compress.c */ - -#ifndef ZSTD_OPT_H_91842398743 -#define ZSTD_OPT_H_91842398743 - -#define ZSTD_LITFREQ_ADD 2 -#define ZSTD_FREQ_DIV 4 -#define ZSTD_MAX_PRICE (1 << 30) - -/*-************************************* -* Price functions for optimal parser -***************************************/ -FORCE_INLINE void ZSTD_setLog2Prices(seqStore_t *ssPtr) -{ - ssPtr->log2matchLengthSum = ZSTD_highbit32(ssPtr->matchLengthSum + 1); - ssPtr->log2litLengthSum = ZSTD_highbit32(ssPtr->litLengthSum + 1); - ssPtr->log2litSum = ZSTD_highbit32(ssPtr->litSum + 1); - ssPtr->log2offCodeSum = ZSTD_highbit32(ssPtr->offCodeSum + 1); - ssPtr->factor = 1 + ((ssPtr->litSum >> 5) / ssPtr->litLengthSum) + ((ssPtr->litSum << 1) / (ssPtr->litSum + ssPtr->matchSum)); -} - -ZSTD_STATIC void ZSTD_rescaleFreqs(seqStore_t *ssPtr, const BYTE *src, size_t srcSize) -{ - unsigned u; - - ssPtr->cachedLiterals = NULL; - ssPtr->cachedPrice = ssPtr->cachedLitLength = 0; - ssPtr->staticPrices = 0; - - if (ssPtr->litLengthSum == 0) { - if (srcSize <= 1024) - ssPtr->staticPrices = 1; - - for (u = 0; u <= MaxLit; u++) - ssPtr->litFreq[u] = 0; - for (u = 0; u < srcSize; u++) - ssPtr->litFreq[src[u]]++; - - ssPtr->litSum = 0; - ssPtr->litLengthSum = MaxLL + 1; - ssPtr->matchLengthSum = MaxML + 1; - ssPtr->offCodeSum = (MaxOff + 1); - ssPtr->matchSum = (ZSTD_LITFREQ_ADD << Litbits); - - for (u = 0; u <= MaxLit; u++) { - ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u] >> ZSTD_FREQ_DIV); - ssPtr->litSum += ssPtr->litFreq[u]; - } - for (u = 0; u <= MaxLL; u++) - ssPtr->litLengthFreq[u] = 1; - for (u = 0; u <= MaxML; u++) - ssPtr->matchLengthFreq[u] = 1; - for (u = 0; u <= MaxOff; u++) - ssPtr->offCodeFreq[u] = 1; - } else { - ssPtr->matchLengthSum = 0; - ssPtr->litLengthSum = 0; - ssPtr->offCodeSum = 0; - ssPtr->matchSum = 0; - ssPtr->litSum = 0; - - for (u = 0; u <= MaxLit; u++) { - ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u] >> (ZSTD_FREQ_DIV + 1)); - ssPtr->litSum += ssPtr->litFreq[u]; - } - for (u = 0; u <= MaxLL; u++) { - ssPtr->litLengthFreq[u] = 1 + (ssPtr->litLengthFreq[u] >> (ZSTD_FREQ_DIV + 1)); - ssPtr->litLengthSum += ssPtr->litLengthFreq[u]; - } - for (u = 0; u <= MaxML; u++) { - ssPtr->matchLengthFreq[u] = 1 + (ssPtr->matchLengthFreq[u] >> ZSTD_FREQ_DIV); - ssPtr->matchLengthSum += ssPtr->matchLengthFreq[u]; - ssPtr->matchSum += ssPtr->matchLengthFreq[u] * (u + 3); - } - ssPtr->matchSum *= ZSTD_LITFREQ_ADD; - for (u = 0; u <= MaxOff; u++) { - ssPtr->offCodeFreq[u] = 1 + (ssPtr->offCodeFreq[u] >> ZSTD_FREQ_DIV); - ssPtr->offCodeSum += ssPtr->offCodeFreq[u]; - } - } - - ZSTD_setLog2Prices(ssPtr); -} - -FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t *ssPtr, U32 litLength, const BYTE *literals) -{ - U32 price, u; - - if (ssPtr->staticPrices) - return ZSTD_highbit32((U32)litLength + 1) + (litLength * 6); - - if (litLength == 0) - return ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[0] + 1); - - /* literals */ - if (ssPtr->cachedLiterals == literals) { - U32 const additional = litLength - ssPtr->cachedLitLength; - const BYTE *literals2 = ssPtr->cachedLiterals + ssPtr->cachedLitLength; - price = ssPtr->cachedPrice + additional * ssPtr->log2litSum; - for (u = 0; u < additional; u++) - price -= ZSTD_highbit32(ssPtr->litFreq[literals2[u]] + 1); - ssPtr->cachedPrice = price; - ssPtr->cachedLitLength = litLength; - } else { - price = litLength * ssPtr->log2litSum; - for (u = 0; u < litLength; u++) - price -= ZSTD_highbit32(ssPtr->litFreq[literals[u]] + 1); - - if (litLength >= 12) { - ssPtr->cachedLiterals = literals; - ssPtr->cachedPrice = price; - ssPtr->cachedLitLength = litLength; - } - } - - /* literal Length */ - { - const BYTE LL_deltaCode = 19; - const BYTE llCode = (litLength > 63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; - price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[llCode] + 1); - } - - return price; -} - -FORCE_INLINE U32 ZSTD_getPrice(seqStore_t *seqStorePtr, U32 litLength, const BYTE *literals, U32 offset, U32 matchLength, const int ultra) -{ - /* offset */ - U32 price; - BYTE const offCode = (BYTE)ZSTD_highbit32(offset + 1); - - if (seqStorePtr->staticPrices) - return ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + ZSTD_highbit32((U32)matchLength + 1) + 16 + offCode; - - price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode] + 1); - if (!ultra && offCode >= 20) - price += (offCode - 19) * 2; - - /* match Length */ - { - const BYTE ML_deltaCode = 36; - const BYTE mlCode = (matchLength > 127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength]; - price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit32(seqStorePtr->matchLengthFreq[mlCode] + 1); - } - - return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor; -} - -ZSTD_STATIC void ZSTD_updatePrice(seqStore_t *seqStorePtr, U32 litLength, const BYTE *literals, U32 offset, U32 matchLength) -{ - U32 u; - - /* literals */ - seqStorePtr->litSum += litLength * ZSTD_LITFREQ_ADD; - for (u = 0; u < litLength; u++) - seqStorePtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD; - - /* literal Length */ - { - const BYTE LL_deltaCode = 19; - const BYTE llCode = (litLength > 63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; - seqStorePtr->litLengthFreq[llCode]++; - seqStorePtr->litLengthSum++; - } - - /* match offset */ - { - BYTE const offCode = (BYTE)ZSTD_highbit32(offset + 1); - seqStorePtr->offCodeSum++; - seqStorePtr->offCodeFreq[offCode]++; - } - - /* match Length */ - { - const BYTE ML_deltaCode = 36; - const BYTE mlCode = (matchLength > 127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength]; - seqStorePtr->matchLengthFreq[mlCode]++; - seqStorePtr->matchLengthSum++; - } - - ZSTD_setLog2Prices(seqStorePtr); -} - -#define SET_PRICE(pos, mlen_, offset_, litlen_, price_) \ - { \ - while (last_pos < pos) { \ - opt[last_pos + 1].price = ZSTD_MAX_PRICE; \ - last_pos++; \ - } \ - opt[pos].mlen = mlen_; \ - opt[pos].off = offset_; \ - opt[pos].litlen = litlen_; \ - opt[pos].price = price_; \ - } - -/* Update hashTable3 up to ip (excluded) - Assumption : always within prefix (i.e. not within extDict) */ -FORCE_INLINE -U32 ZSTD_insertAndFindFirstIndexHash3(ZSTD_CCtx *zc, const BYTE *ip) -{ - U32 *const hashTable3 = zc->hashTable3; - U32 const hashLog3 = zc->hashLog3; - const BYTE *const base = zc->base; - U32 idx = zc->nextToUpdate3; - const U32 target = zc->nextToUpdate3 = (U32)(ip - base); - const size_t hash3 = ZSTD_hash3Ptr(ip, hashLog3); - - while (idx < target) { - hashTable3[ZSTD_hash3Ptr(base + idx, hashLog3)] = idx; - idx++; - } - - return hashTable3[hash3]; -} - -/*-************************************* -* Binary Tree search -***************************************/ -static U32 ZSTD_insertBtAndGetAllMatches(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, U32 nbCompares, const U32 mls, U32 extDict, - ZSTD_match_t *matches, const U32 minMatchLen) -{ - const BYTE *const base = zc->base; - const U32 curr = (U32)(ip - base); - const U32 hashLog = zc->params.cParams.hashLog; - const size_t h = ZSTD_hashPtr(ip, hashLog, mls); - U32 *const hashTable = zc->hashTable; - U32 matchIndex = hashTable[h]; - U32 *const bt = zc->chainTable; - const U32 btLog = zc->params.cParams.chainLog - 1; - const U32 btMask = (1U << btLog) - 1; - size_t commonLengthSmaller = 0, commonLengthLarger = 0; - const BYTE *const dictBase = zc->dictBase; - const U32 dictLimit = zc->dictLimit; - const BYTE *const dictEnd = dictBase + dictLimit; - const BYTE *const prefixStart = base + dictLimit; - const U32 btLow = btMask >= curr ? 0 : curr - btMask; - const U32 windowLow = zc->lowLimit; - U32 *smallerPtr = bt + 2 * (curr & btMask); - U32 *largerPtr = bt + 2 * (curr & btMask) + 1; - U32 matchEndIdx = curr + 8; - U32 dummy32; /* to be nullified at the end */ - U32 mnum = 0; - - const U32 minMatch = (mls == 3) ? 3 : 4; - size_t bestLength = minMatchLen - 1; - - if (minMatch == 3) { /* HC3 match finder */ - U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(zc, ip); - if (matchIndex3 > windowLow && (curr - matchIndex3 < (1 << 18))) { - const BYTE *match; - size_t currMl = 0; - if ((!extDict) || matchIndex3 >= dictLimit) { - match = base + matchIndex3; - if (match[bestLength] == ip[bestLength]) - currMl = ZSTD_count(ip, match, iLimit); - } else { - match = dictBase + matchIndex3; - if (ZSTD_readMINMATCH(match, MINMATCH) == - ZSTD_readMINMATCH(ip, MINMATCH)) /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */ - currMl = ZSTD_count_2segments(ip + MINMATCH, match + MINMATCH, iLimit, dictEnd, prefixStart) + MINMATCH; - } - - /* save best solution */ - if (currMl > bestLength) { - bestLength = currMl; - matches[mnum].off = ZSTD_REP_MOVE_OPT + curr - matchIndex3; - matches[mnum].len = (U32)currMl; - mnum++; - if (currMl > ZSTD_OPT_NUM) - goto update; - if (ip + currMl == iLimit) - goto update; /* best possible, and avoid read overflow*/ - } - } - } - - hashTable[h] = curr; /* Update Hash Table */ - - while (nbCompares-- && (matchIndex > windowLow)) { - U32 *nextPtr = bt + 2 * (matchIndex & btMask); - size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ - const BYTE *match; - - if ((!extDict) || (matchIndex + matchLength >= dictLimit)) { - match = base + matchIndex; - if (match[matchLength] == ip[matchLength]) { - matchLength += ZSTD_count(ip + matchLength + 1, match + matchLength + 1, iLimit) + 1; - } - } else { - match = dictBase + matchIndex; - matchLength += ZSTD_count_2segments(ip + matchLength, match + matchLength, iLimit, dictEnd, prefixStart); - if (matchIndex + matchLength >= dictLimit) - match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ - } - - if (matchLength > bestLength) { - if (matchLength > matchEndIdx - matchIndex) - matchEndIdx = matchIndex + (U32)matchLength; - bestLength = matchLength; - matches[mnum].off = ZSTD_REP_MOVE_OPT + curr - matchIndex; - matches[mnum].len = (U32)matchLength; - mnum++; - if (matchLength > ZSTD_OPT_NUM) - break; - if (ip + matchLength == iLimit) /* equal : no way to know if inf or sup */ - break; /* drop, to guarantee consistency (miss a little bit of compression) */ - } - - if (match[matchLength] < ip[matchLength]) { - /* match is smaller than curr */ - *smallerPtr = matchIndex; /* update smaller idx */ - commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ - if (matchIndex <= btLow) { - smallerPtr = &dummy32; - break; - } /* beyond tree size, stop the search */ - smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */ - matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */ - } else { - /* match is larger than curr */ - *largerPtr = matchIndex; - commonLengthLarger = matchLength; - if (matchIndex <= btLow) { - largerPtr = &dummy32; - break; - } /* beyond tree size, stop the search */ - largerPtr = nextPtr; - matchIndex = nextPtr[0]; - } - } - - *smallerPtr = *largerPtr = 0; - -update: - zc->nextToUpdate = (matchEndIdx > curr + 8) ? matchEndIdx - 8 : curr + 1; - return mnum; -} - -/** Tree updater, providing best match */ -static U32 ZSTD_BtGetAllMatches(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, const U32 maxNbAttempts, const U32 mls, ZSTD_match_t *matches, - const U32 minMatchLen) -{ - if (ip < zc->base + zc->nextToUpdate) - return 0; /* skipped area */ - ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls); - return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 0, matches, minMatchLen); -} - -static U32 ZSTD_BtGetAllMatches_selectMLS(ZSTD_CCtx *zc, /* Index table will be updated */ - const BYTE *ip, const BYTE *const iHighLimit, const U32 maxNbAttempts, const U32 matchLengthSearch, - ZSTD_match_t *matches, const U32 minMatchLen) -{ - switch (matchLengthSearch) { - case 3: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen); - default: - case 4: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen); - case 5: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen); - case 7: - case 6: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen); - } -} - -/** Tree updater, providing best match */ -static U32 ZSTD_BtGetAllMatches_extDict(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, const U32 maxNbAttempts, const U32 mls, - ZSTD_match_t *matches, const U32 minMatchLen) -{ - if (ip < zc->base + zc->nextToUpdate) - return 0; /* skipped area */ - ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls); - return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 1, matches, minMatchLen); -} - -static U32 ZSTD_BtGetAllMatches_selectMLS_extDict(ZSTD_CCtx *zc, /* Index table will be updated */ - const BYTE *ip, const BYTE *const iHighLimit, const U32 maxNbAttempts, const U32 matchLengthSearch, - ZSTD_match_t *matches, const U32 minMatchLen) -{ - switch (matchLengthSearch) { - case 3: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen); - default: - case 4: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen); - case 5: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen); - case 7: - case 6: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen); - } -} - -/*-******************************* -* Optimal parser -*********************************/ -FORCE_INLINE -void ZSTD_compressBlock_opt_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const int ultra) -{ - seqStore_t *seqStorePtr = &(ctx->seqStore); - const BYTE *const istart = (const BYTE *)src; - const BYTE *ip = istart; - const BYTE *anchor = istart; - const BYTE *const iend = istart + srcSize; - const BYTE *const ilimit = iend - 8; - const BYTE *const base = ctx->base; - const BYTE *const prefixStart = base + ctx->dictLimit; - - const U32 maxSearches = 1U << ctx->params.cParams.searchLog; - const U32 sufficient_len = ctx->params.cParams.targetLength; - const U32 mls = ctx->params.cParams.searchLength; - const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4; - - ZSTD_optimal_t *opt = seqStorePtr->priceTable; - ZSTD_match_t *matches = seqStorePtr->matchTable; - const BYTE *inr; - U32 offset, rep[ZSTD_REP_NUM]; - - /* init */ - ctx->nextToUpdate3 = ctx->nextToUpdate; - ZSTD_rescaleFreqs(seqStorePtr, (const BYTE *)src, srcSize); - ip += (ip == prefixStart); - { - U32 i; - for (i = 0; i < ZSTD_REP_NUM; i++) - rep[i] = ctx->rep[i]; - } - - /* Match Loop */ - while (ip < ilimit) { - U32 cur, match_num, last_pos, litlen, price; - U32 u, mlen, best_mlen, best_off, litLength; - memset(opt, 0, sizeof(ZSTD_optimal_t)); - last_pos = 0; - litlen = (U32)(ip - anchor); - - /* check repCode */ - { - U32 i, last_i = ZSTD_REP_CHECK + (ip == anchor); - for (i = (ip == anchor); i < last_i; i++) { - const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : rep[i]; - if ((repCur > 0) && (repCur < (S32)(ip - prefixStart)) && - (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repCur, minMatch))) { - mlen = (U32)ZSTD_count(ip + minMatch, ip + minMatch - repCur, iend) + minMatch; - if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { - best_mlen = mlen; - best_off = i; - cur = 0; - last_pos = 1; - goto _storeSequence; - } - best_off = i - (ip == anchor); - do { - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); - if (mlen > last_pos || price < opt[mlen].price) - SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */ - mlen--; - } while (mlen >= minMatch); - } - } - } - - match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, ip, iend, maxSearches, mls, matches, minMatch); - - if (!last_pos && !match_num) { - ip++; - continue; - } - - if (match_num && (matches[match_num - 1].len > sufficient_len || matches[match_num - 1].len >= ZSTD_OPT_NUM)) { - best_mlen = matches[match_num - 1].len; - best_off = matches[match_num - 1].off; - cur = 0; - last_pos = 1; - goto _storeSequence; - } - - /* set prices using matches at position = 0 */ - best_mlen = (last_pos) ? last_pos : minMatch; - for (u = 0; u < match_num; u++) { - mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen; - best_mlen = matches[u].len; - while (mlen <= best_mlen) { - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra); - if (mlen > last_pos || price < opt[mlen].price) - SET_PRICE(mlen, mlen, matches[u].off, litlen, price); /* note : macro modifies last_pos */ - mlen++; - } - } - - if (last_pos < minMatch) { - ip++; - continue; - } - - /* initialize opt[0] */ - { - U32 i; - for (i = 0; i < ZSTD_REP_NUM; i++) - opt[0].rep[i] = rep[i]; - } - opt[0].mlen = 1; - opt[0].litlen = litlen; - - /* check further positions */ - for (cur = 1; cur <= last_pos; cur++) { - inr = ip + cur; - - if (opt[cur - 1].mlen == 1) { - litlen = opt[cur - 1].litlen + 1; - if (cur > litlen) { - price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - litlen); - } else - price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor); - } else { - litlen = 1; - price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - 1); - } - - if (cur > last_pos || price <= opt[cur].price) - SET_PRICE(cur, 1, 0, litlen, price); - - if (cur == last_pos) - break; - - if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */ - continue; - - mlen = opt[cur].mlen; - if (opt[cur].off > ZSTD_REP_MOVE_OPT) { - opt[cur].rep[2] = opt[cur - mlen].rep[1]; - opt[cur].rep[1] = opt[cur - mlen].rep[0]; - opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT; - } else { - opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur - mlen].rep[1] : opt[cur - mlen].rep[2]; - opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur - mlen].rep[0] : opt[cur - mlen].rep[1]; - opt[cur].rep[0] = - ((opt[cur].off == ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur - mlen].rep[0] - 1) : (opt[cur - mlen].rep[opt[cur].off]); - } - - best_mlen = minMatch; - { - U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1); - for (i = (opt[cur].mlen != 1); i < last_i; i++) { /* check rep */ - const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i]; - if ((repCur > 0) && (repCur < (S32)(inr - prefixStart)) && - (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(inr - repCur, minMatch))) { - mlen = (U32)ZSTD_count(inr + minMatch, inr + minMatch - repCur, iend) + minMatch; - - if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { - best_mlen = mlen; - best_off = i; - last_pos = cur + 1; - goto _storeSequence; - } - - best_off = i - (opt[cur].mlen != 1); - if (mlen > best_mlen) - best_mlen = mlen; - - do { - if (opt[cur].mlen == 1) { - litlen = opt[cur].litlen; - if (cur > litlen) { - price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr - litlen, - best_off, mlen - MINMATCH, ultra); - } else - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); - } else { - litlen = 0; - price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH, ultra); - } - - if (cur + mlen > last_pos || price <= opt[cur + mlen].price) - SET_PRICE(cur + mlen, mlen, i, litlen, price); - mlen--; - } while (mlen >= minMatch); - } - } - } - - match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, iend, maxSearches, mls, matches, best_mlen); - - if (match_num > 0 && (matches[match_num - 1].len > sufficient_len || cur + matches[match_num - 1].len >= ZSTD_OPT_NUM)) { - best_mlen = matches[match_num - 1].len; - best_off = matches[match_num - 1].off; - last_pos = cur + 1; - goto _storeSequence; - } - - /* set prices using matches at position = cur */ - for (u = 0; u < match_num; u++) { - mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen; - best_mlen = matches[u].len; - - while (mlen <= best_mlen) { - if (opt[cur].mlen == 1) { - litlen = opt[cur].litlen; - if (cur > litlen) - price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip + cur - litlen, - matches[u].off - 1, mlen - MINMATCH, ultra); - else - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra); - } else { - litlen = 0; - price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off - 1, mlen - MINMATCH, ultra); - } - - if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) - SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); - - mlen++; - } - } - } - - best_mlen = opt[last_pos].mlen; - best_off = opt[last_pos].off; - cur = last_pos - best_mlen; - - /* store sequence */ -_storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ - opt[0].mlen = 1; - - while (1) { - mlen = opt[cur].mlen; - offset = opt[cur].off; - opt[cur].mlen = best_mlen; - opt[cur].off = best_off; - best_mlen = mlen; - best_off = offset; - if (mlen > cur) - break; - cur -= mlen; - } - - for (u = 0; u <= last_pos;) { - u += opt[u].mlen; - } - - for (cur = 0; cur < last_pos;) { - mlen = opt[cur].mlen; - if (mlen == 1) { - ip++; - cur++; - continue; - } - offset = opt[cur].off; - cur += mlen; - litLength = (U32)(ip - anchor); - - if (offset > ZSTD_REP_MOVE_OPT) { - rep[2] = rep[1]; - rep[1] = rep[0]; - rep[0] = offset - ZSTD_REP_MOVE_OPT; - offset--; - } else { - if (offset != 0) { - best_off = (offset == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]); - if (offset != 1) - rep[2] = rep[1]; - rep[1] = rep[0]; - rep[0] = best_off; - } - if (litLength == 0) - offset--; - } - - ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH); - ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH); - anchor = ip = ip + mlen; - } - } /* for (cur=0; cur < last_pos; ) */ - - /* Save reps for next block */ - { - int i; - for (i = 0; i < ZSTD_REP_NUM; i++) - ctx->repToConfirm[i] = rep[i]; - } - - /* Last Literals */ - { - size_t const lastLLSize = iend - anchor; - memcpy(seqStorePtr->lit, anchor, lastLLSize); - seqStorePtr->lit += lastLLSize; - } -} - -FORCE_INLINE -void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const int ultra) -{ - seqStore_t *seqStorePtr = &(ctx->seqStore); - const BYTE *const istart = (const BYTE *)src; - const BYTE *ip = istart; - const BYTE *anchor = istart; - const BYTE *const iend = istart + srcSize; - const BYTE *const ilimit = iend - 8; - const BYTE *const base = ctx->base; - const U32 lowestIndex = ctx->lowLimit; - const U32 dictLimit = ctx->dictLimit; - const BYTE *const prefixStart = base + dictLimit; - const BYTE *const dictBase = ctx->dictBase; - const BYTE *const dictEnd = dictBase + dictLimit; - - const U32 maxSearches = 1U << ctx->params.cParams.searchLog; - const U32 sufficient_len = ctx->params.cParams.targetLength; - const U32 mls = ctx->params.cParams.searchLength; - const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4; - - ZSTD_optimal_t *opt = seqStorePtr->priceTable; - ZSTD_match_t *matches = seqStorePtr->matchTable; - const BYTE *inr; - - /* init */ - U32 offset, rep[ZSTD_REP_NUM]; - { - U32 i; - for (i = 0; i < ZSTD_REP_NUM; i++) - rep[i] = ctx->rep[i]; - } - - ctx->nextToUpdate3 = ctx->nextToUpdate; - ZSTD_rescaleFreqs(seqStorePtr, (const BYTE *)src, srcSize); - ip += (ip == prefixStart); - - /* Match Loop */ - while (ip < ilimit) { - U32 cur, match_num, last_pos, litlen, price; - U32 u, mlen, best_mlen, best_off, litLength; - U32 curr = (U32)(ip - base); - memset(opt, 0, sizeof(ZSTD_optimal_t)); - last_pos = 0; - opt[0].litlen = (U32)(ip - anchor); - - /* check repCode */ - { - U32 i, last_i = ZSTD_REP_CHECK + (ip == anchor); - for (i = (ip == anchor); i < last_i; i++) { - const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : rep[i]; - const U32 repIndex = (U32)(curr - repCur); - const BYTE *const repBase = repIndex < dictLimit ? dictBase : base; - const BYTE *const repMatch = repBase + repIndex; - if ((repCur > 0 && repCur <= (S32)curr) && - (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ - && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch))) { - /* repcode detected we should take it */ - const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend; - mlen = (U32)ZSTD_count_2segments(ip + minMatch, repMatch + minMatch, iend, repEnd, prefixStart) + minMatch; - - if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { - best_mlen = mlen; - best_off = i; - cur = 0; - last_pos = 1; - goto _storeSequence; - } - - best_off = i - (ip == anchor); - litlen = opt[0].litlen; - do { - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); - if (mlen > last_pos || price < opt[mlen].price) - SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */ - mlen--; - } while (mlen >= minMatch); - } - } - } - - match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, iend, maxSearches, mls, matches, minMatch); /* first search (depth 0) */ - - if (!last_pos && !match_num) { - ip++; - continue; - } - - { - U32 i; - for (i = 0; i < ZSTD_REP_NUM; i++) - opt[0].rep[i] = rep[i]; - } - opt[0].mlen = 1; - - if (match_num && (matches[match_num - 1].len > sufficient_len || matches[match_num - 1].len >= ZSTD_OPT_NUM)) { - best_mlen = matches[match_num - 1].len; - best_off = matches[match_num - 1].off; - cur = 0; - last_pos = 1; - goto _storeSequence; - } - - best_mlen = (last_pos) ? last_pos : minMatch; - - /* set prices using matches at position = 0 */ - for (u = 0; u < match_num; u++) { - mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen; - best_mlen = matches[u].len; - litlen = opt[0].litlen; - while (mlen <= best_mlen) { - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra); - if (mlen > last_pos || price < opt[mlen].price) - SET_PRICE(mlen, mlen, matches[u].off, litlen, price); - mlen++; - } - } - - if (last_pos < minMatch) { - ip++; - continue; - } - - /* check further positions */ - for (cur = 1; cur <= last_pos; cur++) { - inr = ip + cur; - - if (opt[cur - 1].mlen == 1) { - litlen = opt[cur - 1].litlen + 1; - if (cur > litlen) { - price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - litlen); - } else - price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor); - } else { - litlen = 1; - price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - 1); - } - - if (cur > last_pos || price <= opt[cur].price) - SET_PRICE(cur, 1, 0, litlen, price); - - if (cur == last_pos) - break; - - if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */ - continue; - - mlen = opt[cur].mlen; - if (opt[cur].off > ZSTD_REP_MOVE_OPT) { - opt[cur].rep[2] = opt[cur - mlen].rep[1]; - opt[cur].rep[1] = opt[cur - mlen].rep[0]; - opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT; - } else { - opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur - mlen].rep[1] : opt[cur - mlen].rep[2]; - opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur - mlen].rep[0] : opt[cur - mlen].rep[1]; - opt[cur].rep[0] = - ((opt[cur].off == ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur - mlen].rep[0] - 1) : (opt[cur - mlen].rep[opt[cur].off]); - } - - best_mlen = minMatch; - { - U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1); - for (i = (mlen != 1); i < last_i; i++) { - const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i]; - const U32 repIndex = (U32)(curr + cur - repCur); - const BYTE *const repBase = repIndex < dictLimit ? dictBase : base; - const BYTE *const repMatch = repBase + repIndex; - if ((repCur > 0 && repCur <= (S32)(curr + cur)) && - (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ - && (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch))) { - /* repcode detected */ - const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend; - mlen = (U32)ZSTD_count_2segments(inr + minMatch, repMatch + minMatch, iend, repEnd, prefixStart) + minMatch; - - if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { - best_mlen = mlen; - best_off = i; - last_pos = cur + 1; - goto _storeSequence; - } - - best_off = i - (opt[cur].mlen != 1); - if (mlen > best_mlen) - best_mlen = mlen; - - do { - if (opt[cur].mlen == 1) { - litlen = opt[cur].litlen; - if (cur > litlen) { - price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr - litlen, - best_off, mlen - MINMATCH, ultra); - } else - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); - } else { - litlen = 0; - price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH, ultra); - } - - if (cur + mlen > last_pos || price <= opt[cur + mlen].price) - SET_PRICE(cur + mlen, mlen, i, litlen, price); - mlen--; - } while (mlen >= minMatch); - } - } - } - - match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch); - - if (match_num > 0 && (matches[match_num - 1].len > sufficient_len || cur + matches[match_num - 1].len >= ZSTD_OPT_NUM)) { - best_mlen = matches[match_num - 1].len; - best_off = matches[match_num - 1].off; - last_pos = cur + 1; - goto _storeSequence; - } - - /* set prices using matches at position = cur */ - for (u = 0; u < match_num; u++) { - mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen; - best_mlen = matches[u].len; - - while (mlen <= best_mlen) { - if (opt[cur].mlen == 1) { - litlen = opt[cur].litlen; - if (cur > litlen) - price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip + cur - litlen, - matches[u].off - 1, mlen - MINMATCH, ultra); - else - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra); - } else { - litlen = 0; - price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off - 1, mlen - MINMATCH, ultra); - } - - if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) - SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); - - mlen++; - } - } - } /* for (cur = 1; cur <= last_pos; cur++) */ - - best_mlen = opt[last_pos].mlen; - best_off = opt[last_pos].off; - cur = last_pos - best_mlen; - - /* store sequence */ -_storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ - opt[0].mlen = 1; - - while (1) { - mlen = opt[cur].mlen; - offset = opt[cur].off; - opt[cur].mlen = best_mlen; - opt[cur].off = best_off; - best_mlen = mlen; - best_off = offset; - if (mlen > cur) - break; - cur -= mlen; - } - - for (u = 0; u <= last_pos;) { - u += opt[u].mlen; - } - - for (cur = 0; cur < last_pos;) { - mlen = opt[cur].mlen; - if (mlen == 1) { - ip++; - cur++; - continue; - } - offset = opt[cur].off; - cur += mlen; - litLength = (U32)(ip - anchor); - - if (offset > ZSTD_REP_MOVE_OPT) { - rep[2] = rep[1]; - rep[1] = rep[0]; - rep[0] = offset - ZSTD_REP_MOVE_OPT; - offset--; - } else { - if (offset != 0) { - best_off = (offset == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]); - if (offset != 1) - rep[2] = rep[1]; - rep[1] = rep[0]; - rep[0] = best_off; - } - - if (litLength == 0) - offset--; - } - - ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH); - ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH); - anchor = ip = ip + mlen; - } - } /* for (cur=0; cur < last_pos; ) */ - - /* Save reps for next block */ - { - int i; - for (i = 0; i < ZSTD_REP_NUM; i++) - ctx->repToConfirm[i] = rep[i]; - } - - /* Last Literals */ - { - size_t lastLLSize = iend - anchor; - memcpy(seqStorePtr->lit, anchor, lastLLSize); - seqStorePtr->lit += lastLLSize; - } -} - -#endif /* ZSTD_OPT_H_91842398743 */ diff --git a/mm/Kconfig b/mm/Kconfig index 24c045b24b95..dc522a9ff258 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -63,6 +63,43 @@ config SPARSEMEM_MANUAL endchoice +config UNEVICTABLE_FILE + bool "Keep some file pages under memory pressure" + depends on SYSCTL + def_bool y + help + Keep some file pages still mapped under memory pressure + to avoid potential disk thrashing that may occur due to + evicting running executables code. + + The UNEVICTABLE_FILE_KBYTES_LOW value defines a threshold + to activate file pages eviction throttling. + The vm.unevictable_file_kbytes_low sysctl knob is used to + change the amount in the runtime (setting it to 0 + effectively disables this feature). + + Recommended value: 262144 for a typical desktop workload. + + The UNEVICTABLE_FILE_KBYTES_MIN value sets the amount of + pages to keep as a hard limit. + The vm.unevictable_file_kbytes_min sysctl knob is used to + change the amount in the runtime (setting it to 0 + effectively disables this feature). + + Recommended value: 131072 for a typical desktop workload. + + See also: Documentation/admin-guide/sysctl/vm.rst + +config UNEVICTABLE_FILE_KBYTES_LOW + int "Default value for vm.unevictable_file_kbytes_low" + depends on UNEVICTABLE_FILE + default "262144" + +config UNEVICTABLE_FILE_KBYTES_MIN + int "Default value for vm.unevictable_file_kbytes_min" + depends on UNEVICTABLE_FILE + default "131072" + config DISCONTIGMEM def_bool y depends on (!SELECT_MEMORY_MODEL && ARCH_DISCONTIGMEM_ENABLE) || DISCONTIGMEM_MANUAL diff --git a/mm/gup.c b/mm/gup.c index ef7d2da9f03f..333f5dfd8942 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -1551,54 +1551,60 @@ static long check_and_migrate_cma_pages(struct mm_struct *mm, struct vm_area_struct **vmas, unsigned int gup_flags) { - unsigned long i; - unsigned long step; - bool drain_allow = true; - bool migrate_allow = true; + unsigned long i, isolation_error_count; + bool drain_allow; LIST_HEAD(cma_page_list); long ret = nr_pages; + struct page *prev_head, *head; struct migration_target_control mtc = { .nid = NUMA_NO_NODE, .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_NOWARN, }; check_again: - for (i = 0; i < nr_pages;) { - - struct page *head = compound_head(pages[i]); - - /* - * gup may start from a tail page. Advance step by the left - * part. - */ - step = compound_nr(head) - (pages[i] - head); + prev_head = NULL; + isolation_error_count = 0; + drain_allow = true; + for (i = 0; i < nr_pages; i++) { + head = compound_head(pages[i]); + if (head == prev_head) + continue; + prev_head = head; /* * If we get a page from the CMA zone, since we are going to * be pinning these entries, we might as well move them out * of the CMA zone if possible. */ if (is_migrate_cma_page(head)) { - if (PageHuge(head)) - isolate_huge_page(head, &cma_page_list); - else { + if (PageHuge(head)) { + if (!isolate_huge_page(head, &cma_page_list)) + isolation_error_count++; + } else { if (!PageLRU(head) && drain_allow) { lru_add_drain_all(); drain_allow = false; } - if (!isolate_lru_page(head)) { - list_add_tail(&head->lru, &cma_page_list); - mod_node_page_state(page_pgdat(head), - NR_ISOLATED_ANON + - page_is_file_lru(head), - thp_nr_pages(head)); + if (isolate_lru_page(head)) { + isolation_error_count++; + continue; } + list_add_tail(&head->lru, &cma_page_list); + mod_node_page_state(page_pgdat(head), + NR_ISOLATED_ANON + + page_is_file_lru(head), + thp_nr_pages(head)); } } - - i += step; } + /* + * If list is empty, and no isolation errors, means that all pages are + * in the correct zone. + */ + if (list_empty(&cma_page_list) && !isolation_error_count) + return ret; + if (!list_empty(&cma_page_list)) { /* * drop the above get_user_pages reference. @@ -1609,34 +1615,28 @@ static long check_and_migrate_cma_pages(struct mm_struct *mm, for (i = 0; i < nr_pages; i++) put_page(pages[i]); - if (migrate_pages(&cma_page_list, alloc_migration_target, NULL, - (unsigned long)&mtc, MIGRATE_SYNC, MR_CONTIG_RANGE)) { - /* - * some of the pages failed migration. Do get_user_pages - * without migration. - */ - migrate_allow = false; - + ret = migrate_pages(&cma_page_list, alloc_migration_target, + NULL, (unsigned long)&mtc, MIGRATE_SYNC, + MR_CONTIG_RANGE); + if (ret) { if (!list_empty(&cma_page_list)) putback_movable_pages(&cma_page_list); + return ret > 0 ? -ENOMEM : ret; } - /* - * We did migrate all the pages, Try to get the page references - * again migrating any new CMA pages which we failed to isolate - * earlier. - */ - ret = __get_user_pages_locked(mm, start, nr_pages, - pages, vmas, NULL, - gup_flags); - - if ((ret > 0) && migrate_allow) { - nr_pages = ret; - drain_allow = true; - goto check_again; - } + + /* We unpinned pages before migration, pin them again */ + ret = __get_user_pages_locked(mm, start, nr_pages, pages, vmas, + NULL, gup_flags); + if (ret <= 0) + return ret; + nr_pages = ret; } - return ret; + /* + * check again because pages were unpinned, and we also might have + * had isolation errors and need more pages to migrate. + */ + goto check_again; } #else static long check_and_migrate_cma_pages(struct mm_struct *mm, diff --git a/mm/hugetlb.c b/mm/hugetlb.c index a86a58ef132d..96b722af092e 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -743,13 +743,20 @@ void hugetlb_fix_reserve_counts(struct inode *inode) { struct hugepage_subpool *spool = subpool_inode(inode); long rsv_adjust; + bool reserved = false; rsv_adjust = hugepage_subpool_get_pages(spool, 1); - if (rsv_adjust) { + if (rsv_adjust > 0) { struct hstate *h = hstate_inode(inode); - hugetlb_acct_memory(h, 1); + if (!hugetlb_acct_memory(h, 1)) + reserved = true; + } else if (!rsv_adjust) { + reserved = true; } + + if (!reserved) + pr_warn("hugetlb: Huge Page Reserved count may go negative.\n"); } /* @@ -3898,6 +3905,7 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, * See Documentation/vm/mmu_notifier.rst */ huge_ptep_set_wrprotect(src, addr, src_pte); + entry = huge_pte_wrprotect(entry); } page_dup_rmap(ptepage, true); diff --git a/mm/kfence/core.c b/mm/kfence/core.c index d53c91f881a4..f0be2c5038b5 100644 --- a/mm/kfence/core.c +++ b/mm/kfence/core.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -586,6 +587,17 @@ late_initcall(kfence_debugfs_init); /* === Allocation Gate Timer ================================================ */ +#ifdef CONFIG_KFENCE_STATIC_KEYS +/* Wait queue to wake up allocation-gate timer task. */ +static DECLARE_WAIT_QUEUE_HEAD(allocation_wait); + +static void wake_up_kfence_timer(struct irq_work *work) +{ + wake_up(&allocation_wait); +} +static DEFINE_IRQ_WORK(wake_up_kfence_timer_work, wake_up_kfence_timer); +#endif + /* * Set up delayed work, which will enable and disable the static key. We need to * use a work queue (rather than a simple timer), since enabling and disabling a @@ -603,25 +615,13 @@ static void toggle_allocation_gate(struct work_struct *work) if (!READ_ONCE(kfence_enabled)) return; - /* Enable static key, and await allocation to happen. */ atomic_set(&kfence_allocation_gate, 0); #ifdef CONFIG_KFENCE_STATIC_KEYS + /* Enable static key, and await allocation to happen. */ static_branch_enable(&kfence_allocation_key); - /* - * Await an allocation. Timeout after 1 second, in case the kernel stops - * doing allocations, to avoid stalling this worker task for too long. - */ - { - unsigned long end_wait = jiffies + HZ; - - do { - set_current_state(TASK_UNINTERRUPTIBLE); - if (atomic_read(&kfence_allocation_gate) != 0) - break; - schedule_timeout(1); - } while (time_before(jiffies, end_wait)); - __set_current_state(TASK_RUNNING); - } + + wait_event_timeout(allocation_wait, atomic_read(&kfence_allocation_gate), HZ); + /* Disable static key and reset timer. */ static_branch_disable(&kfence_allocation_key); #endif @@ -728,6 +728,19 @@ void *__kfence_alloc(struct kmem_cache *s, size_t size, gfp_t flags) */ if (atomic_read(&kfence_allocation_gate) || atomic_inc_return(&kfence_allocation_gate) > 1) return NULL; +#ifdef CONFIG_KFENCE_STATIC_KEYS + /* + * waitqueue_active() is fully ordered after the update of + * kfence_allocation_gate per atomic_inc_return(). + */ + if (waitqueue_active(&allocation_wait)) { + /* + * Calling wake_up() here may deadlock when allocations happen + * from within timer code. Use an irq_work to defer it. + */ + irq_work_queue(&wake_up_kfence_timer_work); + } +#endif if (!READ_ONCE(kfence_enabled)) return NULL; diff --git a/mm/khugepaged.c b/mm/khugepaged.c index a7d6cb912b05..2680d5ffee7f 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -716,17 +716,17 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, if (pte_write(pteval)) writable = true; } - if (likely(writable)) { - if (likely(referenced)) { - result = SCAN_SUCCEED; - trace_mm_collapse_huge_page_isolate(page, none_or_zero, - referenced, writable, result); - return 1; - } - } else { + + if (unlikely(!writable)) { result = SCAN_PAGE_RO; + } else if (unlikely(!referenced)) { + result = SCAN_LACK_REFERENCED_PAGE; + } else { + result = SCAN_SUCCEED; + trace_mm_collapse_huge_page_isolate(page, none_or_zero, + referenced, writable, result); + return 1; } - out: release_pte_pages(pte, _pte, compound_pagelist); trace_mm_collapse_huge_page_isolate(page, none_or_zero, diff --git a/mm/ksm.c b/mm/ksm.c index 9694ee2c71de..c7ee7e62d424 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -794,6 +794,7 @@ static void remove_rmap_item_from_tree(struct rmap_item *rmap_item) stable_node->rmap_hlist_len--; put_anon_vma(rmap_item->anon_vma); + rmap_item->head = NULL; rmap_item->address &= PAGE_MASK; } else if (rmap_item->address & UNSTABLE_FLAG) { @@ -2434,54 +2435,78 @@ static int ksm_scan_thread(void *nothing) return 0; } -int ksm_madvise(struct vm_area_struct *vma, unsigned long start, - unsigned long end, int advice, unsigned long *vm_flags) +int ksm_madvise_merge(struct mm_struct *mm, struct vm_area_struct *vma, + unsigned long *vm_flags) { - struct mm_struct *mm = vma->vm_mm; int err; - switch (advice) { - case MADV_MERGEABLE: - /* - * Be somewhat over-protective for now! - */ - if (*vm_flags & (VM_MERGEABLE | VM_SHARED | VM_MAYSHARE | - VM_PFNMAP | VM_IO | VM_DONTEXPAND | - VM_HUGETLB | VM_MIXEDMAP)) - return 0; /* just ignore the advice */ + /* + * Be somewhat over-protective for now! + */ + if (*vm_flags & (VM_MERGEABLE | VM_SHARED | VM_MAYSHARE | + VM_PFNMAP | VM_IO | VM_DONTEXPAND | + VM_HUGETLB | VM_MIXEDMAP)) + return 0; /* just ignore the advice */ - if (vma_is_dax(vma)) - return 0; + if (vma_is_dax(vma)) + return 0; #ifdef VM_SAO if (*vm_flags & VM_SAO) return 0; #endif #ifdef VM_SPARC_ADI - if (*vm_flags & VM_SPARC_ADI) - return 0; + if (*vm_flags & VM_SPARC_ADI) + return 0; #endif - if (!test_bit(MMF_VM_MERGEABLE, &mm->flags)) { - err = __ksm_enter(mm); - if (err) - return err; - } + if (!test_bit(MMF_VM_MERGEABLE, &mm->flags)) { + err = __ksm_enter(mm); + if (err) + return err; + } - *vm_flags |= VM_MERGEABLE; - break; + *vm_flags |= VM_MERGEABLE; - case MADV_UNMERGEABLE: - if (!(*vm_flags & VM_MERGEABLE)) - return 0; /* just ignore the advice */ + return 0; +} - if (vma->anon_vma) { - err = unmerge_ksm_pages(vma, start, end); - if (err) - return err; - } +int ksm_madvise_unmerge(struct vm_area_struct *vma, unsigned long start, + unsigned long end, unsigned long *vm_flags) +{ + int err; + + if (!(*vm_flags & VM_MERGEABLE)) + return 0; /* just ignore the advice */ + + if (vma->anon_vma) { + err = unmerge_ksm_pages(vma, start, end); + if (err) + return err; + } - *vm_flags &= ~VM_MERGEABLE; + *vm_flags &= ~VM_MERGEABLE; + + return 0; +} + +int ksm_madvise(struct vm_area_struct *vma, unsigned long start, + unsigned long end, int advice, unsigned long *vm_flags) +{ + struct mm_struct *mm = vma->vm_mm; + int err; + + switch (advice) { + case MADV_MERGEABLE: + err = ksm_madvise_merge(mm, vma, vm_flags); + if (err) + return err; + break; + + case MADV_UNMERGEABLE: + err = ksm_madvise_unmerge(vma, start, end, vm_flags); + if (err) + return err; break; } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index e064ac0d850a..e876ba693998 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -3181,9 +3181,17 @@ static void drain_obj_stock(struct memcg_stock_pcp *stock) unsigned int nr_bytes = stock->nr_bytes & (PAGE_SIZE - 1); if (nr_pages) { + struct mem_cgroup *memcg; + rcu_read_lock(); - __memcg_kmem_uncharge(obj_cgroup_memcg(old), nr_pages); +retry: + memcg = obj_cgroup_memcg(old); + if (unlikely(!css_tryget(&memcg->css))) + goto retry; rcu_read_unlock(); + + __memcg_kmem_uncharge(memcg, nr_pages); + css_put(&memcg->css); } /* diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 24210c9bd843..bd3945446d47 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -1368,7 +1368,7 @@ static int memory_failure_dev_pagemap(unsigned long pfn, int flags, * communicated in siginfo, see kill_proc() */ start = (page->index << PAGE_SHIFT) & ~(size - 1); - unmap_mapping_range(page->mapping, start, start + size, 0); + unmap_mapping_range(page->mapping, start, size, 0); } kill_procs(&tokill, flags & MF_MUST_KILL, !unmap_success, pfn, flags); rc = 0; diff --git a/mm/migrate.c b/mm/migrate.c index 62b81d5257aa..773622cffe77 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -2973,6 +2973,13 @@ static void migrate_vma_insert_page(struct migrate_vma *migrate, swp_entry = make_device_private_entry(page, vma->vm_flags & VM_WRITE); entry = swp_entry_to_pte(swp_entry); + } else { + /* + * For now we only support migrating to un-addressable + * device memory. + */ + pr_warn_once("Unsupported ZONE_DEVICE page type.\n"); + goto abort; } } else { entry = mk_pte(page, vma->vm_page_prot); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index cfc72873961d..4bb3cdfc47f8 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -764,32 +764,36 @@ static inline void clear_page_guard(struct zone *zone, struct page *page, */ void init_mem_debugging_and_hardening(void) { + bool page_poisoning_requested = false; + +#ifdef CONFIG_PAGE_POISONING + /* + * Page poisoning is debug page alloc for some arches. If + * either of those options are enabled, enable poisoning. + */ + if (page_poisoning_enabled() || + (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) && + debug_pagealloc_enabled())) { + static_branch_enable(&_page_poisoning_enabled); + page_poisoning_requested = true; + } +#endif + if (_init_on_alloc_enabled_early) { - if (page_poisoning_enabled()) + if (page_poisoning_requested) pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, " "will take precedence over init_on_alloc\n"); else static_branch_enable(&init_on_alloc); } if (_init_on_free_enabled_early) { - if (page_poisoning_enabled()) + if (page_poisoning_requested) pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, " "will take precedence over init_on_free\n"); else static_branch_enable(&init_on_free); } -#ifdef CONFIG_PAGE_POISONING - /* - * Page poisoning is debug page alloc for some arches. If - * either of those options are enabled, enable poisoning. - */ - if (page_poisoning_enabled() || - (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) && - debug_pagealloc_enabled())) - static_branch_enable(&_page_poisoning_enabled); -#endif - #ifdef CONFIG_DEBUG_PAGEALLOC if (!debug_pagealloc_enabled()) return; diff --git a/mm/shmem.c b/mm/shmem.c index b2db4ed0fbc7..6e99a4ad6e1f 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -2258,25 +2258,11 @@ int shmem_lock(struct file *file, int lock, struct user_struct *user) static int shmem_mmap(struct file *file, struct vm_area_struct *vma) { struct shmem_inode_info *info = SHMEM_I(file_inode(file)); + int ret; - if (info->seals & F_SEAL_FUTURE_WRITE) { - /* - * New PROT_WRITE and MAP_SHARED mmaps are not allowed when - * "future write" seal active. - */ - if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_WRITE)) - return -EPERM; - - /* - * Since an F_SEAL_FUTURE_WRITE sealed memfd can be mapped as - * MAP_SHARED and read-only, take care to not allow mprotect to - * revert protections on such mappings. Do this only for shared - * mappings. For private mappings, don't need to mask - * VM_MAYWRITE as we still want them to be COW-writable. - */ - if (vma->vm_flags & VM_SHARED) - vma->vm_flags &= ~(VM_MAYWRITE); - } + ret = seal_check_future_write(info->seals, vma); + if (ret) + return ret; /* arm64 - allow memory tagging on RAM-based files */ vma->vm_flags |= VM_MTE_ALLOWED; @@ -2375,8 +2361,18 @@ static int shmem_mfill_atomic_pte(struct mm_struct *dst_mm, pgoff_t offset, max_off; ret = -ENOMEM; - if (!shmem_inode_acct_block(inode, 1)) + if (!shmem_inode_acct_block(inode, 1)) { + /* + * We may have got a page, returned -ENOENT triggering a retry, + * and now we find ourselves with -ENOMEM. Release the page, to + * avoid a BUG_ON in our caller. + */ + if (unlikely(*pagep)) { + put_page(*pagep); + *pagep = NULL; + } goto out; + } if (!*pagep) { page = shmem_alloc_page(gfp, info, pgoff); diff --git a/mm/sparse.c b/mm/sparse.c index 7bd23f9d6cef..33406ea2ecc4 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -547,6 +547,7 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin, pr_err("%s: node[%d] memory map backing failed. Some memory will not be available.", __func__, nid); pnum_begin = pnum; + sparse_buffer_fini(); goto failed; } check_usemap_section_nr(nid, usage); diff --git a/mm/vmscan.c b/mm/vmscan.c index 562e87cbd7a1..a20fda05c05c 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -115,6 +115,14 @@ struct scan_control { /* There is easily reclaimable cold cache in the current node */ unsigned int cache_trim_mode:1; +#if defined(CONFIG_UNEVICTABLE_FILE) + /* The file pages on the current node are low */ + unsigned int file_is_low:1; + + /* The file pages on the current node are minimal */ + unsigned int file_is_min:1; +#endif + /* The file pages on the current node are dangerously low */ unsigned int file_is_tiny:1; @@ -164,6 +172,11 @@ struct scan_control { #define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0) #endif +#if defined(CONFIG_UNEVICTABLE_FILE) +extern unsigned long sysctl_unevictable_file_kbytes_low; +extern unsigned long sysctl_unevictable_file_kbytes_min; +#endif + /* * From 0 .. 200. Higher means more swappy. */ @@ -2417,6 +2430,15 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, BUG(); } +#if defined(CONFIG_UNEVICTABLE_FILE) + if (file && scan) { + if (sc->file_is_low) + scan = min(scan, SWAP_CLUSTER_MAX >> sc->priority); + else if (sc->file_is_min) + scan = 0; + } +#endif + nr[lru] = scan; } } @@ -2663,6 +2685,10 @@ static void shrink_node_memcgs(pg_data_t *pgdat, struct scan_control *sc) } while ((memcg = mem_cgroup_iter(target_memcg, memcg, NULL))); } +#if defined(CONFIG_UNEVICTABLE_FILE) +#define K(x) ((x) << (PAGE_SHIFT - 10)) +#endif + static void shrink_node(pg_data_t *pgdat, struct scan_control *sc) { struct reclaim_state *reclaim_state = current->reclaim_state; @@ -2740,11 +2766,18 @@ static void shrink_node(pg_data_t *pgdat, struct scan_control *sc) if (!cgroup_reclaim(sc)) { unsigned long total_high_wmark = 0; unsigned long free, anon; +#if defined(CONFIG_UNEVICTABLE_FILE) + unsigned long reclaimable_file, clean_file, dirty_file; +#endif int z; free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES); file = node_page_state(pgdat, NR_ACTIVE_FILE) + node_page_state(pgdat, NR_INACTIVE_FILE); +#if defined(CONFIG_UNEVICTABLE_FILE) + reclaimable_file = file + node_page_state(pgdat, NR_ISOLATED_FILE); + dirty_file = node_page_state(pgdat, NR_FILE_DIRTY); +#endif for (z = 0; z < MAX_NR_ZONES; z++) { struct zone *zone = &pgdat->node_zones[z]; @@ -2765,6 +2798,26 @@ static void shrink_node(pg_data_t *pgdat, struct scan_control *sc) file + free <= total_high_wmark && !(sc->may_deactivate & DEACTIVATE_ANON) && anon >> sc->priority; + +#if defined(CONFIG_UNEVICTABLE_FILE) + /* + * node_page_state() sum can go out of sync since + * all the values are not read at once + */ + if (unlikely(reclaimable_file < dirty_file)) + /* + * in this case assume the system does not have + * clean file pages anymore + */ + clean_file = 0; + else + clean_file = reclaimable_file - dirty_file; + + sc->file_is_low = K(clean_file) < sysctl_unevictable_file_kbytes_low && + K(clean_file) > sysctl_unevictable_file_kbytes_min; + + sc->file_is_min = K(clean_file) <= sysctl_unevictable_file_kbytes_min; +#endif } shrink_node_memcgs(pgdat, sc); diff --git a/net/bluetooth/ecdh_helper.h b/net/bluetooth/ecdh_helper.h index a6f8d03d4aaf..830723971cf8 100644 --- a/net/bluetooth/ecdh_helper.h +++ b/net/bluetooth/ecdh_helper.h @@ -25,6 +25,6 @@ int compute_ecdh_secret(struct crypto_kpp *tfm, const u8 pair_public_key[64], u8 secret[32]); -int set_ecdh_privkey(struct crypto_kpp *tfm, const u8 *private_key); +int set_ecdh_privkey(struct crypto_kpp *tfm, const u8 private_key[32]); int generate_ecdh_public_key(struct crypto_kpp *tfm, u8 public_key[64]); int generate_ecdh_keys(struct crypto_kpp *tfm, u8 public_key[64]); diff --git a/net/bluetooth/hci_conn.c b/net/bluetooth/hci_conn.c index 6ffa89e3ba0a..f72646690539 100644 --- a/net/bluetooth/hci_conn.c +++ b/net/bluetooth/hci_conn.c @@ -1830,8 +1830,6 @@ u32 hci_conn_get_phy(struct hci_conn *conn) { u32 phys = 0; - hci_dev_lock(conn->hdev); - /* BLUETOOTH CORE SPECIFICATION Version 5.2 | Vol 2, Part B page 471: * Table 6.2: Packets defined for synchronous, asynchronous, and * CSB logical transport types. @@ -1928,7 +1926,5 @@ u32 hci_conn_get_phy(struct hci_conn *conn) break; } - hci_dev_unlock(conn->hdev); - return phys; } diff --git a/net/bluetooth/hci_event.c b/net/bluetooth/hci_event.c index 67668be3461e..82f4973a011d 100644 --- a/net/bluetooth/hci_event.c +++ b/net/bluetooth/hci_event.c @@ -5005,6 +5005,7 @@ static void hci_loglink_complete_evt(struct hci_dev *hdev, struct sk_buff *skb) return; hchan->handle = le16_to_cpu(ev->handle); + hchan->amp = true; BT_DBG("hcon %p mgr %p hchan %p", hcon, hcon->amp_mgr, hchan); @@ -5037,7 +5038,7 @@ static void hci_disconn_loglink_complete_evt(struct hci_dev *hdev, hci_dev_lock(hdev); hchan = hci_chan_lookup_handle(hdev, le16_to_cpu(ev->handle)); - if (!hchan) + if (!hchan || !hchan->amp) goto unlock; amp_destroy_logical_link(hchan, ev->reason); @@ -5911,7 +5912,7 @@ static void hci_le_phy_update_evt(struct hci_dev *hdev, struct sk_buff *skb) BT_DBG("%s status 0x%2.2x", hdev->name, ev->status); - if (!ev->status) + if (ev->status) return; hci_dev_lock(hdev); diff --git a/net/bluetooth/hci_request.c b/net/bluetooth/hci_request.c index e55976db4403..805ce546b813 100644 --- a/net/bluetooth/hci_request.c +++ b/net/bluetooth/hci_request.c @@ -272,12 +272,16 @@ int hci_req_sync(struct hci_dev *hdev, int (*req)(struct hci_request *req, { int ret; - if (!test_bit(HCI_UP, &hdev->flags)) - return -ENETDOWN; - /* Serialize all requests */ hci_req_sync_lock(hdev); - ret = __hci_req_sync(hdev, req, opt, timeout, hci_status); + /* check the state after obtaing the lock to protect the HCI_UP + * against any races from hci_dev_do_close when the controller + * gets removed. + */ + if (test_bit(HCI_UP, &hdev->flags)) + ret = __hci_req_sync(hdev, req, opt, timeout, hci_status); + else + ret = -ENETDOWN; hci_req_sync_unlock(hdev); return ret; diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c index 72c2f5226d67..53ddbee459b9 100644 --- a/net/bluetooth/l2cap_core.c +++ b/net/bluetooth/l2cap_core.c @@ -451,6 +451,8 @@ struct l2cap_chan *l2cap_chan_create(void) if (!chan) return NULL; + skb_queue_head_init(&chan->tx_q); + skb_queue_head_init(&chan->srej_q); mutex_init(&chan->lock); /* Set default lock nesting level */ @@ -516,7 +518,9 @@ void l2cap_chan_set_defaults(struct l2cap_chan *chan) chan->flush_to = L2CAP_DEFAULT_FLUSH_TO; chan->retrans_timeout = L2CAP_DEFAULT_RETRANS_TO; chan->monitor_timeout = L2CAP_DEFAULT_MONITOR_TO; + chan->conf_state = 0; + set_bit(CONF_NOT_COMPLETE, &chan->conf_state); set_bit(FLAG_FORCE_ACTIVE, &chan->flags); } diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c index f1b1edd0b697..c99d65ef13b1 100644 --- a/net/bluetooth/l2cap_sock.c +++ b/net/bluetooth/l2cap_sock.c @@ -179,9 +179,17 @@ static int l2cap_sock_connect(struct socket *sock, struct sockaddr *addr, struct l2cap_chan *chan = l2cap_pi(sk)->chan; struct sockaddr_l2 la; int len, err = 0; + bool zapped; BT_DBG("sk %p", sk); + lock_sock(sk); + zapped = sock_flag(sk, SOCK_ZAPPED); + release_sock(sk); + + if (zapped) + return -EINVAL; + if (!addr || alen < offsetofend(struct sockaddr, sa_family) || addr->sa_family != AF_BLUETOOTH) return -EINVAL; diff --git a/net/bluetooth/mgmt.c b/net/bluetooth/mgmt.c index 74971b4bd457..939c6f77fecc 100644 --- a/net/bluetooth/mgmt.c +++ b/net/bluetooth/mgmt.c @@ -7976,7 +7976,6 @@ static int add_ext_adv_params(struct sock *sk, struct hci_dev *hdev, goto unlock; } - hdev->cur_adv_instance = cp->instance; /* Submit request for advertising params if ext adv available */ if (ext_adv_capable(hdev)) { hci_req_init(&req, hdev); diff --git a/net/bridge/br_arp_nd_proxy.c b/net/bridge/br_arp_nd_proxy.c index dfec65eca8a6..3db1def4437b 100644 --- a/net/bridge/br_arp_nd_proxy.c +++ b/net/bridge/br_arp_nd_proxy.c @@ -160,7 +160,9 @@ void br_do_proxy_suppress_arp(struct sk_buff *skb, struct net_bridge *br, if (br_opt_get(br, BROPT_NEIGH_SUPPRESS_ENABLED)) { if (p && (p->flags & BR_NEIGH_SUPPRESS)) return; - if (ipv4_is_zeronet(sip) || sip == tip) { + if (parp->ar_op != htons(ARPOP_RREQUEST) && + parp->ar_op != htons(ARPOP_RREPLY) && + (ipv4_is_zeronet(sip) || sip == tip)) { /* prevent flooding to neigh suppress ports */ BR_INPUT_SKB_CB(skb)->proxyarp_replied = 1; return; diff --git a/net/bridge/br_multicast.c b/net/bridge/br_multicast.c index 9d265447d654..226bb05c3b42 100644 --- a/net/bridge/br_multicast.c +++ b/net/bridge/br_multicast.c @@ -1593,7 +1593,8 @@ static void br_multicast_port_group_rexmit(struct timer_list *t) spin_unlock(&br->multicast_lock); } -static void br_mc_disabled_update(struct net_device *dev, bool value) +static int br_mc_disabled_update(struct net_device *dev, bool value, + struct netlink_ext_ack *extack) { struct switchdev_attr attr = { .orig_dev = dev, @@ -1602,11 +1603,13 @@ static void br_mc_disabled_update(struct net_device *dev, bool value) .u.mc_disabled = !value, }; - switchdev_port_attr_set(dev, &attr, NULL); + return switchdev_port_attr_set(dev, &attr, extack); } int br_multicast_add_port(struct net_bridge_port *port) { + int err; + port->multicast_router = MDB_RTR_TYPE_TEMP_QUERY; port->multicast_eht_hosts_limit = BR_MCAST_DEFAULT_EHT_HOSTS_LIMIT; @@ -1618,8 +1621,12 @@ int br_multicast_add_port(struct net_bridge_port *port) timer_setup(&port->ip6_own_query.timer, br_ip6_multicast_port_query_expired, 0); #endif - br_mc_disabled_update(port->dev, - br_opt_get(port->br, BROPT_MULTICAST_ENABLED)); + err = br_mc_disabled_update(port->dev, + br_opt_get(port->br, + BROPT_MULTICAST_ENABLED), + NULL); + if (err && err != -EOPNOTSUPP) + return err; port->mcast_stats = netdev_alloc_pcpu_stats(struct bridge_mcast_stats); if (!port->mcast_stats) @@ -3152,25 +3159,14 @@ static int br_multicast_ipv4_rcv(struct net_bridge *br, } #if IS_ENABLED(CONFIG_IPV6) -static int br_ip6_multicast_mrd_rcv(struct net_bridge *br, - struct net_bridge_port *port, - struct sk_buff *skb) +static void br_ip6_multicast_mrd_rcv(struct net_bridge *br, + struct net_bridge_port *port, + struct sk_buff *skb) { - int ret; - - if (ipv6_hdr(skb)->nexthdr != IPPROTO_ICMPV6) - return -ENOMSG; - - ret = ipv6_mc_check_icmpv6(skb); - if (ret < 0) - return ret; - if (icmp6_hdr(skb)->icmp6_type != ICMPV6_MRDISC_ADV) - return -ENOMSG; + return; br_multicast_mark_router(br, port); - - return 0; } static int br_multicast_ipv6_rcv(struct net_bridge *br, @@ -3184,18 +3180,12 @@ static int br_multicast_ipv6_rcv(struct net_bridge *br, err = ipv6_mc_check_mld(skb); - if (err == -ENOMSG) { + if (err == -ENOMSG || err == -ENODATA) { if (!ipv6_addr_is_ll_all_nodes(&ipv6_hdr(skb)->daddr)) BR_INPUT_SKB_CB(skb)->mrouters_only = 1; - - if (ipv6_addr_is_all_snoopers(&ipv6_hdr(skb)->daddr)) { - err = br_ip6_multicast_mrd_rcv(br, port, skb); - - if (err < 0 && err != -ENOMSG) { - br_multicast_err_count(br, port, skb->protocol); - return err; - } - } + if (err == -ENODATA && + ipv6_addr_is_all_snoopers(&ipv6_hdr(skb)->daddr)) + br_ip6_multicast_mrd_rcv(br, port, skb); return 0; } else if (err < 0) { @@ -3560,16 +3550,23 @@ static void br_multicast_start_querier(struct net_bridge *br, rcu_read_unlock(); } -int br_multicast_toggle(struct net_bridge *br, unsigned long val) +int br_multicast_toggle(struct net_bridge *br, unsigned long val, + struct netlink_ext_ack *extack) { struct net_bridge_port *port; bool change_snoopers = false; + int err = 0; spin_lock_bh(&br->multicast_lock); if (!!br_opt_get(br, BROPT_MULTICAST_ENABLED) == !!val) goto unlock; - br_mc_disabled_update(br->dev, val); + err = br_mc_disabled_update(br->dev, val, extack); + if (err == -EOPNOTSUPP) + err = 0; + if (err) + goto unlock; + br_opt_toggle(br, BROPT_MULTICAST_ENABLED, !!val); if (!br_opt_get(br, BROPT_MULTICAST_ENABLED)) { change_snoopers = true; @@ -3607,7 +3604,7 @@ int br_multicast_toggle(struct net_bridge *br, unsigned long val) br_multicast_leave_snoopers(br); } - return 0; + return err; } bool br_multicast_enabled(const struct net_device *dev) diff --git a/net/bridge/br_netlink.c b/net/bridge/br_netlink.c index f2b1343f8332..e4e6e991313e 100644 --- a/net/bridge/br_netlink.c +++ b/net/bridge/br_netlink.c @@ -103,8 +103,9 @@ static size_t br_get_link_af_size_filtered(const struct net_device *dev, rcu_read_lock(); if (netif_is_bridge_port(dev)) { - p = br_port_get_rcu(dev); - vg = nbp_vlan_group_rcu(p); + p = br_port_get_check_rcu(dev); + if (p) + vg = nbp_vlan_group_rcu(p); } else if (dev->priv_flags & IFF_EBRIDGE) { br = netdev_priv(dev); vg = br_vlan_group_rcu(br); @@ -1293,7 +1294,9 @@ static int br_changelink(struct net_device *brdev, struct nlattr *tb[], if (data[IFLA_BR_MCAST_SNOOPING]) { u8 mcast_snooping = nla_get_u8(data[IFLA_BR_MCAST_SNOOPING]); - br_multicast_toggle(br, mcast_snooping); + err = br_multicast_toggle(br, mcast_snooping, extack); + if (err) + return err; } if (data[IFLA_BR_MCAST_QUERY_USE_IFADDR]) { diff --git a/net/bridge/br_private.h b/net/bridge/br_private.h index d7d167e10b70..af3430c2d6ea 100644 --- a/net/bridge/br_private.h +++ b/net/bridge/br_private.h @@ -810,7 +810,8 @@ void br_multicast_flood(struct net_bridge_mdb_entry *mdst, struct sk_buff *skb, bool local_rcv, bool local_orig); int br_multicast_set_router(struct net_bridge *br, unsigned long val); int br_multicast_set_port_router(struct net_bridge_port *p, unsigned long val); -int br_multicast_toggle(struct net_bridge *br, unsigned long val); +int br_multicast_toggle(struct net_bridge *br, unsigned long val, + struct netlink_ext_ack *extack); int br_multicast_set_querier(struct net_bridge *br, unsigned long val); int br_multicast_set_hash_max(struct net_bridge *br, unsigned long val); int br_multicast_set_igmp_version(struct net_bridge *br, unsigned long val); diff --git a/net/bridge/br_sysfs_br.c b/net/bridge/br_sysfs_br.c index 072e29840082..381467b691d5 100644 --- a/net/bridge/br_sysfs_br.c +++ b/net/bridge/br_sysfs_br.c @@ -409,17 +409,11 @@ static ssize_t multicast_snooping_show(struct device *d, return sprintf(buf, "%d\n", br_opt_get(br, BROPT_MULTICAST_ENABLED)); } -static int toggle_multicast(struct net_bridge *br, unsigned long val, - struct netlink_ext_ack *extack) -{ - return br_multicast_toggle(br, val); -} - static ssize_t multicast_snooping_store(struct device *d, struct device_attribute *attr, const char *buf, size_t len) { - return store_bridge_parm(d, buf, len, toggle_multicast); + return store_bridge_parm(d, buf, len, br_multicast_toggle); } static DEVICE_ATTR_RW(multicast_snooping); diff --git a/net/ceph/auth_x.c b/net/ceph/auth_x.c index ca44c327bace..79641c4afee9 100644 --- a/net/ceph/auth_x.c +++ b/net/ceph/auth_x.c @@ -526,7 +526,7 @@ static int ceph_x_build_request(struct ceph_auth_client *ac, if (ret < 0) return ret; - auth->struct_v = 2; /* nautilus+ */ + auth->struct_v = 3; /* nautilus+ */ auth->key = 0; for (u = (u64 *)enc_buf; u + 1 <= (u64 *)(enc_buf + ret); u++) auth->key ^= *(__le64 *)u; diff --git a/net/ceph/decode.c b/net/ceph/decode.c index b44f7651be04..bc109a1a4616 100644 --- a/net/ceph/decode.c +++ b/net/ceph/decode.c @@ -4,6 +4,7 @@ #include #include +#include /* for ceph_pr_addr() */ static int ceph_decode_entity_addr_versioned(void **p, void *end, @@ -110,6 +111,7 @@ int ceph_decode_entity_addrvec(void **p, void *end, bool msgr2, } ceph_decode_32_safe(p, end, addr_cnt, e_inval); + dout("%s addr_cnt %d\n", __func__, addr_cnt); found = false; for (i = 0; i < addr_cnt; i++) { @@ -117,6 +119,7 @@ int ceph_decode_entity_addrvec(void **p, void *end, bool msgr2, if (ret) return ret; + dout("%s i %d addr %s\n", __func__, i, ceph_pr_addr(&tmp_addr)); if (tmp_addr.type == my_type) { if (found) { pr_err("another match of type %d in addrvec\n", @@ -128,13 +131,18 @@ int ceph_decode_entity_addrvec(void **p, void *end, bool msgr2, found = true; } } - if (!found && addr_cnt != 0) { - pr_err("no match of type %d in addrvec\n", - le32_to_cpu(my_type)); - return -ENOENT; - } - return 0; + if (found) + return 0; + + if (!addr_cnt) + return 0; /* normal -- e.g. unused OSD id/slot */ + + if (addr_cnt == 1 && !memchr_inv(&tmp_addr, 0, sizeof(tmp_addr))) + return 0; /* weird but effectively the same as !addr_cnt */ + + pr_err("no match of type %d in addrvec\n", le32_to_cpu(my_type)); + return -ENOENT; e_inval: return -EINVAL; diff --git a/net/core/dev.c b/net/core/dev.c index 1f79b9aa9a3f..70829c568645 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4672,10 +4672,10 @@ static u32 netif_receive_generic_xdp(struct sk_buff *skb, void *orig_data, *orig_data_end, *hard_start; struct netdev_rx_queue *rxqueue; u32 metalen, act = XDP_DROP; + bool orig_bcast, orig_host; u32 mac_len, frame_sz; __be16 orig_eth_type; struct ethhdr *eth; - bool orig_bcast; int off; /* Reinjected packets coming from act_mirred or similar should @@ -4722,6 +4722,7 @@ static u32 netif_receive_generic_xdp(struct sk_buff *skb, orig_data_end = xdp->data_end; orig_data = xdp->data; eth = (struct ethhdr *)xdp->data; + orig_host = ether_addr_equal_64bits(eth->h_dest, skb->dev->dev_addr); orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest); orig_eth_type = eth->h_proto; @@ -4749,8 +4750,11 @@ static u32 netif_receive_generic_xdp(struct sk_buff *skb, /* check if XDP changed eth hdr such SKB needs update */ eth = (struct ethhdr *)xdp->data; if ((orig_eth_type != eth->h_proto) || + (orig_host != ether_addr_equal_64bits(eth->h_dest, + skb->dev->dev_addr)) || (orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) { __skb_push(skb, ETH_HLEN); + skb->pkt_type = PACKET_HOST; skb->protocol = eth_type_trans(skb, skb->dev); } @@ -5914,7 +5918,7 @@ static struct list_head *gro_list_prepare(struct napi_struct *napi, return head; } -static void skb_gro_reset_offset(struct sk_buff *skb) +static inline void skb_gro_reset_offset(struct sk_buff *skb, u32 nhoff) { const struct skb_shared_info *pinfo = skb_shinfo(skb); const skb_frag_t *frag0 = &pinfo->frags[0]; @@ -5925,7 +5929,7 @@ static void skb_gro_reset_offset(struct sk_buff *skb) if (!skb_headlen(skb) && pinfo->nr_frags && !PageHighMem(skb_frag_page(frag0)) && - (!NET_IP_ALIGN || !(skb_frag_off(frag0) & 3))) { + (!NET_IP_ALIGN || !((skb_frag_off(frag0) + nhoff) & 3))) { NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0); NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int, skb_frag_size(frag0), @@ -6143,7 +6147,7 @@ gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) skb_mark_napi_id(skb, napi); trace_napi_gro_receive_entry(skb); - skb_gro_reset_offset(skb); + skb_gro_reset_offset(skb, 0); ret = napi_skb_finish(napi, skb, dev_gro_receive(napi, skb)); trace_napi_gro_receive_exit(ret); @@ -6232,7 +6236,7 @@ static struct sk_buff *napi_frags_skb(struct napi_struct *napi) napi->skb = NULL; skb_reset_mac_header(skb); - skb_gro_reset_offset(skb); + skb_gro_reset_offset(skb, hlen); if (unlikely(skb_gro_header_hard(skb, hlen))) { eth = skb_gro_header_slow(skb, hlen, 0); diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c index a96a4f5de0ce..3f36b04d86a0 100644 --- a/net/core/flow_dissector.c +++ b/net/core/flow_dissector.c @@ -828,8 +828,10 @@ static void __skb_flow_bpf_to_target(const struct bpf_flow_keys *flow_keys, key_addrs = skb_flow_dissector_target(flow_dissector, FLOW_DISSECTOR_KEY_IPV6_ADDRS, target_container); - memcpy(&key_addrs->v6addrs, &flow_keys->ipv6_src, - sizeof(key_addrs->v6addrs)); + memcpy(&key_addrs->v6addrs.src, &flow_keys->ipv6_src, + sizeof(key_addrs->v6addrs.src)); + memcpy(&key_addrs->v6addrs.dst, &flow_keys->ipv6_dst, + sizeof(key_addrs->v6addrs.dst)); key_control->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; } diff --git a/net/core/page_pool.c b/net/core/page_pool.c index ad8b0707af04..f014fd8c19a6 100644 --- a/net/core/page_pool.c +++ b/net/core/page_pool.c @@ -174,8 +174,10 @@ static void page_pool_dma_sync_for_device(struct page_pool *pool, struct page *page, unsigned int dma_sync_size) { + dma_addr_t dma_addr = page_pool_get_dma_addr(page); + dma_sync_size = min(dma_sync_size, pool->p.max_len); - dma_sync_single_range_for_device(pool->p.dev, page->dma_addr, + dma_sync_single_range_for_device(pool->p.dev, dma_addr, pool->p.offset, dma_sync_size, pool->p.dma_dir); } @@ -226,7 +228,7 @@ static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool, put_page(page); return NULL; } - page->dma_addr = dma; + page_pool_set_dma_addr(page, dma); if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV) page_pool_dma_sync_for_device(pool, page, pool->p.max_len); @@ -294,13 +296,13 @@ void page_pool_release_page(struct page_pool *pool, struct page *page) */ goto skip_dma_unmap; - dma = page->dma_addr; + dma = page_pool_get_dma_addr(page); - /* When page is unmapped, it cannot be returned our pool */ + /* When page is unmapped, it cannot be returned to our pool */ dma_unmap_page_attrs(pool->p.dev, dma, PAGE_SIZE << pool->p.order, pool->p.dma_dir, DMA_ATTR_SKIP_CPU_SYNC); - page->dma_addr = 0; + page_pool_set_dma_addr(page, 0); skip_dma_unmap: /* This may be the last page returned, releasing the pool, so * it is not safe to reference pool afterwards. diff --git a/net/ethtool/ioctl.c b/net/ethtool/ioctl.c index 771688e1b0da..2603966da904 100644 --- a/net/ethtool/ioctl.c +++ b/net/ethtool/ioctl.c @@ -489,7 +489,7 @@ store_link_ksettings_for_user(void __user *to, { struct ethtool_link_usettings link_usettings; - memcpy(&link_usettings.base, &from->base, sizeof(link_usettings)); + memcpy(&link_usettings, from, sizeof(link_usettings)); bitmap_to_arr32(link_usettings.link_modes.supported, from->link_modes.supported, __ETHTOOL_LINK_MODE_MASK_NBITS); diff --git a/net/ethtool/netlink.c b/net/ethtool/netlink.c index 50d3c8896f91..25a55086d2b6 100644 --- a/net/ethtool/netlink.c +++ b/net/ethtool/netlink.c @@ -384,7 +384,8 @@ static int ethnl_default_dump_one(struct sk_buff *skb, struct net_device *dev, int ret; ehdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, - ðtool_genl_family, 0, ctx->ops->reply_cmd); + ðtool_genl_family, NLM_F_MULTI, + ctx->ops->reply_cmd); if (!ehdr) return -EMSGSIZE; diff --git a/net/hsr/hsr_forward.c b/net/hsr/hsr_forward.c index b218e4594009..6852e9bccf5b 100644 --- a/net/hsr/hsr_forward.c +++ b/net/hsr/hsr_forward.c @@ -520,6 +520,10 @@ static int fill_frame_info(struct hsr_frame_info *frame, struct ethhdr *ethhdr; __be16 proto; + /* Check if skb contains hsr_ethhdr */ + if (skb->mac_len < sizeof(struct hsr_ethhdr)) + return -EINVAL; + memset(frame, 0, sizeof(*frame)); frame->is_supervision = is_supervision_frame(port->hsr, skb); frame->node_src = hsr_get_node(port, &hsr->node_db, skb, diff --git a/net/ipv4/Kconfig b/net/ipv4/Kconfig index 87983e70f03f..a833a7a67ce7 100644 --- a/net/ipv4/Kconfig +++ b/net/ipv4/Kconfig @@ -669,6 +669,24 @@ config TCP_CONG_BBR AQM schemes that do not provide a delay signal. It requires the fq ("Fair Queue") pacing packet scheduler. +config TCP_CONG_BBR2 + tristate "BBR2 TCP" + default n + help + + BBR2 TCP congestion control is a model-based congestion control + algorithm that aims to maximize network utilization, keep queues and + retransmit rates low, and to be able to coexist with Reno/CUBIC in + common scenarios. It builds an explicit model of the network path. It + tolerates a targeted degree of random packet loss and delay that are + unrelated to congestion. It can operate over LAN, WAN, cellular, wifi, + or cable modem links, and can use DCTCP-L4S-style ECN signals. It can + coexist with flows that use loss-based congestion control, and can + operate with shallow buffers, deep buffers, bufferbloat, policers, or + AQM schemes that do not provide a delay signal. It requires pacing, + using either TCP internal pacing or the fq ("Fair Queue") pacing packet + scheduler. + choice prompt "Default TCP congestion control" default DEFAULT_CUBIC @@ -706,6 +724,9 @@ choice config DEFAULT_BBR bool "BBR" if TCP_CONG_BBR=y + config DEFAULT_BBR2 + bool "BBR2" if TCP_CONG_BBR2=y + config DEFAULT_RENO bool "Reno" endchoice @@ -730,6 +751,7 @@ config DEFAULT_TCP_CONG default "dctcp" if DEFAULT_DCTCP default "cdg" if DEFAULT_CDG default "bbr" if DEFAULT_BBR + default "bbr2" if DEFAULT_BBR2 default "cubic" config TCP_MD5SIG diff --git a/net/ipv4/Makefile b/net/ipv4/Makefile index 5b77a46885b9..8c5779dba462 100644 --- a/net/ipv4/Makefile +++ b/net/ipv4/Makefile @@ -46,6 +46,7 @@ obj-$(CONFIG_INET_TCP_DIAG) += tcp_diag.o obj-$(CONFIG_INET_UDP_DIAG) += udp_diag.o obj-$(CONFIG_INET_RAW_DIAG) += raw_diag.o obj-$(CONFIG_TCP_CONG_BBR) += tcp_bbr.o +obj-$(CONFIG_TCP_CONG_BBR2) += tcp_bbr2.o obj-$(CONFIG_TCP_CONG_BIC) += tcp_bic.o obj-$(CONFIG_TCP_CONG_CDG) += tcp_cdg.o obj-$(CONFIG_TCP_CONG_CUBIC) += tcp_cubic.o diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c index d520e61649c8..22129c1c56a2 100644 --- a/net/ipv4/bpf_tcp_ca.c +++ b/net/ipv4/bpf_tcp_ca.c @@ -16,7 +16,7 @@ static u32 optional_ops[] = { offsetof(struct tcp_congestion_ops, cwnd_event), offsetof(struct tcp_congestion_ops, in_ack_event), offsetof(struct tcp_congestion_ops, pkts_acked), - offsetof(struct tcp_congestion_ops, min_tso_segs), + offsetof(struct tcp_congestion_ops, tso_segs), offsetof(struct tcp_congestion_ops, sndbuf_expand), offsetof(struct tcp_congestion_ops, cong_control), }; diff --git a/net/ipv4/route.c b/net/ipv4/route.c index bba150fdd265..d635b4f32d34 100644 --- a/net/ipv4/route.c +++ b/net/ipv4/route.c @@ -66,6 +66,7 @@ #include #include #include +#include #include #include #include @@ -478,8 +479,10 @@ static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr) __ipv4_confirm_neigh(dev, *(__force u32 *)pkey); } -#define IP_IDENTS_SZ 2048u - +/* Hash tables of size 2048..262144 depending on RAM size. + * Each bucket uses 8 bytes. + */ +static u32 ip_idents_mask __read_mostly; static atomic_t *ip_idents __read_mostly; static u32 *ip_tstamps __read_mostly; @@ -489,12 +492,16 @@ static u32 *ip_tstamps __read_mostly; */ u32 ip_idents_reserve(u32 hash, int segs) { - u32 *p_tstamp = ip_tstamps + hash % IP_IDENTS_SZ; - atomic_t *p_id = ip_idents + hash % IP_IDENTS_SZ; - u32 old = READ_ONCE(*p_tstamp); - u32 now = (u32)jiffies; + u32 bucket, old, now = (u32)jiffies; + atomic_t *p_id; + u32 *p_tstamp; u32 delta = 0; + bucket = hash & ip_idents_mask; + p_tstamp = ip_tstamps + bucket; + p_id = ip_idents + bucket; + old = READ_ONCE(*p_tstamp); + if (old != now && cmpxchg(p_tstamp, old, now) == old) delta = prandom_u32_max(now - old); @@ -3553,18 +3560,25 @@ struct ip_rt_acct __percpu *ip_rt_acct __read_mostly; int __init ip_rt_init(void) { + void *idents_hash; int cpu; - ip_idents = kmalloc_array(IP_IDENTS_SZ, sizeof(*ip_idents), - GFP_KERNEL); - if (!ip_idents) - panic("IP: failed to allocate ip_idents\n"); + /* For modern hosts, this will use 2 MB of memory */ + idents_hash = alloc_large_system_hash("IP idents", + sizeof(*ip_idents) + sizeof(*ip_tstamps), + 0, + 16, /* one bucket per 64 KB */ + HASH_ZERO, + NULL, + &ip_idents_mask, + 2048, + 256*1024); + + ip_idents = idents_hash; - prandom_bytes(ip_idents, IP_IDENTS_SZ * sizeof(*ip_idents)); + prandom_bytes(ip_idents, (ip_idents_mask + 1) * sizeof(*ip_idents)); - ip_tstamps = kcalloc(IP_IDENTS_SZ, sizeof(*ip_tstamps), GFP_KERNEL); - if (!ip_tstamps) - panic("IP: failed to allocate ip_tstamps\n"); + ip_tstamps = idents_hash + (ip_idents_mask + 1) * sizeof(*ip_idents); for_each_possible_cpu(cpu) { struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu); diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index de7cc8445ac0..521f310f2ac1 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -3033,6 +3033,7 @@ int tcp_disconnect(struct sock *sk, int flags) tp->rx_opt.dsack = 0; tp->rx_opt.num_sacks = 0; tp->rcv_ooopack = 0; + tp->fast_ack_mode = 0; /* Clean up fastopen related fields */ diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c index 6ea3dc2e4219..8ef512fefe25 100644 --- a/net/ipv4/tcp_bbr.c +++ b/net/ipv4/tcp_bbr.c @@ -292,26 +292,40 @@ static void bbr_set_pacing_rate(struct sock *sk, u32 bw, int gain) sk->sk_pacing_rate = rate; } -/* override sysctl_tcp_min_tso_segs */ static u32 bbr_min_tso_segs(struct sock *sk) { return sk->sk_pacing_rate < (bbr_min_tso_rate >> 3) ? 1 : 2; } +/* Return the number of segments BBR would like in a TSO/GSO skb, given + * a particular max gso size as a constraint. + */ +static u32 bbr_tso_segs_generic(struct sock *sk, unsigned int mss_now, + u32 gso_max_size) +{ + u32 segs; + u64 bytes; + + /* Budget a TSO/GSO burst size allowance based on bw (pacing_rate). */ + bytes = sk->sk_pacing_rate >> sk->sk_pacing_shift; + + bytes = min_t(u32, bytes, gso_max_size - 1 - MAX_TCP_HEADER); + segs = max_t(u32, bytes / mss_now, bbr_min_tso_segs(sk)); + return segs; +} + +/* Custom tcp_tso_autosize() for BBR, used at transmit time to cap skb size. */ +static u32 bbr_tso_segs(struct sock *sk, unsigned int mss_now) +{ + return bbr_tso_segs_generic(sk, mss_now, sk->sk_gso_max_size); +} + +/* Like bbr_tso_segs(), using mss_cache, ignoring driver's sk_gso_max_size. */ static u32 bbr_tso_segs_goal(struct sock *sk) { struct tcp_sock *tp = tcp_sk(sk); - u32 segs, bytes; - - /* Sort of tcp_tso_autosize() but ignoring - * driver provided sk_gso_max_size. - */ - bytes = min_t(unsigned long, - sk->sk_pacing_rate >> READ_ONCE(sk->sk_pacing_shift), - GSO_MAX_SIZE - 1 - MAX_TCP_HEADER); - segs = max_t(u32, bytes / tp->mss_cache, bbr_min_tso_segs(sk)); - return min(segs, 0x7FU); + return bbr_tso_segs_generic(sk, tp->mss_cache, GSO_MAX_SIZE); } /* Save "last known good" cwnd so we can restore it after losses or PROBE_RTT */ @@ -1147,7 +1161,7 @@ static struct tcp_congestion_ops tcp_bbr_cong_ops __read_mostly = { .undo_cwnd = bbr_undo_cwnd, .cwnd_event = bbr_cwnd_event, .ssthresh = bbr_ssthresh, - .min_tso_segs = bbr_min_tso_segs, + .tso_segs = bbr_tso_segs, .get_info = bbr_get_info, .set_state = bbr_set_state, }; diff --git a/net/ipv4/tcp_bbr2.c b/net/ipv4/tcp_bbr2.c new file mode 100644 index 000000000000..5510adc92bbb --- /dev/null +++ b/net/ipv4/tcp_bbr2.c @@ -0,0 +1,2671 @@ +/* BBR (Bottleneck Bandwidth and RTT) congestion control, v2 + * + * BBRv2 is a model-based congestion control algorithm that aims for low + * queues, low loss, and (bounded) Reno/CUBIC coexistence. To maintain a model + * of the network path, it uses measurements of bandwidth and RTT, as well as + * (if they occur) packet loss and/or DCTCP/L4S-style ECN signals. Note that + * although it can use ECN or loss signals explicitly, it does not require + * either; it can bound its in-flight data based on its estimate of the BDP. + * + * The model has both higher and lower bounds for the operating range: + * lo: bw_lo, inflight_lo: conservative short-term lower bound + * hi: bw_hi, inflight_hi: robust long-term upper bound + * The bandwidth-probing time scale is (a) extended dynamically based on + * estimated BDP to improve coexistence with Reno/CUBIC; (b) bounded by + * an interactive wall-clock time-scale to be more scalable and responsive + * than Reno and CUBIC. + * + * Here is a state transition diagram for BBR: + * + * | + * V + * +---> STARTUP ----+ + * | | | + * | V | + * | DRAIN ----+ + * | | | + * | V | + * +---> PROBE_BW ----+ + * | ^ | | + * | | | | + * | +----+ | + * | | + * +---- PROBE_RTT <--+ + * + * A BBR flow starts in STARTUP, and ramps up its sending rate quickly. + * When it estimates the pipe is full, it enters DRAIN to drain the queue. + * In steady state a BBR flow only uses PROBE_BW and PROBE_RTT. + * A long-lived BBR flow spends the vast majority of its time remaining + * (repeatedly) in PROBE_BW, fully probing and utilizing the pipe's bandwidth + * in a fair manner, with a small, bounded queue. *If* a flow has been + * continuously sending for the entire min_rtt window, and hasn't seen an RTT + * sample that matches or decreases its min_rtt estimate for 10 seconds, then + * it briefly enters PROBE_RTT to cut inflight to a minimum value to re-probe + * the path's two-way propagation delay (min_rtt). When exiting PROBE_RTT, if + * we estimated that we reached the full bw of the pipe then we enter PROBE_BW; + * otherwise we enter STARTUP to try to fill the pipe. + * + * BBR is described in detail in: + * "BBR: Congestion-Based Congestion Control", + * Neal Cardwell, Yuchung Cheng, C. Stephen Gunn, Soheil Hassas Yeganeh, + * Van Jacobson. ACM Queue, Vol. 14 No. 5, September-October 2016. + * + * There is a public e-mail list for discussing BBR development and testing: + * https://groups.google.com/forum/#!forum/bbr-dev + * + * NOTE: BBR might be used with the fq qdisc ("man tc-fq") with pacing enabled, + * otherwise TCP stack falls back to an internal pacing using one high + * resolution timer per TCP socket and may use more resources. + */ +#include +#include +#include +#include +#include + +#include "tcp_dctcp.h" + +/* Scale factor for rate in pkt/uSec unit to avoid truncation in bandwidth + * estimation. The rate unit ~= (1500 bytes / 1 usec / 2^24) ~= 715 bps. + * This handles bandwidths from 0.06pps (715bps) to 256Mpps (3Tbps) in a u32. + * Since the minimum window is >=4 packets, the lower bound isn't + * an issue. The upper bound isn't an issue with existing technologies. + */ +#define BW_SCALE 24 +#define BW_UNIT (1 << BW_SCALE) + +#define BBR_SCALE 8 /* scaling factor for fractions in BBR (e.g. gains) */ +#define BBR_UNIT (1 << BBR_SCALE) + +#define FLAG_DEBUG_VERBOSE 0x1 /* Verbose debugging messages */ +#define FLAG_DEBUG_LOOPBACK 0x2 /* Do NOT skip loopback addr */ + +#define CYCLE_LEN 8 /* number of phases in a pacing gain cycle */ + +/* BBR has the following modes for deciding how fast to send: */ +enum bbr_mode { + BBR_STARTUP, /* ramp up sending rate rapidly to fill pipe */ + BBR_DRAIN, /* drain any queue created during startup */ + BBR_PROBE_BW, /* discover, share bw: pace around estimated bw */ + BBR_PROBE_RTT, /* cut inflight to min to probe min_rtt */ +}; + +/* How does the incoming ACK stream relate to our bandwidth probing? */ +enum bbr_ack_phase { + BBR_ACKS_INIT, /* not probing; not getting probe feedback */ + BBR_ACKS_REFILLING, /* sending at est. bw to fill pipe */ + BBR_ACKS_PROBE_STARTING, /* inflight rising to probe bw */ + BBR_ACKS_PROBE_FEEDBACK, /* getting feedback from bw probing */ + BBR_ACKS_PROBE_STOPPING, /* stopped probing; still getting feedback */ +}; + +/* BBR congestion control block */ +struct bbr { + u32 min_rtt_us; /* min RTT in min_rtt_win_sec window */ + u32 min_rtt_stamp; /* timestamp of min_rtt_us */ + u32 probe_rtt_done_stamp; /* end time for BBR_PROBE_RTT mode */ + u32 probe_rtt_min_us; /* min RTT in bbr_probe_rtt_win_ms window */ + u32 probe_rtt_min_stamp; /* timestamp of probe_rtt_min_us*/ + u32 next_rtt_delivered; /* scb->tx.delivered at end of round */ + u32 prior_rcv_nxt; /* tp->rcv_nxt when CE state last changed */ + u64 cycle_mstamp; /* time of this cycle phase start */ + u32 mode:3, /* current bbr_mode in state machine */ + prev_ca_state:3, /* CA state on previous ACK */ + packet_conservation:1, /* use packet conservation? */ + round_start:1, /* start of packet-timed tx->ack round? */ + ce_state:1, /* If most recent data has CE bit set */ + bw_probe_up_rounds:5, /* cwnd-limited rounds in PROBE_UP */ + try_fast_path:1, /* can we take fast path? */ + unused2:11, + idle_restart:1, /* restarting after idle? */ + probe_rtt_round_done:1, /* a BBR_PROBE_RTT round at 4 pkts? */ + cycle_idx:3, /* current index in pacing_gain cycle array */ + has_seen_rtt:1; /* have we seen an RTT sample yet? */ + u32 pacing_gain:11, /* current gain for setting pacing rate */ + cwnd_gain:11, /* current gain for setting cwnd */ + full_bw_reached:1, /* reached full bw in Startup? */ + full_bw_cnt:2, /* number of rounds without large bw gains */ + init_cwnd:7; /* initial cwnd */ + u32 prior_cwnd; /* prior cwnd upon entering loss recovery */ + u32 full_bw; /* recent bw, to estimate if pipe is full */ + + /* For tracking ACK aggregation: */ + u64 ack_epoch_mstamp; /* start of ACK sampling epoch */ + u16 extra_acked[2]; /* max excess data ACKed in epoch */ + u32 ack_epoch_acked:20, /* packets (S)ACKed in sampling epoch */ + extra_acked_win_rtts:5, /* age of extra_acked, in round trips */ + extra_acked_win_idx:1, /* current index in extra_acked array */ + /* BBR v2 state: */ + unused1:2, + startup_ecn_rounds:2, /* consecutive hi ECN STARTUP rounds */ + loss_in_cycle:1, /* packet loss in this cycle? */ + ecn_in_cycle:1; /* ECN in this cycle? */ + u32 loss_round_delivered; /* scb->tx.delivered ending loss round */ + u32 undo_bw_lo; /* bw_lo before latest losses */ + u32 undo_inflight_lo; /* inflight_lo before latest losses */ + u32 undo_inflight_hi; /* inflight_hi before latest losses */ + u32 bw_latest; /* max delivered bw in last round trip */ + u32 bw_lo; /* lower bound on sending bandwidth */ + u32 bw_hi[2]; /* upper bound of sending bandwidth range*/ + u32 inflight_latest; /* max delivered data in last round trip */ + u32 inflight_lo; /* lower bound of inflight data range */ + u32 inflight_hi; /* upper bound of inflight data range */ + u32 bw_probe_up_cnt; /* packets delivered per inflight_hi incr */ + u32 bw_probe_up_acks; /* packets (S)ACKed since inflight_hi incr */ + u32 probe_wait_us; /* PROBE_DOWN until next clock-driven probe */ + u32 ecn_eligible:1, /* sender can use ECN (RTT, handshake)? */ + ecn_alpha:9, /* EWMA delivered_ce/delivered; 0..256 */ + bw_probe_samples:1, /* rate samples reflect bw probing? */ + prev_probe_too_high:1, /* did last PROBE_UP go too high? */ + stopped_risky_probe:1, /* last PROBE_UP stopped due to risk? */ + rounds_since_probe:8, /* packet-timed rounds since probed bw */ + loss_round_start:1, /* loss_round_delivered round trip? */ + loss_in_round:1, /* loss marked in this round trip? */ + ecn_in_round:1, /* ECN marked in this round trip? */ + ack_phase:3, /* bbr_ack_phase: meaning of ACKs */ + loss_events_in_round:4,/* losses in STARTUP round */ + initialized:1; /* has bbr_init() been called? */ + u32 alpha_last_delivered; /* tp->delivered at alpha update */ + u32 alpha_last_delivered_ce; /* tp->delivered_ce at alpha update */ + + /* Params configurable using setsockopt. Refer to correspoding + * module param for detailed description of params. + */ + struct bbr_params { + u32 high_gain:11, /* max allowed value: 2047 */ + drain_gain:10, /* max allowed value: 1023 */ + cwnd_gain:11; /* max allowed value: 2047 */ + u32 cwnd_min_target:4, /* max allowed value: 15 */ + min_rtt_win_sec:5, /* max allowed value: 31 */ + probe_rtt_mode_ms:9, /* max allowed value: 511 */ + full_bw_cnt:3, /* max allowed value: 7 */ + cwnd_tso_budget:1, /* allowed values: {0, 1} */ + unused3:6, + drain_to_target:1, /* boolean */ + precise_ece_ack:1, /* boolean */ + extra_acked_in_startup:1, /* allowed values: {0, 1} */ + fast_path:1; /* boolean */ + u32 full_bw_thresh:10, /* max allowed value: 1023 */ + startup_cwnd_gain:11, /* max allowed value: 2047 */ + bw_probe_pif_gain:9, /* max allowed value: 511 */ + usage_based_cwnd:1, /* boolean */ + unused2:1; + u16 probe_rtt_win_ms:14, /* max allowed value: 16383 */ + refill_add_inc:2; /* max allowed value: 3 */ + u16 extra_acked_gain:11, /* max allowed value: 2047 */ + extra_acked_win_rtts:5; /* max allowed value: 31*/ + u16 pacing_gain[CYCLE_LEN]; /* max allowed value: 1023 */ + /* Mostly BBR v2 parameters below here: */ + u32 ecn_alpha_gain:8, /* max allowed value: 255 */ + ecn_factor:8, /* max allowed value: 255 */ + ecn_thresh:8, /* max allowed value: 255 */ + beta:8; /* max allowed value: 255 */ + u32 ecn_max_rtt_us:19, /* max allowed value: 524287 */ + bw_probe_reno_gain:9, /* max allowed value: 511 */ + full_loss_cnt:4; /* max allowed value: 15 */ + u32 probe_rtt_cwnd_gain:8, /* max allowed value: 255 */ + inflight_headroom:8, /* max allowed value: 255 */ + loss_thresh:8, /* max allowed value: 255 */ + bw_probe_max_rounds:8; /* max allowed value: 255 */ + u32 bw_probe_rand_rounds:4, /* max allowed value: 15 */ + bw_probe_base_us:26, /* usecs: 0..2^26-1 (67 secs) */ + full_ecn_cnt:2; /* max allowed value: 3 */ + u32 bw_probe_rand_us:26, /* usecs: 0..2^26-1 (67 secs) */ + undo:1, /* boolean */ + tso_rtt_shift:4, /* max allowed value: 15 */ + unused5:1; + u32 ecn_reprobe_gain:9, /* max allowed value: 511 */ + unused1:14, + ecn_alpha_init:9; /* max allowed value: 256 */ + } params; + + struct { + u32 snd_isn; /* Initial sequence number */ + u32 rs_bw; /* last valid rate sample bw */ + u32 target_cwnd; /* target cwnd, based on BDP */ + u8 undo:1, /* Undo even happened but not yet logged */ + unused:7; + char event; /* single-letter event debug codes */ + u16 unused2; + } debug; +}; + +struct bbr_context { + u32 sample_bw; + u32 target_cwnd; + u32 log:1; +}; + +/* Window length of min_rtt filter (in sec). Max allowed value is 31 (0x1F) */ +static u32 bbr_min_rtt_win_sec = 10; +/* Minimum time (in ms) spent at bbr_cwnd_min_target in BBR_PROBE_RTT mode. + * Max allowed value is 511 (0x1FF). + */ +static u32 bbr_probe_rtt_mode_ms = 200; +/* Window length of probe_rtt_min_us filter (in ms), and consequently the + * typical interval between PROBE_RTT mode entries. + * Note that bbr_probe_rtt_win_ms must be <= bbr_min_rtt_win_sec * MSEC_PER_SEC + */ +static u32 bbr_probe_rtt_win_ms = 5000; +/* Skip TSO below the following bandwidth (bits/sec): */ +static int bbr_min_tso_rate = 1200000; + +/* Use min_rtt to help adapt TSO burst size, with smaller min_rtt resulting + * in bigger TSO bursts. By default we cut the RTT-based allowance in half + * for every 2^9 usec (aka 512 us) of RTT, so that the RTT-based allowance + * is below 1500 bytes after 6 * ~500 usec = 3ms. + */ +static u32 bbr_tso_rtt_shift = 9; /* halve allowance per 2^9 usecs, 512us */ + +/* Select cwnd TSO budget approach: + * 0: padding + * 1: flooring + */ +static uint bbr_cwnd_tso_budget = 1; + +/* Pace at ~1% below estimated bw, on average, to reduce queue at bottleneck. + * In order to help drive the network toward lower queues and low latency while + * maintaining high utilization, the average pacing rate aims to be slightly + * lower than the estimated bandwidth. This is an important aspect of the + * design. + */ +static const int bbr_pacing_margin_percent = 1; + +/* We use a high_gain value of 2/ln(2) because it's the smallest pacing gain + * that will allow a smoothly increasing pacing rate that will double each RTT + * and send the same number of packets per RTT that an un-paced, slow-starting + * Reno or CUBIC flow would. Max allowed value is 2047 (0x7FF). + */ +static int bbr_high_gain = BBR_UNIT * 2885 / 1000 + 1; +/* The gain for deriving startup cwnd. Max allowed value is 2047 (0x7FF). */ +static int bbr_startup_cwnd_gain = BBR_UNIT * 2885 / 1000 + 1; +/* The pacing gain of 1/high_gain in BBR_DRAIN is calculated to typically drain + * the queue created in BBR_STARTUP in a single round. Max allowed value + * is 1023 (0x3FF). + */ +static int bbr_drain_gain = BBR_UNIT * 1000 / 2885; +/* The gain for deriving steady-state cwnd tolerates delayed/stretched ACKs. + * Max allowed value is 2047 (0x7FF). + */ +static int bbr_cwnd_gain = BBR_UNIT * 2; +/* The pacing_gain values for the PROBE_BW gain cycle, to discover/share bw. + * Max allowed value for each element is 1023 (0x3FF). + */ +enum bbr_pacing_gain_phase { + BBR_BW_PROBE_UP = 0, /* push up inflight to probe for bw/vol */ + BBR_BW_PROBE_DOWN = 1, /* drain excess inflight from the queue */ + BBR_BW_PROBE_CRUISE = 2, /* use pipe, w/ headroom in queue/pipe */ + BBR_BW_PROBE_REFILL = 3, /* v2: refill the pipe again to 100% */ +}; +static int bbr_pacing_gain[] = { + BBR_UNIT * 5 / 4, /* probe for more available bw */ + BBR_UNIT * 3 / 4, /* drain queue and/or yield bw to other flows */ + BBR_UNIT, BBR_UNIT, BBR_UNIT, /* cruise at 1.0*bw to utilize pipe, */ + BBR_UNIT, BBR_UNIT, BBR_UNIT /* without creating excess queue... */ +}; + +/* Try to keep at least this many packets in flight, if things go smoothly. For + * smooth functioning, a sliding window protocol ACKing every other packet + * needs at least 4 packets in flight. Max allowed value is 15 (0xF). + */ +static u32 bbr_cwnd_min_target = 4; + +/* Cwnd to BDP proportion in PROBE_RTT mode scaled by BBR_UNIT. Default: 50%. + * Use 0 to disable. Max allowed value is 255. + */ +static u32 bbr_probe_rtt_cwnd_gain = BBR_UNIT * 1 / 2; + +/* To estimate if BBR_STARTUP mode (i.e. high_gain) has filled pipe... */ +/* If bw has increased significantly (1.25x), there may be more bw available. + * Max allowed value is 1023 (0x3FF). + */ +static u32 bbr_full_bw_thresh = BBR_UNIT * 5 / 4; +/* But after 3 rounds w/o significant bw growth, estimate pipe is full. + * Max allowed value is 7 (0x7). + */ +static u32 bbr_full_bw_cnt = 3; + +static u32 bbr_flags; /* Debugging related stuff */ + +/* Whether to debug using printk. + */ +static bool bbr_debug_with_printk; + +/* Whether to debug using ftrace event tcp:tcp_bbr_event. + * Ignored when bbr_debug_with_printk is set. + */ +static bool bbr_debug_ftrace; + +/* Experiment: each cycle, try to hold sub-unity gain until inflight <= BDP. */ +static bool bbr_drain_to_target = true; /* default: enabled */ + +/* Experiment: Flags to control BBR with ECN behavior. + */ +static bool bbr_precise_ece_ack = true; /* default: enabled */ + +/* The max rwin scaling shift factor is 14 (RFC 1323), so the max sane rwin is + * (2^(16+14) B)/(1024 B/packet) = 1M packets. + */ +static u32 bbr_cwnd_warn_val = 1U << 20; + +static u16 bbr_debug_port_mask; + +/* BBR module parameters. These are module parameters only in Google prod. + * Upstream these are intentionally not module parameters. + */ +static int bbr_pacing_gain_size = CYCLE_LEN; + +/* Gain factor for adding extra_acked to target cwnd: */ +static int bbr_extra_acked_gain = 256; + +/* Window length of extra_acked window. Max allowed val is 31. */ +static u32 bbr_extra_acked_win_rtts = 5; + +/* Max allowed val for ack_epoch_acked, after which sampling epoch is reset */ +static u32 bbr_ack_epoch_acked_reset_thresh = 1U << 20; + +/* Time period for clamping cwnd increment due to ack aggregation */ +static u32 bbr_extra_acked_max_us = 100 * 1000; + +/* Use extra acked in startup ? + * 0: disabled + * 1: use latest extra_acked value from 1-2 rtt in startup + */ +static int bbr_extra_acked_in_startup = 1; /* default: enabled */ + +/* Experiment: don't grow cwnd beyond twice of what we just probed. */ +static bool bbr_usage_based_cwnd; /* default: disabled */ + +/* For lab testing, researchers can enable BBRv2 ECN support with this flag, + * when they know that any ECN marks that the connections experience will be + * DCTCP/L4S-style ECN marks, rather than RFC3168 ECN marks. + * TODO(ncardwell): Production use of the BBRv2 ECN functionality depends on + * negotiation or configuration that is outside the scope of the BBRv2 + * alpha release. + */ +static bool bbr_ecn_enable = false; + +module_param_named(min_tso_rate, bbr_min_tso_rate, int, 0644); +module_param_named(tso_rtt_shift, bbr_tso_rtt_shift, int, 0644); +module_param_named(high_gain, bbr_high_gain, int, 0644); +module_param_named(drain_gain, bbr_drain_gain, int, 0644); +module_param_named(startup_cwnd_gain, bbr_startup_cwnd_gain, int, 0644); +module_param_named(cwnd_gain, bbr_cwnd_gain, int, 0644); +module_param_array_named(pacing_gain, bbr_pacing_gain, int, + &bbr_pacing_gain_size, 0644); +module_param_named(cwnd_min_target, bbr_cwnd_min_target, uint, 0644); +module_param_named(probe_rtt_cwnd_gain, + bbr_probe_rtt_cwnd_gain, uint, 0664); +module_param_named(cwnd_warn_val, bbr_cwnd_warn_val, uint, 0664); +module_param_named(debug_port_mask, bbr_debug_port_mask, ushort, 0644); +module_param_named(flags, bbr_flags, uint, 0644); +module_param_named(debug_ftrace, bbr_debug_ftrace, bool, 0644); +module_param_named(debug_with_printk, bbr_debug_with_printk, bool, 0644); +module_param_named(min_rtt_win_sec, bbr_min_rtt_win_sec, uint, 0644); +module_param_named(probe_rtt_mode_ms, bbr_probe_rtt_mode_ms, uint, 0644); +module_param_named(probe_rtt_win_ms, bbr_probe_rtt_win_ms, uint, 0644); +module_param_named(full_bw_thresh, bbr_full_bw_thresh, uint, 0644); +module_param_named(full_bw_cnt, bbr_full_bw_cnt, uint, 0644); +module_param_named(cwnd_tso_bduget, bbr_cwnd_tso_budget, uint, 0664); +module_param_named(extra_acked_gain, bbr_extra_acked_gain, int, 0664); +module_param_named(extra_acked_win_rtts, + bbr_extra_acked_win_rtts, uint, 0664); +module_param_named(extra_acked_max_us, + bbr_extra_acked_max_us, uint, 0664); +module_param_named(ack_epoch_acked_reset_thresh, + bbr_ack_epoch_acked_reset_thresh, uint, 0664); +module_param_named(drain_to_target, bbr_drain_to_target, bool, 0664); +module_param_named(precise_ece_ack, bbr_precise_ece_ack, bool, 0664); +module_param_named(extra_acked_in_startup, + bbr_extra_acked_in_startup, int, 0664); +module_param_named(usage_based_cwnd, bbr_usage_based_cwnd, bool, 0664); +module_param_named(ecn_enable, bbr_ecn_enable, bool, 0664); + +static void bbr2_exit_probe_rtt(struct sock *sk); +static void bbr2_reset_congestion_signals(struct sock *sk); + +static void bbr_check_probe_rtt_done(struct sock *sk); + +/* Do we estimate that STARTUP filled the pipe? */ +static bool bbr_full_bw_reached(const struct sock *sk) +{ + const struct bbr *bbr = inet_csk_ca(sk); + + return bbr->full_bw_reached; +} + +/* Return the windowed max recent bandwidth sample, in pkts/uS << BW_SCALE. */ +static u32 bbr_max_bw(const struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + return max(bbr->bw_hi[0], bbr->bw_hi[1]); +} + +/* Return the estimated bandwidth of the path, in pkts/uS << BW_SCALE. */ +static u32 bbr_bw(const struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + return min(bbr_max_bw(sk), bbr->bw_lo); +} + +/* Return maximum extra acked in past k-2k round trips, + * where k = bbr_extra_acked_win_rtts. + */ +static u16 bbr_extra_acked(const struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + return max(bbr->extra_acked[0], bbr->extra_acked[1]); +} + +/* Return rate in bytes per second, optionally with a gain. + * The order here is chosen carefully to avoid overflow of u64. This should + * work for input rates of up to 2.9Tbit/sec and gain of 2.89x. + */ +static u64 bbr_rate_bytes_per_sec(struct sock *sk, u64 rate, int gain, + int margin) +{ + unsigned int mss = tcp_sk(sk)->mss_cache; + + rate *= mss; + rate *= gain; + rate >>= BBR_SCALE; + rate *= USEC_PER_SEC / 100 * (100 - margin); + rate >>= BW_SCALE; + rate = max(rate, 1ULL); + return rate; +} + +static u64 bbr_bw_bytes_per_sec(struct sock *sk, u64 rate) +{ + return bbr_rate_bytes_per_sec(sk, rate, BBR_UNIT, 0); +} + +static u64 bbr_rate_kbps(struct sock *sk, u64 rate) +{ + rate = bbr_bw_bytes_per_sec(sk, rate); + rate *= 8; + do_div(rate, 1000); + return rate; +} + +static u32 bbr_tso_segs_goal(struct sock *sk); +static void bbr_debug(struct sock *sk, u32 acked, + const struct rate_sample *rs, struct bbr_context *ctx) +{ + static const char ca_states[] = { + [TCP_CA_Open] = 'O', + [TCP_CA_Disorder] = 'D', + [TCP_CA_CWR] = 'C', + [TCP_CA_Recovery] = 'R', + [TCP_CA_Loss] = 'L', + }; + static const char mode[] = { + 'G', /* Growing - BBR_STARTUP */ + 'D', /* Drain - BBR_DRAIN */ + 'W', /* Window - BBR_PROBE_BW */ + 'M', /* Min RTT - BBR_PROBE_RTT */ + }; + static const char ack_phase[] = { /* bbr_ack_phase strings */ + 'I', /* BBR_ACKS_INIT - 'Init' */ + 'R', /* BBR_ACKS_REFILLING - 'Refilling' */ + 'B', /* BBR_ACKS_PROBE_STARTING - 'Before' */ + 'F', /* BBR_ACKS_PROBE_FEEDBACK - 'Feedback' */ + 'A', /* BBR_ACKS_PROBE_STOPPING - 'After' */ + }; + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + const u32 una = tp->snd_una - bbr->debug.snd_isn; + const u32 fack = tcp_highest_sack_seq(tp); + const u16 dport = ntohs(inet_sk(sk)->inet_dport); + bool is_port_match = (bbr_debug_port_mask && + ((dport & bbr_debug_port_mask) == 0)); + char debugmsg[320]; + + if (sk->sk_state == TCP_SYN_SENT) + return; /* no bbr_init() yet if SYN retransmit -> CA_Loss */ + + if (!tp->snd_cwnd || tp->snd_cwnd > bbr_cwnd_warn_val) { + char addr[INET6_ADDRSTRLEN + 10] = { 0 }; + + if (sk->sk_family == AF_INET) + snprintf(addr, sizeof(addr), "%pI4:%u", + &inet_sk(sk)->inet_daddr, dport); + else if (sk->sk_family == AF_INET6) + snprintf(addr, sizeof(addr), "%pI6:%u", + &sk->sk_v6_daddr, dport); + + WARN_ONCE(1, + "BBR %s cwnd alert: %u " + "snd_una: %u ca: %d pacing_gain: %u cwnd_gain: %u " + "bw: %u rtt: %u min_rtt: %u " + "acked: %u tso_segs: %u " + "bw: %d %ld %d pif: %u\n", + addr, tp->snd_cwnd, + una, inet_csk(sk)->icsk_ca_state, + bbr->pacing_gain, bbr->cwnd_gain, + bbr_max_bw(sk), (tp->srtt_us >> 3), bbr->min_rtt_us, + acked, bbr_tso_segs_goal(sk), + rs->delivered, rs->interval_us, rs->is_retrans, + tcp_packets_in_flight(tp)); + } + + if (likely(!bbr_debug_with_printk && !bbr_debug_ftrace)) + return; + + if (!sock_flag(sk, SOCK_DBG) && !is_port_match) + return; + + if (!ctx->log && !tp->app_limited && !(bbr_flags & FLAG_DEBUG_VERBOSE)) + return; + + if (ipv4_is_loopback(inet_sk(sk)->inet_daddr) && + !(bbr_flags & FLAG_DEBUG_LOOPBACK)) + return; + + snprintf(debugmsg, sizeof(debugmsg) - 1, + "BBR %pI4:%-5u %5u,%03u:%-7u %c " + "%c %2u br %2u cr %2d rtt %5ld d %2d i %5ld mrtt %d %cbw %llu " + "bw %llu lb %llu ib %llu qb %llu " + "a %u if %2u %c %c dl %u l %u al %u # %u t %u %c %c " + "lr %d er %d ea %d bwl %lld il %d ih %d c %d " + "v %d %c %u %c %s\n", + &inet_sk(sk)->inet_daddr, dport, + una / 1000, una % 1000, fack - tp->snd_una, + ca_states[inet_csk(sk)->icsk_ca_state], + bbr->debug.undo ? '@' : mode[bbr->mode], + tp->snd_cwnd, + bbr_extra_acked(sk), /* br (legacy): extra_acked */ + rs->tx_in_flight, /* cr (legacy): tx_inflight */ + rs->rtt_us, + rs->delivered, + rs->interval_us, + bbr->min_rtt_us, + rs->is_app_limited ? '_' : 'l', + bbr_rate_kbps(sk, ctx->sample_bw), /* lbw: latest sample bw */ + bbr_rate_kbps(sk, bbr_max_bw(sk)), /* bw: max bw */ + 0ULL, /* lb: [obsolete] */ + 0ULL, /* ib: [obsolete] */ + (u64)sk->sk_pacing_rate * 8 / 1000, + acked, + tcp_packets_in_flight(tp), + rs->is_ack_delayed ? 'd' : '.', + bbr->round_start ? '*' : '.', + tp->delivered, tp->lost, + tp->app_limited, + 0, /* #: [obsolete] */ + ctx->target_cwnd, + tp->reord_seen ? 'r' : '.', /* r: reordering seen? */ + ca_states[bbr->prev_ca_state], + (rs->lost + rs->delivered) > 0 ? + (1000 * rs->lost / + (rs->lost + rs->delivered)) : 0, /* lr: loss rate x1000 */ + (rs->delivered) > 0 ? + (1000 * rs->delivered_ce / + (rs->delivered)) : 0, /* er: ECN rate x1000 */ + 1000 * bbr->ecn_alpha >> BBR_SCALE, /* ea: ECN alpha x1000 */ + bbr->bw_lo == ~0U ? + -1 : (s64)bbr_rate_kbps(sk, bbr->bw_lo), /* bwl */ + bbr->inflight_lo, /* il */ + bbr->inflight_hi, /* ih */ + bbr->bw_probe_up_cnt, /* c */ + 2, /* v: version */ + bbr->debug.event, + bbr->cycle_idx, + ack_phase[bbr->ack_phase], + bbr->bw_probe_samples ? "Y" : "N"); + debugmsg[sizeof(debugmsg) - 1] = 0; + + /* printk takes a higher precedence. */ + if (bbr_debug_with_printk) + printk(KERN_DEBUG "%s", debugmsg); + + if (unlikely(bbr->debug.undo)) + bbr->debug.undo = 0; +} + +/* Convert a BBR bw and gain factor to a pacing rate in bytes per second. */ +static unsigned long bbr_bw_to_pacing_rate(struct sock *sk, u32 bw, int gain) +{ + u64 rate = bw; + + rate = bbr_rate_bytes_per_sec(sk, rate, gain, + bbr_pacing_margin_percent); + rate = min_t(u64, rate, sk->sk_max_pacing_rate); + return rate; +} + +/* Initialize pacing rate to: high_gain * init_cwnd / RTT. */ +static void bbr_init_pacing_rate_from_rtt(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u64 bw; + u32 rtt_us; + + if (tp->srtt_us) { /* any RTT sample yet? */ + rtt_us = max(tp->srtt_us >> 3, 1U); + bbr->has_seen_rtt = 1; + } else { /* no RTT sample yet */ + rtt_us = USEC_PER_MSEC; /* use nominal default RTT */ + } + bw = (u64)tp->snd_cwnd * BW_UNIT; + do_div(bw, rtt_us); + sk->sk_pacing_rate = bbr_bw_to_pacing_rate(sk, bw, bbr->params.high_gain); +} + +/* Pace using current bw estimate and a gain factor. */ +static void bbr_set_pacing_rate(struct sock *sk, u32 bw, int gain) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + unsigned long rate = bbr_bw_to_pacing_rate(sk, bw, gain); + + if (unlikely(!bbr->has_seen_rtt && tp->srtt_us)) + bbr_init_pacing_rate_from_rtt(sk); + if (bbr_full_bw_reached(sk) || rate > sk->sk_pacing_rate) + sk->sk_pacing_rate = rate; +} + +static u32 bbr_min_tso_segs(struct sock *sk) +{ + return sk->sk_pacing_rate < (bbr_min_tso_rate >> 3) ? 1 : 2; +} + +/* Return the number of segments BBR would like in a TSO/GSO skb, given + * a particular max gso size as a constraint. + */ +static u32 bbr_tso_segs_generic(struct sock *sk, unsigned int mss_now, + u32 gso_max_size) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 segs, r; + u64 bytes; + + /* Budget a TSO/GSO burst size allowance based on bw (pacing_rate). */ + bytes = sk->sk_pacing_rate >> sk->sk_pacing_shift; + + /* Budget a TSO/GSO burst size allowance based on min_rtt. For every + * K = 2^tso_rtt_shift microseconds of min_rtt, halve the burst. + * The min_rtt-based burst allowance is: 64 KBytes / 2^(min_rtt/K) + */ + if (bbr->params.tso_rtt_shift) { + r = bbr->min_rtt_us >> bbr->params.tso_rtt_shift; + if (r < BITS_PER_TYPE(u32)) /* prevent undefined behavior */ + bytes += GSO_MAX_SIZE >> r; + } + + bytes = min_t(u32, bytes, gso_max_size - 1 - MAX_TCP_HEADER); + segs = max_t(u32, bytes / mss_now, bbr_min_tso_segs(sk)); + return segs; +} + +/* Custom tcp_tso_autosize() for BBR, used at transmit time to cap skb size. */ +static u32 bbr_tso_segs(struct sock *sk, unsigned int mss_now) +{ + return bbr_tso_segs_generic(sk, mss_now, sk->sk_gso_max_size); +} + +/* Like bbr_tso_segs(), using mss_cache, ignoring driver's sk_gso_max_size. */ +static u32 bbr_tso_segs_goal(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + + return bbr_tso_segs_generic(sk, tp->mss_cache, GSO_MAX_SIZE); +} + +/* Save "last known good" cwnd so we can restore it after losses or PROBE_RTT */ +static void bbr_save_cwnd(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr->prev_ca_state < TCP_CA_Recovery && bbr->mode != BBR_PROBE_RTT) + bbr->prior_cwnd = tp->snd_cwnd; /* this cwnd is good enough */ + else /* loss recovery or BBR_PROBE_RTT have temporarily cut cwnd */ + bbr->prior_cwnd = max(bbr->prior_cwnd, tp->snd_cwnd); +} + +static void bbr_cwnd_event(struct sock *sk, enum tcp_ca_event event) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + if (event == CA_EVENT_TX_START && tp->app_limited) { + bbr->idle_restart = 1; + bbr->ack_epoch_mstamp = tp->tcp_mstamp; + bbr->ack_epoch_acked = 0; + /* Avoid pointless buffer overflows: pace at est. bw if we don't + * need more speed (we're restarting from idle and app-limited). + */ + if (bbr->mode == BBR_PROBE_BW) + bbr_set_pacing_rate(sk, bbr_bw(sk), BBR_UNIT); + else if (bbr->mode == BBR_PROBE_RTT) + bbr_check_probe_rtt_done(sk); + } else if ((event == CA_EVENT_ECN_IS_CE || + event == CA_EVENT_ECN_NO_CE) && + bbr_ecn_enable && + bbr->params.precise_ece_ack) { + u32 state = bbr->ce_state; + dctcp_ece_ack_update(sk, event, &bbr->prior_rcv_nxt, &state); + bbr->ce_state = state; + if (tp->fast_ack_mode == 2 && event == CA_EVENT_ECN_IS_CE) + tcp_enter_quickack_mode(sk, TCP_MAX_QUICKACKS); + } +} + +/* Calculate bdp based on min RTT and the estimated bottleneck bandwidth: + * + * bdp = ceil(bw * min_rtt * gain) + * + * The key factor, gain, controls the amount of queue. While a small gain + * builds a smaller queue, it becomes more vulnerable to noise in RTT + * measurements (e.g., delayed ACKs or other ACK compression effects). This + * noise may cause BBR to under-estimate the rate. + */ +static u32 bbr_bdp(struct sock *sk, u32 bw, int gain) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 bdp; + u64 w; + + /* If we've never had a valid RTT sample, cap cwnd at the initial + * default. This should only happen when the connection is not using TCP + * timestamps and has retransmitted all of the SYN/SYNACK/data packets + * ACKed so far. In this case, an RTO can cut cwnd to 1, in which + * case we need to slow-start up toward something safe: initial cwnd. + */ + if (unlikely(bbr->min_rtt_us == ~0U)) /* no valid RTT samples yet? */ + return bbr->init_cwnd; /* be safe: cap at initial cwnd */ + + w = (u64)bw * bbr->min_rtt_us; + + /* Apply a gain to the given value, remove the BW_SCALE shift, and + * round the value up to avoid a negative feedback loop. + */ + bdp = (((w * gain) >> BBR_SCALE) + BW_UNIT - 1) / BW_UNIT; + + return bdp; +} + +/* To achieve full performance in high-speed paths, we budget enough cwnd to + * fit full-sized skbs in-flight on both end hosts to fully utilize the path: + * - one skb in sending host Qdisc, + * - one skb in sending host TSO/GSO engine + * - one skb being received by receiver host LRO/GRO/delayed-ACK engine + * Don't worry, at low rates (bbr_min_tso_rate) this won't bloat cwnd because + * in such cases tso_segs_goal is 1. The minimum cwnd is 4 packets, + * which allows 2 outstanding 2-packet sequences, to try to keep pipe + * full even with ACK-every-other-packet delayed ACKs. + */ +static u32 bbr_quantization_budget(struct sock *sk, u32 cwnd) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 tso_segs_goal; + + tso_segs_goal = 3 * bbr_tso_segs_goal(sk); + + /* Allow enough full-sized skbs in flight to utilize end systems. */ + if (bbr->params.cwnd_tso_budget == 1) { + cwnd = max_t(u32, cwnd, tso_segs_goal); + cwnd = max_t(u32, cwnd, bbr->params.cwnd_min_target); + } else { + cwnd += tso_segs_goal; + cwnd = (cwnd + 1) & ~1U; + } + /* Ensure gain cycling gets inflight above BDP even for small BDPs. */ + if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_UP) + cwnd += 2; + + return cwnd; +} + +/* Find inflight based on min RTT and the estimated bottleneck bandwidth. */ +static u32 bbr_inflight(struct sock *sk, u32 bw, int gain) +{ + u32 inflight; + + inflight = bbr_bdp(sk, bw, gain); + inflight = bbr_quantization_budget(sk, inflight); + + return inflight; +} + +/* With pacing at lower layers, there's often less data "in the network" than + * "in flight". With TSQ and departure time pacing at lower layers (e.g. fq), + * we often have several skbs queued in the pacing layer with a pre-scheduled + * earliest departure time (EDT). BBR adapts its pacing rate based on the + * inflight level that it estimates has already been "baked in" by previous + * departure time decisions. We calculate a rough estimate of the number of our + * packets that might be in the network at the earliest departure time for the + * next skb scheduled: + * in_network_at_edt = inflight_at_edt - (EDT - now) * bw + * If we're increasing inflight, then we want to know if the transmit of the + * EDT skb will push inflight above the target, so inflight_at_edt includes + * bbr_tso_segs_goal() from the skb departing at EDT. If decreasing inflight, + * then estimate if inflight will sink too low just before the EDT transmit. + */ +static u32 bbr_packets_in_net_at_edt(struct sock *sk, u32 inflight_now) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u64 now_ns, edt_ns, interval_us; + u32 interval_delivered, inflight_at_edt; + + now_ns = tp->tcp_clock_cache; + edt_ns = max(tp->tcp_wstamp_ns, now_ns); + interval_us = div_u64(edt_ns - now_ns, NSEC_PER_USEC); + interval_delivered = (u64)bbr_bw(sk) * interval_us >> BW_SCALE; + inflight_at_edt = inflight_now; + if (bbr->pacing_gain > BBR_UNIT) /* increasing inflight */ + inflight_at_edt += bbr_tso_segs_goal(sk); /* include EDT skb */ + if (interval_delivered >= inflight_at_edt) + return 0; + return inflight_at_edt - interval_delivered; +} + +/* Find the cwnd increment based on estimate of ack aggregation */ +static u32 bbr_ack_aggregation_cwnd(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 max_aggr_cwnd, aggr_cwnd = 0; + + if (bbr->params.extra_acked_gain && + (bbr_full_bw_reached(sk) || bbr->params.extra_acked_in_startup)) { + max_aggr_cwnd = ((u64)bbr_bw(sk) * bbr_extra_acked_max_us) + / BW_UNIT; + aggr_cwnd = (bbr->params.extra_acked_gain * bbr_extra_acked(sk)) + >> BBR_SCALE; + aggr_cwnd = min(aggr_cwnd, max_aggr_cwnd); + } + + return aggr_cwnd; +} + +/* Returns the cwnd for PROBE_RTT mode. */ +static u32 bbr_probe_rtt_cwnd(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr->params.probe_rtt_cwnd_gain == 0) + return bbr->params.cwnd_min_target; + return max_t(u32, bbr->params.cwnd_min_target, + bbr_bdp(sk, bbr_bw(sk), bbr->params.probe_rtt_cwnd_gain)); +} + +/* Slow-start up toward target cwnd (if bw estimate is growing, or packet loss + * has drawn us down below target), or snap down to target if we're above it. + */ +static void bbr_set_cwnd(struct sock *sk, const struct rate_sample *rs, + u32 acked, u32 bw, int gain, u32 cwnd, + struct bbr_context *ctx) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u32 target_cwnd = 0, prev_cwnd = tp->snd_cwnd, max_probe; + + if (!acked) + goto done; /* no packet fully ACKed; just apply caps */ + + target_cwnd = bbr_bdp(sk, bw, gain); + + /* Increment the cwnd to account for excess ACKed data that seems + * due to aggregation (of data and/or ACKs) visible in the ACK stream. + */ + target_cwnd += bbr_ack_aggregation_cwnd(sk); + target_cwnd = bbr_quantization_budget(sk, target_cwnd); + + /* If we're below target cwnd, slow start cwnd toward target cwnd. */ + bbr->debug.target_cwnd = target_cwnd; + + /* Update cwnd and enable fast path if cwnd reaches target_cwnd. */ + bbr->try_fast_path = 0; + if (bbr_full_bw_reached(sk)) { /* only cut cwnd if we filled the pipe */ + cwnd += acked; + if (cwnd >= target_cwnd) { + cwnd = target_cwnd; + bbr->try_fast_path = 1; + } + } else if (cwnd < target_cwnd || cwnd < 2 * bbr->init_cwnd) { + cwnd += acked; + } else { + bbr->try_fast_path = 1; + } + + /* When growing cwnd, don't grow beyond twice what we just probed. */ + if (bbr->params.usage_based_cwnd) { + max_probe = max(2 * tp->max_packets_out, tp->snd_cwnd); + cwnd = min(cwnd, max_probe); + } + + cwnd = max_t(u32, cwnd, bbr->params.cwnd_min_target); +done: + tp->snd_cwnd = min(cwnd, tp->snd_cwnd_clamp); /* apply global cap */ + if (bbr->mode == BBR_PROBE_RTT) /* drain queue, refresh min_rtt */ + tp->snd_cwnd = min_t(u32, tp->snd_cwnd, bbr_probe_rtt_cwnd(sk)); + + ctx->target_cwnd = target_cwnd; + ctx->log = (tp->snd_cwnd != prev_cwnd); +} + +/* See if we have reached next round trip */ +static void bbr_update_round_start(struct sock *sk, + const struct rate_sample *rs, struct bbr_context *ctx) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + bbr->round_start = 0; + + /* See if we've reached the next RTT */ + if (rs->interval_us > 0 && + !before(rs->prior_delivered, bbr->next_rtt_delivered)) { + bbr->next_rtt_delivered = tp->delivered; + bbr->round_start = 1; + } +} + +/* Calculate the bandwidth based on how fast packets are delivered */ +static void bbr_calculate_bw_sample(struct sock *sk, + const struct rate_sample *rs, struct bbr_context *ctx) +{ + struct bbr *bbr = inet_csk_ca(sk); + u64 bw = 0; + + /* Divide delivered by the interval to find a (lower bound) bottleneck + * bandwidth sample. Delivered is in packets and interval_us in uS and + * ratio will be <<1 for most connections. So delivered is first scaled. + * Round up to allow growth at low rates, even with integer division. + */ + if (rs->interval_us > 0) { + if (WARN_ONCE(rs->delivered < 0, + "negative delivered: %d interval_us: %ld\n", + rs->delivered, rs->interval_us)) + return; + + bw = DIV_ROUND_UP_ULL((u64)rs->delivered * BW_UNIT, rs->interval_us); + } + + ctx->sample_bw = bw; + bbr->debug.rs_bw = bw; +} + +/* Estimates the windowed max degree of ack aggregation. + * This is used to provision extra in-flight data to keep sending during + * inter-ACK silences. + * + * Degree of ack aggregation is estimated as extra data acked beyond expected. + * + * max_extra_acked = "maximum recent excess data ACKed beyond max_bw * interval" + * cwnd += max_extra_acked + * + * Max extra_acked is clamped by cwnd and bw * bbr_extra_acked_max_us (100 ms). + * Max filter is an approximate sliding window of 5-10 (packet timed) round + * trips for non-startup phase, and 1-2 round trips for startup. + */ +static void bbr_update_ack_aggregation(struct sock *sk, + const struct rate_sample *rs) +{ + u32 epoch_us, expected_acked, extra_acked; + struct bbr *bbr = inet_csk_ca(sk); + struct tcp_sock *tp = tcp_sk(sk); + u32 extra_acked_win_rtts_thresh = bbr->params.extra_acked_win_rtts; + + if (!bbr->params.extra_acked_gain || rs->acked_sacked <= 0 || + rs->delivered < 0 || rs->interval_us <= 0) + return; + + if (bbr->round_start) { + bbr->extra_acked_win_rtts = min(0x1F, + bbr->extra_acked_win_rtts + 1); + if (bbr->params.extra_acked_in_startup && + !bbr_full_bw_reached(sk)) + extra_acked_win_rtts_thresh = 1; + if (bbr->extra_acked_win_rtts >= + extra_acked_win_rtts_thresh) { + bbr->extra_acked_win_rtts = 0; + bbr->extra_acked_win_idx = bbr->extra_acked_win_idx ? + 0 : 1; + bbr->extra_acked[bbr->extra_acked_win_idx] = 0; + } + } + + /* Compute how many packets we expected to be delivered over epoch. */ + epoch_us = tcp_stamp_us_delta(tp->delivered_mstamp, + bbr->ack_epoch_mstamp); + expected_acked = ((u64)bbr_bw(sk) * epoch_us) / BW_UNIT; + + /* Reset the aggregation epoch if ACK rate is below expected rate or + * significantly large no. of ack received since epoch (potentially + * quite old epoch). + */ + if (bbr->ack_epoch_acked <= expected_acked || + (bbr->ack_epoch_acked + rs->acked_sacked >= + bbr_ack_epoch_acked_reset_thresh)) { + bbr->ack_epoch_acked = 0; + bbr->ack_epoch_mstamp = tp->delivered_mstamp; + expected_acked = 0; + } + + /* Compute excess data delivered, beyond what was expected. */ + bbr->ack_epoch_acked = min_t(u32, 0xFFFFF, + bbr->ack_epoch_acked + rs->acked_sacked); + extra_acked = bbr->ack_epoch_acked - expected_acked; + extra_acked = min(extra_acked, tp->snd_cwnd); + if (extra_acked > bbr->extra_acked[bbr->extra_acked_win_idx]) + bbr->extra_acked[bbr->extra_acked_win_idx] = extra_acked; +} + +/* Estimate when the pipe is full, using the change in delivery rate: BBR + * estimates that STARTUP filled the pipe if the estimated bw hasn't changed by + * at least bbr_full_bw_thresh (25%) after bbr_full_bw_cnt (3) non-app-limited + * rounds. Why 3 rounds: 1: rwin autotuning grows the rwin, 2: we fill the + * higher rwin, 3: we get higher delivery rate samples. Or transient + * cross-traffic or radio noise can go away. CUBIC Hystart shares a similar + * design goal, but uses delay and inter-ACK spacing instead of bandwidth. + */ +static void bbr_check_full_bw_reached(struct sock *sk, + const struct rate_sample *rs) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 bw_thresh; + + if (bbr_full_bw_reached(sk) || !bbr->round_start || rs->is_app_limited) + return; + + bw_thresh = (u64)bbr->full_bw * bbr->params.full_bw_thresh >> BBR_SCALE; + if (bbr_max_bw(sk) >= bw_thresh) { + bbr->full_bw = bbr_max_bw(sk); + bbr->full_bw_cnt = 0; + return; + } + ++bbr->full_bw_cnt; + bbr->full_bw_reached = bbr->full_bw_cnt >= bbr->params.full_bw_cnt; +} + +/* If pipe is probably full, drain the queue and then enter steady-state. */ +static bool bbr_check_drain(struct sock *sk, const struct rate_sample *rs, + struct bbr_context *ctx) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr->mode == BBR_STARTUP && bbr_full_bw_reached(sk)) { + bbr->mode = BBR_DRAIN; /* drain queue we created */ + tcp_sk(sk)->snd_ssthresh = + bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT); + bbr2_reset_congestion_signals(sk); + } /* fall through to check if in-flight is already small: */ + if (bbr->mode == BBR_DRAIN && + bbr_packets_in_net_at_edt(sk, tcp_packets_in_flight(tcp_sk(sk))) <= + bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT)) + return true; /* exiting DRAIN now */ + return false; +} + +static void bbr_check_probe_rtt_done(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + if (!(bbr->probe_rtt_done_stamp && + after(tcp_jiffies32, bbr->probe_rtt_done_stamp))) + return; + + bbr->probe_rtt_min_stamp = tcp_jiffies32; /* schedule next PROBE_RTT */ + tp->snd_cwnd = max(tp->snd_cwnd, bbr->prior_cwnd); + bbr2_exit_probe_rtt(sk); +} + +/* The goal of PROBE_RTT mode is to have BBR flows cooperatively and + * periodically drain the bottleneck queue, to converge to measure the true + * min_rtt (unloaded propagation delay). This allows the flows to keep queues + * small (reducing queuing delay and packet loss) and achieve fairness among + * BBR flows. + * + * The min_rtt filter window is 10 seconds. When the min_rtt estimate expires, + * we enter PROBE_RTT mode and cap the cwnd at bbr_cwnd_min_target=4 packets. + * After at least bbr_probe_rtt_mode_ms=200ms and at least one packet-timed + * round trip elapsed with that flight size <= 4, we leave PROBE_RTT mode and + * re-enter the previous mode. BBR uses 200ms to approximately bound the + * performance penalty of PROBE_RTT's cwnd capping to roughly 2% (200ms/10s). + * + * Note that flows need only pay 2% if they are busy sending over the last 10 + * seconds. Interactive applications (e.g., Web, RPCs, video chunks) often have + * natural silences or low-rate periods within 10 seconds where the rate is low + * enough for long enough to drain its queue in the bottleneck. We pick up + * these min RTT measurements opportunistically with our min_rtt filter. :-) + */ +static void bbr_update_min_rtt(struct sock *sk, const struct rate_sample *rs) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + bool probe_rtt_expired, min_rtt_expired; + u32 expire; + + /* Track min RTT in probe_rtt_win_ms to time next PROBE_RTT state. */ + expire = bbr->probe_rtt_min_stamp + + msecs_to_jiffies(bbr->params.probe_rtt_win_ms); + probe_rtt_expired = after(tcp_jiffies32, expire); + if (rs->rtt_us >= 0 && + (rs->rtt_us <= bbr->probe_rtt_min_us || + (probe_rtt_expired && !rs->is_ack_delayed))) { + bbr->probe_rtt_min_us = rs->rtt_us; + bbr->probe_rtt_min_stamp = tcp_jiffies32; + } + /* Track min RTT seen in the min_rtt_win_sec filter window: */ + expire = bbr->min_rtt_stamp + bbr->params.min_rtt_win_sec * HZ; + min_rtt_expired = after(tcp_jiffies32, expire); + if (bbr->probe_rtt_min_us <= bbr->min_rtt_us || + min_rtt_expired) { + bbr->min_rtt_us = bbr->probe_rtt_min_us; + bbr->min_rtt_stamp = bbr->probe_rtt_min_stamp; + } + + if (bbr->params.probe_rtt_mode_ms > 0 && probe_rtt_expired && + !bbr->idle_restart && bbr->mode != BBR_PROBE_RTT) { + bbr->mode = BBR_PROBE_RTT; /* dip, drain queue */ + bbr_save_cwnd(sk); /* note cwnd so we can restore it */ + bbr->probe_rtt_done_stamp = 0; + bbr->ack_phase = BBR_ACKS_PROBE_STOPPING; + bbr->next_rtt_delivered = tp->delivered; + } + + if (bbr->mode == BBR_PROBE_RTT) { + /* Ignore low rate samples during this mode. */ + tp->app_limited = + (tp->delivered + tcp_packets_in_flight(tp)) ? : 1; + /* Maintain min packets in flight for max(200 ms, 1 round). */ + if (!bbr->probe_rtt_done_stamp && + tcp_packets_in_flight(tp) <= bbr_probe_rtt_cwnd(sk)) { + bbr->probe_rtt_done_stamp = tcp_jiffies32 + + msecs_to_jiffies(bbr->params.probe_rtt_mode_ms); + bbr->probe_rtt_round_done = 0; + bbr->next_rtt_delivered = tp->delivered; + } else if (bbr->probe_rtt_done_stamp) { + if (bbr->round_start) + bbr->probe_rtt_round_done = 1; + if (bbr->probe_rtt_round_done) + bbr_check_probe_rtt_done(sk); + } + } + /* Restart after idle ends only once we process a new S/ACK for data */ + if (rs->delivered > 0) + bbr->idle_restart = 0; +} + +static void bbr_update_gains(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + switch (bbr->mode) { + case BBR_STARTUP: + bbr->pacing_gain = bbr->params.high_gain; + bbr->cwnd_gain = bbr->params.startup_cwnd_gain; + break; + case BBR_DRAIN: + bbr->pacing_gain = bbr->params.drain_gain; /* slow, to drain */ + bbr->cwnd_gain = bbr->params.startup_cwnd_gain; /* keep cwnd */ + break; + case BBR_PROBE_BW: + bbr->pacing_gain = bbr->params.pacing_gain[bbr->cycle_idx]; + bbr->cwnd_gain = bbr->params.cwnd_gain; + break; + case BBR_PROBE_RTT: + bbr->pacing_gain = BBR_UNIT; + bbr->cwnd_gain = BBR_UNIT; + break; + default: + WARN_ONCE(1, "BBR bad mode: %u\n", bbr->mode); + break; + } +} + +static void bbr_init(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + int i; + + WARN_ON_ONCE(tp->snd_cwnd >= bbr_cwnd_warn_val); + + bbr->initialized = 1; + bbr->params.high_gain = min(0x7FF, bbr_high_gain); + bbr->params.drain_gain = min(0x3FF, bbr_drain_gain); + bbr->params.startup_cwnd_gain = min(0x7FF, bbr_startup_cwnd_gain); + bbr->params.cwnd_gain = min(0x7FF, bbr_cwnd_gain); + bbr->params.cwnd_tso_budget = min(0x1U, bbr_cwnd_tso_budget); + bbr->params.cwnd_min_target = min(0xFU, bbr_cwnd_min_target); + bbr->params.min_rtt_win_sec = min(0x1FU, bbr_min_rtt_win_sec); + bbr->params.probe_rtt_mode_ms = min(0x1FFU, bbr_probe_rtt_mode_ms); + bbr->params.full_bw_cnt = min(0x7U, bbr_full_bw_cnt); + bbr->params.full_bw_thresh = min(0x3FFU, bbr_full_bw_thresh); + bbr->params.extra_acked_gain = min(0x7FF, bbr_extra_acked_gain); + bbr->params.extra_acked_win_rtts = min(0x1FU, bbr_extra_acked_win_rtts); + bbr->params.drain_to_target = bbr_drain_to_target ? 1 : 0; + bbr->params.precise_ece_ack = bbr_precise_ece_ack ? 1 : 0; + bbr->params.extra_acked_in_startup = bbr_extra_acked_in_startup ? 1 : 0; + bbr->params.probe_rtt_cwnd_gain = min(0xFFU, bbr_probe_rtt_cwnd_gain); + bbr->params.probe_rtt_win_ms = + min(0x3FFFU, + min_t(u32, bbr_probe_rtt_win_ms, + bbr->params.min_rtt_win_sec * MSEC_PER_SEC)); + for (i = 0; i < CYCLE_LEN; i++) + bbr->params.pacing_gain[i] = min(0x3FF, bbr_pacing_gain[i]); + bbr->params.usage_based_cwnd = bbr_usage_based_cwnd ? 1 : 0; + bbr->params.tso_rtt_shift = min(0xFU, bbr_tso_rtt_shift); + + bbr->debug.snd_isn = tp->snd_una; + bbr->debug.target_cwnd = 0; + bbr->debug.undo = 0; + + bbr->init_cwnd = min(0x7FU, tp->snd_cwnd); + bbr->prior_cwnd = tp->prior_cwnd; + tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; + bbr->next_rtt_delivered = 0; + bbr->prev_ca_state = TCP_CA_Open; + bbr->packet_conservation = 0; + + bbr->probe_rtt_done_stamp = 0; + bbr->probe_rtt_round_done = 0; + bbr->probe_rtt_min_us = tcp_min_rtt(tp); + bbr->probe_rtt_min_stamp = tcp_jiffies32; + bbr->min_rtt_us = tcp_min_rtt(tp); + bbr->min_rtt_stamp = tcp_jiffies32; + + bbr->has_seen_rtt = 0; + bbr_init_pacing_rate_from_rtt(sk); + + bbr->round_start = 0; + bbr->idle_restart = 0; + bbr->full_bw_reached = 0; + bbr->full_bw = 0; + bbr->full_bw_cnt = 0; + bbr->cycle_mstamp = 0; + bbr->cycle_idx = 0; + bbr->mode = BBR_STARTUP; + bbr->debug.rs_bw = 0; + + bbr->ack_epoch_mstamp = tp->tcp_mstamp; + bbr->ack_epoch_acked = 0; + bbr->extra_acked_win_rtts = 0; + bbr->extra_acked_win_idx = 0; + bbr->extra_acked[0] = 0; + bbr->extra_acked[1] = 0; + + bbr->ce_state = 0; + bbr->prior_rcv_nxt = tp->rcv_nxt; + bbr->try_fast_path = 0; + + cmpxchg(&sk->sk_pacing_status, SK_PACING_NONE, SK_PACING_NEEDED); +} + +static u32 bbr_sndbuf_expand(struct sock *sk) +{ + /* Provision 3 * cwnd since BBR may slow-start even during recovery. */ + return 3; +} + +/* __________________________________________________________________________ + * + * Functions new to BBR v2 ("bbr") congestion control are below here. + * __________________________________________________________________________ + */ + +/* Incorporate a new bw sample into the current window of our max filter. */ +static void bbr2_take_bw_hi_sample(struct sock *sk, u32 bw) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr->bw_hi[1] = max(bw, bbr->bw_hi[1]); +} + +/* Keep max of last 1-2 cycles. Each PROBE_BW cycle, flip filter window. */ +static void bbr2_advance_bw_hi_filter(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (!bbr->bw_hi[1]) + return; /* no samples in this window; remember old window */ + bbr->bw_hi[0] = bbr->bw_hi[1]; + bbr->bw_hi[1] = 0; +} + +/* How much do we want in flight? Our BDP, unless congestion cut cwnd. */ +static u32 bbr2_target_inflight(struct sock *sk) +{ + u32 bdp = bbr_inflight(sk, bbr_bw(sk), BBR_UNIT); + + return min(bdp, tcp_sk(sk)->snd_cwnd); +} + +static bool bbr2_is_probing_bandwidth(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + return (bbr->mode == BBR_STARTUP) || + (bbr->mode == BBR_PROBE_BW && + (bbr->cycle_idx == BBR_BW_PROBE_REFILL || + bbr->cycle_idx == BBR_BW_PROBE_UP)); +} + +/* Has the given amount of time elapsed since we marked the phase start? */ +static bool bbr2_has_elapsed_in_phase(const struct sock *sk, u32 interval_us) +{ + const struct tcp_sock *tp = tcp_sk(sk); + const struct bbr *bbr = inet_csk_ca(sk); + + return tcp_stamp_us_delta(tp->tcp_mstamp, + bbr->cycle_mstamp + interval_us) > 0; +} + +static void bbr2_handle_queue_too_high_in_startup(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr->full_bw_reached = 1; + bbr->inflight_hi = bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT); +} + +/* Exit STARTUP upon N consecutive rounds with ECN mark rate > ecn_thresh. */ +static void bbr2_check_ecn_too_high_in_startup(struct sock *sk, u32 ce_ratio) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr_full_bw_reached(sk) || !bbr->ecn_eligible || + !bbr->params.full_ecn_cnt || !bbr->params.ecn_thresh) + return; + + if (ce_ratio >= bbr->params.ecn_thresh) + bbr->startup_ecn_rounds++; + else + bbr->startup_ecn_rounds = 0; + + if (bbr->startup_ecn_rounds >= bbr->params.full_ecn_cnt) { + bbr->debug.event = 'E'; /* ECN caused STARTUP exit */ + bbr2_handle_queue_too_high_in_startup(sk); + return; + } +} + +static void bbr2_update_ecn_alpha(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + s32 delivered, delivered_ce; + u64 alpha, ce_ratio; + u32 gain; + + if (bbr->params.ecn_factor == 0) + return; + + delivered = tp->delivered - bbr->alpha_last_delivered; + delivered_ce = tp->delivered_ce - bbr->alpha_last_delivered_ce; + + if (delivered == 0 || /* avoid divide by zero */ + WARN_ON_ONCE(delivered < 0 || delivered_ce < 0)) /* backwards? */ + return; + + /* See if we should use ECN sender logic for this connection. */ + if (!bbr->ecn_eligible && bbr_ecn_enable && + (bbr->min_rtt_us <= bbr->params.ecn_max_rtt_us || + !bbr->params.ecn_max_rtt_us)) + bbr->ecn_eligible = 1; + + ce_ratio = (u64)delivered_ce << BBR_SCALE; + do_div(ce_ratio, delivered); + gain = bbr->params.ecn_alpha_gain; + alpha = ((BBR_UNIT - gain) * bbr->ecn_alpha) >> BBR_SCALE; + alpha += (gain * ce_ratio) >> BBR_SCALE; + bbr->ecn_alpha = min_t(u32, alpha, BBR_UNIT); + + bbr->alpha_last_delivered = tp->delivered; + bbr->alpha_last_delivered_ce = tp->delivered_ce; + + bbr2_check_ecn_too_high_in_startup(sk, ce_ratio); +} + +/* Each round trip of BBR_BW_PROBE_UP, double volume of probing data. */ +static void bbr2_raise_inflight_hi_slope(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u32 growth_this_round, cnt; + + /* Calculate "slope": packets S/Acked per inflight_hi increment. */ + growth_this_round = 1 << bbr->bw_probe_up_rounds; + bbr->bw_probe_up_rounds = min(bbr->bw_probe_up_rounds + 1, 30); + cnt = tp->snd_cwnd / growth_this_round; + cnt = max(cnt, 1U); + bbr->bw_probe_up_cnt = cnt; + bbr->debug.event = 'G'; /* Grow inflight_hi slope */ +} + +/* In BBR_BW_PROBE_UP, not seeing high loss/ECN/queue, so raise inflight_hi. */ +static void bbr2_probe_inflight_hi_upward(struct sock *sk, + const struct rate_sample *rs) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u32 delta; + + if (!tp->is_cwnd_limited || tp->snd_cwnd < bbr->inflight_hi) { + bbr->bw_probe_up_acks = 0; /* don't accmulate unused credits */ + return; /* not fully using inflight_hi, so don't grow it */ + } + + /* For each bw_probe_up_cnt packets ACKed, increase inflight_hi by 1. */ + bbr->bw_probe_up_acks += rs->acked_sacked; + if (bbr->bw_probe_up_acks >= bbr->bw_probe_up_cnt) { + delta = bbr->bw_probe_up_acks / bbr->bw_probe_up_cnt; + bbr->bw_probe_up_acks -= delta * bbr->bw_probe_up_cnt; + bbr->inflight_hi += delta; + bbr->debug.event = 'I'; /* Increment inflight_hi */ + } + + if (bbr->round_start) + bbr2_raise_inflight_hi_slope(sk); +} + +/* Does loss/ECN rate for this sample say inflight is "too high"? + * This is used by both the bbr_check_loss_too_high_in_startup() function, + * which can be used in either v1 or v2, and the PROBE_UP phase of v2, which + * uses it to notice when loss/ECN rates suggest inflight is too high. + */ +static bool bbr2_is_inflight_too_high(const struct sock *sk, + const struct rate_sample *rs) +{ + const struct bbr *bbr = inet_csk_ca(sk); + u32 loss_thresh, ecn_thresh; + + if (rs->lost > 0 && rs->tx_in_flight) { + loss_thresh = (u64)rs->tx_in_flight * bbr->params.loss_thresh >> + BBR_SCALE; + if (rs->lost > loss_thresh) + return true; + } + + if (rs->delivered_ce > 0 && rs->delivered > 0 && + bbr->ecn_eligible && bbr->params.ecn_thresh) { + ecn_thresh = (u64)rs->delivered * bbr->params.ecn_thresh >> + BBR_SCALE; + if (rs->delivered_ce >= ecn_thresh) + return true; + } + + return false; +} + +/* Calculate the tx_in_flight level that corresponded to excessive loss. + * We find "lost_prefix" segs of the skb where loss rate went too high, + * by solving for "lost_prefix" in the following equation: + * lost / inflight >= loss_thresh + * (lost_prev + lost_prefix) / (inflight_prev + lost_prefix) >= loss_thresh + * Then we take that equation, convert it to fixed point, and + * round up to the nearest packet. + */ +static u32 bbr2_inflight_hi_from_lost_skb(const struct sock *sk, + const struct rate_sample *rs, + const struct sk_buff *skb) +{ + const struct bbr *bbr = inet_csk_ca(sk); + u32 loss_thresh = bbr->params.loss_thresh; + u32 pcount, divisor, inflight_hi; + s32 inflight_prev, lost_prev; + u64 loss_budget, lost_prefix; + + pcount = tcp_skb_pcount(skb); + + /* How much data was in flight before this skb? */ + inflight_prev = rs->tx_in_flight - pcount; + if (WARN_ONCE(inflight_prev < 0, + "tx_in_flight: %u pcount: %u reneg: %u", + rs->tx_in_flight, pcount, tcp_sk(sk)->is_sack_reneg)) + return ~0U; + + /* How much inflight data was marked lost before this skb? */ + lost_prev = rs->lost - pcount; + if (WARN_ON_ONCE(lost_prev < 0)) + return ~0U; + + /* At what prefix of this lost skb did losss rate exceed loss_thresh? */ + loss_budget = (u64)inflight_prev * loss_thresh + BBR_UNIT - 1; + loss_budget >>= BBR_SCALE; + if (lost_prev >= loss_budget) { + lost_prefix = 0; /* previous losses crossed loss_thresh */ + } else { + lost_prefix = loss_budget - lost_prev; + lost_prefix <<= BBR_SCALE; + divisor = BBR_UNIT - loss_thresh; + if (WARN_ON_ONCE(!divisor)) /* loss_thresh is 8 bits */ + return ~0U; + do_div(lost_prefix, divisor); + } + + inflight_hi = inflight_prev + lost_prefix; + return inflight_hi; +} + +/* If loss/ECN rates during probing indicated we may have overfilled a + * buffer, return an operating point that tries to leave unutilized headroom in + * the path for other flows, for fairness convergence and lower RTTs and loss. + */ +static u32 bbr2_inflight_with_headroom(const struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 headroom, headroom_fraction; + + if (bbr->inflight_hi == ~0U) + return ~0U; + + headroom_fraction = bbr->params.inflight_headroom; + headroom = ((u64)bbr->inflight_hi * headroom_fraction) >> BBR_SCALE; + headroom = max(headroom, 1U); + return max_t(s32, bbr->inflight_hi - headroom, + bbr->params.cwnd_min_target); +} + +/* Bound cwnd to a sensible level, based on our current probing state + * machine phase and model of a good inflight level (inflight_lo, inflight_hi). + */ +static void bbr2_bound_cwnd_for_inflight_model(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u32 cap; + + /* tcp_rcv_synsent_state_process() currently calls tcp_ack() + * and thus cong_control() without first initializing us(!). + */ + if (!bbr->initialized) + return; + + cap = ~0U; + if (bbr->mode == BBR_PROBE_BW && + bbr->cycle_idx != BBR_BW_PROBE_CRUISE) { + /* Probe to see if more packets fit in the path. */ + cap = bbr->inflight_hi; + } else { + if (bbr->mode == BBR_PROBE_RTT || + (bbr->mode == BBR_PROBE_BW && + bbr->cycle_idx == BBR_BW_PROBE_CRUISE)) + cap = bbr2_inflight_with_headroom(sk); + } + /* Adapt to any loss/ECN since our last bw probe. */ + cap = min(cap, bbr->inflight_lo); + + cap = max_t(u32, cap, bbr->params.cwnd_min_target); + tp->snd_cwnd = min(cap, tp->snd_cwnd); +} + +/* Estimate a short-term lower bound on the capacity available now, based + * on measurements of the current delivery process and recent history. When we + * are seeing loss/ECN at times when we are not probing bw, then conservatively + * move toward flow balance by multiplicatively cutting our short-term + * estimated safe rate and volume of data (bw_lo and inflight_lo). We use a + * multiplicative decrease in order to converge to a lower capacity in time + * logarithmic in the magnitude of the decrease. + * + * However, we do not cut our short-term estimates lower than the current rate + * and volume of delivered data from this round trip, since from the current + * delivery process we can estimate the measured capacity available now. + * + * Anything faster than that approach would knowingly risk high loss, which can + * cause low bw for Reno/CUBIC and high loss recovery latency for + * request/response flows using any congestion control. + */ +static void bbr2_adapt_lower_bounds(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u32 ecn_cut, ecn_inflight_lo, beta; + + /* We only use lower-bound estimates when not probing bw. + * When probing we need to push inflight higher to probe bw. + */ + if (bbr2_is_probing_bandwidth(sk)) + return; + + /* ECN response. */ + if (bbr->ecn_in_round && bbr->ecn_eligible && bbr->params.ecn_factor) { + /* Reduce inflight to (1 - alpha*ecn_factor). */ + ecn_cut = (BBR_UNIT - + ((bbr->ecn_alpha * bbr->params.ecn_factor) >> + BBR_SCALE)); + if (bbr->inflight_lo == ~0U) + bbr->inflight_lo = tp->snd_cwnd; + ecn_inflight_lo = (u64)bbr->inflight_lo * ecn_cut >> BBR_SCALE; + } else { + ecn_inflight_lo = ~0U; + } + + /* Loss response. */ + if (bbr->loss_in_round) { + /* Reduce bw and inflight to (1 - beta). */ + if (bbr->bw_lo == ~0U) + bbr->bw_lo = bbr_max_bw(sk); + if (bbr->inflight_lo == ~0U) + bbr->inflight_lo = tp->snd_cwnd; + beta = bbr->params.beta; + bbr->bw_lo = + max_t(u32, bbr->bw_latest, + (u64)bbr->bw_lo * + (BBR_UNIT - beta) >> BBR_SCALE); + bbr->inflight_lo = + max_t(u32, bbr->inflight_latest, + (u64)bbr->inflight_lo * + (BBR_UNIT - beta) >> BBR_SCALE); + } + + /* Adjust to the lower of the levels implied by loss or ECN. */ + bbr->inflight_lo = min(bbr->inflight_lo, ecn_inflight_lo); +} + +/* Reset any short-term lower-bound adaptation to congestion, so that we can + * push our inflight up. + */ +static void bbr2_reset_lower_bounds(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr->bw_lo = ~0U; + bbr->inflight_lo = ~0U; +} + +/* After bw probing (STARTUP/PROBE_UP), reset signals before entering a state + * machine phase where we adapt our lower bound based on congestion signals. + */ +static void bbr2_reset_congestion_signals(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr->loss_in_round = 0; + bbr->ecn_in_round = 0; + bbr->loss_in_cycle = 0; + bbr->ecn_in_cycle = 0; + bbr->bw_latest = 0; + bbr->inflight_latest = 0; +} + +/* Update (most of) our congestion signals: track the recent rate and volume of + * delivered data, presence of loss, and EWMA degree of ECN marking. + */ +static void bbr2_update_congestion_signals( + struct sock *sk, const struct rate_sample *rs, struct bbr_context *ctx) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u64 bw; + + bbr->loss_round_start = 0; + if (rs->interval_us <= 0 || !rs->acked_sacked) + return; /* Not a valid observation */ + bw = ctx->sample_bw; + + if (!rs->is_app_limited || bw >= bbr_max_bw(sk)) + bbr2_take_bw_hi_sample(sk, bw); + + bbr->loss_in_round |= (rs->losses > 0); + + /* Update rate and volume of delivered data from latest round trip: */ + bbr->bw_latest = max_t(u32, bbr->bw_latest, ctx->sample_bw); + bbr->inflight_latest = max_t(u32, bbr->inflight_latest, rs->delivered); + + if (before(rs->prior_delivered, bbr->loss_round_delivered)) + return; /* skip the per-round-trip updates */ + /* Now do per-round-trip updates. */ + bbr->loss_round_delivered = tp->delivered; /* mark round trip */ + bbr->loss_round_start = 1; + bbr2_adapt_lower_bounds(sk); + + /* Update windowed "latest" (single-round-trip) filters. */ + bbr->loss_in_round = 0; + bbr->ecn_in_round = 0; + bbr->bw_latest = ctx->sample_bw; + bbr->inflight_latest = rs->delivered; +} + +/* Bandwidth probing can cause loss. To help coexistence with loss-based + * congestion control we spread out our probing in a Reno-conscious way. Due to + * the shape of the Reno sawtooth, the time required between loss epochs for an + * idealized Reno flow is a number of round trips that is the BDP of that + * flow. We count packet-timed round trips directly, since measured RTT can + * vary widely, and Reno is driven by packet-timed round trips. + */ +static bool bbr2_is_reno_coexistence_probe_time(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 inflight, rounds, reno_gain, reno_rounds; + + /* Random loss can shave some small percentage off of our inflight + * in each round. To survive this, flows need robust periodic probes. + */ + rounds = bbr->params.bw_probe_max_rounds; + + reno_gain = bbr->params.bw_probe_reno_gain; + if (reno_gain) { + inflight = bbr2_target_inflight(sk); + reno_rounds = ((u64)inflight * reno_gain) >> BBR_SCALE; + rounds = min(rounds, reno_rounds); + } + return bbr->rounds_since_probe >= rounds; +} + +/* How long do we want to wait before probing for bandwidth (and risking + * loss)? We randomize the wait, for better mixing and fairness convergence. + * + * We bound the Reno-coexistence inter-bw-probe time to be 62-63 round trips. + * This is calculated to allow fairness with a 25Mbps, 30ms Reno flow, + * (eg 4K video to a broadband user): + * BDP = 25Mbps * .030sec /(1514bytes) = 61.9 packets + * + * We bound the BBR-native inter-bw-probe wall clock time to be: + * (a) higher than 2 sec: to try to avoid causing loss for a long enough time + * to allow Reno at 30ms to get 4K video bw, the inter-bw-probe time must + * be at least: 25Mbps * .030sec / (1514bytes) * 0.030sec = 1.9secs + * (b) lower than 3 sec: to ensure flows can start probing in a reasonable + * amount of time to discover unutilized bw on human-scale interactive + * time-scales (e.g. perhaps traffic from a web page download that we + * were competing with is now complete). + */ +static void bbr2_pick_probe_wait(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + /* Decide the random round-trip bound for wait until probe: */ + bbr->rounds_since_probe = + prandom_u32_max(bbr->params.bw_probe_rand_rounds); + /* Decide the random wall clock bound for wait until probe: */ + bbr->probe_wait_us = bbr->params.bw_probe_base_us + + prandom_u32_max(bbr->params.bw_probe_rand_us); +} + +static void bbr2_set_cycle_idx(struct sock *sk, int cycle_idx) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr->cycle_idx = cycle_idx; + /* New phase, so need to update cwnd and pacing rate. */ + bbr->try_fast_path = 0; +} + +/* Send at estimated bw to fill the pipe, but not queue. We need this phase + * before PROBE_UP, because as soon as we send faster than the available bw + * we will start building a queue, and if the buffer is shallow we can cause + * loss. If we do not fill the pipe before we cause this loss, our bw_hi and + * inflight_hi estimates will underestimate. + */ +static void bbr2_start_bw_probe_refill(struct sock *sk, u32 bw_probe_up_rounds) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + bbr2_reset_lower_bounds(sk); + if (bbr->inflight_hi != ~0U) + bbr->inflight_hi += bbr->params.refill_add_inc; + bbr->bw_probe_up_rounds = bw_probe_up_rounds; + bbr->bw_probe_up_acks = 0; + bbr->stopped_risky_probe = 0; + bbr->ack_phase = BBR_ACKS_REFILLING; + bbr->next_rtt_delivered = tp->delivered; + bbr2_set_cycle_idx(sk, BBR_BW_PROBE_REFILL); +} + +/* Now probe max deliverable data rate and volume. */ +static void bbr2_start_bw_probe_up(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + bbr->ack_phase = BBR_ACKS_PROBE_STARTING; + bbr->next_rtt_delivered = tp->delivered; + bbr->cycle_mstamp = tp->tcp_mstamp; + bbr2_set_cycle_idx(sk, BBR_BW_PROBE_UP); + bbr2_raise_inflight_hi_slope(sk); +} + +/* Start a new PROBE_BW probing cycle of some wall clock length. Pick a wall + * clock time at which to probe beyond an inflight that we think to be + * safe. This will knowingly risk packet loss, so we want to do this rarely, to + * keep packet loss rates low. Also start a round-trip counter, to probe faster + * if we estimate a Reno flow at our BDP would probe faster. + */ +static void bbr2_start_bw_probe_down(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + bbr2_reset_congestion_signals(sk); + bbr->bw_probe_up_cnt = ~0U; /* not growing inflight_hi any more */ + bbr2_pick_probe_wait(sk); + bbr->cycle_mstamp = tp->tcp_mstamp; /* start wall clock */ + bbr->ack_phase = BBR_ACKS_PROBE_STOPPING; + bbr->next_rtt_delivered = tp->delivered; + bbr2_set_cycle_idx(sk, BBR_BW_PROBE_DOWN); +} + +/* Cruise: maintain what we estimate to be a neutral, conservative + * operating point, without attempting to probe up for bandwidth or down for + * RTT, and only reducing inflight in response to loss/ECN signals. + */ +static void bbr2_start_bw_probe_cruise(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr->inflight_lo != ~0U) + bbr->inflight_lo = min(bbr->inflight_lo, bbr->inflight_hi); + + bbr2_set_cycle_idx(sk, BBR_BW_PROBE_CRUISE); +} + +/* Loss and/or ECN rate is too high while probing. + * Adapt (once per bw probe) by cutting inflight_hi and then restarting cycle. + */ +static void bbr2_handle_inflight_too_high(struct sock *sk, + const struct rate_sample *rs) +{ + struct bbr *bbr = inet_csk_ca(sk); + const u32 beta = bbr->params.beta; + + bbr->prev_probe_too_high = 1; + bbr->bw_probe_samples = 0; /* only react once per probe */ + bbr->debug.event = 'L'; /* Loss/ECN too high */ + /* If we are app-limited then we are not robustly + * probing the max volume of inflight data we think + * might be safe (analogous to how app-limited bw + * samples are not known to be robustly probing bw). + */ + if (!rs->is_app_limited) + bbr->inflight_hi = max_t(u32, rs->tx_in_flight, + (u64)bbr2_target_inflight(sk) * + (BBR_UNIT - beta) >> BBR_SCALE); + if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_UP) + bbr2_start_bw_probe_down(sk); +} + +/* If we're seeing bw and loss samples reflecting our bw probing, adapt + * using the signals we see. If loss or ECN mark rate gets too high, then adapt + * inflight_hi downward. If we're able to push inflight higher without such + * signals, push higher: adapt inflight_hi upward. + */ +static bool bbr2_adapt_upper_bounds(struct sock *sk, + const struct rate_sample *rs) +{ + struct bbr *bbr = inet_csk_ca(sk); + + /* Track when we'll see bw/loss samples resulting from our bw probes. */ + if (bbr->ack_phase == BBR_ACKS_PROBE_STARTING && bbr->round_start) + bbr->ack_phase = BBR_ACKS_PROBE_FEEDBACK; + if (bbr->ack_phase == BBR_ACKS_PROBE_STOPPING && bbr->round_start) { + /* End of samples from bw probing phase. */ + bbr->bw_probe_samples = 0; + bbr->ack_phase = BBR_ACKS_INIT; + /* At this point in the cycle, our current bw sample is also + * our best recent chance at finding the highest available bw + * for this flow. So now is the best time to forget the bw + * samples from the previous cycle, by advancing the window. + */ + if (bbr->mode == BBR_PROBE_BW && !rs->is_app_limited) + bbr2_advance_bw_hi_filter(sk); + /* If we had an inflight_hi, then probed and pushed inflight all + * the way up to hit that inflight_hi without seeing any + * high loss/ECN in all the resulting ACKs from that probing, + * then probe up again, this time letting inflight persist at + * inflight_hi for a round trip, then accelerating beyond. + */ + if (bbr->mode == BBR_PROBE_BW && + bbr->stopped_risky_probe && !bbr->prev_probe_too_high) { + bbr->debug.event = 'R'; /* reprobe */ + bbr2_start_bw_probe_refill(sk, 0); + return true; /* yes, decided state transition */ + } + } + + if (bbr2_is_inflight_too_high(sk, rs)) { + if (bbr->bw_probe_samples) /* sample is from bw probing? */ + bbr2_handle_inflight_too_high(sk, rs); + } else { + /* Loss/ECN rate is declared safe. Adjust upper bound upward. */ + if (bbr->inflight_hi == ~0U) /* no excess queue signals yet? */ + return false; + + /* To be resilient to random loss, we must raise inflight_hi + * if we observe in any phase that a higher level is safe. + */ + if (rs->tx_in_flight > bbr->inflight_hi) { + bbr->inflight_hi = rs->tx_in_flight; + bbr->debug.event = 'U'; /* raise up inflight_hi */ + } + + if (bbr->mode == BBR_PROBE_BW && + bbr->cycle_idx == BBR_BW_PROBE_UP) + bbr2_probe_inflight_hi_upward(sk, rs); + } + + return false; +} + +/* Check if it's time to probe for bandwidth now, and if so, kick it off. */ +static bool bbr2_check_time_to_probe_bw(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 n; + + /* If we seem to be at an operating point where we are not seeing loss + * but we are seeing ECN marks, then when the ECN marks cease we reprobe + * quickly (in case a burst of cross-traffic has ceased and freed up bw, + * or in case we are sharing with multiplicatively probing traffic). + */ + if (bbr->params.ecn_reprobe_gain && bbr->ecn_eligible && + bbr->ecn_in_cycle && !bbr->loss_in_cycle && + inet_csk(sk)->icsk_ca_state == TCP_CA_Open) { + bbr->debug.event = 'A'; /* *A*ll clear to probe *A*gain */ + /* Calculate n so that when bbr2_raise_inflight_hi_slope() + * computes growth_this_round as 2^n it will be roughly the + * desired volume of data (inflight_hi*ecn_reprobe_gain). + */ + n = ilog2((((u64)bbr->inflight_hi * + bbr->params.ecn_reprobe_gain) >> BBR_SCALE)); + bbr2_start_bw_probe_refill(sk, n); + return true; + } + + if (bbr2_has_elapsed_in_phase(sk, bbr->probe_wait_us) || + bbr2_is_reno_coexistence_probe_time(sk)) { + bbr2_start_bw_probe_refill(sk, 0); + return true; + } + return false; +} + +/* Is it time to transition from PROBE_DOWN to PROBE_CRUISE? */ +static bool bbr2_check_time_to_cruise(struct sock *sk, u32 inflight, u32 bw) +{ + struct bbr *bbr = inet_csk_ca(sk); + bool is_under_bdp, is_long_enough; + + /* Always need to pull inflight down to leave headroom in queue. */ + if (inflight > bbr2_inflight_with_headroom(sk)) + return false; + + is_under_bdp = inflight <= bbr_inflight(sk, bw, BBR_UNIT); + if (bbr->params.drain_to_target) + return is_under_bdp; + + is_long_enough = bbr2_has_elapsed_in_phase(sk, bbr->min_rtt_us); + return is_under_bdp || is_long_enough; +} + +/* PROBE_BW state machine: cruise, refill, probe for bw, or drain? */ +static void bbr2_update_cycle_phase(struct sock *sk, + const struct rate_sample *rs) +{ + struct bbr *bbr = inet_csk_ca(sk); + bool is_risky = false, is_queuing = false; + u32 inflight, bw; + + if (!bbr_full_bw_reached(sk)) + return; + + /* In DRAIN, PROBE_BW, or PROBE_RTT, adjust upper bounds. */ + if (bbr2_adapt_upper_bounds(sk, rs)) + return; /* already decided state transition */ + + if (bbr->mode != BBR_PROBE_BW) + return; + + inflight = bbr_packets_in_net_at_edt(sk, rs->prior_in_flight); + bw = bbr_max_bw(sk); + + switch (bbr->cycle_idx) { + /* First we spend most of our time cruising with a pacing_gain of 1.0, + * which paces at the estimated bw, to try to fully use the pipe + * without building queue. If we encounter loss/ECN marks, we adapt + * by slowing down. + */ + case BBR_BW_PROBE_CRUISE: + if (bbr2_check_time_to_probe_bw(sk)) + return; /* already decided state transition */ + break; + + /* After cruising, when it's time to probe, we first "refill": we send + * at the estimated bw to fill the pipe, before probing higher and + * knowingly risking overflowing the bottleneck buffer (causing loss). + */ + case BBR_BW_PROBE_REFILL: + if (bbr->round_start) { + /* After one full round trip of sending in REFILL, we + * start to see bw samples reflecting our REFILL, which + * may be putting too much data in flight. + */ + bbr->bw_probe_samples = 1; + bbr2_start_bw_probe_up(sk); + } + break; + + /* After we refill the pipe, we probe by using a pacing_gain > 1.0, to + * probe for bw. If we have not seen loss/ECN, we try to raise inflight + * to at least pacing_gain*BDP; note that this may take more than + * min_rtt if min_rtt is small (e.g. on a LAN). + * + * We terminate PROBE_UP bandwidth probing upon any of the following: + * + * (1) We've pushed inflight up to hit the inflight_hi target set in the + * most recent previous bw probe phase. Thus we want to start + * draining the queue immediately because it's very likely the most + * recently sent packets will fill the queue and cause drops. + * (checked here) + * (2) We have probed for at least 1*min_rtt_us, and the + * estimated queue is high enough (inflight > 1.25 * estimated_bdp). + * (checked here) + * (3) Loss filter says loss rate is "too high". + * (checked in bbr_is_inflight_too_high()) + * (4) ECN filter says ECN mark rate is "too high". + * (checked in bbr_is_inflight_too_high()) + */ + case BBR_BW_PROBE_UP: + if (bbr->prev_probe_too_high && + inflight >= bbr->inflight_hi) { + bbr->stopped_risky_probe = 1; + is_risky = true; + bbr->debug.event = 'D'; /* D for danger */ + } else if (bbr2_has_elapsed_in_phase(sk, bbr->min_rtt_us) && + inflight >= + bbr_inflight(sk, bw, + bbr->params.bw_probe_pif_gain)) { + is_queuing = true; + bbr->debug.event = 'Q'; /* building Queue */ + } + if (is_risky || is_queuing) { + bbr->prev_probe_too_high = 0; /* no loss/ECN (yet) */ + bbr2_start_bw_probe_down(sk); /* restart w/ down */ + } + break; + + /* After probing in PROBE_UP, we have usually accumulated some data in + * the bottleneck buffer (if bw probing didn't find more bw). We next + * enter PROBE_DOWN to try to drain any excess data from the queue. To + * do this, we use a pacing_gain < 1.0. We hold this pacing gain until + * our inflight is less then that target cruising point, which is the + * minimum of (a) the amount needed to leave headroom, and (b) the + * estimated BDP. Once inflight falls to match the target, we estimate + * the queue is drained; persisting would underutilize the pipe. + */ + case BBR_BW_PROBE_DOWN: + if (bbr2_check_time_to_probe_bw(sk)) + return; /* already decided state transition */ + if (bbr2_check_time_to_cruise(sk, inflight, bw)) + bbr2_start_bw_probe_cruise(sk); + break; + + default: + WARN_ONCE(1, "BBR invalid cycle index %u\n", bbr->cycle_idx); + } +} + +/* Exiting PROBE_RTT, so return to bandwidth probing in STARTUP or PROBE_BW. */ +static void bbr2_exit_probe_rtt(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr2_reset_lower_bounds(sk); + if (bbr_full_bw_reached(sk)) { + bbr->mode = BBR_PROBE_BW; + /* Raising inflight after PROBE_RTT may cause loss, so reset + * the PROBE_BW clock and schedule the next bandwidth probe for + * a friendly and randomized future point in time. + */ + bbr2_start_bw_probe_down(sk); + /* Since we are exiting PROBE_RTT, we know inflight is + * below our estimated BDP, so it is reasonable to cruise. + */ + bbr2_start_bw_probe_cruise(sk); + } else { + bbr->mode = BBR_STARTUP; + } +} + +/* Exit STARTUP based on loss rate > 1% and loss gaps in round >= N. Wait until + * the end of the round in recovery to get a good estimate of how many packets + * have been lost, and how many we need to drain with a low pacing rate. + */ +static void bbr2_check_loss_too_high_in_startup(struct sock *sk, + const struct rate_sample *rs) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr_full_bw_reached(sk)) + return; + + /* For STARTUP exit, check the loss rate at the end of each round trip + * of Recovery episodes in STARTUP. We check the loss rate at the end + * of the round trip to filter out noisy/low loss and have a better + * sense of inflight (extent of loss), so we can drain more accurately. + */ + if (rs->losses && bbr->loss_events_in_round < 0xf) + bbr->loss_events_in_round++; /* update saturating counter */ + if (bbr->params.full_loss_cnt && bbr->loss_round_start && + inet_csk(sk)->icsk_ca_state == TCP_CA_Recovery && + bbr->loss_events_in_round >= bbr->params.full_loss_cnt && + bbr2_is_inflight_too_high(sk, rs)) { + bbr->debug.event = 'P'; /* Packet loss caused STARTUP exit */ + bbr2_handle_queue_too_high_in_startup(sk); + return; + } + if (bbr->loss_round_start) + bbr->loss_events_in_round = 0; +} + +/* If we are done draining, advance into steady state operation in PROBE_BW. */ +static void bbr2_check_drain(struct sock *sk, const struct rate_sample *rs, + struct bbr_context *ctx) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr_check_drain(sk, rs, ctx)) { + bbr->mode = BBR_PROBE_BW; + bbr2_start_bw_probe_down(sk); + } +} + +static void bbr2_update_model(struct sock *sk, const struct rate_sample *rs, + struct bbr_context *ctx) +{ + bbr2_update_congestion_signals(sk, rs, ctx); + bbr_update_ack_aggregation(sk, rs); + bbr2_check_loss_too_high_in_startup(sk, rs); + bbr_check_full_bw_reached(sk, rs); + bbr2_check_drain(sk, rs, ctx); + bbr2_update_cycle_phase(sk, rs); + bbr_update_min_rtt(sk, rs); +} + +/* Fast path for app-limited case. + * + * On each ack, we execute bbr state machine, which primarily consists of: + * 1) update model based on new rate sample, and + * 2) update control based on updated model or state change. + * + * There are certain workload/scenarios, e.g. app-limited case, where + * either we can skip updating model or we can skip update of both model + * as well as control. This provides signifcant softirq cpu savings for + * processing incoming acks. + * + * In case of app-limited, if there is no congestion (loss/ecn) and + * if observed bw sample is less than current estimated bw, then we can + * skip some of the computation in bbr state processing: + * + * - if there is no rtt/mode/phase change: In this case, since all the + * parameters of the network model are constant, we can skip model + * as well control update. + * + * - else we can skip rest of the model update. But we still need to + * update the control to account for the new rtt/mode/phase. + * + * Returns whether we can take fast path or not. + */ +static bool bbr2_fast_path(struct sock *sk, bool *update_model, + const struct rate_sample *rs, struct bbr_context *ctx) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 prev_min_rtt_us, prev_mode; + + if (bbr->params.fast_path && bbr->try_fast_path && + rs->is_app_limited && ctx->sample_bw < bbr_max_bw(sk) && + !bbr->loss_in_round && !bbr->ecn_in_round) { + prev_mode = bbr->mode; + prev_min_rtt_us = bbr->min_rtt_us; + bbr2_check_drain(sk, rs, ctx); + bbr2_update_cycle_phase(sk, rs); + bbr_update_min_rtt(sk, rs); + + if (bbr->mode == prev_mode && + bbr->min_rtt_us == prev_min_rtt_us && + bbr->try_fast_path) + return true; + + /* Skip model update, but control still needs to be updated */ + *update_model = false; + } + return false; +} + +static void bbr2_main(struct sock *sk, const struct rate_sample *rs) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + struct bbr_context ctx = { 0 }; + bool update_model = true; + u32 bw; + + bbr->debug.event = '.'; /* init to default NOP (no event yet) */ + + bbr_update_round_start(sk, rs, &ctx); + if (bbr->round_start) { + bbr->rounds_since_probe = + min_t(s32, bbr->rounds_since_probe + 1, 0xFF); + bbr2_update_ecn_alpha(sk); + } + + bbr->ecn_in_round |= rs->is_ece; + bbr_calculate_bw_sample(sk, rs, &ctx); + + if (bbr2_fast_path(sk, &update_model, rs, &ctx)) + goto out; + + if (update_model) + bbr2_update_model(sk, rs, &ctx); + + bbr_update_gains(sk); + bw = bbr_bw(sk); + bbr_set_pacing_rate(sk, bw, bbr->pacing_gain); + bbr_set_cwnd(sk, rs, rs->acked_sacked, bw, bbr->cwnd_gain, + tp->snd_cwnd, &ctx); + bbr2_bound_cwnd_for_inflight_model(sk); + +out: + bbr->prev_ca_state = inet_csk(sk)->icsk_ca_state; + bbr->loss_in_cycle |= rs->lost > 0; + bbr->ecn_in_cycle |= rs->delivered_ce > 0; + + bbr_debug(sk, rs->acked_sacked, rs, &ctx); +} + +/* Module parameters that are settable by TCP_CONGESTION_PARAMS are declared + * down here, so that the algorithm functions that use the parameters must use + * the per-socket parameters; if they accidentally use the global version + * then there will be a compile error. + * TODO(ncardwell): move all per-socket parameters down to this section. + */ + +/* On losses, scale down inflight and pacing rate by beta scaled by BBR_SCALE. + * No loss response when 0. Max allwed value is 255. + */ +static u32 bbr_beta = BBR_UNIT * 30 / 100; + +/* Gain factor for ECN mark ratio samples, scaled by BBR_SCALE. + * Max allowed value is 255. + */ +static u32 bbr_ecn_alpha_gain = BBR_UNIT * 1 / 16; /* 1/16 = 6.25% */ + +/* The initial value for the ecn_alpha state variable. Default and max + * BBR_UNIT (256), representing 1.0. This allows a flow to respond quickly + * to congestion if the bottleneck is congested when the flow starts up. + */ +static u32 bbr_ecn_alpha_init = BBR_UNIT; /* 1.0, to respond quickly */ + +/* On ECN, cut inflight_lo to (1 - ecn_factor * ecn_alpha) scaled by BBR_SCALE. + * No ECN based bounding when 0. Max allwed value is 255. + */ +static u32 bbr_ecn_factor = BBR_UNIT * 1 / 3; /* 1/3 = 33% */ + +/* Estimate bw probing has gone too far if CE ratio exceeds this threshold. + * Scaled by BBR_SCALE. Disabled when 0. Max allowed is 255. + */ +static u32 bbr_ecn_thresh = BBR_UNIT * 1 / 2; /* 1/2 = 50% */ + +/* Max RTT (in usec) at which to use sender-side ECN logic. + * Disabled when 0 (ECN allowed at any RTT). + * Max allowed for the parameter is 524287 (0x7ffff) us, ~524 ms. + */ +static u32 bbr_ecn_max_rtt_us = 5000; + +/* If non-zero, if in a cycle with no losses but some ECN marks, after ECN + * clears then use a multiplicative increase to quickly reprobe bw by + * starting inflight probing at the given multiple of inflight_hi. + * Default for this experimental knob is 0 (disabled). + * Planned value for experiments: BBR_UNIT * 1 / 2 = 128, representing 0.5. + */ +static u32 bbr_ecn_reprobe_gain; + +/* Estimate bw probing has gone too far if loss rate exceeds this level. */ +static u32 bbr_loss_thresh = BBR_UNIT * 2 / 100; /* 2% loss */ + +/* Exit STARTUP if number of loss marking events in a Recovery round is >= N, + * and loss rate is higher than bbr_loss_thresh. + * Disabled if 0. Max allowed value is 15 (0xF). + */ +static u32 bbr_full_loss_cnt = 8; + +/* Exit STARTUP if number of round trips with ECN mark rate above ecn_thresh + * meets this count. Max allowed value is 3. + */ +static u32 bbr_full_ecn_cnt = 2; + +/* Fraction of unutilized headroom to try to leave in path upon high loss. */ +static u32 bbr_inflight_headroom = BBR_UNIT * 15 / 100; + +/* Multiplier to get target inflight (as multiple of BDP) for PROBE_UP phase. + * Default is 1.25x, as in BBR v1. Max allowed is 511. + */ +static u32 bbr_bw_probe_pif_gain = BBR_UNIT * 5 / 4; + +/* Multiplier to get Reno-style probe epoch duration as: k * BDP round trips. + * If zero, disables this BBR v2 Reno-style BDP-scaled coexistence mechanism. + * Max allowed is 511. + */ +static u32 bbr_bw_probe_reno_gain = BBR_UNIT; + +/* Max number of packet-timed rounds to wait before probing for bandwidth. If + * we want to tolerate 1% random loss per round, and not have this cut our + * inflight too much, we must probe for bw periodically on roughly this scale. + * If low, limits Reno/CUBIC coexistence; if high, limits loss tolerance. + * We aim to be fair with Reno/CUBIC up to a BDP of at least: + * BDP = 25Mbps * .030sec /(1514bytes) = 61.9 packets + */ +static u32 bbr_bw_probe_max_rounds = 63; + +/* Max amount of randomness to inject in round counting for Reno-coexistence. + * Max value is 15. + */ +static u32 bbr_bw_probe_rand_rounds = 2; + +/* Use BBR-native probe time scale starting at this many usec. + * We aim to be fair with Reno/CUBIC up to an inter-loss time epoch of at least: + * BDP*RTT = 25Mbps * .030sec /(1514bytes) * 0.030sec = 1.9 secs + */ +static u32 bbr_bw_probe_base_us = 2 * USEC_PER_SEC; /* 2 secs */ + +/* Use BBR-native probes spread over this many usec: */ +static u32 bbr_bw_probe_rand_us = 1 * USEC_PER_SEC; /* 1 secs */ + +/* Undo the model changes made in loss recovery if recovery was spurious? */ +static bool bbr_undo = true; + +/* Use fast path if app-limited, no loss/ECN, and target cwnd was reached? */ +static bool bbr_fast_path = true; /* default: enabled */ + +/* Use fast ack mode ? */ +static int bbr_fast_ack_mode = 1; /* default: rwnd check off */ + +/* How much to additively increase inflight_hi when entering REFILL? */ +static u32 bbr_refill_add_inc; /* default: disabled */ + +module_param_named(beta, bbr_beta, uint, 0644); +module_param_named(ecn_alpha_gain, bbr_ecn_alpha_gain, uint, 0644); +module_param_named(ecn_alpha_init, bbr_ecn_alpha_init, uint, 0644); +module_param_named(ecn_factor, bbr_ecn_factor, uint, 0644); +module_param_named(ecn_thresh, bbr_ecn_thresh, uint, 0644); +module_param_named(ecn_max_rtt_us, bbr_ecn_max_rtt_us, uint, 0644); +module_param_named(ecn_reprobe_gain, bbr_ecn_reprobe_gain, uint, 0644); +module_param_named(loss_thresh, bbr_loss_thresh, uint, 0664); +module_param_named(full_loss_cnt, bbr_full_loss_cnt, uint, 0664); +module_param_named(full_ecn_cnt, bbr_full_ecn_cnt, uint, 0664); +module_param_named(inflight_headroom, bbr_inflight_headroom, uint, 0664); +module_param_named(bw_probe_pif_gain, bbr_bw_probe_pif_gain, uint, 0664); +module_param_named(bw_probe_reno_gain, bbr_bw_probe_reno_gain, uint, 0664); +module_param_named(bw_probe_max_rounds, bbr_bw_probe_max_rounds, uint, 0664); +module_param_named(bw_probe_rand_rounds, bbr_bw_probe_rand_rounds, uint, 0664); +module_param_named(bw_probe_base_us, bbr_bw_probe_base_us, uint, 0664); +module_param_named(bw_probe_rand_us, bbr_bw_probe_rand_us, uint, 0664); +module_param_named(undo, bbr_undo, bool, 0664); +module_param_named(fast_path, bbr_fast_path, bool, 0664); +module_param_named(fast_ack_mode, bbr_fast_ack_mode, uint, 0664); +module_param_named(refill_add_inc, bbr_refill_add_inc, uint, 0664); + +static void bbr2_init(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + bbr_init(sk); /* run shared init code for v1 and v2 */ + + /* BBR v2 parameters: */ + bbr->params.beta = min_t(u32, 0xFFU, bbr_beta); + bbr->params.ecn_alpha_gain = min_t(u32, 0xFFU, bbr_ecn_alpha_gain); + bbr->params.ecn_alpha_init = min_t(u32, BBR_UNIT, bbr_ecn_alpha_init); + bbr->params.ecn_factor = min_t(u32, 0xFFU, bbr_ecn_factor); + bbr->params.ecn_thresh = min_t(u32, 0xFFU, bbr_ecn_thresh); + bbr->params.ecn_max_rtt_us = min_t(u32, 0x7ffffU, bbr_ecn_max_rtt_us); + bbr->params.ecn_reprobe_gain = min_t(u32, 0x1FF, bbr_ecn_reprobe_gain); + bbr->params.loss_thresh = min_t(u32, 0xFFU, bbr_loss_thresh); + bbr->params.full_loss_cnt = min_t(u32, 0xFU, bbr_full_loss_cnt); + bbr->params.full_ecn_cnt = min_t(u32, 0x3U, bbr_full_ecn_cnt); + bbr->params.inflight_headroom = + min_t(u32, 0xFFU, bbr_inflight_headroom); + bbr->params.bw_probe_pif_gain = + min_t(u32, 0x1FFU, bbr_bw_probe_pif_gain); + bbr->params.bw_probe_reno_gain = + min_t(u32, 0x1FFU, bbr_bw_probe_reno_gain); + bbr->params.bw_probe_max_rounds = + min_t(u32, 0xFFU, bbr_bw_probe_max_rounds); + bbr->params.bw_probe_rand_rounds = + min_t(u32, 0xFU, bbr_bw_probe_rand_rounds); + bbr->params.bw_probe_base_us = + min_t(u32, (1 << 26) - 1, bbr_bw_probe_base_us); + bbr->params.bw_probe_rand_us = + min_t(u32, (1 << 26) - 1, bbr_bw_probe_rand_us); + bbr->params.undo = bbr_undo; + bbr->params.fast_path = bbr_fast_path ? 1 : 0; + bbr->params.refill_add_inc = min_t(u32, 0x3U, bbr_refill_add_inc); + + /* BBR v2 state: */ + bbr->initialized = 1; + /* Start sampling ECN mark rate after first full flight is ACKed: */ + bbr->loss_round_delivered = tp->delivered + 1; + bbr->loss_round_start = 0; + bbr->undo_bw_lo = 0; + bbr->undo_inflight_lo = 0; + bbr->undo_inflight_hi = 0; + bbr->loss_events_in_round = 0; + bbr->startup_ecn_rounds = 0; + bbr2_reset_congestion_signals(sk); + bbr->bw_lo = ~0U; + bbr->bw_hi[0] = 0; + bbr->bw_hi[1] = 0; + bbr->inflight_lo = ~0U; + bbr->inflight_hi = ~0U; + bbr->bw_probe_up_cnt = ~0U; + bbr->bw_probe_up_acks = 0; + bbr->bw_probe_up_rounds = 0; + bbr->probe_wait_us = 0; + bbr->stopped_risky_probe = 0; + bbr->ack_phase = BBR_ACKS_INIT; + bbr->rounds_since_probe = 0; + bbr->bw_probe_samples = 0; + bbr->prev_probe_too_high = 0; + bbr->ecn_eligible = 0; + bbr->ecn_alpha = bbr->params.ecn_alpha_init; + bbr->alpha_last_delivered = 0; + bbr->alpha_last_delivered_ce = 0; + + tp->fast_ack_mode = min_t(u32, 0x2U, bbr_fast_ack_mode); +} + +/* Core TCP stack informs us that the given skb was just marked lost. */ +static void bbr2_skb_marked_lost(struct sock *sk, const struct sk_buff *skb) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + struct tcp_skb_cb *scb = TCP_SKB_CB(skb); + struct rate_sample rs; + + /* Capture "current" data over the full round trip of loss, + * to have a better chance to see the full capacity of the path. + */ + if (!bbr->loss_in_round) /* first loss in this round trip? */ + bbr->loss_round_delivered = tp->delivered; /* set round trip */ + bbr->loss_in_round = 1; + bbr->loss_in_cycle = 1; + + if (!bbr->bw_probe_samples) + return; /* not an skb sent while probing for bandwidth */ + if (unlikely(!scb->tx.delivered_mstamp)) + return; /* skb was SACKed, reneged, marked lost; ignore it */ + /* We are probing for bandwidth. Construct a rate sample that + * estimates what happened in the flight leading up to this lost skb, + * then see if the loss rate went too high, and if so at which packet. + */ + memset(&rs, 0, sizeof(rs)); + rs.tx_in_flight = scb->tx.in_flight; + rs.lost = tp->lost - scb->tx.lost; + rs.is_app_limited = scb->tx.is_app_limited; + if (bbr2_is_inflight_too_high(sk, &rs)) { + rs.tx_in_flight = bbr2_inflight_hi_from_lost_skb(sk, &rs, skb); + bbr2_handle_inflight_too_high(sk, &rs); + } +} + +/* Revert short-term model if current loss recovery event was spurious. */ +static u32 bbr2_undo_cwnd(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + bbr->debug.undo = 1; + bbr->full_bw = 0; /* spurious slow-down; reset full pipe detection */ + bbr->full_bw_cnt = 0; + bbr->loss_in_round = 0; + + if (!bbr->params.undo) + return tp->snd_cwnd; + + /* Revert to cwnd and other state saved before loss episode. */ + bbr->bw_lo = max(bbr->bw_lo, bbr->undo_bw_lo); + bbr->inflight_lo = max(bbr->inflight_lo, bbr->undo_inflight_lo); + bbr->inflight_hi = max(bbr->inflight_hi, bbr->undo_inflight_hi); + return bbr->prior_cwnd; +} + +/* Entering loss recovery, so save state for when we undo recovery. */ +static u32 bbr2_ssthresh(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr_save_cwnd(sk); + /* For undo, save state that adapts based on loss signal. */ + bbr->undo_bw_lo = bbr->bw_lo; + bbr->undo_inflight_lo = bbr->inflight_lo; + bbr->undo_inflight_hi = bbr->inflight_hi; + return tcp_sk(sk)->snd_ssthresh; +} + +static enum tcp_bbr2_phase bbr2_get_phase(struct bbr *bbr) +{ + switch (bbr->mode) { + case BBR_STARTUP: + return BBR2_PHASE_STARTUP; + case BBR_DRAIN: + return BBR2_PHASE_DRAIN; + case BBR_PROBE_BW: + break; + case BBR_PROBE_RTT: + return BBR2_PHASE_PROBE_RTT; + default: + return BBR2_PHASE_INVALID; + } + switch (bbr->cycle_idx) { + case BBR_BW_PROBE_UP: + return BBR2_PHASE_PROBE_BW_UP; + case BBR_BW_PROBE_DOWN: + return BBR2_PHASE_PROBE_BW_DOWN; + case BBR_BW_PROBE_CRUISE: + return BBR2_PHASE_PROBE_BW_CRUISE; + case BBR_BW_PROBE_REFILL: + return BBR2_PHASE_PROBE_BW_REFILL; + default: + return BBR2_PHASE_INVALID; + } +} + +static size_t bbr2_get_info(struct sock *sk, u32 ext, int *attr, + union tcp_cc_info *info) +{ + if (ext & (1 << (INET_DIAG_BBRINFO - 1)) || + ext & (1 << (INET_DIAG_VEGASINFO - 1))) { + struct bbr *bbr = inet_csk_ca(sk); + u64 bw = bbr_bw_bytes_per_sec(sk, bbr_bw(sk)); + u64 bw_hi = bbr_bw_bytes_per_sec(sk, bbr_max_bw(sk)); + u64 bw_lo = bbr->bw_lo == ~0U ? + ~0ULL : bbr_bw_bytes_per_sec(sk, bbr->bw_lo); + + memset(&info->bbr2, 0, sizeof(info->bbr2)); + info->bbr2.bbr_bw_lsb = (u32)bw; + info->bbr2.bbr_bw_msb = (u32)(bw >> 32); + info->bbr2.bbr_min_rtt = bbr->min_rtt_us; + info->bbr2.bbr_pacing_gain = bbr->pacing_gain; + info->bbr2.bbr_cwnd_gain = bbr->cwnd_gain; + info->bbr2.bbr_bw_hi_lsb = (u32)bw_hi; + info->bbr2.bbr_bw_hi_msb = (u32)(bw_hi >> 32); + info->bbr2.bbr_bw_lo_lsb = (u32)bw_lo; + info->bbr2.bbr_bw_lo_msb = (u32)(bw_lo >> 32); + info->bbr2.bbr_mode = bbr->mode; + info->bbr2.bbr_phase = (__u8)bbr2_get_phase(bbr); + info->bbr2.bbr_version = (__u8)2; + info->bbr2.bbr_inflight_lo = bbr->inflight_lo; + info->bbr2.bbr_inflight_hi = bbr->inflight_hi; + info->bbr2.bbr_extra_acked = bbr_extra_acked(sk); + *attr = INET_DIAG_BBRINFO; + return sizeof(info->bbr2); + } + return 0; +} + +static void bbr2_set_state(struct sock *sk, u8 new_state) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + if (new_state == TCP_CA_Loss) { + struct rate_sample rs = { .losses = 1 }; + struct bbr_context ctx = { 0 }; + + bbr->prev_ca_state = TCP_CA_Loss; + bbr->full_bw = 0; + if (!bbr2_is_probing_bandwidth(sk) && bbr->inflight_lo == ~0U) { + /* bbr_adapt_lower_bounds() needs cwnd before + * we suffered an RTO, to update inflight_lo: + */ + bbr->inflight_lo = + max(tp->snd_cwnd, bbr->prior_cwnd); + } + bbr_debug(sk, 0, &rs, &ctx); + } else if (bbr->prev_ca_state == TCP_CA_Loss && + new_state != TCP_CA_Loss) { + tp->snd_cwnd = max(tp->snd_cwnd, bbr->prior_cwnd); + bbr->try_fast_path = 0; /* bound cwnd using latest model */ + } +} + +static struct tcp_congestion_ops tcp_bbr2_cong_ops __read_mostly = { + .flags = TCP_CONG_NON_RESTRICTED | TCP_CONG_WANTS_CE_EVENTS, + .name = "bbr2", + .owner = THIS_MODULE, + .init = bbr2_init, + .cong_control = bbr2_main, + .sndbuf_expand = bbr_sndbuf_expand, + .skb_marked_lost = bbr2_skb_marked_lost, + .undo_cwnd = bbr2_undo_cwnd, + .cwnd_event = bbr_cwnd_event, + .ssthresh = bbr2_ssthresh, + .tso_segs = bbr_tso_segs, + .get_info = bbr2_get_info, + .set_state = bbr2_set_state, +}; + +static int __init bbr_register(void) +{ + BUILD_BUG_ON(sizeof(struct bbr) > ICSK_CA_PRIV_SIZE); + return tcp_register_congestion_control(&tcp_bbr2_cong_ops); +} + +static void __exit bbr_unregister(void) +{ + tcp_unregister_congestion_control(&tcp_bbr2_cong_ops); +} + +module_init(bbr_register); +module_exit(bbr_unregister); + +MODULE_AUTHOR("Van Jacobson "); +MODULE_AUTHOR("Neal Cardwell "); +MODULE_AUTHOR("Yuchung Cheng "); +MODULE_AUTHOR("Soheil Hassas Yeganeh "); +MODULE_AUTHOR("Priyaranjan Jha "); +MODULE_AUTHOR("Yousuk Seung "); +MODULE_AUTHOR("Kevin Yang "); +MODULE_AUTHOR("Arjun Roy "); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_DESCRIPTION("TCP BBR (Bottleneck Bandwidth and RTT)"); diff --git a/net/ipv4/tcp_cong.c b/net/ipv4/tcp_cong.c index 563d016e7478..153ed9010c0c 100644 --- a/net/ipv4/tcp_cong.c +++ b/net/ipv4/tcp_cong.c @@ -179,6 +179,7 @@ void tcp_init_congestion_control(struct sock *sk) struct inet_connection_sock *icsk = inet_csk(sk); tcp_sk(sk)->prior_ssthresh = 0; + tcp_sk(sk)->fast_ack_mode = 0; if (icsk->icsk_ca_ops->init) icsk->icsk_ca_ops->init(sk); if (tcp_ca_needs_ecn(sk)) @@ -230,6 +231,10 @@ int tcp_set_default_congestion_control(struct net *net, const char *name) ret = -ENOENT; } else if (!bpf_try_module_get(ca, ca->owner)) { ret = -EBUSY; + } else if (!net_eq(net, &init_net) && + !(ca->flags & TCP_CONG_NON_RESTRICTED)) { + /* Only init netns can set default to a restricted algorithm */ + ret = -EPERM; } else { prev = xchg(&net->ipv4.tcp_congestion_control, ca); if (prev) diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index 69a545db80d2..45aaba87ce8e 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -348,7 +348,7 @@ static void __tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb) tcp_enter_quickack_mode(sk, 2); break; case INET_ECN_CE: - if (tcp_ca_needs_ecn(sk)) + if (tcp_ca_wants_ce_events(sk)) tcp_ca_event(sk, CA_EVENT_ECN_IS_CE); if (!(tp->ecn_flags & TCP_ECN_DEMAND_CWR)) { @@ -359,7 +359,7 @@ static void __tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb) tp->ecn_flags |= TCP_ECN_SEEN; break; default: - if (tcp_ca_needs_ecn(sk)) + if (tcp_ca_wants_ce_events(sk)) tcp_ca_event(sk, CA_EVENT_ECN_NO_CE); tp->ecn_flags |= TCP_ECN_SEEN; break; @@ -1039,7 +1039,12 @@ static void tcp_verify_retransmit_hint(struct tcp_sock *tp, struct sk_buff *skb) */ static void tcp_notify_skb_loss_event(struct tcp_sock *tp, const struct sk_buff *skb) { + struct sock *sk = (struct sock *)tp; + const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; + tp->lost += tcp_skb_pcount(skb); + if (ca_ops->skb_marked_lost) + ca_ops->skb_marked_lost(sk, skb); } void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb) @@ -1420,6 +1425,17 @@ static bool tcp_shifted_skb(struct sock *sk, struct sk_buff *prev, WARN_ON_ONCE(tcp_skb_pcount(skb) < pcount); tcp_skb_pcount_add(skb, -pcount); + /* Adjust tx.in_flight as pcount is shifted from skb to prev. */ + if (WARN_ONCE(TCP_SKB_CB(skb)->tx.in_flight < pcount, + "prev in_flight: %u skb in_flight: %u pcount: %u", + TCP_SKB_CB(prev)->tx.in_flight, + TCP_SKB_CB(skb)->tx.in_flight, + pcount)) + TCP_SKB_CB(skb)->tx.in_flight = 0; + else + TCP_SKB_CB(skb)->tx.in_flight -= pcount; + TCP_SKB_CB(prev)->tx.in_flight += pcount; + /* When we're adding to gso_segs == 1, gso_size will be zero, * in theory this shouldn't be necessary but as long as DSACK * code can come after this skb later on it's better to keep @@ -3182,7 +3198,6 @@ static int tcp_clean_rtx_queue(struct sock *sk, const struct sk_buff *ack_skb, long seq_rtt_us = -1L; long ca_rtt_us = -1L; u32 pkts_acked = 0; - u32 last_in_flight = 0; bool rtt_update; int flag = 0; @@ -3218,7 +3233,6 @@ static int tcp_clean_rtx_queue(struct sock *sk, const struct sk_buff *ack_skb, if (!first_ackt) first_ackt = last_ackt; - last_in_flight = TCP_SKB_CB(skb)->tx.in_flight; if (before(start_seq, reord)) reord = start_seq; if (!after(scb->end_seq, tp->high_seq)) @@ -3284,8 +3298,8 @@ static int tcp_clean_rtx_queue(struct sock *sk, const struct sk_buff *ack_skb, seq_rtt_us = tcp_stamp_us_delta(tp->tcp_mstamp, first_ackt); ca_rtt_us = tcp_stamp_us_delta(tp->tcp_mstamp, last_ackt); - if (pkts_acked == 1 && last_in_flight < tp->mss_cache && - last_in_flight && !prior_sacked && fully_acked && + if (pkts_acked == 1 && fully_acked && !prior_sacked && + (tp->snd_una - prior_snd_una) < tp->mss_cache && sack->rate->prior_delivered + 1 == tp->delivered && !(flag & (FLAG_CA_ALERT | FLAG_SYN_ACKED))) { /* Conservatively mark a delayed ACK. It's typically @@ -3342,9 +3356,10 @@ static int tcp_clean_rtx_queue(struct sock *sk, const struct sk_buff *ack_skb, if (icsk->icsk_ca_ops->pkts_acked) { struct ack_sample sample = { .pkts_acked = pkts_acked, - .rtt_us = sack->rate->rtt_us, - .in_flight = last_in_flight }; + .rtt_us = sack->rate->rtt_us }; + sample.in_flight = tp->mss_cache * + (tp->delivered - sack->rate->prior_delivered); icsk->icsk_ca_ops->pkts_acked(sk, &sample); } @@ -3742,6 +3757,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) prior_fack = tcp_is_sack(tp) ? tcp_highest_sack_seq(tp) : tp->snd_una; rs.prior_in_flight = tcp_packets_in_flight(tp); + tcp_rate_check_app_limited(sk); /* ts_recent update must be made after we are sure that the packet * is in window. @@ -3839,6 +3855,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) delivered = tcp_newly_delivered(sk, delivered, flag); lost = tp->lost - lost; /* freshly marked lost */ rs.is_ack_delayed = !!(flag & FLAG_ACK_MAYBE_DELAYED); + rs.is_ece = !!(flag & FLAG_ECE); tcp_rate_gen(sk, delivered, lost, is_sack_reneg, sack_state.rate); tcp_cong_control(sk, ack, delivered, flag, sack_state.rate); tcp_xmit_recovery(sk, rexmit); @@ -5399,13 +5416,14 @@ static void __tcp_ack_snd_check(struct sock *sk, int ofo_possible) /* More than one full frame received... */ if (((tp->rcv_nxt - tp->rcv_wup) > inet_csk(sk)->icsk_ack.rcv_mss && + (tp->fast_ack_mode == 1 || /* ... and right edge of window advances far enough. * (tcp_recvmsg() will send ACK otherwise). * If application uses SO_RCVLOWAT, we want send ack now if * we have not received enough bytes to satisfy the condition. */ - (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat || - __tcp_select_window(sk) >= tp->rcv_wnd)) || + (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat || + __tcp_select_window(sk) >= tp->rcv_wnd))) || /* We ACK each frame or... */ tcp_in_quickack_mode(sk) || /* Protocol state mandates a one-time immediate ACK */ diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index fbf140a770d8..90d939375b29 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -1256,8 +1256,6 @@ static int __tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, tp->tcp_wstamp_ns = max(tp->tcp_wstamp_ns, tp->tcp_clock_cache); skb->skb_mstamp_ns = tp->tcp_wstamp_ns; if (clone_it) { - TCP_SKB_CB(skb)->tx.in_flight = TCP_SKB_CB(skb)->end_seq - - tp->snd_una; oskb = skb; tcp_skb_tsorted_save(oskb) { @@ -1536,7 +1534,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, { struct tcp_sock *tp = tcp_sk(sk); struct sk_buff *buff; - int nsize, old_factor; + int nsize, old_factor, inflight_prev; long limit; int nlen; u8 flags; @@ -1615,6 +1613,15 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, if (diff) tcp_adjust_pcount(sk, skb, diff); + + /* Set buff tx.in_flight as if buff were sent by itself. */ + inflight_prev = TCP_SKB_CB(skb)->tx.in_flight - old_factor; + if (WARN_ONCE(inflight_prev < 0, + "inconsistent: tx.in_flight: %u old_factor: %d", + TCP_SKB_CB(skb)->tx.in_flight, old_factor)) + inflight_prev = 0; + TCP_SKB_CB(buff)->tx.in_flight = inflight_prev + + tcp_skb_pcount(buff); } /* Link BUFF into the send queue. */ @@ -1982,13 +1989,12 @@ static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now, static u32 tcp_tso_segs(struct sock *sk, unsigned int mss_now) { const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; - u32 min_tso, tso_segs; - - min_tso = ca_ops->min_tso_segs ? - ca_ops->min_tso_segs(sk) : - sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs; + u32 tso_segs; - tso_segs = tcp_tso_autosize(sk, mss_now, min_tso); + tso_segs = ca_ops->tso_segs ? + ca_ops->tso_segs(sk, mss_now) : + tcp_tso_autosize(sk, mss_now, + sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs); return min_t(u32, tso_segs, sk->sk_gso_max_segs); } @@ -2628,6 +2634,7 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle, skb->skb_mstamp_ns = tp->tcp_wstamp_ns = tp->tcp_clock_cache; list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue); tcp_init_tso_segs(skb, mss_now); + tcp_set_tx_in_flight(sk, skb); goto repair; /* Skip network transmission */ } diff --git a/net/ipv4/tcp_rate.c b/net/ipv4/tcp_rate.c index 0de693565963..796fa6e5310c 100644 --- a/net/ipv4/tcp_rate.c +++ b/net/ipv4/tcp_rate.c @@ -34,6 +34,24 @@ * ready to send in the write queue. */ +void tcp_set_tx_in_flight(struct sock *sk, struct sk_buff *skb) +{ + struct tcp_sock *tp = tcp_sk(sk); + u32 in_flight; + + /* Check, sanitize, and record packets in flight after skb was sent. */ + in_flight = tcp_packets_in_flight(tp) + tcp_skb_pcount(skb); + if (WARN_ONCE(in_flight > TCPCB_IN_FLIGHT_MAX, + "insane in_flight %u cc %s mss %u " + "cwnd %u pif %u %u %u %u\n", + in_flight, inet_csk(sk)->icsk_ca_ops->name, + tp->mss_cache, tp->snd_cwnd, + tp->packets_out, tp->retrans_out, + tp->sacked_out, tp->lost_out)) + in_flight = TCPCB_IN_FLIGHT_MAX; + TCP_SKB_CB(skb)->tx.in_flight = in_flight; +} + /* Snapshot the current delivery information in the skb, to generate * a rate sample later when the skb is (s)acked in tcp_rate_skb_delivered(). */ @@ -65,7 +83,10 @@ void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb) TCP_SKB_CB(skb)->tx.first_tx_mstamp = tp->first_tx_mstamp; TCP_SKB_CB(skb)->tx.delivered_mstamp = tp->delivered_mstamp; TCP_SKB_CB(skb)->tx.delivered = tp->delivered; + TCP_SKB_CB(skb)->tx.delivered_ce = tp->delivered_ce; + TCP_SKB_CB(skb)->tx.lost = tp->lost; TCP_SKB_CB(skb)->tx.is_app_limited = tp->app_limited ? 1 : 0; + tcp_set_tx_in_flight(sk, skb); } /* When an skb is sacked or acked, we fill in the rate sample with the (prior) @@ -86,16 +107,20 @@ void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, if (!rs->prior_delivered || after(scb->tx.delivered, rs->prior_delivered)) { + rs->prior_lost = scb->tx.lost; + rs->prior_delivered_ce = scb->tx.delivered_ce; rs->prior_delivered = scb->tx.delivered; rs->prior_mstamp = scb->tx.delivered_mstamp; rs->is_app_limited = scb->tx.is_app_limited; rs->is_retrans = scb->sacked & TCPCB_RETRANS; + rs->tx_in_flight = scb->tx.in_flight; /* Record send time of most recently ACKed packet: */ tp->first_tx_mstamp = tcp_skb_timestamp_us(skb); /* Find the duration of the "send phase" of this window: */ - rs->interval_us = tcp_stamp_us_delta(tp->first_tx_mstamp, - scb->tx.first_tx_mstamp); + rs->interval_us = tcp_stamp32_us_delta( + tp->first_tx_mstamp, + scb->tx.first_tx_mstamp); } /* Mark off the skb delivered once it's sacked to avoid being @@ -137,6 +162,11 @@ void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, return; } rs->delivered = tp->delivered - rs->prior_delivered; + rs->lost = tp->lost - rs->prior_lost; + + rs->delivered_ce = tp->delivered_ce - rs->prior_delivered_ce; + /* delivered_ce occupies less than 32 bits in the skb control block */ + rs->delivered_ce &= TCPCB_DELIVERED_CE_MASK; /* Model sending data and receiving ACKs as separate pipeline phases * for a window. Usually the ACK phase is longer, but with ACK @@ -144,7 +174,7 @@ void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, * longer phase. */ snd_us = rs->interval_us; /* send phase */ - ack_us = tcp_stamp_us_delta(tp->tcp_mstamp, + ack_us = tcp_stamp32_us_delta(tp->tcp_mstamp, rs->prior_mstamp); /* ack phase */ rs->interval_us = max(snd_us, ack_us); diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c index 4ef08079ccfa..b5b24caa8ba0 100644 --- a/net/ipv4/tcp_timer.c +++ b/net/ipv4/tcp_timer.c @@ -607,6 +607,7 @@ void tcp_write_timer_handler(struct sock *sk) goto out; } + tcp_rate_check_app_limited(sk); tcp_mstamp_refresh(tcp_sk(sk)); event = icsk->icsk_pending; diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index 99d743eb9dc4..c586a6bb8c6d 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -2664,9 +2664,12 @@ int udp_lib_setsockopt(struct sock *sk, int level, int optname, case UDP_GRO: lock_sock(sk); + + /* when enabling GRO, accept the related GSO packet type */ if (valbool) udp_tunnel_encap_enable(sk->sk_socket); up->gro_enabled = valbool; + up->accept_udp_l4 = valbool; release_sock(sk); break; diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c index c5b4b586570f..25134a3548e9 100644 --- a/net/ipv4/udp_offload.c +++ b/net/ipv4/udp_offload.c @@ -515,21 +515,24 @@ struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb, unsigned int off = skb_gro_offset(skb); int flush = 1; + /* we can do L4 aggregation only if the packet can't land in a tunnel + * otherwise we could corrupt the inner stream + */ NAPI_GRO_CB(skb)->is_flist = 0; - if (skb->dev->features & NETIF_F_GRO_FRAGLIST) - NAPI_GRO_CB(skb)->is_flist = sk ? !udp_sk(sk)->gro_enabled: 1; + if (!sk || !udp_sk(sk)->gro_receive) { + if (skb->dev->features & NETIF_F_GRO_FRAGLIST) + NAPI_GRO_CB(skb)->is_flist = sk ? !udp_sk(sk)->gro_enabled : 1; - if ((!sk && (skb->dev->features & NETIF_F_GRO_UDP_FWD)) || - (sk && udp_sk(sk)->gro_enabled) || NAPI_GRO_CB(skb)->is_flist) { - pp = call_gro_receive(udp_gro_receive_segment, head, skb); + if ((!sk && (skb->dev->features & NETIF_F_GRO_UDP_FWD)) || + (sk && udp_sk(sk)->gro_enabled) || NAPI_GRO_CB(skb)->is_flist) + pp = call_gro_receive(udp_gro_receive_segment, head, skb); return pp; } - if (!sk || NAPI_GRO_CB(skb)->encap_mark || + if (NAPI_GRO_CB(skb)->encap_mark || (uh->check && skb->ip_summed != CHECKSUM_PARTIAL && NAPI_GRO_CB(skb)->csum_cnt == 0 && - !NAPI_GRO_CB(skb)->csum_valid) || - !udp_sk(sk)->gro_receive) + !NAPI_GRO_CB(skb)->csum_valid)) goto out; /* mark that this skb passed once through the tunnel gro layer */ diff --git a/net/ipv6/ip6_gre.c b/net/ipv6/ip6_gre.c index 1baf43aacb2e..bc224f917bbd 100644 --- a/net/ipv6/ip6_gre.c +++ b/net/ipv6/ip6_gre.c @@ -387,7 +387,6 @@ static struct ip6_tnl *ip6gre_tunnel_locate(struct net *net, if (!(nt->parms.o_flags & TUNNEL_SEQ)) dev->features |= NETIF_F_LLTX; - dev_hold(dev); ip6gre_tunnel_link(ign, nt); return nt; @@ -1496,6 +1495,7 @@ static int ip6gre_tunnel_init_common(struct net_device *dev) } ip6gre_tnl_init_features(dev); + dev_hold(dev); return 0; cleanup_dst_cache_init: @@ -1538,8 +1538,6 @@ static void ip6gre_fb_tunnel_init(struct net_device *dev) strcpy(tunnel->parms.name, dev->name); tunnel->hlen = sizeof(struct ipv6hdr) + 4; - - dev_hold(dev); } static struct inet6_protocol ip6gre_protocol __read_mostly = { @@ -1889,6 +1887,7 @@ static int ip6erspan_tap_init(struct net_device *dev) dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; ip6erspan_tnl_link_config(tunnel, 1); + dev_hold(dev); return 0; cleanup_dst_cache_init: @@ -1988,8 +1987,6 @@ static int ip6gre_newlink_common(struct net *src_net, struct net_device *dev, if (tb[IFLA_MTU]) ip6_tnl_change_mtu(dev, nla_get_u32(tb[IFLA_MTU])); - dev_hold(dev); - out: return err; } diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c index 42fe7db6bbb3..d42f471b0d65 100644 --- a/net/ipv6/ip6_tunnel.c +++ b/net/ipv6/ip6_tunnel.c @@ -266,7 +266,6 @@ static int ip6_tnl_create2(struct net_device *dev) strcpy(t->parms.name, dev->name); - dev_hold(dev); ip6_tnl_link(ip6n, t); return 0; @@ -1882,6 +1881,7 @@ ip6_tnl_dev_init_gen(struct net_device *dev) dev->min_mtu = ETH_MIN_MTU; dev->max_mtu = IP6_MAX_MTU - dev->hard_header_len; + dev_hold(dev); return 0; destroy_dst: @@ -1925,7 +1925,6 @@ static int __net_init ip6_fb_tnl_dev_init(struct net_device *dev) struct ip6_tnl_net *ip6n = net_generic(net, ip6_tnl_net_id); t->parms.proto = IPPROTO_IPV6; - dev_hold(dev); rcu_assign_pointer(ip6n->tnls_wc[0], t); return 0; diff --git a/net/ipv6/ip6_vti.c b/net/ipv6/ip6_vti.c index e0cc32e45880..2d048e21abbb 100644 --- a/net/ipv6/ip6_vti.c +++ b/net/ipv6/ip6_vti.c @@ -193,7 +193,6 @@ static int vti6_tnl_create2(struct net_device *dev) strcpy(t->parms.name, dev->name); - dev_hold(dev); vti6_tnl_link(ip6n, t); return 0; @@ -934,6 +933,7 @@ static inline int vti6_dev_init_gen(struct net_device *dev) dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); if (!dev->tstats) return -ENOMEM; + dev_hold(dev); return 0; } @@ -965,7 +965,6 @@ static int __net_init vti6_fb_tnl_dev_init(struct net_device *dev) struct vti6_net *ip6n = net_generic(net, vti6_net_id); t->parms.proto = IPPROTO_IPV6; - dev_hold(dev); rcu_assign_pointer(ip6n->tnls_wc[0], t); return 0; diff --git a/net/ipv6/mcast_snoop.c b/net/ipv6/mcast_snoop.c index d3d6b6a66e5f..04d5fcdfa6e0 100644 --- a/net/ipv6/mcast_snoop.c +++ b/net/ipv6/mcast_snoop.c @@ -109,7 +109,7 @@ static int ipv6_mc_check_mld_msg(struct sk_buff *skb) struct mld_msg *mld; if (!ipv6_mc_may_pull(skb, len)) - return -EINVAL; + return -ENODATA; mld = (struct mld_msg *)skb_transport_header(skb); @@ -122,7 +122,7 @@ static int ipv6_mc_check_mld_msg(struct sk_buff *skb) case ICMPV6_MGM_QUERY: return ipv6_mc_check_mld_query(skb); default: - return -ENOMSG; + return -ENODATA; } } @@ -131,7 +131,7 @@ static inline __sum16 ipv6_mc_validate_checksum(struct sk_buff *skb) return skb_checksum_validate(skb, IPPROTO_ICMPV6, ip6_compute_pseudo); } -int ipv6_mc_check_icmpv6(struct sk_buff *skb) +static int ipv6_mc_check_icmpv6(struct sk_buff *skb) { unsigned int len = skb_transport_offset(skb) + sizeof(struct icmp6hdr); unsigned int transport_len = ipv6_transport_len(skb); @@ -150,7 +150,6 @@ int ipv6_mc_check_icmpv6(struct sk_buff *skb) return 0; } -EXPORT_SYMBOL(ipv6_mc_check_icmpv6); /** * ipv6_mc_check_mld - checks whether this is a sane MLD packet @@ -161,7 +160,10 @@ EXPORT_SYMBOL(ipv6_mc_check_icmpv6); * * -EINVAL: A broken packet was detected, i.e. it violates some internet * standard - * -ENOMSG: IP header validation succeeded but it is not an MLD packet. + * -ENOMSG: IP header validation succeeded but it is not an ICMPv6 packet + * with a hop-by-hop option. + * -ENODATA: IP+ICMPv6 header with hop-by-hop option validation succeeded + * but it is not an MLD packet. * -ENOMEM: A memory allocation failure happened. * * Caller needs to set the skb network header and free any returned skb if it diff --git a/net/ipv6/sit.c b/net/ipv6/sit.c index 9fdccf0718b5..fcc9ba2c80e9 100644 --- a/net/ipv6/sit.c +++ b/net/ipv6/sit.c @@ -218,8 +218,6 @@ static int ipip6_tunnel_create(struct net_device *dev) ipip6_tunnel_clone_6rd(dev, sitn); - dev_hold(dev); - ipip6_tunnel_link(sitn, t); return 0; @@ -1456,7 +1454,7 @@ static int ipip6_tunnel_init(struct net_device *dev) dev->tstats = NULL; return err; } - + dev_hold(dev); return 0; } @@ -1472,7 +1470,6 @@ static void __net_init ipip6_fb_tunnel_init(struct net_device *dev) iph->ihl = 5; iph->ttl = 64; - dev_hold(dev); rcu_assign_pointer(sitn->tunnels_wc[0], tunnel); } diff --git a/net/mac80211/main.c b/net/mac80211/main.c index 1b9c82616606..0331f3a3c40e 100644 --- a/net/mac80211/main.c +++ b/net/mac80211/main.c @@ -1141,8 +1141,11 @@ int ieee80211_register_hw(struct ieee80211_hw *hw) if (local->hw.wiphy->max_scan_ie_len) local->hw.wiphy->max_scan_ie_len -= local->scan_ies_len; - WARN_ON(!ieee80211_cs_list_valid(local->hw.cipher_schemes, - local->hw.n_cipher_schemes)); + if (WARN_ON(!ieee80211_cs_list_valid(local->hw.cipher_schemes, + local->hw.n_cipher_schemes))) { + result = -EINVAL; + goto fail_workqueue; + } result = ieee80211_init_cipher_suites(local); if (result < 0) diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index 96f487fc0071..0fe91dc9817e 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -1295,6 +1295,11 @@ static void ieee80211_chswitch_post_beacon(struct ieee80211_sub_if_data *sdata) sdata->vif.csa_active = false; ifmgd->csa_waiting_bcn = false; + /* + * If the CSA IE is still present on the beacon after the switch, + * we need to consider it as a new CSA (possibly to self). + */ + ifmgd->beacon_crc_valid = false; ret = drv_post_channel_switch(sdata); if (ret) { @@ -1400,11 +1405,8 @@ ieee80211_sta_process_chanswitch(struct ieee80211_sub_if_data *sdata, ch_switch.delay = csa_ie.max_switch_time; } - if (res < 0) { - ieee80211_queue_work(&local->hw, - &ifmgd->csa_connection_drop_work); - return; - } + if (res < 0) + goto lock_and_drop_connection; if (beacon && sdata->vif.csa_active && !ifmgd->csa_waiting_bcn) { if (res) diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c index 3b3bcefbf657..28422d687096 100644 --- a/net/mac80211/tx.c +++ b/net/mac80211/tx.c @@ -2267,17 +2267,6 @@ netdev_tx_t ieee80211_monitor_start_xmit(struct sk_buff *skb, payload[7]); } - /* Initialize skb->priority for QoS frames. If the DONT_REORDER flag - * is set, stick to the default value for skb->priority to assure - * frames injected with this flag are not reordered relative to each - * other. - */ - if (ieee80211_is_data_qos(hdr->frame_control) && - !(info->control.flags & IEEE80211_TX_CTRL_DONT_REORDER)) { - u8 *p = ieee80211_get_qos_ctl(hdr); - skb->priority = *p & IEEE80211_QOS_CTL_TAG1D_MASK; - } - rcu_read_lock(); /* @@ -2341,6 +2330,15 @@ netdev_tx_t ieee80211_monitor_start_xmit(struct sk_buff *skb, info->band = chandef->chan->band; + /* Initialize skb->priority according to frame type and TID class, + * with respect to the sub interface that the frame will actually + * be transmitted on. If the DONT_REORDER flag is set, the original + * skb-priority is preserved to assure frames injected with this + * flag are not reordered relative to each other. + */ + ieee80211_select_queue_80211(sdata, skb, hdr); + skb_set_queue_mapping(skb, ieee80211_ac_from_tid(skb->priority)); + /* remove the injection radiotap header */ skb_pull(skb, len_rthdr); diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c index 4bde960e19dc..65e5d3eb1078 100644 --- a/net/mptcp/protocol.c +++ b/net/mptcp/protocol.c @@ -399,6 +399,14 @@ static bool mptcp_pending_data_fin(struct sock *sk, u64 *seq) return false; } +static void mptcp_set_datafin_timeout(const struct sock *sk) +{ + struct inet_connection_sock *icsk = inet_csk(sk); + + mptcp_sk(sk)->timer_ival = min(TCP_RTO_MAX, + TCP_RTO_MIN << icsk->icsk_retransmits); +} + static void mptcp_set_timeout(const struct sock *sk, const struct sock *ssk) { long tout = ssk && inet_csk(ssk)->icsk_pending ? @@ -1052,7 +1060,7 @@ static void __mptcp_clean_una(struct sock *sk) } if (snd_una == READ_ONCE(msk->snd_nxt)) { - if (msk->timer_ival) + if (msk->timer_ival && !mptcp_data_fin_enabled(msk)) mptcp_stop_timer(sk); } else { mptcp_reset_timer(sk); @@ -1275,7 +1283,7 @@ static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk, int avail_size; size_t ret = 0; - pr_debug("msk=%p ssk=%p sending dfrag at seq=%lld len=%d already sent=%d", + pr_debug("msk=%p ssk=%p sending dfrag at seq=%llu len=%u already sent=%u", msk, ssk, dfrag->data_seq, dfrag->data_len, info->sent); /* compute send limit */ @@ -1693,7 +1701,7 @@ static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) if (!msk->first_pending) WRITE_ONCE(msk->first_pending, dfrag); } - pr_debug("msk=%p dfrag at seq=%lld len=%d sent=%d new=%d", msk, + pr_debug("msk=%p dfrag at seq=%llu len=%u sent=%u new=%d", msk, dfrag->data_seq, dfrag->data_len, dfrag->already_sent, !dfrag_collapsed); @@ -2276,8 +2284,19 @@ static void __mptcp_retrans(struct sock *sk) __mptcp_clean_una_wakeup(sk); dfrag = mptcp_rtx_head(sk); - if (!dfrag) + if (!dfrag) { + if (mptcp_data_fin_enabled(msk)) { + struct inet_connection_sock *icsk = inet_csk(sk); + + icsk->icsk_retransmits++; + mptcp_set_datafin_timeout(sk); + mptcp_send_ack(msk); + + goto reset_timer; + } + return; + } ssk = mptcp_subflow_get_retrans(msk); if (!ssk) @@ -2460,6 +2479,8 @@ void mptcp_subflow_shutdown(struct sock *sk, struct sock *ssk, int how) pr_debug("Sending DATA_FIN on subflow %p", ssk); mptcp_set_timeout(sk, ssk); tcp_send_ack(ssk); + if (!mptcp_timer_pending(sk)) + mptcp_reset_timer(sk); } break; } diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c index d17d39ccdf34..4fe7acaa472f 100644 --- a/net/mptcp/subflow.c +++ b/net/mptcp/subflow.c @@ -524,8 +524,7 @@ static void mptcp_sock_destruct(struct sock *sk) * ESTABLISHED state and will not have the SOCK_DEAD flag. * Both result in warnings from inet_sock_destruct. */ - - if (sk->sk_state == TCP_ESTABLISHED) { + if ((1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) { sk->sk_state = TCP_CLOSE; WARN_ON_ONCE(sk->sk_socket); sock_orphan(sk); diff --git a/net/netfilter/nf_conntrack_standalone.c b/net/netfilter/nf_conntrack_standalone.c index c6c0cb465664..313d1c8ff066 100644 --- a/net/netfilter/nf_conntrack_standalone.c +++ b/net/netfilter/nf_conntrack_standalone.c @@ -1060,16 +1060,10 @@ static int nf_conntrack_standalone_init_sysctl(struct net *net) nf_conntrack_standalone_init_dccp_sysctl(net, table); nf_conntrack_standalone_init_gre_sysctl(net, table); - /* Don't allow unprivileged users to alter certain sysctls */ - if (net->user_ns != &init_user_ns) { + /* Don't allow non-init_net ns to alter global sysctls */ + if (!net_eq(&init_net, net)) { table[NF_SYSCTL_CT_MAX].mode = 0444; table[NF_SYSCTL_CT_EXPECT_MAX].mode = 0444; - table[NF_SYSCTL_CT_HELPER].mode = 0444; -#ifdef CONFIG_NF_CONNTRACK_EVENTS - table[NF_SYSCTL_CT_EVENTS].mode = 0444; -#endif - table[NF_SYSCTL_CT_BUCKETS].mode = 0444; - } else if (!net_eq(&init_net, net)) { table[NF_SYSCTL_CT_BUCKETS].mode = 0444; } diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c index 589d2f6978d3..878ed49d0c56 100644 --- a/net/netfilter/nf_tables_api.c +++ b/net/netfilter/nf_tables_api.c @@ -6246,9 +6246,9 @@ static int nf_tables_newobj(struct net *net, struct sock *nlsk, INIT_LIST_HEAD(&obj->list); return err; err_trans: - kfree(obj->key.name); -err_userdata: kfree(obj->udata); +err_userdata: + kfree(obj->key.name); err_strdup: if (obj->ops->destroy) obj->ops->destroy(&ctx, obj); diff --git a/net/netfilter/nf_tables_offload.c b/net/netfilter/nf_tables_offload.c index 9ae14270c543..2b00f7f47693 100644 --- a/net/netfilter/nf_tables_offload.c +++ b/net/netfilter/nf_tables_offload.c @@ -45,6 +45,48 @@ void nft_flow_rule_set_addr_type(struct nft_flow_rule *flow, offsetof(struct nft_flow_key, control); } +struct nft_offload_ethertype { + __be16 value; + __be16 mask; +}; + +static void nft_flow_rule_transfer_vlan(struct nft_offload_ctx *ctx, + struct nft_flow_rule *flow) +{ + struct nft_flow_match *match = &flow->match; + struct nft_offload_ethertype ethertype; + + if (match->dissector.used_keys & BIT(FLOW_DISSECTOR_KEY_CONTROL) && + match->key.basic.n_proto != htons(ETH_P_8021Q) && + match->key.basic.n_proto != htons(ETH_P_8021AD)) + return; + + ethertype.value = match->key.basic.n_proto; + ethertype.mask = match->mask.basic.n_proto; + + if (match->dissector.used_keys & BIT(FLOW_DISSECTOR_KEY_VLAN) && + (match->key.vlan.vlan_tpid == htons(ETH_P_8021Q) || + match->key.vlan.vlan_tpid == htons(ETH_P_8021AD))) { + match->key.basic.n_proto = match->key.cvlan.vlan_tpid; + match->mask.basic.n_proto = match->mask.cvlan.vlan_tpid; + match->key.cvlan.vlan_tpid = match->key.vlan.vlan_tpid; + match->mask.cvlan.vlan_tpid = match->mask.vlan.vlan_tpid; + match->key.vlan.vlan_tpid = ethertype.value; + match->mask.vlan.vlan_tpid = ethertype.mask; + match->dissector.offset[FLOW_DISSECTOR_KEY_CVLAN] = + offsetof(struct nft_flow_key, cvlan); + match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_CVLAN); + } else { + match->key.basic.n_proto = match->key.vlan.vlan_tpid; + match->mask.basic.n_proto = match->mask.vlan.vlan_tpid; + match->key.vlan.vlan_tpid = ethertype.value; + match->mask.vlan.vlan_tpid = ethertype.mask; + match->dissector.offset[FLOW_DISSECTOR_KEY_VLAN] = + offsetof(struct nft_flow_key, vlan); + match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_VLAN); + } +} + struct nft_flow_rule *nft_flow_rule_create(struct net *net, const struct nft_rule *rule) { @@ -89,6 +131,8 @@ struct nft_flow_rule *nft_flow_rule_create(struct net *net, expr = nft_expr_next(expr); } + nft_flow_rule_transfer_vlan(ctx, flow); + flow->proto = ctx->dep.l3num; kfree(ctx); diff --git a/net/netfilter/nfnetlink_osf.c b/net/netfilter/nfnetlink_osf.c index 916a3c7f9eaf..79fbf37291f3 100644 --- a/net/netfilter/nfnetlink_osf.c +++ b/net/netfilter/nfnetlink_osf.c @@ -186,6 +186,8 @@ static const struct tcphdr *nf_osf_hdr_ctx_init(struct nf_osf_hdr_ctx *ctx, ctx->optp = skb_header_pointer(skb, ip_hdrlen(skb) + sizeof(struct tcphdr), ctx->optsize, opts); + if (!ctx->optp) + return NULL; } return tcp; diff --git a/net/netfilter/nft_cmp.c b/net/netfilter/nft_cmp.c index eb6a43a180bb..47b6d05f1ae6 100644 --- a/net/netfilter/nft_cmp.c +++ b/net/netfilter/nft_cmp.c @@ -114,19 +114,56 @@ static int nft_cmp_dump(struct sk_buff *skb, const struct nft_expr *expr) return -1; } +union nft_cmp_offload_data { + u16 val16; + u32 val32; + u64 val64; +}; + +static void nft_payload_n2h(union nft_cmp_offload_data *data, + const u8 *val, u32 len) +{ + switch (len) { + case 2: + data->val16 = ntohs(*((u16 *)val)); + break; + case 4: + data->val32 = ntohl(*((u32 *)val)); + break; + case 8: + data->val64 = be64_to_cpu(*((u64 *)val)); + break; + default: + WARN_ON_ONCE(1); + break; + } +} + static int __nft_cmp_offload(struct nft_offload_ctx *ctx, struct nft_flow_rule *flow, const struct nft_cmp_expr *priv) { struct nft_offload_reg *reg = &ctx->regs[priv->sreg]; + union nft_cmp_offload_data _data, _datamask; u8 *mask = (u8 *)&flow->match.mask; u8 *key = (u8 *)&flow->match.key; + u8 *data, *datamask; if (priv->op != NFT_CMP_EQ || priv->len > reg->len) return -EOPNOTSUPP; - memcpy(key + reg->offset, &priv->data, reg->len); - memcpy(mask + reg->offset, ®->mask, reg->len); + if (reg->flags & NFT_OFFLOAD_F_NETWORK2HOST) { + nft_payload_n2h(&_data, (u8 *)&priv->data, reg->len); + nft_payload_n2h(&_datamask, (u8 *)®->mask, reg->len); + data = (u8 *)&_data; + datamask = (u8 *)&_datamask; + } else { + data = (u8 *)&priv->data; + datamask = (u8 *)®->mask; + } + + memcpy(key + reg->offset, data, reg->len); + memcpy(mask + reg->offset, datamask, reg->len); flow->match.dissector.used_keys |= BIT(reg->key); flow->match.dissector.offset[reg->key] = reg->base_offset; diff --git a/net/netfilter/nft_payload.c b/net/netfilter/nft_payload.c index cb1c8c231880..501c5b24cc39 100644 --- a/net/netfilter/nft_payload.c +++ b/net/netfilter/nft_payload.c @@ -226,8 +226,9 @@ static int nft_payload_offload_ll(struct nft_offload_ctx *ctx, if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) return -EOPNOTSUPP; - NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_VLAN, vlan, - vlan_tci, sizeof(__be16), reg); + NFT_OFFLOAD_MATCH_FLAGS(FLOW_DISSECTOR_KEY_VLAN, vlan, + vlan_tci, sizeof(__be16), reg, + NFT_OFFLOAD_F_NETWORK2HOST); break; case offsetof(struct vlan_ethhdr, h_vlan_encapsulated_proto): if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) @@ -241,16 +242,18 @@ static int nft_payload_offload_ll(struct nft_offload_ctx *ctx, if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) return -EOPNOTSUPP; - NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_CVLAN, vlan, - vlan_tci, sizeof(__be16), reg); + NFT_OFFLOAD_MATCH_FLAGS(FLOW_DISSECTOR_KEY_CVLAN, cvlan, + vlan_tci, sizeof(__be16), reg, + NFT_OFFLOAD_F_NETWORK2HOST); break; case offsetof(struct vlan_ethhdr, h_vlan_encapsulated_proto) + sizeof(struct vlan_hdr): if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) return -EOPNOTSUPP; - NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_CVLAN, vlan, + NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_CVLAN, cvlan, vlan_tpid, sizeof(__be16), reg); + nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_NETWORK); break; default: return -EOPNOTSUPP; diff --git a/net/netfilter/nft_set_hash.c b/net/netfilter/nft_set_hash.c index bf618b7ec1ae..560c2cda52ee 100644 --- a/net/netfilter/nft_set_hash.c +++ b/net/netfilter/nft_set_hash.c @@ -406,9 +406,17 @@ static void nft_rhash_destroy(const struct nft_set *set) (void *)set); } +/* Number of buckets is stored in u32, so cap our result to 1U<<31 */ +#define NFT_MAX_BUCKETS (1U << 31) + static u32 nft_hash_buckets(u32 size) { - return roundup_pow_of_two(size * 4 / 3); + u64 val = div_u64((u64)size * 4, 3); + + if (val >= NFT_MAX_BUCKETS) + return NFT_MAX_BUCKETS; + + return roundup_pow_of_two(val); } static bool nft_rhash_estimate(const struct nft_set_desc *desc, u32 features, diff --git a/net/netfilter/xt_SECMARK.c b/net/netfilter/xt_SECMARK.c index 75625d13e976..498a0bf6f044 100644 --- a/net/netfilter/xt_SECMARK.c +++ b/net/netfilter/xt_SECMARK.c @@ -24,10 +24,9 @@ MODULE_ALIAS("ip6t_SECMARK"); static u8 mode; static unsigned int -secmark_tg(struct sk_buff *skb, const struct xt_action_param *par) +secmark_tg(struct sk_buff *skb, const struct xt_secmark_target_info_v1 *info) { u32 secmark = 0; - const struct xt_secmark_target_info *info = par->targinfo; switch (mode) { case SECMARK_MODE_SEL: @@ -41,7 +40,7 @@ secmark_tg(struct sk_buff *skb, const struct xt_action_param *par) return XT_CONTINUE; } -static int checkentry_lsm(struct xt_secmark_target_info *info) +static int checkentry_lsm(struct xt_secmark_target_info_v1 *info) { int err; @@ -73,15 +72,15 @@ static int checkentry_lsm(struct xt_secmark_target_info *info) return 0; } -static int secmark_tg_check(const struct xt_tgchk_param *par) +static int +secmark_tg_check(const char *table, struct xt_secmark_target_info_v1 *info) { - struct xt_secmark_target_info *info = par->targinfo; int err; - if (strcmp(par->table, "mangle") != 0 && - strcmp(par->table, "security") != 0) { + if (strcmp(table, "mangle") != 0 && + strcmp(table, "security") != 0) { pr_info_ratelimited("only valid in \'mangle\' or \'security\' table, not \'%s\'\n", - par->table); + table); return -EINVAL; } @@ -116,25 +115,76 @@ static void secmark_tg_destroy(const struct xt_tgdtor_param *par) } } -static struct xt_target secmark_tg_reg __read_mostly = { - .name = "SECMARK", - .revision = 0, - .family = NFPROTO_UNSPEC, - .checkentry = secmark_tg_check, - .destroy = secmark_tg_destroy, - .target = secmark_tg, - .targetsize = sizeof(struct xt_secmark_target_info), - .me = THIS_MODULE, +static int secmark_tg_check_v0(const struct xt_tgchk_param *par) +{ + struct xt_secmark_target_info *info = par->targinfo; + struct xt_secmark_target_info_v1 newinfo = { + .mode = info->mode, + }; + int ret; + + memcpy(newinfo.secctx, info->secctx, SECMARK_SECCTX_MAX); + + ret = secmark_tg_check(par->table, &newinfo); + info->secid = newinfo.secid; + + return ret; +} + +static unsigned int +secmark_tg_v0(struct sk_buff *skb, const struct xt_action_param *par) +{ + const struct xt_secmark_target_info *info = par->targinfo; + struct xt_secmark_target_info_v1 newinfo = { + .secid = info->secid, + }; + + return secmark_tg(skb, &newinfo); +} + +static int secmark_tg_check_v1(const struct xt_tgchk_param *par) +{ + return secmark_tg_check(par->table, par->targinfo); +} + +static unsigned int +secmark_tg_v1(struct sk_buff *skb, const struct xt_action_param *par) +{ + return secmark_tg(skb, par->targinfo); +} + +static struct xt_target secmark_tg_reg[] __read_mostly = { + { + .name = "SECMARK", + .revision = 0, + .family = NFPROTO_UNSPEC, + .checkentry = secmark_tg_check_v0, + .destroy = secmark_tg_destroy, + .target = secmark_tg_v0, + .targetsize = sizeof(struct xt_secmark_target_info), + .me = THIS_MODULE, + }, + { + .name = "SECMARK", + .revision = 1, + .family = NFPROTO_UNSPEC, + .checkentry = secmark_tg_check_v1, + .destroy = secmark_tg_destroy, + .target = secmark_tg_v1, + .targetsize = sizeof(struct xt_secmark_target_info_v1), + .usersize = offsetof(struct xt_secmark_target_info_v1, secid), + .me = THIS_MODULE, + }, }; static int __init secmark_tg_init(void) { - return xt_register_target(&secmark_tg_reg); + return xt_register_targets(secmark_tg_reg, ARRAY_SIZE(secmark_tg_reg)); } static void __exit secmark_tg_exit(void) { - xt_unregister_target(&secmark_tg_reg); + xt_unregister_targets(secmark_tg_reg, ARRAY_SIZE(secmark_tg_reg)); } module_init(secmark_tg_init); diff --git a/net/nfc/digital_dep.c b/net/nfc/digital_dep.c index 5971fb6f51cc..dc21b4141b0a 100644 --- a/net/nfc/digital_dep.c +++ b/net/nfc/digital_dep.c @@ -1273,6 +1273,8 @@ static void digital_tg_recv_dep_req(struct nfc_digital_dev *ddev, void *arg, } rc = nfc_tm_data_received(ddev->nfc_dev, resp); + if (rc) + resp = NULL; exit: kfree_skb(ddev->chaining_skb); diff --git a/net/nfc/llcp_sock.c b/net/nfc/llcp_sock.c index a3b46f888803..53dbe733f998 100644 --- a/net/nfc/llcp_sock.c +++ b/net/nfc/llcp_sock.c @@ -109,12 +109,14 @@ static int llcp_sock_bind(struct socket *sock, struct sockaddr *addr, int alen) GFP_KERNEL); if (!llcp_sock->service_name) { nfc_llcp_local_put(llcp_sock->local); + llcp_sock->local = NULL; ret = -ENOMEM; goto put_dev; } llcp_sock->ssap = nfc_llcp_get_sdp_ssap(local, llcp_sock); if (llcp_sock->ssap == LLCP_SAP_MAX) { nfc_llcp_local_put(llcp_sock->local); + llcp_sock->local = NULL; kfree(llcp_sock->service_name); llcp_sock->service_name = NULL; ret = -EADDRINUSE; @@ -709,6 +711,7 @@ static int llcp_sock_connect(struct socket *sock, struct sockaddr *_addr, llcp_sock->ssap = nfc_llcp_get_local_ssap(local); if (llcp_sock->ssap == LLCP_SAP_MAX) { nfc_llcp_local_put(llcp_sock->local); + llcp_sock->local = NULL; ret = -ENOMEM; goto put_dev; } @@ -756,6 +759,7 @@ static int llcp_sock_connect(struct socket *sock, struct sockaddr *_addr, sock_llcp_release: nfc_llcp_put_ssap(local, llcp_sock->ssap); nfc_llcp_local_put(llcp_sock->local); + llcp_sock->local = NULL; put_dev: nfc_put_device(dev); diff --git a/net/openvswitch/actions.c b/net/openvswitch/actions.c index 92a0b67b2728..77d924ab8cdb 100644 --- a/net/openvswitch/actions.c +++ b/net/openvswitch/actions.c @@ -827,17 +827,17 @@ static void ovs_fragment(struct net *net, struct vport *vport, } if (key->eth.type == htons(ETH_P_IP)) { - struct dst_entry ovs_dst; + struct rtable ovs_rt = { 0 }; unsigned long orig_dst; prepare_frag(vport, skb, orig_network_offset, ovs_key_mac_proto(key)); - dst_init(&ovs_dst, &ovs_dst_ops, NULL, 1, + dst_init(&ovs_rt.dst, &ovs_dst_ops, NULL, 1, DST_OBSOLETE_NONE, DST_NOCOUNT); - ovs_dst.dev = vport->dev; + ovs_rt.dst.dev = vport->dev; orig_dst = skb->_skb_refdst; - skb_dst_set_noref(skb, &ovs_dst); + skb_dst_set_noref(skb, &ovs_rt.dst); IPCB(skb)->frag_max_size = mru; ip_do_fragment(net, skb->sk, skb, ovs_vport_output); diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index e24b2841c643..9611e41c7b8b 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -1359,7 +1359,7 @@ static unsigned int fanout_demux_rollover(struct packet_fanout *f, struct packet_sock *po, *po_next, *po_skip = NULL; unsigned int i, j, room = ROOM_NONE; - po = pkt_sk(f->arr[idx]); + po = pkt_sk(rcu_dereference(f->arr[idx])); if (try_self) { room = packet_rcv_has_room(po, skb); @@ -1371,7 +1371,7 @@ static unsigned int fanout_demux_rollover(struct packet_fanout *f, i = j = min_t(int, po->rollover->sock, num - 1); do { - po_next = pkt_sk(f->arr[i]); + po_next = pkt_sk(rcu_dereference(f->arr[i])); if (po_next != po_skip && !READ_ONCE(po_next->pressure) && packet_rcv_has_room(po_next, skb) == ROOM_NORMAL) { if (i != j) @@ -1466,7 +1466,7 @@ static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev, if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER)) idx = fanout_demux_rollover(f, skb, idx, true, num); - po = pkt_sk(f->arr[idx]); + po = pkt_sk(rcu_dereference(f->arr[idx])); return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev); } @@ -1480,7 +1480,7 @@ static void __fanout_link(struct sock *sk, struct packet_sock *po) struct packet_fanout *f = po->fanout; spin_lock(&f->lock); - f->arr[f->num_members] = sk; + rcu_assign_pointer(f->arr[f->num_members], sk); smp_wmb(); f->num_members++; if (f->num_members == 1) @@ -1495,11 +1495,14 @@ static void __fanout_unlink(struct sock *sk, struct packet_sock *po) spin_lock(&f->lock); for (i = 0; i < f->num_members; i++) { - if (f->arr[i] == sk) + if (rcu_dereference_protected(f->arr[i], + lockdep_is_held(&f->lock)) == sk) break; } BUG_ON(i >= f->num_members); - f->arr[i] = f->arr[f->num_members - 1]; + rcu_assign_pointer(f->arr[i], + rcu_dereference_protected(f->arr[f->num_members - 1], + lockdep_is_held(&f->lock))); f->num_members--; if (f->num_members == 0) __dev_remove_pack(&f->prot_hook); diff --git a/net/packet/internal.h b/net/packet/internal.h index 5f61e59ebbff..48af35b1aed2 100644 --- a/net/packet/internal.h +++ b/net/packet/internal.h @@ -94,7 +94,7 @@ struct packet_fanout { spinlock_t lock; refcount_t sk_ref; struct packet_type prot_hook ____cacheline_aligned_in_smp; - struct sock *arr[]; + struct sock __rcu *arr[]; }; struct packet_rollover { diff --git a/net/qrtr/mhi.c b/net/qrtr/mhi.c index 2bf2b1943e61..fa611678af05 100644 --- a/net/qrtr/mhi.c +++ b/net/qrtr/mhi.c @@ -50,6 +50,9 @@ static int qcom_mhi_qrtr_send(struct qrtr_endpoint *ep, struct sk_buff *skb) struct qrtr_mhi_dev *qdev = container_of(ep, struct qrtr_mhi_dev, ep); int rc; + if (skb->sk) + sock_hold(skb->sk); + rc = skb_linearize(skb); if (rc) goto free_skb; @@ -59,12 +62,11 @@ static int qcom_mhi_qrtr_send(struct qrtr_endpoint *ep, struct sk_buff *skb) if (rc) goto free_skb; - if (skb->sk) - sock_hold(skb->sk); - return rc; free_skb: + if (skb->sk) + sock_put(skb->sk); kfree_skb(skb); return rc; diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c index 16e888a9601d..48fdf7293dea 100644 --- a/net/sched/act_ct.c +++ b/net/sched/act_ct.c @@ -732,7 +732,8 @@ static int tcf_ct_handle_fragments(struct net *net, struct sk_buff *skb, #endif } - *qdisc_skb_cb(skb) = cb; + if (err != -EINPROGRESS) + *qdisc_skb_cb(skb) = cb; skb_clear_hash(skb); skb->ignore_df = 1; return err; @@ -967,7 +968,7 @@ static int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a, err = tcf_ct_handle_fragments(net, skb, family, p->zone, &defrag); if (err == -EINPROGRESS) { retval = TC_ACT_STOLEN; - goto out; + goto out_clear; } if (err) goto drop; @@ -1030,7 +1031,6 @@ static int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a, out_push: skb_push_rcsum(skb, nh_ofs); -out: qdisc_skb_cb(skb)->post_ct = true; out_clear: tcf_action_update_bstats(&c->common, skb); diff --git a/net/sched/cls_flower.c b/net/sched/cls_flower.c index c69a4ba9c33f..3035f96c6e6c 100644 --- a/net/sched/cls_flower.c +++ b/net/sched/cls_flower.c @@ -209,16 +209,16 @@ static bool fl_range_port_dst_cmp(struct cls_fl_filter *filter, struct fl_flow_key *key, struct fl_flow_key *mkey) { - __be16 min_mask, max_mask, min_val, max_val; + u16 min_mask, max_mask, min_val, max_val; - min_mask = htons(filter->mask->key.tp_range.tp_min.dst); - max_mask = htons(filter->mask->key.tp_range.tp_max.dst); - min_val = htons(filter->key.tp_range.tp_min.dst); - max_val = htons(filter->key.tp_range.tp_max.dst); + min_mask = ntohs(filter->mask->key.tp_range.tp_min.dst); + max_mask = ntohs(filter->mask->key.tp_range.tp_max.dst); + min_val = ntohs(filter->key.tp_range.tp_min.dst); + max_val = ntohs(filter->key.tp_range.tp_max.dst); if (min_mask && max_mask) { - if (htons(key->tp_range.tp.dst) < min_val || - htons(key->tp_range.tp.dst) > max_val) + if (ntohs(key->tp_range.tp.dst) < min_val || + ntohs(key->tp_range.tp.dst) > max_val) return false; /* skb does not have min and max values */ @@ -232,16 +232,16 @@ static bool fl_range_port_src_cmp(struct cls_fl_filter *filter, struct fl_flow_key *key, struct fl_flow_key *mkey) { - __be16 min_mask, max_mask, min_val, max_val; + u16 min_mask, max_mask, min_val, max_val; - min_mask = htons(filter->mask->key.tp_range.tp_min.src); - max_mask = htons(filter->mask->key.tp_range.tp_max.src); - min_val = htons(filter->key.tp_range.tp_min.src); - max_val = htons(filter->key.tp_range.tp_max.src); + min_mask = ntohs(filter->mask->key.tp_range.tp_min.src); + max_mask = ntohs(filter->mask->key.tp_range.tp_max.src); + min_val = ntohs(filter->key.tp_range.tp_min.src); + max_val = ntohs(filter->key.tp_range.tp_max.src); if (min_mask && max_mask) { - if (htons(key->tp_range.tp.src) < min_val || - htons(key->tp_range.tp.src) > max_val) + if (ntohs(key->tp_range.tp.src) < min_val || + ntohs(key->tp_range.tp.src) > max_val) return false; /* skb does not have min and max values */ @@ -783,16 +783,16 @@ static int fl_set_key_port_range(struct nlattr **tb, struct fl_flow_key *key, TCA_FLOWER_UNSPEC, sizeof(key->tp_range.tp_max.src)); if (mask->tp_range.tp_min.dst && mask->tp_range.tp_max.dst && - htons(key->tp_range.tp_max.dst) <= - htons(key->tp_range.tp_min.dst)) { + ntohs(key->tp_range.tp_max.dst) <= + ntohs(key->tp_range.tp_min.dst)) { NL_SET_ERR_MSG_ATTR(extack, tb[TCA_FLOWER_KEY_PORT_DST_MIN], "Invalid destination port range (min must be strictly smaller than max)"); return -EINVAL; } if (mask->tp_range.tp_min.src && mask->tp_range.tp_max.src && - htons(key->tp_range.tp_max.src) <= - htons(key->tp_range.tp_min.src)) { + ntohs(key->tp_range.tp_max.src) <= + ntohs(key->tp_range.tp_min.src)) { NL_SET_ERR_MSG_ATTR(extack, tb[TCA_FLOWER_KEY_PORT_SRC_MIN], "Invalid source port range (min must be strictly smaller than max)"); diff --git a/net/sched/sch_frag.c b/net/sched/sch_frag.c index e1e77d3fb6c0..8c06381391d6 100644 --- a/net/sched/sch_frag.c +++ b/net/sched/sch_frag.c @@ -90,16 +90,16 @@ static int sch_fragment(struct net *net, struct sk_buff *skb, } if (skb_protocol(skb, true) == htons(ETH_P_IP)) { - struct dst_entry sch_frag_dst; + struct rtable sch_frag_rt = { 0 }; unsigned long orig_dst; sch_frag_prepare_frag(skb, xmit); - dst_init(&sch_frag_dst, &sch_frag_dst_ops, NULL, 1, + dst_init(&sch_frag_rt.dst, &sch_frag_dst_ops, NULL, 1, DST_OBSOLETE_NONE, DST_NOCOUNT); - sch_frag_dst.dev = skb->dev; + sch_frag_rt.dst.dev = skb->dev; orig_dst = skb->_skb_refdst; - skb_dst_set_noref(skb, &sch_frag_dst); + skb_dst_set_noref(skb, &sch_frag_rt.dst); IPCB(skb)->frag_max_size = mru; ret = ip_do_fragment(net, skb->sk, skb, sch_frag_xmit); diff --git a/net/sched/sch_taprio.c b/net/sched/sch_taprio.c index 8287894541e3..909c798b7403 100644 --- a/net/sched/sch_taprio.c +++ b/net/sched/sch_taprio.c @@ -901,6 +901,12 @@ static int parse_taprio_schedule(struct taprio_sched *q, struct nlattr **tb, list_for_each_entry(entry, &new->entries, list) cycle = ktime_add_ns(cycle, entry->interval); + + if (!cycle) { + NL_SET_ERR_MSG(extack, "'cycle_time' can never be 0"); + return -EINVAL; + } + new->cycle_time = cycle; } diff --git a/net/sctp/sm_make_chunk.c b/net/sctp/sm_make_chunk.c index f77484df097b..da4ce0947c3a 100644 --- a/net/sctp/sm_make_chunk.c +++ b/net/sctp/sm_make_chunk.c @@ -3147,7 +3147,7 @@ static __be16 sctp_process_asconf_param(struct sctp_association *asoc, * primary. */ if (af->is_any(&addr)) - memcpy(&addr.v4, sctp_source(asconf), sizeof(addr)); + memcpy(&addr, sctp_source(asconf), sizeof(addr)); if (security_sctp_bind_connect(asoc->ep->base.sk, SCTP_PARAM_SET_PRIMARY, diff --git a/net/sctp/sm_statefuns.c b/net/sctp/sm_statefuns.c index af2b7041fa4e..73bb4c6e9201 100644 --- a/net/sctp/sm_statefuns.c +++ b/net/sctp/sm_statefuns.c @@ -1852,20 +1852,35 @@ static enum sctp_disposition sctp_sf_do_dupcook_a( SCTP_TO(SCTP_EVENT_TIMEOUT_T4_RTO)); sctp_add_cmd_sf(commands, SCTP_CMD_PURGE_ASCONF_QUEUE, SCTP_NULL()); - repl = sctp_make_cookie_ack(new_asoc, chunk); + /* Update the content of current association. */ + if (sctp_assoc_update((struct sctp_association *)asoc, new_asoc)) { + struct sctp_chunk *abort; + + abort = sctp_make_abort(asoc, NULL, sizeof(struct sctp_errhdr)); + if (abort) { + sctp_init_cause(abort, SCTP_ERROR_RSRC_LOW, 0); + sctp_add_cmd_sf(commands, SCTP_CMD_REPLY, SCTP_CHUNK(abort)); + } + sctp_add_cmd_sf(commands, SCTP_CMD_SET_SK_ERR, SCTP_ERROR(ECONNABORTED)); + sctp_add_cmd_sf(commands, SCTP_CMD_ASSOC_FAILED, + SCTP_PERR(SCTP_ERROR_RSRC_LOW)); + SCTP_INC_STATS(net, SCTP_MIB_ABORTEDS); + SCTP_DEC_STATS(net, SCTP_MIB_CURRESTAB); + goto nomem; + } + + repl = sctp_make_cookie_ack(asoc, chunk); if (!repl) goto nomem; /* Report association restart to upper layer. */ ev = sctp_ulpevent_make_assoc_change(asoc, 0, SCTP_RESTART, 0, - new_asoc->c.sinit_num_ostreams, - new_asoc->c.sinit_max_instreams, + asoc->c.sinit_num_ostreams, + asoc->c.sinit_max_instreams, NULL, GFP_ATOMIC); if (!ev) goto nomem_ev; - /* Update the content of current association. */ - sctp_add_cmd_sf(commands, SCTP_CMD_UPDATE_ASSOC, SCTP_ASOC(new_asoc)); sctp_add_cmd_sf(commands, SCTP_CMD_EVENT_ULP, SCTP_ULPEVENT(ev)); if ((sctp_state(asoc, SHUTDOWN_PENDING) || sctp_state(asoc, SHUTDOWN_SENT)) && @@ -1929,7 +1944,8 @@ static enum sctp_disposition sctp_sf_do_dupcook_b( sctp_add_cmd_sf(commands, SCTP_CMD_UPDATE_ASSOC, SCTP_ASOC(new_asoc)); sctp_add_cmd_sf(commands, SCTP_CMD_NEW_STATE, SCTP_STATE(SCTP_STATE_ESTABLISHED)); - SCTP_INC_STATS(net, SCTP_MIB_CURRESTAB); + if (asoc->state < SCTP_STATE_ESTABLISHED) + SCTP_INC_STATS(net, SCTP_MIB_CURRESTAB); sctp_add_cmd_sf(commands, SCTP_CMD_HB_TIMERS_START, SCTP_NULL()); repl = sctp_make_cookie_ack(new_asoc, chunk); diff --git a/net/sctp/socket.c b/net/sctp/socket.c index b9b3d899a611..4ae428f2f2c5 100644 --- a/net/sctp/socket.c +++ b/net/sctp/socket.c @@ -357,6 +357,18 @@ static struct sctp_af *sctp_sockaddr_af(struct sctp_sock *opt, return af; } +static void sctp_auto_asconf_init(struct sctp_sock *sp) +{ + struct net *net = sock_net(&sp->inet.sk); + + if (net->sctp.default_auto_asconf) { + spin_lock(&net->sctp.addr_wq_lock); + list_add_tail(&sp->auto_asconf_list, &net->sctp.auto_asconf_splist); + spin_unlock(&net->sctp.addr_wq_lock); + sp->do_auto_asconf = 1; + } +} + /* Bind a local address either to an endpoint or to an association. */ static int sctp_do_bind(struct sock *sk, union sctp_addr *addr, int len) { @@ -418,8 +430,10 @@ static int sctp_do_bind(struct sock *sk, union sctp_addr *addr, int len) return -EADDRINUSE; /* Refresh ephemeral port. */ - if (!bp->port) + if (!bp->port) { bp->port = inet_sk(sk)->inet_num; + sctp_auto_asconf_init(sp); + } /* Add the address to the bind address list. * Use GFP_ATOMIC since BHs will be disabled. @@ -1520,9 +1534,11 @@ static void sctp_close(struct sock *sk, long timeout) /* Supposedly, no process has access to the socket, but * the net layers still may. + * Also, sctp_destroy_sock() needs to be called with addr_wq_lock + * held and that should be grabbed before socket lock. */ - local_bh_disable(); - bh_lock_sock(sk); + spin_lock_bh(&net->sctp.addr_wq_lock); + bh_lock_sock_nested(sk); /* Hold the sock, since sk_common_release() will put sock_put() * and we have just a little more cleanup. @@ -1531,7 +1547,7 @@ static void sctp_close(struct sock *sk, long timeout) sk_common_release(sk); bh_unlock_sock(sk); - local_bh_enable(); + spin_unlock_bh(&net->sctp.addr_wq_lock); sock_put(sk); @@ -4991,16 +5007,6 @@ static int sctp_init_sock(struct sock *sk) sk_sockets_allocated_inc(sk); sock_prot_inuse_add(net, sk->sk_prot, 1); - if (net->sctp.default_auto_asconf) { - spin_lock(&sock_net(sk)->sctp.addr_wq_lock); - list_add_tail(&sp->auto_asconf_list, - &net->sctp.auto_asconf_splist); - sp->do_auto_asconf = 1; - spin_unlock(&sock_net(sk)->sctp.addr_wq_lock); - } else { - sp->do_auto_asconf = 0; - } - local_bh_enable(); return 0; @@ -5025,9 +5031,7 @@ static void sctp_destroy_sock(struct sock *sk) if (sp->do_auto_asconf) { sp->do_auto_asconf = 0; - spin_lock_bh(&sock_net(sk)->sctp.addr_wq_lock); list_del(&sp->auto_asconf_list); - spin_unlock_bh(&sock_net(sk)->sctp.addr_wq_lock); } sctp_endpoint_free(sp->ep); local_bh_disable(); @@ -9398,6 +9402,8 @@ static int sctp_sock_migrate(struct sock *oldsk, struct sock *newsk, return err; } + sctp_auto_asconf_init(newsp); + /* Move any messages in the old socket's receive queue that are for the * peeled off association to the new socket's receive queue. */ diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c index 47340b3b514f..cb23cca72c24 100644 --- a/net/smc/af_smc.c +++ b/net/smc/af_smc.c @@ -2162,6 +2162,9 @@ static int smc_setsockopt(struct socket *sock, int level, int optname, struct smc_sock *smc; int val, rc; + if (level == SOL_TCP && optname == TCP_ULP) + return -EOPNOTSUPP; + smc = smc_sk(sk); /* generic setsockopts reaching us here always apply to the @@ -2186,7 +2189,6 @@ static int smc_setsockopt(struct socket *sock, int level, int optname, if (rc || smc->use_fallback) goto out; switch (optname) { - case TCP_ULP: case TCP_FASTOPEN: case TCP_FASTOPEN_CONNECT: case TCP_FASTOPEN_KEY: diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index 612f0a641f4c..f555d335e910 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -1799,7 +1799,6 @@ call_allocate(struct rpc_task *task) status = xprt->ops->buf_alloc(task); trace_rpc_buf_alloc(task, status); - xprt_inject_disconnect(xprt); if (status == 0) return; if (status != -ENOMEM) { @@ -2457,12 +2456,6 @@ call_decode(struct rpc_task *task) task->tk_flags &= ~RPC_CALL_MAJORSEEN; } - /* - * Ensure that we see all writes made by xprt_complete_rqst() - * before it changed req->rq_reply_bytes_recvd. - */ - smp_rmb(); - /* * Did we ever call xprt_complete_rqst()? If not, we should assume * the message is incomplete. @@ -2471,6 +2464,11 @@ call_decode(struct rpc_task *task) if (!req->rq_reply_bytes_recvd) goto out; + /* Ensure that we see all writes made by xprt_complete_rqst() + * before it changed req->rq_reply_bytes_recvd. + */ + smp_rmb(); + req->rq_rcv_buf.len = req->rq_private_buf.len; trace_rpc_xdr_recvfrom(task, &req->rq_rcv_buf); diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index d76dc9d95d16..0de918cb3d90 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -846,7 +846,8 @@ void svc_rqst_free(struct svc_rqst *rqstp) { svc_release_buffer(rqstp); - put_page(rqstp->rq_scratch_page); + if (rqstp->rq_scratch_page) + put_page(rqstp->rq_scratch_page); kfree(rqstp->rq_resp); kfree(rqstp->rq_argp); kfree(rqstp->rq_auth_data); diff --git a/net/sunrpc/svcsock.c b/net/sunrpc/svcsock.c index 2e2f007dfc9f..7cde41a936a4 100644 --- a/net/sunrpc/svcsock.c +++ b/net/sunrpc/svcsock.c @@ -1171,7 +1171,7 @@ static int svc_tcp_sendto(struct svc_rqst *rqstp) tcp_sock_set_cork(svsk->sk_sk, true); err = svc_tcp_sendmsg(svsk->sk_sock, xdr, marker, &sent); xdr_free_bvec(xdr); - trace_svcsock_tcp_send(xprt, err < 0 ? err : sent); + trace_svcsock_tcp_send(xprt, err < 0 ? (long)err : sent); if (err < 0 || sent != (xdr->len + sizeof(marker))) goto out_close; if (atomic_dec_and_test(&svsk->sk_sendqlen)) diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c index 691ccf8049a4..20fe31b1b776 100644 --- a/net/sunrpc/xprt.c +++ b/net/sunrpc/xprt.c @@ -698,9 +698,9 @@ int xprt_adjust_timeout(struct rpc_rqst *req) const struct rpc_timeout *to = req->rq_task->tk_client->cl_timeout; int status = 0; - if (time_before(jiffies, req->rq_minortimeo)) - return status; if (time_before(jiffies, req->rq_majortimeo)) { + if (time_before(jiffies, req->rq_minortimeo)) + return status; if (to->to_exponential) req->rq_timeout <<= 1; else @@ -1469,8 +1469,6 @@ bool xprt_prepare_transmit(struct rpc_task *task) struct rpc_xprt *xprt = req->rq_xprt; if (!xprt_lock_write(xprt, task)) { - trace_xprt_transmit_queued(xprt, task); - /* Race breaker: someone may have transmitted us */ if (!test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate)) rpc_wake_up_queued_task_set_status(&xprt->sending, @@ -1483,7 +1481,10 @@ bool xprt_prepare_transmit(struct rpc_task *task) void xprt_end_transmit(struct rpc_task *task) { - xprt_release_write(task->tk_rqstp->rq_xprt, task); + struct rpc_xprt *xprt = task->tk_rqstp->rq_xprt; + + xprt_inject_disconnect(xprt); + xprt_release_write(xprt, task); } /** @@ -1885,7 +1886,6 @@ void xprt_release(struct rpc_task *task) spin_unlock(&xprt->transport_lock); if (req->rq_buffer) xprt->ops->buf_free(task); - xprt_inject_disconnect(xprt); xdr_free_bvec(&req->rq_rcv_buf); xdr_free_bvec(&req->rq_snd_buf); if (req->rq_cred != NULL) diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c index 766a1048a48a..aca2228095db 100644 --- a/net/sunrpc/xprtrdma/frwr_ops.c +++ b/net/sunrpc/xprtrdma/frwr_ops.c @@ -257,6 +257,7 @@ int frwr_query_device(struct rpcrdma_ep *ep, const struct ib_device *device) ep->re_attr.cap.max_send_wr += 1; /* for ib_drain_sq */ ep->re_attr.cap.max_recv_wr = ep->re_max_requests; ep->re_attr.cap.max_recv_wr += RPCRDMA_BACKWARD_WRS; + ep->re_attr.cap.max_recv_wr += RPCRDMA_MAX_RECV_BATCH; ep->re_attr.cap.max_recv_wr += 1; /* for ib_drain_rq */ ep->re_max_rdma_segs = @@ -575,7 +576,6 @@ void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) mr = container_of(frwr, struct rpcrdma_mr, frwr); bad_wr = bad_wr->next; - list_del_init(&mr->mr_list); frwr_mr_recycle(mr); } } diff --git a/net/sunrpc/xprtrdma/rpc_rdma.c b/net/sunrpc/xprtrdma/rpc_rdma.c index 292f066d006e..21ddd78a8c35 100644 --- a/net/sunrpc/xprtrdma/rpc_rdma.c +++ b/net/sunrpc/xprtrdma/rpc_rdma.c @@ -1430,9 +1430,10 @@ void rpcrdma_reply_handler(struct rpcrdma_rep *rep) credits = 1; /* don't deadlock */ else if (credits > r_xprt->rx_ep->re_max_requests) credits = r_xprt->rx_ep->re_max_requests; + rpcrdma_post_recvs(r_xprt, credits + (buf->rb_bc_srv_max_requests << 1), + false); if (buf->rb_credits != credits) rpcrdma_update_cwnd(r_xprt, credits); - rpcrdma_post_recvs(r_xprt, false); req = rpcr_to_rdmar(rqst); if (unlikely(req->rl_reply)) diff --git a/net/sunrpc/xprtrdma/svc_rdma_sendto.c b/net/sunrpc/xprtrdma/svc_rdma_sendto.c index 52c759a8543e..3669661457c1 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_sendto.c +++ b/net/sunrpc/xprtrdma/svc_rdma_sendto.c @@ -958,7 +958,7 @@ int svc_rdma_sendto(struct svc_rqst *rqstp) p = xdr_reserve_space(&sctxt->sc_stream, rpcrdma_fixed_maxsz * sizeof(*p)); if (!p) - goto err0; + goto err1; ret = svc_rdma_send_reply_chunk(rdma, rctxt, &rqstp->rq_res); if (ret < 0) @@ -970,11 +970,11 @@ int svc_rdma_sendto(struct svc_rqst *rqstp) *p = pcl_is_empty(&rctxt->rc_reply_pcl) ? rdma_msg : rdma_nomsg; if (svc_rdma_encode_read_list(sctxt) < 0) - goto err0; + goto err1; if (svc_rdma_encode_write_list(rctxt, sctxt) < 0) - goto err0; + goto err1; if (svc_rdma_encode_reply_chunk(rctxt, sctxt, ret) < 0) - goto err0; + goto err1; ret = svc_rdma_send_reply_msg(rdma, sctxt, rctxt, rqstp); if (ret < 0) diff --git a/net/sunrpc/xprtrdma/transport.c b/net/sunrpc/xprtrdma/transport.c index 78d29d1bcc20..09953597d055 100644 --- a/net/sunrpc/xprtrdma/transport.c +++ b/net/sunrpc/xprtrdma/transport.c @@ -262,8 +262,10 @@ xprt_rdma_connect_worker(struct work_struct *work) * xprt_rdma_inject_disconnect - inject a connection fault * @xprt: transport context * - * If @xprt is connected, disconnect it to simulate spurious connection - * loss. + * If @xprt is connected, disconnect it to simulate spurious + * connection loss. Caller must hold @xprt's send lock to + * ensure that data structures and hardware resources are + * stable during the rdma_disconnect() call. */ static void xprt_rdma_inject_disconnect(struct rpc_xprt *xprt) diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index ec912cf9c618..f3fffc74ab0f 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -535,7 +535,7 @@ int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt) * outstanding Receives. */ rpcrdma_ep_get(ep); - rpcrdma_post_recvs(r_xprt, true); + rpcrdma_post_recvs(r_xprt, 1, true); rc = rdma_connect(ep->re_id, &ep->re_remote_cma); if (rc) @@ -1364,21 +1364,21 @@ int rpcrdma_post_sends(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) /** * rpcrdma_post_recvs - Refill the Receive Queue * @r_xprt: controlling transport instance - * @temp: mark Receive buffers to be deleted after use + * @needed: current credit grant + * @temp: mark Receive buffers to be deleted after one use * */ -void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp) +void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed, bool temp) { struct rpcrdma_buffer *buf = &r_xprt->rx_buf; struct rpcrdma_ep *ep = r_xprt->rx_ep; struct ib_recv_wr *wr, *bad_wr; struct rpcrdma_rep *rep; - int needed, count, rc; + int count, rc; rc = 0; count = 0; - needed = buf->rb_credits + (buf->rb_bc_srv_max_requests << 1); if (likely(ep->re_receive_count > needed)) goto out; needed -= ep->re_receive_count; diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h index fe3be985e239..28af11fbe643 100644 --- a/net/sunrpc/xprtrdma/xprt_rdma.h +++ b/net/sunrpc/xprtrdma/xprt_rdma.h @@ -461,7 +461,7 @@ int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt); void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt); int rpcrdma_post_sends(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req); -void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp); +void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed, bool temp); /* * Buffer calls - xprtrdma/verbs.c diff --git a/net/tipc/crypto.c b/net/tipc/crypto.c index 97710ce36047..c89ce47c56cf 100644 --- a/net/tipc/crypto.c +++ b/net/tipc/crypto.c @@ -1492,6 +1492,8 @@ int tipc_crypto_start(struct tipc_crypto **crypto, struct net *net, /* Allocate statistic structure */ c->stats = alloc_percpu_gfp(struct tipc_crypto_stats, GFP_ATOMIC); if (!c->stats) { + if (c->wq) + destroy_workqueue(c->wq); kfree_sensitive(c); return -ENOMEM; } diff --git a/net/tipc/netlink_compat.c b/net/tipc/netlink_compat.c index 5a1ce64039f7..0749df80454d 100644 --- a/net/tipc/netlink_compat.c +++ b/net/tipc/netlink_compat.c @@ -696,7 +696,7 @@ static int tipc_nl_compat_link_dump(struct tipc_nl_compat_msg *msg, if (err) return err; - link_info.dest = nla_get_flag(link[TIPC_NLA_LINK_DEST]); + link_info.dest = htonl(nla_get_flag(link[TIPC_NLA_LINK_DEST])); link_info.up = htonl(nla_get_flag(link[TIPC_NLA_LINK_UP])); nla_strscpy(link_info.str, link[TIPC_NLA_LINK_NAME], TIPC_MAX_LINK_NAME); diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c index e4370b1b7494..902cb6dd710b 100644 --- a/net/vmw_vsock/virtio_transport_common.c +++ b/net/vmw_vsock/virtio_transport_common.c @@ -733,6 +733,23 @@ static int virtio_transport_reset_no_sock(const struct virtio_transport *t, return t->send_pkt(reply); } +/* This function should be called with sk_lock held and SOCK_DONE set */ +static void virtio_transport_remove_sock(struct vsock_sock *vsk) +{ + struct virtio_vsock_sock *vvs = vsk->trans; + struct virtio_vsock_pkt *pkt, *tmp; + + /* We don't need to take rx_lock, as the socket is closing and we are + * removing it. + */ + list_for_each_entry_safe(pkt, tmp, &vvs->rx_queue, list) { + list_del(&pkt->list); + virtio_transport_free_pkt(pkt); + } + + vsock_remove_sock(vsk); +} + static void virtio_transport_wait_close(struct sock *sk, long timeout) { if (timeout) { @@ -765,7 +782,7 @@ static void virtio_transport_do_close(struct vsock_sock *vsk, (!cancel_timeout || cancel_delayed_work(&vsk->close_work))) { vsk->close_work_scheduled = false; - vsock_remove_sock(vsk); + virtio_transport_remove_sock(vsk); /* Release refcnt obtained when we scheduled the timeout */ sock_put(sk); @@ -828,22 +845,15 @@ static bool virtio_transport_close(struct vsock_sock *vsk) void virtio_transport_release(struct vsock_sock *vsk) { - struct virtio_vsock_sock *vvs = vsk->trans; - struct virtio_vsock_pkt *pkt, *tmp; struct sock *sk = &vsk->sk; bool remove_sock = true; if (sk->sk_type == SOCK_STREAM) remove_sock = virtio_transport_close(vsk); - list_for_each_entry_safe(pkt, tmp, &vvs->rx_queue, list) { - list_del(&pkt->list); - virtio_transport_free_pkt(pkt); - } - if (remove_sock) { sock_set_flag(sk, SOCK_DONE); - vsock_remove_sock(vsk); + virtio_transport_remove_sock(vsk); } } EXPORT_SYMBOL_GPL(virtio_transport_release); diff --git a/net/vmw_vsock/vmci_transport.c b/net/vmw_vsock/vmci_transport.c index 8b65323207db..1c9ecb18b8e6 100644 --- a/net/vmw_vsock/vmci_transport.c +++ b/net/vmw_vsock/vmci_transport.c @@ -568,8 +568,7 @@ vmci_transport_queue_pair_alloc(struct vmci_qp **qpair, peer, flags, VMCI_NO_PRIVILEGE_FLAGS); out: if (err < 0) { - pr_err("Could not attach to queue pair with %d\n", - err); + pr_err_once("Could not attach to queue pair with %d\n", err); err = vmci_transport_error_to_vsock_error(err); } diff --git a/net/wireless/core.c b/net/wireless/core.c index a2785379df6e..589ee5a69a2e 100644 --- a/net/wireless/core.c +++ b/net/wireless/core.c @@ -332,14 +332,29 @@ static void cfg80211_event_work(struct work_struct *work) void cfg80211_destroy_ifaces(struct cfg80211_registered_device *rdev) { struct wireless_dev *wdev, *tmp; + bool found = false; ASSERT_RTNL(); - lockdep_assert_wiphy(&rdev->wiphy); + list_for_each_entry(wdev, &rdev->wiphy.wdev_list, list) { + if (wdev->nl_owner_dead) { + if (wdev->netdev) + dev_close(wdev->netdev); + found = true; + } + } + + if (!found) + return; + + wiphy_lock(&rdev->wiphy); list_for_each_entry_safe(wdev, tmp, &rdev->wiphy.wdev_list, list) { - if (wdev->nl_owner_dead) + if (wdev->nl_owner_dead) { + cfg80211_leave(rdev, wdev); rdev_del_virtual_intf(rdev, wdev); + } } + wiphy_unlock(&rdev->wiphy); } static void cfg80211_destroy_iface_wk(struct work_struct *work) @@ -350,9 +365,7 @@ static void cfg80211_destroy_iface_wk(struct work_struct *work) destroy_work); rtnl_lock(); - wiphy_lock(&rdev->wiphy); cfg80211_destroy_ifaces(rdev); - wiphy_unlock(&rdev->wiphy); rtnl_unlock(); } diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index b1df42e4f1eb..a5224da63832 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -3929,7 +3929,7 @@ static int nl80211_set_interface(struct sk_buff *skb, struct genl_info *info) return err; } -static int nl80211_new_interface(struct sk_buff *skb, struct genl_info *info) +static int _nl80211_new_interface(struct sk_buff *skb, struct genl_info *info) { struct cfg80211_registered_device *rdev = info->user_ptr[0]; struct vif_params params; @@ -3938,9 +3938,6 @@ static int nl80211_new_interface(struct sk_buff *skb, struct genl_info *info) int err; enum nl80211_iftype type = NL80211_IFTYPE_UNSPECIFIED; - /* to avoid failing a new interface creation due to pending removal */ - cfg80211_destroy_ifaces(rdev); - memset(¶ms, 0, sizeof(params)); if (!info->attrs[NL80211_ATTR_IFNAME]) @@ -4028,6 +4025,21 @@ static int nl80211_new_interface(struct sk_buff *skb, struct genl_info *info) return genlmsg_reply(msg, info); } +static int nl80211_new_interface(struct sk_buff *skb, struct genl_info *info) +{ + struct cfg80211_registered_device *rdev = info->user_ptr[0]; + int ret; + + /* to avoid failing a new interface creation due to pending removal */ + cfg80211_destroy_ifaces(rdev); + + wiphy_lock(&rdev->wiphy); + ret = _nl80211_new_interface(skb, info); + wiphy_unlock(&rdev->wiphy); + + return ret; +} + static int nl80211_del_interface(struct sk_buff *skb, struct genl_info *info) { struct cfg80211_registered_device *rdev = info->user_ptr[0]; @@ -15040,7 +15052,9 @@ static const struct genl_small_ops nl80211_small_ops[] = { .doit = nl80211_new_interface, .flags = GENL_UNS_ADMIN_PERM, .internal_flags = NL80211_FLAG_NEED_WIPHY | - NL80211_FLAG_NEED_RTNL, + NL80211_FLAG_NEED_RTNL | + /* we take the wiphy mutex later ourselves */ + NL80211_FLAG_NO_WIPHY_MTX, }, { .cmd = NL80211_CMD_DEL_INTERFACE, diff --git a/net/wireless/scan.c b/net/wireless/scan.c index 758eb7d2a706..caa8eafbd583 100644 --- a/net/wireless/scan.c +++ b/net/wireless/scan.c @@ -1751,6 +1751,8 @@ cfg80211_bss_update(struct cfg80211_registered_device *rdev, if (rdev->bss_entries >= bss_entries_limit && !cfg80211_bss_expire_oldest(rdev)) { + if (!list_empty(&new->hidden_list)) + list_del(&new->hidden_list); kfree(new); goto drop; } diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index 4faabd1ecfd1..143979ea4165 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -454,12 +454,16 @@ static int xsk_generic_xmit(struct sock *sk) struct sk_buff *skb; unsigned long flags; int err = 0; + u32 hr, tr; mutex_lock(&xs->mutex); if (xs->queue_id >= xs->dev->real_num_tx_queues) goto out; + hr = max(NET_SKB_PAD, L1_CACHE_ALIGN(xs->dev->needed_headroom)); + tr = xs->dev->needed_tailroom; + while (xskq_cons_peek_desc(xs->tx, &desc, xs->pool)) { char *buffer; u64 addr; @@ -471,11 +475,13 @@ static int xsk_generic_xmit(struct sock *sk) } len = desc.len; - skb = sock_alloc_send_skb(sk, len, 1, &err); + skb = sock_alloc_send_skb(sk, hr + len + tr, 1, &err); if (unlikely(!skb)) goto out; + skb_reserve(skb, hr); skb_put(skb, len); + addr = desc.addr; buffer = xsk_buff_raw_get_data(xs->pool, addr); err = skb_store_bits(skb, 0, buffer, len); diff --git a/net/xdp/xsk_queue.h b/net/xdp/xsk_queue.h index 2823b7c3302d..40f359bf2044 100644 --- a/net/xdp/xsk_queue.h +++ b/net/xdp/xsk_queue.h @@ -128,13 +128,12 @@ static inline bool xskq_cons_read_addr_unchecked(struct xsk_queue *q, u64 *addr) static inline bool xp_aligned_validate_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc) { - u64 chunk, chunk_end; + u64 chunk; - chunk = xp_aligned_extract_addr(pool, desc->addr); - chunk_end = xp_aligned_extract_addr(pool, desc->addr + desc->len); - if (chunk != chunk_end) + if (desc->len > pool->chunk_size) return false; + chunk = xp_aligned_extract_addr(pool, desc->addr); if (chunk >= pool->addrs_cnt) return false; diff --git a/samples/bpf/tracex1_kern.c b/samples/bpf/tracex1_kern.c index 3f4599c9a202..ef30d2b353b0 100644 --- a/samples/bpf/tracex1_kern.c +++ b/samples/bpf/tracex1_kern.c @@ -26,7 +26,7 @@ SEC("kprobe/__netif_receive_skb_core") int bpf_prog1(struct pt_regs *ctx) { - /* attaches to kprobe netif_receive_skb, + /* attaches to kprobe __netif_receive_skb_core, * looks for packets on loobpack device and prints them */ char devname[IFNAMSIZ]; @@ -35,7 +35,7 @@ int bpf_prog1(struct pt_regs *ctx) int len; /* non-portable! works for the given kernel only */ - skb = (struct sk_buff *) PT_REGS_PARM1(ctx); + bpf_probe_read_kernel(&skb, sizeof(skb), (void *)PT_REGS_PARM1(ctx)); dev = _(skb->dev); len = _(skb->len); diff --git a/samples/kfifo/bytestream-example.c b/samples/kfifo/bytestream-example.c index c406f03ee551..5a90aa527877 100644 --- a/samples/kfifo/bytestream-example.c +++ b/samples/kfifo/bytestream-example.c @@ -122,8 +122,10 @@ static ssize_t fifo_write(struct file *file, const char __user *buf, ret = kfifo_from_user(&test, buf, count, &copied); mutex_unlock(&write_lock); + if (ret) + return ret; - return ret ? ret : copied; + return copied; } static ssize_t fifo_read(struct file *file, char __user *buf, @@ -138,8 +140,10 @@ static ssize_t fifo_read(struct file *file, char __user *buf, ret = kfifo_to_user(&test, buf, count, &copied); mutex_unlock(&read_lock); + if (ret) + return ret; - return ret ? ret : copied; + return copied; } static const struct proc_ops fifo_proc_ops = { diff --git a/samples/kfifo/inttype-example.c b/samples/kfifo/inttype-example.c index 78977fc4a23f..e5403d8c971a 100644 --- a/samples/kfifo/inttype-example.c +++ b/samples/kfifo/inttype-example.c @@ -115,8 +115,10 @@ static ssize_t fifo_write(struct file *file, const char __user *buf, ret = kfifo_from_user(&test, buf, count, &copied); mutex_unlock(&write_lock); + if (ret) + return ret; - return ret ? ret : copied; + return copied; } static ssize_t fifo_read(struct file *file, char __user *buf, @@ -131,8 +133,10 @@ static ssize_t fifo_read(struct file *file, char __user *buf, ret = kfifo_to_user(&test, buf, count, &copied); mutex_unlock(&read_lock); + if (ret) + return ret; - return ret ? ret : copied; + return copied; } static const struct proc_ops fifo_proc_ops = { diff --git a/samples/kfifo/record-example.c b/samples/kfifo/record-example.c index c507998a2617..f64f3d62d6c2 100644 --- a/samples/kfifo/record-example.c +++ b/samples/kfifo/record-example.c @@ -129,8 +129,10 @@ static ssize_t fifo_write(struct file *file, const char __user *buf, ret = kfifo_from_user(&test, buf, count, &copied); mutex_unlock(&write_lock); + if (ret) + return ret; - return ret ? ret : copied; + return copied; } static ssize_t fifo_read(struct file *file, char __user *buf, @@ -145,8 +147,10 @@ static ssize_t fifo_read(struct file *file, char __user *buf, ret = kfifo_to_user(&test, buf, count, &copied); mutex_unlock(&read_lock); + if (ret) + return ret; - return ret ? ret : copied; + return copied; } static const struct proc_ops fifo_proc_ops = { diff --git a/scripts/Makefile.modpost b/scripts/Makefile.modpost index 066beffca09a..4ca5579af4e4 100644 --- a/scripts/Makefile.modpost +++ b/scripts/Makefile.modpost @@ -68,7 +68,20 @@ else ifeq ($(KBUILD_EXTMOD),) input-symdump := vmlinux.symvers -output-symdump := Module.symvers +output-symdump := modules-only.symvers + +quiet_cmd_cat = GEN $@ + cmd_cat = cat $(real-prereqs) > $@ + +ifneq ($(wildcard vmlinux.symvers),) + +__modpost: Module.symvers +Module.symvers: vmlinux.symvers modules-only.symvers FORCE + $(call if_changed,cat) + +targets += Module.symvers + +endif else diff --git a/scripts/kconfig/nconf.c b/scripts/kconfig/nconf.c index e0f965529166..af814b39b876 100644 --- a/scripts/kconfig/nconf.c +++ b/scripts/kconfig/nconf.c @@ -504,8 +504,8 @@ static int get_mext_match(const char *match_str, match_f flag) else if (flag == FIND_NEXT_MATCH_UP) --match_start; + match_start = (match_start + items_num) % items_num; index = match_start; - index = (index + items_num) % items_num; while (true) { char *str = k_menu_items[index].str; if (strcasestr(str, match_str) != NULL) diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index 24725e50c7b4..10c3fba26f03 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -2423,19 +2423,6 @@ static void read_dump(const char *fname) fatal("parse error in symbol dump file\n"); } -/* For normal builds always dump all symbols. - * For external modules only dump symbols - * that are not read from kernel Module.symvers. - **/ -static int dump_sym(struct symbol *sym) -{ - if (!external_module) - return 1; - if (sym->module->from_dump) - return 0; - return 1; -} - static void write_dump(const char *fname) { struct buffer buf = { }; @@ -2446,7 +2433,7 @@ static void write_dump(const char *fname) for (n = 0; n < SYMBOL_HASH_SIZE ; n++) { symbol = symbolhash[n]; while (symbol) { - if (dump_sym(symbol)) { + if (!symbol->module->from_dump) { namespace = symbol->namespace; buf_printf(&buf, "0x%08x\t%s\t%s\t%s\t%s\n", symbol->crc, symbol->name, diff --git a/scripts/recordmcount.pl b/scripts/recordmcount.pl index 867860ea57da..7b83a1aaec98 100755 --- a/scripts/recordmcount.pl +++ b/scripts/recordmcount.pl @@ -392,7 +392,7 @@ sub check_objcopy $mcount_regex = "^\\s*([0-9a-fA-F]+):.*\\s_mcount\$"; } elsif ($arch eq "riscv") { $function_regex = "^([0-9a-fA-F]+)\\s+<([^.0-9][0-9a-zA-Z_\\.]+)>:"; - $mcount_regex = "^\\s*([0-9a-fA-F]+):\\sR_RISCV_CALL\\s_mcount\$"; + $mcount_regex = "^\\s*([0-9a-fA-F]+):\\sR_RISCV_CALL(_PLT)?\\s_?mcount\$"; $type = ".quad"; $alignment = 2; } elsif ($arch eq "nds32") { diff --git a/security/commoncap.c b/security/commoncap.c index 1c519c875217..5cdeb73ca8fa 100644 --- a/security/commoncap.c +++ b/security/commoncap.c @@ -400,7 +400,7 @@ int cap_inode_getsecurity(struct user_namespace *mnt_userns, &tmpbuf, size, GFP_NOFS); dput(dentry); - if (ret < 0) + if (ret < 0 || !tmpbuf) return ret; fs_ns = inode->i_sb->s_user_ns; diff --git a/security/integrity/ima/ima_template.c b/security/integrity/ima/ima_template.c index e22e510ae92d..4e081e650047 100644 --- a/security/integrity/ima/ima_template.c +++ b/security/integrity/ima/ima_template.c @@ -494,8 +494,8 @@ int ima_restore_measurement_list(loff_t size, void *buf) } } - entry->pcr = !ima_canonical_fmt ? *(hdr[HDR_PCR].data) : - le32_to_cpu(*(hdr[HDR_PCR].data)); + entry->pcr = !ima_canonical_fmt ? *(u32 *)(hdr[HDR_PCR].data) : + le32_to_cpu(*(u32 *)(hdr[HDR_PCR].data)); ret = ima_restore_measurement_entry(entry); if (ret < 0) break; diff --git a/security/keys/trusted-keys/trusted_tpm1.c b/security/keys/trusted-keys/trusted_tpm1.c index 493eb91ed017..56c9b48460d9 100644 --- a/security/keys/trusted-keys/trusted_tpm1.c +++ b/security/keys/trusted-keys/trusted_tpm1.c @@ -500,10 +500,12 @@ static int tpm_seal(struct tpm_buf *tb, uint16_t keytype, ret = tpm_get_random(chip, td->nonceodd, TPM_NONCE_SIZE); if (ret < 0) - return ret; + goto out; - if (ret != TPM_NONCE_SIZE) - return -EIO; + if (ret != TPM_NONCE_SIZE) { + ret = -EIO; + goto out; + } ordinal = htonl(TPM_ORD_SEAL); datsize = htonl(datalen); @@ -791,13 +793,33 @@ static int getoptions(char *c, struct trusted_key_payload *pay, return -EINVAL; break; case Opt_blobauth: - if (strlen(args[0].from) != 2 * SHA1_DIGEST_SIZE) - return -EINVAL; - res = hex2bin(opt->blobauth, args[0].from, - SHA1_DIGEST_SIZE); - if (res < 0) - return -EINVAL; + /* + * TPM 1.2 authorizations are sha1 hashes passed in as + * hex strings. TPM 2.0 authorizations are simple + * passwords (although it can take a hash as well) + */ + opt->blobauth_len = strlen(args[0].from); + + if (opt->blobauth_len == 2 * TPM_DIGEST_SIZE) { + res = hex2bin(opt->blobauth, args[0].from, + TPM_DIGEST_SIZE); + if (res < 0) + return -EINVAL; + + opt->blobauth_len = TPM_DIGEST_SIZE; + break; + } + + if (tpm2 && opt->blobauth_len <= sizeof(opt->blobauth)) { + memcpy(opt->blobauth, args[0].from, + opt->blobauth_len); + break; + } + + return -EINVAL; + break; + case Opt_migratable: if (*args[0].from == '0') pay->migratable = 0; diff --git a/security/keys/trusted-keys/trusted_tpm2.c b/security/keys/trusted-keys/trusted_tpm2.c index c87c4df8703d..4c19d3abddbe 100644 --- a/security/keys/trusted-keys/trusted_tpm2.c +++ b/security/keys/trusted-keys/trusted_tpm2.c @@ -97,10 +97,12 @@ int tpm2_seal_trusted(struct tpm_chip *chip, TPM_DIGEST_SIZE); /* sensitive */ - tpm_buf_append_u16(&buf, 4 + TPM_DIGEST_SIZE + payload->key_len + 1); + tpm_buf_append_u16(&buf, 4 + options->blobauth_len + payload->key_len + 1); + + tpm_buf_append_u16(&buf, options->blobauth_len); + if (options->blobauth_len) + tpm_buf_append(&buf, options->blobauth, options->blobauth_len); - tpm_buf_append_u16(&buf, TPM_DIGEST_SIZE); - tpm_buf_append(&buf, options->blobauth, TPM_DIGEST_SIZE); tpm_buf_append_u16(&buf, payload->key_len + 1); tpm_buf_append(&buf, payload->key, payload->key_len); tpm_buf_append_u8(&buf, payload->migratable); @@ -265,7 +267,7 @@ static int tpm2_unseal_cmd(struct tpm_chip *chip, NULL /* nonce */, 0, TPM2_SA_CONTINUE_SESSION, options->blobauth /* hmac */, - TPM_DIGEST_SIZE); + options->blobauth_len); rc = tpm_transmit_cmd(chip, &buf, 6, "unsealing"); if (rc > 0) diff --git a/security/selinux/include/classmap.h b/security/selinux/include/classmap.h index ba2e01a6955c..62d19bccf3de 100644 --- a/security/selinux/include/classmap.h +++ b/security/selinux/include/classmap.h @@ -242,11 +242,12 @@ struct security_class_mapping secclass_map[] = { { "infiniband_endport", { "manage_subnet", NULL } }, { "bpf", - {"map_create", "map_read", "map_write", "prog_load", "prog_run"} }, + { "map_create", "map_read", "map_write", "prog_load", "prog_run", + NULL } }, { "xdp_socket", { COMMON_SOCK_PERMS, NULL } }, { "perf_event", - {"open", "cpu", "kernel", "tracepoint", "read", "write"} }, + { "open", "cpu", "kernel", "tracepoint", "read", "write", NULL } }, { "lockdown", { "integrity", "confidentiality", NULL } }, { "anon_inode", diff --git a/sound/core/init.c b/sound/core/init.c index 45f4b01de23f..ef41f5b3a240 100644 --- a/sound/core/init.c +++ b/sound/core/init.c @@ -398,10 +398,8 @@ int snd_card_disconnect(struct snd_card *card) return 0; } card->shutdown = 1; - spin_unlock(&card->files_lock); /* replace file->f_op with special dummy operations */ - spin_lock(&card->files_lock); list_for_each_entry(mfile, &card->files_list, list) { /* it's critical part, use endless loop */ /* we have no room to fail */ diff --git a/sound/firewire/bebob/bebob_stream.c b/sound/firewire/bebob/bebob_stream.c index bbae04793c50..c18017e0a3d9 100644 --- a/sound/firewire/bebob/bebob_stream.c +++ b/sound/firewire/bebob/bebob_stream.c @@ -517,20 +517,22 @@ int snd_bebob_stream_init_duplex(struct snd_bebob *bebob) static int keep_resources(struct snd_bebob *bebob, struct amdtp_stream *stream, unsigned int rate, unsigned int index) { - struct snd_bebob_stream_formation *formation; + unsigned int pcm_channels; + unsigned int midi_ports; struct cmp_connection *conn; int err; if (stream == &bebob->tx_stream) { - formation = bebob->tx_stream_formations + index; + pcm_channels = bebob->tx_stream_formations[index].pcm; + midi_ports = bebob->midi_input_ports; conn = &bebob->out_conn; } else { - formation = bebob->rx_stream_formations + index; + pcm_channels = bebob->rx_stream_formations[index].pcm; + midi_ports = bebob->midi_output_ports; conn = &bebob->in_conn; } - err = amdtp_am824_set_parameters(stream, rate, formation->pcm, - formation->midi, false); + err = amdtp_am824_set_parameters(stream, rate, pcm_channels, midi_ports, false); if (err < 0) return err; diff --git a/sound/isa/sb/emu8000.c b/sound/isa/sb/emu8000.c index 0aa545ac6e60..1c90421a88dc 100644 --- a/sound/isa/sb/emu8000.c +++ b/sound/isa/sb/emu8000.c @@ -1029,8 +1029,10 @@ snd_emu8000_create_mixer(struct snd_card *card, struct snd_emu8000 *emu) memset(emu->controls, 0, sizeof(emu->controls)); for (i = 0; i < EMU8000_NUM_CONTROLS; i++) { - if ((err = snd_ctl_add(card, emu->controls[i] = snd_ctl_new1(mixer_defs[i], emu))) < 0) + if ((err = snd_ctl_add(card, emu->controls[i] = snd_ctl_new1(mixer_defs[i], emu))) < 0) { + emu->controls[i] = NULL; goto __error; + } } return 0; diff --git a/sound/isa/sb/sb16_csp.c b/sound/isa/sb/sb16_csp.c index 8635a2b6b36b..4789345a8fdd 100644 --- a/sound/isa/sb/sb16_csp.c +++ b/sound/isa/sb/sb16_csp.c @@ -1045,10 +1045,14 @@ static int snd_sb_qsound_build(struct snd_sb_csp * p) spin_lock_init(&p->q_lock); - if ((err = snd_ctl_add(card, p->qsound_switch = snd_ctl_new1(&snd_sb_qsound_switch, p))) < 0) + if ((err = snd_ctl_add(card, p->qsound_switch = snd_ctl_new1(&snd_sb_qsound_switch, p))) < 0) { + p->qsound_switch = NULL; goto __error; - if ((err = snd_ctl_add(card, p->qsound_space = snd_ctl_new1(&snd_sb_qsound_space, p))) < 0) + } + if ((err = snd_ctl_add(card, p->qsound_space = snd_ctl_new1(&snd_sb_qsound_space, p))) < 0) { + p->qsound_space = NULL; goto __error; + } return 0; diff --git a/sound/pci/hda/hda_generic.c b/sound/pci/hda/hda_generic.c index f5cba7afd1c6..ff0fb2d16d82 100644 --- a/sound/pci/hda/hda_generic.c +++ b/sound/pci/hda/hda_generic.c @@ -1202,11 +1202,17 @@ static const char *get_line_out_pfx(struct hda_codec *codec, int ch, *index = ch; return "Headphone"; case AUTO_PIN_LINE_OUT: - /* This deals with the case where we have two DACs and - * one LO, one HP and one Speaker */ - if (!ch && cfg->speaker_outs && cfg->hp_outs) { - bool hp_lo_shared = !path_has_mixer(codec, spec->hp_paths[0], ctl_type); - bool spk_lo_shared = !path_has_mixer(codec, spec->speaker_paths[0], ctl_type); + /* This deals with the case where one HP or one Speaker or + * one HP + one Speaker need to share the DAC with LO + */ + if (!ch) { + bool hp_lo_shared = false, spk_lo_shared = false; + + if (cfg->speaker_outs) + spk_lo_shared = !path_has_mixer(codec, + spec->speaker_paths[0], ctl_type); + if (cfg->hp_outs) + hp_lo_shared = !path_has_mixer(codec, spec->hp_paths[0], ctl_type); if (hp_lo_shared && spk_lo_shared) return spec->vmaster_mute.hook ? "PCM" : "Master"; if (hp_lo_shared) diff --git a/sound/pci/hda/ideapad_s740_helper.c b/sound/pci/hda/ideapad_s740_helper.c new file mode 100644 index 000000000000..564b9086e52d --- /dev/null +++ b/sound/pci/hda/ideapad_s740_helper.c @@ -0,0 +1,492 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Fixes for Lenovo Ideapad S740, to be included from codec driver */ + +static const struct hda_verb alc285_ideapad_s740_coefs[] = { +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x10 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0320 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x24 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0041 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x24 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0041 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x007f }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x007f }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0001 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0001 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0001 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0001 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x003c }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0011 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x003c }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0011 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x000c }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x001a }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x000c }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x001a }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x000f }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0042 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x000f }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0042 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0010 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0040 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0010 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0040 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0003 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0009 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0003 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0009 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x001c }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x004c }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x001c }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x004c }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x001d }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x004e }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x001d }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x004e }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x001b }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0001 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x001b }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0001 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0019 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0025 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0019 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0025 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0018 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0037 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0018 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0037 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x001a }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0040 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x001a }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0040 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0016 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0076 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0016 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0076 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0017 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0010 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0017 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0010 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0015 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0015 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0015 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0015 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0007 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0086 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0007 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0086 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0002 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0001 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0002 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0001 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0002 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0002 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x24 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0042 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x24 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0042 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x007f }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x007f }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0001 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0001 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0001 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0001 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x003c }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0011 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x003c }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0011 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x000c }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x002a }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x000c }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x002a }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x000f }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0046 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x000f }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0046 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0010 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0044 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0010 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0044 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0003 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0009 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0003 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0009 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x001c }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x004c }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x001c }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x004c }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x001b }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0001 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x001b }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0001 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0019 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0025 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0019 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0025 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0018 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0037 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0018 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0037 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x001a }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0040 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x001a }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0040 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0016 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0076 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0016 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0076 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0017 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0010 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0017 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0010 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0015 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0015 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0015 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0015 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0007 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0086 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0007 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0086 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0002 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0001 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0002 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0001 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x29 }, +{ 0x20, AC_VERB_SET_COEF_INDEX, 0x26 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0002 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0x0000 }, +{ 0x20, AC_VERB_SET_PROC_COEF, 0xb020 }, +{} +}; + +static void alc285_fixup_ideapad_s740_coef(struct hda_codec *codec, + const struct hda_fixup *fix, + int action) +{ + switch (action) { + case HDA_FIXUP_ACT_PRE_PROBE: + snd_hda_add_verbs(codec, alc285_ideapad_s740_coefs); + break; + } +} diff --git a/sound/pci/hda/patch_conexant.c b/sound/pci/hda/patch_conexant.c index dfef9c17e140..d111258c6f45 100644 --- a/sound/pci/hda/patch_conexant.c +++ b/sound/pci/hda/patch_conexant.c @@ -930,18 +930,18 @@ static const struct snd_pci_quirk cxt5066_fixups[] = { SND_PCI_QUIRK(0x103c, 0x8079, "HP EliteBook 840 G3", CXT_FIXUP_HP_DOCK), SND_PCI_QUIRK(0x103c, 0x807C, "HP EliteBook 820 G3", CXT_FIXUP_HP_DOCK), SND_PCI_QUIRK(0x103c, 0x80FD, "HP ProBook 640 G2", CXT_FIXUP_HP_DOCK), - SND_PCI_QUIRK(0x103c, 0x828c, "HP EliteBook 840 G4", CXT_FIXUP_HP_DOCK), - SND_PCI_QUIRK(0x103c, 0x83b2, "HP EliteBook 840 G5", CXT_FIXUP_HP_DOCK), - SND_PCI_QUIRK(0x103c, 0x83b3, "HP EliteBook 830 G5", CXT_FIXUP_HP_DOCK), - SND_PCI_QUIRK(0x103c, 0x83d3, "HP ProBook 640 G4", CXT_FIXUP_HP_DOCK), - SND_PCI_QUIRK(0x103c, 0x8174, "HP Spectre x360", CXT_FIXUP_HP_SPECTRE), SND_PCI_QUIRK(0x103c, 0x8115, "HP Z1 Gen3", CXT_FIXUP_HP_GATE_MIC), SND_PCI_QUIRK(0x103c, 0x814f, "HP ZBook 15u G3", CXT_FIXUP_MUTE_LED_GPIO), + SND_PCI_QUIRK(0x103c, 0x8174, "HP Spectre x360", CXT_FIXUP_HP_SPECTRE), SND_PCI_QUIRK(0x103c, 0x822e, "HP ProBook 440 G4", CXT_FIXUP_MUTE_LED_GPIO), - SND_PCI_QUIRK(0x103c, 0x836e, "HP ProBook 455 G5", CXT_FIXUP_MUTE_LED_GPIO), - SND_PCI_QUIRK(0x103c, 0x837f, "HP ProBook 470 G5", CXT_FIXUP_MUTE_LED_GPIO), + SND_PCI_QUIRK(0x103c, 0x828c, "HP EliteBook 840 G4", CXT_FIXUP_HP_DOCK), SND_PCI_QUIRK(0x103c, 0x8299, "HP 800 G3 SFF", CXT_FIXUP_HP_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x103c, 0x829a, "HP 800 G3 DM", CXT_FIXUP_HP_MIC_NO_PRESENCE), + SND_PCI_QUIRK(0x103c, 0x836e, "HP ProBook 455 G5", CXT_FIXUP_MUTE_LED_GPIO), + SND_PCI_QUIRK(0x103c, 0x837f, "HP ProBook 470 G5", CXT_FIXUP_MUTE_LED_GPIO), + SND_PCI_QUIRK(0x103c, 0x83b2, "HP EliteBook 840 G5", CXT_FIXUP_HP_DOCK), + SND_PCI_QUIRK(0x103c, 0x83b3, "HP EliteBook 830 G5", CXT_FIXUP_HP_DOCK), + SND_PCI_QUIRK(0x103c, 0x83d3, "HP ProBook 640 G4", CXT_FIXUP_HP_DOCK), SND_PCI_QUIRK(0x103c, 0x8402, "HP ProBook 645 G4", CXT_FIXUP_MUTE_LED_GPIO), SND_PCI_QUIRK(0x103c, 0x8427, "HP ZBook Studio G5", CXT_FIXUP_HP_ZBOOK_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x844f, "HP ZBook Studio G5", CXT_FIXUP_HP_ZBOOK_MUTE_LED), diff --git a/sound/pci/hda/patch_hdmi.c b/sound/pci/hda/patch_hdmi.c index 45ae845e82df..4b2cc8cb55c4 100644 --- a/sound/pci/hda/patch_hdmi.c +++ b/sound/pci/hda/patch_hdmi.c @@ -1848,16 +1848,12 @@ static int hdmi_add_pin(struct hda_codec *codec, hda_nid_t pin_nid) */ if (spec->intel_hsw_fixup) { /* - * On Intel platforms, device entries number is - * changed dynamically. If there is a DP MST - * hub connected, the device entries number is 3. - * Otherwise, it is 1. - * Here we manually set dev_num to 3, so that - * we can initialize all the device entries when - * bootup statically. + * On Intel platforms, device entries count returned + * by AC_PAR_DEVLIST_LEN is dynamic, and depends on + * the type of receiver that is connected. Allocate pin + * structures based on worst case. */ - dev_num = 3; - spec->dev_num = 3; + dev_num = spec->dev_num; } else if (spec->dyn_pcm_assign && codec->dp_mst) { dev_num = snd_hda_get_num_devices(codec, pin_nid) + 1; /* @@ -2658,7 +2654,7 @@ static void generic_acomp_pin_eld_notify(void *audio_ptr, int port, int dev_id) /* skip notification during system suspend (but not in runtime PM); * the state will be updated at resume */ - if (snd_power_get_state(codec->card) != SNDRV_CTL_POWER_D0) + if (codec->core.dev.power.power_state.event == PM_EVENT_SUSPEND) return; /* ditto during suspend/resume process itself */ if (snd_hdac_is_in_pm(&codec->core)) @@ -2844,7 +2840,7 @@ static void intel_pin_eld_notify(void *audio_ptr, int port, int pipe) /* skip notification during system suspend (but not in runtime PM); * the state will be updated at resume */ - if (snd_power_get_state(codec->card) != SNDRV_CTL_POWER_D0) + if (codec->core.dev.power.power_state.event == PM_EVENT_SUSPEND) return; /* ditto during suspend/resume process itself */ if (snd_hdac_is_in_pm(&codec->core)) @@ -2942,7 +2938,7 @@ static int parse_intel_hdmi(struct hda_codec *codec) /* Intel Haswell and onwards; audio component with eld notifier */ static int intel_hsw_common_init(struct hda_codec *codec, hda_nid_t vendor_nid, - const int *port_map, int port_num) + const int *port_map, int port_num, int dev_num) { struct hdmi_spec *spec; int err; @@ -2957,6 +2953,7 @@ static int intel_hsw_common_init(struct hda_codec *codec, hda_nid_t vendor_nid, spec->port_map = port_map; spec->port_num = port_num; spec->intel_hsw_fixup = true; + spec->dev_num = dev_num; intel_haswell_enable_all_pins(codec, true); intel_haswell_fixup_enable_dp12(codec); @@ -2982,12 +2979,12 @@ static int intel_hsw_common_init(struct hda_codec *codec, hda_nid_t vendor_nid, static int patch_i915_hsw_hdmi(struct hda_codec *codec) { - return intel_hsw_common_init(codec, 0x08, NULL, 0); + return intel_hsw_common_init(codec, 0x08, NULL, 0, 3); } static int patch_i915_glk_hdmi(struct hda_codec *codec) { - return intel_hsw_common_init(codec, 0x0b, NULL, 0); + return intel_hsw_common_init(codec, 0x0b, NULL, 0, 3); } static int patch_i915_icl_hdmi(struct hda_codec *codec) @@ -2998,7 +2995,7 @@ static int patch_i915_icl_hdmi(struct hda_codec *codec) */ static const int map[] = {0x0, 0x4, 0x6, 0x8, 0xa, 0xb}; - return intel_hsw_common_init(codec, 0x02, map, ARRAY_SIZE(map)); + return intel_hsw_common_init(codec, 0x02, map, ARRAY_SIZE(map), 3); } static int patch_i915_tgl_hdmi(struct hda_codec *codec) @@ -3010,7 +3007,7 @@ static int patch_i915_tgl_hdmi(struct hda_codec *codec) static const int map[] = {0x4, 0x6, 0x8, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf}; int ret; - ret = intel_hsw_common_init(codec, 0x02, map, ARRAY_SIZE(map)); + ret = intel_hsw_common_init(codec, 0x02, map, ARRAY_SIZE(map), 4); if (!ret) { struct hdmi_spec *spec = codec->spec; diff --git a/sound/pci/hda/patch_realtek.c b/sound/pci/hda/patch_realtek.c index a7544b77d3f7..1fe70f2fe4fe 100644 --- a/sound/pci/hda/patch_realtek.c +++ b/sound/pci/hda/patch_realtek.c @@ -2470,13 +2470,13 @@ static const struct snd_pci_quirk alc882_fixup_tbl[] = { ALC882_FIXUP_ACER_ASPIRE_8930G), SND_PCI_QUIRK(0x1025, 0x0146, "Acer Aspire 6935G", ALC882_FIXUP_ACER_ASPIRE_8930G), + SND_PCI_QUIRK(0x1025, 0x0142, "Acer Aspire 7730G", + ALC882_FIXUP_ACER_ASPIRE_4930G), + SND_PCI_QUIRK(0x1025, 0x0155, "Packard-Bell M5120", ALC882_FIXUP_PB_M5210), SND_PCI_QUIRK(0x1025, 0x015e, "Acer Aspire 6930G", ALC882_FIXUP_ACER_ASPIRE_4930G), SND_PCI_QUIRK(0x1025, 0x0166, "Acer Aspire 6530G", ALC882_FIXUP_ACER_ASPIRE_4930G), - SND_PCI_QUIRK(0x1025, 0x0142, "Acer Aspire 7730G", - ALC882_FIXUP_ACER_ASPIRE_4930G), - SND_PCI_QUIRK(0x1025, 0x0155, "Packard-Bell M5120", ALC882_FIXUP_PB_M5210), SND_PCI_QUIRK(0x1025, 0x021e, "Acer Aspire 5739G", ALC882_FIXUP_ACER_ASPIRE_4930G), SND_PCI_QUIRK(0x1025, 0x0259, "Acer Aspire 5935", ALC889_FIXUP_DAC_ROUTE), @@ -2489,11 +2489,11 @@ static const struct snd_pci_quirk alc882_fixup_tbl[] = { SND_PCI_QUIRK(0x1043, 0x835f, "Asus Eee 1601", ALC888_FIXUP_EEE1601), SND_PCI_QUIRK(0x1043, 0x84bc, "ASUS ET2700", ALC887_FIXUP_ASUS_BASS), SND_PCI_QUIRK(0x1043, 0x8691, "ASUS ROG Ranger VIII", ALC882_FIXUP_GPIO3), + SND_PCI_QUIRK(0x104d, 0x9043, "Sony Vaio VGC-LN51JGB", ALC882_FIXUP_NO_PRIMARY_HP), + SND_PCI_QUIRK(0x104d, 0x9044, "Sony VAIO AiO", ALC882_FIXUP_NO_PRIMARY_HP), SND_PCI_QUIRK(0x104d, 0x9047, "Sony Vaio TT", ALC889_FIXUP_VAIO_TT), SND_PCI_QUIRK(0x104d, 0x905a, "Sony Vaio Z", ALC882_FIXUP_NO_PRIMARY_HP), SND_PCI_QUIRK(0x104d, 0x9060, "Sony Vaio VPCL14M1R", ALC882_FIXUP_NO_PRIMARY_HP), - SND_PCI_QUIRK(0x104d, 0x9043, "Sony Vaio VGC-LN51JGB", ALC882_FIXUP_NO_PRIMARY_HP), - SND_PCI_QUIRK(0x104d, 0x9044, "Sony VAIO AiO", ALC882_FIXUP_NO_PRIMARY_HP), /* All Apple entries are in codec SSIDs */ SND_PCI_QUIRK(0x106b, 0x00a0, "MacBookPro 3,1", ALC889_FIXUP_MBP_VREF), @@ -2536,9 +2536,19 @@ static const struct snd_pci_quirk alc882_fixup_tbl[] = { SND_PCI_QUIRK(0x1462, 0xda57, "MSI Z270-Gaming", ALC1220_FIXUP_GB_DUAL_CODECS), SND_PCI_QUIRK_VENDOR(0x1462, "MSI", ALC882_FIXUP_GPIO3), SND_PCI_QUIRK(0x147b, 0x107a, "Abit AW9D-MAX", ALC882_FIXUP_ABIT_AW9D_MAX), + SND_PCI_QUIRK(0x1558, 0x50d3, "Clevo PC50[ER][CDF]", ALC1220_FIXUP_CLEVO_PB51ED_PINS), + SND_PCI_QUIRK(0x1558, 0x65d1, "Clevo PB51[ER][CDF]", ALC1220_FIXUP_CLEVO_PB51ED_PINS), + SND_PCI_QUIRK(0x1558, 0x65d2, "Clevo PB51R[CDF]", ALC1220_FIXUP_CLEVO_PB51ED_PINS), + SND_PCI_QUIRK(0x1558, 0x65e1, "Clevo PB51[ED][DF]", ALC1220_FIXUP_CLEVO_PB51ED_PINS), + SND_PCI_QUIRK(0x1558, 0x65e5, "Clevo PC50D[PRS](?:-D|-G)?", ALC1220_FIXUP_CLEVO_PB51ED_PINS), + SND_PCI_QUIRK(0x1558, 0x67d1, "Clevo PB71[ER][CDF]", ALC1220_FIXUP_CLEVO_PB51ED_PINS), + SND_PCI_QUIRK(0x1558, 0x67e1, "Clevo PB71[DE][CDF]", ALC1220_FIXUP_CLEVO_PB51ED_PINS), + SND_PCI_QUIRK(0x1558, 0x67e5, "Clevo PC70D[PRS](?:-D|-G)?", ALC1220_FIXUP_CLEVO_PB51ED_PINS), + SND_PCI_QUIRK(0x1558, 0x70d1, "Clevo PC70[ER][CDF]", ALC1220_FIXUP_CLEVO_PB51ED_PINS), + SND_PCI_QUIRK(0x1558, 0x7714, "Clevo X170", ALC1220_FIXUP_CLEVO_PB51ED_PINS), SND_PCI_QUIRK(0x1558, 0x9501, "Clevo P950HR", ALC1220_FIXUP_CLEVO_P950), SND_PCI_QUIRK(0x1558, 0x9506, "Clevo P955HQ", ALC1220_FIXUP_CLEVO_P950), - SND_PCI_QUIRK(0x1558, 0x950A, "Clevo P955H[PR]", ALC1220_FIXUP_CLEVO_P950), + SND_PCI_QUIRK(0x1558, 0x950a, "Clevo P955H[PR]", ALC1220_FIXUP_CLEVO_P950), SND_PCI_QUIRK(0x1558, 0x95e1, "Clevo P95xER", ALC1220_FIXUP_CLEVO_P950), SND_PCI_QUIRK(0x1558, 0x95e2, "Clevo P950ER", ALC1220_FIXUP_CLEVO_P950), SND_PCI_QUIRK(0x1558, 0x95e3, "Clevo P955[ER]T", ALC1220_FIXUP_CLEVO_P950), @@ -2548,14 +2558,6 @@ static const struct snd_pci_quirk alc882_fixup_tbl[] = { SND_PCI_QUIRK(0x1558, 0x96e1, "Clevo P960[ER][CDFN]-K", ALC1220_FIXUP_CLEVO_P950), SND_PCI_QUIRK(0x1558, 0x97e1, "Clevo P970[ER][CDFN]", ALC1220_FIXUP_CLEVO_P950), SND_PCI_QUIRK(0x1558, 0x97e2, "Clevo P970RC-M", ALC1220_FIXUP_CLEVO_P950), - SND_PCI_QUIRK(0x1558, 0x50d3, "Clevo PC50[ER][CDF]", ALC1220_FIXUP_CLEVO_PB51ED_PINS), - SND_PCI_QUIRK(0x1558, 0x65d1, "Clevo PB51[ER][CDF]", ALC1220_FIXUP_CLEVO_PB51ED_PINS), - SND_PCI_QUIRK(0x1558, 0x65d2, "Clevo PB51R[CDF]", ALC1220_FIXUP_CLEVO_PB51ED_PINS), - SND_PCI_QUIRK(0x1558, 0x65e1, "Clevo PB51[ED][DF]", ALC1220_FIXUP_CLEVO_PB51ED_PINS), - SND_PCI_QUIRK(0x1558, 0x67d1, "Clevo PB71[ER][CDF]", ALC1220_FIXUP_CLEVO_PB51ED_PINS), - SND_PCI_QUIRK(0x1558, 0x67e1, "Clevo PB71[DE][CDF]", ALC1220_FIXUP_CLEVO_PB51ED_PINS), - SND_PCI_QUIRK(0x1558, 0x70d1, "Clevo PC70[ER][CDF]", ALC1220_FIXUP_CLEVO_PB51ED_PINS), - SND_PCI_QUIRK(0x1558, 0x7714, "Clevo X170", ALC1220_FIXUP_CLEVO_PB51ED_PINS), SND_PCI_QUIRK_VENDOR(0x1558, "Clevo laptop", ALC882_FIXUP_EAPD), SND_PCI_QUIRK(0x161f, 0x2054, "Medion laptop", ALC883_FIXUP_EAPD), SND_PCI_QUIRK(0x17aa, 0x3a0d, "Lenovo Y530", ALC882_FIXUP_LENOVO_Y530), @@ -4329,6 +4331,35 @@ static void alc245_fixup_hp_x360_amp(struct hda_codec *codec, } } +/* toggle GPIO2 at each time stream is started; we use PREPARE state instead */ +static void alc274_hp_envy_pcm_hook(struct hda_pcm_stream *hinfo, + struct hda_codec *codec, + struct snd_pcm_substream *substream, + int action) +{ + switch (action) { + case HDA_GEN_PCM_ACT_PREPARE: + alc_update_gpio_data(codec, 0x04, true); + break; + case HDA_GEN_PCM_ACT_CLEANUP: + alc_update_gpio_data(codec, 0x04, false); + break; + } +} + +static void alc274_fixup_hp_envy_gpio(struct hda_codec *codec, + const struct hda_fixup *fix, + int action) +{ + struct alc_spec *spec = codec->spec; + + if (action == HDA_FIXUP_ACT_PROBE) { + spec->gpio_mask |= 0x04; + spec->gpio_dir |= 0x04; + spec->gen.pcm_playback_hook = alc274_hp_envy_pcm_hook; + } +} + static void alc_update_coef_led(struct hda_codec *codec, struct alc_coef_led *led, bool polarity, bool on) @@ -4438,6 +4469,25 @@ static void alc236_fixup_hp_mute_led(struct hda_codec *codec, alc236_fixup_hp_coef_micmute_led(codec, fix, action); } +static void alc236_fixup_hp_micmute_led_vref(struct hda_codec *codec, + const struct hda_fixup *fix, int action) +{ + struct alc_spec *spec = codec->spec; + + if (action == HDA_FIXUP_ACT_PRE_PROBE) { + spec->cap_mute_led_nid = 0x1a; + snd_hda_gen_add_micmute_led_cdev(codec, vref_micmute_led_set); + codec->power_filter = led_power_filter; + } +} + +static void alc236_fixup_hp_mute_led_micmute_vref(struct hda_codec *codec, + const struct hda_fixup *fix, int action) +{ + alc236_fixup_hp_mute_led_coefbit(codec, fix, action); + alc236_fixup_hp_micmute_led_vref(codec, fix, action); +} + #if IS_REACHABLE(CONFIG_INPUT) static void gpio2_mic_hotkey_event(struct hda_codec *codec, struct hda_jack_callback *event) @@ -6232,6 +6282,9 @@ static void alc_fixup_thinkpad_acpi(struct hda_codec *codec, /* for alc295_fixup_hp_top_speakers */ #include "hp_x360_helper.c" +/* for alc285_fixup_ideapad_s740_coef() */ +#include "ideapad_s740_helper.c" + enum { ALC269_FIXUP_GPIO2, ALC269_FIXUP_SONY_VAIO, @@ -6400,6 +6453,7 @@ enum { ALC285_FIXUP_HP_MUTE_LED, ALC236_FIXUP_HP_GPIO_LED, ALC236_FIXUP_HP_MUTE_LED, + ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF, ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET, ALC295_FIXUP_ASUS_MIC_NO_PRESENCE, ALC269VC_FIXUP_ACER_VCOPPERBOX_PINS, @@ -6415,10 +6469,13 @@ enum { ALC269_FIXUP_LEMOTE_A1802, ALC269_FIXUP_LEMOTE_A190X, ALC256_FIXUP_INTEL_NUC8_RUGGED, + ALC233_FIXUP_INTEL_NUC8_DMIC, + ALC233_FIXUP_INTEL_NUC8_BOOST, ALC256_FIXUP_INTEL_NUC10, ALC255_FIXUP_XIAOMI_HEADSET_MIC, ALC274_FIXUP_HP_MIC, ALC274_FIXUP_HP_HEADSET_MIC, + ALC274_FIXUP_HP_ENVY_GPIO, ALC256_FIXUP_ASUS_HPE, ALC285_FIXUP_THINKPAD_NO_BASS_SPK_HEADSET_JACK, ALC287_FIXUP_HP_GPIO_LED, @@ -6427,6 +6484,7 @@ enum { ALC282_FIXUP_ACER_DISABLE_LINEOUT, ALC255_FIXUP_ACER_LIMIT_INT_MIC_BOOST, ALC256_FIXUP_ACER_HEADSET_MIC, + ALC285_FIXUP_IDEAPAD_S740_COEF, }; static const struct hda_fixup alc269_fixups[] = { @@ -7136,6 +7194,16 @@ static const struct hda_fixup alc269_fixups[] = { .type = HDA_FIXUP_FUNC, .v.func = alc233_fixup_lenovo_line2_mic_hotkey, }, + [ALC233_FIXUP_INTEL_NUC8_DMIC] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc_fixup_inv_dmic, + .chained = true, + .chain_id = ALC233_FIXUP_INTEL_NUC8_BOOST, + }, + [ALC233_FIXUP_INTEL_NUC8_BOOST] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc269_fixup_limit_int_mic_boost + }, [ALC255_FIXUP_DELL_SPK_NOISE] = { .type = HDA_FIXUP_FUNC, .v.func = alc_fixup_disable_aamix, @@ -7646,6 +7714,10 @@ static const struct hda_fixup alc269_fixups[] = { .type = HDA_FIXUP_FUNC, .v.func = alc236_fixup_hp_mute_led, }, + [ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc236_fixup_hp_mute_led_micmute_vref, + }, [ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET] = { .type = HDA_FIXUP_VERBS, .v.verbs = (const struct hda_verb[]) { @@ -7844,6 +7916,10 @@ static const struct hda_fixup alc269_fixups[] = { .chained = true, .chain_id = ALC274_FIXUP_HP_MIC }, + [ALC274_FIXUP_HP_ENVY_GPIO] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc274_fixup_hp_envy_gpio, + }, [ALC256_FIXUP_ASUS_HPE] = { .type = HDA_FIXUP_VERBS, .v.verbs = (const struct hda_verb[]) { @@ -7901,6 +7977,12 @@ static const struct hda_fixup alc269_fixups[] = { .chained = true, .chain_id = ALC269_FIXUP_HEADSET_MODE_NO_HP_MIC }, + [ALC285_FIXUP_IDEAPAD_S740_COEF] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc285_fixup_ideapad_s740_coef, + .chained = true, + .chain_id = ALC269_FIXUP_THINKPAD_ACPI, + }, }; static const struct snd_pci_quirk alc269_fixup_tbl[] = { @@ -7909,12 +7991,12 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1025, 0x0349, "Acer AOD260", ALC269_FIXUP_INV_DMIC), SND_PCI_QUIRK(0x1025, 0x047c, "Acer AC700", ALC269_FIXUP_ACER_AC700), SND_PCI_QUIRK(0x1025, 0x072d, "Acer Aspire V5-571G", ALC269_FIXUP_ASPIRE_HEADSET_MIC), - SND_PCI_QUIRK(0x1025, 0x080d, "Acer Aspire V5-122P", ALC269_FIXUP_ASPIRE_HEADSET_MIC), SND_PCI_QUIRK(0x1025, 0x0740, "Acer AO725", ALC271_FIXUP_HP_GATE_MIC_JACK), SND_PCI_QUIRK(0x1025, 0x0742, "Acer AO756", ALC271_FIXUP_HP_GATE_MIC_JACK), SND_PCI_QUIRK(0x1025, 0x0762, "Acer Aspire E1-472", ALC271_FIXUP_HP_GATE_MIC_JACK_E1_572), SND_PCI_QUIRK(0x1025, 0x0775, "Acer Aspire E1-572", ALC271_FIXUP_HP_GATE_MIC_JACK_E1_572), SND_PCI_QUIRK(0x1025, 0x079b, "Acer Aspire V5-573G", ALC282_FIXUP_ASPIRE_V5_PINS), + SND_PCI_QUIRK(0x1025, 0x080d, "Acer Aspire V5-122P", ALC269_FIXUP_ASPIRE_HEADSET_MIC), SND_PCI_QUIRK(0x1025, 0x0840, "Acer Aspire E1", ALC269VB_FIXUP_ASPIRE_E1_COEF), SND_PCI_QUIRK(0x1025, 0x101c, "Acer Veriton N2510G", ALC269_FIXUP_LIFEBOOK), SND_PCI_QUIRK(0x1025, 0x102b, "Acer Aspire C24-860", ALC286_FIXUP_ACER_AIO_MIC_NO_PRESENCE), @@ -7970,8 +8052,8 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1028, 0x0738, "Dell Precision 5820", ALC269_FIXUP_NO_SHUTUP), SND_PCI_QUIRK(0x1028, 0x075c, "Dell XPS 27 7760", ALC298_FIXUP_SPK_VOLUME), SND_PCI_QUIRK(0x1028, 0x075d, "Dell AIO", ALC298_FIXUP_SPK_VOLUME), - SND_PCI_QUIRK(0x1028, 0x07b0, "Dell Precision 7520", ALC295_FIXUP_DISABLE_DAC3), SND_PCI_QUIRK(0x1028, 0x0798, "Dell Inspiron 17 7000 Gaming", ALC256_FIXUP_DELL_INSPIRON_7559_SUBWOOFER), + SND_PCI_QUIRK(0x1028, 0x07b0, "Dell Precision 7520", ALC295_FIXUP_DISABLE_DAC3), SND_PCI_QUIRK(0x1028, 0x080c, "Dell WYSE", ALC225_FIXUP_DELL_WYSE_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1028, 0x084b, "Dell", ALC274_FIXUP_DELL_AIO_LINEOUT_VERB), SND_PCI_QUIRK(0x1028, 0x084e, "Dell", ALC274_FIXUP_DELL_AIO_LINEOUT_VERB), @@ -7981,8 +8063,8 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1028, 0x08ad, "Dell WYSE AIO", ALC225_FIXUP_DELL_WYSE_AIO_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1028, 0x08ae, "Dell WYSE NB", ALC225_FIXUP_DELL1_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1028, 0x0935, "Dell", ALC274_FIXUP_DELL_AIO_LINEOUT_VERB), - SND_PCI_QUIRK(0x1028, 0x097e, "Dell Precision", ALC289_FIXUP_DUAL_SPK), SND_PCI_QUIRK(0x1028, 0x097d, "Dell Precision", ALC289_FIXUP_DUAL_SPK), + SND_PCI_QUIRK(0x1028, 0x097e, "Dell Precision", ALC289_FIXUP_DUAL_SPK), SND_PCI_QUIRK(0x1028, 0x098d, "Dell Precision", ALC233_FIXUP_ASUS_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1028, 0x09bf, "Dell Precision", ALC233_FIXUP_ASUS_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1028, 0x0a2e, "Dell", ALC236_FIXUP_DELL_AIO_HEADSET_MIC), @@ -7993,35 +8075,18 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x1586, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC2), SND_PCI_QUIRK(0x103c, 0x18e6, "HP", ALC269_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x218b, "HP", ALC269_FIXUP_LIMIT_INT_MIC_BOOST_MUTE_LED), - SND_PCI_QUIRK(0x103c, 0x225f, "HP", ALC280_FIXUP_HP_GPIO2_MIC_HOTKEY), - /* ALC282 */ SND_PCI_QUIRK(0x103c, 0x21f9, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), SND_PCI_QUIRK(0x103c, 0x2210, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), SND_PCI_QUIRK(0x103c, 0x2214, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), + SND_PCI_QUIRK(0x103c, 0x221b, "HP", ALC269_FIXUP_HP_GPIO_MIC1_LED), + SND_PCI_QUIRK(0x103c, 0x221c, "HP EliteBook 755 G2", ALC280_FIXUP_HP_HEADSET_MIC), + SND_PCI_QUIRK(0x103c, 0x2221, "HP", ALC269_FIXUP_HP_GPIO_MIC1_LED), + SND_PCI_QUIRK(0x103c, 0x2225, "HP", ALC269_FIXUP_HP_GPIO_MIC1_LED), SND_PCI_QUIRK(0x103c, 0x2236, "HP", ALC269_FIXUP_HP_LINE1_MIC1_LED), SND_PCI_QUIRK(0x103c, 0x2237, "HP", ALC269_FIXUP_HP_LINE1_MIC1_LED), SND_PCI_QUIRK(0x103c, 0x2238, "HP", ALC269_FIXUP_HP_LINE1_MIC1_LED), SND_PCI_QUIRK(0x103c, 0x2239, "HP", ALC269_FIXUP_HP_LINE1_MIC1_LED), SND_PCI_QUIRK(0x103c, 0x224b, "HP", ALC269_FIXUP_HP_LINE1_MIC1_LED), - SND_PCI_QUIRK(0x103c, 0x2268, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), - SND_PCI_QUIRK(0x103c, 0x226a, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), - SND_PCI_QUIRK(0x103c, 0x226b, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), - SND_PCI_QUIRK(0x103c, 0x226e, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), - SND_PCI_QUIRK(0x103c, 0x2271, "HP", ALC286_FIXUP_HP_GPIO_LED), - SND_PCI_QUIRK(0x103c, 0x2272, "HP", ALC280_FIXUP_HP_DOCK_PINS), - SND_PCI_QUIRK(0x103c, 0x2273, "HP", ALC280_FIXUP_HP_DOCK_PINS), - SND_PCI_QUIRK(0x103c, 0x229e, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), - SND_PCI_QUIRK(0x103c, 0x22b2, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), - SND_PCI_QUIRK(0x103c, 0x22b7, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), - SND_PCI_QUIRK(0x103c, 0x22bf, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), - SND_PCI_QUIRK(0x103c, 0x22cf, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), - SND_PCI_QUIRK(0x103c, 0x22db, "HP", ALC280_FIXUP_HP_9480M), - SND_PCI_QUIRK(0x103c, 0x22dc, "HP", ALC269_FIXUP_HP_GPIO_MIC1_LED), - SND_PCI_QUIRK(0x103c, 0x22fb, "HP", ALC269_FIXUP_HP_GPIO_MIC1_LED), - /* ALC290 */ - SND_PCI_QUIRK(0x103c, 0x221b, "HP", ALC269_FIXUP_HP_GPIO_MIC1_LED), - SND_PCI_QUIRK(0x103c, 0x2221, "HP", ALC269_FIXUP_HP_GPIO_MIC1_LED), - SND_PCI_QUIRK(0x103c, 0x2225, "HP", ALC269_FIXUP_HP_GPIO_MIC1_LED), SND_PCI_QUIRK(0x103c, 0x2253, "HP", ALC269_FIXUP_HP_GPIO_MIC1_LED), SND_PCI_QUIRK(0x103c, 0x2254, "HP", ALC269_FIXUP_HP_GPIO_MIC1_LED), SND_PCI_QUIRK(0x103c, 0x2255, "HP", ALC269_FIXUP_HP_GPIO_MIC1_LED), @@ -8029,28 +8094,45 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x2257, "HP", ALC269_FIXUP_HP_GPIO_MIC1_LED), SND_PCI_QUIRK(0x103c, 0x2259, "HP", ALC269_FIXUP_HP_GPIO_MIC1_LED), SND_PCI_QUIRK(0x103c, 0x225a, "HP", ALC269_FIXUP_HP_DOCK_GPIO_MIC1_LED), + SND_PCI_QUIRK(0x103c, 0x225f, "HP", ALC280_FIXUP_HP_GPIO2_MIC_HOTKEY), SND_PCI_QUIRK(0x103c, 0x2260, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), SND_PCI_QUIRK(0x103c, 0x2263, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), SND_PCI_QUIRK(0x103c, 0x2264, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), SND_PCI_QUIRK(0x103c, 0x2265, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), + SND_PCI_QUIRK(0x103c, 0x2268, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), + SND_PCI_QUIRK(0x103c, 0x226a, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), + SND_PCI_QUIRK(0x103c, 0x226b, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), + SND_PCI_QUIRK(0x103c, 0x226e, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), + SND_PCI_QUIRK(0x103c, 0x2271, "HP", ALC286_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x2272, "HP", ALC269_FIXUP_HP_GPIO_MIC1_LED), + SND_PCI_QUIRK(0x103c, 0x2272, "HP", ALC280_FIXUP_HP_DOCK_PINS), SND_PCI_QUIRK(0x103c, 0x2273, "HP", ALC269_FIXUP_HP_GPIO_MIC1_LED), + SND_PCI_QUIRK(0x103c, 0x2273, "HP", ALC280_FIXUP_HP_DOCK_PINS), SND_PCI_QUIRK(0x103c, 0x2278, "HP", ALC269_FIXUP_HP_GPIO_MIC1_LED), SND_PCI_QUIRK(0x103c, 0x227f, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), SND_PCI_QUIRK(0x103c, 0x2282, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), SND_PCI_QUIRK(0x103c, 0x228b, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), SND_PCI_QUIRK(0x103c, 0x228e, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), + SND_PCI_QUIRK(0x103c, 0x229e, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), + SND_PCI_QUIRK(0x103c, 0x22b2, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), + SND_PCI_QUIRK(0x103c, 0x22b7, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), + SND_PCI_QUIRK(0x103c, 0x22bf, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), + SND_PCI_QUIRK(0x103c, 0x22c4, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), SND_PCI_QUIRK(0x103c, 0x22c5, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), SND_PCI_QUIRK(0x103c, 0x22c7, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), SND_PCI_QUIRK(0x103c, 0x22c8, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), - SND_PCI_QUIRK(0x103c, 0x22c4, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), + SND_PCI_QUIRK(0x103c, 0x22cf, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), + SND_PCI_QUIRK(0x103c, 0x22db, "HP", ALC280_FIXUP_HP_9480M), + SND_PCI_QUIRK(0x103c, 0x22dc, "HP", ALC269_FIXUP_HP_GPIO_MIC1_LED), + SND_PCI_QUIRK(0x103c, 0x22fb, "HP", ALC269_FIXUP_HP_GPIO_MIC1_LED), SND_PCI_QUIRK(0x103c, 0x2334, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), SND_PCI_QUIRK(0x103c, 0x2335, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), SND_PCI_QUIRK(0x103c, 0x2336, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), SND_PCI_QUIRK(0x103c, 0x2337, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), - SND_PCI_QUIRK(0x103c, 0x221c, "HP EliteBook 755 G2", ALC280_FIXUP_HP_HEADSET_MIC), SND_PCI_QUIRK(0x103c, 0x802e, "HP Z240 SFF", ALC221_FIXUP_HP_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x103c, 0x802f, "HP Z240", ALC221_FIXUP_HP_MIC_NO_PRESENCE), + SND_PCI_QUIRK(0x103c, 0x8077, "HP", ALC256_FIXUP_HP_HEADSET_MIC), + SND_PCI_QUIRK(0x103c, 0x8158, "HP", ALC256_FIXUP_HP_HEADSET_MIC), SND_PCI_QUIRK(0x103c, 0x820d, "HP Pavilion 15", ALC269_FIXUP_HP_MUTE_LED_MIC3), SND_PCI_QUIRK(0x103c, 0x8256, "HP", ALC221_FIXUP_HP_FRONT_MIC), SND_PCI_QUIRK(0x103c, 0x827e, "HP x360", ALC295_FIXUP_HP_X360), @@ -8061,8 +8143,10 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8497, "HP Envy x360", ALC269_FIXUP_HP_MUTE_LED_MIC3), SND_PCI_QUIRK(0x103c, 0x84e7, "HP Pavilion 15", ALC269_FIXUP_HP_MUTE_LED_MIC3), SND_PCI_QUIRK(0x103c, 0x869d, "HP", ALC236_FIXUP_HP_MUTE_LED), + SND_PCI_QUIRK(0x103c, 0x86c7, "HP Envy AiO 32", ALC274_FIXUP_HP_ENVY_GPIO), SND_PCI_QUIRK(0x103c, 0x8724, "HP EliteBook 850 G7", ALC285_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8729, "HP", ALC285_FIXUP_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x8730, "HP ProBook 445 G7", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF), SND_PCI_QUIRK(0x103c, 0x8736, "HP", ALC285_FIXUP_HP_GPIO_AMP_INIT), SND_PCI_QUIRK(0x103c, 0x8760, "HP", ALC285_FIXUP_HP_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x877a, "HP", ALC285_FIXUP_HP_MUTE_LED), @@ -8087,16 +8171,18 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1043, 0x10d0, "ASUS X540LA/X540LJ", ALC255_FIXUP_ASUS_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1043, 0x115d, "Asus 1015E", ALC269_FIXUP_LIMIT_INT_MIC_BOOST), SND_PCI_QUIRK(0x1043, 0x11c0, "ASUS X556UR", ALC255_FIXUP_ASUS_MIC_NO_PRESENCE), + SND_PCI_QUIRK(0x1043, 0x125e, "ASUS Q524UQK", ALC255_FIXUP_ASUS_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1043, 0x1271, "ASUS X430UN", ALC256_FIXUP_ASUS_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1043, 0x1290, "ASUS X441SA", ALC233_FIXUP_EAPD_COEF_AND_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1043, 0x12a0, "ASUS X441UV", ALC233_FIXUP_EAPD_COEF_AND_MIC_NO_PRESENCE), - SND_PCI_QUIRK(0x1043, 0x12f0, "ASUS X541UV", ALC256_FIXUP_ASUS_MIC), SND_PCI_QUIRK(0x1043, 0x12e0, "ASUS X541SA", ALC256_FIXUP_ASUS_MIC), + SND_PCI_QUIRK(0x1043, 0x12f0, "ASUS X541UV", ALC256_FIXUP_ASUS_MIC), SND_PCI_QUIRK(0x1043, 0x13b0, "ASUS Z550SA", ALC256_FIXUP_ASUS_MIC), SND_PCI_QUIRK(0x1043, 0x1427, "Asus Zenbook UX31E", ALC269VB_FIXUP_ASUS_ZENBOOK), SND_PCI_QUIRK(0x1043, 0x1517, "Asus Zenbook UX31A", ALC269VB_FIXUP_ASUS_ZENBOOK_UX31A), SND_PCI_QUIRK(0x1043, 0x16e3, "ASUS UX50", ALC269_FIXUP_STEREO_DMIC), SND_PCI_QUIRK(0x1043, 0x17d1, "ASUS UX431FL", ALC294_FIXUP_ASUS_DUAL_SPK), + SND_PCI_QUIRK(0x1043, 0x1881, "ASUS Zephyrus S/M", ALC294_FIXUP_ASUS_GX502_PINS), SND_PCI_QUIRK(0x1043, 0x18b1, "Asus MJ401TA", ALC256_FIXUP_ASUS_HEADSET_MIC), SND_PCI_QUIRK(0x1043, 0x18f1, "Asus FX505DT", ALC256_FIXUP_ASUS_HEADSET_MIC), SND_PCI_QUIRK(0x1043, 0x194e, "ASUS UX563FD", ALC294_FIXUP_ASUS_HPE), @@ -8109,31 +8195,31 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1043, 0x1b13, "Asus U41SV", ALC269_FIXUP_INV_DMIC), SND_PCI_QUIRK(0x1043, 0x1bbd, "ASUS Z550MA", ALC255_FIXUP_ASUS_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1043, 0x1c23, "Asus X55U", ALC269_FIXUP_LIMIT_INT_MIC_BOOST), - SND_PCI_QUIRK(0x1043, 0x125e, "ASUS Q524UQK", ALC255_FIXUP_ASUS_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1043, 0x1ccd, "ASUS X555UB", ALC256_FIXUP_ASUS_MIC), SND_PCI_QUIRK(0x1043, 0x1d4e, "ASUS TM420", ALC256_FIXUP_ASUS_HPE), SND_PCI_QUIRK(0x1043, 0x1e11, "ASUS Zephyrus G15", ALC289_FIXUP_ASUS_GA502), + SND_PCI_QUIRK(0x1043, 0x1e8e, "ASUS Zephyrus G15", ALC289_FIXUP_ASUS_GA401), SND_PCI_QUIRK(0x1043, 0x1f11, "ASUS Zephyrus G14", ALC289_FIXUP_ASUS_GA401), - SND_PCI_QUIRK(0x1043, 0x1881, "ASUS Zephyrus S/M", ALC294_FIXUP_ASUS_GX502_PINS), SND_PCI_QUIRK(0x1043, 0x3030, "ASUS ZN270IE", ALC256_FIXUP_ASUS_AIO_GPIO2), SND_PCI_QUIRK(0x1043, 0x831a, "ASUS P901", ALC269_FIXUP_STEREO_DMIC), SND_PCI_QUIRK(0x1043, 0x834a, "ASUS S101", ALC269_FIXUP_STEREO_DMIC), SND_PCI_QUIRK(0x1043, 0x8398, "ASUS P1005", ALC269_FIXUP_STEREO_DMIC), SND_PCI_QUIRK(0x1043, 0x83ce, "ASUS P1005", ALC269_FIXUP_STEREO_DMIC), SND_PCI_QUIRK(0x1043, 0x8516, "ASUS X101CH", ALC269_FIXUP_ASUS_X101), - SND_PCI_QUIRK(0x104d, 0x90b5, "Sony VAIO Pro 11", ALC286_FIXUP_SONY_MIC_NO_PRESENCE), - SND_PCI_QUIRK(0x104d, 0x90b6, "Sony VAIO Pro 13", ALC286_FIXUP_SONY_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x104d, 0x9073, "Sony VAIO", ALC275_FIXUP_SONY_VAIO_GPIO2), SND_PCI_QUIRK(0x104d, 0x907b, "Sony VAIO", ALC275_FIXUP_SONY_HWEQ), SND_PCI_QUIRK(0x104d, 0x9084, "Sony VAIO", ALC275_FIXUP_SONY_HWEQ), SND_PCI_QUIRK(0x104d, 0x9099, "Sony VAIO S13", ALC275_FIXUP_SONY_DISABLE_AAMIX), + SND_PCI_QUIRK(0x104d, 0x90b5, "Sony VAIO Pro 11", ALC286_FIXUP_SONY_MIC_NO_PRESENCE), + SND_PCI_QUIRK(0x104d, 0x90b6, "Sony VAIO Pro 13", ALC286_FIXUP_SONY_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x10cf, 0x1475, "Lifebook", ALC269_FIXUP_LIFEBOOK), SND_PCI_QUIRK(0x10cf, 0x159f, "Lifebook E780", ALC269_FIXUP_LIFEBOOK_NO_HP_TO_LINEOUT), SND_PCI_QUIRK(0x10cf, 0x15dc, "Lifebook T731", ALC269_FIXUP_LIFEBOOK_HP_PIN), - SND_PCI_QUIRK(0x10cf, 0x1757, "Lifebook E752", ALC269_FIXUP_LIFEBOOK_HP_PIN), SND_PCI_QUIRK(0x10cf, 0x1629, "Lifebook U7x7", ALC255_FIXUP_LIFEBOOK_U7x7_HEADSET_MIC), + SND_PCI_QUIRK(0x10cf, 0x1757, "Lifebook E752", ALC269_FIXUP_LIFEBOOK_HP_PIN), SND_PCI_QUIRK(0x10cf, 0x1845, "Lifebook U904", ALC269_FIXUP_LIFEBOOK_EXTMIC), SND_PCI_QUIRK(0x10ec, 0x10f2, "Intel Reference board", ALC700_FIXUP_INTEL_REFERENCE), + SND_PCI_QUIRK(0x10ec, 0x118c, "Medion EE4254 MD62100", ALC256_FIXUP_MEDION_HEADSET_NO_PRESENCE), SND_PCI_QUIRK(0x10ec, 0x1230, "Intel Reference board", ALC295_FIXUP_CHROME_BOOK), SND_PCI_QUIRK(0x10ec, 0x1252, "Intel Reference board", ALC295_FIXUP_CHROME_BOOK), SND_PCI_QUIRK(0x10ec, 0x1254, "Intel Reference board", ALC295_FIXUP_CHROME_BOOK), @@ -8143,9 +8229,9 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x144d, 0xc176, "Samsung Notebook 9 Pro (NP930MBE-K04US)", ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET), SND_PCI_QUIRK(0x144d, 0xc189, "Samsung Galaxy Flex Book (NT950QCG-X716)", ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET), SND_PCI_QUIRK(0x144d, 0xc18a, "Samsung Galaxy Book Ion (NP930XCJ-K01US)", ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET), - SND_PCI_QUIRK(0x144d, 0xc830, "Samsung Galaxy Book Ion (NT950XCJ-X716A)", ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET), SND_PCI_QUIRK(0x144d, 0xc740, "Samsung Ativ book 8 (NP870Z5G)", ALC269_FIXUP_ATIV_BOOK_8), SND_PCI_QUIRK(0x144d, 0xc812, "Samsung Notebook Pen S (NT950SBE-X58)", ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET), + SND_PCI_QUIRK(0x144d, 0xc830, "Samsung Galaxy Book Ion (NT950XCJ-X716A)", ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET), SND_PCI_QUIRK(0x1458, 0xfa53, "Gigabyte BXBT-2807", ALC283_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1462, 0xb120, "MSI Cubi MS-B120", ALC283_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1462, 0xb171, "Cubi N 8GL (MS-B171)", ALC283_FIXUP_HEADSET_MIC), @@ -8201,9 +8287,9 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x17aa, 0x21b8, "Thinkpad Edge 14", ALC269_FIXUP_SKU_IGNORE), SND_PCI_QUIRK(0x17aa, 0x21ca, "Thinkpad L412", ALC269_FIXUP_SKU_IGNORE), SND_PCI_QUIRK(0x17aa, 0x21e9, "Thinkpad Edge 15", ALC269_FIXUP_SKU_IGNORE), + SND_PCI_QUIRK(0x17aa, 0x21f3, "Thinkpad T430", ALC269_FIXUP_LENOVO_DOCK), SND_PCI_QUIRK(0x17aa, 0x21f6, "Thinkpad T530", ALC269_FIXUP_LENOVO_DOCK_LIMIT_BOOST), SND_PCI_QUIRK(0x17aa, 0x21fa, "Thinkpad X230", ALC269_FIXUP_LENOVO_DOCK), - SND_PCI_QUIRK(0x17aa, 0x21f3, "Thinkpad T430", ALC269_FIXUP_LENOVO_DOCK), SND_PCI_QUIRK(0x17aa, 0x21fb, "Thinkpad T430s", ALC269_FIXUP_LENOVO_DOCK), SND_PCI_QUIRK(0x17aa, 0x2203, "Thinkpad X230 Tablet", ALC269_FIXUP_LENOVO_DOCK), SND_PCI_QUIRK(0x17aa, 0x2208, "Thinkpad T431s", ALC269_FIXUP_LENOVO_DOCK), @@ -8244,9 +8330,11 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x17aa, 0x3176, "ThinkCentre Station", ALC283_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x17aa, 0x3178, "ThinkCentre Station", ALC283_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x17aa, 0x3818, "Lenovo C940", ALC298_FIXUP_LENOVO_SPK_VOLUME), + SND_PCI_QUIRK(0x17aa, 0x3827, "Ideapad S740", ALC285_FIXUP_IDEAPAD_S740_COEF), SND_PCI_QUIRK(0x17aa, 0x3902, "Lenovo E50-80", ALC269_FIXUP_DMIC_THINKPAD_ACPI), SND_PCI_QUIRK(0x17aa, 0x3977, "IdeaPad S210", ALC283_FIXUP_INT_MIC), SND_PCI_QUIRK(0x17aa, 0x3978, "Lenovo B50-70", ALC269_FIXUP_DMIC_THINKPAD_ACPI), + SND_PCI_QUIRK(0x17aa, 0x3bf8, "Quanta FL1", ALC269_FIXUP_PCM_44K), SND_PCI_QUIRK(0x17aa, 0x5013, "Thinkpad", ALC269_FIXUP_LIMIT_INT_MIC_BOOST), SND_PCI_QUIRK(0x17aa, 0x501a, "Thinkpad", ALC283_FIXUP_INT_MIC), SND_PCI_QUIRK(0x17aa, 0x501e, "Thinkpad L440", ALC292_FIXUP_TPT440_DOCK), @@ -8265,20 +8353,19 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x17aa, 0x5109, "Thinkpad", ALC269_FIXUP_LIMIT_INT_MIC_BOOST), SND_PCI_QUIRK(0x17aa, 0x511e, "Thinkpad", ALC298_FIXUP_TPT470_DOCK), SND_PCI_QUIRK(0x17aa, 0x511f, "Thinkpad", ALC298_FIXUP_TPT470_DOCK), - SND_PCI_QUIRK(0x17aa, 0x3bf8, "Quanta FL1", ALC269_FIXUP_PCM_44K), SND_PCI_QUIRK(0x17aa, 0x9e54, "LENOVO NB", ALC269_FIXUP_LENOVO_EAPD), SND_PCI_QUIRK(0x19e5, 0x3204, "Huawei MACH-WX9", ALC256_FIXUP_HUAWEI_MACH_WX9_PINS), SND_PCI_QUIRK(0x1b35, 0x1235, "CZC B20", ALC269_FIXUP_CZC_B20), SND_PCI_QUIRK(0x1b35, 0x1236, "CZC TMI", ALC269_FIXUP_CZC_TMI), SND_PCI_QUIRK(0x1b35, 0x1237, "CZC L101", ALC269_FIXUP_CZC_L101), SND_PCI_QUIRK(0x1b7d, 0xa831, "Ordissimo EVE2 ", ALC269VB_FIXUP_ORDISSIMO_EVE2), /* Also known as Malata PC-B1303 */ + SND_PCI_QUIRK(0x1c06, 0x2013, "Lemote A1802", ALC269_FIXUP_LEMOTE_A1802), + SND_PCI_QUIRK(0x1c06, 0x2015, "Lemote A190X", ALC269_FIXUP_LEMOTE_A190X), SND_PCI_QUIRK(0x1d72, 0x1602, "RedmiBook", ALC255_FIXUP_XIAOMI_HEADSET_MIC), SND_PCI_QUIRK(0x1d72, 0x1701, "XiaomiNotebook Pro", ALC298_FIXUP_DELL1_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1d72, 0x1901, "RedmiBook 14", ALC256_FIXUP_ASUS_HEADSET_MIC), SND_PCI_QUIRK(0x1d72, 0x1947, "RedmiBook Air", ALC255_FIXUP_XIAOMI_HEADSET_MIC), - SND_PCI_QUIRK(0x10ec, 0x118c, "Medion EE4254 MD62100", ALC256_FIXUP_MEDION_HEADSET_NO_PRESENCE), - SND_PCI_QUIRK(0x1c06, 0x2013, "Lemote A1802", ALC269_FIXUP_LEMOTE_A1802), - SND_PCI_QUIRK(0x1c06, 0x2015, "Lemote A190X", ALC269_FIXUP_LEMOTE_A190X), + SND_PCI_QUIRK(0x8086, 0x2074, "Intel NUC 8", ALC233_FIXUP_INTEL_NUC8_DMIC), SND_PCI_QUIRK(0x8086, 0x2080, "Intel NUC 8 Rugged", ALC256_FIXUP_INTEL_NUC8_RUGGED), SND_PCI_QUIRK(0x8086, 0x2081, "Intel NUC 10", ALC256_FIXUP_INTEL_NUC10), @@ -8733,12 +8820,17 @@ static const struct snd_hda_pin_quirk alc269_pin_fixup_tbl[] = { {0x12, 0x90a60130}, {0x19, 0x03a11020}, {0x21, 0x0321101f}), - SND_HDA_PIN_QUIRK(0x10ec0285, 0x17aa, "Lenovo", ALC285_FIXUP_THINKPAD_NO_BASS_SPK_HEADSET_JACK, + SND_HDA_PIN_QUIRK(0x10ec0285, 0x17aa, "Lenovo", ALC285_FIXUP_LENOVO_PC_BEEP_IN_NOISE, + {0x12, 0x90a60130}, {0x14, 0x90170110}, {0x19, 0x04a11040}, {0x21, 0x04211020}), SND_HDA_PIN_QUIRK(0x10ec0285, 0x17aa, "Lenovo", ALC285_FIXUP_LENOVO_PC_BEEP_IN_NOISE, - {0x12, 0x90a60130}, + {0x14, 0x90170110}, + {0x19, 0x04a11040}, + {0x1d, 0x40600001}, + {0x21, 0x04211020}), + SND_HDA_PIN_QUIRK(0x10ec0285, 0x17aa, "Lenovo", ALC285_FIXUP_THINKPAD_NO_BASS_SPK_HEADSET_JACK, {0x14, 0x90170110}, {0x19, 0x04a11040}, {0x21, 0x04211020}), @@ -9224,8 +9316,7 @@ static const struct snd_pci_quirk alc861_fixup_tbl[] = { SND_PCI_QUIRK(0x1043, 0x1393, "ASUS A6Rp", ALC861_FIXUP_ASUS_A6RP), SND_PCI_QUIRK_VENDOR(0x1043, "ASUS laptop", ALC861_FIXUP_AMP_VREF_0F), SND_PCI_QUIRK(0x1462, 0x7254, "HP DX2200", ALC861_FIXUP_NO_JACK_DETECT), - SND_PCI_QUIRK(0x1584, 0x2b01, "Haier W18", ALC861_FIXUP_AMP_VREF_0F), - SND_PCI_QUIRK(0x1584, 0x0000, "Uniwill ECS M31EI", ALC861_FIXUP_AMP_VREF_0F), + SND_PCI_QUIRK_VENDOR(0x1584, "Haier/Uniwill", ALC861_FIXUP_AMP_VREF_0F), SND_PCI_QUIRK(0x1734, 0x10c7, "FSC Amilo Pi1505", ALC861_FIXUP_FSC_AMILO_PI1505), {} }; @@ -10020,6 +10111,7 @@ static const struct snd_pci_quirk alc662_fixup_tbl[] = { SND_PCI_QUIRK(0x1025, 0x0349, "eMachines eM250", ALC662_FIXUP_INV_DMIC), SND_PCI_QUIRK(0x1025, 0x034a, "Gateway LT27", ALC662_FIXUP_INV_DMIC), SND_PCI_QUIRK(0x1025, 0x038b, "Acer Aspire 8943G", ALC662_FIXUP_ASPIRE), + SND_PCI_QUIRK(0x1025, 0x0566, "Acer Aspire Ethos 8951G", ALC669_FIXUP_ACER_ASPIRE_ETHOS), SND_PCI_QUIRK(0x1025, 0x123c, "Acer Nitro N50-600", ALC662_FIXUP_ACER_NITRO_HEADSET_MODE), SND_PCI_QUIRK(0x1025, 0x124e, "Acer 2660G", ALC662_FIXUP_ACER_X2660G_HEADSET_MODE), SND_PCI_QUIRK(0x1028, 0x05d8, "Dell", ALC668_FIXUP_DELL_MIC_NO_PRESENCE), @@ -10036,9 +10128,9 @@ static const struct snd_pci_quirk alc662_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x873e, "HP", ALC671_FIXUP_HP_HEADSET_MIC2), SND_PCI_QUIRK(0x1043, 0x1080, "Asus UX501VW", ALC668_FIXUP_HEADSET_MODE), SND_PCI_QUIRK(0x1043, 0x11cd, "Asus N550", ALC662_FIXUP_ASUS_Nx50), - SND_PCI_QUIRK(0x1043, 0x13df, "Asus N550JX", ALC662_FIXUP_BASS_1A), SND_PCI_QUIRK(0x1043, 0x129d, "Asus N750", ALC662_FIXUP_ASUS_Nx50), SND_PCI_QUIRK(0x1043, 0x12ff, "ASUS G751", ALC668_FIXUP_ASUS_G751), + SND_PCI_QUIRK(0x1043, 0x13df, "Asus N550JX", ALC662_FIXUP_BASS_1A), SND_PCI_QUIRK(0x1043, 0x1477, "ASUS N56VZ", ALC662_FIXUP_BASS_MODE4_CHMAP), SND_PCI_QUIRK(0x1043, 0x15a7, "ASUS UX51VZH", ALC662_FIXUP_BASS_16), SND_PCI_QUIRK(0x1043, 0x177d, "ASUS N551", ALC668_FIXUP_ASUS_Nx51), @@ -10058,7 +10150,6 @@ static const struct snd_pci_quirk alc662_fixup_tbl[] = { SND_PCI_QUIRK(0x1b0a, 0x01b8, "ACER Veriton", ALC662_FIXUP_ACER_VERITON), SND_PCI_QUIRK(0x1b35, 0x1234, "CZC ET26", ALC662_FIXUP_CZC_ET26), SND_PCI_QUIRK(0x1b35, 0x2206, "CZC P10T", ALC662_FIXUP_CZC_P10T), - SND_PCI_QUIRK(0x1025, 0x0566, "Acer Aspire Ethos 8951G", ALC669_FIXUP_ACER_ASPIRE_ETHOS), #if 0 /* Below is a quirk table taken from the old code. diff --git a/sound/pci/rme9652/hdsp.c b/sound/pci/rme9652/hdsp.c index 4cf879c42dc4..720297cbdf87 100644 --- a/sound/pci/rme9652/hdsp.c +++ b/sound/pci/rme9652/hdsp.c @@ -5390,7 +5390,8 @@ static int snd_hdsp_free(struct hdsp *hdsp) if (hdsp->port) pci_release_regions(hdsp->pci); - pci_disable_device(hdsp->pci); + if (pci_is_enabled(hdsp->pci)) + pci_disable_device(hdsp->pci); return 0; } diff --git a/sound/pci/rme9652/hdspm.c b/sound/pci/rme9652/hdspm.c index 8d900c132f0f..97a0bff96b28 100644 --- a/sound/pci/rme9652/hdspm.c +++ b/sound/pci/rme9652/hdspm.c @@ -6883,7 +6883,8 @@ static int snd_hdspm_free(struct hdspm * hdspm) if (hdspm->port) pci_release_regions(hdspm->pci); - pci_disable_device(hdspm->pci); + if (pci_is_enabled(hdspm->pci)) + pci_disable_device(hdspm->pci); return 0; } diff --git a/sound/pci/rme9652/rme9652.c b/sound/pci/rme9652/rme9652.c index 4df992e846f2..7a4d395abcee 100644 --- a/sound/pci/rme9652/rme9652.c +++ b/sound/pci/rme9652/rme9652.c @@ -1731,7 +1731,8 @@ static int snd_rme9652_free(struct snd_rme9652 *rme9652) if (rme9652->port) pci_release_regions(rme9652->pci); - pci_disable_device(rme9652->pci); + if (pci_is_enabled(rme9652->pci)) + pci_disable_device(rme9652->pci); return 0; } diff --git a/sound/soc/codecs/ak5558.c b/sound/soc/codecs/ak5558.c index 85bdd0534180..80b3b162ca5b 100644 --- a/sound/soc/codecs/ak5558.c +++ b/sound/soc/codecs/ak5558.c @@ -272,7 +272,7 @@ static void ak5558_power_off(struct ak5558_priv *ak5558) if (!ak5558->reset_gpiod) return; - gpiod_set_value_cansleep(ak5558->reset_gpiod, 0); + gpiod_set_value_cansleep(ak5558->reset_gpiod, 1); usleep_range(1000, 2000); } @@ -281,7 +281,7 @@ static void ak5558_power_on(struct ak5558_priv *ak5558) if (!ak5558->reset_gpiod) return; - gpiod_set_value_cansleep(ak5558->reset_gpiod, 1); + gpiod_set_value_cansleep(ak5558->reset_gpiod, 0); usleep_range(1000, 2000); } diff --git a/sound/soc/codecs/rt286.c b/sound/soc/codecs/rt286.c index 8abe232ca4a4..ff23a7d4d2ac 100644 --- a/sound/soc/codecs/rt286.c +++ b/sound/soc/codecs/rt286.c @@ -171,6 +171,9 @@ static bool rt286_readable_register(struct device *dev, unsigned int reg) case RT286_PROC_COEF: case RT286_SET_AMP_GAIN_ADC_IN1: case RT286_SET_AMP_GAIN_ADC_IN2: + case RT286_SET_GPIO_MASK: + case RT286_SET_GPIO_DIRECTION: + case RT286_SET_GPIO_DATA: case RT286_SET_POWER(RT286_DAC_OUT1): case RT286_SET_POWER(RT286_DAC_OUT2): case RT286_SET_POWER(RT286_ADC_IN1): @@ -1117,12 +1120,11 @@ static const struct dmi_system_id force_combo_jack_table[] = { { } }; -static const struct dmi_system_id dmi_dell_dino[] = { +static const struct dmi_system_id dmi_dell[] = { { - .ident = "Dell Dino", + .ident = "Dell", .matches = { DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), - DMI_MATCH(DMI_PRODUCT_NAME, "XPS 13 9343") } }, { } @@ -1133,7 +1135,7 @@ static int rt286_i2c_probe(struct i2c_client *i2c, { struct rt286_platform_data *pdata = dev_get_platdata(&i2c->dev); struct rt286_priv *rt286; - int i, ret, val; + int i, ret, vendor_id; rt286 = devm_kzalloc(&i2c->dev, sizeof(*rt286), GFP_KERNEL); @@ -1149,14 +1151,15 @@ static int rt286_i2c_probe(struct i2c_client *i2c, } ret = regmap_read(rt286->regmap, - RT286_GET_PARAM(AC_NODE_ROOT, AC_PAR_VENDOR_ID), &val); + RT286_GET_PARAM(AC_NODE_ROOT, AC_PAR_VENDOR_ID), &vendor_id); if (ret != 0) { dev_err(&i2c->dev, "I2C error %d\n", ret); return ret; } - if (val != RT286_VENDOR_ID && val != RT288_VENDOR_ID) { + if (vendor_id != RT286_VENDOR_ID && vendor_id != RT288_VENDOR_ID) { dev_err(&i2c->dev, - "Device with ID register %#x is not rt286\n", val); + "Device with ID register %#x is not rt286\n", + vendor_id); return -ENODEV; } @@ -1180,8 +1183,8 @@ static int rt286_i2c_probe(struct i2c_client *i2c, if (pdata) rt286->pdata = *pdata; - if (dmi_check_system(force_combo_jack_table) || - dmi_check_system(dmi_dell_dino)) + if ((vendor_id == RT288_VENDOR_ID && dmi_check_system(dmi_dell)) || + dmi_check_system(force_combo_jack_table)) rt286->pdata.cbj_en = true; regmap_write(rt286->regmap, RT286_SET_AUDIO_POWER, AC_PWRST_D3); @@ -1220,7 +1223,7 @@ static int rt286_i2c_probe(struct i2c_client *i2c, regmap_update_bits(rt286->regmap, RT286_DEPOP_CTRL3, 0xf777, 0x4737); regmap_update_bits(rt286->regmap, RT286_DEPOP_CTRL4, 0x00ff, 0x003f); - if (dmi_check_system(dmi_dell_dino)) { + if (vendor_id == RT288_VENDOR_ID && dmi_check_system(dmi_dell)) { regmap_update_bits(rt286->regmap, RT286_SET_GPIO_MASK, 0x40, 0x40); regmap_update_bits(rt286->regmap, diff --git a/sound/soc/codecs/rt5670.c b/sound/soc/codecs/rt5670.c index 4063aac2a443..dd69d874bad2 100644 --- a/sound/soc/codecs/rt5670.c +++ b/sound/soc/codecs/rt5670.c @@ -2980,6 +2980,18 @@ static const struct dmi_system_id dmi_platform_intel_quirks[] = { RT5670_GPIO1_IS_IRQ | RT5670_JD_MODE3), }, + { + .callback = rt5670_quirk_cb, + .ident = "Dell Venue 10 Pro 5055", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_MATCH(DMI_PRODUCT_NAME, "Venue 10 Pro 5055"), + }, + .driver_data = (unsigned long *)(RT5670_DMIC_EN | + RT5670_DMIC2_INR | + RT5670_GPIO1_IS_IRQ | + RT5670_JD_MODE1), + }, { .callback = rt5670_quirk_cb, .ident = "Aegex 10 tablet (RU2)", diff --git a/sound/soc/codecs/tlv320aic32x4.c b/sound/soc/codecs/tlv320aic32x4.c index f04f88c8d425..b689f26fc4be 100644 --- a/sound/soc/codecs/tlv320aic32x4.c +++ b/sound/soc/codecs/tlv320aic32x4.c @@ -577,12 +577,12 @@ static const struct regmap_range_cfg aic32x4_regmap_pages[] = { .window_start = 0, .window_len = 128, .range_min = 0, - .range_max = AIC32X4_RMICPGAVOL, + .range_max = AIC32X4_REFPOWERUP, }, }; const struct regmap_config aic32x4_regmap_config = { - .max_register = AIC32X4_RMICPGAVOL, + .max_register = AIC32X4_REFPOWERUP, .ranges = aic32x4_regmap_pages, .num_ranges = ARRAY_SIZE(aic32x4_regmap_pages), }; @@ -1243,6 +1243,10 @@ int aic32x4_probe(struct device *dev, struct regmap *regmap) if (ret) goto err_disable_regulators; + ret = aic32x4_register_clocks(dev, aic32x4->mclk_name); + if (ret) + goto err_disable_regulators; + ret = devm_snd_soc_register_component(dev, &soc_component_dev_aic32x4, &aic32x4_dai, 1); if (ret) { @@ -1250,10 +1254,6 @@ int aic32x4_probe(struct device *dev, struct regmap *regmap) goto err_disable_regulators; } - ret = aic32x4_register_clocks(dev, aic32x4->mclk_name); - if (ret) - goto err_disable_regulators; - return 0; err_disable_regulators: diff --git a/sound/soc/codecs/wm8960.c b/sound/soc/codecs/wm8960.c index cda9cd935d4f..9e621a254392 100644 --- a/sound/soc/codecs/wm8960.c +++ b/sound/soc/codecs/wm8960.c @@ -608,10 +608,6 @@ static const int bclk_divs[] = { * - lrclk = sysclk / dac_divs * - 10 * bclk = sysclk / bclk_divs * - * If we cannot find an exact match for (sysclk, lrclk, bclk) - * triplet, we relax the bclk such that bclk is chosen as the - * closest available frequency greater than expected bclk. - * * @wm8960: codec private data * @mclk: MCLK used to derive sysclk * @sysclk_idx: sysclk_divs index for found sysclk @@ -629,7 +625,7 @@ int wm8960_configure_sysclk(struct wm8960_priv *wm8960, int mclk, { int sysclk, bclk, lrclk; int i, j, k; - int diff, closest = mclk; + int diff; /* marker for no match */ *bclk_idx = -1; @@ -653,12 +649,6 @@ int wm8960_configure_sysclk(struct wm8960_priv *wm8960, int mclk, *bclk_idx = k; break; } - if (diff > 0 && closest > diff) { - *sysclk_idx = i; - *dac_idx = j; - *bclk_idx = k; - closest = diff; - } } if (k != ARRAY_SIZE(bclk_divs)) break; diff --git a/sound/soc/generic/audio-graph-card.c b/sound/soc/generic/audio-graph-card.c index 8c5cdcdc8713..e81b5cf0d37a 100644 --- a/sound/soc/generic/audio-graph-card.c +++ b/sound/soc/generic/audio-graph-card.c @@ -380,7 +380,7 @@ static int graph_dai_link_of(struct asoc_simple_priv *priv, struct device_node *top = dev->of_node; struct asoc_simple_dai *cpu_dai; struct asoc_simple_dai *codec_dai; - int ret, single_cpu; + int ret, single_cpu = 0; /* Do it only CPU turn */ if (!li->cpu) diff --git a/sound/soc/generic/simple-card.c b/sound/soc/generic/simple-card.c index 75365c7bb393..d916ec69c24f 100644 --- a/sound/soc/generic/simple-card.c +++ b/sound/soc/generic/simple-card.c @@ -258,7 +258,7 @@ static int simple_dai_link_of(struct asoc_simple_priv *priv, struct device_node *plat = NULL; char prop[128]; char *prefix = ""; - int ret, single_cpu; + int ret, single_cpu = 0; /* * |CPU |Codec : turn diff --git a/sound/soc/intel/Makefile b/sound/soc/intel/Makefile index 4e0248d2accc..7c5038803be7 100644 --- a/sound/soc/intel/Makefile +++ b/sound/soc/intel/Makefile @@ -5,7 +5,7 @@ obj-$(CONFIG_SND_SOC) += common/ # Platform Support obj-$(CONFIG_SND_SST_ATOM_HIFI2_PLATFORM) += atom/ obj-$(CONFIG_SND_SOC_INTEL_CATPT) += catpt/ -obj-$(CONFIG_SND_SOC_INTEL_SKYLAKE) += skylake/ +obj-$(CONFIG_SND_SOC_INTEL_SKYLAKE_COMMON) += skylake/ obj-$(CONFIG_SND_SOC_INTEL_KEEMBAY) += keembay/ # Machine support diff --git a/sound/soc/intel/boards/bytcr_rt5640.c b/sound/soc/intel/boards/bytcr_rt5640.c index 5d48cc359c3d..22912cab5e63 100644 --- a/sound/soc/intel/boards/bytcr_rt5640.c +++ b/sound/soc/intel/boards/bytcr_rt5640.c @@ -482,6 +482,9 @@ static const struct dmi_system_id byt_rt5640_quirk_table[] = { DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "T100TAF"), }, .driver_data = (void *)(BYT_RT5640_IN1_MAP | + BYT_RT5640_JD_SRC_JD2_IN4N | + BYT_RT5640_OVCD_TH_2000UA | + BYT_RT5640_OVCD_SF_0P75 | BYT_RT5640_MONO_SPEAKER | BYT_RT5640_DIFF_MIC | BYT_RT5640_SSP0_AIF2 | @@ -515,6 +518,23 @@ static const struct dmi_system_id byt_rt5640_quirk_table[] = { BYT_RT5640_SSP0_AIF1 | BYT_RT5640_MCLK_EN), }, + { + /* Chuwi Hi8 (CWI509) */ + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "Hampoo"), + DMI_MATCH(DMI_BOARD_NAME, "BYT-PA03C"), + DMI_MATCH(DMI_SYS_VENDOR, "ilife"), + DMI_MATCH(DMI_PRODUCT_NAME, "S806"), + }, + .driver_data = (void *)(BYT_RT5640_IN1_MAP | + BYT_RT5640_JD_SRC_JD2_IN4N | + BYT_RT5640_OVCD_TH_2000UA | + BYT_RT5640_OVCD_SF_0P75 | + BYT_RT5640_MONO_SPEAKER | + BYT_RT5640_DIFF_MIC | + BYT_RT5640_SSP0_AIF1 | + BYT_RT5640_MCLK_EN), + }, { .matches = { DMI_MATCH(DMI_SYS_VENDOR, "Circuitco"), diff --git a/sound/soc/intel/boards/kbl_da7219_max98927.c b/sound/soc/intel/boards/kbl_da7219_max98927.c index cc9a2509ace2..e0149cf6127d 100644 --- a/sound/soc/intel/boards/kbl_da7219_max98927.c +++ b/sound/soc/intel/boards/kbl_da7219_max98927.c @@ -282,11 +282,33 @@ static int kabylake_ssp_fixup(struct snd_soc_pcm_runtime *rtd, struct snd_interval *chan = hw_param_interval(params, SNDRV_PCM_HW_PARAM_CHANNELS); struct snd_mask *fmt = hw_param_mask(params, SNDRV_PCM_HW_PARAM_FORMAT); - struct snd_soc_dpcm *dpcm = container_of( - params, struct snd_soc_dpcm, hw_params); - struct snd_soc_dai_link *fe_dai_link = dpcm->fe->dai_link; - struct snd_soc_dai_link *be_dai_link = dpcm->be->dai_link; + struct snd_soc_dpcm *dpcm, *rtd_dpcm = NULL; + /* + * The following loop will be called only for playback stream + * In this platform, there is only one playback device on every SSP + */ + for_each_dpcm_fe(rtd, SNDRV_PCM_STREAM_PLAYBACK, dpcm) { + rtd_dpcm = dpcm; + break; + } + + /* + * This following loop will be called only for capture stream + * In this platform, there is only one capture device on every SSP + */ + for_each_dpcm_fe(rtd, SNDRV_PCM_STREAM_CAPTURE, dpcm) { + rtd_dpcm = dpcm; + break; + } + + if (!rtd_dpcm) + return -EINVAL; + + /* + * The above 2 loops are mutually exclusive based on the stream direction, + * thus rtd_dpcm variable will never be overwritten + */ /* * Topology for kblda7219m98373 & kblmax98373 supports only S24_LE, * where as kblda7219m98927 & kblmax98927 supports S16_LE by default. @@ -309,9 +331,9 @@ static int kabylake_ssp_fixup(struct snd_soc_pcm_runtime *rtd, /* * The ADSP will convert the FE rate to 48k, stereo, 24 bit */ - if (!strcmp(fe_dai_link->name, "Kbl Audio Port") || - !strcmp(fe_dai_link->name, "Kbl Audio Headset Playback") || - !strcmp(fe_dai_link->name, "Kbl Audio Capture Port")) { + if (!strcmp(rtd_dpcm->fe->dai_link->name, "Kbl Audio Port") || + !strcmp(rtd_dpcm->fe->dai_link->name, "Kbl Audio Headset Playback") || + !strcmp(rtd_dpcm->fe->dai_link->name, "Kbl Audio Capture Port")) { rate->min = rate->max = 48000; chan->min = chan->max = 2; snd_mask_none(fmt); @@ -322,7 +344,7 @@ static int kabylake_ssp_fixup(struct snd_soc_pcm_runtime *rtd, * The speaker on the SSP0 supports S16_LE and not S24_LE. * thus changing the mask here */ - if (!strcmp(be_dai_link->name, "SSP0-Codec")) + if (!strcmp(rtd_dpcm->be->dai_link->name, "SSP0-Codec")) snd_mask_set_format(fmt, SNDRV_PCM_FORMAT_S16_LE); return 0; diff --git a/sound/soc/intel/boards/sof_sdw.c b/sound/soc/intel/boards/sof_sdw.c index 8adce6417b02..ecd3f90f4bbe 100644 --- a/sound/soc/intel/boards/sof_sdw.c +++ b/sound/soc/intel/boards/sof_sdw.c @@ -187,6 +187,17 @@ static const struct dmi_system_id sof_sdw_quirk_table[] = { SOF_RT715_DAI_ID_FIX | SOF_SDW_FOUR_SPK), }, + /* AlderLake devices */ + { + .callback = sof_sdw_quirk_cb, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Intel Corporation"), + DMI_MATCH(DMI_PRODUCT_NAME, "Alder Lake Client Platform"), + }, + .driver_data = (void *)(SOF_RT711_JD_SRC_JD1 | + SOF_SDW_TGL_HDMI | + SOF_SDW_PCH_DMIC), + }, {} }; diff --git a/sound/soc/intel/boards/sof_wm8804.c b/sound/soc/intel/boards/sof_wm8804.c index a46ba13e8eb0..6a181e45143d 100644 --- a/sound/soc/intel/boards/sof_wm8804.c +++ b/sound/soc/intel/boards/sof_wm8804.c @@ -124,7 +124,11 @@ static int sof_wm8804_hw_params(struct snd_pcm_substream *substream, } snd_soc_dai_set_clkdiv(codec_dai, WM8804_MCLK_DIV, mclk_div); - snd_soc_dai_set_pll(codec_dai, 0, 0, sysclk, mclk_freq); + ret = snd_soc_dai_set_pll(codec_dai, 0, 0, sysclk, mclk_freq); + if (ret < 0) { + dev_err(rtd->card->dev, "Failed to set WM8804 PLL\n"); + return ret; + } ret = snd_soc_dai_set_sysclk(codec_dai, WM8804_TX_CLKSRC_PLL, sysclk, SND_SOC_CLOCK_OUT); diff --git a/sound/soc/intel/skylake/Makefile b/sound/soc/intel/skylake/Makefile index dd39149b89b1..1c4649bccec5 100644 --- a/sound/soc/intel/skylake/Makefile +++ b/sound/soc/intel/skylake/Makefile @@ -7,7 +7,7 @@ ifdef CONFIG_DEBUG_FS snd-soc-skl-objs += skl-debug.o endif -obj-$(CONFIG_SND_SOC_INTEL_SKYLAKE) += snd-soc-skl.o +obj-$(CONFIG_SND_SOC_INTEL_SKYLAKE_COMMON) += snd-soc-skl.o #Skylake Clock device support snd-soc-skl-ssp-clk-objs := skl-ssp-clk.o diff --git a/sound/soc/qcom/qdsp6/q6afe-clocks.c b/sound/soc/qcom/qdsp6/q6afe-clocks.c index f0362f061652..9431656283cd 100644 --- a/sound/soc/qcom/qdsp6/q6afe-clocks.c +++ b/sound/soc/qcom/qdsp6/q6afe-clocks.c @@ -11,33 +11,29 @@ #include #include "q6afe.h" -#define Q6AFE_CLK(id) &(struct q6afe_clk) { \ +#define Q6AFE_CLK(id) { \ .clk_id = id, \ .afe_clk_id = Q6AFE_##id, \ .name = #id, \ - .attributes = LPASS_CLK_ATTRIBUTE_COUPLE_NO, \ .rate = 19200000, \ - .hw.init = &(struct clk_init_data) { \ - .ops = &clk_q6afe_ops, \ - .name = #id, \ - }, \ } -#define Q6AFE_VOTE_CLK(id, blkid, n) &(struct q6afe_clk) { \ +#define Q6AFE_VOTE_CLK(id, blkid, n) { \ .clk_id = id, \ .afe_clk_id = blkid, \ - .name = #n, \ - .hw.init = &(struct clk_init_data) { \ - .ops = &clk_vote_q6afe_ops, \ - .name = #id, \ - }, \ + .name = n, \ } -struct q6afe_clk { - struct device *dev; +struct q6afe_clk_init { int clk_id; int afe_clk_id; char *name; + int rate; +}; + +struct q6afe_clk { + struct device *dev; + int afe_clk_id; int attributes; int rate; uint32_t handle; @@ -48,8 +44,7 @@ struct q6afe_clk { struct q6afe_cc { struct device *dev; - struct q6afe_clk **clks; - int num_clks; + struct q6afe_clk *clks[Q6AFE_MAX_CLK_ID]; }; static int clk_q6afe_prepare(struct clk_hw *hw) @@ -105,7 +100,7 @@ static int clk_vote_q6afe_block(struct clk_hw *hw) struct q6afe_clk *clk = to_q6afe_clk(hw); return q6afe_vote_lpass_core_hw(clk->dev, clk->afe_clk_id, - clk->name, &clk->handle); + clk_hw_get_name(&clk->hw), &clk->handle); } static void clk_unvote_q6afe_block(struct clk_hw *hw) @@ -120,84 +115,76 @@ static const struct clk_ops clk_vote_q6afe_ops = { .unprepare = clk_unvote_q6afe_block, }; -static struct q6afe_clk *q6afe_clks[Q6AFE_MAX_CLK_ID] = { - [LPASS_CLK_ID_PRI_MI2S_IBIT] = Q6AFE_CLK(LPASS_CLK_ID_PRI_MI2S_IBIT), - [LPASS_CLK_ID_PRI_MI2S_EBIT] = Q6AFE_CLK(LPASS_CLK_ID_PRI_MI2S_EBIT), - [LPASS_CLK_ID_SEC_MI2S_IBIT] = Q6AFE_CLK(LPASS_CLK_ID_SEC_MI2S_IBIT), - [LPASS_CLK_ID_SEC_MI2S_EBIT] = Q6AFE_CLK(LPASS_CLK_ID_SEC_MI2S_EBIT), - [LPASS_CLK_ID_TER_MI2S_IBIT] = Q6AFE_CLK(LPASS_CLK_ID_TER_MI2S_IBIT), - [LPASS_CLK_ID_TER_MI2S_EBIT] = Q6AFE_CLK(LPASS_CLK_ID_TER_MI2S_EBIT), - [LPASS_CLK_ID_QUAD_MI2S_IBIT] = Q6AFE_CLK(LPASS_CLK_ID_QUAD_MI2S_IBIT), - [LPASS_CLK_ID_QUAD_MI2S_EBIT] = Q6AFE_CLK(LPASS_CLK_ID_QUAD_MI2S_EBIT), - [LPASS_CLK_ID_SPEAKER_I2S_IBIT] = - Q6AFE_CLK(LPASS_CLK_ID_SPEAKER_I2S_IBIT), - [LPASS_CLK_ID_SPEAKER_I2S_EBIT] = - Q6AFE_CLK(LPASS_CLK_ID_SPEAKER_I2S_EBIT), - [LPASS_CLK_ID_SPEAKER_I2S_OSR] = - Q6AFE_CLK(LPASS_CLK_ID_SPEAKER_I2S_OSR), - [LPASS_CLK_ID_QUI_MI2S_IBIT] = Q6AFE_CLK(LPASS_CLK_ID_QUI_MI2S_IBIT), - [LPASS_CLK_ID_QUI_MI2S_EBIT] = Q6AFE_CLK(LPASS_CLK_ID_QUI_MI2S_EBIT), - [LPASS_CLK_ID_SEN_MI2S_IBIT] = Q6AFE_CLK(LPASS_CLK_ID_SEN_MI2S_IBIT), - [LPASS_CLK_ID_SEN_MI2S_EBIT] = Q6AFE_CLK(LPASS_CLK_ID_SEN_MI2S_EBIT), - [LPASS_CLK_ID_INT0_MI2S_IBIT] = Q6AFE_CLK(LPASS_CLK_ID_INT0_MI2S_IBIT), - [LPASS_CLK_ID_INT1_MI2S_IBIT] = Q6AFE_CLK(LPASS_CLK_ID_INT1_MI2S_IBIT), - [LPASS_CLK_ID_INT2_MI2S_IBIT] = Q6AFE_CLK(LPASS_CLK_ID_INT2_MI2S_IBIT), - [LPASS_CLK_ID_INT3_MI2S_IBIT] = Q6AFE_CLK(LPASS_CLK_ID_INT3_MI2S_IBIT), - [LPASS_CLK_ID_INT4_MI2S_IBIT] = Q6AFE_CLK(LPASS_CLK_ID_INT4_MI2S_IBIT), - [LPASS_CLK_ID_INT5_MI2S_IBIT] = Q6AFE_CLK(LPASS_CLK_ID_INT5_MI2S_IBIT), - [LPASS_CLK_ID_INT6_MI2S_IBIT] = Q6AFE_CLK(LPASS_CLK_ID_INT6_MI2S_IBIT), - [LPASS_CLK_ID_QUI_MI2S_OSR] = Q6AFE_CLK(LPASS_CLK_ID_QUI_MI2S_OSR), - [LPASS_CLK_ID_PRI_PCM_IBIT] = Q6AFE_CLK(LPASS_CLK_ID_PRI_PCM_IBIT), - [LPASS_CLK_ID_PRI_PCM_EBIT] = Q6AFE_CLK(LPASS_CLK_ID_PRI_PCM_EBIT), - [LPASS_CLK_ID_SEC_PCM_IBIT] = Q6AFE_CLK(LPASS_CLK_ID_SEC_PCM_IBIT), - [LPASS_CLK_ID_SEC_PCM_EBIT] = Q6AFE_CLK(LPASS_CLK_ID_SEC_PCM_EBIT), - [LPASS_CLK_ID_TER_PCM_IBIT] = Q6AFE_CLK(LPASS_CLK_ID_TER_PCM_IBIT), - [LPASS_CLK_ID_TER_PCM_EBIT] = Q6AFE_CLK(LPASS_CLK_ID_TER_PCM_EBIT), - [LPASS_CLK_ID_QUAD_PCM_IBIT] = Q6AFE_CLK(LPASS_CLK_ID_QUAD_PCM_IBIT), - [LPASS_CLK_ID_QUAD_PCM_EBIT] = Q6AFE_CLK(LPASS_CLK_ID_QUAD_PCM_EBIT), - [LPASS_CLK_ID_QUIN_PCM_IBIT] = Q6AFE_CLK(LPASS_CLK_ID_QUIN_PCM_IBIT), - [LPASS_CLK_ID_QUIN_PCM_EBIT] = Q6AFE_CLK(LPASS_CLK_ID_QUIN_PCM_EBIT), - [LPASS_CLK_ID_QUI_PCM_OSR] = Q6AFE_CLK(LPASS_CLK_ID_QUI_PCM_OSR), - [LPASS_CLK_ID_PRI_TDM_IBIT] = Q6AFE_CLK(LPASS_CLK_ID_PRI_TDM_IBIT), - [LPASS_CLK_ID_PRI_TDM_EBIT] = Q6AFE_CLK(LPASS_CLK_ID_PRI_TDM_EBIT), - [LPASS_CLK_ID_SEC_TDM_IBIT] = Q6AFE_CLK(LPASS_CLK_ID_SEC_TDM_IBIT), - [LPASS_CLK_ID_SEC_TDM_EBIT] = Q6AFE_CLK(LPASS_CLK_ID_SEC_TDM_EBIT), - [LPASS_CLK_ID_TER_TDM_IBIT] = Q6AFE_CLK(LPASS_CLK_ID_TER_TDM_IBIT), - [LPASS_CLK_ID_TER_TDM_EBIT] = Q6AFE_CLK(LPASS_CLK_ID_TER_TDM_EBIT), - [LPASS_CLK_ID_QUAD_TDM_IBIT] = Q6AFE_CLK(LPASS_CLK_ID_QUAD_TDM_IBIT), - [LPASS_CLK_ID_QUAD_TDM_EBIT] = Q6AFE_CLK(LPASS_CLK_ID_QUAD_TDM_EBIT), - [LPASS_CLK_ID_QUIN_TDM_IBIT] = Q6AFE_CLK(LPASS_CLK_ID_QUIN_TDM_IBIT), - [LPASS_CLK_ID_QUIN_TDM_EBIT] = Q6AFE_CLK(LPASS_CLK_ID_QUIN_TDM_EBIT), - [LPASS_CLK_ID_QUIN_TDM_OSR] = Q6AFE_CLK(LPASS_CLK_ID_QUIN_TDM_OSR), - [LPASS_CLK_ID_MCLK_1] = Q6AFE_CLK(LPASS_CLK_ID_MCLK_1), - [LPASS_CLK_ID_MCLK_2] = Q6AFE_CLK(LPASS_CLK_ID_MCLK_2), - [LPASS_CLK_ID_MCLK_3] = Q6AFE_CLK(LPASS_CLK_ID_MCLK_3), - [LPASS_CLK_ID_MCLK_4] = Q6AFE_CLK(LPASS_CLK_ID_MCLK_4), - [LPASS_CLK_ID_INTERNAL_DIGITAL_CODEC_CORE] = - Q6AFE_CLK(LPASS_CLK_ID_INTERNAL_DIGITAL_CODEC_CORE), - [LPASS_CLK_ID_INT_MCLK_0] = Q6AFE_CLK(LPASS_CLK_ID_INT_MCLK_0), - [LPASS_CLK_ID_INT_MCLK_1] = Q6AFE_CLK(LPASS_CLK_ID_INT_MCLK_1), - [LPASS_CLK_ID_WSA_CORE_MCLK] = Q6AFE_CLK(LPASS_CLK_ID_WSA_CORE_MCLK), - [LPASS_CLK_ID_WSA_CORE_NPL_MCLK] = - Q6AFE_CLK(LPASS_CLK_ID_WSA_CORE_NPL_MCLK), - [LPASS_CLK_ID_VA_CORE_MCLK] = Q6AFE_CLK(LPASS_CLK_ID_VA_CORE_MCLK), - [LPASS_CLK_ID_TX_CORE_MCLK] = Q6AFE_CLK(LPASS_CLK_ID_TX_CORE_MCLK), - [LPASS_CLK_ID_TX_CORE_NPL_MCLK] = - Q6AFE_CLK(LPASS_CLK_ID_TX_CORE_NPL_MCLK), - [LPASS_CLK_ID_RX_CORE_MCLK] = Q6AFE_CLK(LPASS_CLK_ID_RX_CORE_MCLK), - [LPASS_CLK_ID_RX_CORE_NPL_MCLK] = - Q6AFE_CLK(LPASS_CLK_ID_RX_CORE_NPL_MCLK), - [LPASS_CLK_ID_VA_CORE_2X_MCLK] = - Q6AFE_CLK(LPASS_CLK_ID_VA_CORE_2X_MCLK), - [LPASS_HW_AVTIMER_VOTE] = Q6AFE_VOTE_CLK(LPASS_HW_AVTIMER_VOTE, - Q6AFE_LPASS_CORE_AVTIMER_BLOCK, - "LPASS_AVTIMER_MACRO"), - [LPASS_HW_MACRO_VOTE] = Q6AFE_VOTE_CLK(LPASS_HW_MACRO_VOTE, - Q6AFE_LPASS_CORE_HW_MACRO_BLOCK, - "LPASS_HW_MACRO"), - [LPASS_HW_DCODEC_VOTE] = Q6AFE_VOTE_CLK(LPASS_HW_DCODEC_VOTE, - Q6AFE_LPASS_CORE_HW_DCODEC_BLOCK, - "LPASS_HW_DCODEC"), +static const struct q6afe_clk_init q6afe_clks[] = { + Q6AFE_CLK(LPASS_CLK_ID_PRI_MI2S_IBIT), + Q6AFE_CLK(LPASS_CLK_ID_PRI_MI2S_EBIT), + Q6AFE_CLK(LPASS_CLK_ID_SEC_MI2S_IBIT), + Q6AFE_CLK(LPASS_CLK_ID_SEC_MI2S_EBIT), + Q6AFE_CLK(LPASS_CLK_ID_TER_MI2S_IBIT), + Q6AFE_CLK(LPASS_CLK_ID_TER_MI2S_EBIT), + Q6AFE_CLK(LPASS_CLK_ID_QUAD_MI2S_IBIT), + Q6AFE_CLK(LPASS_CLK_ID_QUAD_MI2S_EBIT), + Q6AFE_CLK(LPASS_CLK_ID_SPEAKER_I2S_IBIT), + Q6AFE_CLK(LPASS_CLK_ID_SPEAKER_I2S_EBIT), + Q6AFE_CLK(LPASS_CLK_ID_SPEAKER_I2S_OSR), + Q6AFE_CLK(LPASS_CLK_ID_QUI_MI2S_IBIT), + Q6AFE_CLK(LPASS_CLK_ID_QUI_MI2S_EBIT), + Q6AFE_CLK(LPASS_CLK_ID_SEN_MI2S_IBIT), + Q6AFE_CLK(LPASS_CLK_ID_SEN_MI2S_EBIT), + Q6AFE_CLK(LPASS_CLK_ID_INT0_MI2S_IBIT), + Q6AFE_CLK(LPASS_CLK_ID_INT1_MI2S_IBIT), + Q6AFE_CLK(LPASS_CLK_ID_INT2_MI2S_IBIT), + Q6AFE_CLK(LPASS_CLK_ID_INT3_MI2S_IBIT), + Q6AFE_CLK(LPASS_CLK_ID_INT4_MI2S_IBIT), + Q6AFE_CLK(LPASS_CLK_ID_INT5_MI2S_IBIT), + Q6AFE_CLK(LPASS_CLK_ID_INT6_MI2S_IBIT), + Q6AFE_CLK(LPASS_CLK_ID_QUI_MI2S_OSR), + Q6AFE_CLK(LPASS_CLK_ID_PRI_PCM_IBIT), + Q6AFE_CLK(LPASS_CLK_ID_PRI_PCM_EBIT), + Q6AFE_CLK(LPASS_CLK_ID_SEC_PCM_IBIT), + Q6AFE_CLK(LPASS_CLK_ID_SEC_PCM_EBIT), + Q6AFE_CLK(LPASS_CLK_ID_TER_PCM_IBIT), + Q6AFE_CLK(LPASS_CLK_ID_TER_PCM_EBIT), + Q6AFE_CLK(LPASS_CLK_ID_QUAD_PCM_IBIT), + Q6AFE_CLK(LPASS_CLK_ID_QUAD_PCM_EBIT), + Q6AFE_CLK(LPASS_CLK_ID_QUIN_PCM_IBIT), + Q6AFE_CLK(LPASS_CLK_ID_QUIN_PCM_EBIT), + Q6AFE_CLK(LPASS_CLK_ID_QUI_PCM_OSR), + Q6AFE_CLK(LPASS_CLK_ID_PRI_TDM_IBIT), + Q6AFE_CLK(LPASS_CLK_ID_PRI_TDM_EBIT), + Q6AFE_CLK(LPASS_CLK_ID_SEC_TDM_IBIT), + Q6AFE_CLK(LPASS_CLK_ID_SEC_TDM_EBIT), + Q6AFE_CLK(LPASS_CLK_ID_TER_TDM_IBIT), + Q6AFE_CLK(LPASS_CLK_ID_TER_TDM_EBIT), + Q6AFE_CLK(LPASS_CLK_ID_QUAD_TDM_IBIT), + Q6AFE_CLK(LPASS_CLK_ID_QUAD_TDM_EBIT), + Q6AFE_CLK(LPASS_CLK_ID_QUIN_TDM_IBIT), + Q6AFE_CLK(LPASS_CLK_ID_QUIN_TDM_EBIT), + Q6AFE_CLK(LPASS_CLK_ID_QUIN_TDM_OSR), + Q6AFE_CLK(LPASS_CLK_ID_MCLK_1), + Q6AFE_CLK(LPASS_CLK_ID_MCLK_2), + Q6AFE_CLK(LPASS_CLK_ID_MCLK_3), + Q6AFE_CLK(LPASS_CLK_ID_MCLK_4), + Q6AFE_CLK(LPASS_CLK_ID_INTERNAL_DIGITAL_CODEC_CORE), + Q6AFE_CLK(LPASS_CLK_ID_INT_MCLK_0), + Q6AFE_CLK(LPASS_CLK_ID_INT_MCLK_1), + Q6AFE_CLK(LPASS_CLK_ID_WSA_CORE_MCLK), + Q6AFE_CLK(LPASS_CLK_ID_WSA_CORE_NPL_MCLK), + Q6AFE_CLK(LPASS_CLK_ID_VA_CORE_MCLK), + Q6AFE_CLK(LPASS_CLK_ID_TX_CORE_MCLK), + Q6AFE_CLK(LPASS_CLK_ID_TX_CORE_NPL_MCLK), + Q6AFE_CLK(LPASS_CLK_ID_RX_CORE_MCLK), + Q6AFE_CLK(LPASS_CLK_ID_RX_CORE_NPL_MCLK), + Q6AFE_CLK(LPASS_CLK_ID_VA_CORE_2X_MCLK), + Q6AFE_VOTE_CLK(LPASS_HW_AVTIMER_VOTE, + Q6AFE_LPASS_CORE_AVTIMER_BLOCK, + "LPASS_AVTIMER_MACRO"), + Q6AFE_VOTE_CLK(LPASS_HW_MACRO_VOTE, + Q6AFE_LPASS_CORE_HW_MACRO_BLOCK, + "LPASS_HW_MACRO"), + Q6AFE_VOTE_CLK(LPASS_HW_DCODEC_VOTE, + Q6AFE_LPASS_CORE_HW_DCODEC_BLOCK, + "LPASS_HW_DCODEC"), }; static struct clk_hw *q6afe_of_clk_hw_get(struct of_phandle_args *clkspec, @@ -207,7 +194,7 @@ static struct clk_hw *q6afe_of_clk_hw_get(struct of_phandle_args *clkspec, unsigned int idx = clkspec->args[0]; unsigned int attr = clkspec->args[1]; - if (idx >= cc->num_clks || attr > LPASS_CLK_ATTRIBUTE_COUPLE_DIVISOR) { + if (idx >= Q6AFE_MAX_CLK_ID || attr > LPASS_CLK_ATTRIBUTE_COUPLE_DIVISOR) { dev_err(cc->dev, "Invalid clk specifier (%d, %d)\n", idx, attr); return ERR_PTR(-EINVAL); } @@ -230,20 +217,36 @@ static int q6afe_clock_dev_probe(struct platform_device *pdev) if (!cc) return -ENOMEM; - cc->clks = &q6afe_clks[0]; - cc->num_clks = ARRAY_SIZE(q6afe_clks); + cc->dev = dev; for (i = 0; i < ARRAY_SIZE(q6afe_clks); i++) { - if (!q6afe_clks[i]) - continue; + unsigned int id = q6afe_clks[i].clk_id; + struct clk_init_data init = { + .name = q6afe_clks[i].name, + }; + struct q6afe_clk *clk; + + clk = devm_kzalloc(dev, sizeof(*clk), GFP_KERNEL); + if (!clk) + return -ENOMEM; + + clk->dev = dev; + clk->afe_clk_id = q6afe_clks[i].afe_clk_id; + clk->rate = q6afe_clks[i].rate; + clk->hw.init = &init; + + if (clk->rate) + init.ops = &clk_q6afe_ops; + else + init.ops = &clk_vote_q6afe_ops; - q6afe_clks[i]->dev = dev; + cc->clks[id] = clk; - ret = devm_clk_hw_register(dev, &q6afe_clks[i]->hw); + ret = devm_clk_hw_register(dev, &clk->hw); if (ret) return ret; } - ret = of_clk_add_hw_provider(dev->of_node, q6afe_of_clk_hw_get, cc); + ret = devm_of_clk_add_hw_provider(dev, q6afe_of_clk_hw_get, cc); if (ret) return ret; diff --git a/sound/soc/qcom/qdsp6/q6afe.c b/sound/soc/qcom/qdsp6/q6afe.c index cad1cd1bfdf0..4327b72162ec 100644 --- a/sound/soc/qcom/qdsp6/q6afe.c +++ b/sound/soc/qcom/qdsp6/q6afe.c @@ -1681,7 +1681,7 @@ int q6afe_unvote_lpass_core_hw(struct device *dev, uint32_t hw_block_id, EXPORT_SYMBOL(q6afe_unvote_lpass_core_hw); int q6afe_vote_lpass_core_hw(struct device *dev, uint32_t hw_block_id, - char *client_name, uint32_t *client_handle) + const char *client_name, uint32_t *client_handle) { struct q6afe *afe = dev_get_drvdata(dev->parent); struct afe_cmd_remote_lpass_core_hw_vote_request *vote_cfg; diff --git a/sound/soc/qcom/qdsp6/q6afe.h b/sound/soc/qcom/qdsp6/q6afe.h index 22e10269aa10..3845b56c0ed3 100644 --- a/sound/soc/qcom/qdsp6/q6afe.h +++ b/sound/soc/qcom/qdsp6/q6afe.h @@ -236,7 +236,7 @@ int q6afe_port_set_sysclk(struct q6afe_port *port, int clk_id, int q6afe_set_lpass_clock(struct device *dev, int clk_id, int clk_src, int clk_root, unsigned int freq); int q6afe_vote_lpass_core_hw(struct device *dev, uint32_t hw_block_id, - char *client_name, uint32_t *client_handle); + const char *client_name, uint32_t *client_handle); int q6afe_unvote_lpass_core_hw(struct device *dev, uint32_t hw_block_id, uint32_t client_handle); #endif /* __Q6AFE_H__ */ diff --git a/sound/soc/samsung/tm2_wm5110.c b/sound/soc/samsung/tm2_wm5110.c index 9300fef9bf26..125e07f65d2b 100644 --- a/sound/soc/samsung/tm2_wm5110.c +++ b/sound/soc/samsung/tm2_wm5110.c @@ -553,7 +553,7 @@ static int tm2_probe(struct platform_device *pdev) ret = of_parse_phandle_with_args(dev->of_node, "i2s-controller", cells_name, i, &args); - if (!args.np) { + if (ret) { dev_err(dev, "i2s-controller property parse error: %d\n", i); ret = -EINVAL; goto dai_node_put; diff --git a/sound/soc/sh/rcar/core.c b/sound/soc/sh/rcar/core.c index 1029d8d9d800..d2b4632d9c2a 100644 --- a/sound/soc/sh/rcar/core.c +++ b/sound/soc/sh/rcar/core.c @@ -1428,8 +1428,75 @@ static int rsnd_hw_params(struct snd_soc_component *component, } if (io->converted_chan) dev_dbg(dev, "convert channels = %d\n", io->converted_chan); - if (io->converted_rate) + if (io->converted_rate) { + /* + * SRC supports convert rates from params_rate(hw_params)/k_down + * to params_rate(hw_params)*k_up, where k_up is always 6, and + * k_down depends on number of channels and SRC unit. + * So all SRC units can upsample audio up to 6 times regardless + * its number of channels. And all SRC units can downsample + * 2 channel audio up to 6 times too. + */ + int k_up = 6; + int k_down = 6; + int channel; + struct rsnd_mod *src_mod = rsnd_io_to_mod_src(io); + dev_dbg(dev, "convert rate = %d\n", io->converted_rate); + + channel = io->converted_chan ? io->converted_chan : + params_channels(hw_params); + + switch (rsnd_mod_id(src_mod)) { + /* + * SRC0 can downsample 4, 6 and 8 channel audio up to 4 times. + * SRC1, SRC3 and SRC4 can downsample 4 channel audio + * up to 4 times. + * SRC1, SRC3 and SRC4 can downsample 6 and 8 channel audio + * no more than twice. + */ + case 1: + case 3: + case 4: + if (channel > 4) { + k_down = 2; + break; + } + fallthrough; + case 0: + if (channel > 2) + k_down = 4; + break; + + /* Other SRC units do not support more than 2 channels */ + default: + if (channel > 2) + return -EINVAL; + } + + if (params_rate(hw_params) > io->converted_rate * k_down) { + hw_param_interval(hw_params, SNDRV_PCM_HW_PARAM_RATE)->min = + io->converted_rate * k_down; + hw_param_interval(hw_params, SNDRV_PCM_HW_PARAM_RATE)->max = + io->converted_rate * k_down; + hw_params->cmask |= SNDRV_PCM_HW_PARAM_RATE; + } else if (params_rate(hw_params) * k_up < io->converted_rate) { + hw_param_interval(hw_params, SNDRV_PCM_HW_PARAM_RATE)->min = + (io->converted_rate + k_up - 1) / k_up; + hw_param_interval(hw_params, SNDRV_PCM_HW_PARAM_RATE)->max = + (io->converted_rate + k_up - 1) / k_up; + hw_params->cmask |= SNDRV_PCM_HW_PARAM_RATE; + } + + /* + * TBD: Max SRC input and output rates also depend on number + * of channels and SRC unit: + * SRC1, SRC3 and SRC4 do not support more than 128kHz + * for 6 channel and 96kHz for 8 channel audio. + * Perhaps this function should return EINVAL if the input or + * the output rate exceeds the limitation. + */ + } } return rsnd_dai_call(hw_params, io, substream, hw_params); diff --git a/sound/soc/sh/rcar/ssi.c b/sound/soc/sh/rcar/ssi.c index d0ded427a836..042207c11651 100644 --- a/sound/soc/sh/rcar/ssi.c +++ b/sound/soc/sh/rcar/ssi.c @@ -507,10 +507,15 @@ static int rsnd_ssi_init(struct rsnd_mod *mod, struct rsnd_priv *priv) { struct rsnd_ssi *ssi = rsnd_mod_to_ssi(mod); + int ret; if (!rsnd_ssi_is_run_mods(mod, io)) return 0; + ret = rsnd_ssi_master_clk_start(mod, io); + if (ret < 0) + return ret; + ssi->usrcnt++; rsnd_mod_power_on(mod); @@ -792,7 +797,6 @@ static void __rsnd_ssi_interrupt(struct rsnd_mod *mod, SSI_SYS_STATUS(i * 2), 0xf << (id * 4)); stop = true; - break; } } break; @@ -810,7 +814,6 @@ static void __rsnd_ssi_interrupt(struct rsnd_mod *mod, SSI_SYS_STATUS((i * 2) + 1), 0xf << 4); stop = true; - break; } } break; @@ -1060,13 +1063,6 @@ static int rsnd_ssi_pio_pointer(struct rsnd_mod *mod, return 0; } -static int rsnd_ssi_prepare(struct rsnd_mod *mod, - struct rsnd_dai_stream *io, - struct rsnd_priv *priv) -{ - return rsnd_ssi_master_clk_start(mod, io); -} - static struct rsnd_mod_ops rsnd_ssi_pio_ops = { .name = SSI_NAME, .probe = rsnd_ssi_common_probe, @@ -1079,7 +1075,6 @@ static struct rsnd_mod_ops rsnd_ssi_pio_ops = { .pointer = rsnd_ssi_pio_pointer, .pcm_new = rsnd_ssi_pcm_new, .hw_params = rsnd_ssi_hw_params, - .prepare = rsnd_ssi_prepare, .get_status = rsnd_ssi_get_status, }; @@ -1166,7 +1161,6 @@ static struct rsnd_mod_ops rsnd_ssi_dma_ops = { .pcm_new = rsnd_ssi_pcm_new, .fallback = rsnd_ssi_fallback, .hw_params = rsnd_ssi_hw_params, - .prepare = rsnd_ssi_prepare, .get_status = rsnd_ssi_get_status, }; diff --git a/sound/soc/soc-compress.c b/sound/soc/soc-compress.c index 246a5e32e22a..b4810266f5e5 100644 --- a/sound/soc/soc-compress.c +++ b/sound/soc/soc-compress.c @@ -153,7 +153,9 @@ static int soc_compr_open_fe(struct snd_compr_stream *cstream) fe->dpcm[stream].state = SND_SOC_DPCM_STATE_OPEN; fe->dpcm[stream].runtime_update = SND_SOC_DPCM_UPDATE_NO; + mutex_lock_nested(&fe->card->pcm_mutex, fe->card->pcm_subclass); snd_soc_runtime_activate(fe, stream); + mutex_unlock(&fe->card->pcm_mutex); mutex_unlock(&fe->card->mutex); @@ -181,7 +183,9 @@ static int soc_compr_free_fe(struct snd_compr_stream *cstream) mutex_lock_nested(&fe->card->mutex, SND_SOC_CARD_CLASS_RUNTIME); + mutex_lock_nested(&fe->card->pcm_mutex, fe->card->pcm_subclass); snd_soc_runtime_deactivate(fe, stream); + mutex_unlock(&fe->card->pcm_mutex); fe->dpcm[stream].runtime_update = SND_SOC_DPCM_UPDATE_FE; diff --git a/sound/soc/tegra/tegra30_i2s.c b/sound/soc/tegra/tegra30_i2s.c index 6740df541508..3d22c1be6f3d 100644 --- a/sound/soc/tegra/tegra30_i2s.c +++ b/sound/soc/tegra/tegra30_i2s.c @@ -58,8 +58,18 @@ static int tegra30_i2s_runtime_resume(struct device *dev) } regcache_cache_only(i2s->regmap, false); + regcache_mark_dirty(i2s->regmap); + + ret = regcache_sync(i2s->regmap); + if (ret) + goto disable_clocks; return 0; + +disable_clocks: + clk_disable_unprepare(i2s->clk_i2s); + + return ret; } static int tegra30_i2s_set_fmt(struct snd_soc_dai *dai, @@ -551,37 +561,11 @@ static int tegra30_i2s_platform_remove(struct platform_device *pdev) return 0; } -#ifdef CONFIG_PM_SLEEP -static int tegra30_i2s_suspend(struct device *dev) -{ - struct tegra30_i2s *i2s = dev_get_drvdata(dev); - - regcache_mark_dirty(i2s->regmap); - - return 0; -} - -static int tegra30_i2s_resume(struct device *dev) -{ - struct tegra30_i2s *i2s = dev_get_drvdata(dev); - int ret; - - ret = pm_runtime_get_sync(dev); - if (ret < 0) { - pm_runtime_put(dev); - return ret; - } - ret = regcache_sync(i2s->regmap); - pm_runtime_put(dev); - - return ret; -} -#endif - static const struct dev_pm_ops tegra30_i2s_pm_ops = { SET_RUNTIME_PM_OPS(tegra30_i2s_runtime_suspend, tegra30_i2s_runtime_resume, NULL) - SET_SYSTEM_SLEEP_PM_OPS(tegra30_i2s_suspend, tegra30_i2s_resume) + SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, + pm_runtime_force_resume) }; static struct platform_driver tegra30_i2s_driver = { diff --git a/sound/usb/card.c b/sound/usb/card.c index 0826a437f8fc..7b7526d3a56e 100644 --- a/sound/usb/card.c +++ b/sound/usb/card.c @@ -181,9 +181,8 @@ static int snd_usb_create_stream(struct snd_usb_audio *chip, int ctrlif, int int ctrlif, interface); return -EINVAL; } - usb_driver_claim_interface(&usb_audio_driver, iface, (void *)-1L); - - return 0; + return usb_driver_claim_interface(&usb_audio_driver, iface, + USB_AUDIO_IFACE_UNUSED); } if ((altsd->bInterfaceClass != USB_CLASS_AUDIO && @@ -203,7 +202,8 @@ static int snd_usb_create_stream(struct snd_usb_audio *chip, int ctrlif, int int if (! snd_usb_parse_audio_interface(chip, interface)) { usb_set_interface(dev, interface, 0); /* reset the current interface */ - usb_driver_claim_interface(&usb_audio_driver, iface, (void *)-1L); + return usb_driver_claim_interface(&usb_audio_driver, iface, + USB_AUDIO_IFACE_UNUSED); } return 0; @@ -862,7 +862,7 @@ static void usb_audio_disconnect(struct usb_interface *intf) struct snd_card *card; struct list_head *p; - if (chip == (void *)-1L) + if (chip == USB_AUDIO_IFACE_UNUSED) return; card = chip->card; @@ -992,7 +992,7 @@ static int usb_audio_suspend(struct usb_interface *intf, pm_message_t message) struct usb_mixer_interface *mixer; struct list_head *p; - if (chip == (void *)-1L) + if (chip == USB_AUDIO_IFACE_UNUSED) return 0; if (!chip->num_suspended_intf++) { @@ -1022,7 +1022,7 @@ static int __usb_audio_resume(struct usb_interface *intf, bool reset_resume) struct list_head *p; int err = 0; - if (chip == (void *)-1L) + if (chip == USB_AUDIO_IFACE_UNUSED) return 0; atomic_inc(&chip->active); /* avoid autopm */ diff --git a/sound/usb/clock.c b/sound/usb/clock.c index a746802d0ac3..17bbde73d4d1 100644 --- a/sound/usb/clock.c +++ b/sound/usb/clock.c @@ -296,7 +296,7 @@ static int __uac_clock_find_source(struct snd_usb_audio *chip, selector = snd_usb_find_clock_selector(chip->ctrl_intf, entity_id); if (selector) { - int ret, i, cur; + int ret, i, cur, err; if (selector->bNrInPins == 1) { ret = 1; @@ -324,13 +324,17 @@ static int __uac_clock_find_source(struct snd_usb_audio *chip, ret = __uac_clock_find_source(chip, fmt, selector->baCSourceID[ret - 1], visited, validate); + if (ret > 0) { + err = uac_clock_selector_set_val(chip, entity_id, cur); + if (err < 0) + return err; + } + if (!validate || ret > 0 || !chip->autoclock) return ret; /* The current clock source is invalid, try others. */ for (i = 1; i <= selector->bNrInPins; i++) { - int err; - if (i == cur) continue; @@ -396,7 +400,7 @@ static int __uac3_clock_find_source(struct snd_usb_audio *chip, selector = snd_usb_find_clock_selector_v3(chip->ctrl_intf, entity_id); if (selector) { - int ret, i, cur; + int ret, i, cur, err; /* the entity ID we are looking for is a selector. * find out what it currently selects */ @@ -418,6 +422,12 @@ static int __uac3_clock_find_source(struct snd_usb_audio *chip, ret = __uac3_clock_find_source(chip, fmt, selector->baCSourceID[ret - 1], visited, validate); + if (ret > 0) { + err = uac_clock_selector_set_val(chip, entity_id, cur); + if (err < 0) + return err; + } + if (!validate || ret > 0 || !chip->autoclock) return ret; diff --git a/sound/usb/endpoint.c b/sound/usb/endpoint.c index 102d53515a76..933586a895e7 100644 --- a/sound/usb/endpoint.c +++ b/sound/usb/endpoint.c @@ -1442,11 +1442,11 @@ void snd_usb_endpoint_stop(struct snd_usb_endpoint *ep) if (snd_BUG_ON(!atomic_read(&ep->running))) return; - if (ep->sync_source) - WRITE_ONCE(ep->sync_source->sync_sink, NULL); - - if (!atomic_dec_return(&ep->running)) + if (!atomic_dec_return(&ep->running)) { + if (ep->sync_source) + WRITE_ONCE(ep->sync_source->sync_sink, NULL); stop_urbs(ep, false); + } } /** diff --git a/sound/usb/midi.c b/sound/usb/midi.c index 0c23fa6d8525..cd46ca7cd28d 100644 --- a/sound/usb/midi.c +++ b/sound/usb/midi.c @@ -1332,7 +1332,7 @@ static int snd_usbmidi_in_endpoint_create(struct snd_usb_midi *umidi, error: snd_usbmidi_in_endpoint_delete(ep); - return -ENOMEM; + return err; } /* diff --git a/sound/usb/mixer_maps.c b/sound/usb/mixer_maps.c index 646deb6244b1..c5794e83fd80 100644 --- a/sound/usb/mixer_maps.c +++ b/sound/usb/mixer_maps.c @@ -337,6 +337,13 @@ static const struct usbmix_name_map bose_companion5_map[] = { { 0 } /* terminator */ }; +/* Sennheiser Communications Headset [PC 8], the dB value is reported as -6 negative maximum */ +static const struct usbmix_dB_map sennheiser_pc8_dB = {-9500, 0}; +static const struct usbmix_name_map sennheiser_pc8_map[] = { + { 9, NULL, .dB = &sennheiser_pc8_dB }, + { 0 } /* terminator */ +}; + /* * Dell usb dock with ALC4020 codec had a firmware problem where it got * screwed up when zero volume is passed; just skip it as a workaround @@ -593,6 +600,11 @@ static const struct usbmix_ctl_map usbmix_ctl_maps[] = { .id = USB_ID(0x17aa, 0x1046), .map = lenovo_p620_rear_map, }, + { + /* Sennheiser Communications Headset [PC 8] */ + .id = USB_ID(0x1395, 0x0025), + .map = sennheiser_pc8_map, + }, { 0 } /* terminator */ }; diff --git a/sound/usb/quirks-table.h b/sound/usb/quirks-table.h index 1165a5ac60f2..8a8fe2b980a1 100644 --- a/sound/usb/quirks-table.h +++ b/sound/usb/quirks-table.h @@ -2376,6 +2376,16 @@ YAMAHA_DEVICE(0x7010, "UB99"), } }, +{ + USB_DEVICE_VENDOR_SPEC(0x0944, 0x0204), + .driver_info = (unsigned long) & (const struct snd_usb_audio_quirk) { + .vendor_name = "KORG, Inc.", + /* .product_name = "ToneLab EX", */ + .ifnum = 3, + .type = QUIRK_MIDI_STANDARD_INTERFACE, + } +}, + /* AKAI devices */ { USB_DEVICE(0x09e8, 0x0062), @@ -3817,6 +3827,69 @@ AU0828_DEVICE(0x2040, 0x7270, "Hauppauge", "HVR-950Q"), } } }, +{ + /* + * Pioneer DJ DJM-850 + * 8 channels playback and 8 channels capture @ 44.1/48/96kHz S24LE + * Playback on EP 0x05 + * Capture on EP 0x86 + */ + USB_DEVICE_VENDOR_SPEC(0x08e4, 0x0163), + .driver_info = (unsigned long) &(const struct snd_usb_audio_quirk) { + .ifnum = QUIRK_ANY_INTERFACE, + .type = QUIRK_COMPOSITE, + .data = (const struct snd_usb_audio_quirk[]) { + { + .ifnum = 0, + .type = QUIRK_AUDIO_FIXED_ENDPOINT, + .data = &(const struct audioformat) { + .formats = SNDRV_PCM_FMTBIT_S24_3LE, + .channels = 8, + .iface = 0, + .altsetting = 1, + .altset_idx = 1, + .endpoint = 0x05, + .ep_attr = USB_ENDPOINT_XFER_ISOC| + USB_ENDPOINT_SYNC_ASYNC| + USB_ENDPOINT_USAGE_DATA, + .rates = SNDRV_PCM_RATE_44100| + SNDRV_PCM_RATE_48000| + SNDRV_PCM_RATE_96000, + .rate_min = 44100, + .rate_max = 96000, + .nr_rates = 3, + .rate_table = (unsigned int[]) { 44100, 48000, 96000 } + } + }, + { + .ifnum = 0, + .type = QUIRK_AUDIO_FIXED_ENDPOINT, + .data = &(const struct audioformat) { + .formats = SNDRV_PCM_FMTBIT_S24_3LE, + .channels = 8, + .iface = 0, + .altsetting = 1, + .altset_idx = 1, + .endpoint = 0x86, + .ep_idx = 1, + .ep_attr = USB_ENDPOINT_XFER_ISOC| + USB_ENDPOINT_SYNC_ASYNC| + USB_ENDPOINT_USAGE_DATA, + .rates = SNDRV_PCM_RATE_44100| + SNDRV_PCM_RATE_48000| + SNDRV_PCM_RATE_96000, + .rate_min = 44100, + .rate_max = 96000, + .nr_rates = 3, + .rate_table = (unsigned int[]) { 44100, 48000, 96000 } + } + }, + { + .ifnum = -1 + } + } + } +}, { /* * Pioneer DJ DJM-450 diff --git a/sound/usb/quirks.c b/sound/usb/quirks.c index 176437a441e6..7c6e83eee71d 100644 --- a/sound/usb/quirks.c +++ b/sound/usb/quirks.c @@ -55,8 +55,12 @@ static int create_composite_quirk(struct snd_usb_audio *chip, if (!iface) continue; if (quirk->ifnum != probed_ifnum && - !usb_interface_claimed(iface)) - usb_driver_claim_interface(driver, iface, (void *)-1L); + !usb_interface_claimed(iface)) { + err = usb_driver_claim_interface(driver, iface, + USB_AUDIO_IFACE_UNUSED); + if (err < 0) + return err; + } } return 0; @@ -426,8 +430,12 @@ static int create_autodetect_quirks(struct snd_usb_audio *chip, continue; err = create_autodetect_quirk(chip, iface, driver); - if (err >= 0) - usb_driver_claim_interface(driver, iface, (void *)-1L); + if (err >= 0) { + err = usb_driver_claim_interface(driver, iface, + USB_AUDIO_IFACE_UNUSED); + if (err < 0) + return err; + } } return 0; diff --git a/sound/usb/usbaudio.h b/sound/usb/usbaudio.h index 60b9dd7df6bb..8794c8658ab9 100644 --- a/sound/usb/usbaudio.h +++ b/sound/usb/usbaudio.h @@ -61,6 +61,8 @@ struct snd_usb_audio { struct media_intf_devnode *ctl_intf_media_devnode; }; +#define USB_AUDIO_IFACE_UNUSED ((void *)-1L) + #define usb_audio_err(chip, fmt, args...) \ dev_err(&(chip)->dev->dev, fmt, ##args) #define usb_audio_warn(chip, fmt, args...) \ diff --git a/tools/bpf/bpftool/btf.c b/tools/bpf/bpftool/btf.c index fe9e7b3a4b50..1326fff3629b 100644 --- a/tools/bpf/bpftool/btf.c +++ b/tools/bpf/bpftool/btf.c @@ -538,6 +538,7 @@ static int do_dump(int argc, char **argv) NEXT_ARG(); if (argc < 1) { p_err("expecting value for 'format' option\n"); + err = -EINVAL; goto done; } if (strcmp(*argv, "c") == 0) { @@ -547,11 +548,13 @@ static int do_dump(int argc, char **argv) } else { p_err("unrecognized format specifier: '%s', possible values: raw, c", *argv); + err = -EINVAL; goto done; } NEXT_ARG(); } else { p_err("unrecognized option: '%s'", *argv); + err = -EINVAL; goto done; } } diff --git a/tools/bpf/bpftool/main.c b/tools/bpf/bpftool/main.c index b86f450e6fce..d9afb730136a 100644 --- a/tools/bpf/bpftool/main.c +++ b/tools/bpf/bpftool/main.c @@ -276,7 +276,7 @@ static int do_batch(int argc, char **argv) int n_argc; FILE *fp; char *cp; - int err; + int err = 0; int i; if (argc < 2) { @@ -370,7 +370,6 @@ static int do_batch(int argc, char **argv) } else { if (!json_output) printf("processed %d commands\n", lines); - err = 0; } err_close: if (fp != stdin) diff --git a/tools/bpf/bpftool/map.c b/tools/bpf/bpftool/map.c index b400364ee054..09ae0381205b 100644 --- a/tools/bpf/bpftool/map.c +++ b/tools/bpf/bpftool/map.c @@ -100,7 +100,7 @@ static int do_dump_btf(const struct btf_dumper *d, void *value) { __u32 value_id; - int ret; + int ret = 0; /* start of key-value pair */ jsonw_start_object(d->jw); diff --git a/tools/lib/bpf/bpf_core_read.h b/tools/lib/bpf/bpf_core_read.h index 53b3e199fb25..09ebe3db5f2f 100644 --- a/tools/lib/bpf/bpf_core_read.h +++ b/tools/lib/bpf/bpf_core_read.h @@ -88,11 +88,19 @@ enum bpf_enum_value_kind { const void *p = (const void *)s + __CORE_RELO(s, field, BYTE_OFFSET); \ unsigned long long val; \ \ + /* This is a so-called barrier_var() operation that makes specified \ + * variable "a black box" for optimizing compiler. \ + * It forces compiler to perform BYTE_OFFSET relocation on p and use \ + * its calculated value in the switch below, instead of applying \ + * the same relocation 4 times for each individual memory load. \ + */ \ + asm volatile("" : "=r"(p) : "0"(p)); \ + \ switch (__CORE_RELO(s, field, BYTE_SIZE)) { \ - case 1: val = *(const unsigned char *)p; \ - case 2: val = *(const unsigned short *)p; \ - case 4: val = *(const unsigned int *)p; \ - case 8: val = *(const unsigned long long *)p; \ + case 1: val = *(const unsigned char *)p; break; \ + case 2: val = *(const unsigned short *)p; break; \ + case 4: val = *(const unsigned int *)p; break; \ + case 8: val = *(const unsigned long long *)p; break; \ } \ val <<= __CORE_RELO(s, field, LSHIFT_U64); \ if (__CORE_RELO(s, field, SIGNED)) \ diff --git a/tools/lib/bpf/bpf_tracing.h b/tools/lib/bpf/bpf_tracing.h index f9ef37707888..1c2e91ee041d 100644 --- a/tools/lib/bpf/bpf_tracing.h +++ b/tools/lib/bpf/bpf_tracing.h @@ -413,20 +413,38 @@ typeof(name(0)) name(struct pt_regs *ctx) \ } \ static __always_inline typeof(name(0)) ____##name(struct pt_regs *ctx, ##args) +#define ___bpf_fill0(arr, p, x) do {} while (0) +#define ___bpf_fill1(arr, p, x) arr[p] = x +#define ___bpf_fill2(arr, p, x, args...) arr[p] = x; ___bpf_fill1(arr, p + 1, args) +#define ___bpf_fill3(arr, p, x, args...) arr[p] = x; ___bpf_fill2(arr, p + 1, args) +#define ___bpf_fill4(arr, p, x, args...) arr[p] = x; ___bpf_fill3(arr, p + 1, args) +#define ___bpf_fill5(arr, p, x, args...) arr[p] = x; ___bpf_fill4(arr, p + 1, args) +#define ___bpf_fill6(arr, p, x, args...) arr[p] = x; ___bpf_fill5(arr, p + 1, args) +#define ___bpf_fill7(arr, p, x, args...) arr[p] = x; ___bpf_fill6(arr, p + 1, args) +#define ___bpf_fill8(arr, p, x, args...) arr[p] = x; ___bpf_fill7(arr, p + 1, args) +#define ___bpf_fill9(arr, p, x, args...) arr[p] = x; ___bpf_fill8(arr, p + 1, args) +#define ___bpf_fill10(arr, p, x, args...) arr[p] = x; ___bpf_fill9(arr, p + 1, args) +#define ___bpf_fill11(arr, p, x, args...) arr[p] = x; ___bpf_fill10(arr, p + 1, args) +#define ___bpf_fill12(arr, p, x, args...) arr[p] = x; ___bpf_fill11(arr, p + 1, args) +#define ___bpf_fill(arr, args...) \ + ___bpf_apply(___bpf_fill, ___bpf_narg(args))(arr, 0, args) + /* * BPF_SEQ_PRINTF to wrap bpf_seq_printf to-be-printed values * in a structure. */ -#define BPF_SEQ_PRINTF(seq, fmt, args...) \ - ({ \ - _Pragma("GCC diagnostic push") \ - _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ - static const char ___fmt[] = fmt; \ - unsigned long long ___param[] = { args }; \ - _Pragma("GCC diagnostic pop") \ - int ___ret = bpf_seq_printf(seq, ___fmt, sizeof(___fmt), \ - ___param, sizeof(___param)); \ - ___ret; \ - }) +#define BPF_SEQ_PRINTF(seq, fmt, args...) \ +({ \ + static const char ___fmt[] = fmt; \ + unsigned long long ___param[___bpf_narg(args)]; \ + \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ + ___bpf_fill(___param, args); \ + _Pragma("GCC diagnostic pop") \ + \ + bpf_seq_printf(seq, ___fmt, sizeof(___fmt), \ + ___param, sizeof(___param)); \ +}) #endif diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h index 1237bcd1dd17..5b8a6ea44b38 100644 --- a/tools/lib/bpf/btf.h +++ b/tools/lib/bpf/btf.h @@ -173,6 +173,7 @@ struct btf_dump_emit_type_decl_opts { int indent_level; /* strip all the const/volatile/restrict mods */ bool strip_mods; + size_t :0; }; #define btf_dump_emit_type_decl_opts__last_field strip_mods diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h index 3c35eb401931..3d690d4e785c 100644 --- a/tools/lib/bpf/libbpf.h +++ b/tools/lib/bpf/libbpf.h @@ -507,6 +507,7 @@ struct xdp_link_info { struct bpf_xdp_set_link_opts { size_t sz; int old_fd; + size_t :0; }; #define bpf_xdp_set_link_opts__last_field old_fd diff --git a/tools/lib/bpf/ringbuf.c b/tools/lib/bpf/ringbuf.c index e7a8d847161f..1d80ad4e0de8 100644 --- a/tools/lib/bpf/ringbuf.c +++ b/tools/lib/bpf/ringbuf.c @@ -202,9 +202,11 @@ static inline int roundup_len(__u32 len) return (len + 7) / 8 * 8; } -static int ringbuf_process_ring(struct ring* r) +static int64_t ringbuf_process_ring(struct ring* r) { - int *len_ptr, len, err, cnt = 0; + int *len_ptr, len, err; + /* 64-bit to avoid overflow in case of extreme application behavior */ + int64_t cnt = 0; unsigned long cons_pos, prod_pos; bool got_new_data; void *sample; @@ -244,12 +246,14 @@ static int ringbuf_process_ring(struct ring* r) } /* Consume available ring buffer(s) data without event polling. - * Returns number of records consumed across all registered ring buffers, or - * negative number if any of the callbacks return error. + * Returns number of records consumed across all registered ring buffers (or + * INT_MAX, whichever is less), or negative number if any of the callbacks + * return error. */ int ring_buffer__consume(struct ring_buffer *rb) { - int i, err, res = 0; + int64_t err, res = 0; + int i; for (i = 0; i < rb->ring_cnt; i++) { struct ring *ring = &rb->rings[i]; @@ -259,18 +263,24 @@ int ring_buffer__consume(struct ring_buffer *rb) return err; res += err; } + if (res > INT_MAX) + return INT_MAX; return res; } /* Poll for available data and consume records, if any are available. - * Returns number of records consumed, or negative number, if any of the - * registered callbacks returned error. + * Returns number of records consumed (or INT_MAX, whichever is less), or + * negative number, if any of the registered callbacks returned error. */ int ring_buffer__poll(struct ring_buffer *rb, int timeout_ms) { - int i, cnt, err, res = 0; + int i, cnt; + int64_t err, res = 0; cnt = epoll_wait(rb->epoll_fd, rb->events, rb->ring_cnt, timeout_ms); + if (cnt < 0) + return -errno; + for (i = 0; i < cnt; i++) { __u32 ring_id = rb->events[i].data.fd; struct ring *ring = &rb->rings[ring_id]; @@ -280,7 +290,9 @@ int ring_buffer__poll(struct ring_buffer *rb, int timeout_ms) return err; res += err; } - return cnt < 0 ? -errno : res; + if (res > INT_MAX) + return INT_MAX; + return res; } /* Get an fd that can be used to sleep until data is available in the ring(s) */ diff --git a/tools/lib/perf/include/perf/event.h b/tools/lib/perf/include/perf/event.h index d82054225fcc..4d0c02ba3f7d 100644 --- a/tools/lib/perf/include/perf/event.h +++ b/tools/lib/perf/include/perf/event.h @@ -8,6 +8,8 @@ #include #include /* pid_t */ +#define event_contains(obj, mem) ((obj).header.size > offsetof(typeof(obj), mem)) + struct perf_record_mmap { struct perf_event_header header; __u32 pid, tid; @@ -346,8 +348,9 @@ struct perf_record_time_conv { __u64 time_zero; __u64 time_cycles; __u64 time_mask; - bool cap_user_time_zero; - bool cap_user_time_short; + __u8 cap_user_time_zero; + __u8 cap_user_time_short; + __u8 reserved[6]; /* For alignment */ }; struct perf_record_header_feature { diff --git a/tools/perf/Makefile.config b/tools/perf/Makefile.config index d8e59d31399a..c955cd683e22 100644 --- a/tools/perf/Makefile.config +++ b/tools/perf/Makefile.config @@ -530,6 +530,7 @@ ifndef NO_LIBELF ifdef LIBBPF_DYNAMIC ifeq ($(feature-libbpf), 1) EXTLIBS += -lbpf + $(call detected,CONFIG_LIBBPF_DYNAMIC) else dummy := $(error Error: No libbpf devel library found, please install libbpf-devel); endif diff --git a/tools/perf/pmu-events/arch/x86/amdzen1/cache.json b/tools/perf/pmu-events/arch/x86/amdzen1/cache.json index 4ea7ec4f496e..008f1683e540 100644 --- a/tools/perf/pmu-events/arch/x86/amdzen1/cache.json +++ b/tools/perf/pmu-events/arch/x86/amdzen1/cache.json @@ -275,7 +275,7 @@ { "EventName": "l2_pf_hit_l2", "EventCode": "0x70", - "BriefDescription": "L2 prefetch hit in L2.", + "BriefDescription": "L2 prefetch hit in L2. Use l2_cache_hits_from_l2_hwpf instead.", "UMask": "0xff" }, { diff --git a/tools/perf/pmu-events/arch/x86/amdzen1/recommended.json b/tools/perf/pmu-events/arch/x86/amdzen1/recommended.json index 2cfe2d2f3bfd..3c954543d1ae 100644 --- a/tools/perf/pmu-events/arch/x86/amdzen1/recommended.json +++ b/tools/perf/pmu-events/arch/x86/amdzen1/recommended.json @@ -79,10 +79,10 @@ "UMask": "0x70" }, { - "MetricName": "l2_cache_hits_from_l2_hwpf", + "EventName": "l2_cache_hits_from_l2_hwpf", + "EventCode": "0x70", "BriefDescription": "L2 Cache Hits from L2 HWPF", - "MetricExpr": "l2_pf_hit_l2 + l2_pf_miss_l2_hit_l3 + l2_pf_miss_l2_l3", - "MetricGroup": "l2_cache" + "UMask": "0xff" }, { "EventName": "l3_accesses", diff --git a/tools/perf/pmu-events/arch/x86/amdzen2/cache.json b/tools/perf/pmu-events/arch/x86/amdzen2/cache.json index f61b982f83ca..8ba84a48188d 100644 --- a/tools/perf/pmu-events/arch/x86/amdzen2/cache.json +++ b/tools/perf/pmu-events/arch/x86/amdzen2/cache.json @@ -205,7 +205,7 @@ { "EventName": "l2_pf_hit_l2", "EventCode": "0x70", - "BriefDescription": "L2 prefetch hit in L2.", + "BriefDescription": "L2 prefetch hit in L2. Use l2_cache_hits_from_l2_hwpf instead.", "UMask": "0xff" }, { diff --git a/tools/perf/pmu-events/arch/x86/amdzen2/recommended.json b/tools/perf/pmu-events/arch/x86/amdzen2/recommended.json index 2ef91e25e661..1c624cee9ef4 100644 --- a/tools/perf/pmu-events/arch/x86/amdzen2/recommended.json +++ b/tools/perf/pmu-events/arch/x86/amdzen2/recommended.json @@ -79,10 +79,10 @@ "UMask": "0x70" }, { - "MetricName": "l2_cache_hits_from_l2_hwpf", + "EventName": "l2_cache_hits_from_l2_hwpf", + "EventCode": "0x70", "BriefDescription": "L2 Cache Hits from L2 HWPF", - "MetricExpr": "l2_pf_hit_l2 + l2_pf_miss_l2_hit_l3 + l2_pf_miss_l2_l3", - "MetricGroup": "l2_cache" + "UMask": "0xff" }, { "EventName": "l3_accesses", diff --git a/tools/perf/trace/beauty/fsconfig.sh b/tools/perf/trace/beauty/fsconfig.sh index 83fb24df05c9..bc6ef7bb7a5f 100755 --- a/tools/perf/trace/beauty/fsconfig.sh +++ b/tools/perf/trace/beauty/fsconfig.sh @@ -10,8 +10,7 @@ fi linux_mount=${linux_header_dir}/mount.h printf "static const char *fsconfig_cmds[] = {\n" -regex='^[[:space:]]*+FSCONFIG_([[:alnum:]_]+)[[:space:]]*=[[:space:]]*([[:digit:]]+)[[:space:]]*,[[:space:]]*.*' -egrep $regex ${linux_mount} | \ - sed -r "s/$regex/\2 \1/g" | \ - xargs printf "\t[%s] = \"%s\",\n" +ms='[[:space:]]*' +sed -nr "s/^${ms}FSCONFIG_([[:alnum:]_]+)${ms}=${ms}([[:digit:]]+)${ms},.*/\t[\2] = \"\1\",/p" \ + ${linux_mount} printf "};\n" diff --git a/tools/perf/util/Build b/tools/perf/util/Build index e3e12f9d4733..5a296ac69415 100644 --- a/tools/perf/util/Build +++ b/tools/perf/util/Build @@ -141,7 +141,14 @@ perf-$(CONFIG_LIBELF) += symbol-elf.o perf-$(CONFIG_LIBELF) += probe-file.o perf-$(CONFIG_LIBELF) += probe-event.o +ifdef CONFIG_LIBBPF_DYNAMIC + hashmap := 1 +endif ifndef CONFIG_LIBBPF + hashmap := 1 +endif + +ifdef hashmap perf-y += hashmap.o endif diff --git a/tools/perf/util/jitdump.c b/tools/perf/util/jitdump.c index 9760d8e7b386..917a9c707371 100644 --- a/tools/perf/util/jitdump.c +++ b/tools/perf/util/jitdump.c @@ -396,21 +396,31 @@ static pid_t jr_entry_tid(struct jit_buf_desc *jd, union jr_entry *jr) static uint64_t convert_timestamp(struct jit_buf_desc *jd, uint64_t timestamp) { - struct perf_tsc_conversion tc; + struct perf_tsc_conversion tc = { .time_shift = 0, }; + struct perf_record_time_conv *time_conv = &jd->session->time_conv; if (!jd->use_arch_timestamp) return timestamp; - tc.time_shift = jd->session->time_conv.time_shift; - tc.time_mult = jd->session->time_conv.time_mult; - tc.time_zero = jd->session->time_conv.time_zero; - tc.time_cycles = jd->session->time_conv.time_cycles; - tc.time_mask = jd->session->time_conv.time_mask; - tc.cap_user_time_zero = jd->session->time_conv.cap_user_time_zero; - tc.cap_user_time_short = jd->session->time_conv.cap_user_time_short; + tc.time_shift = time_conv->time_shift; + tc.time_mult = time_conv->time_mult; + tc.time_zero = time_conv->time_zero; - if (!tc.cap_user_time_zero) - return 0; + /* + * The event TIME_CONV was extended for the fields from "time_cycles" + * when supported cap_user_time_short, for backward compatibility, + * checks the event size and assigns these extended fields if these + * fields are contained in the event. + */ + if (event_contains(*time_conv, time_cycles)) { + tc.time_cycles = time_conv->time_cycles; + tc.time_mask = time_conv->time_mask; + tc.cap_user_time_zero = time_conv->cap_user_time_zero; + tc.cap_user_time_short = time_conv->cap_user_time_short; + + if (!tc.cap_user_time_zero) + return 0; + } return tsc_to_perf_time(timestamp, &tc); } diff --git a/tools/perf/util/session.c b/tools/perf/util/session.c index 859832a82496..e9d4e6f4bdf3 100644 --- a/tools/perf/util/session.c +++ b/tools/perf/util/session.c @@ -949,6 +949,19 @@ static void perf_event__stat_round_swap(union perf_event *event, event->stat_round.time = bswap_64(event->stat_round.time); } +static void perf_event__time_conv_swap(union perf_event *event, + bool sample_id_all __maybe_unused) +{ + event->time_conv.time_shift = bswap_64(event->time_conv.time_shift); + event->time_conv.time_mult = bswap_64(event->time_conv.time_mult); + event->time_conv.time_zero = bswap_64(event->time_conv.time_zero); + + if (event_contains(event->time_conv, time_cycles)) { + event->time_conv.time_cycles = bswap_64(event->time_conv.time_cycles); + event->time_conv.time_mask = bswap_64(event->time_conv.time_mask); + } +} + typedef void (*perf_event__swap_op)(union perf_event *event, bool sample_id_all); @@ -985,7 +998,7 @@ static perf_event__swap_op perf_event__swap_ops[] = { [PERF_RECORD_STAT] = perf_event__stat_swap, [PERF_RECORD_STAT_ROUND] = perf_event__stat_round_swap, [PERF_RECORD_EVENT_UPDATE] = perf_event__event_update_swap, - [PERF_RECORD_TIME_CONV] = perf_event__all64_swap, + [PERF_RECORD_TIME_CONV] = perf_event__time_conv_swap, [PERF_RECORD_HEADER_MAX] = NULL, }; diff --git a/tools/perf/util/symbol_fprintf.c b/tools/perf/util/symbol_fprintf.c index 35c936ce33ef..2664fb65e47a 100644 --- a/tools/perf/util/symbol_fprintf.c +++ b/tools/perf/util/symbol_fprintf.c @@ -68,7 +68,7 @@ size_t dso__fprintf_symbols_by_name(struct dso *dso, for (nd = rb_first_cached(&dso->symbol_names); nd; nd = rb_next(nd)) { pos = rb_entry(nd, struct symbol_name_rb_node, rb_node); - fprintf(fp, "%s\n", pos->sym.name); + ret += fprintf(fp, "%s\n", pos->sym.name); } return ret; diff --git a/tools/power/x86/intel-speed-select/isst-display.c b/tools/power/x86/intel-speed-select/isst-display.c index 8e54ce47648e..3bf1820c0da1 100644 --- a/tools/power/x86/intel-speed-select/isst-display.c +++ b/tools/power/x86/intel-speed-select/isst-display.c @@ -25,10 +25,14 @@ static void printcpulist(int str_len, char *str, int mask_size, index = snprintf(&str[curr_index], str_len - curr_index, ","); curr_index += index; + if (curr_index >= str_len) + break; } index = snprintf(&str[curr_index], str_len - curr_index, "%d", i); curr_index += index; + if (curr_index >= str_len) + break; first = 0; } } @@ -64,10 +68,14 @@ static void printcpumask(int str_len, char *str, int mask_size, index = snprintf(&str[curr_index], str_len - curr_index, "%08x", mask[i]); curr_index += index; + if (curr_index >= str_len) + break; if (i) { strncat(&str[curr_index], ",", str_len - curr_index); curr_index++; } + if (curr_index >= str_len) + break; } free(mask); @@ -185,7 +193,7 @@ static void _isst_pbf_display_information(int cpu, FILE *outf, int level, int disp_level) { char header[256]; - char value[256]; + char value[512]; snprintf(header, sizeof(header), "speed-select-base-freq-properties"); format_and_print(outf, disp_level, header, NULL); @@ -349,7 +357,7 @@ void isst_ctdp_display_information(int cpu, FILE *outf, int tdp_level, struct isst_pkg_ctdp *pkg_dev) { char header[256]; - char value[256]; + char value[512]; static int level; int i; diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c index a7c4f0772e53..002697021474 100644 --- a/tools/power/x86/turbostat/turbostat.c +++ b/tools/power/x86/turbostat/turbostat.c @@ -291,13 +291,16 @@ struct msr_sum_array { /* The percpu MSR sum array.*/ struct msr_sum_array *per_cpu_msr_sum; -int idx_to_offset(int idx) +off_t idx_to_offset(int idx) { - int offset; + off_t offset; switch (idx) { case IDX_PKG_ENERGY: - offset = MSR_PKG_ENERGY_STATUS; + if (do_rapl & RAPL_AMD_F17H) + offset = MSR_PKG_ENERGY_STAT; + else + offset = MSR_PKG_ENERGY_STATUS; break; case IDX_DRAM_ENERGY: offset = MSR_DRAM_ENERGY_STATUS; @@ -320,12 +323,13 @@ int idx_to_offset(int idx) return offset; } -int offset_to_idx(int offset) +int offset_to_idx(off_t offset) { int idx; switch (offset) { case MSR_PKG_ENERGY_STATUS: + case MSR_PKG_ENERGY_STAT: idx = IDX_PKG_ENERGY; break; case MSR_DRAM_ENERGY_STATUS: @@ -353,7 +357,7 @@ int idx_valid(int idx) { switch (idx) { case IDX_PKG_ENERGY: - return do_rapl & RAPL_PKG; + return do_rapl & (RAPL_PKG | RAPL_AMD_F17H); case IDX_DRAM_ENERGY: return do_rapl & RAPL_DRAM; case IDX_PP0_ENERGY: @@ -3272,7 +3276,7 @@ static int update_msr_sum(struct thread_data *t, struct core_data *c, struct pkg for (i = IDX_PKG_ENERGY; i < IDX_COUNT; i++) { unsigned long long msr_cur, msr_last; - int offset; + off_t offset; if (!idx_valid(i)) continue; @@ -3281,7 +3285,8 @@ static int update_msr_sum(struct thread_data *t, struct core_data *c, struct pkg continue; ret = get_msr(cpu, offset, &msr_cur); if (ret) { - fprintf(outf, "Can not update msr(0x%x)\n", offset); + fprintf(outf, "Can not update msr(0x%llx)\n", + (unsigned long long)offset); continue; } @@ -4817,33 +4822,12 @@ double discover_bclk(unsigned int family, unsigned int model) * below this value, including the Digital Thermal Sensor (DTS), * Package Thermal Management Sensor (PTM), and thermal event thresholds. */ -int read_tcc_activation_temp() +int set_temperature_target(struct thread_data *t, struct core_data *c, struct pkg_data *p) { unsigned long long msr; - unsigned int tcc, target_c, offset_c; - - /* Temperature Target MSR is Nehalem and newer only */ - if (!do_nhm_platform_info) - return 0; - - if (get_msr(base_cpu, MSR_IA32_TEMPERATURE_TARGET, &msr)) - return 0; - - target_c = (msr >> 16) & 0xFF; - - offset_c = (msr >> 24) & 0xF; - - tcc = target_c - offset_c; - - if (!quiet) - fprintf(outf, "cpu%d: MSR_IA32_TEMPERATURE_TARGET: 0x%08llx (%d C) (%d default - %d offset)\n", - base_cpu, msr, tcc, target_c, offset_c); - - return tcc; -} + unsigned int target_c_local; + int cpu; -int set_temperature_target(struct thread_data *t, struct core_data *c, struct pkg_data *p) -{ /* tcc_activation_temp is used only for dts or ptm */ if (!(do_dts || do_ptm)) return 0; @@ -4852,18 +4836,43 @@ int set_temperature_target(struct thread_data *t, struct core_data *c, struct pk if (!(t->flags & CPU_IS_FIRST_THREAD_IN_CORE) || !(t->flags & CPU_IS_FIRST_CORE_IN_PACKAGE)) return 0; + cpu = t->cpu_id; + if (cpu_migrate(cpu)) { + fprintf(outf, "Could not migrate to CPU %d\n", cpu); + return -1; + } + if (tcc_activation_temp_override != 0) { tcc_activation_temp = tcc_activation_temp_override; - fprintf(outf, "Using cmdline TCC Target (%d C)\n", tcc_activation_temp); + fprintf(outf, "cpu%d: Using cmdline TCC Target (%d C)\n", + cpu, tcc_activation_temp); return 0; } - tcc_activation_temp = read_tcc_activation_temp(); - if (tcc_activation_temp) - return 0; + /* Temperature Target MSR is Nehalem and newer only */ + if (!do_nhm_platform_info) + goto guess; + + if (get_msr(base_cpu, MSR_IA32_TEMPERATURE_TARGET, &msr)) + goto guess; + + target_c_local = (msr >> 16) & 0xFF; + + if (!quiet) + fprintf(outf, "cpu%d: MSR_IA32_TEMPERATURE_TARGET: 0x%08llx (%d C)\n", + cpu, msr, target_c_local); + + if (!target_c_local) + goto guess; + + tcc_activation_temp = target_c_local; + + return 0; +guess: tcc_activation_temp = TJMAX_DEFAULT; - fprintf(outf, "Guessing tjMax %d C, Please use -T to specify\n", tcc_activation_temp); + fprintf(outf, "cpu%d: Guessing tjMax %d C, Please use -T to specify\n", + cpu, tcc_activation_temp); return 0; } diff --git a/tools/spi/Makefile b/tools/spi/Makefile index ada881afb489..0aa6dbd31fb8 100644 --- a/tools/spi/Makefile +++ b/tools/spi/Makefile @@ -25,11 +25,12 @@ include $(srctree)/tools/build/Makefile.include # # We need the following to be outside of kernel tree # -$(OUTPUT)include/linux/spi/spidev.h: ../../include/uapi/linux/spi/spidev.h +$(OUTPUT)include/linux/spi: ../../include/uapi/linux/spi mkdir -p $(OUTPUT)include/linux/spi 2>&1 || true ln -sf $(CURDIR)/../../include/uapi/linux/spi/spidev.h $@ + ln -sf $(CURDIR)/../../include/uapi/linux/spi/spi.h $@ -prepare: $(OUTPUT)include/linux/spi/spidev.h +prepare: $(OUTPUT)include/linux/spi # # spidev_test diff --git a/tools/testing/selftests/arm64/mte/Makefile b/tools/testing/selftests/arm64/mte/Makefile index 0b3af552632a..df15d44aeb8d 100644 --- a/tools/testing/selftests/arm64/mte/Makefile +++ b/tools/testing/selftests/arm64/mte/Makefile @@ -6,9 +6,7 @@ SRCS := $(filter-out mte_common_util.c,$(wildcard *.c)) PROGS := $(patsubst %.c,%,$(SRCS)) #Add mte compiler option -ifneq ($(shell $(CC) --version 2>&1 | head -n 1 | grep gcc),) CFLAGS += -march=armv8.5-a+memtag -endif #check if the compiler works well mte_cc_support := $(shell if ($(CC) $(CFLAGS) -E -x c /dev/null -o /dev/null 2>&1) then echo "1"; fi) diff --git a/tools/testing/selftests/arm64/mte/mte_common_util.c b/tools/testing/selftests/arm64/mte/mte_common_util.c index 39f8908988ea..70665ba88cbb 100644 --- a/tools/testing/selftests/arm64/mte/mte_common_util.c +++ b/tools/testing/selftests/arm64/mte/mte_common_util.c @@ -278,22 +278,13 @@ int mte_switch_mode(int mte_option, unsigned long incl_mask) return 0; } -#define ID_AA64PFR1_MTE_SHIFT 8 -#define ID_AA64PFR1_MTE 2 - int mte_default_setup(void) { - unsigned long hwcaps = getauxval(AT_HWCAP); + unsigned long hwcaps2 = getauxval(AT_HWCAP2); unsigned long en = 0; int ret; - if (!(hwcaps & HWCAP_CPUID)) { - ksft_print_msg("FAIL: CPUID registers unavailable\n"); - return KSFT_FAIL; - } - /* Read ID_AA64PFR1_EL1 register */ - asm volatile("mrs %0, id_aa64pfr1_el1" : "=r"(hwcaps) : : "memory"); - if (((hwcaps >> ID_AA64PFR1_MTE_SHIFT) & MT_TAG_MASK) != ID_AA64PFR1_MTE) { + if (!(hwcaps2 & HWCAP2_MTE)) { ksft_print_msg("FAIL: MTE features unavailable\n"); return KSFT_SKIP; } diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 044bfdcf5b74..76a325862119 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -221,7 +221,7 @@ $(HOST_BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile) \ DESTDIR=$(HOST_SCRATCH_DIR)/ prefix= all install_headers endif -$(INCLUDE_DIR)/vmlinux.h: $(VMLINUX_BTF) | $(BPFTOOL) $(INCLUDE_DIR) +$(INCLUDE_DIR)/vmlinux.h: $(VMLINUX_BTF) $(BPFTOOL) | $(INCLUDE_DIR) ifeq ($(VMLINUX_H),) $(call msg,GEN,,$@) $(Q)$(BPFTOOL) btf dump file $(VMLINUX_BTF) format c > $@ @@ -346,7 +346,8 @@ $(TRUNNER_BPF_OBJS): $(TRUNNER_OUTPUT)/%.o: \ $(TRUNNER_BPF_SKELS): $(TRUNNER_OUTPUT)/%.skel.h: \ $(TRUNNER_OUTPUT)/%.o \ - | $(BPFTOOL) $(TRUNNER_OUTPUT) + $(BPFTOOL) \ + | $(TRUNNER_OUTPUT) $$(call msg,GEN-SKEL,$(TRUNNER_BINARY),$$@) $(Q)$$(BPFTOOL) gen skeleton $$< > $$@ endif diff --git a/tools/testing/selftests/bpf/prog_tests/core_reloc.c b/tools/testing/selftests/bpf/prog_tests/core_reloc.c index 06eb956ff7bb..4b517d76257d 100644 --- a/tools/testing/selftests/bpf/prog_tests/core_reloc.c +++ b/tools/testing/selftests/bpf/prog_tests/core_reloc.c @@ -210,11 +210,6 @@ static int duration = 0; .bpf_obj_file = "test_core_reloc_existence.o", \ .btf_src_file = "btf__core_reloc_" #name ".o" \ -#define FIELD_EXISTS_ERR_CASE(name) { \ - FIELD_EXISTS_CASE_COMMON(name), \ - .fails = true, \ -} - #define BITFIELDS_CASE_COMMON(objfile, test_name_prefix, name) \ .case_name = test_name_prefix#name, \ .bpf_obj_file = objfile, \ @@ -222,7 +217,7 @@ static int duration = 0; #define BITFIELDS_CASE(name, ...) { \ BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_probed.o", \ - "direct:", name), \ + "probed:", name), \ .input = STRUCT_TO_CHAR_PTR(core_reloc_##name) __VA_ARGS__, \ .input_len = sizeof(struct core_reloc_##name), \ .output = STRUCT_TO_CHAR_PTR(core_reloc_bitfields_output) \ @@ -230,7 +225,7 @@ static int duration = 0; .output_len = sizeof(struct core_reloc_bitfields_output), \ }, { \ BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_direct.o", \ - "probed:", name), \ + "direct:", name), \ .input = STRUCT_TO_CHAR_PTR(core_reloc_##name) __VA_ARGS__, \ .input_len = sizeof(struct core_reloc_##name), \ .output = STRUCT_TO_CHAR_PTR(core_reloc_bitfields_output) \ @@ -550,8 +545,7 @@ static struct core_reloc_test_case test_cases[] = { ARRAYS_ERR_CASE(arrays___err_too_small), ARRAYS_ERR_CASE(arrays___err_too_shallow), ARRAYS_ERR_CASE(arrays___err_non_array), - ARRAYS_ERR_CASE(arrays___err_wrong_val_type1), - ARRAYS_ERR_CASE(arrays___err_wrong_val_type2), + ARRAYS_ERR_CASE(arrays___err_wrong_val_type), ARRAYS_ERR_CASE(arrays___err_bad_zero_sz_arr), /* enum/ptr/int handling scenarios */ @@ -642,13 +636,25 @@ static struct core_reloc_test_case test_cases[] = { }, .output_len = sizeof(struct core_reloc_existence_output), }, - - FIELD_EXISTS_ERR_CASE(existence__err_int_sz), - FIELD_EXISTS_ERR_CASE(existence__err_int_type), - FIELD_EXISTS_ERR_CASE(existence__err_int_kind), - FIELD_EXISTS_ERR_CASE(existence__err_arr_kind), - FIELD_EXISTS_ERR_CASE(existence__err_arr_value_type), - FIELD_EXISTS_ERR_CASE(existence__err_struct_type), + { + FIELD_EXISTS_CASE_COMMON(existence___wrong_field_defs), + .input = STRUCT_TO_CHAR_PTR(core_reloc_existence___wrong_field_defs) { + }, + .input_len = sizeof(struct core_reloc_existence___wrong_field_defs), + .output = STRUCT_TO_CHAR_PTR(core_reloc_existence_output) { + .a_exists = 0, + .b_exists = 0, + .c_exists = 0, + .arr_exists = 0, + .s_exists = 0, + .a_value = 0xff000001u, + .b_value = 0xff000002u, + .c_value = 0xff000003u, + .arr_value = 0xff000004u, + .s_value = 0xff000005u, + }, + .output_len = sizeof(struct core_reloc_existence_output), + }, /* bitfield relocation checks */ BITFIELDS_CASE(bitfields, { @@ -857,13 +863,20 @@ void test_core_reloc(void) "prog '%s' not found\n", probe_name)) goto cleanup; + + if (test_case->btf_src_file) { + err = access(test_case->btf_src_file, R_OK); + if (!ASSERT_OK(err, "btf_src_file")) + goto cleanup; + } + load_attr.obj = obj; load_attr.log_level = 0; load_attr.target_btf_path = test_case->btf_src_file; err = bpf_object__load_xattr(&load_attr); if (err) { if (!test_case->fails) - CHECK(false, "obj_load", "failed to load prog '%s': %d\n", probe_name, err); + ASSERT_OK(err, "obj_load"); goto cleanup; } @@ -902,10 +915,8 @@ void test_core_reloc(void) goto cleanup; } - if (test_case->fails) { - CHECK(false, "obj_load_fail", "should fail to load prog '%s'\n", probe_name); + if (!ASSERT_FALSE(test_case->fails, "obj_load_should_fail")) goto cleanup; - } equal = memcmp(data->out, test_case->output, test_case->output_len) == 0; diff --git a/tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_arr_kind.c b/tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_arr_kind.c deleted file mode 100644 index dd0ffa518f36..000000000000 --- a/tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_arr_kind.c +++ /dev/null @@ -1,3 +0,0 @@ -#include "core_reloc_types.h" - -void f(struct core_reloc_existence___err_wrong_arr_kind x) {} diff --git a/tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_arr_value_type.c b/tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_arr_value_type.c deleted file mode 100644 index bc83372088ad..000000000000 --- a/tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_arr_value_type.c +++ /dev/null @@ -1,3 +0,0 @@ -#include "core_reloc_types.h" - -void f(struct core_reloc_existence___err_wrong_arr_value_type x) {} diff --git a/tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_int_kind.c b/tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_int_kind.c deleted file mode 100644 index 917bec41be08..000000000000 --- a/tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_int_kind.c +++ /dev/null @@ -1,3 +0,0 @@ -#include "core_reloc_types.h" - -void f(struct core_reloc_existence___err_wrong_int_kind x) {} diff --git a/tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_int_sz.c b/tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_int_sz.c deleted file mode 100644 index 6ec7e6ec1c91..000000000000 --- a/tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_int_sz.c +++ /dev/null @@ -1,3 +0,0 @@ -#include "core_reloc_types.h" - -void f(struct core_reloc_existence___err_wrong_int_sz x) {} diff --git a/tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_int_type.c b/tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_int_type.c deleted file mode 100644 index 7bbcacf2b0d1..000000000000 --- a/tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_int_type.c +++ /dev/null @@ -1,3 +0,0 @@ -#include "core_reloc_types.h" - -void f(struct core_reloc_existence___err_wrong_int_type x) {} diff --git a/tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_struct_type.c b/tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_struct_type.c deleted file mode 100644 index f384dd38ec70..000000000000 --- a/tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_struct_type.c +++ /dev/null @@ -1,3 +0,0 @@ -#include "core_reloc_types.h" - -void f(struct core_reloc_existence___err_wrong_struct_type x) {} diff --git a/tools/testing/selftests/bpf/progs/btf__core_reloc_existence___wrong_field_defs.c b/tools/testing/selftests/bpf/progs/btf__core_reloc_existence___wrong_field_defs.c new file mode 100644 index 000000000000..d14b496190c3 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/btf__core_reloc_existence___wrong_field_defs.c @@ -0,0 +1,3 @@ +#include "core_reloc_types.h" + +void f(struct core_reloc_existence___wrong_field_defs x) {} diff --git a/tools/testing/selftests/bpf/progs/core_reloc_types.h b/tools/testing/selftests/bpf/progs/core_reloc_types.h index 9a2850850121..664eea1013aa 100644 --- a/tools/testing/selftests/bpf/progs/core_reloc_types.h +++ b/tools/testing/selftests/bpf/progs/core_reloc_types.h @@ -700,27 +700,11 @@ struct core_reloc_existence___minimal { int a; }; -struct core_reloc_existence___err_wrong_int_sz { - short a; -}; - -struct core_reloc_existence___err_wrong_int_type { +struct core_reloc_existence___wrong_field_defs { + void *a; int b[1]; -}; - -struct core_reloc_existence___err_wrong_int_kind { struct{ int x; } c; -}; - -struct core_reloc_existence___err_wrong_arr_kind { int arr; -}; - -struct core_reloc_existence___err_wrong_arr_value_type { - short arr[1]; -}; - -struct core_reloc_existence___err_wrong_struct_type { int s; }; diff --git a/tools/testing/selftests/bpf/verifier/array_access.c b/tools/testing/selftests/bpf/verifier/array_access.c index 1b138cd2b187..1b1c798e9248 100644 --- a/tools/testing/selftests/bpf/verifier/array_access.c +++ b/tools/testing/selftests/bpf/verifier/array_access.c @@ -186,7 +186,7 @@ }, .fixup_map_hash_48b = { 3 }, .errstr_unpriv = "R0 leaks addr", - .errstr = "invalid access to map value, value_size=48 off=44 size=8", + .errstr = "R0 unbounded memory access", .result_unpriv = REJECT, .result = REJECT, .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, diff --git a/tools/testing/selftests/drivers/net/mlxsw/mirror_gre_scale.sh b/tools/testing/selftests/drivers/net/mlxsw/mirror_gre_scale.sh index 6f3a70df63bc..e00435753008 100644 --- a/tools/testing/selftests/drivers/net/mlxsw/mirror_gre_scale.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/mirror_gre_scale.sh @@ -120,12 +120,13 @@ __mirror_gre_test() sleep 5 for ((i = 0; i < count; ++i)); do + local sip=$(mirror_gre_ipv6_addr 1 $i)::1 local dip=$(mirror_gre_ipv6_addr 1 $i)::2 local htun=h3-gt6-$i local message icmp6_capture_install $htun - mirror_test v$h1 "" $dip $htun 100 10 + mirror_test v$h1 $sip $dip $htun 100 10 icmp6_capture_uninstall $htun done } diff --git a/tools/testing/selftests/drivers/net/mlxsw/port_scale.sh b/tools/testing/selftests/drivers/net/mlxsw/port_scale.sh index f813ffefc07e..65f43a7ce9c9 100644 --- a/tools/testing/selftests/drivers/net/mlxsw/port_scale.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/port_scale.sh @@ -55,10 +55,6 @@ port_test() | jq '.[][][] | select(.name=="physical_ports") |.["occ"]') [[ $occ -eq $max_ports ]] - if [[ $should_fail -eq 0 ]]; then - check_err $? "Mismatch ports number: Expected $max_ports, got $occ." - else - check_err_fail $should_fail $? "Reached more ports than expected" - fi + check_err_fail $should_fail $? "Attempt to create $max_ports ports (actual result $occ)" } diff --git a/tools/testing/selftests/drivers/net/mlxsw/sch_red_core.sh b/tools/testing/selftests/drivers/net/mlxsw/sch_red_core.sh index b0cb1aaffdda..33ddd01689be 100644 --- a/tools/testing/selftests/drivers/net/mlxsw/sch_red_core.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/sch_red_core.sh @@ -507,8 +507,8 @@ do_red_test() check_err $? "backlog $backlog / $limit Got $pct% marked packets, expected == 0." local diff=$((limit - backlog)) pct=$((100 * diff / limit)) - ((0 <= pct && pct <= 5)) - check_err $? "backlog $backlog / $limit expected <= 5% distance" + ((0 <= pct && pct <= 10)) + check_err $? "backlog $backlog / $limit expected <= 10% distance" log_test "TC $((vlan - 10)): RED backlog > limit" stop_traffic diff --git a/tools/testing/selftests/drivers/net/mlxsw/tc_flower_scale.sh b/tools/testing/selftests/drivers/net/mlxsw/tc_flower_scale.sh index cc0f07e72cf2..aa74be9f47c8 100644 --- a/tools/testing/selftests/drivers/net/mlxsw/tc_flower_scale.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/tc_flower_scale.sh @@ -98,11 +98,7 @@ __tc_flower_test() jq -r '[ .[] | select(.kind == "flower") | .options | .in_hw ]' | jq .[] | wc -l) [[ $((offload_count - 1)) -eq $count ]] - if [[ $should_fail -eq 0 ]]; then - check_err $? "Offload mismatch" - else - check_err_fail $should_fail $? "Offload more than expacted" - fi + check_err_fail $should_fail $? "Attempt to offload $count rules (actual result $((offload_count - 1)))" } tc_flower_test() diff --git a/tools/testing/selftests/kvm/dirty_log_test.c b/tools/testing/selftests/kvm/dirty_log_test.c index bb2752d78fe3..81edbd23d371 100644 --- a/tools/testing/selftests/kvm/dirty_log_test.c +++ b/tools/testing/selftests/kvm/dirty_log_test.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "kvm_util.h" #include "test_util.h" @@ -137,12 +138,20 @@ static uint64_t host_clear_count; static uint64_t host_track_next_count; /* Whether dirty ring reset is requested, or finished */ -static sem_t dirty_ring_vcpu_stop; -static sem_t dirty_ring_vcpu_cont; +static sem_t sem_vcpu_stop; +static sem_t sem_vcpu_cont; +/* + * This is only set by main thread, and only cleared by vcpu thread. It is + * used to request vcpu thread to stop at the next GUEST_SYNC, since GUEST_SYNC + * is the only place that we'll guarantee both "dirty bit" and "dirty data" + * will match. E.g., SIG_IPI won't guarantee that if the vcpu is interrupted + * after setting dirty bit but before the data is written. + */ +static atomic_t vcpu_sync_stop_requested; /* * This is updated by the vcpu thread to tell the host whether it's a * ring-full event. It should only be read until a sem_wait() of - * dirty_ring_vcpu_stop and before vcpu continues to run. + * sem_vcpu_stop and before vcpu continues to run. */ static bool dirty_ring_vcpu_ring_full; /* @@ -234,6 +243,17 @@ static void clear_log_collect_dirty_pages(struct kvm_vm *vm, int slot, kvm_vm_clear_dirty_log(vm, slot, bitmap, 0, num_pages); } +/* Should only be called after a GUEST_SYNC */ +static void vcpu_handle_sync_stop(void) +{ + if (atomic_read(&vcpu_sync_stop_requested)) { + /* It means main thread is sleeping waiting */ + atomic_set(&vcpu_sync_stop_requested, false); + sem_post(&sem_vcpu_stop); + sem_wait_until(&sem_vcpu_cont); + } +} + static void default_after_vcpu_run(struct kvm_vm *vm, int ret, int err) { struct kvm_run *run = vcpu_state(vm, VCPU_ID); @@ -244,6 +264,8 @@ static void default_after_vcpu_run(struct kvm_vm *vm, int ret, int err) TEST_ASSERT(get_ucall(vm, VCPU_ID, NULL) == UCALL_SYNC, "Invalid guest sync status: exit_reason=%s\n", exit_reason_str(run->exit_reason)); + + vcpu_handle_sync_stop(); } static bool dirty_ring_supported(void) @@ -301,13 +323,13 @@ static void dirty_ring_wait_vcpu(void) { /* This makes sure that hardware PML cache flushed */ vcpu_kick(); - sem_wait_until(&dirty_ring_vcpu_stop); + sem_wait_until(&sem_vcpu_stop); } static void dirty_ring_continue_vcpu(void) { pr_info("Notifying vcpu to continue\n"); - sem_post(&dirty_ring_vcpu_cont); + sem_post(&sem_vcpu_cont); } static void dirty_ring_collect_dirty_pages(struct kvm_vm *vm, int slot, @@ -361,11 +383,11 @@ static void dirty_ring_after_vcpu_run(struct kvm_vm *vm, int ret, int err) /* Update the flag first before pause */ WRITE_ONCE(dirty_ring_vcpu_ring_full, run->exit_reason == KVM_EXIT_DIRTY_RING_FULL); - sem_post(&dirty_ring_vcpu_stop); + sem_post(&sem_vcpu_stop); pr_info("vcpu stops because %s...\n", dirty_ring_vcpu_ring_full ? "dirty ring is full" : "vcpu is kicked out"); - sem_wait_until(&dirty_ring_vcpu_cont); + sem_wait_until(&sem_vcpu_cont); pr_info("vcpu continues now.\n"); } else { TEST_ASSERT(false, "Invalid guest sync status: " @@ -377,7 +399,7 @@ static void dirty_ring_after_vcpu_run(struct kvm_vm *vm, int ret, int err) static void dirty_ring_before_vcpu_join(void) { /* Kick another round of vcpu just to make sure it will quit */ - sem_post(&dirty_ring_vcpu_cont); + sem_post(&sem_vcpu_cont); } struct log_mode { @@ -505,9 +527,8 @@ static void *vcpu_worker(void *data) */ sigmask->len = 8; pthread_sigmask(0, NULL, sigset); + sigdelset(sigset, SIG_IPI); vcpu_ioctl(vm, VCPU_ID, KVM_SET_SIGNAL_MASK, sigmask); - sigaddset(sigset, SIG_IPI); - pthread_sigmask(SIG_BLOCK, sigset, NULL); sigemptyset(sigset); sigaddset(sigset, SIG_IPI); @@ -768,7 +789,25 @@ static void run_test(enum vm_guest_mode mode, void *arg) usleep(p->interval * 1000); log_mode_collect_dirty_pages(vm, TEST_MEM_SLOT_INDEX, bmap, host_num_pages); + + /* + * See vcpu_sync_stop_requested definition for details on why + * we need to stop vcpu when verify data. + */ + atomic_set(&vcpu_sync_stop_requested, true); + sem_wait_until(&sem_vcpu_stop); + /* + * NOTE: for dirty ring, it's possible that we didn't stop at + * GUEST_SYNC but instead we stopped because ring is full; + * that's okay too because ring full means we're only missing + * the flush of the last page, and since we handle the last + * page specially verification will succeed anyway. + */ + assert(host_log_mode == LOG_MODE_DIRTY_RING || + atomic_read(&vcpu_sync_stop_requested) == false); vm_dirty_log_verify(mode, bmap); + sem_post(&sem_vcpu_cont); + iteration++; sync_global_to_guest(vm, iteration); } @@ -818,9 +857,10 @@ int main(int argc, char *argv[]) .interval = TEST_HOST_LOOP_INTERVAL, }; int opt, i; + sigset_t sigset; - sem_init(&dirty_ring_vcpu_stop, 0, 0); - sem_init(&dirty_ring_vcpu_cont, 0, 0); + sem_init(&sem_vcpu_stop, 0, 0); + sem_init(&sem_vcpu_cont, 0, 0); guest_modes_append_default(); @@ -876,6 +916,11 @@ int main(int argc, char *argv[]) srandom(time(0)); + /* Ensure that vCPU threads start with SIG_IPI blocked. */ + sigemptyset(&sigset); + sigaddset(&sigset, SIG_IPI); + pthread_sigmask(SIG_BLOCK, &sigset, NULL); + if (host_log_mode_option == LOG_MODE_ALL) { /* Run each log mode */ for (i = 0; i < LOG_MODE_NUM; i++) { diff --git a/tools/testing/selftests/lib.mk b/tools/testing/selftests/lib.mk index a5ce26d548e4..0af84ad48aa7 100644 --- a/tools/testing/selftests/lib.mk +++ b/tools/testing/selftests/lib.mk @@ -1,6 +1,10 @@ # This mimics the top-level Makefile. We do it explicitly here so that this # Makefile can operate with or without the kbuild infrastructure. +ifneq ($(LLVM),) +CC := clang +else CC := $(CROSS_COMPILE)gcc +endif ifeq (0,$(MAKELEVEL)) ifeq ($(OUTPUT),) @@ -74,7 +78,8 @@ ifdef building_out_of_srctree rsync -aq $(TEST_PROGS) $(TEST_PROGS_EXTENDED) $(TEST_FILES) $(OUTPUT); \ fi @if [ "X$(TEST_PROGS)" != "X" ]; then \ - $(call RUN_TESTS, $(TEST_GEN_PROGS) $(TEST_CUSTOM_PROGS) $(OUTPUT)/$(TEST_PROGS)) ; \ + $(call RUN_TESTS, $(TEST_GEN_PROGS) $(TEST_CUSTOM_PROGS) \ + $(addprefix $(OUTPUT)/,$(TEST_PROGS))) ; \ else \ $(call RUN_TESTS, $(TEST_GEN_PROGS) $(TEST_CUSTOM_PROGS)); \ fi diff --git a/tools/testing/selftests/net/forwarding/mirror_gre_vlan_bridge_1q.sh b/tools/testing/selftests/net/forwarding/mirror_gre_vlan_bridge_1q.sh index c02291e9841e..880e3ab9d088 100755 --- a/tools/testing/selftests/net/forwarding/mirror_gre_vlan_bridge_1q.sh +++ b/tools/testing/selftests/net/forwarding/mirror_gre_vlan_bridge_1q.sh @@ -271,7 +271,7 @@ test_span_gre_fdb_roaming() while ((RET == 0)); do bridge fdb del dev $swp3 $h3mac vlan 555 master 2>/dev/null - bridge fdb add dev $swp2 $h3mac vlan 555 master + bridge fdb add dev $swp2 $h3mac vlan 555 master static sleep 1 fail_test_span_gre_dir $tundev ingress diff --git a/tools/testing/selftests/net/forwarding/mirror_lib.sh b/tools/testing/selftests/net/forwarding/mirror_lib.sh index 13db1cb50e57..6406cd76a19d 100644 --- a/tools/testing/selftests/net/forwarding/mirror_lib.sh +++ b/tools/testing/selftests/net/forwarding/mirror_lib.sh @@ -20,6 +20,13 @@ mirror_uninstall() tc filter del dev $swp1 $direction pref 1000 } +is_ipv6() +{ + local addr=$1; shift + + [[ -z ${addr//[0-9a-fA-F:]/} ]] +} + mirror_test() { local vrf_name=$1; shift @@ -29,9 +36,17 @@ mirror_test() local pref=$1; shift local expect=$1; shift + if is_ipv6 $dip; then + local proto=-6 + local type="icmp6 type=128" # Echo request. + else + local proto= + local type="icmp echoreq" + fi + local t0=$(tc_rule_stats_get $dev $pref) - $MZ $vrf_name ${sip:+-A $sip} -B $dip -a own -b bc -q \ - -c 10 -d 100msec -t icmp type=8 + $MZ $proto $vrf_name ${sip:+-A $sip} -B $dip -a own -b bc -q \ + -c 10 -d 100msec -t $type sleep 0.5 local t1=$(tc_rule_stats_get $dev $pref) local delta=$((t1 - t0)) diff --git a/tools/testing/selftests/net/mptcp/diag.sh b/tools/testing/selftests/net/mptcp/diag.sh index 39edce4f541c..2674ba20d524 100755 --- a/tools/testing/selftests/net/mptcp/diag.sh +++ b/tools/testing/selftests/net/mptcp/diag.sh @@ -5,8 +5,9 @@ rndh=$(printf %x $sec)-$(mktemp -u XXXXXX) ns="ns1-$rndh" ksft_skip=4 test_cnt=1 +timeout_poll=100 +timeout_test=$((timeout_poll * 2 + 1)) ret=0 -pids=() flush_pids() { @@ -14,18 +15,14 @@ flush_pids() # give it some time sleep 1.1 - for pid in ${pids[@]}; do - [ -d /proc/$pid ] && kill -SIGUSR1 $pid >/dev/null 2>&1 - done - pids=() + ip netns pids "${ns}" | xargs --no-run-if-empty kill -SIGUSR1 &>/dev/null } cleanup() { + ip netns pids "${ns}" | xargs --no-run-if-empty kill -SIGKILL &>/dev/null + ip netns del $ns - for pid in ${pids[@]}; do - [ -d /proc/$pid ] && kill -9 $pid >/dev/null 2>&1 - done } ip -Version > /dev/null 2>&1 @@ -79,39 +76,57 @@ trap cleanup EXIT ip netns add $ns ip -n $ns link set dev lo up -echo "a" | ip netns exec $ns ./mptcp_connect -p 10000 -l 0.0.0.0 -t 100 >/dev/null & +echo "a" | \ + timeout ${timeout_test} \ + ip netns exec $ns \ + ./mptcp_connect -p 10000 -l -t ${timeout_poll} \ + 0.0.0.0 >/dev/null & sleep 0.1 -pids[0]=$! chk_msk_nr 0 "no msk on netns creation" -echo "b" | ip netns exec $ns ./mptcp_connect -p 10000 127.0.0.1 -j -t 100 >/dev/null & +echo "b" | \ + timeout ${timeout_test} \ + ip netns exec $ns \ + ./mptcp_connect -p 10000 -j -t ${timeout_poll} \ + 127.0.0.1 >/dev/null & sleep 0.1 -pids[1]=$! chk_msk_nr 2 "after MPC handshake " chk_msk_remote_key_nr 2 "....chk remote_key" chk_msk_fallback_nr 0 "....chk no fallback" flush_pids -echo "a" | ip netns exec $ns ./mptcp_connect -p 10001 -s TCP -l 0.0.0.0 -t 100 >/dev/null & -pids[0]=$! +echo "a" | \ + timeout ${timeout_test} \ + ip netns exec $ns \ + ./mptcp_connect -p 10001 -l -s TCP -t ${timeout_poll} \ + 0.0.0.0 >/dev/null & sleep 0.1 -echo "b" | ip netns exec $ns ./mptcp_connect -p 10001 127.0.0.1 -j -t 100 >/dev/null & -pids[1]=$! +echo "b" | \ + timeout ${timeout_test} \ + ip netns exec $ns \ + ./mptcp_connect -p 10001 -j -t ${timeout_poll} \ + 127.0.0.1 >/dev/null & sleep 0.1 chk_msk_fallback_nr 1 "check fallback" flush_pids NR_CLIENTS=100 for I in `seq 1 $NR_CLIENTS`; do - echo "a" | ip netns exec $ns ./mptcp_connect -p $((I+10001)) -l 0.0.0.0 -t 100 -w 10 >/dev/null & - pids[$((I*2))]=$! + echo "a" | \ + timeout ${timeout_test} \ + ip netns exec $ns \ + ./mptcp_connect -p $((I+10001)) -l -w 10 \ + -t ${timeout_poll} 0.0.0.0 >/dev/null & done sleep 0.1 for I in `seq 1 $NR_CLIENTS`; do - echo "b" | ip netns exec $ns ./mptcp_connect -p $((I+10001)) 127.0.0.1 -t 100 -w 10 >/dev/null & - pids[$((I*2 + 1))]=$! + echo "b" | \ + timeout ${timeout_test} \ + ip netns exec $ns \ + ./mptcp_connect -p $((I+10001)) -w 10 \ + -t ${timeout_poll} 127.0.0.1 >/dev/null & done sleep 1.5 diff --git a/tools/testing/selftests/net/mptcp/mptcp_connect.sh b/tools/testing/selftests/net/mptcp/mptcp_connect.sh index 10a030b53b23..65b3b983efc2 100755 --- a/tools/testing/selftests/net/mptcp/mptcp_connect.sh +++ b/tools/testing/selftests/net/mptcp/mptcp_connect.sh @@ -11,7 +11,8 @@ cin="" cout="" ksft_skip=4 capture=false -timeout=30 +timeout_poll=30 +timeout_test=$((timeout_poll * 2 + 1)) ipv6=true ethtool_random_on=true tc_delay="$((RANDOM%50))" @@ -273,7 +274,7 @@ check_mptcp_disabled() ip netns exec ${disabled_ns} sysctl -q net.mptcp.enabled=0 local err=0 - LANG=C ip netns exec ${disabled_ns} ./mptcp_connect -t $timeout -p 10000 -s MPTCP 127.0.0.1 < "$cin" 2>&1 | \ + LANG=C ip netns exec ${disabled_ns} ./mptcp_connect -p 10000 -s MPTCP 127.0.0.1 < "$cin" 2>&1 | \ grep -q "^socket: Protocol not available$" && err=1 ip netns delete ${disabled_ns} @@ -430,14 +431,20 @@ do_transfer() local stat_cookietx_last=$(get_mib_counter "${listener_ns}" "TcpExtSyncookiesSent") local stat_cookierx_last=$(get_mib_counter "${listener_ns}" "TcpExtSyncookiesRecv") - ip netns exec ${listener_ns} ./mptcp_connect -t $timeout -l -p $port -s ${srv_proto} $extra_args $local_addr < "$sin" > "$sout" & + timeout ${timeout_test} \ + ip netns exec ${listener_ns} \ + ./mptcp_connect -t ${timeout_poll} -l -p $port -s ${srv_proto} \ + $extra_args $local_addr < "$sin" > "$sout" & local spid=$! wait_local_port_listen "${listener_ns}" "${port}" local start start=$(date +%s%3N) - ip netns exec ${connector_ns} ./mptcp_connect -t $timeout -p $port -s ${cl_proto} $extra_args $connect_addr < "$cin" > "$cout" & + timeout ${timeout_test} \ + ip netns exec ${connector_ns} \ + ./mptcp_connect -t ${timeout_poll} -p $port -s ${cl_proto} \ + $extra_args $connect_addr < "$cin" > "$cout" & local cpid=$! wait $cpid diff --git a/tools/testing/selftests/net/mptcp/mptcp_join.sh b/tools/testing/selftests/net/mptcp/mptcp_join.sh index ad32240fbfda..43ed99de7734 100755 --- a/tools/testing/selftests/net/mptcp/mptcp_join.sh +++ b/tools/testing/selftests/net/mptcp/mptcp_join.sh @@ -8,7 +8,8 @@ cin="" cinsent="" cout="" ksft_skip=4 -timeout=30 +timeout_poll=30 +timeout_test=$((timeout_poll * 2 + 1)) mptcp_connect="" capture=0 do_all_tests=1 @@ -245,17 +246,26 @@ do_transfer() local_addr="0.0.0.0" fi - ip netns exec ${listener_ns} $mptcp_connect -t $timeout -l -p $port \ - -s ${srv_proto} ${local_addr} < "$sin" > "$sout" & + timeout ${timeout_test} \ + ip netns exec ${listener_ns} \ + $mptcp_connect -t ${timeout_poll} -l -p $port -s ${srv_proto} \ + ${local_addr} < "$sin" > "$sout" & spid=$! sleep 1 if [ "$test_link_fail" -eq 0 ];then - ip netns exec ${connector_ns} $mptcp_connect -t $timeout -p $port -s ${cl_proto} $connect_addr < "$cin" > "$cout" & + timeout ${timeout_test} \ + ip netns exec ${connector_ns} \ + $mptcp_connect -t ${timeout_poll} -p $port -s ${cl_proto} \ + $connect_addr < "$cin" > "$cout" & else - ( cat "$cin" ; sleep 2; link_failure $listener_ns ; cat "$cin" ) | tee "$cinsent" | \ - ip netns exec ${connector_ns} $mptcp_connect -t $timeout -p $port -s ${cl_proto} $connect_addr > "$cout" & + ( cat "$cin" ; sleep 2; link_failure $listener_ns ; cat "$cin" ) | \ + tee "$cinsent" | \ + timeout ${timeout_test} \ + ip netns exec ${connector_ns} \ + $mptcp_connect -t ${timeout_poll} -p $port -s ${cl_proto} \ + $connect_addr > "$cout" & fi cpid=$! diff --git a/tools/testing/selftests/net/mptcp/simult_flows.sh b/tools/testing/selftests/net/mptcp/simult_flows.sh index f039ee57eb3c..3aeef3bcb101 100755 --- a/tools/testing/selftests/net/mptcp/simult_flows.sh +++ b/tools/testing/selftests/net/mptcp/simult_flows.sh @@ -7,7 +7,8 @@ ns2="ns2-$rndh" ns3="ns3-$rndh" capture=false ksft_skip=4 -timeout=30 +timeout_poll=30 +timeout_test=$((timeout_poll * 2 + 1)) test_cnt=1 ret=0 bail=0 @@ -157,14 +158,20 @@ do_transfer() sleep 1 fi - ip netns exec ${ns3} ./mptcp_connect -jt $timeout -l -p $port 0.0.0.0 < "$sin" > "$sout" & + timeout ${timeout_test} \ + ip netns exec ${ns3} \ + ./mptcp_connect -jt ${timeout_poll} -l -p $port \ + 0.0.0.0 < "$sin" > "$sout" & local spid=$! wait_local_port_listen "${ns3}" "${port}" local start start=$(date +%s%3N) - ip netns exec ${ns1} ./mptcp_connect -jt $timeout -p $port 10.0.3.3 < "$cin" > "$cout" & + timeout ${timeout_test} \ + ip netns exec ${ns1} \ + ./mptcp_connect -jt ${timeout_poll} -p $port \ + 10.0.3.3 < "$cin" > "$cout" & local cpid=$! wait $cpid diff --git a/tools/testing/selftests/powerpc/security/entry_flush.c b/tools/testing/selftests/powerpc/security/entry_flush.c index 78cf914fa321..68ce377b205e 100644 --- a/tools/testing/selftests/powerpc/security/entry_flush.c +++ b/tools/testing/selftests/powerpc/security/entry_flush.c @@ -53,7 +53,7 @@ int entry_flush_test(void) entry_flush = entry_flush_orig; - fd = perf_event_open_counter(PERF_TYPE_RAW, /* L1d miss */ 0x400f0, -1); + fd = perf_event_open_counter(PERF_TYPE_HW_CACHE, PERF_L1D_READ_MISS_CONFIG, -1); FAIL_IF(fd < 0); p = (char *)memalign(zero_size, CACHELINE_SIZE); diff --git a/tools/testing/selftests/powerpc/security/flush_utils.h b/tools/testing/selftests/powerpc/security/flush_utils.h index 07a5eb301466..7a3d60292916 100644 --- a/tools/testing/selftests/powerpc/security/flush_utils.h +++ b/tools/testing/selftests/powerpc/security/flush_utils.h @@ -9,6 +9,10 @@ #define CACHELINE_SIZE 128 +#define PERF_L1D_READ_MISS_CONFIG ((PERF_COUNT_HW_CACHE_L1D) | \ + (PERF_COUNT_HW_CACHE_OP_READ << 8) | \ + (PERF_COUNT_HW_CACHE_RESULT_MISS << 16)) + void syscall_loop(char *p, unsigned long iterations, unsigned long zero_size); diff --git a/tools/testing/selftests/powerpc/security/rfi_flush.c b/tools/testing/selftests/powerpc/security/rfi_flush.c index 7565fd786640..f73484a6470f 100644 --- a/tools/testing/selftests/powerpc/security/rfi_flush.c +++ b/tools/testing/selftests/powerpc/security/rfi_flush.c @@ -54,7 +54,7 @@ int rfi_flush_test(void) rfi_flush = rfi_flush_orig; - fd = perf_event_open_counter(PERF_TYPE_RAW, /* L1d miss */ 0x400f0, -1); + fd = perf_event_open_counter(PERF_TYPE_HW_CACHE, PERF_L1D_READ_MISS_CONFIG, -1); FAIL_IF(fd < 0); p = (char *)memalign(zero_size, CACHELINE_SIZE); diff --git a/tools/testing/selftests/resctrl/Makefile b/tools/testing/selftests/resctrl/Makefile index d585cc1948cc..6bcee2ec91a9 100644 --- a/tools/testing/selftests/resctrl/Makefile +++ b/tools/testing/selftests/resctrl/Makefile @@ -1,5 +1,5 @@ CC = $(CROSS_COMPILE)gcc -CFLAGS = -g -Wall +CFLAGS = -g -Wall -O2 -D_FORTIFY_SOURCE=2 SRCS=$(wildcard *.c) OBJS=$(SRCS:.c=.o) diff --git a/tools/testing/selftests/resctrl/cache.c b/tools/testing/selftests/resctrl/cache.c index 38dbf4962e33..5922cc1b0386 100644 --- a/tools/testing/selftests/resctrl/cache.c +++ b/tools/testing/selftests/resctrl/cache.c @@ -182,7 +182,7 @@ int measure_cache_vals(struct resctrl_val_param *param, int bm_pid) /* * Measure cache miss from perf. */ - if (!strcmp(param->resctrl_val, "cat")) { + if (!strncmp(param->resctrl_val, CAT_STR, sizeof(CAT_STR))) { ret = get_llc_perf(&llc_perf_miss); if (ret < 0) return ret; @@ -192,7 +192,7 @@ int measure_cache_vals(struct resctrl_val_param *param, int bm_pid) /* * Measure llc occupancy from resctrl. */ - if (!strcmp(param->resctrl_val, "cqm")) { + if (!strncmp(param->resctrl_val, CQM_STR, sizeof(CQM_STR))) { ret = get_llc_occu_resctrl(&llc_occu_resc); if (ret < 0) return ret; @@ -234,7 +234,7 @@ int cat_val(struct resctrl_val_param *param) if (ret) return ret; - if ((strcmp(resctrl_val, "cat") == 0)) { + if (!strncmp(resctrl_val, CAT_STR, sizeof(CAT_STR))) { ret = initialize_llc_perf(); if (ret) return ret; @@ -242,7 +242,7 @@ int cat_val(struct resctrl_val_param *param) /* Test runs until the callback setup() tells the test to stop. */ while (1) { - if (strcmp(resctrl_val, "cat") == 0) { + if (!strncmp(resctrl_val, CAT_STR, sizeof(CAT_STR))) { ret = param->setup(1, param); if (ret) { ret = 0; diff --git a/tools/testing/selftests/resctrl/cat_test.c b/tools/testing/selftests/resctrl/cat_test.c index 5da43767b973..20823725daca 100644 --- a/tools/testing/selftests/resctrl/cat_test.c +++ b/tools/testing/selftests/resctrl/cat_test.c @@ -17,10 +17,10 @@ #define MAX_DIFF_PERCENT 4 #define MAX_DIFF 1000000 -int count_of_bits; -char cbm_mask[256]; -unsigned long long_mask; -unsigned long cache_size; +static int count_of_bits; +static char cbm_mask[256]; +static unsigned long long_mask; +static unsigned long cache_size; /* * Change schemata. Write schemata to specified @@ -136,7 +136,7 @@ int cat_perf_miss_val(int cpu_no, int n, char *cache_type) return -1; /* Get default cbm mask for L3/L2 cache */ - ret = get_cbm_mask(cache_type); + ret = get_cbm_mask(cache_type, cbm_mask); if (ret) return ret; @@ -164,7 +164,7 @@ int cat_perf_miss_val(int cpu_no, int n, char *cache_type) return -1; struct resctrl_val_param param = { - .resctrl_val = "cat", + .resctrl_val = CAT_STR, .cpu_no = cpu_no, .mum_resctrlfs = 0, .setup = cat_setup, diff --git a/tools/testing/selftests/resctrl/cqm_test.c b/tools/testing/selftests/resctrl/cqm_test.c index c8756152bd61..271752e9ef5b 100644 --- a/tools/testing/selftests/resctrl/cqm_test.c +++ b/tools/testing/selftests/resctrl/cqm_test.c @@ -16,10 +16,10 @@ #define MAX_DIFF 2000000 #define MAX_DIFF_PERCENT 15 -int count_of_bits; -char cbm_mask[256]; -unsigned long long_mask; -unsigned long cache_size; +static int count_of_bits; +static char cbm_mask[256]; +static unsigned long long_mask; +static unsigned long cache_size; static int cqm_setup(int num, ...) { @@ -86,7 +86,7 @@ static int check_results(struct resctrl_val_param *param, int no_of_bits) return errno; } - while (fgets(temp, 1024, fp)) { + while (fgets(temp, sizeof(temp), fp)) { char *token = strtok(temp, ":\t"); int fields = 0; @@ -125,7 +125,7 @@ int cqm_resctrl_val(int cpu_no, int n, char **benchmark_cmd) if (!validate_resctrl_feature_request("cqm")) return -1; - ret = get_cbm_mask("L3"); + ret = get_cbm_mask("L3", cbm_mask); if (ret) return ret; @@ -145,7 +145,7 @@ int cqm_resctrl_val(int cpu_no, int n, char **benchmark_cmd) } struct resctrl_val_param param = { - .resctrl_val = "cqm", + .resctrl_val = CQM_STR, .ctrlgrp = "c1", .mongrp = "m1", .cpu_no = cpu_no, diff --git a/tools/testing/selftests/resctrl/fill_buf.c b/tools/testing/selftests/resctrl/fill_buf.c index 79c611c99a3d..51e5cf22632f 100644 --- a/tools/testing/selftests/resctrl/fill_buf.c +++ b/tools/testing/selftests/resctrl/fill_buf.c @@ -115,7 +115,7 @@ static int fill_cache_read(unsigned char *start_ptr, unsigned char *end_ptr, while (1) { ret = fill_one_span_read(start_ptr, end_ptr); - if (!strcmp(resctrl_val, "cat")) + if (!strncmp(resctrl_val, CAT_STR, sizeof(CAT_STR))) break; } @@ -134,7 +134,7 @@ static int fill_cache_write(unsigned char *start_ptr, unsigned char *end_ptr, { while (1) { fill_one_span_write(start_ptr, end_ptr); - if (!strcmp(resctrl_val, "cat")) + if (!strncmp(resctrl_val, CAT_STR, sizeof(CAT_STR))) break; } diff --git a/tools/testing/selftests/resctrl/mba_test.c b/tools/testing/selftests/resctrl/mba_test.c index 7bf8eaa6204b..6449fbd96096 100644 --- a/tools/testing/selftests/resctrl/mba_test.c +++ b/tools/testing/selftests/resctrl/mba_test.c @@ -141,7 +141,7 @@ void mba_test_cleanup(void) int mba_schemata_change(int cpu_no, char *bw_report, char **benchmark_cmd) { struct resctrl_val_param param = { - .resctrl_val = "mba", + .resctrl_val = MBA_STR, .ctrlgrp = "c1", .mongrp = "m1", .cpu_no = cpu_no, diff --git a/tools/testing/selftests/resctrl/mbm_test.c b/tools/testing/selftests/resctrl/mbm_test.c index 4700f7453f81..ec6cfe01c9c2 100644 --- a/tools/testing/selftests/resctrl/mbm_test.c +++ b/tools/testing/selftests/resctrl/mbm_test.c @@ -114,7 +114,7 @@ void mbm_test_cleanup(void) int mbm_bw_change(int span, int cpu_no, char *bw_report, char **benchmark_cmd) { struct resctrl_val_param param = { - .resctrl_val = "mbm", + .resctrl_val = MBM_STR, .ctrlgrp = "c1", .mongrp = "m1", .span = span, diff --git a/tools/testing/selftests/resctrl/resctrl.h b/tools/testing/selftests/resctrl/resctrl.h index 39bf59c6b9c5..9dcc96e1ad3d 100644 --- a/tools/testing/selftests/resctrl/resctrl.h +++ b/tools/testing/selftests/resctrl/resctrl.h @@ -28,6 +28,10 @@ #define RESCTRL_PATH "/sys/fs/resctrl" #define PHYS_ID_PATH "/sys/devices/system/cpu/cpu" #define CBM_MASK_PATH "/sys/fs/resctrl/info" +#define L3_PATH "/sys/fs/resctrl/info/L3" +#define MB_PATH "/sys/fs/resctrl/info/MB" +#define L3_MON_PATH "/sys/fs/resctrl/info/L3_MON" +#define L3_MON_FEATURES_PATH "/sys/fs/resctrl/info/L3_MON/mon_features" #define PARENT_EXIT(err_msg) \ do { \ @@ -62,11 +66,16 @@ struct resctrl_val_param { int (*setup)(int num, ...); }; -pid_t bm_pid, ppid; -int tests_run; +#define MBM_STR "mbm" +#define MBA_STR "mba" +#define CQM_STR "cqm" +#define CAT_STR "cat" -char llc_occup_path[1024]; -bool is_amd; +extern pid_t bm_pid, ppid; +extern int tests_run; + +extern char llc_occup_path[1024]; +extern bool is_amd; bool check_resctrlfs_support(void); int filter_dmesg(void); @@ -74,7 +83,7 @@ int remount_resctrlfs(bool mum_resctrlfs); int get_resource_id(int cpu_no, int *resource_id); int umount_resctrlfs(void); int validate_bw_report_request(char *bw_report); -bool validate_resctrl_feature_request(char *resctrl_val); +bool validate_resctrl_feature_request(const char *resctrl_val); char *fgrep(FILE *inf, const char *str); int taskset_benchmark(pid_t bm_pid, int cpu_no); void run_benchmark(int signum, siginfo_t *info, void *ucontext); @@ -92,7 +101,7 @@ void tests_cleanup(void); void mbm_test_cleanup(void); int mba_schemata_change(int cpu_no, char *bw_report, char **benchmark_cmd); void mba_test_cleanup(void); -int get_cbm_mask(char *cache_type); +int get_cbm_mask(char *cache_type, char *cbm_mask); int get_cache_size(int cpu_no, char *cache_type, unsigned long *cache_size); void ctrlc_handler(int signum, siginfo_t *info, void *ptr); int cat_val(struct resctrl_val_param *param); diff --git a/tools/testing/selftests/resctrl/resctrl_tests.c b/tools/testing/selftests/resctrl/resctrl_tests.c index 425cc85ac883..ac2269610aa9 100644 --- a/tools/testing/selftests/resctrl/resctrl_tests.c +++ b/tools/testing/selftests/resctrl/resctrl_tests.c @@ -73,7 +73,7 @@ int main(int argc, char **argv) } } - while ((c = getopt(argc_new, argv, "ht:b:")) != -1) { + while ((c = getopt(argc_new, argv, "ht:b:n:p:")) != -1) { char *token; switch (c) { @@ -85,13 +85,13 @@ int main(int argc, char **argv) cqm_test = false; cat_test = false; while (token) { - if (!strcmp(token, "mbm")) { + if (!strncmp(token, MBM_STR, sizeof(MBM_STR))) { mbm_test = true; - } else if (!strcmp(token, "mba")) { + } else if (!strncmp(token, MBA_STR, sizeof(MBA_STR))) { mba_test = true; - } else if (!strcmp(token, "cqm")) { + } else if (!strncmp(token, CQM_STR, sizeof(CQM_STR))) { cqm_test = true; - } else if (!strcmp(token, "cat")) { + } else if (!strncmp(token, CAT_STR, sizeof(CAT_STR))) { cat_test = true; } else { printf("invalid argument\n"); @@ -161,7 +161,7 @@ int main(int argc, char **argv) if (!is_amd && mbm_test) { printf("# Starting MBM BW change ...\n"); if (!has_ben) - sprintf(benchmark_cmd[5], "%s", "mba"); + sprintf(benchmark_cmd[5], "%s", MBA_STR); res = mbm_bw_change(span, cpu_no, bw_report, benchmark_cmd); printf("%sok MBM: bw change\n", res ? "not " : ""); mbm_test_cleanup(); @@ -181,7 +181,7 @@ int main(int argc, char **argv) if (cqm_test) { printf("# Starting CQM test ...\n"); if (!has_ben) - sprintf(benchmark_cmd[5], "%s", "cqm"); + sprintf(benchmark_cmd[5], "%s", CQM_STR); res = cqm_resctrl_val(cpu_no, no_of_bits, benchmark_cmd); printf("%sok CQM: test\n", res ? "not " : ""); cqm_test_cleanup(); diff --git a/tools/testing/selftests/resctrl/resctrl_val.c b/tools/testing/selftests/resctrl/resctrl_val.c index 520fea3606d1..8df557894059 100644 --- a/tools/testing/selftests/resctrl/resctrl_val.c +++ b/tools/testing/selftests/resctrl/resctrl_val.c @@ -221,8 +221,8 @@ static int read_from_imc_dir(char *imc_dir, int count) */ static int num_of_imcs(void) { + char imc_dir[512], *temp; unsigned int count = 0; - char imc_dir[512]; struct dirent *ep; int ret; DIR *dp; @@ -230,7 +230,25 @@ static int num_of_imcs(void) dp = opendir(DYN_PMU_PATH); if (dp) { while ((ep = readdir(dp))) { - if (strstr(ep->d_name, UNCORE_IMC)) { + temp = strstr(ep->d_name, UNCORE_IMC); + if (!temp) + continue; + + /* + * imc counters are named as "uncore_imc_", hence + * increment the pointer to point to . Note that + * sizeof(UNCORE_IMC) would count for null character as + * well and hence the last underscore character in + * uncore_imc'_' need not be counted. + */ + temp = temp + sizeof(UNCORE_IMC); + + /* + * Some directories under "DYN_PMU_PATH" could have + * names like "uncore_imc_free_running", hence, check if + * first character is a numerical digit or not. + */ + if (temp[0] >= '0' && temp[0] <= '9') { sprintf(imc_dir, "%s/%s/", DYN_PMU_PATH, ep->d_name); ret = read_from_imc_dir(imc_dir, count); @@ -282,9 +300,9 @@ static int initialize_mem_bw_imc(void) * Memory B/W utilized by a process on a socket can be calculated using * iMC counters. Perf events are used to read these counters. * - * Return: >= 0 on success. < 0 on failure. + * Return: = 0 on success. < 0 on failure. */ -static float get_mem_bw_imc(int cpu_no, char *bw_report) +static int get_mem_bw_imc(int cpu_no, char *bw_report, float *bw_imc) { float reads, writes, of_mul_read, of_mul_write; int imc, j, ret; @@ -355,13 +373,18 @@ static float get_mem_bw_imc(int cpu_no, char *bw_report) close(imc_counters_config[imc][WRITE].fd); } - if (strcmp(bw_report, "reads") == 0) - return reads; + if (strcmp(bw_report, "reads") == 0) { + *bw_imc = reads; + return 0; + } - if (strcmp(bw_report, "writes") == 0) - return writes; + if (strcmp(bw_report, "writes") == 0) { + *bw_imc = writes; + return 0; + } - return (reads + writes); + *bw_imc = reads + writes; + return 0; } void set_mbm_path(const char *ctrlgrp, const char *mongrp, int resource_id) @@ -397,10 +420,10 @@ static void initialize_mem_bw_resctrl(const char *ctrlgrp, const char *mongrp, return; } - if (strcmp(resctrl_val, "mbm") == 0) + if (!strncmp(resctrl_val, MBM_STR, sizeof(MBM_STR))) set_mbm_path(ctrlgrp, mongrp, resource_id); - if ((strcmp(resctrl_val, "mba") == 0)) { + if (!strncmp(resctrl_val, MBA_STR, sizeof(MBA_STR))) { if (ctrlgrp) sprintf(mbm_total_path, CON_MBM_LOCAL_BYTES_PATH, RESCTRL_PATH, ctrlgrp, resource_id); @@ -420,9 +443,8 @@ static void initialize_mem_bw_resctrl(const char *ctrlgrp, const char *mongrp, * 1. If con_mon grp is given, then read from it * 2. If con_mon grp is not given, then read from root con_mon grp */ -static unsigned long get_mem_bw_resctrl(void) +static int get_mem_bw_resctrl(unsigned long *mbm_total) { - unsigned long mbm_total = 0; FILE *fp; fp = fopen(mbm_total_path, "r"); @@ -431,7 +453,7 @@ static unsigned long get_mem_bw_resctrl(void) return -1; } - if (fscanf(fp, "%lu", &mbm_total) <= 0) { + if (fscanf(fp, "%lu", mbm_total) <= 0) { perror("Could not get mbm local bytes"); fclose(fp); @@ -439,7 +461,7 @@ static unsigned long get_mem_bw_resctrl(void) } fclose(fp); - return mbm_total; + return 0; } pid_t bm_pid, ppid; @@ -524,14 +546,15 @@ static void initialize_llc_occu_resctrl(const char *ctrlgrp, const char *mongrp, return; } - if (strcmp(resctrl_val, "cqm") == 0) + if (!strncmp(resctrl_val, CQM_STR, sizeof(CQM_STR))) set_cqm_path(ctrlgrp, mongrp, resource_id); } static int measure_vals(struct resctrl_val_param *param, unsigned long *bw_resc_start) { - unsigned long bw_imc, bw_resc, bw_resc_end; + unsigned long bw_resc, bw_resc_end; + float bw_imc; int ret; /* @@ -541,13 +564,13 @@ measure_vals(struct resctrl_val_param *param, unsigned long *bw_resc_start) * Compare the two values to validate resctrl value. * It takes 1sec to measure the data. */ - bw_imc = get_mem_bw_imc(param->cpu_no, param->bw_report); - if (bw_imc <= 0) - return bw_imc; + ret = get_mem_bw_imc(param->cpu_no, param->bw_report, &bw_imc); + if (ret < 0) + return ret; - bw_resc_end = get_mem_bw_resctrl(); - if (bw_resc_end <= 0) - return bw_resc_end; + ret = get_mem_bw_resctrl(&bw_resc_end); + if (ret < 0) + return ret; bw_resc = (bw_resc_end - *bw_resc_start) / MB; ret = print_results_bw(param->filename, bm_pid, bw_imc, bw_resc); @@ -579,8 +602,8 @@ int resctrl_val(char **benchmark_cmd, struct resctrl_val_param *param) if (strcmp(param->filename, "") == 0) sprintf(param->filename, "stdio"); - if ((strcmp(resctrl_val, "mba")) == 0 || - (strcmp(resctrl_val, "mbm")) == 0) { + if (!strncmp(resctrl_val, MBA_STR, sizeof(MBA_STR)) || + !strncmp(resctrl_val, MBM_STR, sizeof(MBM_STR))) { ret = validate_bw_report_request(param->bw_report); if (ret) return ret; @@ -674,15 +697,15 @@ int resctrl_val(char **benchmark_cmd, struct resctrl_val_param *param) if (ret) goto out; - if ((strcmp(resctrl_val, "mbm") == 0) || - (strcmp(resctrl_val, "mba") == 0)) { + if (!strncmp(resctrl_val, MBM_STR, sizeof(MBM_STR)) || + !strncmp(resctrl_val, MBA_STR, sizeof(MBA_STR))) { ret = initialize_mem_bw_imc(); if (ret) goto out; initialize_mem_bw_resctrl(param->ctrlgrp, param->mongrp, param->cpu_no, resctrl_val); - } else if (strcmp(resctrl_val, "cqm") == 0) + } else if (!strncmp(resctrl_val, CQM_STR, sizeof(CQM_STR))) initialize_llc_occu_resctrl(param->ctrlgrp, param->mongrp, param->cpu_no, resctrl_val); @@ -710,8 +733,8 @@ int resctrl_val(char **benchmark_cmd, struct resctrl_val_param *param) /* Test runs until the callback setup() tells the test to stop. */ while (1) { - if ((strcmp(resctrl_val, "mbm") == 0) || - (strcmp(resctrl_val, "mba") == 0)) { + if (!strncmp(resctrl_val, MBM_STR, sizeof(MBM_STR)) || + !strncmp(resctrl_val, MBA_STR, sizeof(MBA_STR))) { ret = param->setup(1, param); if (ret) { ret = 0; @@ -721,7 +744,7 @@ int resctrl_val(char **benchmark_cmd, struct resctrl_val_param *param) ret = measure_vals(param, &bw_resc_start); if (ret) break; - } else if (strcmp(resctrl_val, "cqm") == 0) { + } else if (!strncmp(resctrl_val, CQM_STR, sizeof(CQM_STR))) { ret = param->setup(1, param); if (ret) { ret = 0; diff --git a/tools/testing/selftests/resctrl/resctrlfs.c b/tools/testing/selftests/resctrl/resctrlfs.c index 19c0ec4045a4..b57170f53861 100644 --- a/tools/testing/selftests/resctrl/resctrlfs.c +++ b/tools/testing/selftests/resctrl/resctrlfs.c @@ -49,8 +49,6 @@ static int find_resctrl_mount(char *buffer) return -ENOENT; } -char cbm_mask[256]; - /* * remount_resctrlfs - Remount resctrl FS at /sys/fs/resctrl * @mum_resctrlfs: Should the resctrl FS be remounted? @@ -205,16 +203,18 @@ int get_cache_size(int cpu_no, char *cache_type, unsigned long *cache_size) /* * get_cbm_mask - Get cbm mask for given cache * @cache_type: Cache level L2/L3 - * - * Mask is stored in cbm_mask which is global variable. + * @cbm_mask: cbm_mask returned as a string * * Return: = 0 on success, < 0 on failure. */ -int get_cbm_mask(char *cache_type) +int get_cbm_mask(char *cache_type, char *cbm_mask) { char cbm_mask_path[1024]; FILE *fp; + if (!cbm_mask) + return -1; + sprintf(cbm_mask_path, "%s/%s/cbm_mask", CBM_MASK_PATH, cache_type); fp = fopen(cbm_mask_path, "r"); @@ -334,7 +334,7 @@ void run_benchmark(int signum, siginfo_t *info, void *ucontext) operation = atoi(benchmark_cmd[4]); sprintf(resctrl_val, "%s", benchmark_cmd[5]); - if (strcmp(resctrl_val, "cqm") != 0) + if (strncmp(resctrl_val, CQM_STR, sizeof(CQM_STR))) buffer_span = span * MB; else buffer_span = span; @@ -459,8 +459,8 @@ int write_bm_pid_to_resctrl(pid_t bm_pid, char *ctrlgrp, char *mongrp, goto out; /* Create mon grp and write pid into it for "mbm" and "cqm" test */ - if ((strcmp(resctrl_val, "cqm") == 0) || - (strcmp(resctrl_val, "mbm") == 0)) { + if (!strncmp(resctrl_val, CQM_STR, sizeof(CQM_STR)) || + !strncmp(resctrl_val, MBM_STR, sizeof(MBM_STR))) { if (strlen(mongrp)) { sprintf(monitorgroup_p, "%s/mon_groups", controlgroup); sprintf(monitorgroup, "%s/%s", monitorgroup_p, mongrp); @@ -505,9 +505,9 @@ int write_schemata(char *ctrlgrp, char *schemata, int cpu_no, char *resctrl_val) int resource_id, ret = 0; FILE *fp; - if ((strcmp(resctrl_val, "mba") != 0) && - (strcmp(resctrl_val, "cat") != 0) && - (strcmp(resctrl_val, "cqm") != 0)) + if (strncmp(resctrl_val, MBA_STR, sizeof(MBA_STR)) && + strncmp(resctrl_val, CAT_STR, sizeof(CAT_STR)) && + strncmp(resctrl_val, CQM_STR, sizeof(CQM_STR))) return -ENOENT; if (!schemata) { @@ -528,9 +528,10 @@ int write_schemata(char *ctrlgrp, char *schemata, int cpu_no, char *resctrl_val) else sprintf(controlgroup, "%s/schemata", RESCTRL_PATH); - if (!strcmp(resctrl_val, "cat") || !strcmp(resctrl_val, "cqm")) + if (!strncmp(resctrl_val, CAT_STR, sizeof(CAT_STR)) || + !strncmp(resctrl_val, CQM_STR, sizeof(CQM_STR))) sprintf(schema, "%s%d%c%s", "L3:", resource_id, '=', schemata); - if (strcmp(resctrl_val, "mba") == 0) + if (!strncmp(resctrl_val, MBA_STR, sizeof(MBA_STR))) sprintf(schema, "%s%d%c%s", "MB:", resource_id, '=', schemata); fp = fopen(controlgroup, "w"); @@ -615,26 +616,56 @@ char *fgrep(FILE *inf, const char *str) * validate_resctrl_feature_request - Check if requested feature is valid. * @resctrl_val: Requested feature * - * Return: 0 on success, non-zero on failure + * Return: True if the feature is supported, else false */ -bool validate_resctrl_feature_request(char *resctrl_val) +bool validate_resctrl_feature_request(const char *resctrl_val) { - FILE *inf = fopen("/proc/cpuinfo", "r"); + struct stat statbuf; bool found = false; char *res; + FILE *inf; - if (!inf) + if (!resctrl_val) return false; - res = fgrep(inf, "flags"); - - if (res) { - char *s = strchr(res, ':'); + if (remount_resctrlfs(false)) + return false; - found = s && !strstr(s, resctrl_val); - free(res); + if (!strncmp(resctrl_val, CAT_STR, sizeof(CAT_STR))) { + if (!stat(L3_PATH, &statbuf)) + return true; + } else if (!strncmp(resctrl_val, MBA_STR, sizeof(MBA_STR))) { + if (!stat(MB_PATH, &statbuf)) + return true; + } else if (!strncmp(resctrl_val, MBM_STR, sizeof(MBM_STR)) || + !strncmp(resctrl_val, CMT_STR, sizeof(CMT_STR))) { + if (!stat(L3_MON_PATH, &statbuf)) { + inf = fopen(L3_MON_FEATURES_PATH, "r"); + if (!inf) + return false; + + if (!strncmp(resctrl_val, CMT_STR, sizeof(CMT_STR))) { + res = fgrep(inf, "llc_occupancy"); + if (res) { + found = true; + free(res); + } + } + + if (!strncmp(resctrl_val, MBM_STR, sizeof(MBM_STR))) { + res = fgrep(inf, "mbm_total_bytes"); + if (res) { + free(res); + res = fgrep(inf, "mbm_local_bytes"); + if (res) { + found = true; + free(res); + } + } + } + fclose(inf); + } } - fclose(inf); return found; } diff --git a/tools/testing/selftests/x86/thunks_32.S b/tools/testing/selftests/x86/thunks_32.S index a71d92da8f46..f3f56e681e9f 100644 --- a/tools/testing/selftests/x86/thunks_32.S +++ b/tools/testing/selftests/x86/thunks_32.S @@ -45,3 +45,5 @@ call64_from_32: ret .size call64_from_32, .-call64_from_32 + +.section .note.GNU-stack,"",%progbits diff --git a/virt/kvm/coalesced_mmio.c b/virt/kvm/coalesced_mmio.c index 62bd908ecd58..f08f5e82460b 100644 --- a/virt/kvm/coalesced_mmio.c +++ b/virt/kvm/coalesced_mmio.c @@ -174,21 +174,36 @@ int kvm_vm_ioctl_unregister_coalesced_mmio(struct kvm *kvm, struct kvm_coalesced_mmio_zone *zone) { struct kvm_coalesced_mmio_dev *dev, *tmp; + int r; if (zone->pio != 1 && zone->pio != 0) return -EINVAL; mutex_lock(&kvm->slots_lock); - list_for_each_entry_safe(dev, tmp, &kvm->coalesced_zones, list) + list_for_each_entry_safe(dev, tmp, &kvm->coalesced_zones, list) { if (zone->pio == dev->zone.pio && coalesced_mmio_in_range(dev, zone->addr, zone->size)) { - kvm_io_bus_unregister_dev(kvm, + r = kvm_io_bus_unregister_dev(kvm, zone->pio ? KVM_PIO_BUS : KVM_MMIO_BUS, &dev->dev); kvm_iodevice_destructor(&dev->dev); + + /* + * On failure, unregister destroys all devices on the + * bus _except_ the target device, i.e. coalesced_zones + * has been modified. No need to restart the walk as + * there aren't any zones left. + */ + if (r) + break; } + } mutex_unlock(&kvm->slots_lock); + /* + * Ignore the result of kvm_io_bus_unregister_dev(), from userspace's + * perspective, the coalesced MMIO is most definitely unregistered. + */ return 0; } diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 383df23514b9..5cabc6c748db 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -2758,8 +2758,8 @@ static void grow_halt_poll_ns(struct kvm_vcpu *vcpu) if (val < grow_start) val = grow_start; - if (val > halt_poll_ns) - val = halt_poll_ns; + if (val > vcpu->kvm->max_halt_poll_ns) + val = vcpu->kvm->max_halt_poll_ns; vcpu->halt_poll_ns = val; out: @@ -2838,7 +2838,8 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu) goto out; } poll_end = cur = ktime_get(); - } while (single_task_running() && ktime_before(cur, stop)); + } while (single_task_running() && !need_resched() && + ktime_before(cur, stop)); } prepare_to_rcuwait(&vcpu->wait); @@ -4486,15 +4487,15 @@ int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, } /* Caller must hold slots_lock. */ -void kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, - struct kvm_io_device *dev) +int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, + struct kvm_io_device *dev) { int i, j; struct kvm_io_bus *new_bus, *bus; bus = kvm_get_bus(kvm, bus_idx); if (!bus) - return; + return 0; for (i = 0; i < bus->dev_count; i++) if (bus->range[i].dev == dev) { @@ -4502,7 +4503,7 @@ void kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, } if (i == bus->dev_count) - return; + return 0; new_bus = kmalloc(struct_size(bus, range, bus->dev_count - 1), GFP_KERNEL_ACCOUNT); @@ -4511,7 +4512,13 @@ void kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, new_bus->dev_count--; memcpy(new_bus->range + i, bus->range + i + 1, flex_array_size(new_bus, range, new_bus->dev_count - i)); - } else { + } + + rcu_assign_pointer(kvm->buses[bus_idx], new_bus); + synchronize_srcu_expedited(&kvm->srcu); + + /* Destroy the old bus _after_ installing the (null) bus. */ + if (!new_bus) { pr_err("kvm: failed to shrink bus, removing it completely\n"); for (j = 0; j < bus->dev_count; j++) { if (j == i) @@ -4520,10 +4527,8 @@ void kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, } } - rcu_assign_pointer(kvm->buses[bus_idx], new_bus); - synchronize_srcu_expedited(&kvm->srcu); kfree(bus); - return; + return new_bus ? 0 : -ENOMEM; } struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,