diff --git a/Documentation/device-mapper/dm-integrity.txt b/Documentation/device-mapper/dm-integrity.txt
index 297251b0d2d5..bf6af2ade0a6 100644
--- a/Documentation/device-mapper/dm-integrity.txt
+++ b/Documentation/device-mapper/dm-integrity.txt
@@ -146,6 +146,13 @@ block_size:number
Supported values are 512, 1024, 2048 and 4096 bytes. If not
specified the default block size is 512 bytes.
+legacy_recalculate
+ Allow recalculating of volumes with HMAC keys. This is disabled by
+ default for security reasons - an attacker could modify the volume,
+ set recalc_sector to zero, and the kernel would not detect the
+ modification.
+
+
The journal mode (D/J), buffer_sectors, journal_watermark, commit_time can
be changed when reloading the target (load an inactive table and swap the
tables with suspend and resume). The other arguments should not be changed
diff --git a/Makefile b/Makefile
index 98fa05359c2c..7e00ebfb0fcf 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: GPL-2.0
VERSION = 4
PATCHLEVEL = 19
-SUBLEVEL = 164
+SUBLEVEL = 172
EXTRAVERSION =
NAME = "People's Front"
@@ -400,7 +400,7 @@ YACC = bison
AWK = awk
GENKSYMS = scripts/genksyms/genksyms
INSTALLKERNEL := installkernel
-DEPMOD = /sbin/depmod
+DEPMOD = depmod
PERL = perl
PYTHON = python
PYTHON2 = python2
@@ -438,7 +438,7 @@ KBUILD_AFLAGS := -D__ASSEMBLY__
KBUILD_CFLAGS := -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs \
-fno-strict-aliasing -fno-common -fshort-wchar \
-Werror-implicit-function-declaration \
- -Wno-format-security \
+ -Werror=return-type -Wno-format-security \
-std=gnu89
KBUILD_CPPFLAGS := -D__KERNEL__
KBUILD_AFLAGS_KERNEL :=
diff --git a/android/abi_gki_aarch64 b/android/abi_gki_aarch64
index 7bb7e004c3de..d7d0576299d0 100644
--- a/android/abi_gki_aarch64
+++ b/android/abi_gki_aarch64
@@ -139,7 +139,9 @@
kstrtoull
__ll_sc_atomic64_fetch_andnot_release
__ll_sc_atomic64_fetch_or_acquire
+ __ll_sc_atomic_add
__ll_sc_atomic_sub_return
+ __ll_sc___cmpxchg_case_mb_4
memcpy
memset
memzero_explicit
diff --git a/android/abi_gki_aarch64.xml b/android/abi_gki_aarch64.xml
index 9d6093253121..c68af339baf6 100644
--- a/android/abi_gki_aarch64.xml
+++ b/android/abi_gki_aarch64.xml
@@ -82,6 +82,7 @@
+
@@ -162,6 +163,8 @@
+
+
@@ -1042,6 +1045,7 @@
+
@@ -1176,6 +1180,7 @@
+
@@ -1282,6 +1287,7 @@
+
@@ -1336,6 +1342,7 @@
+
@@ -1990,6 +1997,7 @@
+
@@ -2152,6 +2160,7 @@
+
@@ -2245,6 +2254,7 @@
+
@@ -2569,6 +2579,7 @@
+
@@ -2683,9 +2694,13 @@
+
+
+
+
@@ -2694,6 +2709,7 @@
+
@@ -2771,37 +2787,642 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -2926,20 +3547,34 @@
-
-
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -2991,6 +3626,20 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -3150,7 +3799,7 @@
-
+
@@ -3451,21 +4100,21 @@
-
+
-
+
-
+
-
+
-
+
@@ -3473,7 +4122,7 @@
-
+
@@ -3644,6 +4293,17 @@
+
+
+
+
+
+
+
+
+
+
+
@@ -3914,8 +4574,8 @@
-
-
+
+
@@ -3955,7 +4615,7 @@
-
+
@@ -3996,13 +4656,13 @@
-
+
-
+
@@ -4160,8 +4820,8 @@
-
-
+
+
@@ -4730,31 +5390,22 @@
-
+
-
+
-
-
-
-
-
-
-
-
-
-
+
-
+
-
+
-
+
@@ -5185,7 +5836,7 @@
-
+
@@ -5245,21 +5896,21 @@
-
+
-
+
-
+
-
+
-
+
-
+
@@ -5279,7 +5930,7 @@
-
+
@@ -5327,7 +5978,7 @@
-
+
@@ -5417,8 +6068,8 @@
-
-
+
+
@@ -5475,36 +6126,36 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
@@ -5512,7 +6163,7 @@
-
+
@@ -5536,7 +6187,7 @@
-
+
@@ -5547,7 +6198,7 @@
-
+
@@ -5565,26 +6216,26 @@
-
+
-
+
-
+
-
+
-
+
-
+
@@ -5595,7 +6246,7 @@
-
+
@@ -5603,7 +6254,7 @@
-
+
@@ -5611,7 +6262,7 @@
-
+
@@ -5648,8 +6299,8 @@
-
-
+
+
@@ -5671,7 +6322,7 @@
-
+
@@ -5683,30 +6334,30 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
@@ -5720,9 +6371,9 @@
-
+
-
+
@@ -5731,18 +6382,18 @@
-
+
-
+
-
+
-
+
@@ -5753,7 +6404,18 @@
-
+
+
+
+
+
+
+
+
+
+
+
+
@@ -5764,7 +6426,7 @@
-
+
@@ -5778,7 +6440,7 @@
-
+
@@ -5789,7 +6451,7 @@
-
+
@@ -5807,7 +6469,15 @@
-
+
+
+
+
+
+
+
+
+
@@ -5818,12 +6488,26 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
@@ -5845,15 +6529,15 @@
-
+
-
+
-
+
-
+
@@ -5862,7 +6546,7 @@
-
+
@@ -5890,17 +6574,45 @@
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
@@ -5965,7 +6677,7 @@
-
+
@@ -6013,8 +6725,8 @@
-
-
+
+
@@ -6067,14 +6779,22 @@
-
-
+
+
-
+
+
+
+
+
+
+
+
+
@@ -6083,7 +6803,7 @@
-
+
@@ -6165,6 +6885,17 @@
+
+
+
+
+
+
+
+
+
+
+
@@ -6377,7 +7108,7 @@
-
+
@@ -6632,6 +7363,29 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -6905,109 +7659,17285 @@
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
-
+
-
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
-
+
-
-
-
-
+
-
+
-
+
-
-
-
-
-
-
-
+
-
+
-
+
-
-
-
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
-
-
-
-
+
-
+
-
+
-
-
-
-
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
-
-
-
-
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -7035,6 +24965,8 @@
+
+
@@ -7119,6 +25051,9 @@
+
+
+
@@ -7152,6 +25087,28 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -7171,6 +25128,1273 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -7287,6 +26511,7 @@
+
@@ -7393,6 +26618,7 @@
+
@@ -7409,114 +26635,30 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -7620,35 +26762,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -7696,23 +26809,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -7845,7 +26941,7 @@
-
+
@@ -7858,7 +26954,7 @@
-
+
@@ -8011,6 +27107,34 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -8025,6 +27149,72 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -8063,14 +27253,6 @@
-
-
-
-
-
-
-
-
@@ -8083,43 +27265,8 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -8172,11 +27319,739 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -8421,11 +28296,12 @@
-
+
+
@@ -8502,10 +28378,136 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
+
+
@@ -8516,23 +28518,375 @@
-
-
+
+
-
-
-
+
+
+
-
-
+
+
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -8623,6 +28977,9 @@
+
+
+
@@ -8634,9 +28991,9 @@
-
+
-
+
@@ -8660,7 +29017,7 @@
-
+
@@ -8668,7 +29025,7 @@
-
+
@@ -8679,7 +29036,7 @@
-
+
@@ -8713,7 +29070,7 @@
-
+
@@ -8802,45 +29159,16 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
-
+
@@ -8866,65 +29194,36 @@
-
+
-
+
-
+
-
+
-
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
@@ -8935,50 +29234,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -9034,6 +29289,7 @@
+
@@ -9099,6 +29355,7 @@
+
@@ -9198,6 +29455,15 @@
+
+
+
+
+
+
+
+
+
@@ -9296,6 +29562,18 @@
+
+
+
+
+
+
+
+
+
+
+
+
@@ -9348,6 +29626,24 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -9395,28 +29691,710 @@
-
+
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -9429,6 +30407,32 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -9738,7 +30742,7 @@
-
+
@@ -9909,7 +30913,7 @@
-
+
@@ -9930,7 +30934,7 @@
-
+
@@ -9942,7 +30946,7 @@
-
+
@@ -9969,7 +30973,7 @@
-
+
@@ -9990,7 +30994,7 @@
-
+
@@ -10005,13 +31009,13 @@
-
+
-
+
@@ -10103,7 +31107,7 @@
-
+
@@ -10410,7 +31414,7 @@
-
+
@@ -10422,7 +31426,7 @@
-
+
@@ -10557,7 +31561,7 @@
-
+
@@ -10635,7 +31639,18 @@
-
+
+
+
+
+
+
+
+
+
+
+
+
@@ -10646,7 +31661,39 @@
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -10689,6 +31736,17 @@
+
+
+
+
+
+
+
+
+
+
+
@@ -10707,8 +31765,16 @@
+
+
+
+
+
+
+
+
-
+
@@ -10716,6 +31782,30 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -10836,8 +31926,8 @@
-
-
+
+
@@ -10845,8 +31935,8 @@
-
-
+
+
@@ -10854,8 +31944,8 @@
-
-
+
+
@@ -10878,14 +31968,14 @@
-
-
+
+
-
-
+
+
@@ -10893,8 +31983,8 @@
-
-
+
+
@@ -10905,8 +31995,8 @@
-
-
+
+
@@ -11179,7 +32269,7 @@
-
+
@@ -11196,38 +32286,38 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
@@ -11235,25 +32325,25 @@
-
+
-
+
-
+
-
+
-
+
-
+
@@ -12129,7 +33219,7 @@
-
+
@@ -12176,7 +33266,7 @@
-
+
@@ -12235,8 +33325,8 @@
-
-
+
+
@@ -12342,10 +33432,10 @@
-
+
-
+
@@ -12450,7 +33540,7 @@
-
+
@@ -12459,17 +33549,34 @@
-
+
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -12648,7 +33755,7 @@
-
+
@@ -12675,7 +33782,7 @@
-
+
@@ -12684,7 +33791,7 @@
-
+
@@ -12978,13 +34085,13 @@
-
+
-
+
@@ -13247,8 +34354,8 @@
-
-
+
+
@@ -13288,7 +34395,7 @@
-
+
@@ -13327,19 +34434,19 @@
-
+
-
+
-
+
-
+
@@ -13363,16 +34470,16 @@
-
+
-
+
-
+
@@ -13526,7 +34633,7 @@
-
+
@@ -14085,71 +35192,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -14511,20 +35553,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -14548,17 +35576,6 @@
-
-
-
-
-
-
-
-
-
-
-
@@ -14566,6 +35583,17 @@
+
+
+
+
+
+
+
+
+
+
+
@@ -14592,7 +35620,7 @@
-
+
@@ -14601,7 +35629,7 @@
-
+
@@ -14615,7 +35643,7 @@
-
+
@@ -14638,8 +35666,8 @@
-
-
+
+
@@ -14665,8 +35693,8 @@
-
-
+
+
@@ -14683,7 +35711,7 @@
-
+
@@ -14771,11 +35799,11 @@
-
+
-
-
+
+
@@ -14809,7 +35837,7 @@
-
+
@@ -14820,7 +35848,7 @@
-
+
@@ -14883,10 +35911,6 @@
-
-
-
-
@@ -14899,7 +35923,11 @@
-
+
+
+
+
+
@@ -14907,14 +35935,20 @@
+
+
+
+
+
+
-
-
+
+
-
+
-
+
@@ -15202,12 +36236,12 @@
-
+
-
+
@@ -15342,7 +36376,7 @@
-
+
@@ -15402,7 +36436,7 @@
-
+
@@ -15416,23 +36450,23 @@
-
+
-
+
-
+
-
+
@@ -15508,7 +36542,7 @@
-
+
@@ -15546,109 +36580,17 @@
-
-
-
+
+
+
+
-
-
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
@@ -15733,46 +36675,43 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
+
+
+
-
-
+
+
-
-
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
@@ -16075,21 +37014,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -16179,6 +37103,41 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -16206,30 +37165,7 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
@@ -16246,21 +37182,7 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
@@ -16319,6 +37241,7 @@
+
@@ -16331,39 +37254,131 @@
+
+
+
+
+
+
+
+
+
-
-
+
+
-
-
-
-
-
-
-
-
+
-
+
+
-
-
+
+
+
+
+
+
+
+
-
-
+
+
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -16401,6 +37416,8 @@
+
+
@@ -16417,6 +37434,12 @@
+
+
+
+
+
+
@@ -16519,6 +37542,160 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -16628,7 +37805,7 @@
-
+
@@ -16683,15 +37860,15 @@
-
+
-
+
-
+
@@ -16856,6 +38033,496 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -16958,16 +38625,9 @@
+
-
-
-
-
-
-
-
-
@@ -16993,6 +38653,448 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -17013,6 +39115,950 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -17126,7 +40172,7 @@
-
+
@@ -17203,6 +40249,20 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -17554,6 +40614,7 @@
+
@@ -17607,11 +40668,25 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
@@ -17794,7 +40869,15 @@
-
+
+
+
+
+
+
+
+
+
@@ -18305,6 +41388,7 @@
+
@@ -18387,14 +41471,6 @@
-
-
-
-
-
-
-
-
@@ -18441,10 +41517,25 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
+
+
@@ -18466,6 +41557,8 @@
+
+
@@ -18480,7 +41573,7 @@
-
+
@@ -18497,7 +41590,7 @@
-
+
@@ -18547,7 +41640,7 @@
-
+
@@ -18583,14 +41676,14 @@
-
+
-
+
@@ -18709,6 +41802,7 @@
+
@@ -18717,6 +41811,29 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -18874,6 +41991,14 @@
+
+
+
+
+
+
+
+
@@ -18970,7 +42095,132 @@
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -19158,7 +42408,7 @@
-
+
@@ -19265,7 +42515,7 @@
-
+
@@ -19280,154 +42530,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -19595,8 +42697,159 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -19655,6 +42908,8 @@
+
+
@@ -19810,12 +43065,12 @@
-
+
-
-
-
+
+
+
@@ -19861,6 +43116,216 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -20181,10 +43646,7 @@
-
-
-
-
+
@@ -20227,6 +43689,82 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -20262,6 +43800,136 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -20471,6 +44139,7 @@
+
@@ -20499,8 +44168,197 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -20785,6 +44643,300 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -20810,6 +44962,7 @@
+
@@ -20848,6 +45001,347 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -21134,6 +45628,125 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -21440,8 +46053,8 @@
-
-
+
+
@@ -21482,8 +46095,8 @@
-
-
+
+
@@ -21500,8 +46113,8 @@
-
-
+
+
@@ -21550,8 +46163,8 @@
-
-
+
+
@@ -21586,8 +46199,8 @@
-
-
+
+
@@ -21603,8 +46216,8 @@
-
-
+
+
@@ -21632,8 +46245,8 @@
-
-
+
+
@@ -21649,8 +46262,8 @@
-
-
+
+
@@ -21989,6 +46602,7 @@
+
@@ -22058,7 +46672,7 @@
-
+
@@ -22205,61 +46819,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -22441,6 +47000,8 @@
+
+
@@ -22497,6 +47058,9 @@
+
+
+
@@ -22507,6 +47071,3966 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -22567,20 +51091,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -22814,6 +51324,7 @@
+
@@ -22836,12 +51347,22 @@
+
+
+
+
+
+
+
+
+
+
-
-
-
-
+
+
+
+
@@ -22910,6 +51431,15 @@
+
+
+
+
+
+
+
+
+
@@ -23009,6 +51539,8 @@
+
+
@@ -23021,13 +51553,72 @@
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -23220,7 +51811,7 @@
-
+
@@ -23265,7 +51856,7 @@
-
+
@@ -23405,7 +51996,7 @@
-
+
@@ -23715,7 +52306,7 @@
-
+
@@ -23982,7 +52573,7 @@
-
+
@@ -24164,7 +52755,7 @@
-
+
@@ -24960,7 +53551,7 @@
-
+
@@ -25063,6 +53654,17 @@
+
+
+
+
+
+
+
+
+
+
+
@@ -25731,7 +54333,7 @@
-
+
@@ -25766,7 +54368,7 @@
-
+
@@ -26289,7 +54891,7 @@
-
+
@@ -26311,7 +54913,7 @@
-
+
@@ -26346,7 +54948,7 @@
-
+
@@ -27275,17 +55877,25 @@
-
-
-
+
+
+
-
-
+
+
-
-
+
+
-
+
+
+
+
+
+
+
+
+
@@ -28083,13 +56693,13 @@
-
-
-
-
-
-
-
+
+
+
+
+
+
+
@@ -28172,29 +56782,30 @@
-
-
-
+
+
+
-
-
-
+
+
+
-
-
+
+
-
-
-
+
+
+
-
-
+
+
+
@@ -28210,6 +56821,326 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -28384,7 +57315,7 @@
-
+
@@ -28483,7 +57414,7 @@
-
+
@@ -28507,26 +57438,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -28597,6 +57508,8 @@
+
+
@@ -28694,6 +57607,68 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -28728,14 +57703,6 @@
-
-
-
-
-
-
-
-
@@ -28814,7 +57781,7 @@
-
+
@@ -28949,23 +57916,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -29005,7 +57955,7 @@
-
+
@@ -30007,18 +58957,18 @@
-
+
-
+
-
+
-
+
-
+
@@ -30038,7 +58988,7 @@
-
+
@@ -30228,17 +59178,6 @@
-
-
-
-
-
-
-
-
-
-
-
@@ -30355,6 +59294,508 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -30368,13 +59809,13 @@
-
+
-
+
@@ -30552,7 +59993,7 @@
-
+
@@ -30895,7 +60336,7 @@
-
+
@@ -31095,6 +60536,575 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -31119,8 +61129,19 @@
-
+
+
+
+
+
+
+
+
+
+
+
+
@@ -31245,17 +61266,6 @@
-
-
-
-
-
-
-
-
-
-
-
@@ -31556,7 +61566,7 @@
-
+
@@ -31593,6 +61603,7 @@
+
@@ -31614,6 +61625,128 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -31630,17 +61763,6 @@
-
-
-
-
-
-
-
-
-
-
-
@@ -31730,6 +61852,32 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -31775,11 +61923,6 @@
-
-
-
-
-
@@ -31875,7 +62018,7 @@
-
+
@@ -31904,7 +62047,7 @@
-
+
@@ -31995,7 +62138,7 @@
-
+
@@ -32333,9 +62476,11 @@
+
+
-
+
@@ -32347,8 +62492,8 @@
-
-
+
+
@@ -32375,18 +62520,18 @@
-
+
-
+
-
+
-
+
@@ -32394,7 +62539,7 @@
-
+
@@ -32404,17 +62549,17 @@
-
+
-
+
-
+
@@ -32427,9 +62572,33 @@
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -33296,6 +63465,915 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -33422,6 +64500,10 @@
+
+
+
+
@@ -33459,29 +64541,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -33535,6 +64594,7 @@
+
@@ -33584,26 +64644,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -33640,7 +64680,7 @@
-
+
@@ -33660,21 +64700,21 @@
-
+
-
+
-
+
-
+
-
+
-
+
@@ -33685,7 +64725,7 @@
-
+
@@ -33693,7 +64733,7 @@
-
+
@@ -33707,7 +64747,7 @@
-
+
@@ -33821,6 +64861,21 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -33843,41 +64898,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -34017,6 +65037,23 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -34025,43 +65062,9 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
+
@@ -34070,18 +65073,18 @@
-
+
-
+
-
+
@@ -34826,6 +65829,86 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -34850,7 +65933,7 @@
-
+
@@ -34860,6 +65943,18 @@
+
+
+
+
+
+
+
+
+
+
+
+
@@ -34870,6 +65965,40 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -34883,7 +66012,7 @@
-
+
@@ -34912,7 +66041,7 @@
-
+
@@ -35017,7 +66146,7 @@
-
+
@@ -35235,7 +66364,7 @@
-
+
@@ -35496,7 +66625,7 @@
-
+
@@ -35544,7 +66673,7 @@
-
+
@@ -35553,7 +66682,7 @@
-
+
@@ -35641,7 +66770,7 @@
-
+
@@ -35650,38 +66779,38 @@
-
+
-
+
-
+
-
+
-
+
-
+
@@ -35744,17 +66873,17 @@
-
+
-
+
-
+
@@ -35793,7 +66922,7 @@
-
+
@@ -35803,16 +66932,16 @@
-
+
-
+
-
+
@@ -35877,38 +67006,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -36141,8 +67238,8 @@
-
-
+
+
@@ -36342,11 +67439,363 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -36379,20 +67828,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -36407,29 +67842,111 @@
-
-
-
+
+
+
-
-
+
+
-
-
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -36519,12 +68036,467 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
@@ -36609,7 +68581,7 @@
-
+
@@ -36661,7 +68633,7 @@
-
+
@@ -36716,7 +68688,7 @@
-
+
@@ -36828,7 +68800,12 @@
-
+
+
+
+
+
+
@@ -36923,11 +68900,18 @@
-
-
-
+
+
+
+
-
+
+
+
+
+
+
+
@@ -37043,6 +69027,274 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -37096,8 +69348,34 @@
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -37127,7 +69405,7 @@
-
+
@@ -38329,7 +70607,7 @@
-
+
@@ -38342,7 +70620,7 @@
-
+
@@ -38350,13 +70628,13 @@
-
+
-
+
-
+
@@ -38364,7 +70642,7 @@
-
+
@@ -39213,10 +71491,10 @@
-
+
-
+
@@ -39608,15 +71886,15 @@
-
+
-
+
-
+
-
+
@@ -39629,26 +71907,26 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
@@ -39656,7 +71934,7 @@
-
+
@@ -40013,7 +72291,7 @@
-
+
@@ -40069,7 +72347,7 @@
-
+
@@ -40103,6 +72381,10 @@
+
+
+
+
@@ -40718,6 +73000,25 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -40744,15 +73045,9 @@
-
-
-
-
-
-
@@ -40818,7 +73113,6 @@
-
@@ -40868,8 +73162,117 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -40902,6 +73305,52 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -40915,7 +73364,7 @@
-
+
@@ -41065,7 +73514,7 @@
-
+
@@ -41154,13 +73603,13 @@
-
+
-
+
@@ -41389,6 +73838,17 @@
+
+
+
+
+
+
+
+
+
+
+
@@ -41450,35 +73910,9 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
@@ -41625,7 +74059,7 @@
-
+
@@ -41634,7 +74068,7 @@
-
+
@@ -41781,30 +74215,8 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -41966,7 +74378,7 @@
-
+
@@ -42254,80 +74666,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -42337,22 +74675,11 @@
-
+
-
-
-
-
-
-
-
-
-
-
-
@@ -42562,43 +74889,20 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
+
@@ -42606,41 +74910,15 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
+
-
+
@@ -42650,7 +74928,7 @@
-
+
@@ -42686,19 +74964,19 @@
-
+
-
-
+
+
-
+
@@ -42775,7 +75053,7 @@
-
+
@@ -42788,7 +75066,110 @@
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -42954,11 +75335,88 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -42992,35 +75450,15 @@
-
+
-
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -43046,15 +75484,15 @@
-
+
-
+
-
+
@@ -43155,7 +75593,7 @@
-
+
@@ -43240,7 +75678,7 @@
-
+
@@ -43264,10 +75702,10 @@
-
+
-
+
@@ -43500,13 +75938,13 @@
-
+
-
+
@@ -43575,8 +76013,16 @@
+
+
+
+
+
+
+
+
-
+
@@ -43584,8 +76030,24 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
@@ -43593,6 +76055,14 @@
+
+
+
+
+
+
+
+
@@ -43777,24 +76247,15 @@
-
-
-
+
+
+
-
-
+
+
-
-
-
-
-
-
-
-
-
-
-
+
+
@@ -43835,7 +76296,7 @@
-
+
@@ -43862,7 +76323,7 @@
-
+
@@ -43937,6 +76398,14 @@
+
+
+
+
+
+
+
+
@@ -44043,7 +76512,7 @@
-
+
@@ -44389,13 +76858,13 @@
-
+
-
+
-
+
@@ -44404,10 +76873,10 @@
-
+
-
+
@@ -44415,7 +76884,7 @@
-
+
@@ -44423,7 +76892,7 @@
-
+
@@ -44432,7 +76901,7 @@
-
+
@@ -44633,7 +77102,7 @@
-
+
@@ -44645,7 +77114,7 @@
-
+
@@ -44741,6 +77210,20 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -44760,8 +77243,38 @@
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -44772,7 +77285,7 @@
-
+
@@ -44794,7 +77307,7 @@
-
+
@@ -44847,7 +77360,7 @@
-
+
@@ -44907,7 +77420,7 @@
-
+
@@ -44948,7 +77461,7 @@
-
+
@@ -44979,43 +77492,43 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
@@ -45053,7 +77566,7 @@
-
+
@@ -45075,7 +77588,7 @@
-
+
@@ -45113,7 +77626,7 @@
-
+
@@ -45124,7 +77637,7 @@
-
+
@@ -45138,7 +77651,7 @@
-
+
@@ -45164,7 +77677,7 @@
-
+
@@ -45189,7 +77702,7 @@
-
+
@@ -45200,7 +77713,7 @@
-
+
@@ -45220,7 +77733,7 @@
-
+
@@ -45228,7 +77741,7 @@
-
+
@@ -45245,7 +77758,7 @@
-
+
@@ -45429,7 +77942,7 @@
-
+
@@ -45453,11 +77966,14 @@
-
-
-
+
+
+
-
+
+
+
+
@@ -45480,6 +77996,22 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -45494,7 +78026,7 @@
-
+
@@ -45587,7 +78119,7 @@
-
+
@@ -45601,7 +78133,7 @@
-
+
@@ -45646,6 +78178,14 @@
+
+
+
+
+
+
+
+
@@ -45669,8 +78209,8 @@
-
-
+
+
@@ -45776,8 +78316,8 @@
-
-
+
+
@@ -45802,18 +78342,18 @@
-
+
-
+
-
+
-
+
@@ -45833,20 +78373,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -45855,7 +78381,7 @@
-
+
@@ -46065,38 +78591,7 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
@@ -46116,173 +78611,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -46291,112 +78619,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -46445,9 +78667,9 @@
-
+
-
+
@@ -46462,7 +78684,7 @@
-
+
@@ -46480,7 +78702,7 @@
-
+
@@ -46557,7 +78779,7 @@
-
+
@@ -46586,7 +78808,7 @@
-
+
@@ -46671,7 +78893,7 @@
-
+
@@ -46764,29 +78986,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -46856,7 +79055,7 @@
-
+
@@ -46901,21 +79100,21 @@
-
+
-
+
-
+
-
+
-
+
@@ -46923,7 +79122,7 @@
-
+
@@ -47042,7 +79241,7 @@
-
+
@@ -47060,10 +79259,10 @@
-
+
-
+
@@ -47128,7 +79327,7 @@
-
+
@@ -47391,13 +79590,21 @@
-
+
-
+
+
+
+
+
+
+
+
+
@@ -47405,7 +79612,7 @@
-
+
@@ -47419,17 +79626,6 @@
-
-
-
-
-
-
-
-
-
-
-
@@ -47525,18 +79721,18 @@
-
+
-
+
-
-
-
+
+
+
@@ -47564,7 +79760,7 @@
-
+
@@ -47590,14 +79786,14 @@
-
+
-
+
-
+
@@ -47607,7 +79803,7 @@
-
+
@@ -47688,17 +79884,17 @@
-
+
-
+
-
-
+
+
@@ -47712,15 +79908,15 @@
-
-
+
+
-
+
@@ -47801,12 +79997,12 @@
-
+
-
-
+
+
@@ -47834,7 +80030,7 @@
-
+
@@ -47918,7 +80114,7 @@
-
+
@@ -47937,32 +80133,32 @@
-
+
-
+
-
+
-
+
-
+
-
+
@@ -47971,31 +80167,31 @@
-
+
-
+
-
+
-
+
-
+
-
+
@@ -48024,7 +80220,7 @@
-
+
@@ -48050,7 +80246,7 @@
-
+
@@ -48115,11 +80311,11 @@
-
+
-
+
@@ -48165,7 +80361,7 @@
-
+
@@ -48367,7 +80563,7 @@
-
+
@@ -48377,30 +80573,30 @@
-
+
-
+
-
+
-
+
-
+
@@ -48412,16 +80608,16 @@
-
+
-
+
-
+
@@ -48434,7 +80630,7 @@
-
+
@@ -48486,7 +80682,7 @@
-
+
@@ -48495,7 +80691,7 @@
-
+
@@ -48509,32 +80705,32 @@
-
+
-
+
-
+
-
+
-
+
-
+
@@ -48555,7 +80751,7 @@
-
+
@@ -48584,15 +80780,15 @@
-
+
-
+
-
+
@@ -48600,7 +80796,7 @@
-
+
@@ -48666,28 +80862,28 @@
-
+
-
+
-
+
-
+
-
+
@@ -48714,130 +80910,91 @@
-
-
-
+
+
+
+
+
-
-
+
+
-
-
-
-
+
+
-
-
+
+
-
-
-
-
+
+
-
-
+
+
-
-
-
-
+
+
-
-
+
+
+
+
-
-
-
-
+
+
-
-
+
+
-
-
-
-
+
+
-
-
+
+
-
-
+
+
-
-
+
+
-
-
-
-
+
+
-
-
+
+
-
-
+
+
-
-
-
-
+
+
-
-
+
+
-
-
-
-
+
+
-
-
+
+
+
+
+
+
+
+
+
-
-
-
-
+
+
-
-
+
+
-
-
-
-
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
@@ -48849,6 +81006,104 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -48956,7 +81211,7 @@
-
+
@@ -49027,80 +81282,10 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
+
@@ -49124,7 +81309,7 @@
-
+
@@ -49133,7 +81318,7 @@
-
+
@@ -49199,7 +81384,7 @@
-
+
@@ -49379,7 +81564,7 @@
-
+
@@ -49423,7 +81608,7 @@
-
+
@@ -49437,10 +81622,10 @@
-
-
-
-
+
+
+
+
@@ -49512,47 +81697,47 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
@@ -49561,8 +81746,64 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -49613,53 +81854,78 @@
-
+
-
+
-
-
-
-
+
-
-
-
-
+
-
+
-
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -49710,6 +81976,32 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -49822,44 +82114,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -50043,6 +82297,8 @@
+
+
@@ -50054,6 +82310,20 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -50130,14 +82400,6 @@
-
-
-
-
-
-
-
-
@@ -50154,7 +82416,7 @@
-
+
@@ -50207,7 +82469,7 @@
-
+
@@ -50300,23 +82562,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -50716,6 +82961,39 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -50725,9 +83003,50 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
@@ -51089,13 +83408,13 @@
-
+
-
+
@@ -51159,7 +83478,7 @@
-
+
@@ -51243,7 +83562,7 @@
-
+
@@ -51497,6 +83816,274 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -51814,7 +84401,7 @@
-
+
@@ -51825,23 +84412,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -51955,17 +84525,546 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
-
+
-
+
@@ -51993,7 +85092,6 @@
-
@@ -52013,6 +85111,23 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -52504,7 +85619,7 @@
-
+
@@ -52541,169 +85656,17 @@
-
-
-
+
+
+
-
-
+
+
-
-
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
@@ -52723,7 +85686,6 @@
-
@@ -52773,10 +85735,6 @@
-
-
-
-
@@ -52787,12 +85745,6 @@
-
-
-
-
-
-
@@ -52931,32 +85883,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -53005,6 +85931,33 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -53025,6 +85978,167 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -53142,6 +86256,56 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -53159,7 +86323,7 @@
-
+
@@ -53521,6 +86685,8 @@
+
+
@@ -53533,6 +86699,414 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -53540,6 +87114,436 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -53549,23 +87553,7 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
@@ -53602,7 +87590,7 @@
-
+
@@ -53663,16 +87651,210 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
-
-
@@ -53703,7 +87885,7 @@
-
+
@@ -54010,7 +88192,7 @@
-
+
@@ -54081,7 +88263,7 @@
-
+
@@ -54240,17 +88422,28 @@
-
-
-
+
+
+
-
-
+
+
-
-
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
@@ -54675,6 +88868,14 @@
+
+
+
+
+
+
+
+
@@ -54746,6 +88947,7 @@
+
@@ -54771,7 +88973,7 @@
-
+
@@ -54976,19 +89178,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -55010,30 +89199,8 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -55047,19 +89214,19 @@
-
+
-
+
-
+
-
+
-
+
@@ -55067,7 +89234,7 @@
-
+
@@ -55078,7 +89245,7 @@
-
+
@@ -55086,7 +89253,7 @@
-
+
@@ -55149,40 +89316,40 @@
-
-
+
+
-
-
-
+
+
+
-
-
+
+
-
-
-
-
+
+
+
+
-
-
+
+
-
-
-
+
+
+
-
-
+
+
-
-
+
+
@@ -55360,6 +89527,7 @@
+
@@ -55567,7 +89735,7 @@
-
+
@@ -55793,7 +89961,7 @@
-
+
@@ -55942,10 +90110,6 @@
-
-
-
-
@@ -56069,14 +90233,6 @@
-
-
-
-
-
-
-
-
@@ -56259,6 +90415,32 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -56454,7 +90636,7 @@
-
+
@@ -57626,7 +91808,7 @@
-
+
@@ -57886,8 +92068,8 @@
-
-
+
+
@@ -58594,9 +92776,9 @@
-
-
-
+
+
+
@@ -59108,6 +93290,7 @@
+
@@ -59170,6 +93353,494 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -59247,19 +93918,19 @@
-
-
+
+
-
-
-
-
-
+
+
+
+
+
-
-
+
+
@@ -59282,12 +93953,548 @@
-
-
+
+
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -59302,6 +94509,423 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -59313,12 +94937,12 @@
-
-
-
-
-
-
+
+
+
+
+
+
@@ -59335,8 +94959,45 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -60094,6 +95755,9 @@
+
+
+
@@ -60173,14 +95837,278 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -60264,7 +96192,7 @@
-
+
@@ -61293,7 +97221,7 @@
-
+
@@ -61688,23 +97616,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -63416,10 +99327,10 @@
-
+
-
+
@@ -64052,15 +99963,15 @@
-
+
-
+
-
+
@@ -64358,7 +100269,6 @@
-
@@ -64446,6 +100356,75 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -64828,24 +100807,11 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -64856,11 +100822,6 @@
-
-
-
-
-
@@ -65970,10 +101931,11 @@
-
+
+
@@ -65993,9 +101955,9 @@
-
+
-
+
@@ -66019,21 +101981,21 @@
-
+
-
+
-
+
-
+
-
+
@@ -66059,12 +102021,12 @@
-
+
-
+
-
+
@@ -66080,7 +102042,7 @@
-
+
@@ -66104,7 +102066,7 @@
-
+
@@ -66145,7 +102107,8 @@
-
+
+
@@ -66173,10 +102136,10 @@
-
+
-
+
@@ -66189,28 +102152,27 @@
-
+
-
+
-
+
-
+
-
+
-
@@ -66227,24 +102189,12 @@
-
-
-
-
-
-
-
-
-
-
-
-
@@ -66324,12 +102274,73 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -66351,40 +102362,7 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -66404,6 +102382,94 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -66442,39 +102508,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -66485,28 +102518,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -66516,6 +102527,17 @@
+
+
+
+
+
+
+
+
+
+
+
@@ -66527,44 +102549,46 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
+
+
@@ -66608,7 +102632,7 @@
-
+
@@ -66616,49 +102640,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -66680,21 +102661,21 @@
-
+
-
+
-
+
-
+
-
+
@@ -66708,7 +102689,7 @@
-
+
@@ -66728,41 +102709,18 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
+
-
+
-
+
@@ -66798,17 +102756,17 @@
-
-
-
+
+
+
-
-
+
+
-
+
@@ -66817,10 +102775,10 @@
-
+
-
+
@@ -66832,8 +102790,8 @@
-
-
+
+
@@ -66866,19 +102824,19 @@
-
-
+
+
-
+
-
+
@@ -66887,7 +102845,7 @@
-
+
@@ -66896,9 +102854,9 @@
-
+
-
+
@@ -66932,10 +102890,9 @@
-
+
-
@@ -66944,18 +102901,18 @@
-
+
-
+
-
+
-
+
@@ -67017,7 +102974,7 @@
-
+
@@ -67195,7 +103152,7 @@
-
+
@@ -67269,39 +103226,17 @@
-
+
-
+
-
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -67310,7 +103245,28 @@
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -67321,34 +103277,25 @@
-
+
-
+
-
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
-
@@ -67361,6 +103308,10 @@
+
+
+
+
@@ -67553,7 +103504,7 @@
-
+
@@ -67690,6 +103641,17 @@
+
+
+
+
+
+
+
+
+
+
+
@@ -67726,6 +103688,8 @@
+
+
@@ -67738,53 +103702,15 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
+
-
+
@@ -67856,7 +103782,6 @@
-
@@ -67877,6 +103802,8 @@
+
+
@@ -67893,14 +103820,14 @@
-
+
-
+
@@ -67916,11 +103843,98 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -67982,16 +103996,16 @@
-
-
+
+
-
-
+
+
@@ -68063,6 +104077,26 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -68097,7 +104131,7 @@
-
+
@@ -68118,43 +104152,17 @@
-
+
-
+
-
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -68192,6 +104200,23 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -68285,12 +104310,12 @@
-
+
-
+
@@ -68326,6 +104351,14 @@
+
+
+
+
+
+
+
+
@@ -68347,7 +104380,418 @@
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -68383,146 +104827,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -68623,39 +104927,39 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
@@ -68663,7 +104967,7 @@
-
+
@@ -68692,7 +104996,7 @@
-
+
@@ -68719,7 +105023,7 @@
-
+
@@ -68727,7 +105031,7 @@
-
+
@@ -68741,7 +105045,7 @@
-
+
@@ -68752,7 +105056,7 @@
-
+
@@ -68760,7 +105064,7 @@
-
+
@@ -68768,12 +105072,12 @@
-
+
-
+
@@ -68829,15 +105133,15 @@
-
+
-
+
-
+
@@ -68846,44 +105150,11 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -68916,12 +105187,12 @@
-
+
-
+
-
+
@@ -68940,15 +105211,15 @@
-
+
-
+
-
+
@@ -68964,7 +105235,6 @@
-
@@ -68973,12 +105243,12 @@
-
-
-
+
+
+
-
-
+
+
@@ -68986,7 +105256,7 @@
-
+
@@ -68995,41 +105265,41 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
-
-
+
+
+
-
-
+
+
-
+
@@ -69044,6 +105314,20 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -69217,13 +105501,13 @@
-
-
-
-
-
+
+
+
+
+
-
+
@@ -69231,7 +105515,7 @@
-
+
@@ -69248,30 +105532,30 @@
-
-
-
-
+
+
+
+
-
-
-
-
-
-
-
+
+
+
+
+
+
+
-
-
-
-
+
+
+
+
-
-
-
+
+
+
@@ -69302,34 +105586,9 @@
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -69365,6 +105624,8 @@
+
+
@@ -69399,104 +105660,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -69520,7 +105683,7 @@
-
+
@@ -69568,15 +105731,15 @@
-
+
-
+
-
+
@@ -69596,14 +105759,6 @@
-
-
-
-
-
-
-
-
@@ -69648,26 +105803,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -69682,14 +105817,6 @@
-
-
-
-
-
-
-
-
@@ -69701,6 +105828,12 @@
+
+
+
+
+
+
@@ -69712,6 +105845,74 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -69720,9 +105921,9 @@
-
-
-
+
+
+
@@ -69739,29 +105940,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -69817,34 +105995,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -69888,6 +106038,11 @@
+
+
+
+
+
@@ -69918,6 +106073,11 @@
+
+
+
+
+
@@ -69934,6 +106094,13 @@
+
+
+
+
+
+
+
@@ -70015,6 +106182,7 @@
+
@@ -70034,15 +106202,15 @@
-
+
-
+
-
+
@@ -70051,7 +106219,7 @@
-
+
@@ -70060,13 +106228,13 @@
-
+
-
+
@@ -70345,17 +106513,6 @@
-
-
-
-
-
-
-
-
-
-
-
@@ -70364,17 +106521,6 @@
-
-
-
-
-
-
-
-
-
-
-
@@ -70390,12 +106536,12 @@
-
-
-
-
-
-
+
+
+
+
+
+
@@ -70458,6 +106604,10 @@
+
+
+
+
@@ -70513,17 +106663,6 @@
-
-
-
-
-
-
-
-
-
-
-
@@ -70824,11 +106963,24 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -70943,365 +107095,302 @@
-
+
-
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
@@ -71312,12 +107401,12 @@
-
+
-
+
-
+
@@ -71325,129 +107414,129 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
@@ -71491,26 +107580,26 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
@@ -71687,50 +107776,50 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
@@ -71771,42 +107860,19 @@
-
+
-
+
-
+
-
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
@@ -71872,8 +107938,9 @@
+
-
+
@@ -71920,6 +107987,7 @@
+
@@ -71970,6 +108038,15 @@
+
+
+
+
+
+
+
+
+
@@ -71983,6 +108060,8 @@
+
+
@@ -72082,38 +108161,1038 @@
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -72123,7 +109202,7 @@
-
+
@@ -72421,15 +109500,23 @@
+
+
+
+
+
+
+
+
-
+
-
+
-
+
@@ -72440,7 +109527,7 @@
-
+
@@ -72495,21 +109582,21 @@
-
+
-
+
-
+
-
+
@@ -72606,7 +109693,7 @@
-
+
@@ -72657,7 +109744,7 @@
-
+
@@ -72873,7 +109960,7 @@
-
+
@@ -72894,7 +109981,7 @@
-
+
@@ -73096,7 +110183,7 @@
-
+
@@ -73166,6 +110253,14 @@
+
+
+
+
+
+
+
+
@@ -73495,7 +110590,7 @@
-
+
@@ -73670,10 +110765,10 @@
-
+
-
+
@@ -73841,7 +110936,7 @@
-
+
@@ -73999,26 +111094,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -74137,16 +111212,16 @@
-
+
-
+
-
+
-
+
@@ -74170,7 +111245,7 @@
-
+
@@ -74217,10 +111292,10 @@
-
+
-
+
@@ -74236,7 +111311,31 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -74248,7 +111347,7 @@
-
+
@@ -74256,7 +111355,15 @@
-
+
+
+
+
+
+
+
+
+
@@ -74277,7 +111384,7 @@
-
+
@@ -74307,10 +111414,10 @@
-
+
-
+
@@ -74412,15 +111519,15 @@
-
+
-
+
-
+
@@ -74434,7 +111541,7 @@
-
+
@@ -74471,7 +111578,7 @@
-
+
@@ -74495,15 +111602,15 @@
-
+
-
+
-
+
@@ -74605,6 +111712,14 @@
+
+
+
+
+
+
+
+
@@ -74638,7 +111753,7 @@
-
+
@@ -74659,12 +111774,12 @@
-
+
-
+
@@ -74681,7 +111796,7 @@
-
+
@@ -74695,59 +111810,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -74793,26 +111855,49 @@
-
-
+
+
-
-
+
+
-
-
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -74997,7 +112082,7 @@
-
+
@@ -75026,7 +112111,7 @@
-
+
@@ -75121,7 +112206,7 @@
-
+
@@ -75162,7 +112247,7 @@
-
+
@@ -75856,14 +112941,6 @@
-
-
-
-
-
-
-
-
@@ -75876,6 +112953,14 @@
+
+
+
+
+
+
+
+
@@ -75911,13 +112996,13 @@
-
+
-
+
-
+
@@ -76028,8 +113113,8 @@
-
-
+
+
@@ -76037,21 +113122,21 @@
-
+
-
+
-
+
-
+
@@ -76096,7 +113181,15 @@
-
+
+
+
+
+
+
+
+
+
@@ -76277,7 +113370,7 @@
-
+
@@ -76319,7 +113412,7 @@
-
+
@@ -76406,7 +113499,7 @@
-
+
@@ -76432,7 +113525,7 @@
-
+
@@ -76491,7 +113584,7 @@
-
+
@@ -76559,7 +113652,7 @@
-
+
@@ -76607,7 +113700,7 @@
-
+
@@ -76616,7 +113709,7 @@
-
+
@@ -76736,7 +113829,7 @@
-
+
@@ -76754,7 +113847,7 @@
-
+
@@ -76793,7 +113886,7 @@
-
+
@@ -76822,7 +113915,7 @@
-
+
@@ -76855,19 +113948,19 @@
-
+
-
+
-
+
-
+
@@ -76906,7 +113999,7 @@
-
+
@@ -76975,7 +114068,7 @@
-
+
@@ -76987,7 +114080,7 @@
-
+
@@ -77079,15 +114172,31 @@
-
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -77213,7 +114322,37 @@
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -77221,7 +114360,7 @@
-
+
@@ -77239,6 +114378,14 @@
+
+
+
+
+
+
+
+
@@ -77259,7 +114406,7 @@
-
+
@@ -77341,6 +114488,14 @@
+
+
+
+
+
+
+
+
@@ -77808,7 +114963,7 @@
-
+
@@ -78677,7 +115832,7 @@
-
+
@@ -78751,7 +115906,7 @@
-
+
@@ -79196,23 +116351,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -79239,7 +116377,7 @@
-
+
@@ -79253,7 +116391,7 @@
-
+
@@ -79281,7 +116419,7 @@
-
+
@@ -79448,7 +116586,7 @@
-
+
@@ -79593,14 +116731,14 @@
-
-
+
+
-
-
+
+
@@ -79794,7 +116932,7 @@
-
+
@@ -79833,27 +116971,27 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
@@ -79899,17 +117037,17 @@
-
+
-
+
-
+
@@ -79917,7 +117055,7 @@
-
+
@@ -79953,7 +117091,7 @@
-
+
@@ -80026,13 +117164,13 @@
-
+
-
+
-
+
@@ -80086,7 +117224,7 @@
-
+
@@ -80098,7 +117236,7 @@
-
+
@@ -80192,10 +117330,10 @@
-
+
-
+
@@ -80224,7 +117362,7 @@
-
+
@@ -80232,7 +117370,7 @@
-
+
@@ -80256,17 +117394,6 @@
-
-
-
-
-
-
-
-
-
-
-
@@ -80428,8 +117555,8 @@
-
-
+
+
@@ -80843,8 +117970,8 @@
-
-
+
+
@@ -80890,7 +118017,7 @@
-
+
@@ -80902,15 +118029,15 @@
-
+
-
+
-
+
@@ -80983,7 +118110,23 @@
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -81159,7 +118302,7 @@
-
+
@@ -81220,6 +118363,17 @@
+
+
+
+
+
+
+
+
+
+
+
@@ -81236,7 +118390,7 @@
-
+
@@ -81258,7 +118412,7 @@
-
+
@@ -81266,7 +118420,7 @@
-
+
@@ -81277,18 +118431,18 @@
-
+
-
+
-
+
-
+
@@ -81296,6 +118450,25 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -81358,7 +118531,7 @@
-
+
@@ -81367,24 +118540,24 @@
-
+
-
-
-
-
+
+
+
+
-
+
-
+
-
+
@@ -81423,7 +118596,7 @@
-
+
@@ -81443,13 +118616,13 @@
-
-
+
+
-
+
@@ -81512,19 +118685,19 @@
-
+
-
+
-
+
@@ -81538,7 +118711,7 @@
-
+
@@ -81578,16 +118751,16 @@
-
+
-
+
-
+
@@ -81718,10 +118891,10 @@
-
+
-
+
@@ -81811,7 +118984,7 @@
-
+
@@ -81823,7 +118996,7 @@
-
+
@@ -81889,7 +119062,7 @@
-
+
@@ -81994,7 +119167,7 @@
-
+
@@ -82017,7 +119190,7 @@
-
+
@@ -82031,30 +119204,30 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
@@ -82121,7 +119294,7 @@
-
+
@@ -82257,33 +119430,33 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
@@ -82467,18 +119640,18 @@
-
+
-
+
-
+
@@ -82486,28 +119659,28 @@
-
+
-
+
-
+
-
+
-
+
@@ -82567,10 +119740,10 @@
-
+
-
+
@@ -82701,7 +119874,7 @@
-
+
@@ -82795,13 +119968,13 @@
-
+
-
+
-
+
@@ -82869,7 +120042,7 @@
-
+
@@ -82888,11 +120061,11 @@
-
+
-
+
@@ -82968,7 +120141,7 @@
-
+
@@ -83118,24 +120291,24 @@
-
+
-
+
-
+
-
+
@@ -83174,11 +120347,11 @@
-
+
-
+
@@ -83513,188 +120686,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -83793,6 +120784,18 @@
+
+
+
+
+
+
+
+
+
+
+
+
@@ -83803,8 +120806,9 @@
+
-
+
@@ -83843,7 +120847,7 @@
-
+
@@ -83864,25 +120868,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -84088,17 +121073,6 @@
-
-
-
-
-
-
-
-
-
-
-
@@ -84182,17 +121156,6 @@
-
-
-
-
-
-
-
-
-
-
-
@@ -84219,7 +121182,6 @@
-
@@ -84322,6 +121284,7 @@
+
@@ -84349,17 +121312,9 @@
-
-
-
-
-
-
-
-
-
+
@@ -84405,7 +121360,7 @@
-
+
@@ -84415,7 +121370,7 @@
-
+
@@ -84427,7 +121382,7 @@
-
+
@@ -84441,7 +121396,7 @@
-
+
@@ -84456,7 +121411,7 @@
-
+
@@ -84476,26 +121431,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -84609,106 +121544,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -84767,77 +121602,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -84876,35 +121640,7 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
@@ -85046,71 +121782,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -85294,12 +121965,12 @@
-
+
-
+
-
+
@@ -85308,12 +121979,12 @@
-
-
-
-
-
-
+
+
+
+
+
+
@@ -85515,14 +122186,14 @@
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
@@ -85570,6 +122241,21 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -85585,6 +122271,18 @@
+
+
+
+
+
+
+
+
+
+
+
+
@@ -85630,7 +122328,7 @@
-
+
@@ -86080,8 +122778,8 @@
-
-
+
+
@@ -86102,8 +122800,8 @@
-
-
+
+
@@ -86162,8 +122860,8 @@
-
-
+
+
@@ -86270,26 +122968,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -86316,34 +122994,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -86356,13 +123006,6 @@
-
-
-
-
-
-
-
@@ -86519,6 +123162,7 @@
+
@@ -86595,15 +123239,6 @@
-
-
-
-
-
-
-
-
-
@@ -86627,92 +123262,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -86737,18 +123286,18 @@
-
+
-
+
-
+
-
+
-
+
@@ -86770,7 +123319,7 @@
-
+
@@ -86865,14 +123414,6 @@
-
-
-
-
-
-
-
-
@@ -86901,6 +123442,11 @@
+
+
+
+
+
@@ -86987,6 +123533,62 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -87068,293 +123670,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -87368,19 +123683,7 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
+
@@ -87489,41 +123792,30 @@
-
-
+
+
-
-
-
+
+
+
-
-
+
+
-
-
-
+
+
+
-
-
-
-
+
+
+
+
-
-
-
-
-
-
-
-
-
-
-
@@ -88232,48 +124524,11 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
@@ -88287,78 +124542,78 @@
-
-
-
-
+
+
+
+
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
-
+
-
-
-
+
+
+
+
+
-
-
-
+
+
+
+
+
+
+
+
-
-
-
+
+
+
-
-
-
-
+
+
+
+
-
-
+
+
-
-
-
-
-
-
+
+
+
+
+
+
-
-
-
-
-
+
+
+
+
+
-
-
-
-
-
+
+
+
+
+
-
-
-
+
+
+
@@ -88535,17 +124790,6 @@
-
-
-
-
-
-
-
-
-
-
-
@@ -88603,6 +124847,27 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -88713,6 +124978,59 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -88721,38 +125039,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -88785,39 +125071,9 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -88905,14 +125161,6 @@
-
-
-
-
-
-
-
-
@@ -88920,78 +125168,15 @@
+
+
+
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -89000,7 +125185,7 @@
-
+
@@ -89045,9 +125230,9 @@
-
+
-
+
@@ -89068,7 +125253,7 @@
-
+
@@ -89126,23 +125311,12 @@
-
+
-
+
-
-
-
-
-
-
-
-
-
-
-
@@ -89278,10 +125452,9 @@
-
-
-
+
+
@@ -89531,49 +125704,8 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
@@ -89582,34 +125714,12 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -89661,16 +125771,8 @@
-
-
-
-
-
-
-
-
-
+
@@ -89797,7 +125899,7 @@
-
+
@@ -89808,13 +125910,13 @@
-
-
-
-
-
-
-
+
+
+
+
+
+
+
@@ -89830,76 +125932,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -89934,22 +125966,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -90269,14 +126285,6 @@
-
-
-
-
-
-
-
-
@@ -90310,32 +126318,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -90362,14 +126344,6 @@
-
-
-
-
-
-
-
-
@@ -90409,7 +126383,7 @@
-
+
@@ -90466,22 +126440,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -90561,22 +126519,174 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -90611,7 +126721,7 @@
-
+
@@ -90651,12 +126761,12 @@
-
+
-
+
@@ -90947,7 +127057,7 @@
-
+
@@ -91039,62 +127149,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -91102,17 +127156,6 @@
-
-
-
-
-
-
-
-
-
-
-
@@ -91159,7 +127202,7 @@
-
+
@@ -91336,7 +127379,7 @@
-
+
@@ -91348,6 +127391,33 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -91360,6 +127430,9 @@
+
+
+
@@ -91400,6 +127473,13 @@
+
+
+
+
+
+
+
@@ -91567,7 +127647,7 @@
-
+
@@ -92000,6 +128080,195 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -93298,7 +129567,7 @@
-
+
@@ -94680,6 +130949,10 @@
+
+
+
+
@@ -94704,6 +130977,7 @@
+
@@ -94787,7 +131061,13 @@
+
+
+
+
+
+
@@ -94833,6 +131113,28 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -95386,6 +131688,10 @@
+
+
+
+
@@ -95480,84 +131786,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -96135,22 +132363,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -96230,45 +132442,45 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
@@ -96282,12 +132494,12 @@
-
+
-
+
@@ -96298,7 +132510,7 @@
-
+
@@ -96312,7 +132524,7 @@
-
+
@@ -96326,7 +132538,7 @@
-
+
@@ -96340,73 +132552,73 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
@@ -96420,7 +132632,7 @@
-
+
@@ -96428,7 +132640,7 @@
-
+
@@ -96453,7 +132665,7 @@
-
+
@@ -96484,7 +132696,7 @@
-
+
@@ -96501,7 +132713,7 @@
-
+
@@ -96518,7 +132730,7 @@
-
+
@@ -96529,7 +132741,7 @@
-
+
@@ -96537,7 +132749,7 @@
-
+
@@ -96545,7 +132757,7 @@
-
+
@@ -96553,7 +132765,7 @@
-
+
@@ -96567,7 +132779,7 @@
-
+
@@ -96575,7 +132787,7 @@
-
+
@@ -96586,7 +132798,7 @@
-
+
@@ -96611,7 +132823,7 @@
-
+
@@ -96688,32 +132900,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -96729,23 +132915,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -96856,17 +133025,6 @@
-
-
-
-
-
-
-
-
-
-
-
@@ -96913,18 +133071,18 @@
-
+
-
+
-
-
+
+
-
+
@@ -97680,6 +133838,104 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -97742,41 +133998,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -97795,7 +134016,7 @@
-
+
@@ -97805,21 +134026,21 @@
-
+
-
+
-
+
-
+
-
+
@@ -97830,7 +134051,7 @@
-
+
@@ -97841,7 +134062,7 @@
-
+
@@ -98763,17 +134984,6 @@
-
-
-
-
-
-
-
-
-
-
-
@@ -98815,13 +135025,13 @@
-
-
-
+
+
+
-
-
+
+
@@ -98896,6 +135106,14 @@
+
+
+
+
+
+
+
+
@@ -99043,14 +135261,6 @@
-
-
-
-
-
-
-
-
@@ -99071,6 +135281,1446 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -101026,7 +138676,7 @@
-
+
@@ -101046,69 +138696,74 @@
-
-
-
+
+
+
+
+
+
+
+
-
-
-
+
+
+
-
-
-
-
+
+
+
+
-
-
-
-
-
+
+
+
+
+
-
-
-
-
-
-
+
+
+
+
+
+
-
-
-
-
-
-
+
+
+
+
+
+
-
-
-
-
-
-
+
+
+
+
+
+
-
-
-
-
-
-
-
-
-
+
-
+
-
+
+
+
+
+
+
+
+
+
@@ -101125,54 +138780,54 @@
-
-
+
+
-
-
-
+
+
+
-
-
+
+
-
-
-
-
-
+
+
+
+
+
-
-
+
+
-
-
-
+
+
+
-
-
-
+
+
+
-
-
-
+
+
+
-
-
-
+
+
+
-
-
-
-
+
+
+
+
@@ -101583,42 +139238,34 @@
-
-
-
+
+
+
-
-
-
+
+
+
-
-
-
+
+
+
-
-
-
+
+
+
-
-
-
+
+
+
-
-
-
-
-
-
-
-
@@ -101640,46 +139287,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -101947,7 +139554,7 @@
-
+
@@ -102327,7 +139934,7 @@
-
+
@@ -102390,7 +139997,7 @@
-
+
@@ -102427,6 +140034,6 @@
diff --git a/android/abi_gki_aarch64_qcom b/android/abi_gki_aarch64_qcom
index 4e124c7051ce..6397060495da 100644
--- a/android/abi_gki_aarch64_qcom
+++ b/android/abi_gki_aarch64_qcom
@@ -53,6 +53,7 @@
__check_object_size
__class_create
class_destroy
+ _cleanup_srcu_struct
clk_disable
clk_enable
clk_fixed_factor_ops
@@ -366,6 +367,7 @@
iomem_resource
iommu_attach_device
iommu_detach_device
+ iommu_dma_enable_best_fit_algo
iommu_domain_alloc
iommu_domain_free
iommu_domain_get_attr
@@ -400,6 +402,7 @@
irq_set_affinity_notifier
irq_set_chip_and_handler_name
irq_set_chip_data
+ irq_set_irqchip_state
irq_set_irq_type
irq_set_irq_wake
irq_to_desc
@@ -907,6 +910,7 @@
strpbrk
strrchr
strsep
+ strstr
__sw_hweight32
__sw_hweight64
__sw_hweight8
@@ -963,6 +967,7 @@
trace_raw_output_prep
trace_seq_printf
try_module_get
+ try_to_del_timer_sync
tty_flip_buffer_push
typec_register_partner
typec_register_port
@@ -1052,9 +1057,6 @@
try_wait_for_completion
vfs_statx
-# required by apr_dlkm.ko
- strstr
-
# required by arm-memlat-mon.ko
perf_event_create_kernel_counter
perf_event_enable
@@ -1147,7 +1149,6 @@
led_trigger_unregister_simple
# required by cam_smmu_api.ko
- iommu_dma_enable_best_fit_algo
iommu_dma_reserve_iova
# required by cam_utils.ko
@@ -1472,6 +1473,7 @@
pci_release_region
pci_request_region
remove_wait_queue
+ vmemdup_user
vm_iomap_memory
wait_woken
woken_wake_function
@@ -1503,7 +1505,7 @@
snd_soc_of_parse_card_name
snd_soc_pm_ops
snd_soc_unregister_card
- soc_find_component
+ soc_find_component_locked
# required by mbhc_dlkm.ko
snd_jack_set_key
@@ -1512,6 +1514,7 @@
# required by minidump_log.ko
__bss_stop
+ irq_stack_ptr
log_buf_addr_get
log_buf_len_get
__per_cpu_end
@@ -1551,7 +1554,6 @@
# required by msm_adreno.ko
bpf_trace_run10
- _cleanup_srcu_struct
__clk_get_name
devfreq_cooling_unregister
device_show_int
@@ -1567,6 +1569,7 @@
getboottime64
get_random_u32
get_task_mm
+ get_task_pid
get_unmapped_area
get_user_pages
idr_for_each
@@ -1588,6 +1591,7 @@
security_mmap_addr
set_page_dirty_lock
sg_alloc_table_from_pages
+ shmem_read_mapping_page_gfp
sysfs_remove_files
__tracepoint_gpu_mem_total
unmapped_area_topdown
@@ -1963,9 +1967,6 @@
crc8
crc8_populate_msb
-# required by pac193x.ko
- try_to_del_timer_sync
-
# required by peripheral-loader.ko
__iowrite32_copy
memblock_overlaps_memory
@@ -1987,6 +1988,8 @@
irq_chip_set_wake_parent
irq_create_fwspec_mapping
irq_domain_free_irqs_top
+ __irq_set_handler
+ irq_set_handler_data
of_irq_domain_map
register_restart_handler
unregister_restart_handler
@@ -2177,6 +2180,12 @@
gen_pool_set_algo
gen_pool_virt_to_phys
+# required by ramdump.ko
+ init_srcu_struct
+ __srcu_read_lock
+ __srcu_read_unlock
+ synchronize_srcu
+
# required by regmap-spmi.ko
spmi_ext_register_read
spmi_ext_register_readl
@@ -2556,7 +2565,6 @@
device_wakeup_disable
extcon_get_edev_name
extcon_get_property
- irq_set_irqchip_state
pm_runtime_barrier
regulator_register_notifier
regulator_unregister_notifier
@@ -2599,6 +2607,10 @@
disable_percpu_irq
enable_percpu_irq
free_percpu_irq
+ irq_stat
+ kstat
+ kstat_irqs_usr
+ nr_irqs
panic_timeout
__request_percpu_irq
@@ -2722,7 +2734,7 @@
# preserved by --additions-only
crc32_le
- get_next_event_cpu
simple_strtoull
+ soc_find_component
vm_map_ram
vm_unmap_ram
diff --git a/arch/arc/Makefile b/arch/arc/Makefile
index df00578c279d..eb43e57fc80f 100644
--- a/arch/arc/Makefile
+++ b/arch/arc/Makefile
@@ -91,14 +91,9 @@ libs-y += arch/arc/lib/ $(LIBGCC)
boot := arch/arc/boot
-#default target for make without any arguments.
-KBUILD_IMAGE := $(boot)/bootpImage
-
-all: bootpImage
-bootpImage: vmlinux
-
-boot_targets += uImage uImage.bin uImage.gz
+boot_targets := uImage uImage.bin uImage.gz uImage.lzma
+PHONY += $(boot_targets)
$(boot_targets): vmlinux
$(Q)$(MAKE) $(build)=$(boot) $(boot)/$@
diff --git a/arch/arc/include/asm/page.h b/arch/arc/include/asm/page.h
index 09ddddf71cc5..a70fef79c405 100644
--- a/arch/arc/include/asm/page.h
+++ b/arch/arc/include/asm/page.h
@@ -13,6 +13,7 @@
#ifndef __ASSEMBLY__
#define clear_page(paddr) memset((paddr), 0, PAGE_SIZE)
+#define copy_user_page(to, from, vaddr, pg) copy_page(to, from)
#define copy_page(to, from) memcpy((to), (from), PAGE_SIZE)
struct vm_area_struct;
diff --git a/arch/arm/boot/dts/picoxcell-pc3x2.dtsi b/arch/arm/boot/dts/picoxcell-pc3x2.dtsi
index a1266cf8776c..8362c6a3bcd3 100644
--- a/arch/arm/boot/dts/picoxcell-pc3x2.dtsi
+++ b/arch/arm/boot/dts/picoxcell-pc3x2.dtsi
@@ -54,18 +54,21 @@
emac: gem@30000 {
compatible = "cadence,gem";
reg = <0x30000 0x10000>;
+ interrupt-parent = <&vic0>;
interrupts = <31>;
};
dmac1: dmac@40000 {
compatible = "snps,dw-dmac";
reg = <0x40000 0x10000>;
+ interrupt-parent = <&vic0>;
interrupts = <25>;
};
dmac2: dmac@50000 {
compatible = "snps,dw-dmac";
reg = <0x50000 0x10000>;
+ interrupt-parent = <&vic0>;
interrupts = <26>;
};
@@ -243,6 +246,7 @@
axi2pico@c0000000 {
compatible = "picochip,axi2pico-pc3x2";
reg = <0xc0000000 0x10000>;
+ interrupt-parent = <&vic0>;
interrupts = <13 14 15 16 17 18 19 20 21>;
};
};
diff --git a/arch/arm/mach-omap2/omap_device.c b/arch/arm/mach-omap2/omap_device.c
index 41c7b905980a..23e8146b9b32 100644
--- a/arch/arm/mach-omap2/omap_device.c
+++ b/arch/arm/mach-omap2/omap_device.c
@@ -239,10 +239,12 @@ static int _omap_device_notifier_call(struct notifier_block *nb,
break;
case BUS_NOTIFY_BIND_DRIVER:
od = to_omap_device(pdev);
- if (od && (od->_state == OMAP_DEVICE_STATE_ENABLED) &&
- pm_runtime_status_suspended(dev)) {
+ if (od) {
od->_driver_status = BUS_NOTIFY_BIND_DRIVER;
- pm_runtime_set_active(dev);
+ if (od->_state == OMAP_DEVICE_STATE_ENABLED &&
+ pm_runtime_status_suspended(dev)) {
+ pm_runtime_set_active(dev);
+ }
}
break;
case BUS_NOTIFY_ADD_DEVICE:
diff --git a/arch/arm/xen/enlighten.c b/arch/arm/xen/enlighten.c
index 07060e5b5864..8aa901e20ca8 100644
--- a/arch/arm/xen/enlighten.c
+++ b/arch/arm/xen/enlighten.c
@@ -405,7 +405,7 @@ static int __init xen_guest_init(void)
}
gnttab_init();
if (!xen_initial_domain())
- xenbus_probe(NULL);
+ xenbus_probe();
/*
* Making sure board specific code will not set up ops for
diff --git a/arch/arm64/configs/gki_defconfig b/arch/arm64/configs/gki_defconfig
index 7446c16b744d..429f829b9d1d 100644
--- a/arch/arm64/configs/gki_defconfig
+++ b/arch/arm64/configs/gki_defconfig
@@ -101,6 +101,7 @@ CONFIG_PACKET=y
CONFIG_UNIX=y
CONFIG_XFRM_USER=y
CONFIG_XFRM_INTERFACE=y
+CONFIG_XFRM_MIGRATE=y
CONFIG_XFRM_STATISTICS=y
CONFIG_NET_KEY=y
CONFIG_XDP_SOCKETS=y
diff --git a/arch/arm64/include/asm/lse.h b/arch/arm64/include/asm/lse.h
index d712ab2804f9..13536c4da2c2 100644
--- a/arch/arm64/include/asm/lse.h
+++ b/arch/arm64/include/asm/lse.h
@@ -22,13 +22,6 @@
#else /* __ASSEMBLER__ */
-#ifdef CONFIG_LTO_CLANG
-#define __LSE_PREAMBLE ".arch_extension lse\n"
-#else
-__asm__(".arch_extension lse");
-#define __LSE_PREAMBLE
-#endif
-
/* Move the ll/sc atomics out-of-line */
#define __LL_SC_INLINE notrace
#define __LL_SC_PREFIX(x) __ll_sc_##x
diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c
index c4ac832ef54c..108e4ac9567f 100644
--- a/arch/arm64/kernel/irq.c
+++ b/arch/arm64/kernel/irq.c
@@ -38,6 +38,7 @@ unsigned long irq_err_count;
DEFINE_PER_CPU(struct nmi_ctx, nmi_contexts);
DEFINE_PER_CPU(unsigned long *, irq_stack_ptr);
+EXPORT_PER_CPU_SYMBOL_GPL(irq_stack_ptr);
int arch_show_interrupts(struct seq_file *p, int prec)
{
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 847b2d80ce87..fe97b2ad82b9 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -619,6 +619,10 @@ static void reset_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
{
u64 pmcr, val;
+ /* No PMU available, PMCR_EL0 may UNDEF... */
+ if (!kvm_arm_support_pmu_v3())
+ return;
+
pmcr = read_sysreg(pmcr_el0);
/*
* Writable bits of PMCR_EL0 (ARMV8_PMU_PMCR_MASK) are reset to UNKNOWN
diff --git a/arch/mips/boot/compressed/decompress.c b/arch/mips/boot/compressed/decompress.c
index 81df9047e110..40218be0b7ce 100644
--- a/arch/mips/boot/compressed/decompress.c
+++ b/arch/mips/boot/compressed/decompress.c
@@ -17,6 +17,7 @@
#include
#include
+#include
/*
* These two variables specify the free mem region
@@ -117,7 +118,7 @@ void decompress_kernel(unsigned long boot_heap_start)
dtb_size = fdt_totalsize((void *)&__appended_dtb);
/* last four bytes is always image size in little endian */
- image_size = le32_to_cpup((void *)&__image_end - 4);
+ image_size = get_unaligned_le32((void *)&__image_end - 4);
/* copy dtb to where the booted kernel will expect it */
memcpy((void *)VMLINUX_LOAD_ADDRESS_ULL + image_size,
diff --git a/arch/mips/kernel/relocate.c b/arch/mips/kernel/relocate.c
index cbf4cc0b0b6c..934caf204078 100644
--- a/arch/mips/kernel/relocate.c
+++ b/arch/mips/kernel/relocate.c
@@ -187,8 +187,14 @@ static int __init relocate_exception_table(long offset)
static inline __init unsigned long rotate_xor(unsigned long hash,
const void *area, size_t size)
{
- size_t i;
- unsigned long *ptr = (unsigned long *)area;
+ const typeof(hash) *ptr = PTR_ALIGN(area, sizeof(hash));
+ size_t diff, i;
+
+ diff = (void *)ptr - area;
+ if (unlikely(size < diff + sizeof(hash)))
+ return hash;
+
+ size = ALIGN_DOWN(size - diff, sizeof(hash));
for (i = 0; i < size / sizeof(hash); i++) {
/* Rotate by odd number of bits and XOR. */
diff --git a/arch/powerpc/include/asm/bitops.h b/arch/powerpc/include/asm/bitops.h
index ff71566dadee..76db1c5000bd 100644
--- a/arch/powerpc/include/asm/bitops.h
+++ b/arch/powerpc/include/asm/bitops.h
@@ -221,15 +221,34 @@ static __inline__ void __clear_bit_unlock(int nr, volatile unsigned long *addr)
*/
static __inline__ int fls(unsigned int x)
{
- return 32 - __builtin_clz(x);
+ int lz;
+
+ if (__builtin_constant_p(x))
+ return x ? 32 - __builtin_clz(x) : 0;
+ asm("cntlzw %0,%1" : "=r" (lz) : "r" (x));
+ return 32 - lz;
}
#include
+/*
+ * 64-bit can do this using one cntlzd (count leading zeroes doubleword)
+ * instruction; for 32-bit we use the generic version, which does two
+ * 32-bit fls calls.
+ */
+#ifdef CONFIG_PPC64
static __inline__ int fls64(__u64 x)
{
- return 64 - __builtin_clzll(x);
+ int lz;
+
+ if (__builtin_constant_p(x))
+ return x ? 64 - __builtin_clzll(x) : 0;
+ asm("cntlzd %0,%1" : "=r" (lz) : "r" (x));
+ return 64 - lz;
}
+#else
+#include
+#endif
#ifdef CONFIG_PPC64
unsigned int __arch_hweight8(unsigned int w);
diff --git a/arch/powerpc/sysdev/mpic_msgr.c b/arch/powerpc/sysdev/mpic_msgr.c
index 280e964e1aa8..497e86cfb12e 100644
--- a/arch/powerpc/sysdev/mpic_msgr.c
+++ b/arch/powerpc/sysdev/mpic_msgr.c
@@ -196,7 +196,7 @@ static int mpic_msgr_probe(struct platform_device *dev)
/* IO map the message register block. */
of_address_to_resource(np, 0, &rsrc);
- msgr_block_addr = ioremap(rsrc.start, resource_size(&rsrc));
+ msgr_block_addr = devm_ioremap(&dev->dev, rsrc.start, resource_size(&rsrc));
if (!msgr_block_addr) {
dev_err(&dev->dev, "Failed to iomap MPIC message registers");
return -EFAULT;
diff --git a/arch/riscv/kernel/time.c b/arch/riscv/kernel/time.c
index 1911c8f6b8a6..15f4ab40e222 100644
--- a/arch/riscv/kernel/time.c
+++ b/arch/riscv/kernel/time.c
@@ -12,6 +12,7 @@
* GNU General Public License for more details.
*/
+#include
#include
#include
#include
@@ -29,5 +30,7 @@ void __init time_init(void)
riscv_timebase = prop;
lpj_fine = riscv_timebase / HZ;
+
+ of_clk_init(NULL);
timer_probe();
}
diff --git a/arch/sh/drivers/dma/Kconfig b/arch/sh/drivers/dma/Kconfig
index d0de378beefe..7d54f284ce10 100644
--- a/arch/sh/drivers/dma/Kconfig
+++ b/arch/sh/drivers/dma/Kconfig
@@ -63,8 +63,7 @@ config PVR2_DMA
config G2_DMA
tristate "G2 Bus DMA support"
- depends on SH_DREAMCAST
- select SH_DMA_API
+ depends on SH_DREAMCAST && SH_DMA_API
help
This enables support for the DMA controller for the Dreamcast's
G2 bus. Drivers that want this will generally enable this on
diff --git a/arch/x86/configs/gki_defconfig b/arch/x86/configs/gki_defconfig
index 92b84c795071..ea5aa466c13e 100644
--- a/arch/x86/configs/gki_defconfig
+++ b/arch/x86/configs/gki_defconfig
@@ -73,6 +73,7 @@ CONFIG_PACKET=y
CONFIG_UNIX=y
CONFIG_XFRM_USER=y
CONFIG_XFRM_INTERFACE=y
+CONFIG_XFRM_MIGRATE=y
CONFIG_XFRM_STATISTICS=y
CONFIG_NET_KEY=y
CONFIG_XDP_SOCKETS=y
diff --git a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
index d9b734d0c8cc..3c6e01520a97 100644
--- a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
+++ b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
@@ -170,7 +170,7 @@ continue_block:
## branch into array
lea jump_table(%rip), bufp
- movzxw (bufp, %rax, 2), len
+ movzwq (bufp, %rax, 2), len
lea crc_array(%rip), bufp
lea (bufp, len, 1), bufp
JMP_NOSPEC bufp
diff --git a/arch/x86/hyperv/mmu.c b/arch/x86/hyperv/mmu.c
index 2f34d5275352..e666f7eaf32d 100644
--- a/arch/x86/hyperv/mmu.c
+++ b/arch/x86/hyperv/mmu.c
@@ -66,11 +66,17 @@ static void hyperv_flush_tlb_others(const struct cpumask *cpus,
if (!hv_hypercall_pg)
goto do_native;
- if (cpumask_empty(cpus))
- return;
-
local_irq_save(flags);
+ /*
+ * Only check the mask _after_ interrupt has been disabled to avoid the
+ * mask changing under our feet.
+ */
+ if (cpumask_empty(cpus)) {
+ local_irq_restore(flags);
+ return;
+ }
+
flush_pcpu = (struct hv_tlb_flush **)
this_cpu_ptr(hyperv_pcpu_input_arg);
diff --git a/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c b/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c
index 12083f200e09..f406e3b85bdb 100644
--- a/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c
+++ b/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c
@@ -533,85 +533,70 @@ static void rdtgroup_remove(struct rdtgroup *rdtgrp)
kfree(rdtgrp);
}
-struct task_move_callback {
- struct callback_head work;
- struct rdtgroup *rdtgrp;
-};
-
-static void move_myself(struct callback_head *head)
+static void _update_task_closid_rmid(void *task)
{
- struct task_move_callback *callback;
- struct rdtgroup *rdtgrp;
-
- callback = container_of(head, struct task_move_callback, work);
- rdtgrp = callback->rdtgrp;
-
/*
- * If resource group was deleted before this task work callback
- * was invoked, then assign the task to root group and free the
- * resource group.
+ * If the task is still current on this CPU, update PQR_ASSOC MSR.
+ * Otherwise, the MSR is updated when the task is scheduled in.
*/
- if (atomic_dec_and_test(&rdtgrp->waitcount) &&
- (rdtgrp->flags & RDT_DELETED)) {
- current->closid = 0;
- current->rmid = 0;
- rdtgroup_remove(rdtgrp);
- }
+ if (task == current)
+ intel_rdt_sched_in();
+}
- preempt_disable();
- /* update PQR_ASSOC MSR to make resource group go into effect */
- intel_rdt_sched_in();
- preempt_enable();
-
- kfree(callback);
+static void update_task_closid_rmid(struct task_struct *t)
+{
+ if (IS_ENABLED(CONFIG_SMP) && task_curr(t))
+ smp_call_function_single(task_cpu(t), _update_task_closid_rmid, t, 1);
+ else
+ _update_task_closid_rmid(t);
}
static int __rdtgroup_move_task(struct task_struct *tsk,
struct rdtgroup *rdtgrp)
{
- struct task_move_callback *callback;
- int ret;
-
- callback = kzalloc(sizeof(*callback), GFP_KERNEL);
- if (!callback)
- return -ENOMEM;
- callback->work.func = move_myself;
- callback->rdtgrp = rdtgrp;
+ /* If the task is already in rdtgrp, no need to move the task. */
+ if ((rdtgrp->type == RDTCTRL_GROUP && tsk->closid == rdtgrp->closid &&
+ tsk->rmid == rdtgrp->mon.rmid) ||
+ (rdtgrp->type == RDTMON_GROUP && tsk->rmid == rdtgrp->mon.rmid &&
+ tsk->closid == rdtgrp->mon.parent->closid))
+ return 0;
/*
- * Take a refcount, so rdtgrp cannot be freed before the
- * callback has been invoked.
+ * Set the task's closid/rmid before the PQR_ASSOC MSR can be
+ * updated by them.
+ *
+ * For ctrl_mon groups, move both closid and rmid.
+ * For monitor groups, can move the tasks only from
+ * their parent CTRL group.
*/
- atomic_inc(&rdtgrp->waitcount);
- ret = task_work_add(tsk, &callback->work, true);
- if (ret) {
- /*
- * Task is exiting. Drop the refcount and free the callback.
- * No need to check the refcount as the group cannot be
- * deleted before the write function unlocks rdtgroup_mutex.
- */
- atomic_dec(&rdtgrp->waitcount);
- kfree(callback);
- rdt_last_cmd_puts("task exited\n");
- } else {
- /*
- * For ctrl_mon groups move both closid and rmid.
- * For monitor groups, can move the tasks only from
- * their parent CTRL group.
- */
- if (rdtgrp->type == RDTCTRL_GROUP) {
- tsk->closid = rdtgrp->closid;
+
+ if (rdtgrp->type == RDTCTRL_GROUP) {
+ tsk->closid = rdtgrp->closid;
+ tsk->rmid = rdtgrp->mon.rmid;
+ } else if (rdtgrp->type == RDTMON_GROUP) {
+ if (rdtgrp->mon.parent->closid == tsk->closid) {
tsk->rmid = rdtgrp->mon.rmid;
- } else if (rdtgrp->type == RDTMON_GROUP) {
- if (rdtgrp->mon.parent->closid == tsk->closid) {
- tsk->rmid = rdtgrp->mon.rmid;
- } else {
- rdt_last_cmd_puts("Can't move task to different control group\n");
- ret = -EINVAL;
- }
+ } else {
+ rdt_last_cmd_puts("Can't move task to different control group\n");
+ return -EINVAL;
}
}
- return ret;
+
+ /*
+ * Ensure the task's closid and rmid are written before determining if
+ * the task is current that will decide if it will be interrupted.
+ */
+ barrier();
+
+ /*
+ * By now, the task's closid and rmid are set. If the task is current
+ * on a CPU, the PQR_ASSOC MSR needs to be updated to make the resource
+ * group go into effect. If the task is not current, the MSR will be
+ * updated when the task is scheduled in.
+ */
+ update_task_closid_rmid(tsk);
+
+ return 0;
}
/**
diff --git a/arch/x86/kernel/cpu/mtrr/generic.c b/arch/x86/kernel/cpu/mtrr/generic.c
index e12ee86906c6..9436f3452049 100644
--- a/arch/x86/kernel/cpu/mtrr/generic.c
+++ b/arch/x86/kernel/cpu/mtrr/generic.c
@@ -166,9 +166,6 @@ static u8 mtrr_type_lookup_variable(u64 start, u64 end, u64 *partial_end,
*repeat = 0;
*uniform = 1;
- /* Make end inclusive instead of exclusive */
- end--;
-
prev_match = MTRR_TYPE_INVALID;
for (i = 0; i < num_var_ranges; ++i) {
unsigned short start_state, end_state, inclusive;
@@ -260,6 +257,9 @@ u8 mtrr_type_lookup(u64 start, u64 end, u8 *uniform)
int repeat;
u64 partial_end;
+ /* Make end inclusive instead of exclusive */
+ end--;
+
if (!mtrr_state_set)
return MTRR_TYPE_INVALID;
diff --git a/arch/x86/kvm/cpuid.h b/arch/x86/kvm/cpuid.h
index d78a61408243..7dec43b2c420 100644
--- a/arch/x86/kvm/cpuid.h
+++ b/arch/x86/kvm/cpuid.h
@@ -154,6 +154,20 @@ static inline int guest_cpuid_stepping(struct kvm_vcpu *vcpu)
return x86_stepping(best->eax);
}
+static inline bool guest_has_spec_ctrl_msr(struct kvm_vcpu *vcpu)
+{
+ return (guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL) ||
+ guest_cpuid_has(vcpu, X86_FEATURE_AMD_STIBP) ||
+ guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBRS) ||
+ guest_cpuid_has(vcpu, X86_FEATURE_AMD_SSBD));
+}
+
+static inline bool guest_has_pred_cmd_msr(struct kvm_vcpu *vcpu)
+{
+ return (guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL) ||
+ guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBPB));
+}
+
static inline bool supports_cpuid_fault(struct kvm_vcpu *vcpu)
{
return vcpu->arch.msr_platform_info & MSR_PLATFORM_INFO_CPUID_FAULT;
diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h
index 0b62c817f63f..05a02b8ace6b 100644
--- a/arch/x86/kvm/mmu.h
+++ b/arch/x86/kvm/mmu.h
@@ -53,7 +53,7 @@ static inline u64 rsvd_bits(int s, int e)
if (e < s)
return 0;
- return ((1ULL << (e - s + 1)) - 1) << s;
+ return ((2ULL << (e - s)) - 1) << s;
}
void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask, u64 mmio_value);
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index a0c3d1b4b295..d2dc734f5bd0 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -4209,8 +4209,7 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
break;
case MSR_IA32_SPEC_CTRL:
if (!msr_info->host_initiated &&
- !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBRS) &&
- !guest_cpuid_has(vcpu, X86_FEATURE_AMD_SSBD))
+ !guest_has_spec_ctrl_msr(vcpu))
return 1;
msr_info->data = svm->spec_ctrl;
@@ -4312,8 +4311,7 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
break;
case MSR_IA32_SPEC_CTRL:
if (!msr->host_initiated &&
- !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBRS) &&
- !guest_cpuid_has(vcpu, X86_FEATURE_AMD_SSBD))
+ !guest_has_spec_ctrl_msr(vcpu))
return 1;
/* The STIBP bit doesn't fault even if it's not advertised */
@@ -4340,12 +4338,11 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
break;
case MSR_IA32_PRED_CMD:
if (!msr->host_initiated &&
- !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBPB))
+ !guest_has_pred_cmd_msr(vcpu))
return 1;
if (data & ~PRED_CMD_IBPB)
return 1;
-
if (!data)
break;
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index d6bcbce6c15c..77b9ed5223f3 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -4066,7 +4066,7 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
return kvm_get_msr_common(vcpu, msr_info);
case MSR_IA32_SPEC_CTRL:
if (!msr_info->host_initiated &&
- !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL))
+ !guest_has_spec_ctrl_msr(vcpu))
return 1;
msr_info->data = to_vmx(vcpu)->spec_ctrl;
@@ -4180,7 +4180,7 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
break;
case MSR_IA32_SPEC_CTRL:
if (!msr_info->host_initiated &&
- !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL))
+ !guest_has_spec_ctrl_msr(vcpu))
return 1;
/* The STIBP bit doesn't fault even if it's not advertised */
@@ -4210,7 +4210,7 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
break;
case MSR_IA32_PRED_CMD:
if (!msr_info->host_initiated &&
- !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL))
+ !guest_has_pred_cmd_msr(vcpu))
return 1;
if (data & ~PRED_CMD_IBPB)
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index bf52106ab9c4..c0e9c00402ac 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -838,6 +838,8 @@ int pud_free_pmd_page(pud_t *pud, unsigned long addr)
}
free_page((unsigned long)pmd_sv);
+
+ pgtable_pmd_page_dtor(virt_to_page(pmd));
free_page((unsigned long)pmd);
return 1;
diff --git a/arch/x86/xen/efi.c b/arch/x86/xen/efi.c
index 66bcdeeee639..90d2e4ce7064 100644
--- a/arch/x86/xen/efi.c
+++ b/arch/x86/xen/efi.c
@@ -172,7 +172,7 @@ static enum efi_secureboot_mode xen_efi_get_secureboot(void)
return efi_secureboot_mode_unknown;
}
-void __init xen_efi_init(void)
+void __init xen_efi_init(struct boot_params *boot_params)
{
efi_system_table_t *efi_systab_xen;
@@ -181,12 +181,12 @@ void __init xen_efi_init(void)
if (efi_systab_xen == NULL)
return;
- strncpy((char *)&boot_params.efi_info.efi_loader_signature, "Xen",
- sizeof(boot_params.efi_info.efi_loader_signature));
- boot_params.efi_info.efi_systab = (__u32)__pa(efi_systab_xen);
- boot_params.efi_info.efi_systab_hi = (__u32)(__pa(efi_systab_xen) >> 32);
+ strncpy((char *)&boot_params->efi_info.efi_loader_signature, "Xen",
+ sizeof(boot_params->efi_info.efi_loader_signature));
+ boot_params->efi_info.efi_systab = (__u32)__pa(efi_systab_xen);
+ boot_params->efi_info.efi_systab_hi = (__u32)(__pa(efi_systab_xen) >> 32);
- boot_params.secure_boot = xen_efi_get_secureboot();
+ boot_params->secure_boot = xen_efi_get_secureboot();
set_bit(EFI_BOOT, &efi.flags);
set_bit(EFI_PARAVIRT, &efi.flags);
diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c
index 9f8995cd28f6..1c3e9185934c 100644
--- a/arch/x86/xen/enlighten_pv.c
+++ b/arch/x86/xen/enlighten_pv.c
@@ -1409,7 +1409,7 @@ asmlinkage __visible void __init xen_start_kernel(void)
/* We need this for printk timestamps */
xen_setup_runstate_info(0);
- xen_efi_init();
+ xen_efi_init(&boot_params);
/* Start the world */
#ifdef CONFIG_X86_32
diff --git a/arch/x86/xen/enlighten_pvh.c b/arch/x86/xen/enlighten_pvh.c
index dab07827d25e..f04d22bcf08d 100644
--- a/arch/x86/xen/enlighten_pvh.c
+++ b/arch/x86/xen/enlighten_pvh.c
@@ -14,6 +14,8 @@
#include
#include
+#include "xen-ops.h"
+
/*
* PVH variables.
*
@@ -79,6 +81,8 @@ static void __init init_pvh_bootparams(void)
pvh_bootparams.hdr.type_of_loader = (9 << 4) | 0; /* Xen loader */
x86_init.acpi.get_root_pointer = pvh_get_root_pointer;
+
+ xen_efi_init(&pvh_bootparams);
}
/*
diff --git a/arch/x86/xen/xen-ops.h b/arch/x86/xen/xen-ops.h
index 0e60bd918695..2f111f47ba98 100644
--- a/arch/x86/xen/xen-ops.h
+++ b/arch/x86/xen/xen-ops.h
@@ -122,9 +122,9 @@ static inline void __init xen_init_vga(const struct dom0_vga_console_info *info,
void __init xen_init_apic(void);
#ifdef CONFIG_XEN_EFI
-extern void xen_efi_init(void);
+extern void xen_efi_init(struct boot_params *boot_params);
#else
-static inline void __init xen_efi_init(void)
+static inline void __init xen_efi_init(struct boot_params *boot_params)
{
}
#endif
diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c
index ceb72a70b32a..4edec850ae1b 100644
--- a/block/bfq-iosched.c
+++ b/block/bfq-iosched.c
@@ -5276,13 +5276,13 @@ static unsigned int bfq_update_depths(struct bfq_data *bfqd,
* limit 'something'.
*/
/* no more than 50% of tags for async I/O */
- bfqd->word_depths[0][0] = max((1U << bt->sb.shift) >> 1, 1U);
+ bfqd->word_depths[0][0] = max(bt->sb.depth >> 1, 1U);
/*
* no more than 75% of tags for sync writes (25% extra tags
* w.r.t. async I/O, to prevent async I/O from starving sync
* writes)
*/
- bfqd->word_depths[0][1] = max(((1U << bt->sb.shift) * 3) >> 2, 1U);
+ bfqd->word_depths[0][1] = max((bt->sb.depth * 3) >> 2, 1U);
/*
* In-word depths in case some bfq_queue is being weight-
@@ -5292,9 +5292,9 @@ static unsigned int bfq_update_depths(struct bfq_data *bfqd,
* shortage.
*/
/* no more than ~18% of tags for async I/O */
- bfqd->word_depths[1][0] = max(((1U << bt->sb.shift) * 3) >> 4, 1U);
+ bfqd->word_depths[1][0] = max((bt->sb.depth * 3) >> 4, 1U);
/* no more than ~37% of tags for sync writes (~20% extra tags) */
- bfqd->word_depths[1][1] = max(((1U << bt->sb.shift) * 6) >> 4, 1U);
+ bfqd->word_depths[1][1] = max((bt->sb.depth * 6) >> 4, 1U);
for (i = 0; i < 2; i++)
for (j = 0; j < 2; j++)
diff --git a/block/genhd.c b/block/genhd.c
index 2b2a936cf848..2b536ab570ac 100644
--- a/block/genhd.c
+++ b/block/genhd.c
@@ -208,14 +208,17 @@ struct hd_struct *disk_part_iter_next(struct disk_part_iter *piter)
part = rcu_dereference(ptbl->part[piter->idx]);
if (!part)
continue;
+ get_device(part_to_dev(part));
+ piter->part = part;
if (!part_nr_sects_read(part) &&
!(piter->flags & DISK_PITER_INCL_EMPTY) &&
!(piter->flags & DISK_PITER_INCL_EMPTY_PART0 &&
- piter->idx == 0))
+ piter->idx == 0)) {
+ put_device(part_to_dev(part));
+ piter->part = NULL;
continue;
+ }
- get_device(part_to_dev(part));
- piter->part = part;
piter->idx += inc;
break;
}
diff --git a/build.config.aarch64 b/build.config.aarch64
index 569b9747661d..3b9ccac718a9 100644
--- a/build.config.aarch64
+++ b/build.config.aarch64
@@ -1,5 +1,6 @@
ARCH=arm64
+LLVM_IAS=1
CROSS_COMPILE=aarch64-linux-gnu-
CROSS_COMPILE_COMPAT=arm-linux-gnueabi-
LINUX_GCC_CROSS_COMPILE_PREBUILTS_BIN=prebuilts/gas/linux-x86
diff --git a/build.config.common b/build.config.common
index c9467028ea7f..eecf0385d0a3 100644
--- a/build.config.common
+++ b/build.config.common
@@ -3,7 +3,7 @@ KMI_GENERATION=0
LLVM=1
DEPMOD=depmod
-CLANG_PREBUILT_BIN=prebuilts-master/clang/host/linux-x86/clang-r399163b/bin
+CLANG_PREBUILT_BIN=prebuilts-master/clang/host/linux-x86/clang-r407598/bin
BUILDTOOLS_PREBUILT_BIN=build/build-tools/path/linux-x86
EXTRA_CMDS=''
diff --git a/crypto/ecdh.c b/crypto/ecdh.c
index a6e1a5d43fa7..34605509b41a 100644
--- a/crypto/ecdh.c
+++ b/crypto/ecdh.c
@@ -43,7 +43,8 @@ static int ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,
struct ecdh params;
unsigned int ndigits;
- if (crypto_ecdh_decode_key(buf, len, ¶ms) < 0)
+ if (crypto_ecdh_decode_key(buf, len, ¶ms) < 0 ||
+ params.key_size > sizeof(ctx->private_key))
return -EINVAL;
ndigits = ecdh_supported_curve(params.curve_id);
diff --git a/drivers/acpi/internal.h b/drivers/acpi/internal.h
index f59d0b9e2683..6def196cc23c 100644
--- a/drivers/acpi/internal.h
+++ b/drivers/acpi/internal.h
@@ -98,7 +98,7 @@ void acpi_scan_table_handler(u32 event, void *table, void *context);
extern struct list_head acpi_bus_id_list;
struct acpi_device_bus_id {
- char bus_id[15];
+ const char *bus_id;
unsigned int instance_no;
struct list_head node;
};
diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c
index 1cfa3ac1d91f..d614cb72041e 100644
--- a/drivers/acpi/scan.c
+++ b/drivers/acpi/scan.c
@@ -486,6 +486,7 @@ static void acpi_device_del(struct acpi_device *device)
acpi_device_bus_id->instance_no--;
else {
list_del(&acpi_device_bus_id->node);
+ kfree_const(acpi_device_bus_id->bus_id);
kfree(acpi_device_bus_id);
}
break;
@@ -585,6 +586,8 @@ static int acpi_get_device_data(acpi_handle handle, struct acpi_device **device,
if (!device)
return -EINVAL;
+ *device = NULL;
+
status = acpi_get_data_full(handle, acpi_scan_drop_device,
(void **)device, callback);
if (ACPI_FAILURE(status) || !*device) {
@@ -674,7 +677,14 @@ int acpi_device_add(struct acpi_device *device,
}
if (!found) {
acpi_device_bus_id = new_bus_id;
- strcpy(acpi_device_bus_id->bus_id, acpi_device_hid(device));
+ acpi_device_bus_id->bus_id =
+ kstrdup_const(acpi_device_hid(device), GFP_KERNEL);
+ if (!acpi_device_bus_id->bus_id) {
+ pr_err(PREFIX "Memory allocation error for bus id\n");
+ result = -ENOMEM;
+ goto err_free_new_bus_id;
+ }
+
acpi_device_bus_id->instance_no = 0;
list_add_tail(&acpi_device_bus_id->node, &acpi_bus_id_list);
}
@@ -709,6 +719,11 @@ int acpi_device_add(struct acpi_device *device,
if (device->parent)
list_del(&device->node);
list_del(&device->wakeup_list);
+
+ err_free_new_bus_id:
+ if (!found)
+ kfree(new_bus_id);
+
mutex_unlock(&acpi_device_lock);
err_detach:
diff --git a/drivers/atm/idt77252.c b/drivers/atm/idt77252.c
index 6e737142ceaa..3e00ab8a8890 100644
--- a/drivers/atm/idt77252.c
+++ b/drivers/atm/idt77252.c
@@ -3607,7 +3607,7 @@ static int idt77252_init_one(struct pci_dev *pcidev,
if ((err = dma_set_mask_and_coherent(&pcidev->dev, DMA_BIT_MASK(32)))) {
printk("idt77252: can't enable DMA for PCI device at %s\n", pci_name(pcidev));
- return err;
+ goto err_out_disable_pdev;
}
card = kzalloc(sizeof(struct idt77252_dev), GFP_KERNEL);
diff --git a/drivers/base/core.c b/drivers/base/core.c
index 8746d7d7329b..63e1e8f41426 100644
--- a/drivers/base/core.c
+++ b/drivers/base/core.c
@@ -100,6 +100,16 @@ void device_links_read_unlock(int not_used)
}
#endif /* !CONFIG_SRCU */
+static bool device_is_ancestor(struct device *dev, struct device *target)
+{
+ while (target->parent) {
+ target = target->parent;
+ if (dev == target)
+ return true;
+ }
+ return false;
+}
+
/**
* device_is_dependent - Check if one device depends on another one
* @dev: Device to check dependencies for.
@@ -113,7 +123,12 @@ static int device_is_dependent(struct device *dev, void *target)
struct device_link *link;
int ret;
- if (dev == target)
+ /*
+ * The "ancestors" check is needed to catch the case when the target
+ * device has not been completely initialized yet and it is still
+ * missing from the list of children of its parent device.
+ */
+ if (dev == target || device_is_ancestor(dev, target))
return 1;
ret = device_for_each_child(dev, target, device_is_dependent);
@@ -3818,7 +3833,7 @@ void set_primary_fwnode(struct device *dev, struct fwnode_handle *fwnode)
if (fwnode_is_primary(fn)) {
dev->fwnode = fn->secondary;
if (!(parent && fn == parent->fwnode))
- fn->secondary = ERR_PTR(-ENODEV);
+ fn->secondary = NULL;
} else {
dev->fwnode = NULL;
}
diff --git a/drivers/base/regmap/regmap-debugfs.c b/drivers/base/regmap/regmap-debugfs.c
index 6c859c6a2a79..51f648939612 100644
--- a/drivers/base/regmap/regmap-debugfs.c
+++ b/drivers/base/regmap/regmap-debugfs.c
@@ -642,8 +642,12 @@ void regmap_debugfs_init(struct regmap *map, const char *name)
devname = dev_name(map->dev);
if (name) {
- map->debugfs_name = kasprintf(GFP_KERNEL, "%s-%s",
+ if (!map->debugfs_name) {
+ map->debugfs_name = kasprintf(GFP_KERNEL, "%s-%s",
devname, name);
+ if (!map->debugfs_name)
+ return;
+ }
name = map->debugfs_name;
} else {
name = devname;
@@ -651,9 +655,10 @@ void regmap_debugfs_init(struct regmap *map, const char *name)
if (!strcmp(name, "dummy")) {
kfree(map->debugfs_name);
-
map->debugfs_name = kasprintf(GFP_KERNEL, "dummy%d",
dummy_index);
+ if (!map->debugfs_name)
+ return;
name = map->debugfs_name;
dummy_index++;
}
diff --git a/drivers/block/Kconfig b/drivers/block/Kconfig
index a2c3c895d111..01a9bb865302 100644
--- a/drivers/block/Kconfig
+++ b/drivers/block/Kconfig
@@ -474,6 +474,7 @@ config BLK_DEV_RBD
config BLK_DEV_RSXX
tristate "IBM Flash Adapter 900GB Full Height PCIe Device Driver"
depends on PCI
+ select CRC32
help
Device driver for IBM's high speed PCIe SSD
storage device: Flash Adapter 900GB Full Height.
diff --git a/drivers/block/null_blk_zoned.c b/drivers/block/null_blk_zoned.c
index d1725ac636c0..079ed33fd806 100644
--- a/drivers/block/null_blk_zoned.c
+++ b/drivers/block/null_blk_zoned.c
@@ -1,9 +1,9 @@
// SPDX-License-Identifier: GPL-2.0
#include
+#include
#include "null_blk.h"
-/* zone_size in MBs to sectors. */
-#define ZONE_SIZE_SHIFT 11
+#define MB_TO_SECTS(mb) (((sector_t)mb * SZ_1M) >> SECTOR_SHIFT)
static inline unsigned int null_zone_no(struct nullb_device *dev, sector_t sect)
{
@@ -12,7 +12,7 @@ static inline unsigned int null_zone_no(struct nullb_device *dev, sector_t sect)
int null_zone_init(struct nullb_device *dev)
{
- sector_t dev_size = (sector_t)dev->size * 1024 * 1024;
+ sector_t dev_capacity_sects;
sector_t sector = 0;
unsigned int i;
@@ -25,9 +25,12 @@ int null_zone_init(struct nullb_device *dev)
return -EINVAL;
}
- dev->zone_size_sects = dev->zone_size << ZONE_SIZE_SHIFT;
- dev->nr_zones = dev_size >>
- (SECTOR_SHIFT + ilog2(dev->zone_size_sects));
+ dev_capacity_sects = MB_TO_SECTS(dev->size);
+ dev->zone_size_sects = MB_TO_SECTS(dev->zone_size);
+ dev->nr_zones = dev_capacity_sects >> ilog2(dev->zone_size_sects);
+ if (dev_capacity_sects & (dev->zone_size_sects - 1))
+ dev->nr_zones++;
+
dev->zones = kvmalloc_array(dev->nr_zones, sizeof(struct blk_zone),
GFP_KERNEL | __GFP_ZERO);
if (!dev->zones)
@@ -37,7 +40,10 @@ int null_zone_init(struct nullb_device *dev)
struct blk_zone *zone = &dev->zones[i];
zone->start = zone->wp = sector;
- zone->len = dev->zone_size_sects;
+ if (zone->start + dev->zone_size_sects > dev_capacity_sects)
+ zone->len = dev_capacity_sects - zone->start;
+ else
+ zone->len = dev->zone_size_sects;
zone->type = BLK_ZONE_TYPE_SEQWRITE_REQ;
zone->cond = BLK_ZONE_COND_EMPTY;
diff --git a/drivers/clk/tegra/clk-tegra30.c b/drivers/clk/tegra/clk-tegra30.c
index e0aaecd98fbf..678019f86bc7 100644
--- a/drivers/clk/tegra/clk-tegra30.c
+++ b/drivers/clk/tegra/clk-tegra30.c
@@ -1274,6 +1274,8 @@ static struct tegra_clk_init_table init_table[] __initdata = {
{ TEGRA30_CLK_I2S3_SYNC, TEGRA30_CLK_CLK_MAX, 24000000, 0 },
{ TEGRA30_CLK_I2S4_SYNC, TEGRA30_CLK_CLK_MAX, 24000000, 0 },
{ TEGRA30_CLK_VIMCLK_SYNC, TEGRA30_CLK_CLK_MAX, 24000000, 0 },
+ { TEGRA30_CLK_HDA, TEGRA30_CLK_PLL_P, 102000000, 0 },
+ { TEGRA30_CLK_HDA2CODEC_2X, TEGRA30_CLK_PLL_P, 48000000, 0 },
/* must be the last entry */
{ TEGRA30_CLK_CLK_MAX, TEGRA30_CLK_CLK_MAX, 0, 0 },
};
diff --git a/drivers/cpufreq/powernow-k8.c b/drivers/cpufreq/powernow-k8.c
index fb77b39a4ce3..818f92798fb9 100644
--- a/drivers/cpufreq/powernow-k8.c
+++ b/drivers/cpufreq/powernow-k8.c
@@ -881,9 +881,9 @@ static int get_transition_latency(struct powernow_k8_data *data)
/* Take a frequency, and issue the fid/vid transition command */
static int transition_frequency_fidvid(struct powernow_k8_data *data,
- unsigned int index)
+ unsigned int index,
+ struct cpufreq_policy *policy)
{
- struct cpufreq_policy *policy;
u32 fid = 0;
u32 vid = 0;
int res;
@@ -915,9 +915,6 @@ static int transition_frequency_fidvid(struct powernow_k8_data *data,
freqs.old = find_khz_freq_from_fid(data->currfid);
freqs.new = find_khz_freq_from_fid(fid);
- policy = cpufreq_cpu_get(smp_processor_id());
- cpufreq_cpu_put(policy);
-
cpufreq_freq_transition_begin(policy, &freqs);
res = transition_fid_vid(data, fid, vid);
cpufreq_freq_transition_end(policy, &freqs, res);
@@ -972,7 +969,7 @@ static long powernowk8_target_fn(void *arg)
powernow_k8_acpi_pst_values(data, newstate);
- ret = transition_frequency_fidvid(data, newstate);
+ ret = transition_frequency_fidvid(data, newstate, pol);
if (ret) {
pr_err("transition frequency failed\n");
diff --git a/drivers/crypto/chelsio/chtls/chtls_cm.c b/drivers/crypto/chelsio/chtls/chtls_cm.c
index 35d6bd1b3099..fd3092a4378e 100644
--- a/drivers/crypto/chelsio/chtls/chtls_cm.c
+++ b/drivers/crypto/chelsio/chtls/chtls_cm.c
@@ -581,7 +581,7 @@ static void chtls_reset_synq(struct listen_ctx *listen_ctx)
while (!skb_queue_empty(&listen_ctx->synq)) {
struct chtls_sock *csk =
- container_of((struct synq *)__skb_dequeue
+ container_of((struct synq *)skb_peek
(&listen_ctx->synq), struct chtls_sock, synq);
struct sock *child = csk->sk;
@@ -1024,6 +1024,7 @@ static struct sock *chtls_recv_sock(struct sock *lsk,
const struct cpl_pass_accept_req *req,
struct chtls_dev *cdev)
{
+ struct adapter *adap = pci_get_drvdata(cdev->pdev);
const struct tcphdr *tcph;
struct inet_sock *newinet;
const struct iphdr *iph;
@@ -1033,9 +1034,10 @@ static struct sock *chtls_recv_sock(struct sock *lsk,
struct neighbour *n;
struct tcp_sock *tp;
struct sock *newsk;
+ bool found = false;
u16 port_id;
int rxq_idx;
- int step;
+ int step, i;
iph = (const struct iphdr *)network_hdr;
newsk = tcp_create_openreq_child(lsk, oreq, cdev->askb);
@@ -1048,7 +1050,7 @@ static struct sock *chtls_recv_sock(struct sock *lsk,
tcph = (struct tcphdr *)(iph + 1);
n = dst_neigh_lookup(dst, &iph->saddr);
- if (!n)
+ if (!n || !n->dev)
goto free_sk;
ndev = n->dev;
@@ -1057,6 +1059,13 @@ static struct sock *chtls_recv_sock(struct sock *lsk,
if (is_vlan_dev(ndev))
ndev = vlan_dev_real_dev(ndev);
+ for_each_port(adap, i)
+ if (cdev->ports[i] == ndev)
+ found = true;
+
+ if (!found)
+ goto free_dst;
+
port_id = cxgb4_port_idx(ndev);
csk = chtls_sock_create(cdev);
@@ -1108,6 +1117,7 @@ static struct sock *chtls_recv_sock(struct sock *lsk,
free_csk:
chtls_sock_release(&csk->kref);
free_dst:
+ neigh_release(n);
dst_release(dst);
free_sk:
inet_csk_prepare_forced_close(newsk);
@@ -1422,6 +1432,11 @@ static int chtls_pass_establish(struct chtls_dev *cdev, struct sk_buff *skb)
sk_wake_async(sk, 0, POLL_OUT);
data = lookup_stid(cdev->tids, stid);
+ if (!data) {
+ /* listening server close */
+ kfree_skb(skb);
+ goto unlock;
+ }
lsk = ((struct listen_ctx *)data)->lsk;
bh_lock_sock(lsk);
@@ -1807,39 +1822,6 @@ static void send_defer_abort_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
kfree_skb(skb);
}
-static void send_abort_rpl(struct sock *sk, struct sk_buff *skb,
- struct chtls_dev *cdev, int status, int queue)
-{
- struct cpl_abort_req_rss *req = cplhdr(skb);
- struct sk_buff *reply_skb;
- struct chtls_sock *csk;
-
- csk = rcu_dereference_sk_user_data(sk);
-
- reply_skb = alloc_skb(sizeof(struct cpl_abort_rpl),
- GFP_KERNEL);
-
- if (!reply_skb) {
- req->status = (queue << 1);
- send_defer_abort_rpl(cdev, skb);
- return;
- }
-
- set_abort_rpl_wr(reply_skb, GET_TID(req), status);
- kfree_skb(skb);
-
- set_wr_txq(reply_skb, CPL_PRIORITY_DATA, queue);
- if (csk_conn_inline(csk)) {
- struct l2t_entry *e = csk->l2t_entry;
-
- if (e && sk->sk_state != TCP_SYN_RECV) {
- cxgb4_l2t_send(csk->egress_dev, reply_skb, e);
- return;
- }
- }
- cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
-}
-
/*
* Add an skb to the deferred skb queue for processing from process context.
*/
@@ -1902,9 +1884,9 @@ static void bl_abort_syn_rcv(struct sock *lsk, struct sk_buff *skb)
queue = csk->txq_idx;
skb->sk = NULL;
+ chtls_send_abort_rpl(child, skb, BLOG_SKB_CB(skb)->cdev,
+ CPL_ABORT_NO_RST, queue);
do_abort_syn_rcv(child, lsk);
- send_abort_rpl(child, skb, BLOG_SKB_CB(skb)->cdev,
- CPL_ABORT_NO_RST, queue);
}
static int abort_syn_rcv(struct sock *sk, struct sk_buff *skb)
@@ -1934,8 +1916,8 @@ static int abort_syn_rcv(struct sock *sk, struct sk_buff *skb)
if (!sock_owned_by_user(psk)) {
int queue = csk->txq_idx;
+ chtls_send_abort_rpl(sk, skb, cdev, CPL_ABORT_NO_RST, queue);
do_abort_syn_rcv(sk, psk);
- send_abort_rpl(sk, skb, cdev, CPL_ABORT_NO_RST, queue);
} else {
skb->sk = sk;
BLOG_SKB_CB(skb)->backlog_rcv = bl_abort_syn_rcv;
@@ -1953,9 +1935,6 @@ static void chtls_abort_req_rss(struct sock *sk, struct sk_buff *skb)
int queue = csk->txq_idx;
if (is_neg_adv(req->status)) {
- if (sk->sk_state == TCP_SYN_RECV)
- chtls_set_tcb_tflag(sk, 0, 0);
-
kfree_skb(skb);
return;
}
@@ -1981,12 +1960,11 @@ static void chtls_abort_req_rss(struct sock *sk, struct sk_buff *skb)
if (sk->sk_state == TCP_SYN_RECV && !abort_syn_rcv(sk, skb))
return;
-
- chtls_release_resources(sk);
- chtls_conn_done(sk);
}
chtls_send_abort_rpl(sk, skb, csk->cdev, rst_status, queue);
+ chtls_release_resources(sk);
+ chtls_conn_done(sk);
}
static void chtls_abort_rpl_rss(struct sock *sk, struct sk_buff *skb)
diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c
index 86427f6ba78c..c52718b37f8f 100644
--- a/drivers/dma/at_hdmac.c
+++ b/drivers/dma/at_hdmac.c
@@ -1683,9 +1683,11 @@ static struct dma_chan *at_dma_xlate(struct of_phandle_args *dma_spec,
dma_cap_zero(mask);
dma_cap_set(DMA_SLAVE, mask);
- atslave = kzalloc(sizeof(*atslave), GFP_KERNEL);
- if (!atslave)
+ atslave = kmalloc(sizeof(*atslave), GFP_KERNEL);
+ if (!atslave) {
+ put_device(&dmac_pdev->dev);
return NULL;
+ }
atslave->cfg = ATC_DST_H2SEL_HW | ATC_SRC_H2SEL_HW;
/*
@@ -1714,8 +1716,11 @@ static struct dma_chan *at_dma_xlate(struct of_phandle_args *dma_spec,
atslave->dma_dev = &dmac_pdev->dev;
chan = dma_request_channel(mask, at_dma_filter, atslave);
- if (!chan)
+ if (!chan) {
+ put_device(&dmac_pdev->dev);
+ kfree(atslave);
return NULL;
+ }
atchan = to_at_dma_chan(chan);
atchan->per_if = dma_spec->args[0] & 0xff;
diff --git a/drivers/dma/mediatek/mtk-hsdma.c b/drivers/dma/mediatek/mtk-hsdma.c
index fca232b1d4a6..1d44b02831a0 100644
--- a/drivers/dma/mediatek/mtk-hsdma.c
+++ b/drivers/dma/mediatek/mtk-hsdma.c
@@ -1007,6 +1007,7 @@ static int mtk_hsdma_probe(struct platform_device *pdev)
return 0;
err_free:
+ mtk_hsdma_hw_deinit(hsdma);
of_dma_controller_free(pdev->dev.of_node);
err_unregister:
dma_async_device_unregister(dd);
diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c
index 28592137fb67..0c5668e897fe 100644
--- a/drivers/dma/xilinx/xilinx_dma.c
+++ b/drivers/dma/xilinx/xilinx_dma.c
@@ -2426,7 +2426,7 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev,
has_dre = false;
if (!has_dre)
- xdev->common.copy_align = fls(width - 1);
+ xdev->common.copy_align = (enum dmaengine_alignment)fls(width - 1);
if (of_device_is_compatible(node, "xlnx,axi-vdma-mm2s-channel") ||
of_device_is_compatible(node, "xlnx,axi-dma-mm2s-channel") ||
@@ -2529,7 +2529,8 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev,
static int xilinx_dma_child_probe(struct xilinx_dma_device *xdev,
struct device_node *node)
{
- int ret, i, nr_channels = 1;
+ int ret, i;
+ u32 nr_channels = 1;
ret = of_property_read_u32(node, "dma-channels", &nr_channels);
if ((ret < 0) && xdev->mcdma)
@@ -2713,7 +2714,11 @@ static int xilinx_dma_probe(struct platform_device *pdev)
}
/* Register the DMA engine with the core */
- dma_async_device_register(&xdev->common);
+ err = dma_async_device_register(&xdev->common);
+ if (err) {
+ dev_err(xdev->dev, "failed to register the dma device\n");
+ goto error;
+ }
err = of_dma_controller_register(node, of_dma_xilinx_xlate,
xdev);
diff --git a/drivers/gpio/gpio-mvebu.c b/drivers/gpio/gpio-mvebu.c
index 3b78dcda4736..874caed72390 100644
--- a/drivers/gpio/gpio-mvebu.c
+++ b/drivers/gpio/gpio-mvebu.c
@@ -650,9 +650,8 @@ static void mvebu_pwm_get_state(struct pwm_chip *chip,
spin_lock_irqsave(&mvpwm->lock, flags);
- val = (unsigned long long)
- readl_relaxed(mvebu_pwmreg_blink_on_duration(mvpwm));
- val *= NSEC_PER_SEC;
+ u = readl_relaxed(mvebu_pwmreg_blink_on_duration(mvpwm));
+ val = (unsigned long long) u * NSEC_PER_SEC;
do_div(val, mvpwm->clk_rate);
if (val > UINT_MAX)
state->duty_cycle = UINT_MAX;
@@ -661,21 +660,17 @@ static void mvebu_pwm_get_state(struct pwm_chip *chip,
else
state->duty_cycle = 1;
- val = (unsigned long long)
- readl_relaxed(mvebu_pwmreg_blink_off_duration(mvpwm));
+ val = (unsigned long long) u; /* on duration */
+ /* period = on + off duration */
+ val += readl_relaxed(mvebu_pwmreg_blink_off_duration(mvpwm));
val *= NSEC_PER_SEC;
do_div(val, mvpwm->clk_rate);
- if (val < state->duty_cycle) {
+ if (val > UINT_MAX)
+ state->period = UINT_MAX;
+ else if (val)
+ state->period = val;
+ else
state->period = 1;
- } else {
- val -= state->duty_cycle;
- if (val > UINT_MAX)
- state->period = UINT_MAX;
- else if (val)
- state->period = val;
- else
- state->period = 1;
- }
regmap_read(mvchip->regs, GPIO_BLINK_EN_OFF + mvchip->offset, &u);
if (u)
diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c
index e2350dc56372..2992f4f5ab12 100644
--- a/drivers/gpu/drm/drm_atomic_helper.c
+++ b/drivers/gpu/drm/drm_atomic_helper.c
@@ -2920,7 +2920,7 @@ int drm_atomic_helper_set_config(struct drm_mode_set *set,
ret = handle_conflicting_encoders(state, true);
if (ret)
- return ret;
+ goto fail;
ret = drm_atomic_commit(state);
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index 52894816167c..8b5b147cdfd1 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -380,7 +380,7 @@ eb_vma_misplaced(const struct drm_i915_gem_exec_object2 *entry,
return true;
if (!(flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) &&
- (vma->node.start + vma->node.size - 1) >> 32)
+ (vma->node.start + vma->node.size + 4095) >> 32)
return true;
if (flags & __EXEC_OBJECT_NEEDS_MAP &&
diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c
index e11299869677..7e30b41db1e5 100644
--- a/drivers/gpu/drm/msm/msm_drv.c
+++ b/drivers/gpu/drm/msm/msm_drv.c
@@ -485,15 +485,15 @@ static int msm_drm_init(struct device *dev, struct drm_driver *drv)
drm_mode_config_init(ddev);
+ ret = msm_init_vram(ddev);
+ if (ret)
+ goto err_destroy_mdss;
+
/* Bind all our sub-components: */
ret = component_bind_all(dev, ddev);
if (ret)
goto err_destroy_mdss;
- ret = msm_init_vram(ddev);
- if (ret)
- goto err_msm_uninit;
-
if (!dev->dma_parms) {
dev->dma_parms = devm_kzalloc(dev, sizeof(*dev->dma_parms),
GFP_KERNEL);
diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.c b/drivers/gpu/drm/nouveau/dispnv50/disp.c
index 1bb0a9f6fa73..fbe156302ee8 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/disp.c
+++ b/drivers/gpu/drm/nouveau/dispnv50/disp.c
@@ -131,7 +131,7 @@ nv50_dmac_destroy(struct nv50_dmac *dmac)
int
nv50_dmac_create(struct nvif_device *device, struct nvif_object *disp,
- const s32 *oclass, u8 head, void *data, u32 size, u64 syncbuf,
+ const s32 *oclass, u8 head, void *data, u32 size, s64 syncbuf,
struct nv50_dmac *dmac)
{
struct nouveau_cli *cli = (void *)device->object.client;
@@ -166,7 +166,7 @@ nv50_dmac_create(struct nvif_device *device, struct nvif_object *disp,
if (ret)
return ret;
- if (!syncbuf)
+ if (syncbuf < 0)
return 0;
ret = nvif_object_init(&dmac->base.user, 0xf0000000, NV_DMA_IN_MEMORY,
diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.h b/drivers/gpu/drm/nouveau/dispnv50/disp.h
index 66c125a6b0b3..55205d23360c 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/disp.h
+++ b/drivers/gpu/drm/nouveau/dispnv50/disp.h
@@ -68,7 +68,7 @@ struct nv50_dmac {
int nv50_dmac_create(struct nvif_device *device, struct nvif_object *disp,
const s32 *oclass, u8 head, void *data, u32 size,
- u64 syncbuf, struct nv50_dmac *dmac);
+ s64 syncbuf, struct nv50_dmac *dmac);
void nv50_dmac_destroy(struct nv50_dmac *);
u32 *evo_wait(struct nv50_dmac *, int nr);
diff --git a/drivers/gpu/drm/nouveau/dispnv50/wimmc37b.c b/drivers/gpu/drm/nouveau/dispnv50/wimmc37b.c
index f7dbd965e4e7..b49a212af4d8 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/wimmc37b.c
+++ b/drivers/gpu/drm/nouveau/dispnv50/wimmc37b.c
@@ -68,7 +68,7 @@ wimmc37b_init_(const struct nv50_wimm_func *func, struct nouveau_drm *drm,
int ret;
ret = nv50_dmac_create(&drm->client.device, &disp->disp->object,
- &oclass, 0, &args, sizeof(args), 0,
+ &oclass, 0, &args, sizeof(args), -1,
&wndw->wimm);
if (ret) {
NV_ERROR(drm, "wimm%04x allocation failed: %d\n", oclass, ret);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadow.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadow.c
index 7deb81b6dbac..4b571cc6bc70 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadow.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadow.c
@@ -75,7 +75,7 @@ shadow_image(struct nvkm_bios *bios, int idx, u32 offset, struct shadow *mthd)
nvkm_debug(subdev, "%08x: type %02x, %d bytes\n",
image.base, image.type, image.size);
- if (!shadow_fetch(bios, mthd, image.size)) {
+ if (!shadow_fetch(bios, mthd, image.base + image.size)) {
nvkm_debug(subdev, "%08x: fetch failed\n", image.base);
return 0;
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/auxgm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/auxgm200.c
index edb6148cbca0..d0e80ad52684 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/auxgm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/auxgm200.c
@@ -33,7 +33,7 @@ static void
gm200_i2c_aux_fini(struct gm200_i2c_aux *aux)
{
struct nvkm_device *device = aux->base.pad->i2c->subdev.device;
- nvkm_mask(device, 0x00d954 + (aux->ch * 0x50), 0x00310000, 0x00000000);
+ nvkm_mask(device, 0x00d954 + (aux->ch * 0x50), 0x00710000, 0x00000000);
}
static int
@@ -54,10 +54,10 @@ gm200_i2c_aux_init(struct gm200_i2c_aux *aux)
AUX_ERR(&aux->base, "begin idle timeout %08x", ctrl);
return -EBUSY;
}
- } while (ctrl & 0x03010000);
+ } while (ctrl & 0x07010000);
/* set some magic, and wait up to 1ms for it to appear */
- nvkm_mask(device, 0x00d954 + (aux->ch * 0x50), 0x00300000, ureq);
+ nvkm_mask(device, 0x00d954 + (aux->ch * 0x50), 0x00700000, ureq);
timeout = 1000;
do {
ctrl = nvkm_rd32(device, 0x00d954 + (aux->ch * 0x50));
@@ -67,7 +67,7 @@ gm200_i2c_aux_init(struct gm200_i2c_aux *aux)
gm200_i2c_aux_fini(aux);
return -EBUSY;
}
- } while ((ctrl & 0x03000000) != urep);
+ } while ((ctrl & 0x07000000) != urep);
return 0;
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ibus/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ibus/gf100.c
index d80dbc8f09b2..55a4ea4393c6 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/ibus/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ibus/gf100.c
@@ -22,6 +22,7 @@
* Authors: Ben Skeggs
*/
#include "priv.h"
+#include
static void
gf100_ibus_intr_hub(struct nvkm_subdev *ibus, int i)
@@ -31,7 +32,6 @@ gf100_ibus_intr_hub(struct nvkm_subdev *ibus, int i)
u32 data = nvkm_rd32(device, 0x122124 + (i * 0x0400));
u32 stat = nvkm_rd32(device, 0x122128 + (i * 0x0400));
nvkm_debug(ibus, "HUB%d: %06x %08x (%08x)\n", i, addr, data, stat);
- nvkm_mask(device, 0x122128 + (i * 0x0400), 0x00000200, 0x00000000);
}
static void
@@ -42,7 +42,6 @@ gf100_ibus_intr_rop(struct nvkm_subdev *ibus, int i)
u32 data = nvkm_rd32(device, 0x124124 + (i * 0x0400));
u32 stat = nvkm_rd32(device, 0x124128 + (i * 0x0400));
nvkm_debug(ibus, "ROP%d: %06x %08x (%08x)\n", i, addr, data, stat);
- nvkm_mask(device, 0x124128 + (i * 0x0400), 0x00000200, 0x00000000);
}
static void
@@ -53,7 +52,6 @@ gf100_ibus_intr_gpc(struct nvkm_subdev *ibus, int i)
u32 data = nvkm_rd32(device, 0x128124 + (i * 0x0400));
u32 stat = nvkm_rd32(device, 0x128128 + (i * 0x0400));
nvkm_debug(ibus, "GPC%d: %06x %08x (%08x)\n", i, addr, data, stat);
- nvkm_mask(device, 0x128128 + (i * 0x0400), 0x00000200, 0x00000000);
}
void
@@ -90,6 +88,12 @@ gf100_ibus_intr(struct nvkm_subdev *ibus)
intr1 &= ~stat;
}
}
+
+ nvkm_mask(device, 0x121c4c, 0x0000003f, 0x00000002);
+ nvkm_msec(device, 2000,
+ if (!(nvkm_rd32(device, 0x121c4c) & 0x0000003f))
+ break;
+ );
}
static int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ibus/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ibus/gk104.c
index 9025ed1bd2a9..4caf3ef087e1 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/ibus/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ibus/gk104.c
@@ -22,6 +22,7 @@
* Authors: Ben Skeggs
*/
#include "priv.h"
+#include
static void
gk104_ibus_intr_hub(struct nvkm_subdev *ibus, int i)
@@ -31,7 +32,6 @@ gk104_ibus_intr_hub(struct nvkm_subdev *ibus, int i)
u32 data = nvkm_rd32(device, 0x122124 + (i * 0x0800));
u32 stat = nvkm_rd32(device, 0x122128 + (i * 0x0800));
nvkm_debug(ibus, "HUB%d: %06x %08x (%08x)\n", i, addr, data, stat);
- nvkm_mask(device, 0x122128 + (i * 0x0800), 0x00000200, 0x00000000);
}
static void
@@ -42,7 +42,6 @@ gk104_ibus_intr_rop(struct nvkm_subdev *ibus, int i)
u32 data = nvkm_rd32(device, 0x124124 + (i * 0x0800));
u32 stat = nvkm_rd32(device, 0x124128 + (i * 0x0800));
nvkm_debug(ibus, "ROP%d: %06x %08x (%08x)\n", i, addr, data, stat);
- nvkm_mask(device, 0x124128 + (i * 0x0800), 0x00000200, 0x00000000);
}
static void
@@ -53,7 +52,6 @@ gk104_ibus_intr_gpc(struct nvkm_subdev *ibus, int i)
u32 data = nvkm_rd32(device, 0x128124 + (i * 0x0800));
u32 stat = nvkm_rd32(device, 0x128128 + (i * 0x0800));
nvkm_debug(ibus, "GPC%d: %06x %08x (%08x)\n", i, addr, data, stat);
- nvkm_mask(device, 0x128128 + (i * 0x0800), 0x00000200, 0x00000000);
}
void
@@ -90,6 +88,12 @@ gk104_ibus_intr(struct nvkm_subdev *ibus)
intr1 &= ~stat;
}
}
+
+ nvkm_mask(device, 0x12004c, 0x0000003f, 0x00000002);
+ nvkm_msec(device, 2000,
+ if (!(nvkm_rd32(device, 0x12004c) & 0x0000003f))
+ break;
+ );
}
static int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c
index ee11ccaf0563..cb51e248cb41 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c
@@ -316,9 +316,9 @@ nvkm_mmu_vram(struct nvkm_mmu *mmu)
{
struct nvkm_device *device = mmu->subdev.device;
struct nvkm_mm *mm = &device->fb->ram->vram;
- const u32 sizeN = nvkm_mm_heap_size(mm, NVKM_RAM_MM_NORMAL);
- const u32 sizeU = nvkm_mm_heap_size(mm, NVKM_RAM_MM_NOMAP);
- const u32 sizeM = nvkm_mm_heap_size(mm, NVKM_RAM_MM_MIXED);
+ const u64 sizeN = nvkm_mm_heap_size(mm, NVKM_RAM_MM_NORMAL);
+ const u64 sizeU = nvkm_mm_heap_size(mm, NVKM_RAM_MM_NOMAP);
+ const u64 sizeM = nvkm_mm_heap_size(mm, NVKM_RAM_MM_MIXED);
u8 type = NVKM_MEM_KIND * !!mmu->func->kind;
u8 heap = NVKM_MEM_VRAM;
int heapM, heapN, heapU;
diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h
index b26a3599523c..bb10e152a894 100644
--- a/drivers/hid/hid-ids.h
+++ b/drivers/hid/hid-ids.h
@@ -386,6 +386,7 @@
#define USB_DEVICE_ID_TOSHIBA_CLICK_L9W 0x0401
#define USB_DEVICE_ID_HP_X2 0x074d
#define USB_DEVICE_ID_HP_X2_10_COVER 0x0755
+#define USB_DEVICE_ID_ASUS_UX550_TOUCHSCREEN 0x2706
#define USB_VENDOR_ID_ELECOM 0x056e
#define USB_DEVICE_ID_ELECOM_BM084 0x0061
diff --git a/drivers/hid/hid-input.c b/drivers/hid/hid-input.c
index 13deb9a67685..4dd151b2924e 100644
--- a/drivers/hid/hid-input.c
+++ b/drivers/hid/hid-input.c
@@ -334,6 +334,8 @@ static const struct hid_device_id hid_battery_quirks[] = {
{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH,
USB_DEVICE_ID_LOGITECH_DINOVO_EDGE_KBD),
HID_BATTERY_QUIRK_IGNORE },
+ { HID_USB_DEVICE(USB_VENDOR_ID_ELAN, USB_DEVICE_ID_ASUS_UX550_TOUCHSCREEN),
+ HID_BATTERY_QUIRK_IGNORE },
{}
};
diff --git a/drivers/hid/wacom_sys.c b/drivers/hid/wacom_sys.c
index 8249ff3a5a8d..8006732b8f42 100644
--- a/drivers/hid/wacom_sys.c
+++ b/drivers/hid/wacom_sys.c
@@ -150,9 +150,9 @@ static int wacom_wac_pen_serial_enforce(struct hid_device *hdev,
}
if (flush)
- wacom_wac_queue_flush(hdev, &wacom_wac->pen_fifo);
+ wacom_wac_queue_flush(hdev, wacom_wac->pen_fifo);
else if (insert)
- wacom_wac_queue_insert(hdev, &wacom_wac->pen_fifo,
+ wacom_wac_queue_insert(hdev, wacom_wac->pen_fifo,
raw_data, report_size);
return insert && !flush;
@@ -1241,6 +1241,38 @@ static int wacom_devm_sysfs_create_group(struct wacom *wacom,
group);
}
+static void wacom_devm_kfifo_release(struct device *dev, void *res)
+{
+ struct kfifo_rec_ptr_2 *devres = res;
+
+ kfifo_free(devres);
+}
+
+static int wacom_devm_kfifo_alloc(struct wacom *wacom)
+{
+ struct wacom_wac *wacom_wac = &wacom->wacom_wac;
+ struct kfifo_rec_ptr_2 *pen_fifo;
+ int error;
+
+ pen_fifo = devres_alloc(wacom_devm_kfifo_release,
+ sizeof(struct kfifo_rec_ptr_2),
+ GFP_KERNEL);
+
+ if (!pen_fifo)
+ return -ENOMEM;
+
+ error = kfifo_alloc(pen_fifo, WACOM_PKGLEN_MAX, GFP_KERNEL);
+ if (error) {
+ devres_free(pen_fifo);
+ return error;
+ }
+
+ devres_add(&wacom->hdev->dev, pen_fifo);
+ wacom_wac->pen_fifo = pen_fifo;
+
+ return 0;
+}
+
enum led_brightness wacom_leds_brightness_get(struct wacom_led *led)
{
struct wacom *wacom = led->wacom;
@@ -2697,7 +2729,7 @@ static int wacom_probe(struct hid_device *hdev,
goto fail;
}
- error = kfifo_alloc(&wacom_wac->pen_fifo, WACOM_PKGLEN_MAX, GFP_KERNEL);
+ error = wacom_devm_kfifo_alloc(wacom);
if (error)
goto fail;
@@ -2764,8 +2796,6 @@ static void wacom_remove(struct hid_device *hdev)
if (wacom->wacom_wac.features.type != REMOTE)
wacom_release_resources(wacom);
- kfifo_free(&wacom_wac->pen_fifo);
-
hid_set_drvdata(hdev, NULL);
}
diff --git a/drivers/hid/wacom_wac.h b/drivers/hid/wacom_wac.h
index f67d871841c0..46da97162ef4 100644
--- a/drivers/hid/wacom_wac.h
+++ b/drivers/hid/wacom_wac.h
@@ -344,7 +344,7 @@ struct wacom_wac {
struct input_dev *pen_input;
struct input_dev *touch_input;
struct input_dev *pad_input;
- struct kfifo_rec_ptr_2 pen_fifo;
+ struct kfifo_rec_ptr_2 *pen_fifo;
int pid;
int num_contacts_left;
u8 bt_features;
diff --git a/drivers/hwtracing/intel_th/pci.c b/drivers/hwtracing/intel_th/pci.c
index a775d7acfa47..2a1617103394 100644
--- a/drivers/hwtracing/intel_th/pci.c
+++ b/drivers/hwtracing/intel_th/pci.c
@@ -230,6 +230,11 @@ static const struct pci_device_id intel_th_pci_id_table[] = {
PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x4b26),
.driver_data = (kernel_ulong_t)&intel_th_2x,
},
+ {
+ /* Alder Lake-P */
+ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x51a6),
+ .driver_data = (kernel_ulong_t)&intel_th_2x,
+ },
{
/* Emmitsburg PCH */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x1bcc),
diff --git a/drivers/hwtracing/stm/heartbeat.c b/drivers/hwtracing/stm/heartbeat.c
index 7db42395e131..825c59a77a0e 100644
--- a/drivers/hwtracing/stm/heartbeat.c
+++ b/drivers/hwtracing/stm/heartbeat.c
@@ -64,7 +64,7 @@ static void stm_heartbeat_unlink(struct stm_source_data *data)
static int stm_heartbeat_init(void)
{
- int i, ret = -ENOMEM;
+ int i, ret;
if (nr_devs < 0 || nr_devs > STM_HEARTBEAT_MAX)
return -EINVAL;
@@ -72,8 +72,10 @@ static int stm_heartbeat_init(void)
for (i = 0; i < nr_devs; i++) {
stm_heartbeat[i].data.name =
kasprintf(GFP_KERNEL, "heartbeat.%d", i);
- if (!stm_heartbeat[i].data.name)
+ if (!stm_heartbeat[i].data.name) {
+ ret = -ENOMEM;
goto fail_unregister;
+ }
stm_heartbeat[i].data.nr_chans = 1;
stm_heartbeat[i].data.link = stm_heartbeat_link;
diff --git a/drivers/i2c/busses/i2c-octeon-core.c b/drivers/i2c/busses/i2c-octeon-core.c
index d9607905dc2f..845eda70b8ca 100644
--- a/drivers/i2c/busses/i2c-octeon-core.c
+++ b/drivers/i2c/busses/i2c-octeon-core.c
@@ -347,7 +347,7 @@ static int octeon_i2c_read(struct octeon_i2c *i2c, int target,
if (result)
return result;
if (recv_len && i == 0) {
- if (data[i] > I2C_SMBUS_BLOCK_MAX + 1)
+ if (data[i] > I2C_SMBUS_BLOCK_MAX)
return -EPROTO;
length += data[i];
}
diff --git a/drivers/i2c/busses/i2c-sprd.c b/drivers/i2c/busses/i2c-sprd.c
index a94e724f51dc..bb1478e781c4 100644
--- a/drivers/i2c/busses/i2c-sprd.c
+++ b/drivers/i2c/busses/i2c-sprd.c
@@ -71,6 +71,8 @@
/* timeout (ms) for pm runtime autosuspend */
#define SPRD_I2C_PM_TIMEOUT 1000
+/* timeout (ms) for transfer message */
+#define I2C_XFER_TIMEOUT 1000
/* SPRD i2c data structure */
struct sprd_i2c {
@@ -244,6 +246,7 @@ static int sprd_i2c_handle_msg(struct i2c_adapter *i2c_adap,
struct i2c_msg *msg, bool is_last_msg)
{
struct sprd_i2c *i2c_dev = i2c_adap->algo_data;
+ unsigned long time_left;
i2c_dev->msg = msg;
i2c_dev->buf = msg->buf;
@@ -273,7 +276,10 @@ static int sprd_i2c_handle_msg(struct i2c_adapter *i2c_adap,
sprd_i2c_opt_start(i2c_dev);
- wait_for_completion(&i2c_dev->complete);
+ time_left = wait_for_completion_timeout(&i2c_dev->complete,
+ msecs_to_jiffies(I2C_XFER_TIMEOUT));
+ if (!time_left)
+ return -ETIMEDOUT;
return i2c_dev->err;
}
diff --git a/drivers/i2c/busses/i2c-tegra-bpmp.c b/drivers/i2c/busses/i2c-tegra-bpmp.c
index f6cd35d0a2ac..240bd1e90892 100644
--- a/drivers/i2c/busses/i2c-tegra-bpmp.c
+++ b/drivers/i2c/busses/i2c-tegra-bpmp.c
@@ -91,7 +91,7 @@ static int tegra_bpmp_xlate_flags(u16 flags, u16 *out)
flags &= ~I2C_M_RECV_LEN;
}
- return (flags != 0) ? -EINVAL : 0;
+ return 0;
}
/**
diff --git a/drivers/ide/ide-atapi.c b/drivers/ide/ide-atapi.c
index 8b2b72b93885..4224c4dd8963 100644
--- a/drivers/ide/ide-atapi.c
+++ b/drivers/ide/ide-atapi.c
@@ -213,7 +213,6 @@ void ide_prep_sense(ide_drive_t *drive, struct request *rq)
sense_rq->rq_disk = rq->rq_disk;
sense_rq->cmd_flags = REQ_OP_DRV_IN;
ide_req(sense_rq)->type = ATA_PRIV_SENSE;
- sense_rq->rq_flags |= RQF_PREEMPT;
req->cmd[0] = GPCMD_REQUEST_SENSE;
req->cmd[4] = cmd_len;
diff --git a/drivers/ide/ide-io.c b/drivers/ide/ide-io.c
index 0d93e0cfbeaf..438176084610 100644
--- a/drivers/ide/ide-io.c
+++ b/drivers/ide/ide-io.c
@@ -527,11 +527,6 @@ repeat:
* above to return us whatever is in the queue. Since we call
* ide_do_request() ourselves, we end up taking requests while
* the queue is blocked...
- *
- * We let requests forced at head of queue with ide-preempt
- * though. I hope that doesn't happen too much, hopefully not
- * unless the subdriver triggers such a thing in its own PM
- * state machine.
*/
if ((drive->dev_flags & IDE_DFLAG_BLOCKED) &&
ata_pm_request(rq) == 0 &&
diff --git a/drivers/iio/dac/ad5504.c b/drivers/iio/dac/ad5504.c
index d9037ea59168..97b9ce305a68 100644
--- a/drivers/iio/dac/ad5504.c
+++ b/drivers/iio/dac/ad5504.c
@@ -189,9 +189,9 @@ static ssize_t ad5504_write_dac_powerdown(struct iio_dev *indio_dev,
return ret;
if (pwr_down)
- st->pwr_down_mask |= (1 << chan->channel);
- else
st->pwr_down_mask &= ~(1 << chan->channel);
+ else
+ st->pwr_down_mask |= (1 << chan->channel);
ret = ad5504_spi_write(st, AD5504_ADDR_CTRL,
AD5504_DAC_PWRDWN_MODE(st->pwr_down_mode) |
diff --git a/drivers/iio/imu/bmi160/bmi160_core.c b/drivers/iio/imu/bmi160/bmi160_core.c
index e95d817c8390..1e413bb233ae 100644
--- a/drivers/iio/imu/bmi160/bmi160_core.c
+++ b/drivers/iio/imu/bmi160/bmi160_core.c
@@ -110,6 +110,13 @@ enum bmi160_sensor_type {
struct bmi160_data {
struct regmap *regmap;
+ /*
+ * Ensure natural alignment for timestamp if present.
+ * Max length needed: 2 * 3 channels + 4 bytes padding + 8 byte ts.
+ * If fewer channels are enabled, less space may be needed, as
+ * long as the timestamp is still aligned to 8 bytes.
+ */
+ __le16 buf[12] __aligned(8);
};
const struct regmap_config bmi160_regmap_config = {
@@ -385,8 +392,6 @@ static irqreturn_t bmi160_trigger_handler(int irq, void *p)
struct iio_poll_func *pf = p;
struct iio_dev *indio_dev = pf->indio_dev;
struct bmi160_data *data = iio_priv(indio_dev);
- __le16 buf[12];
- /* 2 sens x 3 axis x __le16 + 2 x __le16 pad + 4 x __le16 tstamp */
int i, ret, j = 0, base = BMI160_REG_DATA_MAGN_XOUT_L;
__le16 sample;
@@ -396,10 +401,10 @@ static irqreturn_t bmi160_trigger_handler(int irq, void *p)
&sample, sizeof(sample));
if (ret < 0)
goto done;
- buf[j++] = sample;
+ data->buf[j++] = sample;
}
- iio_push_to_buffers_with_timestamp(indio_dev, buf,
+ iio_push_to_buffers_with_timestamp(indio_dev, data->buf,
iio_get_time_ns(indio_dev));
done:
iio_trigger_notify_done(indio_dev->trig);
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c
index 631360b14ca7..4d89de0be58b 100644
--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c
@@ -475,13 +475,29 @@ static irqreturn_t st_lsm6dsx_handler_irq(int irq, void *private)
static irqreturn_t st_lsm6dsx_handler_thread(int irq, void *private)
{
struct st_lsm6dsx_hw *hw = private;
- int count;
+ int fifo_len = 0, len;
- mutex_lock(&hw->fifo_lock);
- count = st_lsm6dsx_read_fifo(hw);
- mutex_unlock(&hw->fifo_lock);
+ /*
+ * If we are using edge IRQs, new samples can arrive while
+ * processing current interrupt since there are no hw
+ * guarantees the irq line stays "low" long enough to properly
+ * detect the new interrupt. In this case the new sample will
+ * be missed.
+ * Polling FIFO status register allow us to read new
+ * samples even if the interrupt arrives while processing
+ * previous data and the timeslot where the line is "low" is
+ * too short to be properly detected.
+ */
+ do {
+ mutex_lock(&hw->fifo_lock);
+ len = st_lsm6dsx_read_fifo(hw);
+ mutex_unlock(&hw->fifo_lock);
- return !count ? IRQ_NONE : IRQ_HANDLED;
+ if (len > 0)
+ fifo_len += len;
+ } while (len > 0);
+
+ return fifo_len ? IRQ_HANDLED : IRQ_NONE;
}
static int st_lsm6dsx_buffer_preenable(struct iio_dev *iio_dev)
diff --git a/drivers/iio/magnetometer/mag3110.c b/drivers/iio/magnetometer/mag3110.c
index f063355480ba..57fbe5bfe883 100644
--- a/drivers/iio/magnetometer/mag3110.c
+++ b/drivers/iio/magnetometer/mag3110.c
@@ -56,6 +56,12 @@ struct mag3110_data {
struct mutex lock;
u8 ctrl_reg1;
int sleep_val;
+ /* Ensure natural alignment of timestamp */
+ struct {
+ __be16 channels[3];
+ u8 temperature;
+ s64 ts __aligned(8);
+ } scan;
};
static int mag3110_request(struct mag3110_data *data)
@@ -387,10 +393,9 @@ static irqreturn_t mag3110_trigger_handler(int irq, void *p)
struct iio_poll_func *pf = p;
struct iio_dev *indio_dev = pf->indio_dev;
struct mag3110_data *data = iio_priv(indio_dev);
- u8 buffer[16]; /* 3 16-bit channels + 1 byte temp + padding + ts */
int ret;
- ret = mag3110_read(data, (__be16 *) buffer);
+ ret = mag3110_read(data, data->scan.channels);
if (ret < 0)
goto done;
@@ -399,10 +404,10 @@ static irqreturn_t mag3110_trigger_handler(int irq, void *p)
MAG3110_DIE_TEMP);
if (ret < 0)
goto done;
- buffer[6] = ret;
+ data->scan.temperature = ret;
}
- iio_push_to_buffers_with_timestamp(indio_dev, buffer,
+ iio_push_to_buffers_with_timestamp(indio_dev, &data->scan,
iio_get_time_ns(indio_dev));
done:
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index f41f3ff689c5..1695605eeb52 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -6094,7 +6094,7 @@ int mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev *dev)
err = mlx5_alloc_bfreg(dev->mdev, &dev->fp_bfreg, false, true);
if (err)
- mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
+ mlx5_free_bfreg(dev->mdev, &dev->bfreg);
return err;
}
diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
index e611f133aa97..e6c11b5a1669 100644
--- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
+++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
@@ -212,6 +212,7 @@ find_free_vf_and_create_qp_grp(struct usnic_ib_dev *us_ibdev,
}
usnic_uiom_free_dev_list(dev_list);
+ dev_list = NULL;
}
/* Try to find resources on an unused vf */
@@ -236,6 +237,8 @@ find_free_vf_and_create_qp_grp(struct usnic_ib_dev *us_ibdev,
qp_grp_check:
if (IS_ERR_OR_NULL(qp_grp)) {
usnic_err("Failed to allocate qp_grp\n");
+ if (usnic_ib_share_vf)
+ usnic_uiom_free_dev_list(dev_list);
return ERR_PTR(qp_grp ? PTR_ERR(qp_grp) : -ENOMEM);
}
return qp_grp;
diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c
index 9d2d03545bb0..cd2e5b44119a 100644
--- a/drivers/iommu/intel_irq_remapping.c
+++ b/drivers/iommu/intel_irq_remapping.c
@@ -1373,6 +1373,8 @@ static int intel_irq_remapping_alloc(struct irq_domain *domain,
irq_data = irq_domain_get_irq_data(domain, virq + i);
irq_cfg = irqd_cfg(irq_data);
if (!irq_data || !irq_cfg) {
+ if (!i)
+ kfree(data);
ret = -EINVAL;
goto out_free_data;
}
diff --git a/drivers/irqchip/irq-mips-cpu.c b/drivers/irqchip/irq-mips-cpu.c
index 66f97fde13d8..51e09f6c653c 100644
--- a/drivers/irqchip/irq-mips-cpu.c
+++ b/drivers/irqchip/irq-mips-cpu.c
@@ -201,6 +201,13 @@ static int mips_cpu_ipi_alloc(struct irq_domain *domain, unsigned int virq,
if (ret)
return ret;
+ ret = irq_domain_set_hwirq_and_chip(domain->parent, virq + i, hwirq,
+ &mips_mt_cpu_irq_controller,
+ NULL);
+
+ if (ret)
+ return ret;
+
ret = irq_set_irq_type(virq + i, IRQ_TYPE_LEVEL_HIGH);
if (ret)
return ret;
diff --git a/drivers/isdn/mISDN/Kconfig b/drivers/isdn/mISDN/Kconfig
index c0730d5c734d..fb61181a5c4f 100644
--- a/drivers/isdn/mISDN/Kconfig
+++ b/drivers/isdn/mISDN/Kconfig
@@ -12,6 +12,7 @@ if MISDN != n
config MISDN_DSP
tristate "Digital Audio Processing of transparent data"
depends on MISDN
+ select BITREVERSE
help
Enable support for digital audio processing capability.
diff --git a/drivers/lightnvm/Kconfig b/drivers/lightnvm/Kconfig
index 439bf90d084d..20706da7aa1c 100644
--- a/drivers/lightnvm/Kconfig
+++ b/drivers/lightnvm/Kconfig
@@ -19,6 +19,7 @@ if NVM
config NVM_PBLK
tristate "Physical Block Device Open-Channel SSD target"
+ select CRC32
help
Allows an open-channel SSD to be exposed as a block device to the
host. The target assumes the device exposes raw flash and must be
diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig
index 72c3d6658670..e4b3ed031a53 100644
--- a/drivers/md/Kconfig
+++ b/drivers/md/Kconfig
@@ -655,4 +655,18 @@ config DM_BOW
If unsure, say N.
+config DM_USER
+ tristate "Block device in userspace"
+ depends on BLK_DEV_DM
+ default y
+ help
+ This device-mapper target allows a userspace daemon to provide the
+ contents of a block device. See
+ for more information.
+
+ To compile this code as a module, choose M here: the module will be
+ called dm-user.
+
+ If unsure, say N.
+
endif # MD
diff --git a/drivers/md/Makefile b/drivers/md/Makefile
index 2b5f53d3b17c..4874b859644b 100644
--- a/drivers/md/Makefile
+++ b/drivers/md/Makefile
@@ -70,6 +70,7 @@ obj-$(CONFIG_DM_INTEGRITY) += dm-integrity.o
obj-$(CONFIG_DM_ZONED) += dm-zoned.o
obj-$(CONFIG_DM_WRITECACHE) += dm-writecache.o
obj-$(CONFIG_DM_BOW) += dm-bow.o
+obj-$(CONFIG_DM_USER) += dm-user.o
ifeq ($(CONFIG_DM_UEVENT),y)
dm-mod-objs += dm-uevent.o
diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c
index b0db05232d53..8d7719d48674 100644
--- a/drivers/md/dm-bufio.c
+++ b/drivers/md/dm-bufio.c
@@ -1450,6 +1450,12 @@ sector_t dm_bufio_get_device_size(struct dm_bufio_client *c)
}
EXPORT_SYMBOL_GPL(dm_bufio_get_device_size);
+struct dm_io_client *dm_bufio_get_dm_io_client(struct dm_bufio_client *c)
+{
+ return c->dm_io;
+}
+EXPORT_SYMBOL_GPL(dm_bufio_get_dm_io_client);
+
sector_t dm_bufio_get_block_number(struct dm_buffer *b)
{
return b->block;
diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c
index 50aabb560e18..fc9ef30c8178 100644
--- a/drivers/md/dm-integrity.c
+++ b/drivers/md/dm-integrity.c
@@ -240,6 +240,7 @@ struct dm_integrity_c {
bool journal_uptodate;
bool just_formatted;
+ bool legacy_recalculate;
struct alg_spec internal_hash_alg;
struct alg_spec journal_crypt_alg;
@@ -345,6 +346,14 @@ static int dm_integrity_failed(struct dm_integrity_c *ic)
return READ_ONCE(ic->failed);
}
+static bool dm_integrity_disable_recalculate(struct dm_integrity_c *ic)
+{
+ if ((ic->internal_hash_alg.key || ic->journal_mac_alg.key) &&
+ !ic->legacy_recalculate)
+ return true;
+ return false;
+}
+
static commit_id_t dm_integrity_commit_id(struct dm_integrity_c *ic, unsigned i,
unsigned j, unsigned char seq)
{
@@ -1158,12 +1167,52 @@ static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, se
return 0;
}
-static void dm_integrity_flush_buffers(struct dm_integrity_c *ic)
+struct flush_request {
+ struct dm_io_request io_req;
+ struct dm_io_region io_reg;
+ struct dm_integrity_c *ic;
+ struct completion comp;
+};
+
+static void flush_notify(unsigned long error, void *fr_)
+{
+ struct flush_request *fr = fr_;
+ if (unlikely(error != 0))
+ dm_integrity_io_error(fr->ic, "flusing disk cache", -EIO);
+ complete(&fr->comp);
+}
+
+static void dm_integrity_flush_buffers(struct dm_integrity_c *ic, bool flush_data)
{
int r;
+
+ struct flush_request fr;
+
+ if (!ic->meta_dev)
+ flush_data = false;
+ if (flush_data) {
+ fr.io_req.bi_op = REQ_OP_WRITE,
+ fr.io_req.bi_op_flags = REQ_PREFLUSH | REQ_SYNC,
+ fr.io_req.mem.type = DM_IO_KMEM,
+ fr.io_req.mem.ptr.addr = NULL,
+ fr.io_req.notify.fn = flush_notify,
+ fr.io_req.notify.context = &fr;
+ fr.io_req.client = dm_bufio_get_dm_io_client(ic->bufio),
+ fr.io_reg.bdev = ic->dev->bdev,
+ fr.io_reg.sector = 0,
+ fr.io_reg.count = 0,
+ fr.ic = ic;
+ init_completion(&fr.comp);
+ r = dm_io(&fr.io_req, 1, &fr.io_reg, NULL);
+ BUG_ON(r);
+ }
+
r = dm_bufio_write_dirty_buffers(ic->bufio);
if (unlikely(r))
dm_integrity_io_error(ic, "writing tags", r);
+
+ if (flush_data)
+ wait_for_completion(&fr.comp);
}
static void sleep_on_endio_wait(struct dm_integrity_c *ic)
@@ -1857,7 +1906,7 @@ static void integrity_commit(struct work_struct *w)
flushes = bio_list_get(&ic->flush_bio_list);
if (unlikely(ic->mode != 'J')) {
spin_unlock_irq(&ic->endio_wait.lock);
- dm_integrity_flush_buffers(ic);
+ dm_integrity_flush_buffers(ic, true);
goto release_flush_bios;
}
@@ -2068,7 +2117,7 @@ skip_io:
complete_journal_op(&comp);
wait_for_completion_io(&comp.comp);
- dm_integrity_flush_buffers(ic);
+ dm_integrity_flush_buffers(ic, true);
}
static void integrity_writer(struct work_struct *w)
@@ -2110,7 +2159,7 @@ static void recalc_write_super(struct dm_integrity_c *ic)
{
int r;
- dm_integrity_flush_buffers(ic);
+ dm_integrity_flush_buffers(ic, false);
if (dm_integrity_failed(ic))
return;
@@ -2420,7 +2469,7 @@ static void dm_integrity_postsuspend(struct dm_target *ti)
if (ic->meta_dev)
queue_work(ic->writer_wq, &ic->writer_work);
drain_workqueue(ic->writer_wq);
- dm_integrity_flush_buffers(ic);
+ dm_integrity_flush_buffers(ic, true);
}
BUG_ON(!RB_EMPTY_ROOT(&ic->in_progress));
@@ -2474,6 +2523,7 @@ static void dm_integrity_status(struct dm_target *ti, status_type_t type,
arg_count += !!ic->internal_hash_alg.alg_string;
arg_count += !!ic->journal_crypt_alg.alg_string;
arg_count += !!ic->journal_mac_alg.alg_string;
+ arg_count += ic->legacy_recalculate;
DMEMIT("%s %llu %u %c %u", ic->dev->name, (unsigned long long)ic->start,
ic->tag_size, ic->mode, arg_count);
if (ic->meta_dev)
@@ -2487,6 +2537,8 @@ static void dm_integrity_status(struct dm_target *ti, status_type_t type,
DMEMIT(" buffer_sectors:%u", 1U << ic->log2_buffer_sectors);
DMEMIT(" journal_watermark:%u", (unsigned)watermark_percentage);
DMEMIT(" commit_time:%u", ic->autocommit_msec);
+ if (ic->legacy_recalculate)
+ DMEMIT(" legacy_recalculate");
#define EMIT_ALG(a, n) \
do { \
@@ -3089,7 +3141,7 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv)
unsigned extra_args;
struct dm_arg_set as;
static const struct dm_arg _args[] = {
- {0, 9, "Invalid number of feature args"},
+ {0, 12, "Invalid number of feature args"},
};
unsigned journal_sectors, interleave_sectors, buffer_sectors, journal_watermark, sync_msec;
bool recalculate;
@@ -3219,6 +3271,8 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv)
goto bad;
} else if (!strcmp(opt_string, "recalculate")) {
recalculate = true;
+ } else if (!strcmp(opt_string, "legacy_recalculate")) {
+ ic->legacy_recalculate = true;
} else {
r = -EINVAL;
ti->error = "Invalid argument";
@@ -3486,6 +3540,20 @@ try_smaller_buffer:
r = -ENOMEM;
goto bad;
}
+ } else {
+ if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) {
+ ti->error = "Recalculate can only be specified with internal_hash";
+ r = -EINVAL;
+ goto bad;
+ }
+ }
+
+ if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) &&
+ le64_to_cpu(ic->sb->recalc_sector) < ic->provided_data_sectors &&
+ dm_integrity_disable_recalculate(ic)) {
+ ti->error = "Recalculating with HMAC is disabled for security reasons - if you really need it, use the argument \"legacy_recalculate\"";
+ r = -EOPNOTSUPP;
+ goto bad;
}
ic->bufio = dm_bufio_client_create(ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev,
diff --git a/drivers/md/dm-snap.c b/drivers/md/dm-snap.c
index d3f28a9e3fd9..9e930a150aa2 100644
--- a/drivers/md/dm-snap.c
+++ b/drivers/md/dm-snap.c
@@ -137,6 +137,11 @@ struct dm_snapshot {
* for them to be committed.
*/
struct bio_list bios_queued_during_merge;
+
+ /*
+ * Flush data after merge.
+ */
+ struct bio flush_bio;
};
/*
@@ -1061,6 +1066,17 @@ shut:
static void error_bios(struct bio *bio);
+static int flush_data(struct dm_snapshot *s)
+{
+ struct bio *flush_bio = &s->flush_bio;
+
+ bio_reset(flush_bio);
+ bio_set_dev(flush_bio, s->origin->bdev);
+ flush_bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
+
+ return submit_bio_wait(flush_bio);
+}
+
static void merge_callback(int read_err, unsigned long write_err, void *context)
{
struct dm_snapshot *s = context;
@@ -1074,6 +1090,11 @@ static void merge_callback(int read_err, unsigned long write_err, void *context)
goto shut;
}
+ if (flush_data(s) < 0) {
+ DMERR("Flush after merge failed: shutting down merge");
+ goto shut;
+ }
+
if (s->store->type->commit_merge(s->store,
s->num_merging_chunks) < 0) {
DMERR("Write error in exception store: shutting down merge");
@@ -1198,6 +1219,7 @@ static int snapshot_ctr(struct dm_target *ti, unsigned int argc, char **argv)
s->first_merging_chunk = 0;
s->num_merging_chunks = 0;
bio_list_init(&s->bios_queued_during_merge);
+ bio_init(&s->flush_bio, NULL, 0);
/* Allocate hash table for COW data */
if (init_hash_tables(s)) {
@@ -1391,6 +1413,8 @@ static void snapshot_dtr(struct dm_target *ti)
mutex_destroy(&s->lock);
+ bio_uninit(&s->flush_bio);
+
dm_put_device(ti, s->cow);
dm_put_device(ti, s->origin);
diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c
index b641e5be43a2..80f3deccb64c 100644
--- a/drivers/md/dm-table.c
+++ b/drivers/md/dm-table.c
@@ -434,14 +434,23 @@ int dm_get_device(struct dm_target *ti, const char *path, fmode_t mode,
{
int r;
dev_t dev;
+ unsigned int major, minor;
+ char dummy;
struct dm_dev_internal *dd;
struct dm_table *t = ti->table;
BUG_ON(!t);
- dev = dm_get_dev_t(path);
- if (!dev)
- return -ENODEV;
+ if (sscanf(path, "%u:%u%c", &major, &minor, &dummy) == 2) {
+ /* Extract the major/minor numbers */
+ dev = MKDEV(major, minor);
+ if (MAJOR(dev) != major || MINOR(dev) != minor)
+ return -EOVERFLOW;
+ } else {
+ dev = dm_get_dev_t(path);
+ if (!dev)
+ return -ENODEV;
+ }
dd = find_device(&t->devices, dev);
if (!dd) {
diff --git a/drivers/md/dm-user.c b/drivers/md/dm-user.c
new file mode 100644
index 000000000000..4e64adaefdbd
--- /dev/null
+++ b/drivers/md/dm-user.c
@@ -0,0 +1,1141 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2020 Google, Inc
+ * Copyright (C) 2020 Palmer Dabbelt
+ */
+
+#include
+#include
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#define DM_MSG_PREFIX "user"
+
+#define MAX_OUTSTANDING_MESSAGES 128
+
+/*
+ * dm-user uses four structures:
+ *
+ * - "struct target", the outermost structure, corresponds to a single device
+ * mapper target. This contains the set of outstanding BIOs that have been
+ * provided by DM and are not actively being processed by the user, along
+ * with a misc device that userspace can open to communicate with the
+ * kernel. Each time userspaces opens the misc device a new channel is
+ * created.
+ * - "struct channel", which represents a single active communication channel
+ * with userspace. Userspace may choose arbitrary read/write sizes to use
+ * when processing messages, channels form these into logical accesses.
+ * When userspace responds to a full message the channel completes the BIO
+ * and obtains a new message to process from the target.
+ * - "struct message", which wraps a BIO with the additional information
+ * required by the kernel to sort out what to do with BIOs when they return
+ * from userspace.
+ * - "struct dm_user_message", which is the exact message format that
+ * userspace sees.
+ *
+ * The hot path contains three distinct operations:
+ *
+ * - user_map(), which is provided a BIO from device mapper that is queued
+ * into the target. This allocates and enqueues a new message.
+ * - dev_read(), which dequeues a message, copies it to userspace.
+ * - dev_write(), which looks up a message (keyed by sequence number) and
+ * completes the corresponding BIO.
+ *
+ * Lock ordering (outer to inner)
+ *
+ * 1) miscdevice's global lock. This is held around dev_open, so it has to be
+ * the outermost lock.
+ * 2) target->lock
+ * 3) channel->lock
+ */
+
+struct message {
+ /*
+ * Messages themselves do not need a lock, they're protected by either
+ * the target or channel's lock, depending on which can reference them
+ * directly.
+ */
+ struct dm_user_message msg;
+ struct bio *bio;
+ size_t posn_to_user;
+ size_t total_to_user;
+ size_t posn_from_user;
+ size_t total_from_user;
+
+ struct list_head from_user;
+ struct list_head to_user;
+
+ /*
+ * These are written back from the user. They live in the same spot in
+ * the message, but we need to either keep the old values around or
+ * call a bunch more BIO helpers. These are only valid after write has
+ * adopted the message.
+ */
+ u64 return_type;
+ u64 return_flags;
+};
+
+struct target {
+ /*
+ * A target has a single lock, which protects everything in the target
+ * (but does not protect the channels associated with a target).
+ */
+ struct mutex lock;
+
+ /*
+ * There is only one point at which anything blocks: userspace blocks
+ * reading a new message, which is woken up by device mapper providing
+ * a new BIO to process (or tearing down the target). The
+ * corresponding write side doesn't block, instead we treat userspace's
+ * response containing a message that has yet to be mapped as an
+ * invalid operation.
+ */
+ struct wait_queue_head wq;
+
+ /*
+ * Messages are delivered to userspace in order, but may be returned
+ * out of order. This allows userspace to schedule IO if it wants to.
+ */
+ mempool_t message_pool;
+ u64 next_seq_to_map;
+ u64 next_seq_to_user;
+ struct list_head to_user;
+
+ /*
+ * There is a misc device per target. The name is selected by
+ * userspace (via a DM create ioctl argument), and each ends up in
+ * /dev/dm-user/. It looks like a better way to do this may be to have
+ * a filesystem to manage these, but this was more expedient. The
+ * current mechanism is functional, but does result in an arbitrary
+ * number of dynamically created misc devices.
+ */
+ struct miscdevice miscdev;
+
+ /*
+ * Device mapper's target destructor triggers tearing this all down,
+ * but we can't actually free until every channel associated with this
+ * target has been destroyed. Channels each have a reference to their
+ * target, and there is an additional single reference that corresponds
+ * to both DM and the misc device (both of which are destroyed by DM).
+ *
+ * In the common case userspace will be asleep waiting for a new
+ * message when device mapper decides to destroy the target, which
+ * means no new messages will appear. The destroyed flag triggers a
+ * wakeup, which will end up removing the reference.
+ */
+ struct kref references;
+ int dm_destroyed;
+};
+
+struct channel {
+ struct target *target;
+
+ /*
+ * A channel has a single lock, which prevents multiple reads (or
+ * multiple writes) from conflicting with each other.
+ */
+ struct mutex lock;
+
+ struct message *cur_to_user;
+ struct message *cur_from_user;
+ ssize_t to_user_error;
+ ssize_t from_user_error;
+
+ /*
+ * Once a message has been forwarded to userspace on a channel it must
+ * be responded to on the same channel. This allows us to error out
+ * the messages that have not yet been responded to by a channel when
+ * that channel closes, which makes handling errors more reasonable for
+ * fault-tolerant userspace daemons. It also happens to make avoiding
+ * shared locks between user_map() and dev_read() a lot easier.
+ *
+ * This does preclude a multi-threaded work stealing userspace
+ * implementation (or at least, force a degree of head-of-line blocking
+ * on the response path).
+ */
+ struct list_head from_user;
+
+ /*
+ * Responses from userspace can arrive in arbitrarily small chunks.
+ * We need some place to buffer one up until we can find the
+ * corresponding kernel-side message to continue processing, so instead
+ * of allocating them we just keep one off to the side here. This can
+ * only ever be pointer to by from_user_cur, and will never have a BIO.
+ */
+ struct message scratch_message_from_user;
+};
+
+static inline struct target *target_from_target(struct dm_target *target)
+{
+ WARN_ON(target->private == NULL);
+ return target->private;
+}
+
+static inline struct target *target_from_miscdev(struct miscdevice *miscdev)
+{
+ return container_of(miscdev, struct target, miscdev);
+}
+
+static inline struct channel *channel_from_file(struct file *file)
+{
+ WARN_ON(file->private_data == NULL);
+ return file->private_data;
+}
+
+static inline struct target *target_from_channel(struct channel *c)
+{
+ WARN_ON(c->target == NULL);
+ return c->target;
+}
+
+static inline size_t bio_size(struct bio *bio)
+{
+ struct bio_vec bvec;
+ struct bvec_iter iter;
+ size_t out = 0;
+
+ bio_for_each_segment (bvec, bio, iter)
+ out += bio_iter_len(bio, iter);
+ return out;
+}
+
+static inline size_t bio_bytes_needed_to_user(struct bio *bio)
+{
+ switch (bio_op(bio)) {
+ case REQ_OP_WRITE:
+ return sizeof(struct dm_user_message) + bio_size(bio);
+ case REQ_OP_READ:
+ case REQ_OP_FLUSH:
+ case REQ_OP_DISCARD:
+ case REQ_OP_SECURE_ERASE:
+ case REQ_OP_WRITE_SAME:
+ case REQ_OP_WRITE_ZEROES:
+ return sizeof(struct dm_user_message);
+
+ /*
+ * These ops are not passed to userspace under the assumption that
+ * they're not going to be particularly useful in that context.
+ */
+ default:
+ return -EOPNOTSUPP;
+ }
+}
+
+static inline size_t bio_bytes_needed_from_user(struct bio *bio)
+{
+ switch (bio_op(bio)) {
+ case REQ_OP_READ:
+ return sizeof(struct dm_user_message) + bio_size(bio);
+ case REQ_OP_WRITE:
+ case REQ_OP_FLUSH:
+ case REQ_OP_DISCARD:
+ case REQ_OP_SECURE_ERASE:
+ case REQ_OP_WRITE_SAME:
+ case REQ_OP_WRITE_ZEROES:
+ return sizeof(struct dm_user_message);
+
+ /*
+ * These ops are not passed to userspace under the assumption that
+ * they're not going to be particularly useful in that context.
+ */
+ default:
+ return -EOPNOTSUPP;
+ }
+}
+
+static inline long bio_type_to_user_type(struct bio *bio)
+{
+ switch (bio_op(bio)) {
+ case REQ_OP_READ:
+ return DM_USER_REQ_MAP_READ;
+ case REQ_OP_WRITE:
+ return DM_USER_REQ_MAP_WRITE;
+ case REQ_OP_FLUSH:
+ return DM_USER_REQ_MAP_FLUSH;
+ case REQ_OP_DISCARD:
+ return DM_USER_REQ_MAP_DISCARD;
+ case REQ_OP_SECURE_ERASE:
+ return DM_USER_REQ_MAP_SECURE_ERASE;
+ case REQ_OP_WRITE_SAME:
+ return DM_USER_REQ_MAP_WRITE_SAME;
+ case REQ_OP_WRITE_ZEROES:
+ return DM_USER_REQ_MAP_WRITE_ZEROES;
+
+ /*
+ * These ops are not passed to userspace under the assumption that
+ * they're not going to be particularly useful in that context.
+ */
+ default:
+ return -EOPNOTSUPP;
+ }
+}
+
+static inline long bio_flags_to_user_flags(struct bio *bio)
+{
+ u64 out = 0;
+ typeof(bio->bi_opf) opf = bio->bi_opf & ~REQ_OP_MASK;
+
+ if (opf & REQ_FAILFAST_DEV) {
+ opf &= ~REQ_FAILFAST_DEV;
+ out |= DM_USER_REQ_MAP_FLAG_FAILFAST_DEV;
+ }
+
+ if (opf & REQ_FAILFAST_TRANSPORT) {
+ opf &= ~REQ_FAILFAST_TRANSPORT;
+ out |= DM_USER_REQ_MAP_FLAG_FAILFAST_TRANSPORT;
+ }
+
+ if (opf & REQ_FAILFAST_DRIVER) {
+ opf &= ~REQ_FAILFAST_DRIVER;
+ out |= DM_USER_REQ_MAP_FLAG_FAILFAST_DRIVER;
+ }
+
+ if (opf & REQ_SYNC) {
+ opf &= ~REQ_SYNC;
+ out |= DM_USER_REQ_MAP_FLAG_SYNC;
+ }
+
+ if (opf & REQ_META) {
+ opf &= ~REQ_META;
+ out |= DM_USER_REQ_MAP_FLAG_META;
+ }
+
+ if (opf & REQ_PRIO) {
+ opf &= ~REQ_PRIO;
+ out |= DM_USER_REQ_MAP_FLAG_PRIO;
+ }
+
+ if (opf & REQ_NOMERGE) {
+ opf &= ~REQ_NOMERGE;
+ out |= DM_USER_REQ_MAP_FLAG_NOMERGE;
+ }
+
+ if (opf & REQ_IDLE) {
+ opf &= ~REQ_IDLE;
+ out |= DM_USER_REQ_MAP_FLAG_IDLE;
+ }
+
+ if (opf & REQ_INTEGRITY) {
+ opf &= ~REQ_INTEGRITY;
+ out |= DM_USER_REQ_MAP_FLAG_INTEGRITY;
+ }
+
+ if (opf & REQ_FUA) {
+ opf &= ~REQ_FUA;
+ out |= DM_USER_REQ_MAP_FLAG_FUA;
+ }
+
+ if (opf & REQ_PREFLUSH) {
+ opf &= ~REQ_PREFLUSH;
+ out |= DM_USER_REQ_MAP_FLAG_PREFLUSH;
+ }
+
+ if (opf & REQ_RAHEAD) {
+ opf &= ~REQ_RAHEAD;
+ out |= DM_USER_REQ_MAP_FLAG_RAHEAD;
+ }
+
+ if (opf & REQ_BACKGROUND) {
+ opf &= ~REQ_BACKGROUND;
+ out |= DM_USER_REQ_MAP_FLAG_BACKGROUND;
+ }
+
+ if (opf & REQ_NOWAIT) {
+ opf &= ~REQ_NOWAIT;
+ out |= DM_USER_REQ_MAP_FLAG_NOWAIT;
+ }
+
+ if (opf & REQ_NOUNMAP) {
+ opf &= ~REQ_NOUNMAP;
+ out |= DM_USER_REQ_MAP_FLAG_NOUNMAP;
+ }
+
+ if (unlikely(opf)) {
+ pr_warn("unsupported BIO type %x\n", opf);
+ return -EOPNOTSUPP;
+ }
+ WARN_ON(out < 0);
+ return out;
+}
+
+/*
+ * Not quite what's in blk-map.c, but instead what I thought the functions in
+ * blk-map did. This one seems more generally useful and I think we could
+ * write the blk-map version in terms of this one. The differences are that
+ * this has a return value that counts, and blk-map uses the BIO _all iters.
+ * Neither advance the BIO iter but don't advance the IOV iter, which is a bit
+ * odd here.
+ */
+static ssize_t bio_copy_from_iter(struct bio *bio, struct iov_iter *iter)
+{
+ struct bio_vec bvec;
+ struct bvec_iter biter;
+ ssize_t out = 0;
+
+ bio_for_each_segment (bvec, bio, biter) {
+ ssize_t ret;
+
+ ret = copy_page_from_iter(bvec.bv_page, bvec.bv_offset,
+ bvec.bv_len, iter);
+
+ /*
+ * FIXME: I thought that IOV copies had a mechanism for
+ * terminating early, if for example a signal came in while
+ * sleeping waiting for a page to be mapped, but I don't see
+ * where that would happen.
+ */
+ WARN_ON(ret < 0);
+ out += ret;
+
+ if (!iov_iter_count(iter))
+ break;
+
+ if (ret < bvec.bv_len)
+ return ret;
+ }
+
+ return out;
+}
+
+static ssize_t bio_copy_to_iter(struct bio *bio, struct iov_iter *iter)
+{
+ struct bio_vec bvec;
+ struct bvec_iter biter;
+ ssize_t out = 0;
+
+ bio_for_each_segment (bvec, bio, biter) {
+ ssize_t ret;
+
+ ret = copy_page_to_iter(bvec.bv_page, bvec.bv_offset,
+ bvec.bv_len, iter);
+
+ /* as above */
+ WARN_ON(ret < 0);
+ out += ret;
+
+ if (!iov_iter_count(iter))
+ break;
+
+ if (ret < bvec.bv_len)
+ return ret;
+ }
+
+ return out;
+}
+
+static ssize_t msg_copy_to_iov(struct message *msg, struct iov_iter *to)
+{
+ ssize_t copied = 0;
+
+ if (!iov_iter_count(to))
+ return 0;
+
+ if (msg->posn_to_user < sizeof(msg->msg)) {
+ copied = copy_to_iter((char *)(&msg->msg) + msg->posn_to_user,
+ sizeof(msg->msg) - msg->posn_to_user, to);
+ } else {
+ copied = bio_copy_to_iter(msg->bio, to);
+ if (copied > 0)
+ bio_advance(msg->bio, copied);
+ }
+
+ if (copied < 0)
+ return copied;
+
+ msg->posn_to_user += copied;
+ return copied;
+}
+
+static ssize_t msg_copy_from_iov(struct message *msg, struct iov_iter *from)
+{
+ ssize_t copied = 0;
+
+ if (!iov_iter_count(from))
+ return 0;
+
+ if (msg->posn_from_user < sizeof(msg->msg)) {
+ copied = copy_from_iter(
+ (char *)(&msg->msg) + msg->posn_from_user,
+ sizeof(msg->msg) - msg->posn_from_user, from);
+ } else {
+ copied = bio_copy_from_iter(msg->bio, from);
+ if (copied > 0)
+ bio_advance(msg->bio, copied);
+ }
+
+ if (copied < 0)
+ return copied;
+
+ msg->posn_from_user += copied;
+ return copied;
+}
+
+static struct message *msg_get_map(struct target *t)
+{
+ struct message *m;
+
+ lockdep_assert_held(&t->lock);
+
+ m = mempool_alloc(&t->message_pool, GFP_NOIO);
+ m->msg.seq = t->next_seq_to_map++;
+ INIT_LIST_HEAD(&m->to_user);
+ INIT_LIST_HEAD(&m->from_user);
+ return m;
+}
+
+static struct message *msg_get_to_user(struct target *t)
+{
+ struct message *m;
+
+ lockdep_assert_held(&t->lock);
+
+ if (list_empty(&t->to_user))
+ return NULL;
+
+ m = list_first_entry(&t->to_user, struct message, to_user);
+ list_del(&m->to_user);
+ return m;
+}
+
+static struct message *msg_get_from_user(struct channel *c, u64 seq)
+{
+ struct message *m;
+ struct list_head *cur, *tmp;
+
+ lockdep_assert_held(&c->lock);
+
+ list_for_each_safe (cur, tmp, &c->from_user) {
+ m = list_entry(cur, struct message, from_user);
+ if (m->msg.seq == seq) {
+ list_del(&m->from_user);
+ return m;
+ }
+ }
+
+ return NULL;
+}
+
+static void message_kill(struct message *m, mempool_t *pool)
+{
+ m->bio->bi_status = BLK_STS_IOERR;
+ bio_endio(m->bio);
+ bio_put(m->bio);
+ mempool_free(m, pool);
+}
+
+/*
+ * Returns 0 when there is no work left to do. This must be callable without
+ * holding the target lock, as it is part of the waitqueue's check expression.
+ * When called without the lock it may spuriously indicate there is remaining
+ * work, but when called with the lock it must be accurate.
+ */
+static int target_poll(struct target *t)
+{
+ return !list_empty(&t->to_user) || t->dm_destroyed;
+}
+
+static void target_release(struct kref *ref)
+{
+ struct target *t = container_of(ref, struct target, references);
+ struct list_head *cur, *tmp;
+
+ /*
+ * There may be outstanding BIOs that have not yet been given to
+ * userspace. At this point there's nothing we can do about them, as
+ * there are and will never be any channels.
+ */
+ list_for_each_safe (cur, tmp, &t->to_user) {
+ message_kill(list_entry(cur, struct message, to_user),
+ &t->message_pool);
+ }
+
+ mempool_exit(&t->message_pool);
+ mutex_unlock(&t->lock);
+ mutex_destroy(&t->lock);
+ kfree(t);
+}
+
+static void target_put(struct target *t)
+{
+ /*
+ * This both releases a reference to the target and the lock. We leave
+ * it up to the caller to hold the lock, as they probably needed it for
+ * something else.
+ */
+ lockdep_assert_held(&t->lock);
+
+ if (!kref_put(&t->references, target_release))
+ mutex_unlock(&t->lock);
+}
+
+static struct channel *channel_alloc(struct target *t)
+{
+ struct channel *c;
+
+ lockdep_assert_held(&t->lock);
+
+ c = kzalloc(sizeof(*c), GFP_KERNEL);
+ if (c == NULL)
+ return NULL;
+
+ kref_get(&t->references);
+ c->target = t;
+ c->cur_from_user = &c->scratch_message_from_user;
+ mutex_init(&c->lock);
+ INIT_LIST_HEAD(&c->from_user);
+ return c;
+}
+
+static void channel_free(struct channel *c)
+{
+ struct list_head *cur, *tmp;
+
+ lockdep_assert_held(&c->lock);
+
+ /*
+ * There may be outstanding BIOs that have been given to userspace but
+ * have not yet been completed. The channel has been shut down so
+ * there's no way to process the rest of those messages, so we just go
+ * ahead and error out the BIOs. Hopefully whatever's on the other end
+ * can handle the errors. One could imagine splitting the BIOs and
+ * completing as much as we got, but that seems like overkill here.
+ *
+ * Our only other options would be to let the BIO hang around (which
+ * seems way worse) or to resubmit it to userspace in the hope there's
+ * another channel. I don't really like the idea of submitting a
+ * message twice.
+ */
+ if (c->cur_to_user != NULL)
+ message_kill(c->cur_to_user, &c->target->message_pool);
+ if (c->cur_from_user != &c->scratch_message_from_user)
+ message_kill(c->cur_from_user, &c->target->message_pool);
+ list_for_each_safe (cur, tmp, &c->from_user)
+ message_kill(list_entry(cur, struct message, from_user),
+ &c->target->message_pool);
+
+ mutex_lock(&c->target->lock);
+ target_put(c->target);
+ mutex_unlock(&c->lock);
+ mutex_destroy(&c->lock);
+ kfree(c);
+}
+
+static int dev_open(struct inode *inode, struct file *file)
+{
+ struct channel *c;
+ struct target *t;
+
+ /*
+ * This is called by miscdev, which sets private_data to point to the
+ * struct miscdevice that was opened. The rest of our file operations
+ * want to refer to the channel that's been opened, so we swap that
+ * pointer out with a fresh channel.
+ *
+ * This is called with the miscdev lock held, which is also held while
+ * registering/unregistering the miscdev. The miscdev must be
+ * registered for this to get called, which means there must be an
+ * outstanding reference to the target, which means it cannot be freed
+ * out from under us despite us not holding a reference yet.
+ */
+ t = container_of(file->private_data, struct target, miscdev);
+ mutex_lock(&t->lock);
+ file->private_data = c = channel_alloc(t);
+
+ if (c == NULL) {
+ mutex_unlock(&t->lock);
+ return -ENOMEM;
+ }
+
+ mutex_unlock(&t->lock);
+ return 0;
+}
+
+static ssize_t dev_read(struct kiocb *iocb, struct iov_iter *to)
+{
+ struct channel *c = channel_from_file(iocb->ki_filp);
+ ssize_t total_processed = 0;
+ ssize_t processed;
+
+ mutex_lock(&c->lock);
+
+ if (unlikely(c->to_user_error)) {
+ total_processed = c->to_user_error;
+ goto cleanup_unlock;
+ }
+
+ if (c->cur_to_user == NULL) {
+ struct target *t = target_from_channel(c);
+
+ mutex_lock(&t->lock);
+
+ while (!target_poll(t)) {
+ int e;
+
+ mutex_unlock(&t->lock);
+ mutex_unlock(&c->lock);
+ e = wait_event_interruptible(t->wq, target_poll(t));
+ mutex_lock(&c->lock);
+ mutex_lock(&t->lock);
+
+ if (unlikely(e != 0)) {
+ /*
+ * We haven't processed any bytes in either the
+ * BIO or the IOV, so we can just terminate
+ * right now. Elsewhere in the kernel handles
+ * restarting the syscall when appropriate.
+ */
+ total_processed = e;
+ mutex_unlock(&t->lock);
+ goto cleanup_unlock;
+ }
+ }
+
+ if (unlikely(t->dm_destroyed)) {
+ /*
+ * DM has destroyed this target, so just lock
+ * the user out. There's really nothing else
+ * we can do here. Note that we don't actually
+ * tear any thing down until userspace has
+ * closed the FD, as there may still be
+ * outstanding BIOs.
+ *
+ * This is kind of a wacky error code to
+ * return. My goal was really just to try and
+ * find something that wasn't likely to be
+ * returned by anything else in the miscdev
+ * path. The message "block device required"
+ * seems like a somewhat reasonable thing to
+ * say when the target has disappeared out from
+ * under us, but "not block" isn't sensible.
+ */
+ c->to_user_error = total_processed = -ENOTBLK;
+ mutex_unlock(&t->lock);
+ goto cleanup_unlock;
+ }
+
+ /*
+ * Ensures that accesses to the message data are not ordered
+ * before the remote accesses that produce that message data.
+ *
+ * This pairs with the barrier in user_map(), via the
+ * conditional within the while loop above. Also see the lack
+ * of barrier in user_dtr(), which is why this can be after the
+ * destroyed check.
+ */
+ smp_rmb();
+
+ c->cur_to_user = msg_get_to_user(t);
+ WARN_ON(c->cur_to_user == NULL);
+ mutex_unlock(&t->lock);
+ }
+
+ processed = msg_copy_to_iov(c->cur_to_user, to);
+ total_processed += processed;
+
+ WARN_ON(c->cur_to_user->posn_to_user > c->cur_to_user->total_to_user);
+ if (c->cur_to_user->posn_to_user == c->cur_to_user->total_to_user) {
+ struct message *m = c->cur_to_user;
+
+ c->cur_to_user = NULL;
+ list_add_tail(&m->from_user, &c->from_user);
+ }
+
+cleanup_unlock:
+ mutex_unlock(&c->lock);
+ return total_processed;
+}
+
+static ssize_t dev_write(struct kiocb *iocb, struct iov_iter *from)
+{
+ struct channel *c = channel_from_file(iocb->ki_filp);
+ ssize_t total_processed = 0;
+ ssize_t processed;
+
+ mutex_lock(&c->lock);
+
+ if (unlikely(c->from_user_error)) {
+ total_processed = c->from_user_error;
+ goto cleanup_unlock;
+ }
+
+ /*
+ * cur_from_user can never be NULL. If there's no real message it must
+ * point to the scratch space.
+ */
+ WARN_ON(c->cur_from_user == NULL);
+ if (c->cur_from_user->posn_from_user < sizeof(struct dm_user_message)) {
+ struct message *msg, *old;
+
+ processed = msg_copy_from_iov(c->cur_from_user, from);
+ if (processed <= 0) {
+ pr_warn("msg_copy_from_iov() returned %zu\n",
+ processed);
+ c->from_user_error = -EINVAL;
+ goto cleanup_unlock;
+ }
+ total_processed += processed;
+
+ /*
+ * In the unlikely event the user has provided us a very short
+ * write, not even big enough to fill a message, just succeed.
+ * We'll eventually build up enough bytes to do something.
+ */
+ if (unlikely(c->cur_from_user->posn_from_user <
+ sizeof(struct dm_user_message)))
+ goto cleanup_unlock;
+
+ old = c->cur_from_user;
+ mutex_lock(&c->target->lock);
+ msg = msg_get_from_user(c, c->cur_from_user->msg.seq);
+ if (msg == NULL) {
+ pr_info("user provided an invalid messag seq of %llx\n",
+ old->msg.seq);
+ mutex_unlock(&c->target->lock);
+ c->from_user_error = -EINVAL;
+ goto cleanup_unlock;
+ }
+ mutex_unlock(&c->target->lock);
+
+ WARN_ON(old->posn_from_user != sizeof(struct dm_user_message));
+ msg->posn_from_user = sizeof(struct dm_user_message);
+ msg->return_type = old->msg.type;
+ msg->return_flags = old->msg.flags;
+ WARN_ON(msg->posn_from_user > msg->total_from_user);
+ c->cur_from_user = msg;
+ WARN_ON(old != &c->scratch_message_from_user);
+ }
+
+ /*
+ * Userspace can signal an error for single requests by overwriting the
+ * seq field.
+ */
+ switch (c->cur_from_user->return_type) {
+ case DM_USER_RESP_SUCCESS:
+ c->cur_from_user->bio->bi_status = BLK_STS_OK;
+ break;
+ case DM_USER_RESP_ERROR:
+ case DM_USER_RESP_UNSUPPORTED:
+ default:
+ c->cur_from_user->bio->bi_status = BLK_STS_IOERR;
+ goto finish_bio;
+ }
+
+ /*
+ * The op was a success as far as userspace is concerned, so process
+ * whatever data may come along with it. The user may provide the BIO
+ * data in multiple chunks, in which case we don't need to finish the
+ * BIO.
+ */
+ processed = msg_copy_from_iov(c->cur_from_user, from);
+ total_processed += processed;
+
+ if (c->cur_from_user->posn_from_user <
+ c->cur_from_user->total_from_user)
+ goto cleanup_unlock;
+
+finish_bio:
+ /*
+ * When we set up this message the BIO's size matched the
+ * message size, if that's not still the case then something
+ * has gone off the rails.
+ */
+ WARN_ON(bio_size(c->cur_from_user->bio) != 0);
+ bio_endio(c->cur_from_user->bio);
+ bio_put(c->cur_from_user->bio);
+
+ /*
+ * We don't actually need to take the target lock here, as all
+ * we're doing is freeing the message and mempools have their
+ * own lock. Each channel has its ows scratch message.
+ */
+ WARN_ON(c->cur_from_user == &c->scratch_message_from_user);
+ mempool_free(c->cur_from_user, &c->target->message_pool);
+ c->scratch_message_from_user.posn_from_user = 0;
+ c->cur_from_user = &c->scratch_message_from_user;
+
+cleanup_unlock:
+ mutex_unlock(&c->lock);
+ return total_processed;
+}
+
+static int dev_release(struct inode *inode, struct file *file)
+{
+ struct channel *c;
+
+ c = channel_from_file(file);
+ mutex_lock(&c->lock);
+ channel_free(c);
+
+ return 0;
+}
+
+static const struct file_operations file_operations = {
+ .owner = THIS_MODULE,
+ .open = dev_open,
+ .llseek = no_llseek,
+ .read_iter = dev_read,
+ .write_iter = dev_write,
+ .release = dev_release,
+};
+
+static int user_ctr(struct dm_target *ti, unsigned int argc, char **argv)
+{
+ struct target *t;
+ int r;
+
+ if (argc != 3) {
+ ti->error = "Invalid argument count";
+ r = -EINVAL;
+ goto cleanup_none;
+ }
+
+ t = kzalloc(sizeof(*t), GFP_KERNEL);
+ if (t == NULL) {
+ r = -ENOMEM;
+ goto cleanup_none;
+ }
+ ti->private = t;
+
+ /* Enable more BIO types. */
+ ti->num_discard_bios = 1;
+ ti->discards_supported = true;
+ ti->num_flush_bios = 1;
+ ti->flush_supported = true;
+
+ /*
+ * We begin with a single reference to the target, which is miscdev's
+ * reference. This ensures that the target won't be freed
+ * until after the miscdev has been unregistered and all extant
+ * channels have been closed.
+ */
+ kref_init(&t->references);
+ kref_get(&t->references);
+
+ mutex_init(&t->lock);
+ init_waitqueue_head(&t->wq);
+ INIT_LIST_HEAD(&t->to_user);
+ mempool_init_kmalloc_pool(&t->message_pool, MAX_OUTSTANDING_MESSAGES,
+ sizeof(struct message));
+
+ t->miscdev.minor = MISC_DYNAMIC_MINOR;
+ t->miscdev.fops = &file_operations;
+ t->miscdev.name = kasprintf(GFP_KERNEL, "dm-user/%s", argv[2]);
+ if (t->miscdev.name == NULL) {
+ r = -ENOMEM;
+ goto cleanup_message_pool;
+ }
+
+ /*
+ * Once the miscdev is registered it can be opened and therefor
+ * concurrent references to the channel can happen. Holding the target
+ * lock during misc_register() could deadlock. If registration
+ * succeeds then we will not access the target again so we just stick a
+ * barrier here, which pairs with taking the target lock everywhere
+ * else the target is accessed.
+ *
+ * I forgot where we ended up on the RCpc/RCsc locks. IIU RCsc locks
+ * would mean that we could take the target lock earlier and release it
+ * here instead of the memory barrier. I'm not sure that's any better,
+ * though, and this isn't on a hot path so it probably doesn't matter
+ * either way.
+ */
+ smp_mb();
+
+ r = misc_register(&t->miscdev);
+ if (r) {
+ DMERR("Unable to register miscdev %s for dm-user",
+ t->miscdev.name);
+ r = -ENOMEM;
+ goto cleanup_misc_name;
+ }
+
+ return 0;
+
+cleanup_misc_name:
+ kfree(t->miscdev.name);
+cleanup_message_pool:
+ mempool_exit(&t->message_pool);
+ kfree(t);
+cleanup_none:
+ return r;
+}
+
+static void user_dtr(struct dm_target *ti)
+{
+ struct target *t = target_from_target(ti);
+
+ /*
+ * Removes the miscdev. This must be called without the target lock
+ * held to avoid a possible deadlock because our open implementation is
+ * called holding the miscdev lock and must later take the target lock.
+ *
+ * There is no race here because only DM can register/unregister the
+ * miscdev, and DM ensures that doesn't happen twice. The internal
+ * miscdev lock is sufficient to ensure there are no races between
+ * deregistering the miscdev and open.
+ */
+ misc_deregister(&t->miscdev);
+
+ /*
+ * We are now free to take the target's lock and drop our reference to
+ * the target. There are almost certainly tasks sleeping in read on at
+ * least one of the channels associated with this target, this
+ * explicitly wakes them up and terminates the read.
+ */
+ mutex_lock(&t->lock);
+ /*
+ * No barrier here, as wait/wake ensures that the flag visibility is
+ * correct WRT the wake/sleep state of the target tasks.
+ */
+ t->dm_destroyed = true;
+ wake_up_all(&t->wq);
+ target_put(t);
+}
+
+/*
+ * Consumes a BIO from device mapper, queueing it up for userspace.
+ */
+static int user_map(struct dm_target *ti, struct bio *bio)
+{
+ struct target *t;
+ struct message *entry;
+
+ t = target_from_target(ti);
+ /*
+ * FIXME
+ *
+ * This seems like a bad idea. Specifically, here we're
+ * directly on the IO path when we take the target lock, which may also
+ * be taken from a user context. The user context doesn't actively
+ * trigger anything that may sleep while holding the lock, but this
+ * still seems like a bad idea.
+ *
+ * The obvious way to fix this would be to use a proper queue, which
+ * would result in no shared locks between the direct IO path and user
+ * tasks. I had a version that did this, but the head-of-line blocking
+ * from the circular buffer resulted in us needing a fairly large
+ * allocation in order to avoid situations in which the queue fills up
+ * and everything goes off the rails.
+ *
+ * I could jump through a some hoops to avoid a shared lock while still
+ * allowing for a large queue, but I'm not actually sure that allowing
+ * for very large queues is the right thing to do here. Intuitively it
+ * seems better to keep the queues small in here (essentially sized to
+ * the user latency for performance reasons only) and rely on returning
+ * DM_MAPIO_REQUEUE regularly, as that would give the rest of the
+ * kernel more information.
+ *
+ * I'll spend some time trying to figure out what's going on with
+ * DM_MAPIO_REQUEUE, but if someone has a better idea of how to fix
+ * this I'm all ears.
+ */
+ mutex_lock(&t->lock);
+
+ /*
+ * FIXME
+ *
+ * The assumption here is that there's no benefit to returning
+ * DM_MAPIO_KILL as opposed to just erroring out the BIO, but I'm not
+ * sure that's actually true -- for example, I could imagine users
+ * expecting that submitted BIOs are unlikely to fail and therefor
+ * relying on submission failure to indicate an unsupported type.
+ *
+ * There's two ways I can think of to fix this:
+ * - Add DM arguments that are parsed during the constructor that
+ * allow various dm_target flags to be set that indicate the op
+ * types supported by this target. This may make sense for things
+ * like discard, where DM can already transform the BIOs to a form
+ * that's likely to be supported.
+ * - Some sort of pre-filter that allows userspace to hook in here
+ * and kill BIOs before marking them as submitted. My guess would
+ * be that a userspace round trip is a bad idea here, but a BPF
+ * call seems resonable.
+ *
+ * My guess is that we'd likely want to do both. The first one is easy
+ * and gives DM the proper info, so it seems better. The BPF call
+ * seems overly complex for just this, but one could imagine wanting to
+ * sometimes return _MAPPED and a BPF filter would be the way to do
+ * that.
+ *
+ * For example, in Android we have an in-kernel DM device called
+ * "dm-bow" that takes advange of some portion of the space that has
+ * been discarded on a device to provide opportunistic block-level
+ * backups. While one could imagine just implementing this entirely in
+ * userspace, that would come with an appreciable performance penalty.
+ * Instead one could keep a BPF program that forwards most accesses
+ * directly to the backing block device while informing a userspace
+ * daemon of any discarded space and on writes to blocks that are to be
+ * backed up.
+ */
+ if (unlikely((bio_type_to_user_type(bio) < 0) ||
+ (bio_flags_to_user_flags(bio) < 0))) {
+ mutex_unlock(&t->lock);
+ return DM_MAPIO_KILL;
+ }
+
+ entry = msg_get_map(t);
+ if (unlikely(entry == NULL)) {
+ mutex_unlock(&t->lock);
+ return DM_MAPIO_REQUEUE;
+ }
+
+ bio_get(bio);
+ entry->msg.type = bio_type_to_user_type(bio);
+ entry->msg.flags = bio_flags_to_user_flags(bio);
+ entry->msg.sector = bio->bi_iter.bi_sector;
+ entry->msg.len = bio_size(bio);
+ entry->bio = bio;
+ entry->posn_to_user = 0;
+ entry->total_to_user = bio_bytes_needed_to_user(bio);
+ entry->posn_from_user = 0;
+ entry->total_from_user = bio_bytes_needed_from_user(bio);
+ /* Pairs with the barrier in dev_read() */
+ smp_wmb();
+ list_add_tail(&entry->to_user, &t->to_user);
+ wake_up_interruptible(&t->wq);
+ mutex_unlock(&t->lock);
+ return DM_MAPIO_SUBMITTED;
+}
+
+static struct target_type user_target = {
+ .name = "user",
+ .version = { 1, 0, 0 },
+ .module = THIS_MODULE,
+ .ctr = user_ctr,
+ .dtr = user_dtr,
+ .map = user_map,
+};
+
+static int __init dm_user_init(void)
+{
+ int r;
+
+ r = dm_register_target(&user_target);
+ if (r) {
+ DMERR("register failed %d", r);
+ goto error;
+ }
+
+ return 0;
+
+error:
+ return r;
+}
+
+static void __exit dm_user_exit(void)
+{
+ dm_unregister_target(&user_target);
+}
+
+module_init(dm_user_init);
+module_exit(dm_user_exit);
+MODULE_AUTHOR("Palmer Dabbelt ");
+MODULE_DESCRIPTION(DM_NAME " target returning blocks from userspace");
+MODULE_LICENSE("GPL");
diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c
index 47d756cd1f16..1ef4289ef492 100644
--- a/drivers/md/dm-verity-target.c
+++ b/drivers/md/dm-verity-target.c
@@ -62,14 +62,6 @@ struct dm_verity_prefetch_work {
struct buffer_aux {
int hash_verified;
};
-/*
- * While system shutdown, skip verity work for I/O error.
- */
-static inline bool verity_is_system_shutting_down(void)
-{
- return system_state == SYSTEM_HALT || system_state == SYSTEM_POWER_OFF
- || system_state == SYSTEM_RESTART;
-}
/*
* Initialize struct buffer_aux for a freshly created buffer.
@@ -545,6 +537,15 @@ static int verity_verify_io(struct dm_verity_io *io)
return 0;
}
+/*
+ * Skip verity work in response to I/O error when system is shutting down.
+ */
+static inline bool verity_is_system_shutting_down(void)
+{
+ return system_state == SYSTEM_HALT || system_state == SYSTEM_POWER_OFF
+ || system_state == SYSTEM_RESTART;
+}
+
/*
* End one "io" structure with a given error.
*/
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index 1116097bd780..3b6b2b8b7324 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -517,7 +517,7 @@ static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
* subset of the parent bdev; require extra privileges.
*/
if (!capable(CAP_SYS_RAWIO)) {
- DMWARN_LIMIT(
+ DMDEBUG_LIMIT(
"%s: sending ioctl %x to DM device without required privilege.",
current->comm, cmd);
r = -ENOIOCTLCMD;
diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c
index 02c5e390f89f..8e0f936b3e37 100644
--- a/drivers/md/raid10.c
+++ b/drivers/md/raid10.c
@@ -1138,7 +1138,7 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio,
struct md_rdev *err_rdev = NULL;
gfp_t gfp = GFP_NOIO;
- if (r10_bio->devs[slot].rdev) {
+ if (slot >= 0 && r10_bio->devs[slot].rdev) {
/*
* This is an error retry, but we cannot
* safely dereference the rdev in the r10_bio,
@@ -1547,6 +1547,7 @@ static void __make_request(struct mddev *mddev, struct bio *bio, int sectors)
r10_bio->mddev = mddev;
r10_bio->sector = bio->bi_iter.bi_sector;
r10_bio->state = 0;
+ r10_bio->read_slot = -1;
memset(r10_bio->devs, 0, sizeof(r10_bio->devs[0]) * conf->copies);
if (bio_data_dir(bio) == READ)
diff --git a/drivers/media/usb/dvb-usb/gp8psk.c b/drivers/media/usb/dvb-usb/gp8psk.c
index 13e96b0aeb0f..d97eab01cb8c 100644
--- a/drivers/media/usb/dvb-usb/gp8psk.c
+++ b/drivers/media/usb/dvb-usb/gp8psk.c
@@ -185,7 +185,7 @@ out_rel_fw:
static int gp8psk_power_ctrl(struct dvb_usb_device *d, int onoff)
{
- u8 status, buf;
+ u8 status = 0, buf;
int gp_product_id = le16_to_cpu(d->udev->descriptor.idProduct);
if (onoff) {
diff --git a/drivers/misc/vmw_vmci/vmci_context.c b/drivers/misc/vmw_vmci/vmci_context.c
index bc089e634a75..26e20b091160 100644
--- a/drivers/misc/vmw_vmci/vmci_context.c
+++ b/drivers/misc/vmw_vmci/vmci_context.c
@@ -751,7 +751,7 @@ static int vmci_ctx_get_chkpt_doorbells(struct vmci_ctx *context,
return VMCI_ERROR_MORE_DATA;
}
- dbells = kmalloc(data_size, GFP_ATOMIC);
+ dbells = kzalloc(data_size, GFP_ATOMIC);
if (!dbells)
return VMCI_ERROR_NO_MEM;
diff --git a/drivers/mmc/core/queue.c b/drivers/mmc/core/queue.c
index 33ed018d3a90..6c658faa8c20 100644
--- a/drivers/mmc/core/queue.c
+++ b/drivers/mmc/core/queue.c
@@ -383,8 +383,10 @@ static void mmc_setup_queue(struct mmc_queue *mq, struct mmc_card *card)
if (host->ops->init)
host->ops->init(host);
- if (mmc_card_mmc(card))
+ if (mmc_card_mmc(card) && card->ext_csd.data_sector_size) {
block_size = card->ext_csd.data_sector_size;
+ WARN_ON(block_size != 512 && block_size != 4096);
+ }
blk_queue_logical_block_size(mq->queue, block_size);
blk_queue_max_segment_size(mq->queue,
diff --git a/drivers/mmc/host/sdhci-xenon.c b/drivers/mmc/host/sdhci-xenon.c
index fafb02644efd..ca34fa424634 100644
--- a/drivers/mmc/host/sdhci-xenon.c
+++ b/drivers/mmc/host/sdhci-xenon.c
@@ -170,7 +170,12 @@ static void xenon_reset_exit(struct sdhci_host *host,
/* Disable tuning request and auto-retuning again */
xenon_retune_setup(host);
- xenon_set_acg(host, true);
+ /*
+ * The ACG should be turned off at the early init time, in order
+ * to solve a possible issues with the 1.8V regulator stabilization.
+ * The feature is enabled in later stage.
+ */
+ xenon_set_acg(host, false);
xenon_set_sdclk_off_idle(host, sdhc_id, false);
diff --git a/drivers/mtd/nand/spi/core.c b/drivers/mtd/nand/spi/core.c
index 83954f424d41..1d61ae7aaa66 100644
--- a/drivers/mtd/nand/spi/core.c
+++ b/drivers/mtd/nand/spi/core.c
@@ -378,10 +378,6 @@ static int spinand_write_to_cache_op(struct spinand_device *spinand,
}
}
- if (req->ooblen)
- memcpy(req->oobbuf.in, spinand->oobbuf + req->ooboffs,
- req->ooblen);
-
return 0;
}
diff --git a/drivers/net/can/dev.c b/drivers/net/can/dev.c
index f88590074569..953c6fdc75cc 100644
--- a/drivers/net/can/dev.c
+++ b/drivers/net/can/dev.c
@@ -579,11 +579,11 @@ static void can_restart(struct net_device *dev)
}
cf->can_id |= CAN_ERR_RESTARTED;
- netif_rx_ni(skb);
-
stats->rx_packets++;
stats->rx_bytes += cf->can_dlc;
+ netif_rx_ni(skb);
+
restart:
netdev_dbg(dev, "restarted\n");
priv->can_stats.restarts++;
diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_fd.c b/drivers/net/can/usb/peak_usb/pcan_usb_fd.c
index 19600d35aac5..40ac37fe9dcd 100644
--- a/drivers/net/can/usb/peak_usb/pcan_usb_fd.c
+++ b/drivers/net/can/usb/peak_usb/pcan_usb_fd.c
@@ -520,11 +520,11 @@ static int pcan_usb_fd_decode_canmsg(struct pcan_usb_fd_if *usb_if,
else
memcpy(cfd->data, rm->d, cfd->len);
- peak_usb_netif_rx(skb, &usb_if->time_ref, le32_to_cpu(rm->ts_low));
-
netdev->stats.rx_packets++;
netdev->stats.rx_bytes += cfd->len;
+ peak_usb_netif_rx(skb, &usb_if->time_ref, le32_to_cpu(rm->ts_low));
+
return 0;
}
@@ -586,11 +586,11 @@ static int pcan_usb_fd_decode_status(struct pcan_usb_fd_if *usb_if,
if (!skb)
return -ENOMEM;
- peak_usb_netif_rx(skb, &usb_if->time_ref, le32_to_cpu(sm->ts_low));
-
netdev->stats.rx_packets++;
netdev->stats.rx_bytes += cf->can_dlc;
+ peak_usb_netif_rx(skb, &usb_if->time_ref, le32_to_cpu(sm->ts_low));
+
return 0;
}
diff --git a/drivers/net/can/vxcan.c b/drivers/net/can/vxcan.c
index ed6828821fbd..ccd758ba3fb0 100644
--- a/drivers/net/can/vxcan.c
+++ b/drivers/net/can/vxcan.c
@@ -49,6 +49,7 @@ static netdev_tx_t vxcan_xmit(struct sk_buff *skb, struct net_device *dev)
struct net_device *peer;
struct canfd_frame *cfd = (struct canfd_frame *)skb->data;
struct net_device_stats *peerstats, *srcstats = &dev->stats;
+ u8 len;
if (can_dropped_invalid_skb(dev, skb))
return NETDEV_TX_OK;
@@ -71,12 +72,13 @@ static netdev_tx_t vxcan_xmit(struct sk_buff *skb, struct net_device *dev)
skb->dev = peer;
skb->ip_summed = CHECKSUM_UNNECESSARY;
+ len = cfd->len;
if (netif_rx_ni(skb) == NET_RX_SUCCESS) {
srcstats->tx_packets++;
- srcstats->tx_bytes += cfd->len;
+ srcstats->tx_bytes += len;
peerstats = &peer->stats;
peerstats->rx_packets++;
- peerstats->rx_bytes += cfd->len;
+ peerstats->rx_bytes += len;
}
out_unlock:
diff --git a/drivers/net/dsa/b53/b53_common.c b/drivers/net/dsa/b53/b53_common.c
index 294be86420b6..335ce1e84904 100644
--- a/drivers/net/dsa/b53/b53_common.c
+++ b/drivers/net/dsa/b53/b53_common.c
@@ -1142,7 +1142,7 @@ int b53_vlan_prepare(struct dsa_switch *ds, int port,
if ((is5325(dev) || is5365(dev)) && vlan->vid_begin == 0)
return -EOPNOTSUPP;
- if (vlan->vid_end > dev->num_vlans)
+ if (vlan->vid_end >= dev->num_vlans)
return -ERANGE;
b53_enable_vlan(dev, true, dev->vlan_filtering_enabled);
diff --git a/drivers/net/ethernet/broadcom/bcmsysport.c b/drivers/net/ethernet/broadcom/bcmsysport.c
index 9a614c5cdfa2..0c69becc3c17 100644
--- a/drivers/net/ethernet/broadcom/bcmsysport.c
+++ b/drivers/net/ethernet/broadcom/bcmsysport.c
@@ -2507,6 +2507,7 @@ static int bcm_sysport_probe(struct platform_device *pdev)
/* HW supported features, none enabled by default */
dev->hw_features |= NETIF_F_RXCSUM | NETIF_F_HIGHDMA |
NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
+ dev->max_mtu = UMAC_MAX_MTU_SIZE;
/* Request the WOL interrupt and advertise suspend if available */
priv->wol_irq_disabled = 1;
diff --git a/drivers/net/ethernet/ethoc.c b/drivers/net/ethernet/ethoc.c
index 60da0499ad66..a2280c4be0f5 100644
--- a/drivers/net/ethernet/ethoc.c
+++ b/drivers/net/ethernet/ethoc.c
@@ -1213,7 +1213,7 @@ static int ethoc_probe(struct platform_device *pdev)
ret = mdiobus_register(priv->mdio);
if (ret) {
dev_err(&netdev->dev, "failed to register MDIO bus\n");
- goto free2;
+ goto free3;
}
ret = ethoc_mdio_probe(netdev);
@@ -1245,6 +1245,7 @@ error2:
netif_napi_del(&priv->napi);
error:
mdiobus_unregister(priv->mdio);
+free3:
mdiobus_free(priv->mdio);
free2:
clk_disable_unprepare(priv->clk);
diff --git a/drivers/net/ethernet/freescale/fs_enet/mii-bitbang.c b/drivers/net/ethernet/freescale/fs_enet/mii-bitbang.c
index c8e5d889bd81..21de56345503 100644
--- a/drivers/net/ethernet/freescale/fs_enet/mii-bitbang.c
+++ b/drivers/net/ethernet/freescale/fs_enet/mii-bitbang.c
@@ -223,3 +223,4 @@ static struct platform_driver fs_enet_bb_mdio_driver = {
};
module_platform_driver(fs_enet_bb_mdio_driver);
+MODULE_LICENSE("GPL");
diff --git a/drivers/net/ethernet/freescale/fs_enet/mii-fec.c b/drivers/net/ethernet/freescale/fs_enet/mii-fec.c
index 1582d82483ec..4e6a9c5d8af5 100644
--- a/drivers/net/ethernet/freescale/fs_enet/mii-fec.c
+++ b/drivers/net/ethernet/freescale/fs_enet/mii-fec.c
@@ -224,3 +224,4 @@ static struct platform_driver fs_enet_fec_mdio_driver = {
};
module_platform_driver(fs_enet_fec_mdio_driver);
+MODULE_LICENSE("GPL");
diff --git a/drivers/net/ethernet/freescale/ucc_geth.c b/drivers/net/ethernet/freescale/ucc_geth.c
index 5de6f7c73c1f..d43173917ce4 100644
--- a/drivers/net/ethernet/freescale/ucc_geth.c
+++ b/drivers/net/ethernet/freescale/ucc_geth.c
@@ -3902,6 +3902,7 @@ static int ucc_geth_probe(struct platform_device* ofdev)
INIT_WORK(&ugeth->timeout_work, ucc_geth_timeout_work);
netif_napi_add(dev, &ugeth->napi, ucc_geth_poll, 64);
dev->mtu = 1500;
+ dev->max_mtu = 1518;
ugeth->msg_enable = netif_msg_init(debug.msg_enable, UGETH_MSG_DEFAULT);
ugeth->phy_interface = phy_interface;
@@ -3947,12 +3948,12 @@ static int ucc_geth_remove(struct platform_device* ofdev)
struct device_node *np = ofdev->dev.of_node;
unregister_netdev(dev);
- free_netdev(dev);
ucc_geth_memclean(ugeth);
if (of_phy_is_fixed_link(np))
of_phy_deregister_fixed_link(np);
of_node_put(ugeth->ug_info->tbi_node);
of_node_put(ugeth->ug_info->phy_node);
+ free_netdev(dev);
return 0;
}
diff --git a/drivers/net/ethernet/freescale/ucc_geth.h b/drivers/net/ethernet/freescale/ucc_geth.h
index 5da19b440a6a..bf25e49d4fe3 100644
--- a/drivers/net/ethernet/freescale/ucc_geth.h
+++ b/drivers/net/ethernet/freescale/ucc_geth.h
@@ -580,7 +580,14 @@ struct ucc_geth_tx_global_pram {
u32 vtagtable[0x8]; /* 8 4-byte VLAN tags */
u32 tqptr; /* a base pointer to the Tx Queues Memory
Region */
- u8 res2[0x80 - 0x74];
+ u8 res2[0x78 - 0x74];
+ u64 snums_en;
+ u32 l2l3baseptr; /* top byte consists of a few other bit fields */
+
+ u16 mtu[8];
+ u8 res3[0xa8 - 0x94];
+ u32 wrrtablebase; /* top byte is reserved */
+ u8 res4[0xc0 - 0xac];
} __packed;
/* structure representing Extended Filtering Global Parameters in PRAM */
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c b/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c
index 1fa0cd527ead..f453cebf758c 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c
@@ -419,6 +419,10 @@ static void __lb_other_process(struct hns_nic_ring_data *ring_data,
/* for mutl buffer*/
new_skb = skb_copy(skb, GFP_ATOMIC);
dev_kfree_skb_any(skb);
+ if (!new_skb) {
+ netdev_err(ndev, "skb alloc failed\n");
+ return;
+ }
skb = new_skb;
check_ok = 0;
diff --git a/drivers/net/ethernet/hisilicon/hns3/hclge_mbx.h b/drivers/net/ethernet/hisilicon/hns3/hclge_mbx.h
index be9dc08ccf67..4f56ffcdfc93 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hclge_mbx.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hclge_mbx.h
@@ -102,7 +102,7 @@ struct hclgevf_mbx_arq_ring {
#define hclge_mbx_ring_ptr_move_crq(crq) \
(crq->next_to_use = (crq->next_to_use + 1) % crq->desc_num)
#define hclge_mbx_tail_ptr_move_arq(arq) \
- (arq.tail = (arq.tail + 1) % HCLGE_MBX_MAX_ARQ_MSG_SIZE)
+ (arq.tail = (arq.tail + 1) % HCLGE_MBX_MAX_ARQ_MSG_NUM)
#define hclge_mbx_head_ptr_move_arq(arq) \
- (arq.head = (arq.head + 1) % HCLGE_MBX_MAX_ARQ_MSG_SIZE)
+ (arq.head = (arq.head + 1) % HCLGE_MBX_MAX_ARQ_MSG_NUM)
#endif
diff --git a/drivers/net/ethernet/intel/i40e/i40e.h b/drivers/net/ethernet/intel/i40e/i40e.h
index 3ad46d1f58f6..738acba7a9a3 100644
--- a/drivers/net/ethernet/intel/i40e/i40e.h
+++ b/drivers/net/ethernet/intel/i40e/i40e.h
@@ -127,6 +127,7 @@ enum i40e_state_t {
__I40E_RESET_INTR_RECEIVED,
__I40E_REINIT_REQUESTED,
__I40E_PF_RESET_REQUESTED,
+ __I40E_PF_RESET_AND_REBUILD_REQUESTED,
__I40E_CORE_RESET_REQUESTED,
__I40E_GLOBAL_RESET_REQUESTED,
__I40E_EMP_RESET_REQUESTED,
@@ -153,6 +154,8 @@ enum i40e_state_t {
};
#define I40E_PF_RESET_FLAG BIT_ULL(__I40E_PF_RESET_REQUESTED)
+#define I40E_PF_RESET_AND_REBUILD_FLAG \
+ BIT_ULL(__I40E_PF_RESET_AND_REBUILD_REQUESTED)
/* VSI state flags */
enum i40e_vsi_state_t {
diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c
index 985601a65def..a728b6a7872c 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_main.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_main.c
@@ -42,6 +42,8 @@ static int i40e_setup_misc_vector(struct i40e_pf *pf);
static void i40e_determine_queue_usage(struct i40e_pf *pf);
static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired);
+static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
+ bool lock_acquired);
static int i40e_reset(struct i40e_pf *pf);
static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired);
static void i40e_fdir_sb_setup(struct i40e_pf *pf);
@@ -7929,6 +7931,14 @@ void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired)
dev_dbg(&pf->pdev->dev, "PFR requested\n");
i40e_handle_reset_warning(pf, lock_acquired);
+ } else if (reset_flags & I40E_PF_RESET_AND_REBUILD_FLAG) {
+ /* Request a PF Reset
+ *
+ * Resets PF and reinitializes PFs VSI.
+ */
+ i40e_prep_for_reset(pf, lock_acquired);
+ i40e_reset_and_rebuild(pf, true, lock_acquired);
+
} else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) {
int v;
diff --git a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
index dd0c9604d3c9..5d782148d35f 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
@@ -1567,7 +1567,7 @@ int i40e_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
if (num_vfs) {
if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
- i40e_do_reset_safe(pf, I40E_PF_RESET_FLAG);
+ i40e_do_reset_safe(pf, I40E_PF_RESET_AND_REBUILD_FLAG);
}
return i40e_pci_sriov_enable(pdev, num_vfs);
}
@@ -1575,7 +1575,7 @@ int i40e_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
if (!pci_vfs_assigned(pf->pdev)) {
i40e_free_vfs(pf);
pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
- i40e_do_reset_safe(pf, I40E_PF_RESET_FLAG);
+ i40e_do_reset_safe(pf, I40E_PF_RESET_AND_REBUILD_FLAG);
} else {
dev_warn(&pdev->dev, "Unable to free VFs because some are assigned to VMs.\n");
return -EINVAL;
diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
index e4e43519710d..bc5cfe062b10 100644
--- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
+++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
@@ -954,7 +954,7 @@ static void mvpp22_gop_init_rgmii(struct mvpp2_port *port)
regmap_read(priv->sysctrl_base, GENCONF_CTRL0, &val);
if (port->gop_id == 2)
- val |= GENCONF_CTRL0_PORT0_RGMII | GENCONF_CTRL0_PORT1_RGMII;
+ val |= GENCONF_CTRL0_PORT0_RGMII;
else if (port->gop_id == 3)
val |= GENCONF_CTRL0_PORT1_RGMII_MII;
regmap_write(priv->sysctrl_base, GENCONF_CTRL0, val);
@@ -4266,8 +4266,6 @@ static void mvpp2_phylink_validate(struct net_device *dev,
phylink_set(mask, Autoneg);
phylink_set_port_modes(mask);
- phylink_set(mask, Pause);
- phylink_set(mask, Asym_Pause);
switch (state->interface) {
case PHY_INTERFACE_MODE_10GKR:
diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_prs.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_prs.c
index 5692c6087bbb..a30eb90ba3d2 100644
--- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_prs.c
+++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_prs.c
@@ -405,6 +405,38 @@ static int mvpp2_prs_tcam_first_free(struct mvpp2 *priv, unsigned char start,
return -EINVAL;
}
+/* Drop flow control pause frames */
+static void mvpp2_prs_drop_fc(struct mvpp2 *priv)
+{
+ unsigned char da[ETH_ALEN] = { 0x01, 0x80, 0xC2, 0x00, 0x00, 0x01 };
+ struct mvpp2_prs_entry pe;
+ unsigned int len;
+
+ memset(&pe, 0, sizeof(pe));
+
+ /* For all ports - drop flow control frames */
+ pe.index = MVPP2_PE_FC_DROP;
+ mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_MAC);
+
+ /* Set match on DA */
+ len = ETH_ALEN;
+ while (len--)
+ mvpp2_prs_tcam_data_byte_set(&pe, len, da[len], 0xff);
+
+ mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_DROP_MASK,
+ MVPP2_PRS_RI_DROP_MASK);
+
+ mvpp2_prs_sram_bits_set(&pe, MVPP2_PRS_SRAM_LU_GEN_BIT, 1);
+ mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_FLOWS);
+
+ /* Mask all ports */
+ mvpp2_prs_tcam_port_map_set(&pe, MVPP2_PRS_PORT_MASK);
+
+ /* Update shadow table and hw entry */
+ mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_MAC);
+ mvpp2_prs_hw_write(priv, &pe);
+}
+
/* Enable/disable dropping all mac da's */
static void mvpp2_prs_mac_drop_all_set(struct mvpp2 *priv, int port, bool add)
{
@@ -1162,6 +1194,7 @@ static void mvpp2_prs_mac_init(struct mvpp2 *priv)
mvpp2_prs_hw_write(priv, &pe);
/* Create dummy entries for drop all and promiscuous modes */
+ mvpp2_prs_drop_fc(priv);
mvpp2_prs_mac_drop_all_set(priv, 0, false);
mvpp2_prs_mac_promisc_set(priv, 0, MVPP2_PRS_L2_UNI_CAST, false);
mvpp2_prs_mac_promisc_set(priv, 0, MVPP2_PRS_L2_MULTI_CAST, false);
@@ -1647,8 +1680,9 @@ static int mvpp2_prs_pppoe_init(struct mvpp2 *priv)
mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_IP6);
mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_L3_IP6,
MVPP2_PRS_RI_L3_PROTO_MASK);
- /* Skip eth_type + 4 bytes of IPv6 header */
- mvpp2_prs_sram_shift_set(&pe, MVPP2_ETH_TYPE_LEN + 4,
+ /* Jump to DIP of IPV6 header */
+ mvpp2_prs_sram_shift_set(&pe, MVPP2_ETH_TYPE_LEN + 8 +
+ MVPP2_MAX_L3_ADDR_SIZE,
MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD);
/* Set L3 offset */
mvpp2_prs_sram_offset_set(&pe, MVPP2_PRS_SRAM_UDF_TYPE_L3,
diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_prs.h b/drivers/net/ethernet/marvell/mvpp2/mvpp2_prs.h
index e22f6c85d380..4b68dd374733 100644
--- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_prs.h
+++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_prs.h
@@ -129,7 +129,7 @@
#define MVPP2_PE_VID_EDSA_FLTR_DEFAULT (MVPP2_PRS_TCAM_SRAM_SIZE - 7)
#define MVPP2_PE_VLAN_DBL (MVPP2_PRS_TCAM_SRAM_SIZE - 6)
#define MVPP2_PE_VLAN_NONE (MVPP2_PRS_TCAM_SRAM_SIZE - 5)
-/* reserved */
+#define MVPP2_PE_FC_DROP (MVPP2_PRS_TCAM_SRAM_SIZE - 4)
#define MVPP2_PE_MAC_MC_PROMISCUOUS (MVPP2_PRS_TCAM_SRAM_SIZE - 3)
#define MVPP2_PE_MAC_UC_PROMISCUOUS (MVPP2_PRS_TCAM_SRAM_SIZE - 2)
#define MVPP2_PE_MAC_NON_PROMISCUOUS (MVPP2_PRS_TCAM_SRAM_SIZE - 1)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
index 7ddacc9e4fe4..c6eea6b6b1bb 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
@@ -893,6 +893,7 @@ static int mlx5e_create_ttc_table_groups(struct mlx5e_ttc_table *ttc,
in = kvzalloc(inlen, GFP_KERNEL);
if (!in) {
kfree(ft->g);
+ ft->g = NULL;
return -ENOMEM;
}
@@ -1033,6 +1034,7 @@ static int mlx5e_create_inner_ttc_table_groups(struct mlx5e_ttc_table *ttc)
in = kvzalloc(inlen, GFP_KERNEL);
if (!in) {
kfree(ft->g);
+ ft->g = NULL;
return -ENOMEM;
}
@@ -1312,6 +1314,7 @@ err_destroy_groups:
ft->g[ft->num_groups] = NULL;
mlx5e_destroy_groups(ft);
kvfree(in);
+ kfree(ft->g);
return err;
}
diff --git a/drivers/net/ethernet/mscc/ocelot.c b/drivers/net/ethernet/mscc/ocelot.c
index a29a6a618110..ea30da1c53f0 100644
--- a/drivers/net/ethernet/mscc/ocelot.c
+++ b/drivers/net/ethernet/mscc/ocelot.c
@@ -1549,10 +1549,8 @@ static int ocelot_netdevice_event(struct notifier_block *unused,
struct net_device *dev = netdev_notifier_info_to_dev(ptr);
int ret = 0;
- if (!ocelot_netdevice_dev_check(dev))
- return 0;
-
if (event == NETDEV_PRECHANGEUPPER &&
+ ocelot_netdevice_dev_check(dev) &&
netif_is_lag_master(info->upper_dev)) {
struct netdev_lag_upper_info *lag_upper_info = info->upper_info;
struct netlink_ext_ack *extack;
diff --git a/drivers/net/ethernet/natsemi/macsonic.c b/drivers/net/ethernet/natsemi/macsonic.c
index 0937fc2a928e..23c9394cd5d2 100644
--- a/drivers/net/ethernet/natsemi/macsonic.c
+++ b/drivers/net/ethernet/natsemi/macsonic.c
@@ -540,10 +540,14 @@ static int mac_sonic_platform_probe(struct platform_device *pdev)
err = register_netdev(dev);
if (err)
- goto out;
+ goto undo_probe;
return 0;
+undo_probe:
+ dma_free_coherent(lp->device,
+ SIZEOF_SONIC_DESC * SONIC_BUS_SCALE(lp->dma_bitmode),
+ lp->descriptors, lp->descriptors_laddr);
out:
free_netdev(dev);
@@ -618,12 +622,16 @@ static int mac_sonic_nubus_probe(struct nubus_board *board)
err = register_netdev(ndev);
if (err)
- goto out;
+ goto undo_probe;
nubus_set_drvdata(board, ndev);
return 0;
+undo_probe:
+ dma_free_coherent(lp->device,
+ SIZEOF_SONIC_DESC * SONIC_BUS_SCALE(lp->dma_bitmode),
+ lp->descriptors, lp->descriptors_laddr);
out:
free_netdev(ndev);
return err;
diff --git a/drivers/net/ethernet/natsemi/xtsonic.c b/drivers/net/ethernet/natsemi/xtsonic.c
index e1b886e87a76..44171d7bb434 100644
--- a/drivers/net/ethernet/natsemi/xtsonic.c
+++ b/drivers/net/ethernet/natsemi/xtsonic.c
@@ -265,11 +265,14 @@ int xtsonic_probe(struct platform_device *pdev)
sonic_msg_init(dev);
if ((err = register_netdev(dev)))
- goto out1;
+ goto undo_probe1;
return 0;
-out1:
+undo_probe1:
+ dma_free_coherent(lp->device,
+ SIZEOF_SONIC_DESC * SONIC_BUS_SCALE(lp->dma_bitmode),
+ lp->descriptors, lp->descriptors_laddr);
release_region(dev->base_addr, SONIC_MEM_SIZE);
out:
free_netdev(dev);
diff --git a/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c b/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c
index 59c70be22a84..42b99b182616 100644
--- a/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c
+++ b/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c
@@ -580,11 +580,6 @@ static const struct net_device_ops netxen_netdev_ops = {
.ndo_set_features = netxen_set_features,
};
-static inline bool netxen_function_zero(struct pci_dev *pdev)
-{
- return (PCI_FUNC(pdev->devfn) == 0) ? true : false;
-}
-
static inline void netxen_set_interrupt_mode(struct netxen_adapter *adapter,
u32 mode)
{
@@ -680,7 +675,7 @@ static int netxen_setup_intr(struct netxen_adapter *adapter)
netxen_initialize_interrupt_registers(adapter);
netxen_set_msix_bit(pdev, 0);
- if (netxen_function_zero(pdev)) {
+ if (adapter->portnum == 0) {
if (!netxen_setup_msi_interrupts(adapter, num_msix))
netxen_set_interrupt_mode(adapter, NETXEN_MSI_MODE);
else
diff --git a/drivers/net/ethernet/qlogic/qede/qede_fp.c b/drivers/net/ethernet/qlogic/qede/qede_fp.c
index a96da16f3404..1976279800cd 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_fp.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_fp.c
@@ -1747,6 +1747,11 @@ netdev_features_t qede_features_check(struct sk_buff *skb,
ntohs(udp_hdr(skb)->dest) != gnv_port))
return features & ~(NETIF_F_CSUM_MASK |
NETIF_F_GSO_MASK);
+ } else if (l4_proto == IPPROTO_IPIP) {
+ /* IPIP tunnels are unknown to the device or at least unsupported natively,
+ * offloads for them can't be done trivially, so disable them for such skb.
+ */
+ return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
}
}
diff --git a/drivers/net/ethernet/realtek/r8169.c b/drivers/net/ethernet/realtek/r8169.c
index c7ce167c67a0..6b901bf1cd54 100644
--- a/drivers/net/ethernet/realtek/r8169.c
+++ b/drivers/net/ethernet/realtek/r8169.c
@@ -4237,7 +4237,8 @@ static void r8168_pll_power_down(struct rtl8169_private *tp)
return;
switch (tp->mac_version) {
- case RTL_GIGA_MAC_VER_25 ... RTL_GIGA_MAC_VER_33:
+ case RTL_GIGA_MAC_VER_25 ... RTL_GIGA_MAC_VER_26:
+ case RTL_GIGA_MAC_VER_32 ... RTL_GIGA_MAC_VER_33:
case RTL_GIGA_MAC_VER_37:
case RTL_GIGA_MAC_VER_39:
case RTL_GIGA_MAC_VER_43:
@@ -4263,7 +4264,8 @@ static void r8168_pll_power_down(struct rtl8169_private *tp)
static void r8168_pll_power_up(struct rtl8169_private *tp)
{
switch (tp->mac_version) {
- case RTL_GIGA_MAC_VER_25 ... RTL_GIGA_MAC_VER_33:
+ case RTL_GIGA_MAC_VER_25 ... RTL_GIGA_MAC_VER_26:
+ case RTL_GIGA_MAC_VER_32 ... RTL_GIGA_MAC_VER_33:
case RTL_GIGA_MAC_VER_37:
case RTL_GIGA_MAC_VER_39:
case RTL_GIGA_MAC_VER_43:
diff --git a/drivers/net/ethernet/renesas/sh_eth.c b/drivers/net/ethernet/renesas/sh_eth.c
index 441643670ac0..9ac7b09badca 100644
--- a/drivers/net/ethernet/renesas/sh_eth.c
+++ b/drivers/net/ethernet/renesas/sh_eth.c
@@ -2620,10 +2620,10 @@ static int sh_eth_close(struct net_device *ndev)
/* Free all the skbuffs in the Rx queue and the DMA buffer. */
sh_eth_ring_free(ndev);
- pm_runtime_put_sync(&mdp->pdev->dev);
-
mdp->is_opened = 0;
+ pm_runtime_put(&mdp->pdev->dev);
+
return 0;
}
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c
index ef13a462c36d..1e5e831718db 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c
@@ -73,6 +73,7 @@ struct emac_variant {
* @variant: reference to the current board variant
* @regmap: regmap for using the syscon
* @internal_phy_powered: Does the internal PHY is enabled
+ * @use_internal_phy: Is the internal PHY selected for use
* @mux_handle: Internal pointer used by mdio-mux lib
*/
struct sunxi_priv_data {
@@ -83,6 +84,7 @@ struct sunxi_priv_data {
const struct emac_variant *variant;
struct regmap_field *regmap_field;
bool internal_phy_powered;
+ bool use_internal_phy;
void *mux_handle;
};
@@ -518,8 +520,11 @@ static const struct stmmac_dma_ops sun8i_dwmac_dma_ops = {
.dma_interrupt = sun8i_dwmac_dma_interrupt,
};
+static int sun8i_dwmac_power_internal_phy(struct stmmac_priv *priv);
+
static int sun8i_dwmac_init(struct platform_device *pdev, void *priv)
{
+ struct net_device *ndev = platform_get_drvdata(pdev);
struct sunxi_priv_data *gmac = priv;
int ret;
@@ -533,13 +538,25 @@ static int sun8i_dwmac_init(struct platform_device *pdev, void *priv)
ret = clk_prepare_enable(gmac->tx_clk);
if (ret) {
- if (gmac->regulator)
- regulator_disable(gmac->regulator);
dev_err(&pdev->dev, "Could not enable AHB clock\n");
- return ret;
+ goto err_disable_regulator;
+ }
+
+ if (gmac->use_internal_phy) {
+ ret = sun8i_dwmac_power_internal_phy(netdev_priv(ndev));
+ if (ret)
+ goto err_disable_clk;
}
return 0;
+
+err_disable_clk:
+ clk_disable_unprepare(gmac->tx_clk);
+err_disable_regulator:
+ if (gmac->regulator)
+ regulator_disable(gmac->regulator);
+
+ return ret;
}
static void sun8i_dwmac_core_init(struct mac_device_info *hw,
@@ -809,7 +826,6 @@ static int mdio_mux_syscon_switch_fn(int current_child, int desired_child,
struct sunxi_priv_data *gmac = priv->plat->bsp_priv;
u32 reg, val;
int ret = 0;
- bool need_power_ephy = false;
if (current_child ^ desired_child) {
regmap_field_read(gmac->regmap_field, ®);
@@ -817,13 +833,12 @@ static int mdio_mux_syscon_switch_fn(int current_child, int desired_child,
case DWMAC_SUN8I_MDIO_MUX_INTERNAL_ID:
dev_info(priv->device, "Switch mux to internal PHY");
val = (reg & ~H3_EPHY_MUX_MASK) | H3_EPHY_SELECT;
-
- need_power_ephy = true;
+ gmac->use_internal_phy = true;
break;
case DWMAC_SUN8I_MDIO_MUX_EXTERNAL_ID:
dev_info(priv->device, "Switch mux to external PHY");
val = (reg & ~H3_EPHY_MUX_MASK) | H3_EPHY_SHUTDOWN;
- need_power_ephy = false;
+ gmac->use_internal_phy = false;
break;
default:
dev_err(priv->device, "Invalid child ID %x\n",
@@ -831,7 +846,7 @@ static int mdio_mux_syscon_switch_fn(int current_child, int desired_child,
return -EINVAL;
}
regmap_field_write(gmac->regmap_field, val);
- if (need_power_ephy) {
+ if (gmac->use_internal_phy) {
ret = sun8i_dwmac_power_internal_phy(priv);
if (ret)
return ret;
@@ -977,17 +992,12 @@ static void sun8i_dwmac_exit(struct platform_device *pdev, void *priv)
struct sunxi_priv_data *gmac = priv;
if (gmac->variant->soc_has_internal_phy) {
- /* sun8i_dwmac_exit could be called with mdiomux uninit */
- if (gmac->mux_handle)
- mdio_mux_uninit(gmac->mux_handle);
if (gmac->internal_phy_powered)
sun8i_dwmac_unpower_internal_phy(gmac);
}
sun8i_dwmac_unset_syscon(gmac);
- reset_control_put(gmac->rst_ephy);
-
clk_disable_unprepare(gmac->tx_clk);
if (gmac->regulator)
@@ -1200,12 +1210,32 @@ static int sun8i_dwmac_probe(struct platform_device *pdev)
return ret;
dwmac_mux:
+ reset_control_put(gmac->rst_ephy);
+ clk_put(gmac->ephy_clk);
sun8i_dwmac_unset_syscon(gmac);
dwmac_exit:
stmmac_pltfr_remove(pdev);
return ret;
}
+static int sun8i_dwmac_remove(struct platform_device *pdev)
+{
+ struct net_device *ndev = platform_get_drvdata(pdev);
+ struct stmmac_priv *priv = netdev_priv(ndev);
+ struct sunxi_priv_data *gmac = priv->plat->bsp_priv;
+
+ if (gmac->variant->soc_has_internal_phy) {
+ mdio_mux_uninit(gmac->mux_handle);
+ sun8i_dwmac_unpower_internal_phy(gmac);
+ reset_control_put(gmac->rst_ephy);
+ clk_put(gmac->ephy_clk);
+ }
+
+ stmmac_pltfr_remove(pdev);
+
+ return 0;
+}
+
static const struct of_device_id sun8i_dwmac_match[] = {
{ .compatible = "allwinner,sun8i-h3-emac",
.data = &emac_variant_h3 },
@@ -1223,7 +1253,7 @@ MODULE_DEVICE_TABLE(of, sun8i_dwmac_match);
static struct platform_driver sun8i_dwmac_driver = {
.probe = sun8i_dwmac_probe,
- .remove = stmmac_pltfr_remove,
+ .remove = sun8i_dwmac_remove,
.driver = {
.name = "dwmac-sun8i",
.pm = &stmmac_pltfr_pm_ops,
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
index 4ac507b4d101..76d4b8e6ac3e 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
@@ -3596,6 +3596,7 @@ static int stmmac_change_mtu(struct net_device *dev, int new_mtu)
{
struct stmmac_priv *priv = netdev_priv(dev);
int txfifosz = priv->plat->tx_fifo_size;
+ const int mtu = new_mtu;
if (txfifosz == 0)
txfifosz = priv->dma_cap.tx_fifo_size;
@@ -3613,7 +3614,7 @@ static int stmmac_change_mtu(struct net_device *dev, int new_mtu)
if ((txfifosz < new_mtu) || (new_mtu > BUF_SIZE_16KiB))
return -EINVAL;
- dev->mtu = new_mtu;
+ dev->mtu = mtu;
netdev_update_features(dev);
diff --git a/drivers/net/ethernet/ti/cpts.c b/drivers/net/ethernet/ti/cpts.c
index d7543811dfae..10b301e79086 100644
--- a/drivers/net/ethernet/ti/cpts.c
+++ b/drivers/net/ethernet/ti/cpts.c
@@ -476,6 +476,7 @@ void cpts_unregister(struct cpts *cpts)
ptp_clock_unregister(cpts->clock);
cpts->clock = NULL;
+ cpts->phc_index = -1;
cpts_write32(cpts, 0, int_enable);
cpts_write32(cpts, 0, control);
@@ -577,6 +578,7 @@ struct cpts *cpts_create(struct device *dev, void __iomem *regs,
cpts->cc.read = cpts_systim_read;
cpts->cc.mask = CLOCKSOURCE_MASK(32);
cpts->info = cpts_info;
+ cpts->phc_index = -1;
cpts_calc_mult_shift(cpts);
/* save cc.mult original value as it can be modified
diff --git a/drivers/net/tun.c b/drivers/net/tun.c
index a8392f24f95a..2ac83701db93 100644
--- a/drivers/net/tun.c
+++ b/drivers/net/tun.c
@@ -1450,7 +1450,7 @@ static struct sk_buff *tun_napi_alloc_frags(struct tun_file *tfile,
int i;
if (it->nr_segs > MAX_SKB_FRAGS + 1)
- return ERR_PTR(-ENOMEM);
+ return ERR_PTR(-EMSGSIZE);
local_bh_disable();
skb = napi_get_frags(&tfile->napi);
diff --git a/drivers/net/usb/cdc_ether.c b/drivers/net/usb/cdc_ether.c
index 1de97b69ce4e..529c8fac1531 100644
--- a/drivers/net/usb/cdc_ether.c
+++ b/drivers/net/usb/cdc_ether.c
@@ -800,6 +800,13 @@ static const struct usb_device_id products[] = {
.driver_info = 0,
},
+/* Lenovo Powered USB-C Travel Hub (4X90S92381, based on Realtek RTL8153) */
+{
+ USB_DEVICE_AND_INTERFACE_INFO(LENOVO_VENDOR_ID, 0x721e, USB_CLASS_COMM,
+ USB_CDC_SUBCLASS_ETHERNET, USB_CDC_PROTO_NONE),
+ .driver_info = 0,
+},
+
/* ThinkPad USB-C Dock Gen 2 (based on Realtek RTL8153) */
{
USB_DEVICE_AND_INTERFACE_INFO(LENOVO_VENDOR_ID, 0xa387, USB_CLASS_COMM,
diff --git a/drivers/net/usb/cdc_ncm.c b/drivers/net/usb/cdc_ncm.c
index 1f57a6a2b8a2..faca70c3647d 100644
--- a/drivers/net/usb/cdc_ncm.c
+++ b/drivers/net/usb/cdc_ncm.c
@@ -1127,7 +1127,10 @@ cdc_ncm_fill_tx_frame(struct usbnet *dev, struct sk_buff *skb, __le32 sign)
* accordingly. Otherwise, we should check here.
*/
if (ctx->drvflags & CDC_NCM_FLAG_NDP_TO_END)
- delayed_ndp_size = ALIGN(ctx->max_ndp_size, ctx->tx_ndp_modulus);
+ delayed_ndp_size = ctx->max_ndp_size +
+ max_t(u32,
+ ctx->tx_ndp_modulus,
+ ctx->tx_modulus + ctx->tx_remainder) - 1;
else
delayed_ndp_size = 0;
@@ -1308,7 +1311,8 @@ cdc_ncm_fill_tx_frame(struct usbnet *dev, struct sk_buff *skb, __le32 sign)
if (!(dev->driver_info->flags & FLAG_SEND_ZLP) &&
skb_out->len > ctx->min_tx_pkt) {
padding_count = ctx->tx_curr_size - skb_out->len;
- skb_put_zero(skb_out, padding_count);
+ if (!WARN_ON(padding_count > ctx->tx_curr_size))
+ skb_put_zero(skb_out, padding_count);
} else if (skb_out->len < ctx->tx_curr_size &&
(skb_out->len % dev->maxpacket) == 0) {
skb_put_u8(skb_out, 0); /* force short packet */
@@ -1629,9 +1633,6 @@ static void cdc_ncm_status(struct usbnet *dev, struct urb *urb)
* USB_CDC_NOTIFY_NETWORK_CONNECTION notification shall be
* sent by device after USB_CDC_NOTIFY_SPEED_CHANGE.
*/
- netif_info(dev, link, dev->net,
- "network connection: %sconnected\n",
- !!event->wValue ? "" : "dis");
usbnet_link_change(dev, !!event->wValue, 0);
break;
diff --git a/drivers/net/usb/qmi_wwan.c b/drivers/net/usb/qmi_wwan.c
index ebd630a94571..1b48d71dbc28 100644
--- a/drivers/net/usb/qmi_wwan.c
+++ b/drivers/net/usb/qmi_wwan.c
@@ -995,6 +995,7 @@ static const struct usb_device_id products[] = {
{QMI_MATCH_FF_FF_FF(0x2c7c, 0x0125)}, /* Quectel EC25, EC20 R2.0 Mini PCIe */
{QMI_MATCH_FF_FF_FF(0x2c7c, 0x0306)}, /* Quectel EP06/EG06/EM06 */
{QMI_MATCH_FF_FF_FF(0x2c7c, 0x0512)}, /* Quectel EG12/EM12 */
+ {QMI_MATCH_FF_FF_FF(0x2c7c, 0x0620)}, /* Quectel EM160R-GL */
{QMI_MATCH_FF_FF_FF(0x2c7c, 0x0800)}, /* Quectel RM500Q-GL */
/* 3. Combined interface devices matching on interface number */
diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c
index 1b1ec4197830..7dc605585535 100644
--- a/drivers/net/usb/r8152.c
+++ b/drivers/net/usb/r8152.c
@@ -5352,6 +5352,7 @@ static const struct usb_device_id rtl8152_table[] = {
{REALTEK_USB_DEVICE(VENDOR_ID_LENOVO, 0x7205)},
{REALTEK_USB_DEVICE(VENDOR_ID_LENOVO, 0x720c)},
{REALTEK_USB_DEVICE(VENDOR_ID_LENOVO, 0x7214)},
+ {REALTEK_USB_DEVICE(VENDOR_ID_LENOVO, 0x721e)},
{REALTEK_USB_DEVICE(VENDOR_ID_LENOVO, 0xa387)},
{REALTEK_USB_DEVICE(VENDOR_ID_LINKSYS, 0x0041)},
{REALTEK_USB_DEVICE(VENDOR_ID_NVIDIA, 0x09ff)},
diff --git a/drivers/net/usb/rndis_host.c b/drivers/net/usb/rndis_host.c
index a22ae3137a3f..d3f79a4067e2 100644
--- a/drivers/net/usb/rndis_host.c
+++ b/drivers/net/usb/rndis_host.c
@@ -399,7 +399,7 @@ generic_rndis_bind(struct usbnet *dev, struct usb_interface *intf, int flags)
reply_len = sizeof *phym;
retval = rndis_query(dev, intf, u.buf,
RNDIS_OID_GEN_PHYSICAL_MEDIUM,
- 0, (void **) &phym, &reply_len);
+ reply_len, (void **)&phym, &reply_len);
if (retval != 0 || !phym) {
/* OID is optional so don't fail here. */
phym_unspec = cpu_to_le32(RNDIS_PHYSICAL_MEDIUM_UNSPECIFIED);
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index b21223be93c8..d41d5f63f211 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -2077,14 +2077,16 @@ static int virtnet_set_channels(struct net_device *dev,
get_online_cpus();
err = _virtnet_set_queues(vi, queue_pairs);
- if (!err) {
- netif_set_real_num_tx_queues(dev, queue_pairs);
- netif_set_real_num_rx_queues(dev, queue_pairs);
-
- virtnet_set_affinity(vi);
+ if (err) {
+ put_online_cpus();
+ goto err;
}
+ virtnet_set_affinity(vi);
put_online_cpus();
+ netif_set_real_num_tx_queues(dev, queue_pairs);
+ netif_set_real_num_rx_queues(dev, queue_pairs);
+ err:
return err;
}
diff --git a/drivers/net/wan/Kconfig b/drivers/net/wan/Kconfig
index 21190dfbabb1..17ed5107b90a 100644
--- a/drivers/net/wan/Kconfig
+++ b/drivers/net/wan/Kconfig
@@ -295,6 +295,7 @@ config SLIC_DS26522
tristate "Slic Maxim ds26522 card support"
depends on SPI
depends on FSL_SOC || ARCH_MXC || ARCH_LAYERSCAPE || COMPILE_TEST
+ select BITREVERSE
help
This module initializes and configures the slic maxim card
in T1 or E1 mode.
diff --git a/drivers/net/wan/hdlc_ppp.c b/drivers/net/wan/hdlc_ppp.c
index 20d9b6585fba..d67623d61dc4 100644
--- a/drivers/net/wan/hdlc_ppp.c
+++ b/drivers/net/wan/hdlc_ppp.c
@@ -572,6 +572,13 @@ static void ppp_timer(struct timer_list *t)
unsigned long flags;
spin_lock_irqsave(&ppp->lock, flags);
+ /* mod_timer could be called after we entered this function but
+ * before we got the lock.
+ */
+ if (timer_pending(&proto->timer)) {
+ spin_unlock_irqrestore(&ppp->lock, flags);
+ return;
+ }
switch (proto->state) {
case STOPPING:
case REQ_SENT:
diff --git a/drivers/net/wireless/ath/wil6210/Kconfig b/drivers/net/wireless/ath/wil6210/Kconfig
index 1c0bf47f55d4..2bb03396f992 100644
--- a/drivers/net/wireless/ath/wil6210/Kconfig
+++ b/drivers/net/wireless/ath/wil6210/Kconfig
@@ -1,6 +1,7 @@
config WIL6210
tristate "Wilocity 60g WiFi card wil6210 support"
select WANT_DEV_COREDUMP
+ select CRC32
depends on CFG80211
depends on PCI
default n
diff --git a/drivers/net/wireless/marvell/mwifiex/join.c b/drivers/net/wireless/marvell/mwifiex/join.c
index d87aeff70cef..c2cb1e711c06 100644
--- a/drivers/net/wireless/marvell/mwifiex/join.c
+++ b/drivers/net/wireless/marvell/mwifiex/join.c
@@ -877,6 +877,8 @@ mwifiex_cmd_802_11_ad_hoc_start(struct mwifiex_private *priv,
memset(adhoc_start->ssid, 0, IEEE80211_MAX_SSID_LEN);
+ if (req_ssid->ssid_len > IEEE80211_MAX_SSID_LEN)
+ req_ssid->ssid_len = IEEE80211_MAX_SSID_LEN;
memcpy(adhoc_start->ssid, req_ssid->ssid, req_ssid->ssid_len);
mwifiex_dbg(adapter, INFO, "info: ADHOC_S_CMD: SSID = %s\n",
diff --git a/drivers/platform/x86/intel-vbtn.c b/drivers/platform/x86/intel-vbtn.c
index 36d6e72f5073..f5774372c387 100644
--- a/drivers/platform/x86/intel-vbtn.c
+++ b/drivers/platform/x86/intel-vbtn.c
@@ -191,12 +191,6 @@ static const struct dmi_system_id dmi_switches_allow_list[] = {
DMI_MATCH(DMI_PRODUCT_NAME, "Venue 11 Pro 7130"),
},
},
- {
- .matches = {
- DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
- DMI_MATCH(DMI_PRODUCT_NAME, "HP Stream x360 Convertible PC 11"),
- },
- },
{
.matches = {
DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
diff --git a/drivers/rtc/rtc-sun6i.c b/drivers/rtc/rtc-sun6i.c
index 2cd5a7b1a2e3..e85abe805606 100644
--- a/drivers/rtc/rtc-sun6i.c
+++ b/drivers/rtc/rtc-sun6i.c
@@ -232,7 +232,7 @@ static void __init sun6i_rtc_clk_init(struct device_node *node)
300000000);
if (IS_ERR(rtc->int_osc)) {
pr_crit("Couldn't register the internal oscillator\n");
- return;
+ goto err;
}
parents[0] = clk_hw_get_name(rtc->int_osc);
@@ -248,7 +248,7 @@ static void __init sun6i_rtc_clk_init(struct device_node *node)
rtc->losc = clk_register(NULL, &rtc->hw);
if (IS_ERR(rtc->losc)) {
pr_crit("Couldn't register the LOSC clock\n");
- return;
+ goto err_register;
}
of_property_read_string_index(node, "clock-output-names", 1,
@@ -259,7 +259,7 @@ static void __init sun6i_rtc_clk_init(struct device_node *node)
&rtc->lock);
if (IS_ERR(rtc->ext_losc)) {
pr_crit("Couldn't register the LOSC external gate\n");
- return;
+ goto err_register;
}
clk_data->num = 2;
@@ -268,6 +268,8 @@ static void __init sun6i_rtc_clk_init(struct device_node *node)
of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data);
return;
+err_register:
+ clk_hw_unregister_fixed_rate(rtc->int_osc);
err:
kfree(clk_data);
}
diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c
index 83d25ee88f02..8877a21102f1 100644
--- a/drivers/scsi/megaraid/megaraid_sas_base.c
+++ b/drivers/scsi/megaraid/megaraid_sas_base.c
@@ -7323,11 +7323,9 @@ megasas_mgmt_fw_ioctl(struct megasas_instance *instance,
goto out;
}
+ /* always store 64 bits regardless of addressing */
sense_ptr = (void *)cmd->frame + ioc->sense_off;
- if (instance->consistent_mask_64bit)
- put_unaligned_le64(sense_handle, sense_ptr);
- else
- put_unaligned_le32(sense_handle, sense_ptr);
+ put_unaligned_le64(sense_handle, sense_ptr);
}
/*
diff --git a/drivers/scsi/qedi/qedi_main.c b/drivers/scsi/qedi/qedi_main.c
index eaa50328de90..e201c163ea1c 100644
--- a/drivers/scsi/qedi/qedi_main.c
+++ b/drivers/scsi/qedi/qedi_main.c
@@ -2129,7 +2129,7 @@ qedi_show_boot_tgt_info(struct qedi_ctx *qedi, int type,
chap_name);
break;
case ISCSI_BOOT_TGT_CHAP_SECRET:
- rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_CHAP_NAME_MAX_LEN,
+ rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_CHAP_PWD_MAX_LEN,
chap_secret);
break;
case ISCSI_BOOT_TGT_REV_CHAP_NAME:
@@ -2137,7 +2137,7 @@ qedi_show_boot_tgt_info(struct qedi_ctx *qedi, int type,
mchap_name);
break;
case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
- rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_CHAP_NAME_MAX_LEN,
+ rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_CHAP_PWD_MAX_LEN,
mchap_secret);
break;
case ISCSI_BOOT_TGT_FLAGS:
diff --git a/drivers/scsi/scsi_transport_spi.c b/drivers/scsi/scsi_transport_spi.c
index 69213842e63e..efb9c3d90213 100644
--- a/drivers/scsi/scsi_transport_spi.c
+++ b/drivers/scsi/scsi_transport_spi.c
@@ -130,12 +130,16 @@ static int spi_execute(struct scsi_device *sdev, const void *cmd,
sshdr = &sshdr_tmp;
for(i = 0; i < DV_RETRIES; i++) {
+ /*
+ * The purpose of the RQF_PM flag below is to bypass the
+ * SDEV_QUIESCE state.
+ */
result = scsi_execute(sdev, cmd, dir, buffer, bufflen, sense,
sshdr, DV_TIMEOUT, /* retries */ 1,
REQ_FAILFAST_DEV |
REQ_FAILFAST_TRANSPORT |
REQ_FAILFAST_DRIVER,
- 0, NULL);
+ RQF_PM, NULL);
if (driver_byte(result) != DRIVER_SENSE ||
sshdr->sense_key != UNIT_ATTENTION)
break;
@@ -1018,23 +1022,26 @@ spi_dv_device(struct scsi_device *sdev)
*/
lock_system_sleep();
+ if (scsi_autopm_get_device(sdev))
+ goto unlock_system_sleep;
+
if (unlikely(spi_dv_in_progress(starget)))
- goto unlock;
+ goto put_autopm;
if (unlikely(scsi_device_get(sdev)))
- goto unlock;
+ goto put_autopm;
spi_dv_in_progress(starget) = 1;
buffer = kzalloc(len, GFP_KERNEL);
if (unlikely(!buffer))
- goto out_put;
+ goto put_sdev;
/* We need to verify that the actual device will quiesce; the
* later target quiesce is just a nice to have */
if (unlikely(scsi_device_quiesce(sdev)))
- goto out_free;
+ goto free_buffer;
scsi_target_quiesce(starget);
@@ -1054,12 +1061,16 @@ spi_dv_device(struct scsi_device *sdev)
spi_initial_dv(starget) = 1;
- out_free:
+free_buffer:
kfree(buffer);
- out_put:
+
+put_sdev:
spi_dv_in_progress(starget) = 0;
scsi_device_put(sdev);
-unlock:
+put_autopm:
+ scsi_autopm_put_device(sdev);
+
+unlock_system_sleep:
unlock_system_sleep();
}
EXPORT_SYMBOL(spi_dv_device);
diff --git a/drivers/scsi/ufs/ufshcd-pci.c b/drivers/scsi/ufs/ufshcd-pci.c
index ffe6f82182ba..68f4f67c5ff8 100644
--- a/drivers/scsi/ufs/ufshcd-pci.c
+++ b/drivers/scsi/ufs/ufshcd-pci.c
@@ -96,6 +96,30 @@ static int ufshcd_pci_resume(struct device *dev)
{
return ufshcd_system_resume(dev_get_drvdata(dev));
}
+
+/**
+ * ufshcd_pci_poweroff - suspend-to-disk poweroff function
+ * @dev: pointer to PCI device handle
+ *
+ * Returns 0 if successful
+ * Returns non-zero otherwise
+ */
+static int ufshcd_pci_poweroff(struct device *dev)
+{
+ struct ufs_hba *hba = dev_get_drvdata(dev);
+ int spm_lvl = hba->spm_lvl;
+ int ret;
+
+ /*
+ * For poweroff we need to set the UFS device to PowerDown mode.
+ * Force spm_lvl to ensure that.
+ */
+ hba->spm_lvl = 5;
+ ret = ufshcd_system_suspend(hba);
+ hba->spm_lvl = spm_lvl;
+ return ret;
+}
+
#endif /* !CONFIG_PM_SLEEP */
#ifdef CONFIG_PM
@@ -190,8 +214,14 @@ ufshcd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
}
static const struct dev_pm_ops ufshcd_pci_pm_ops = {
- SET_SYSTEM_SLEEP_PM_OPS(ufshcd_pci_suspend,
- ufshcd_pci_resume)
+#ifdef CONFIG_PM_SLEEP
+ .suspend = ufshcd_pci_suspend,
+ .resume = ufshcd_pci_resume,
+ .freeze = ufshcd_pci_suspend,
+ .thaw = ufshcd_pci_resume,
+ .poweroff = ufshcd_pci_poweroff,
+ .restore = ufshcd_pci_resume,
+#endif
SET_RUNTIME_PM_OPS(ufshcd_pci_runtime_suspend,
ufshcd_pci_runtime_resume,
ufshcd_pci_runtime_idle)
diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c
index f8a9d28d138e..35539533ca6d 100644
--- a/drivers/scsi/ufs/ufshcd.c
+++ b/drivers/scsi/ufs/ufshcd.c
@@ -4963,7 +4963,7 @@ static int ufshcd_dme_enable(struct ufs_hba *hba)
ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
if (ret)
dev_err(hba->dev,
- "dme-reset: error code %d\n", ret);
+ "dme-enable: error code %d\n", ret);
return ret;
}
@@ -7887,19 +7887,16 @@ static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
{
struct Scsi_Host *host;
struct ufs_hba *hba;
- unsigned int tag;
u32 pos;
int err;
- u8 resp = 0xF;
- struct ufshcd_lrb *lrbp;
+ u8 resp = 0xF, lun;
unsigned long flags;
host = cmd->device->host;
hba = shost_priv(host);
- tag = cmd->request->tag;
- lrbp = &hba->lrb[tag];
- err = ufshcd_issue_tm_cmd(hba, lrbp->lun, 0, UFS_LOGICAL_RESET, &resp);
+ lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
+ err = ufshcd_issue_tm_cmd(hba, lun, 0, UFS_LOGICAL_RESET, &resp);
if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
if (!err)
err = resp;
@@ -7908,7 +7905,7 @@ static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
/* clear the commands that were pending for corresponding LUN */
for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs) {
- if (hba->lrb[pos].lun == lrbp->lun) {
+ if (hba->lrb[pos].lun == lun) {
err = ufshcd_clear_cmd(hba, pos);
if (err)
break;
diff --git a/drivers/spi/spi-cadence.c b/drivers/spi/spi-cadence.c
index 94cc0a152449..f5055ceb7529 100644
--- a/drivers/spi/spi-cadence.c
+++ b/drivers/spi/spi-cadence.c
@@ -119,6 +119,7 @@ struct cdns_spi {
void __iomem *regs;
struct clk *ref_clk;
struct clk *pclk;
+ unsigned int clk_rate;
u32 speed_hz;
const u8 *txbuf;
u8 *rxbuf;
@@ -258,7 +259,7 @@ static void cdns_spi_config_clock_freq(struct spi_device *spi,
u32 ctrl_reg, baud_rate_val;
unsigned long frequency;
- frequency = clk_get_rate(xspi->ref_clk);
+ frequency = xspi->clk_rate;
ctrl_reg = cdns_spi_read(xspi, CDNS_SPI_CR);
@@ -628,8 +629,9 @@ static int cdns_spi_probe(struct platform_device *pdev)
master->auto_runtime_pm = true;
master->mode_bits = SPI_CPOL | SPI_CPHA;
+ xspi->clk_rate = clk_get_rate(xspi->ref_clk);
/* Set to default valid value */
- master->max_speed_hz = clk_get_rate(xspi->ref_clk) / 4;
+ master->max_speed_hz = xspi->clk_rate / 4;
xspi->speed_hz = master->max_speed_hz;
master->bits_per_word_mask = SPI_BPW_MASK(8);
diff --git a/drivers/spi/spi-pxa2xx.c b/drivers/spi/spi-pxa2xx.c
index eafd0c2135a1..a889505e978b 100644
--- a/drivers/spi/spi-pxa2xx.c
+++ b/drivers/spi/spi-pxa2xx.c
@@ -1572,7 +1572,7 @@ static int pxa2xx_spi_probe(struct platform_device *pdev)
return -ENODEV;
}
- master = spi_alloc_master(dev, sizeof(struct driver_data));
+ master = devm_spi_alloc_master(dev, sizeof(*drv_data));
if (!master) {
dev_err(&pdev->dev, "cannot alloc spi_master\n");
pxa_ssp_free(ssp);
@@ -1759,7 +1759,6 @@ out_error_dma_irq_alloc:
free_irq(ssp->irq, drv_data);
out_error_master_alloc:
- spi_controller_put(master);
pxa_ssp_free(ssp);
return status;
}
diff --git a/drivers/spi/spi-stm32.c b/drivers/spi/spi-stm32.c
index 391a20b3d2fd..8eb2644506dd 100644
--- a/drivers/spi/spi-stm32.c
+++ b/drivers/spi/spi-stm32.c
@@ -299,9 +299,9 @@ static u32 stm32_spi_prepare_fthlv(struct stm32_spi *spi)
/* align packet size with data registers access */
if (spi->cur_bpw > 8)
- fthlv -= (fthlv % 2); /* multiple of 2 */
+ fthlv += (fthlv % 2) ? 1 : 0;
else
- fthlv -= (fthlv % 4); /* multiple of 4 */
+ fthlv += (fthlv % 4) ? (4 - (fthlv % 4)) : 0;
return fthlv;
}
diff --git a/drivers/staging/mt7621-dma/mtk-hsdma.c b/drivers/staging/mt7621-dma/mtk-hsdma.c
index 5831f816c17b..f60302b71181 100644
--- a/drivers/staging/mt7621-dma/mtk-hsdma.c
+++ b/drivers/staging/mt7621-dma/mtk-hsdma.c
@@ -723,7 +723,7 @@ static int mtk_hsdma_probe(struct platform_device *pdev)
ret = dma_async_device_register(dd);
if (ret) {
dev_err(&pdev->dev, "failed to register dma device\n");
- return ret;
+ goto err_uninit_hsdma;
}
ret = of_dma_controller_register(pdev->dev.of_node,
@@ -739,6 +739,8 @@ static int mtk_hsdma_probe(struct platform_device *pdev)
err_unregister:
dma_async_device_unregister(dd);
+err_uninit_hsdma:
+ mtk_hsdma_uninit(hsdma);
return ret;
}
diff --git a/drivers/target/target_core_xcopy.c b/drivers/target/target_core_xcopy.c
index 7cdb5d7f6538..1709b8a99bd7 100644
--- a/drivers/target/target_core_xcopy.c
+++ b/drivers/target/target_core_xcopy.c
@@ -55,60 +55,83 @@ static int target_xcopy_gen_naa_ieee(struct se_device *dev, unsigned char *buf)
return 0;
}
-struct xcopy_dev_search_info {
- const unsigned char *dev_wwn;
- struct se_device *found_dev;
-};
-
+/**
+ * target_xcopy_locate_se_dev_e4_iter - compare XCOPY NAA device identifiers
+ *
+ * @se_dev: device being considered for match
+ * @dev_wwn: XCOPY requested NAA dev_wwn
+ * @return: 1 on match, 0 on no-match
+ */
static int target_xcopy_locate_se_dev_e4_iter(struct se_device *se_dev,
- void *data)
+ const unsigned char *dev_wwn)
{
- struct xcopy_dev_search_info *info = data;
unsigned char tmp_dev_wwn[XCOPY_NAA_IEEE_REGEX_LEN];
int rc;
- if (!se_dev->dev_attrib.emulate_3pc)
+ if (!se_dev->dev_attrib.emulate_3pc) {
+ pr_debug("XCOPY: emulate_3pc disabled on se_dev %p\n", se_dev);
return 0;
+ }
memset(&tmp_dev_wwn[0], 0, XCOPY_NAA_IEEE_REGEX_LEN);
target_xcopy_gen_naa_ieee(se_dev, &tmp_dev_wwn[0]);
- rc = memcmp(&tmp_dev_wwn[0], info->dev_wwn, XCOPY_NAA_IEEE_REGEX_LEN);
- if (rc != 0)
+ rc = memcmp(&tmp_dev_wwn[0], dev_wwn, XCOPY_NAA_IEEE_REGEX_LEN);
+ if (rc != 0) {
+ pr_debug("XCOPY: skip non-matching: %*ph\n",
+ XCOPY_NAA_IEEE_REGEX_LEN, tmp_dev_wwn);
return 0;
-
- info->found_dev = se_dev;
+ }
pr_debug("XCOPY 0xe4: located se_dev: %p\n", se_dev);
- rc = target_depend_item(&se_dev->dev_group.cg_item);
- if (rc != 0) {
- pr_err("configfs_depend_item attempt failed: %d for se_dev: %p\n",
- rc, se_dev);
- return rc;
- }
-
- pr_debug("Called configfs_depend_item for se_dev: %p se_dev->se_dev_group: %p\n",
- se_dev, &se_dev->dev_group);
return 1;
}
-static int target_xcopy_locate_se_dev_e4(const unsigned char *dev_wwn,
- struct se_device **found_dev)
+static int target_xcopy_locate_se_dev_e4(struct se_session *sess,
+ const unsigned char *dev_wwn,
+ struct se_device **_found_dev,
+ struct percpu_ref **_found_lun_ref)
{
- struct xcopy_dev_search_info info;
- int ret;
+ struct se_dev_entry *deve;
+ struct se_node_acl *nacl;
+ struct se_lun *this_lun = NULL;
+ struct se_device *found_dev = NULL;
- memset(&info, 0, sizeof(info));
- info.dev_wwn = dev_wwn;
+ /* cmd with NULL sess indicates no associated $FABRIC_MOD */
+ if (!sess)
+ goto err_out;
- ret = target_for_each_device(target_xcopy_locate_se_dev_e4_iter, &info);
- if (ret == 1) {
- *found_dev = info.found_dev;
- return 0;
- } else {
- pr_debug_ratelimited("Unable to locate 0xe4 descriptor for EXTENDED_COPY\n");
- return -EINVAL;
+ pr_debug("XCOPY 0xe4: searching for: %*ph\n",
+ XCOPY_NAA_IEEE_REGEX_LEN, dev_wwn);
+
+ nacl = sess->se_node_acl;
+ rcu_read_lock();
+ hlist_for_each_entry_rcu(deve, &nacl->lun_entry_hlist, link) {
+ struct se_device *this_dev;
+ int rc;
+
+ this_lun = rcu_dereference(deve->se_lun);
+ this_dev = rcu_dereference_raw(this_lun->lun_se_dev);
+
+ rc = target_xcopy_locate_se_dev_e4_iter(this_dev, dev_wwn);
+ if (rc) {
+ if (percpu_ref_tryget_live(&this_lun->lun_ref))
+ found_dev = this_dev;
+ break;
+ }
}
+ rcu_read_unlock();
+ if (found_dev == NULL)
+ goto err_out;
+
+ pr_debug("lun_ref held for se_dev: %p se_dev->se_dev_group: %p\n",
+ found_dev, &found_dev->dev_group);
+ *_found_dev = found_dev;
+ *_found_lun_ref = &this_lun->lun_ref;
+ return 0;
+err_out:
+ pr_debug_ratelimited("Unable to locate 0xe4 descriptor for EXTENDED_COPY\n");
+ return -EINVAL;
}
static int target_xcopy_parse_tiddesc_e4(struct se_cmd *se_cmd, struct xcopy_op *xop,
@@ -255,12 +278,16 @@ static int target_xcopy_parse_target_descriptors(struct se_cmd *se_cmd,
switch (xop->op_origin) {
case XCOL_SOURCE_RECV_OP:
- rc = target_xcopy_locate_se_dev_e4(xop->dst_tid_wwn,
- &xop->dst_dev);
+ rc = target_xcopy_locate_se_dev_e4(se_cmd->se_sess,
+ xop->dst_tid_wwn,
+ &xop->dst_dev,
+ &xop->remote_lun_ref);
break;
case XCOL_DEST_RECV_OP:
- rc = target_xcopy_locate_se_dev_e4(xop->src_tid_wwn,
- &xop->src_dev);
+ rc = target_xcopy_locate_se_dev_e4(se_cmd->se_sess,
+ xop->src_tid_wwn,
+ &xop->src_dev,
+ &xop->remote_lun_ref);
break;
default:
pr_err("XCOPY CSCD descriptor IDs not found in CSCD list - "
@@ -412,18 +439,12 @@ static int xcopy_pt_get_cmd_state(struct se_cmd *se_cmd)
static void xcopy_pt_undepend_remotedev(struct xcopy_op *xop)
{
- struct se_device *remote_dev;
-
if (xop->op_origin == XCOL_SOURCE_RECV_OP)
- remote_dev = xop->dst_dev;
+ pr_debug("putting dst lun_ref for %p\n", xop->dst_dev);
else
- remote_dev = xop->src_dev;
+ pr_debug("putting src lun_ref for %p\n", xop->src_dev);
- pr_debug("Calling configfs_undepend_item for"
- " remote_dev: %p remote_dev->dev_group: %p\n",
- remote_dev, &remote_dev->dev_group.cg_item);
-
- target_undepend_item(&remote_dev->dev_group.cg_item);
+ percpu_ref_put(xop->remote_lun_ref);
}
static void xcopy_pt_release_cmd(struct se_cmd *se_cmd)
diff --git a/drivers/target/target_core_xcopy.h b/drivers/target/target_core_xcopy.h
index 26ba4c3c9cff..974bc1e19ff2 100644
--- a/drivers/target/target_core_xcopy.h
+++ b/drivers/target/target_core_xcopy.h
@@ -29,6 +29,7 @@ struct xcopy_op {
struct se_device *dst_dev;
unsigned char dst_tid_wwn[XCOPY_NAA_IEEE_REGEX_LEN];
unsigned char local_dev_wwn[XCOPY_NAA_IEEE_REGEX_LEN];
+ struct percpu_ref *remote_lun_ref;
sector_t src_lba;
sector_t dst_lba;
diff --git a/drivers/tty/serial/mvebu-uart.c b/drivers/tty/serial/mvebu-uart.c
index fb9d369e0f50..330522be708f 100644
--- a/drivers/tty/serial/mvebu-uart.c
+++ b/drivers/tty/serial/mvebu-uart.c
@@ -637,6 +637,14 @@ static void wait_for_xmitr(struct uart_port *port)
(val & STAT_TX_RDY(port)), 1, 10000);
}
+static void wait_for_xmite(struct uart_port *port)
+{
+ u32 val;
+
+ readl_poll_timeout_atomic(port->membase + UART_STAT, val,
+ (val & STAT_TX_EMP), 1, 10000);
+}
+
static void mvebu_uart_console_putchar(struct uart_port *port, int ch)
{
wait_for_xmitr(port);
@@ -664,7 +672,7 @@ static void mvebu_uart_console_write(struct console *co, const char *s,
uart_console_write(port, s, count, mvebu_uart_console_putchar);
- wait_for_xmitr(port);
+ wait_for_xmite(port);
if (ier)
writel(ier, port->membase + UART_CTRL(port));
diff --git a/drivers/usb/chipidea/ci_hdrc_imx.c b/drivers/usb/chipidea/ci_hdrc_imx.c
index 7335dc855218..af420ec2a9f4 100644
--- a/drivers/usb/chipidea/ci_hdrc_imx.c
+++ b/drivers/usb/chipidea/ci_hdrc_imx.c
@@ -128,9 +128,13 @@ static struct imx_usbmisc_data *usbmisc_get_init_data(struct device *dev)
misc_pdev = of_find_device_by_node(args.np);
of_node_put(args.np);
- if (!misc_pdev || !platform_get_drvdata(misc_pdev))
+ if (!misc_pdev)
return ERR_PTR(-EPROBE_DEFER);
+ if (!platform_get_drvdata(misc_pdev)) {
+ put_device(&misc_pdev->dev);
+ return ERR_PTR(-EPROBE_DEFER);
+ }
data->dev = &misc_pdev->dev;
if (of_find_property(np, "disable-over-current", NULL))
diff --git a/drivers/usb/class/cdc-acm.c b/drivers/usb/class/cdc-acm.c
index e0d8da4e3967..e847d0de6760 100644
--- a/drivers/usb/class/cdc-acm.c
+++ b/drivers/usb/class/cdc-acm.c
@@ -1939,6 +1939,10 @@ static const struct usb_device_id acm_ids[] = {
{ USB_DEVICE(0x04d8, 0x0083), /* Bootloader mode */
.driver_info = IGNORE_DEVICE,
},
+
+ { USB_DEVICE(0x04d8, 0xf58b),
+ .driver_info = IGNORE_DEVICE,
+ },
#endif
/*Samsung phone in firmware update mode */
diff --git a/drivers/usb/class/cdc-wdm.c b/drivers/usb/class/cdc-wdm.c
index 55ad4c43b380..ae69635bb1fb 100644
--- a/drivers/usb/class/cdc-wdm.c
+++ b/drivers/usb/class/cdc-wdm.c
@@ -465,13 +465,23 @@ static int service_outstanding_interrupt(struct wdm_device *desc)
if (!desc->resp_count || !--desc->resp_count)
goto out;
+ if (test_bit(WDM_DISCONNECTING, &desc->flags)) {
+ rv = -ENODEV;
+ goto out;
+ }
+ if (test_bit(WDM_RESETTING, &desc->flags)) {
+ rv = -EIO;
+ goto out;
+ }
+
set_bit(WDM_RESPONDING, &desc->flags);
spin_unlock_irq(&desc->iuspin);
rv = usb_submit_urb(desc->response, GFP_KERNEL);
spin_lock_irq(&desc->iuspin);
if (rv) {
- dev_err(&desc->intf->dev,
- "usb_submit_urb failed with result %d\n", rv);
+ if (!test_bit(WDM_DISCONNECTING, &desc->flags))
+ dev_err(&desc->intf->dev,
+ "usb_submit_urb failed with result %d\n", rv);
/* make sure the next notification trigger a submit */
clear_bit(WDM_RESPONDING, &desc->flags);
@@ -1026,9 +1036,9 @@ static void wdm_disconnect(struct usb_interface *intf)
wake_up_all(&desc->wait);
mutex_lock(&desc->rlock);
mutex_lock(&desc->wlock);
- kill_urbs(desc);
cancel_work_sync(&desc->rxwork);
cancel_work_sync(&desc->service_outs_intr);
+ kill_urbs(desc);
mutex_unlock(&desc->wlock);
mutex_unlock(&desc->rlock);
diff --git a/drivers/usb/class/usblp.c b/drivers/usb/class/usblp.c
index db36a796af8c..5c00d79ac7a1 100644
--- a/drivers/usb/class/usblp.c
+++ b/drivers/usb/class/usblp.c
@@ -274,8 +274,25 @@ static int usblp_ctrl_msg(struct usblp *usblp, int request, int type, int dir, i
#define usblp_reset(usblp)\
usblp_ctrl_msg(usblp, USBLP_REQ_RESET, USB_TYPE_CLASS, USB_DIR_OUT, USB_RECIP_OTHER, 0, NULL, 0)
-#define usblp_hp_channel_change_request(usblp, channel, buffer) \
- usblp_ctrl_msg(usblp, USBLP_REQ_HP_CHANNEL_CHANGE_REQUEST, USB_TYPE_VENDOR, USB_DIR_IN, USB_RECIP_INTERFACE, channel, buffer, 1)
+static int usblp_hp_channel_change_request(struct usblp *usblp, int channel, u8 *new_channel)
+{
+ u8 *buf;
+ int ret;
+
+ buf = kzalloc(1, GFP_KERNEL);
+ if (!buf)
+ return -ENOMEM;
+
+ ret = usblp_ctrl_msg(usblp, USBLP_REQ_HP_CHANNEL_CHANGE_REQUEST,
+ USB_TYPE_VENDOR, USB_DIR_IN, USB_RECIP_INTERFACE,
+ channel, buf, 1);
+ if (ret == 0)
+ *new_channel = buf[0];
+
+ kfree(buf);
+
+ return ret;
+}
/*
* See the description for usblp_select_alts() below for the usage
diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h
index 9422ea5896be..1527f71c3229 100644
--- a/drivers/usb/dwc3/core.h
+++ b/drivers/usb/dwc3/core.h
@@ -302,6 +302,7 @@
/* Global USB2 PHY Vendor Control Register */
#define DWC3_GUSB2PHYACC_NEWREGREQ BIT(25)
+#define DWC3_GUSB2PHYACC_DONE BIT(24)
#define DWC3_GUSB2PHYACC_BUSY BIT(23)
#define DWC3_GUSB2PHYACC_WRITE BIT(22)
#define DWC3_GUSB2PHYACC_ADDR(n) (n << 16)
diff --git a/drivers/usb/dwc3/ulpi.c b/drivers/usb/dwc3/ulpi.c
index f62b5f3c2d67..bb8271531da7 100644
--- a/drivers/usb/dwc3/ulpi.c
+++ b/drivers/usb/dwc3/ulpi.c
@@ -24,7 +24,7 @@ static int dwc3_ulpi_busyloop(struct dwc3 *dwc)
while (count--) {
reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYACC(0));
- if (!(reg & DWC3_GUSB2PHYACC_BUSY))
+ if (reg & DWC3_GUSB2PHYACC_DONE)
return 0;
cpu_relax();
}
diff --git a/drivers/usb/gadget/Kconfig b/drivers/usb/gadget/Kconfig
index 90681a129a63..0cd4d0d4bdf5 100644
--- a/drivers/usb/gadget/Kconfig
+++ b/drivers/usb/gadget/Kconfig
@@ -310,6 +310,7 @@ config USB_CONFIGFS_NCM
depends on NET
select USB_U_ETHER
select USB_F_NCM
+ select CRC32
help
NCM is an advanced protocol for Ethernet encapsulation, allows
grouping of several ethernet frames into one USB transfer and
@@ -381,6 +382,7 @@ config USB_CONFIGFS_EEM
depends on NET
select USB_U_ETHER
select USB_F_EEM
+ select CRC32
help
CDC EEM is a newer USB standard that is somewhat simpler than CDC ECM
and therefore can be supported by more hardware. Technically ECM and
diff --git a/drivers/usb/gadget/configfs.c b/drivers/usb/gadget/configfs.c
index 60652e81a627..581564e31d3c 100644
--- a/drivers/usb/gadget/configfs.c
+++ b/drivers/usb/gadget/configfs.c
@@ -278,9 +278,16 @@ static ssize_t gadget_dev_desc_bcdUSB_store(struct config_item *item,
static ssize_t gadget_dev_desc_UDC_show(struct config_item *item, char *page)
{
- char *udc_name = to_gadget_info(item)->composite.gadget_driver.udc_name;
+ struct gadget_info *gi = to_gadget_info(item);
+ char *udc_name;
+ int ret;
- return sprintf(page, "%s\n", udc_name ?: "");
+ mutex_lock(&gi->lock);
+ udc_name = gi->composite.gadget_driver.udc_name;
+ ret = sprintf(page, "%s\n", udc_name ?: "");
+ mutex_unlock(&gi->lock);
+
+ return ret;
}
static int unregister_gadget(struct gadget_info *gi)
@@ -1841,7 +1848,7 @@ static struct config_group *gadgets_make(
gi->composite.unbind = configfs_do_nothing;
gi->composite.suspend = NULL;
gi->composite.resume = NULL;
- gi->composite.max_speed = USB_SPEED_SUPER;
+ gi->composite.max_speed = USB_SPEED_SUPER_PLUS;
spin_lock_init(&gi->spinlock);
mutex_init(&gi->lock);
diff --git a/drivers/usb/gadget/function/f_accessory.c b/drivers/usb/gadget/function/f_accessory.c
index 3341c92a07d0..dfa2b8102314 100644
--- a/drivers/usb/gadget/function/f_accessory.c
+++ b/drivers/usb/gadget/function/f_accessory.c
@@ -27,6 +27,7 @@
#include
#include
#include
+#include
#include
#include
@@ -73,6 +74,7 @@ struct acc_dev {
struct usb_function function;
struct usb_composite_dev *cdev;
spinlock_t lock;
+ struct acc_dev_ref *ref;
struct usb_ep *ep_in;
struct usb_ep *ep_out;
@@ -80,13 +82,13 @@ struct acc_dev {
/* online indicates state of function_set_alt & function_unbind
* set to 1 when we connect
*/
- int online:1;
+ int online;
/* disconnected indicates state of open & release
* Set to 1 when we disconnect.
* Not cleared until our file is closed.
*/
- int disconnected:1;
+ int disconnected;
/* strings sent by the host */
char manufacturer[ACC_STRING_SIZE];
@@ -199,14 +201,48 @@ static struct usb_gadget_strings *acc_strings[] = {
NULL,
};
-/* temporary variable used between acc_open() and acc_gadget_bind() */
-static struct acc_dev *_acc_dev;
+struct acc_dev_ref {
+ struct kref kref;
+ struct acc_dev *acc_dev;
+};
+
+static struct acc_dev_ref _acc_dev_ref = {
+ .kref = KREF_INIT(0),
+};
struct acc_instance {
struct usb_function_instance func_inst;
const char *name;
};
+static struct acc_dev *get_acc_dev(void)
+{
+ struct acc_dev_ref *ref = &_acc_dev_ref;
+
+ return kref_get_unless_zero(&ref->kref) ? ref->acc_dev : NULL;
+}
+
+static void __put_acc_dev(struct kref *kref)
+{
+ struct acc_dev_ref *ref = container_of(kref, struct acc_dev_ref, kref);
+ struct acc_dev *dev = ref->acc_dev;
+
+ /* Cancel any async work */
+ cancel_delayed_work_sync(&dev->start_work);
+ cancel_work_sync(&dev->hid_work);
+
+ ref->acc_dev = NULL;
+ kfree(dev);
+}
+
+static void put_acc_dev(struct acc_dev *dev)
+{
+ struct acc_dev_ref *ref = dev->ref;
+
+ WARN_ON(ref->acc_dev != dev);
+ kref_put(&ref->kref, __put_acc_dev);
+}
+
static inline struct acc_dev *func_to_dev(struct usb_function *f)
{
return container_of(f, struct acc_dev, function);
@@ -272,7 +308,10 @@ static void acc_set_disconnected(struct acc_dev *dev)
static void acc_complete_in(struct usb_ep *ep, struct usb_request *req)
{
- struct acc_dev *dev = _acc_dev;
+ struct acc_dev *dev = get_acc_dev();
+
+ if (!dev)
+ return;
if (req->status == -ESHUTDOWN) {
pr_debug("acc_complete_in set disconnected");
@@ -282,11 +321,15 @@ static void acc_complete_in(struct usb_ep *ep, struct usb_request *req)
req_put(dev, &dev->tx_idle, req);
wake_up(&dev->write_wq);
+ put_acc_dev(dev);
}
static void acc_complete_out(struct usb_ep *ep, struct usb_request *req)
{
- struct acc_dev *dev = _acc_dev;
+ struct acc_dev *dev = get_acc_dev();
+
+ if (!dev)
+ return;
dev->rx_done = 1;
if (req->status == -ESHUTDOWN) {
@@ -295,6 +338,7 @@ static void acc_complete_out(struct usb_ep *ep, struct usb_request *req)
}
wake_up(&dev->read_wq);
+ put_acc_dev(dev);
}
static void acc_complete_set_string(struct usb_ep *ep, struct usb_request *req)
@@ -567,7 +611,9 @@ static ssize_t acc_read(struct file *fp, char __user *buf,
{
struct acc_dev *dev = fp->private_data;
struct usb_request *req;
- ssize_t r = count, xfer, len;
+ ssize_t r = count;
+ ssize_t data_length;
+ unsigned xfer;
int ret = 0;
pr_debug("acc_read(%zu)\n", count);
@@ -588,7 +634,14 @@ static ssize_t acc_read(struct file *fp, char __user *buf,
goto done;
}
- len = ALIGN(count, dev->ep_out->maxpacket);
+ /*
+ * Calculate the data length by considering termination character.
+ * Then compansite the difference of rounding up to
+ * integer multiple of maxpacket size.
+ */
+ data_length = count;
+ data_length += dev->ep_out->maxpacket - 1;
+ data_length -= data_length % dev->ep_out->maxpacket;
if (dev->rx_done) {
// last req cancelled. try to get it.
@@ -599,7 +652,7 @@ static ssize_t acc_read(struct file *fp, char __user *buf,
requeue_req:
/* queue a request */
req = dev->rx_req[0];
- req->length = len;
+ req->length = data_length;
dev->rx_done = 0;
ret = usb_ep_queue(dev->ep_out, req, GFP_KERNEL);
if (ret < 0) {
@@ -753,24 +806,36 @@ static long acc_ioctl(struct file *fp, unsigned code, unsigned long value)
static int acc_open(struct inode *ip, struct file *fp)
{
- printk(KERN_INFO "acc_open\n");
- if (atomic_xchg(&_acc_dev->open_excl, 1))
- return -EBUSY;
+ struct acc_dev *dev = get_acc_dev();
- _acc_dev->disconnected = 0;
- fp->private_data = _acc_dev;
+ if (!dev)
+ return -ENODEV;
+
+ if (atomic_xchg(&dev->open_excl, 1)) {
+ put_acc_dev(dev);
+ return -EBUSY;
+ }
+
+ dev->disconnected = 0;
+ fp->private_data = dev;
return 0;
}
static int acc_release(struct inode *ip, struct file *fp)
{
- printk(KERN_INFO "acc_release\n");
+ struct acc_dev *dev = fp->private_data;
+
+ if (!dev)
+ return -ENOENT;
- WARN_ON(!atomic_xchg(&_acc_dev->open_excl, 0));
/* indicate that we are disconnected
* still could be online so don't touch online flag
*/
- _acc_dev->disconnected = 1;
+ dev->disconnected = 1;
+
+ fp->private_data = NULL;
+ WARN_ON(!atomic_xchg(&dev->open_excl, 0));
+ put_acc_dev(dev);
return 0;
}
@@ -823,7 +888,7 @@ static void acc_complete_setup_noop(struct usb_ep *ep, struct usb_request *req)
int acc_ctrlrequest(struct usb_composite_dev *cdev,
const struct usb_ctrlrequest *ctrl)
{
- struct acc_dev *dev = _acc_dev;
+ struct acc_dev *dev = get_acc_dev();
int value = -EOPNOTSUPP;
struct acc_hid_dev *hid;
int offset;
@@ -840,12 +905,6 @@ int acc_ctrlrequest(struct usb_composite_dev *cdev,
*/
if (!dev)
return -ENODEV;
-/*
- printk(KERN_INFO "acc_ctrlrequest "
- "%02x.%02x v%04x i%04x l%u\n",
- b_requestType, b_request,
- w_value, w_index, w_length);
-*/
if (b_requestType == (USB_DIR_OUT | USB_TYPE_VENDOR)) {
if (b_request == ACCESSORY_START) {
@@ -934,6 +993,7 @@ err:
"%02x.%02x v%04x i%04x l%u\n",
ctrl->bRequestType, ctrl->bRequest,
w_value, w_index, w_length);
+ put_acc_dev(dev);
return value;
}
EXPORT_SYMBOL_GPL(acc_ctrlrequest);
@@ -1004,10 +1064,6 @@ kill_all_hid_devices(struct acc_dev *dev)
struct list_head *entry, *temp;
unsigned long flags;
- /* do nothing if usb accessory device doesn't exist */
- if (!dev)
- return;
-
spin_lock_irqsave(&dev->lock, flags);
list_for_each_safe(entry, temp, &dev->hid_list) {
hid = list_entry(entry, struct acc_hid_dev, list);
@@ -1092,12 +1148,15 @@ static void acc_hid_delete(struct acc_hid_dev *hid)
static void acc_hid_work(struct work_struct *data)
{
- struct acc_dev *dev = _acc_dev;
+ struct acc_dev *dev = get_acc_dev();
struct list_head *entry, *temp;
struct acc_hid_dev *hid;
struct list_head new_list, dead_list;
unsigned long flags;
+ if (!dev)
+ return;
+
INIT_LIST_HEAD(&new_list);
spin_lock_irqsave(&dev->lock, flags);
@@ -1143,6 +1202,8 @@ static void acc_hid_work(struct work_struct *data)
hid_destroy_device(hid->hid);
acc_hid_delete(hid);
}
+
+ put_acc_dev(dev);
}
static int acc_function_set_alt(struct usb_function *f,
@@ -1199,9 +1260,13 @@ static void acc_function_disable(struct usb_function *f)
static int acc_setup(void)
{
+ struct acc_dev_ref *ref = &_acc_dev_ref;
struct acc_dev *dev;
int ret;
+ if (kref_read(&ref->kref))
+ return -EBUSY;
+
dev = kzalloc(sizeof(*dev), GFP_KERNEL);
if (!dev)
return -ENOMEM;
@@ -1217,16 +1282,22 @@ static int acc_setup(void)
INIT_DELAYED_WORK(&dev->start_work, acc_start_work);
INIT_WORK(&dev->hid_work, acc_hid_work);
+ dev->ref = ref;
+ if (cmpxchg_relaxed(&ref->acc_dev, NULL, dev)) {
+ ret = -EBUSY;
+ goto err_free_dev;
+ }
+
ret = misc_register(&acc_device);
if (ret)
- goto err;
-
- /* _acc_dev must be set before calling usb_gadget_register_driver */
- _acc_dev = dev;
+ goto err_zap_ptr;
+ kref_init(&ref->kref);
return 0;
-err:
+err_zap_ptr:
+ ref->acc_dev = NULL;
+err_free_dev:
kfree(dev);
pr_err("USB accessory gadget driver failed to initialize\n");
return ret;
@@ -1234,16 +1305,24 @@ err:
void acc_disconnect(void)
{
+ struct acc_dev *dev = get_acc_dev();
+
+ if (!dev)
+ return;
+
/* unregister all HID devices if USB is disconnected */
- kill_all_hid_devices(_acc_dev);
+ kill_all_hid_devices(dev);
+ put_acc_dev(dev);
}
EXPORT_SYMBOL_GPL(acc_disconnect);
static void acc_cleanup(void)
{
+ struct acc_dev *dev = get_acc_dev();
+
misc_deregister(&acc_device);
- kfree(_acc_dev);
- _acc_dev = NULL;
+ put_acc_dev(dev);
+ put_acc_dev(dev); /* Pairs with kref_init() in acc_setup() */
}
static struct acc_instance *to_acc_instance(struct config_item *item)
{
@@ -1303,7 +1382,6 @@ static void acc_free_inst(struct usb_function_instance *fi)
static struct usb_function_instance *acc_alloc_inst(void)
{
struct acc_instance *fi_acc;
- struct acc_dev *dev;
int err;
fi_acc = kzalloc(sizeof(*fi_acc), GFP_KERNEL);
@@ -1315,19 +1393,19 @@ static struct usb_function_instance *acc_alloc_inst(void)
err = acc_setup();
if (err) {
kfree(fi_acc);
- pr_err("Error setting ACCESSORY\n");
return ERR_PTR(err);
}
config_group_init_type_name(&fi_acc->func_inst.group,
"", &acc_func_type);
- dev = _acc_dev;
return &fi_acc->func_inst;
}
static void acc_free(struct usb_function *f)
{
-/*NO-OP: no function specific resource allocation in mtp_alloc*/
+ struct acc_dev *dev = func_to_dev(f);
+
+ put_acc_dev(dev);
}
int acc_ctrlrequest_configfs(struct usb_function *f,
@@ -1340,9 +1418,7 @@ int acc_ctrlrequest_configfs(struct usb_function *f,
static struct usb_function *acc_alloc(struct usb_function_instance *fi)
{
- struct acc_dev *dev = _acc_dev;
-
- pr_info("acc_alloc\n");
+ struct acc_dev *dev = get_acc_dev();
dev->function.name = "accessory";
dev->function.strings = acc_strings,
diff --git a/drivers/usb/gadget/function/f_printer.c b/drivers/usb/gadget/function/f_printer.c
index 8ed1295d7e35..0f47cd398d60 100644
--- a/drivers/usb/gadget/function/f_printer.c
+++ b/drivers/usb/gadget/function/f_printer.c
@@ -1126,6 +1126,7 @@ fail_tx_reqs:
printer_req_free(dev->in_ep, req);
}
+ usb_free_all_descriptors(f);
return ret;
}
diff --git a/drivers/usb/gadget/function/f_uac2.c b/drivers/usb/gadget/function/f_uac2.c
index 65e9562c0d04..b05900de6053 100644
--- a/drivers/usb/gadget/function/f_uac2.c
+++ b/drivers/usb/gadget/function/f_uac2.c
@@ -278,7 +278,7 @@ static struct usb_endpoint_descriptor fs_epout_desc = {
.bEndpointAddress = USB_DIR_OUT,
.bmAttributes = USB_ENDPOINT_XFER_ISOC | USB_ENDPOINT_SYNC_SYNC,
- .wMaxPacketSize = cpu_to_le16(1023),
+ /* .wMaxPacketSize = DYNAMIC */
.bInterval = 1,
};
@@ -287,7 +287,7 @@ static struct usb_endpoint_descriptor hs_epout_desc = {
.bDescriptorType = USB_DT_ENDPOINT,
.bmAttributes = USB_ENDPOINT_XFER_ISOC | USB_ENDPOINT_SYNC_SYNC,
- .wMaxPacketSize = cpu_to_le16(1024),
+ /* .wMaxPacketSize = DYNAMIC */
.bInterval = 4,
};
@@ -362,7 +362,7 @@ static struct usb_endpoint_descriptor fs_epin_desc = {
.bEndpointAddress = USB_DIR_IN,
.bmAttributes = USB_ENDPOINT_XFER_ISOC | USB_ENDPOINT_SYNC_SYNC,
- .wMaxPacketSize = cpu_to_le16(1023),
+ /* .wMaxPacketSize = DYNAMIC */
.bInterval = 1,
};
@@ -371,7 +371,7 @@ static struct usb_endpoint_descriptor hs_epin_desc = {
.bDescriptorType = USB_DT_ENDPOINT,
.bmAttributes = USB_ENDPOINT_XFER_ISOC | USB_ENDPOINT_SYNC_SYNC,
- .wMaxPacketSize = cpu_to_le16(1024),
+ /* .wMaxPacketSize = DYNAMIC */
.bInterval = 4,
};
@@ -497,12 +497,28 @@ struct cntrl_range_lay3 {
__le32 dRES;
} __packed;
-static void set_ep_max_packet_size(const struct f_uac2_opts *uac2_opts,
+static int set_ep_max_packet_size(const struct f_uac2_opts *uac2_opts,
struct usb_endpoint_descriptor *ep_desc,
- unsigned int factor, bool is_playback)
+ enum usb_device_speed speed, bool is_playback)
{
int chmask, srate, ssize;
- u16 max_packet_size;
+ u16 max_size_bw, max_size_ep;
+ unsigned int factor;
+
+ switch (speed) {
+ case USB_SPEED_FULL:
+ max_size_ep = 1023;
+ factor = 1000;
+ break;
+
+ case USB_SPEED_HIGH:
+ max_size_ep = 1024;
+ factor = 8000;
+ break;
+
+ default:
+ return -EINVAL;
+ }
if (is_playback) {
chmask = uac2_opts->p_chmask;
@@ -514,10 +530,12 @@ static void set_ep_max_packet_size(const struct f_uac2_opts *uac2_opts,
ssize = uac2_opts->c_ssize;
}
- max_packet_size = num_channels(chmask) * ssize *
+ max_size_bw = num_channels(chmask) * ssize *
DIV_ROUND_UP(srate, factor / (1 << (ep_desc->bInterval - 1)));
- ep_desc->wMaxPacketSize = cpu_to_le16(min_t(u16, max_packet_size,
- le16_to_cpu(ep_desc->wMaxPacketSize)));
+ ep_desc->wMaxPacketSize = cpu_to_le16(min_t(u16, max_size_bw,
+ max_size_ep));
+
+ return 0;
}
static int
@@ -601,10 +619,33 @@ afunc_bind(struct usb_configuration *cfg, struct usb_function *fn)
uac2->as_in_alt = 0;
/* Calculate wMaxPacketSize according to audio bandwidth */
- set_ep_max_packet_size(uac2_opts, &fs_epin_desc, 1000, true);
- set_ep_max_packet_size(uac2_opts, &fs_epout_desc, 1000, false);
- set_ep_max_packet_size(uac2_opts, &hs_epin_desc, 8000, true);
- set_ep_max_packet_size(uac2_opts, &hs_epout_desc, 8000, false);
+ ret = set_ep_max_packet_size(uac2_opts, &fs_epin_desc, USB_SPEED_FULL,
+ true);
+ if (ret < 0) {
+ dev_err(dev, "%s:%d Error!\n", __func__, __LINE__);
+ return ret;
+ }
+
+ ret = set_ep_max_packet_size(uac2_opts, &fs_epout_desc, USB_SPEED_FULL,
+ false);
+ if (ret < 0) {
+ dev_err(dev, "%s:%d Error!\n", __func__, __LINE__);
+ return ret;
+ }
+
+ ret = set_ep_max_packet_size(uac2_opts, &hs_epin_desc, USB_SPEED_HIGH,
+ true);
+ if (ret < 0) {
+ dev_err(dev, "%s:%d Error!\n", __func__, __LINE__);
+ return ret;
+ }
+
+ ret = set_ep_max_packet_size(uac2_opts, &hs_epout_desc, USB_SPEED_HIGH,
+ false);
+ if (ret < 0) {
+ dev_err(dev, "%s:%d Error!\n", __func__, __LINE__);
+ return ret;
+ }
agdev->out_ep = usb_ep_autoconfig(gadget, &fs_epout_desc);
if (!agdev->out_ep) {
diff --git a/drivers/usb/gadget/function/u_ether.c b/drivers/usb/gadget/function/u_ether.c
index ddab9e1fa082..28fba8db3b42 100644
--- a/drivers/usb/gadget/function/u_ether.c
+++ b/drivers/usb/gadget/function/u_ether.c
@@ -45,9 +45,10 @@
#define UETH__VERSION "29-May-2008"
/* Experiments show that both Linux and Windows hosts allow up to 16k
- * frame sizes. Set the max size to 15k+52 to prevent allocating 32k
+ * frame sizes. Set the max MTU size to 15k+52 to prevent allocating 32k
* blocks and still have efficient handling. */
-#define GETHER_MAX_ETH_FRAME_LEN 15412
+#define GETHER_MAX_MTU_SIZE 15412
+#define GETHER_MAX_ETH_FRAME_LEN (GETHER_MAX_MTU_SIZE + ETH_HLEN)
struct eth_dev {
/* lock is held while accessing port_usb
@@ -791,7 +792,7 @@ struct eth_dev *gether_setup_name(struct usb_gadget *g,
/* MTU range: 14 - 15412 */
net->min_mtu = ETH_HLEN;
- net->max_mtu = GETHER_MAX_ETH_FRAME_LEN;
+ net->max_mtu = GETHER_MAX_MTU_SIZE;
dev->gadget = g;
SET_NETDEV_DEV(net, &g->dev);
@@ -853,7 +854,7 @@ struct net_device *gether_setup_name_default(const char *netname)
/* MTU range: 14 - 15412 */
net->min_mtu = ETH_HLEN;
- net->max_mtu = GETHER_MAX_ETH_FRAME_LEN;
+ net->max_mtu = GETHER_MAX_MTU_SIZE;
return net;
}
diff --git a/drivers/usb/gadget/legacy/acm_ms.c b/drivers/usb/gadget/legacy/acm_ms.c
index af16672d5118..6680dcfe660e 100644
--- a/drivers/usb/gadget/legacy/acm_ms.c
+++ b/drivers/usb/gadget/legacy/acm_ms.c
@@ -203,8 +203,10 @@ static int acm_ms_bind(struct usb_composite_dev *cdev)
struct usb_descriptor_header *usb_desc;
usb_desc = usb_otg_descriptor_alloc(gadget);
- if (!usb_desc)
+ if (!usb_desc) {
+ status = -ENOMEM;
goto fail_string_ids;
+ }
usb_otg_descriptor_init(gadget, usb_desc);
otg_desc[0] = usb_desc;
otg_desc[1] = NULL;
diff --git a/drivers/usb/gadget/udc/aspeed-vhub/epn.c b/drivers/usb/gadget/udc/aspeed-vhub/epn.c
index 5939eb1e97f2..ae853cf36966 100644
--- a/drivers/usb/gadget/udc/aspeed-vhub/epn.c
+++ b/drivers/usb/gadget/udc/aspeed-vhub/epn.c
@@ -420,7 +420,10 @@ static void ast_vhub_stop_active_req(struct ast_vhub_ep *ep,
u32 state, reg, loops;
/* Stop DMA activity */
- writel(0, ep->epn.regs + AST_VHUB_EP_DMA_CTLSTAT);
+ if (ep->epn.desc_mode)
+ writel(VHUB_EP_DMA_CTRL_RESET, ep->epn.regs + AST_VHUB_EP_DMA_CTLSTAT);
+ else
+ writel(0, ep->epn.regs + AST_VHUB_EP_DMA_CTLSTAT);
/* Wait for it to complete */
for (loops = 0; loops < 1000; loops++) {
diff --git a/drivers/usb/gadget/udc/bdc/Kconfig b/drivers/usb/gadget/udc/bdc/Kconfig
index c74ac25dddcd..051091bd265b 100644
--- a/drivers/usb/gadget/udc/bdc/Kconfig
+++ b/drivers/usb/gadget/udc/bdc/Kconfig
@@ -15,7 +15,7 @@ if USB_BDC_UDC
comment "Platform Support"
config USB_BDC_PCI
tristate "BDC support for PCIe based platforms"
- depends on USB_PCI
+ depends on USB_PCI && BROKEN
default USB_BDC_UDC
help
Enable support for platforms which have BDC connected through PCIe, such as Lego3 FPGA platform.
diff --git a/drivers/usb/gadget/udc/core.c b/drivers/usb/gadget/udc/core.c
index 538d56e89ec8..567b4372c49d 100644
--- a/drivers/usb/gadget/udc/core.c
+++ b/drivers/usb/gadget/udc/core.c
@@ -1512,10 +1512,13 @@ static ssize_t soft_connect_store(struct device *dev,
struct device_attribute *attr, const char *buf, size_t n)
{
struct usb_udc *udc = container_of(dev, struct usb_udc, dev);
+ ssize_t ret;
+ mutex_lock(&udc_lock);
if (!udc->driver) {
dev_err(dev, "soft-connect without a gadget driver\n");
- return -EOPNOTSUPP;
+ ret = -EOPNOTSUPP;
+ goto out;
}
if (sysfs_streq(buf, "connect")) {
@@ -1527,10 +1530,14 @@ static ssize_t soft_connect_store(struct device *dev,
usb_gadget_udc_stop(udc);
} else {
dev_err(dev, "unsupported command '%s'\n", buf);
- return -EINVAL;
+ ret = -EINVAL;
+ goto out;
}
- return n;
+ ret = n;
+out:
+ mutex_unlock(&udc_lock);
+ return ret;
}
static DEVICE_ATTR_WO(soft_connect);
diff --git a/drivers/usb/host/ehci-hcd.c b/drivers/usb/host/ehci-hcd.c
index caf9f6b1cd34..d9282ca7ae8c 100644
--- a/drivers/usb/host/ehci-hcd.c
+++ b/drivers/usb/host/ehci-hcd.c
@@ -574,6 +574,7 @@ static int ehci_run (struct usb_hcd *hcd)
struct ehci_hcd *ehci = hcd_to_ehci (hcd);
u32 temp;
u32 hcc_params;
+ int rc;
hcd->uses_new_polling = 1;
@@ -629,9 +630,20 @@ static int ehci_run (struct usb_hcd *hcd)
down_write(&ehci_cf_port_reset_rwsem);
ehci->rh_state = EHCI_RH_RUNNING;
ehci_writel(ehci, FLAG_CF, &ehci->regs->configured_flag);
+
+ /* Wait until HC become operational */
ehci_readl(ehci, &ehci->regs->command); /* unblock posted writes */
msleep(5);
+ rc = ehci_handshake(ehci, &ehci->regs->status, STS_HALT, 0, 100 * 1000);
+
up_write(&ehci_cf_port_reset_rwsem);
+
+ if (rc) {
+ ehci_err(ehci, "USB %x.%x, controller refused to start: %d\n",
+ ((ehci->sbrn & 0xf0)>>4), (ehci->sbrn & 0x0f), rc);
+ return rc;
+ }
+
ehci->last_periodic_enable = ktime_get_real();
temp = HC_VERSION(ehci, ehci_readl(ehci, &ehci->caps->hc_capbase));
diff --git a/drivers/usb/host/ehci-hub.c b/drivers/usb/host/ehci-hub.c
index 087402aec5cb..9f9ab5ccea88 100644
--- a/drivers/usb/host/ehci-hub.c
+++ b/drivers/usb/host/ehci-hub.c
@@ -345,6 +345,9 @@ static int ehci_bus_suspend (struct usb_hcd *hcd)
unlink_empty_async_suspended(ehci);
+ /* Some Synopsys controllers mistakenly leave IAA turned on */
+ ehci_writel(ehci, STS_IAA, &ehci->regs->status);
+
/* Any IAA cycle that started before the suspend is now invalid */
end_iaa_cycle(ehci);
ehci_handle_start_intr_unlinks(ehci);
diff --git a/drivers/usb/host/ohci-hcd.c b/drivers/usb/host/ohci-hcd.c
index e88486d8084a..5916235adf35 100644
--- a/drivers/usb/host/ohci-hcd.c
+++ b/drivers/usb/host/ohci-hcd.c
@@ -101,7 +101,7 @@ static void io_watchdog_func(struct timer_list *t);
/* Some boards misreport power switching/overcurrent */
-static bool distrust_firmware = true;
+static bool distrust_firmware;
module_param (distrust_firmware, bool, 0);
MODULE_PARM_DESC (distrust_firmware,
"true to distrust firmware power/overcurrent setup");
diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c
index 20ea7f5cb044..fb40483178f3 100644
--- a/drivers/usb/host/xhci-ring.c
+++ b/drivers/usb/host/xhci-ring.c
@@ -2835,6 +2835,8 @@ static void queue_trb(struct xhci_hcd *xhci, struct xhci_ring *ring,
trb->field[0] = cpu_to_le32(field1);
trb->field[1] = cpu_to_le32(field2);
trb->field[2] = cpu_to_le32(field3);
+ /* make sure TRB is fully written before giving it to the controller */
+ wmb();
trb->field[3] = cpu_to_le32(field4);
trace_xhci_queue_trb(ring, trb);
diff --git a/drivers/usb/host/xhci-tegra.c b/drivers/usb/host/xhci-tegra.c
index fe37dacc695f..dc9cd8c519cf 100644
--- a/drivers/usb/host/xhci-tegra.c
+++ b/drivers/usb/host/xhci-tegra.c
@@ -578,6 +578,13 @@ static void tegra_xusb_mbox_handle(struct tegra_xusb *tegra,
enable);
if (err < 0)
break;
+
+ /*
+ * wait 500us for LFPS detector to be disabled before
+ * sending ACK
+ */
+ if (!enable)
+ usleep_range(500, 1000);
}
if (err < 0) {
diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c
index 64a69b3c8ce9..53c88b95e4f3 100644
--- a/drivers/usb/host/xhci.c
+++ b/drivers/usb/host/xhci.c
@@ -4593,19 +4593,19 @@ static u16 xhci_calculate_u1_timeout(struct xhci_hcd *xhci,
{
unsigned long long timeout_ns;
- /* Prevent U1 if service interval is shorter than U1 exit latency */
- if (usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc)) {
- if (xhci_service_interval_to_ns(desc) <= udev->u1_params.mel) {
- dev_dbg(&udev->dev, "Disable U1, ESIT shorter than exit latency\n");
- return USB3_LPM_DISABLED;
- }
- }
-
if (xhci->quirks & XHCI_INTEL_HOST)
timeout_ns = xhci_calculate_intel_u1_timeout(udev, desc);
else
timeout_ns = udev->u1_params.sel;
+ /* Prevent U1 if service interval is shorter than U1 exit latency */
+ if (usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc)) {
+ if (xhci_service_interval_to_ns(desc) <= timeout_ns) {
+ dev_dbg(&udev->dev, "Disable U1, ESIT shorter than exit latency\n");
+ return USB3_LPM_DISABLED;
+ }
+ }
+
/* The U1 timeout is encoded in 1us intervals.
* Don't return a timeout of zero, because that's USB3_LPM_DISABLED.
*/
@@ -4657,19 +4657,19 @@ static u16 xhci_calculate_u2_timeout(struct xhci_hcd *xhci,
{
unsigned long long timeout_ns;
- /* Prevent U2 if service interval is shorter than U2 exit latency */
- if (usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc)) {
- if (xhci_service_interval_to_ns(desc) <= udev->u2_params.mel) {
- dev_dbg(&udev->dev, "Disable U2, ESIT shorter than exit latency\n");
- return USB3_LPM_DISABLED;
- }
- }
-
if (xhci->quirks & XHCI_INTEL_HOST)
timeout_ns = xhci_calculate_intel_u2_timeout(udev, desc);
else
timeout_ns = udev->u2_params.sel;
+ /* Prevent U2 if service interval is shorter than U2 exit latency */
+ if (usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc)) {
+ if (xhci_service_interval_to_ns(desc) <= timeout_ns) {
+ dev_dbg(&udev->dev, "Disable U2, ESIT shorter than exit latency\n");
+ return USB3_LPM_DISABLED;
+ }
+ }
+
/* The U2 timeout is encoded in 256us intervals */
timeout_ns = DIV_ROUND_UP_ULL(timeout_ns, 256 * 1000);
/* If the necessary timeout value is bigger than what we can set in the
diff --git a/drivers/usb/misc/yurex.c b/drivers/usb/misc/yurex.c
index 785080f79073..08b72bb22b7e 100644
--- a/drivers/usb/misc/yurex.c
+++ b/drivers/usb/misc/yurex.c
@@ -497,6 +497,9 @@ static ssize_t yurex_write(struct file *file, const char __user *user_buffer,
timeout = schedule_timeout(YUREX_WRITE_TIMEOUT);
finish_wait(&dev->waitq, &wait);
+ /* make sure URB is idle after timeout or (spurious) CMD_ACK */
+ usb_kill_urb(dev->cntl_urb);
+
mutex_unlock(&dev->io_mutex);
if (retval < 0) {
diff --git a/drivers/usb/serial/iuu_phoenix.c b/drivers/usb/serial/iuu_phoenix.c
index 734f18d0a7f7..6dc93652afc1 100644
--- a/drivers/usb/serial/iuu_phoenix.c
+++ b/drivers/usb/serial/iuu_phoenix.c
@@ -536,23 +536,29 @@ static int iuu_uart_flush(struct usb_serial_port *port)
struct device *dev = &port->dev;
int i;
int status;
- u8 rxcmd = IUU_UART_RX;
+ u8 *rxcmd;
struct iuu_private *priv = usb_get_serial_port_data(port);
if (iuu_led(port, 0xF000, 0, 0, 0xFF) < 0)
return -EIO;
+ rxcmd = kmalloc(1, GFP_KERNEL);
+ if (!rxcmd)
+ return -ENOMEM;
+
+ rxcmd[0] = IUU_UART_RX;
+
for (i = 0; i < 2; i++) {
- status = bulk_immediate(port, &rxcmd, 1);
+ status = bulk_immediate(port, rxcmd, 1);
if (status != IUU_OPERATION_OK) {
dev_dbg(dev, "%s - uart_flush_write error\n", __func__);
- return status;
+ goto out_free;
}
status = read_immediate(port, &priv->len, 1);
if (status != IUU_OPERATION_OK) {
dev_dbg(dev, "%s - uart_flush_read error\n", __func__);
- return status;
+ goto out_free;
}
if (priv->len > 0) {
@@ -560,12 +566,16 @@ static int iuu_uart_flush(struct usb_serial_port *port)
status = read_immediate(port, priv->buf, priv->len);
if (status != IUU_OPERATION_OK) {
dev_dbg(dev, "%s - uart_flush_read error\n", __func__);
- return status;
+ goto out_free;
}
}
}
dev_dbg(dev, "%s - uart_flush_read OK!\n", __func__);
iuu_led(port, 0, 0xF000, 0, 0xFF);
+
+out_free:
+ kfree(rxcmd);
+
return status;
}
diff --git a/drivers/usb/serial/keyspan_pda.c b/drivers/usb/serial/keyspan_pda.c
index e7a2aa1747db..766969cf72b2 100644
--- a/drivers/usb/serial/keyspan_pda.c
+++ b/drivers/usb/serial/keyspan_pda.c
@@ -555,10 +555,8 @@ exit:
static void keyspan_pda_write_bulk_callback(struct urb *urb)
{
struct usb_serial_port *port = urb->context;
- struct keyspan_pda_private *priv;
set_bit(0, &port->write_urbs_free);
- priv = usb_get_serial_port_data(port);
/* queue up a wakeup at scheduler time */
usb_serial_port_softint(port);
diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c
index 6fd6012ad7b3..1024aca25221 100644
--- a/drivers/usb/serial/option.c
+++ b/drivers/usb/serial/option.c
@@ -1117,6 +1117,8 @@ static const struct usb_device_id option_ids[] = {
{ USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EM12, 0xff, 0xff, 0xff),
.driver_info = RSVD(1) | RSVD(2) | RSVD(3) | RSVD(4) | NUMEP2 },
{ USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EM12, 0xff, 0, 0) },
+ { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, 0x0620, 0xff, 0xff, 0x30) }, /* EM160R-GL */
+ { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, 0x0620, 0xff, 0, 0) },
{ USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RM500Q, 0xff, 0xff, 0x30) },
{ USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RM500Q, 0xff, 0, 0) },
{ USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RM500Q, 0xff, 0xff, 0x10),
@@ -2057,6 +2059,7 @@ static const struct usb_device_id option_ids[] = {
{ USB_DEVICE_INTERFACE_CLASS(0x2cb7, 0x0105, 0xff), /* Fibocom NL678 series */
.driver_info = RSVD(6) },
{ USB_DEVICE_INTERFACE_CLASS(0x2cb7, 0x01a0, 0xff) }, /* Fibocom NL668-AM/NL652-EU (laptop MBIM) */
+ { USB_DEVICE_INTERFACE_CLASS(0x2df3, 0x9d03, 0xff) }, /* LongSung M5710 */
{ USB_DEVICE_INTERFACE_CLASS(0x305a, 0x1404, 0xff) }, /* GosunCn GM500 RNDIS */
{ USB_DEVICE_INTERFACE_CLASS(0x305a, 0x1405, 0xff) }, /* GosunCn GM500 MBIM */
{ USB_DEVICE_INTERFACE_CLASS(0x305a, 0x1406, 0xff) }, /* GosunCn GM500 ECM/NCM */
diff --git a/drivers/usb/storage/unusual_uas.h b/drivers/usb/storage/unusual_uas.h
index 749c69be091c..cb7b15ecb7ab 100644
--- a/drivers/usb/storage/unusual_uas.h
+++ b/drivers/usb/storage/unusual_uas.h
@@ -90,6 +90,13 @@ UNUSUAL_DEV(0x152d, 0x0578, 0x0000, 0x9999,
USB_SC_DEVICE, USB_PR_DEVICE, NULL,
US_FL_BROKEN_FUA),
+/* Reported-by: Thinh Nguyen */
+UNUSUAL_DEV(0x154b, 0xf00b, 0x0000, 0x9999,
+ "PNY",
+ "Pro Elite SSD",
+ USB_SC_DEVICE, USB_PR_DEVICE, NULL,
+ US_FL_NO_ATA_1X),
+
/* Reported-by: Thinh Nguyen */
UNUSUAL_DEV(0x154b, 0xf00d, 0x0000, 0x9999,
"PNY",
diff --git a/drivers/usb/usbip/vhci_hcd.c b/drivers/usb/usbip/vhci_hcd.c
index d5a036bf904b..0909cfcbf720 100644
--- a/drivers/usb/usbip/vhci_hcd.c
+++ b/drivers/usb/usbip/vhci_hcd.c
@@ -396,6 +396,8 @@ static int vhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
default:
usbip_dbg_vhci_rh(" ClearPortFeature: default %x\n",
wValue);
+ if (wValue >= 32)
+ goto error;
vhci_hcd->port_status[rhport] &= ~(1 << wValue);
break;
}
diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c
index 5e23e4aa5b0a..c48e1d84efb6 100644
--- a/drivers/vfio/pci/vfio_pci.c
+++ b/drivers/vfio/pci/vfio_pci.c
@@ -118,8 +118,6 @@ static void vfio_pci_probe_mmaps(struct vfio_pci_device *vdev)
int bar;
struct vfio_pci_dummy_resource *dummy_res;
- INIT_LIST_HEAD(&vdev->dummy_resources_list);
-
for (bar = PCI_STD_RESOURCES; bar <= PCI_STD_RESOURCE_END; bar++) {
res = vdev->pdev->resource + bar;
@@ -1522,6 +1520,7 @@ static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
mutex_init(&vdev->igate);
spin_lock_init(&vdev->irqlock);
mutex_init(&vdev->ioeventfds_lock);
+ INIT_LIST_HEAD(&vdev->dummy_resources_list);
INIT_LIST_HEAD(&vdev->ioeventfds_list);
mutex_init(&vdev->vma_lock);
INIT_LIST_HEAD(&vdev->vma_list);
diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
index 88c8c158ec25..0c7bbc92b22a 100644
--- a/drivers/vhost/net.c
+++ b/drivers/vhost/net.c
@@ -613,6 +613,7 @@ static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock)
size_t len, total_len = 0;
int err;
struct vhost_net_ubuf_ref *uninitialized_var(ubufs);
+ struct ubuf_info *ubuf;
bool zcopy_used;
int sent_pkts = 0;
@@ -645,9 +646,7 @@ static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock)
/* use msg_control to pass vhost zerocopy ubuf info to skb */
if (zcopy_used) {
- struct ubuf_info *ubuf;
ubuf = nvq->ubuf_info + nvq->upend_idx;
-
vq->heads[nvq->upend_idx].id = cpu_to_vhost32(vq, head);
vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS;
ubuf->callback = vhost_zerocopy_callback;
@@ -675,7 +674,8 @@ static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock)
err = sock->ops->sendmsg(sock, &msg, len);
if (unlikely(err < 0)) {
if (zcopy_used) {
- vhost_net_ubuf_put(ubufs);
+ if (vq->heads[ubuf->desc].len == VHOST_DMA_IN_PROGRESS)
+ vhost_net_ubuf_put(ubufs);
nvq->upend_idx = ((unsigned)nvq->upend_idx - 1)
% UIO_MAXIOV;
}
diff --git a/drivers/video/fbdev/hyperv_fb.c b/drivers/video/fbdev/hyperv_fb.c
index 56e70f12c996..c907f96d6890 100644
--- a/drivers/video/fbdev/hyperv_fb.c
+++ b/drivers/video/fbdev/hyperv_fb.c
@@ -713,11 +713,9 @@ static int hvfb_getmem(struct hv_device *hdev, struct fb_info *info)
}
/*
- * Map the VRAM cacheable for performance. This is also required for
- * VM Connect to display properly for ARM64 Linux VM, as the host also
- * maps the VRAM cacheable.
+ * Map the VRAM cacheable for performance.
*/
- fb_virt = ioremap_cache(par->mem->start, screen_fb_size);
+ fb_virt = ioremap_wc(par->mem->start, screen_fb_size);
if (!fb_virt)
goto err2;
diff --git a/drivers/xen/events/events_base.c b/drivers/xen/events/events_base.c
index aca845675279..8c08c7d46d3d 100644
--- a/drivers/xen/events/events_base.c
+++ b/drivers/xen/events/events_base.c
@@ -1987,16 +1987,6 @@ static struct irq_chip xen_percpu_chip __read_mostly = {
.irq_ack = ack_dynirq,
};
-int xen_set_callback_via(uint64_t via)
-{
- struct xen_hvm_param a;
- a.domid = DOMID_SELF;
- a.index = HVM_PARAM_CALLBACK_IRQ;
- a.value = via;
- return HYPERVISOR_hvm_op(HVMOP_set_param, &a);
-}
-EXPORT_SYMBOL_GPL(xen_set_callback_via);
-
#ifdef CONFIG_XEN_PVHVM
/* Vector callbacks are better than PCI interrupts to receive event
* channel notifications because we can receive vector callbacks on any
diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c
index 9d8e02cfd480..3cfbec482efb 100644
--- a/drivers/xen/gntdev.c
+++ b/drivers/xen/gntdev.c
@@ -842,17 +842,18 @@ struct gntdev_copy_batch {
s16 __user *status[GNTDEV_COPY_BATCH];
unsigned int nr_ops;
unsigned int nr_pages;
+ bool writeable;
};
static int gntdev_get_page(struct gntdev_copy_batch *batch, void __user *virt,
- bool writeable, unsigned long *gfn)
+ unsigned long *gfn)
{
unsigned long addr = (unsigned long)virt;
struct page *page;
unsigned long xen_pfn;
int ret;
- ret = get_user_pages_fast(addr, 1, writeable, &page);
+ ret = get_user_pages_fast(addr, 1, batch->writeable, &page);
if (ret < 0)
return ret;
@@ -868,9 +869,13 @@ static void gntdev_put_pages(struct gntdev_copy_batch *batch)
{
unsigned int i;
- for (i = 0; i < batch->nr_pages; i++)
+ for (i = 0; i < batch->nr_pages; i++) {
+ if (batch->writeable && !PageDirty(batch->pages[i]))
+ set_page_dirty_lock(batch->pages[i]);
put_page(batch->pages[i]);
+ }
batch->nr_pages = 0;
+ batch->writeable = false;
}
static int gntdev_copy(struct gntdev_copy_batch *batch)
@@ -959,8 +964,9 @@ static int gntdev_grant_copy_seg(struct gntdev_copy_batch *batch,
virt = seg->source.virt + copied;
off = (unsigned long)virt & ~XEN_PAGE_MASK;
len = min(len, (size_t)XEN_PAGE_SIZE - off);
+ batch->writeable = false;
- ret = gntdev_get_page(batch, virt, false, &gfn);
+ ret = gntdev_get_page(batch, virt, &gfn);
if (ret < 0)
return ret;
@@ -978,8 +984,9 @@ static int gntdev_grant_copy_seg(struct gntdev_copy_batch *batch,
virt = seg->dest.virt + copied;
off = (unsigned long)virt & ~XEN_PAGE_MASK;
len = min(len, (size_t)XEN_PAGE_SIZE - off);
+ batch->writeable = true;
- ret = gntdev_get_page(batch, virt, true, &gfn);
+ ret = gntdev_get_page(batch, virt, &gfn);
if (ret < 0)
return ret;
diff --git a/drivers/xen/platform-pci.c b/drivers/xen/platform-pci.c
index 5d7dcad0b0a0..4cec8146609a 100644
--- a/drivers/xen/platform-pci.c
+++ b/drivers/xen/platform-pci.c
@@ -162,7 +162,6 @@ static int platform_pci_probe(struct pci_dev *pdev,
ret = gnttab_init();
if (ret)
goto grant_out;
- xenbus_probe(NULL);
return 0;
grant_out:
gnttab_free_auto_xlat_frames();
diff --git a/drivers/xen/xenbus/xenbus.h b/drivers/xen/xenbus/xenbus.h
index 88516a8a9f93..a9bb5f91082d 100644
--- a/drivers/xen/xenbus/xenbus.h
+++ b/drivers/xen/xenbus/xenbus.h
@@ -115,6 +115,7 @@ int xenbus_probe_node(struct xen_bus_type *bus,
const char *type,
const char *nodename);
int xenbus_probe_devices(struct xen_bus_type *bus);
+void xenbus_probe(void);
void xenbus_dev_changed(const char *node, struct xen_bus_type *bus);
diff --git a/drivers/xen/xenbus/xenbus_comms.c b/drivers/xen/xenbus/xenbus_comms.c
index eb5151fc8efa..e5fda0256feb 100644
--- a/drivers/xen/xenbus/xenbus_comms.c
+++ b/drivers/xen/xenbus/xenbus_comms.c
@@ -57,16 +57,8 @@ DEFINE_MUTEX(xs_response_mutex);
static int xenbus_irq;
static struct task_struct *xenbus_task;
-static DECLARE_WORK(probe_work, xenbus_probe);
-
-
static irqreturn_t wake_waiting(int irq, void *unused)
{
- if (unlikely(xenstored_ready == 0)) {
- xenstored_ready = 1;
- schedule_work(&probe_work);
- }
-
wake_up(&xb_waitq);
return IRQ_HANDLED;
}
diff --git a/drivers/xen/xenbus/xenbus_probe.c b/drivers/xen/xenbus/xenbus_probe.c
index e6d0903459e1..14ccf13ab8fa 100644
--- a/drivers/xen/xenbus/xenbus_probe.c
+++ b/drivers/xen/xenbus/xenbus_probe.c
@@ -683,29 +683,76 @@ void unregister_xenstore_notifier(struct notifier_block *nb)
}
EXPORT_SYMBOL_GPL(unregister_xenstore_notifier);
-void xenbus_probe(struct work_struct *unused)
+void xenbus_probe(void)
{
xenstored_ready = 1;
+ /*
+ * In the HVM case, xenbus_init() deferred its call to
+ * xs_init() in case callbacks were not operational yet.
+ * So do it now.
+ */
+ if (xen_store_domain_type == XS_HVM)
+ xs_init();
+
/* Notify others that xenstore is up */
blocking_notifier_call_chain(&xenstore_chain, 0, NULL);
}
-EXPORT_SYMBOL_GPL(xenbus_probe);
+
+/*
+ * Returns true when XenStore init must be deferred in order to
+ * allow the PCI platform device to be initialised, before we
+ * can actually have event channel interrupts working.
+ */
+static bool xs_hvm_defer_init_for_callback(void)
+{
+#ifdef CONFIG_XEN_PVHVM
+ return xen_store_domain_type == XS_HVM &&
+ !xen_have_vector_callback;
+#else
+ return false;
+#endif
+}
static int __init xenbus_probe_initcall(void)
{
- if (!xen_domain())
- return -ENODEV;
+ /*
+ * Probe XenBus here in the XS_PV case, and also XS_HVM unless we
+ * need to wait for the platform PCI device to come up.
+ */
+ if (xen_store_domain_type == XS_PV ||
+ (xen_store_domain_type == XS_HVM &&
+ !xs_hvm_defer_init_for_callback()))
+ xenbus_probe();
- if (xen_initial_domain() || xen_hvm_domain())
- return 0;
-
- xenbus_probe(NULL);
return 0;
}
-
device_initcall(xenbus_probe_initcall);
+int xen_set_callback_via(uint64_t via)
+{
+ struct xen_hvm_param a;
+ int ret;
+
+ a.domid = DOMID_SELF;
+ a.index = HVM_PARAM_CALLBACK_IRQ;
+ a.value = via;
+
+ ret = HYPERVISOR_hvm_op(HVMOP_set_param, &a);
+ if (ret)
+ return ret;
+
+ /*
+ * If xenbus_probe_initcall() deferred the xenbus_probe()
+ * due to the callback not functioning yet, we can do it now.
+ */
+ if (!xenstored_ready && xs_hvm_defer_init_for_callback())
+ xenbus_probe();
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(xen_set_callback_via);
+
/* Set up event channel for xenstored which is run as a local process
* (this is normally used only in dom0)
*/
@@ -818,11 +865,17 @@ static int __init xenbus_init(void)
break;
}
- /* Initialize the interface to xenstore. */
- err = xs_init();
- if (err) {
- pr_warn("Error initializing xenstore comms: %i\n", err);
- goto out_error;
+ /*
+ * HVM domains may not have a functional callback yet. In that
+ * case let xs_init() be called from xenbus_probe(), which will
+ * get invoked at an appropriate time.
+ */
+ if (xen_store_domain_type != XS_HVM) {
+ err = xs_init();
+ if (err) {
+ pr_warn("Error initializing xenstore comms: %i\n", err);
+ goto out_error;
+ }
}
if ((xen_store_domain_type != XS_LOCAL) &&
diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c
index 9936e4b991aa..7bda5586e433 100644
--- a/fs/btrfs/qgroup.c
+++ b/fs/btrfs/qgroup.c
@@ -2774,6 +2774,12 @@ out:
return ret;
}
+static bool rescan_should_stop(struct btrfs_fs_info *fs_info)
+{
+ return btrfs_fs_closing(fs_info) ||
+ test_bit(BTRFS_FS_STATE_REMOUNTING, &fs_info->fs_state);
+}
+
static void btrfs_qgroup_rescan_worker(struct btrfs_work *work)
{
struct btrfs_fs_info *fs_info = container_of(work, struct btrfs_fs_info,
@@ -2782,6 +2788,7 @@ static void btrfs_qgroup_rescan_worker(struct btrfs_work *work)
struct btrfs_trans_handle *trans = NULL;
int err = -ENOMEM;
int ret = 0;
+ bool stopped = false;
path = btrfs_alloc_path();
if (!path)
@@ -2794,7 +2801,7 @@ static void btrfs_qgroup_rescan_worker(struct btrfs_work *work)
path->skip_locking = 1;
err = 0;
- while (!err && !btrfs_fs_closing(fs_info)) {
+ while (!err && !(stopped = rescan_should_stop(fs_info))) {
trans = btrfs_start_transaction(fs_info->fs_root, 0);
if (IS_ERR(trans)) {
err = PTR_ERR(trans);
@@ -2837,7 +2844,7 @@ out:
}
mutex_lock(&fs_info->qgroup_rescan_lock);
- if (!btrfs_fs_closing(fs_info))
+ if (!stopped)
fs_info->qgroup_flags &= ~BTRFS_QGROUP_STATUS_FLAG_RESCAN;
if (trans) {
ret = update_qgroup_status_item(trans);
@@ -2856,7 +2863,7 @@ out:
btrfs_end_transaction(trans);
- if (btrfs_fs_closing(fs_info)) {
+ if (stopped) {
btrfs_info(fs_info, "qgroup scan paused");
} else if (err >= 0) {
btrfs_info(fs_info, "qgroup scan completed%s",
diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c
index ed61c0daef41..128398dde081 100644
--- a/fs/btrfs/send.c
+++ b/fs/btrfs/send.c
@@ -238,6 +238,7 @@ struct waiting_dir_move {
* after this directory is moved, we can try to rmdir the ino rmdir_ino.
*/
u64 rmdir_ino;
+ u64 rmdir_gen;
bool orphanized;
};
@@ -308,7 +309,7 @@ static int is_waiting_for_move(struct send_ctx *sctx, u64 ino);
static struct waiting_dir_move *
get_waiting_dir_move(struct send_ctx *sctx, u64 ino);
-static int is_waiting_for_rm(struct send_ctx *sctx, u64 dir_ino);
+static int is_waiting_for_rm(struct send_ctx *sctx, u64 dir_ino, u64 gen);
static int need_send_hole(struct send_ctx *sctx)
{
@@ -2304,7 +2305,7 @@ static int get_cur_path(struct send_ctx *sctx, u64 ino, u64 gen,
fs_path_reset(name);
- if (is_waiting_for_rm(sctx, ino)) {
+ if (is_waiting_for_rm(sctx, ino, gen)) {
ret = gen_unique_name(sctx, ino, gen, name);
if (ret < 0)
goto out;
@@ -2863,8 +2864,8 @@ out:
return ret;
}
-static struct orphan_dir_info *
-add_orphan_dir_info(struct send_ctx *sctx, u64 dir_ino)
+static struct orphan_dir_info *add_orphan_dir_info(struct send_ctx *sctx,
+ u64 dir_ino, u64 dir_gen)
{
struct rb_node **p = &sctx->orphan_dirs.rb_node;
struct rb_node *parent = NULL;
@@ -2873,20 +2874,23 @@ add_orphan_dir_info(struct send_ctx *sctx, u64 dir_ino)
while (*p) {
parent = *p;
entry = rb_entry(parent, struct orphan_dir_info, node);
- if (dir_ino < entry->ino) {
+ if (dir_ino < entry->ino)
p = &(*p)->rb_left;
- } else if (dir_ino > entry->ino) {
+ else if (dir_ino > entry->ino)
p = &(*p)->rb_right;
- } else {
+ else if (dir_gen < entry->gen)
+ p = &(*p)->rb_left;
+ else if (dir_gen > entry->gen)
+ p = &(*p)->rb_right;
+ else
return entry;
- }
}
odi = kmalloc(sizeof(*odi), GFP_KERNEL);
if (!odi)
return ERR_PTR(-ENOMEM);
odi->ino = dir_ino;
- odi->gen = 0;
+ odi->gen = dir_gen;
odi->last_dir_index_offset = 0;
rb_link_node(&odi->node, parent, p);
@@ -2894,8 +2898,8 @@ add_orphan_dir_info(struct send_ctx *sctx, u64 dir_ino)
return odi;
}
-static struct orphan_dir_info *
-get_orphan_dir_info(struct send_ctx *sctx, u64 dir_ino)
+static struct orphan_dir_info *get_orphan_dir_info(struct send_ctx *sctx,
+ u64 dir_ino, u64 gen)
{
struct rb_node *n = sctx->orphan_dirs.rb_node;
struct orphan_dir_info *entry;
@@ -2906,15 +2910,19 @@ get_orphan_dir_info(struct send_ctx *sctx, u64 dir_ino)
n = n->rb_left;
else if (dir_ino > entry->ino)
n = n->rb_right;
+ else if (gen < entry->gen)
+ n = n->rb_left;
+ else if (gen > entry->gen)
+ n = n->rb_right;
else
return entry;
}
return NULL;
}
-static int is_waiting_for_rm(struct send_ctx *sctx, u64 dir_ino)
+static int is_waiting_for_rm(struct send_ctx *sctx, u64 dir_ino, u64 gen)
{
- struct orphan_dir_info *odi = get_orphan_dir_info(sctx, dir_ino);
+ struct orphan_dir_info *odi = get_orphan_dir_info(sctx, dir_ino, gen);
return odi != NULL;
}
@@ -2959,7 +2967,7 @@ static int can_rmdir(struct send_ctx *sctx, u64 dir, u64 dir_gen,
key.type = BTRFS_DIR_INDEX_KEY;
key.offset = 0;
- odi = get_orphan_dir_info(sctx, dir);
+ odi = get_orphan_dir_info(sctx, dir, dir_gen);
if (odi)
key.offset = odi->last_dir_index_offset;
@@ -2990,7 +2998,7 @@ static int can_rmdir(struct send_ctx *sctx, u64 dir, u64 dir_gen,
dm = get_waiting_dir_move(sctx, loc.objectid);
if (dm) {
- odi = add_orphan_dir_info(sctx, dir);
+ odi = add_orphan_dir_info(sctx, dir, dir_gen);
if (IS_ERR(odi)) {
ret = PTR_ERR(odi);
goto out;
@@ -2998,12 +3006,13 @@ static int can_rmdir(struct send_ctx *sctx, u64 dir, u64 dir_gen,
odi->gen = dir_gen;
odi->last_dir_index_offset = found_key.offset;
dm->rmdir_ino = dir;
+ dm->rmdir_gen = dir_gen;
ret = 0;
goto out;
}
if (loc.objectid > send_progress) {
- odi = add_orphan_dir_info(sctx, dir);
+ odi = add_orphan_dir_info(sctx, dir, dir_gen);
if (IS_ERR(odi)) {
ret = PTR_ERR(odi);
goto out;
@@ -3043,6 +3052,7 @@ static int add_waiting_dir_move(struct send_ctx *sctx, u64 ino, bool orphanized)
return -ENOMEM;
dm->ino = ino;
dm->rmdir_ino = 0;
+ dm->rmdir_gen = 0;
dm->orphanized = orphanized;
while (*p) {
@@ -3188,7 +3198,7 @@ static int path_loop(struct send_ctx *sctx, struct fs_path *name,
while (ino != BTRFS_FIRST_FREE_OBJECTID) {
fs_path_reset(name);
- if (is_waiting_for_rm(sctx, ino))
+ if (is_waiting_for_rm(sctx, ino, gen))
break;
if (is_waiting_for_move(sctx, ino)) {
if (*ancestor_ino == 0)
@@ -3228,6 +3238,7 @@ static int apply_dir_move(struct send_ctx *sctx, struct pending_dir_move *pm)
u64 parent_ino, parent_gen;
struct waiting_dir_move *dm = NULL;
u64 rmdir_ino = 0;
+ u64 rmdir_gen;
u64 ancestor;
bool is_orphan;
int ret;
@@ -3242,6 +3253,7 @@ static int apply_dir_move(struct send_ctx *sctx, struct pending_dir_move *pm)
dm = get_waiting_dir_move(sctx, pm->ino);
ASSERT(dm);
rmdir_ino = dm->rmdir_ino;
+ rmdir_gen = dm->rmdir_gen;
is_orphan = dm->orphanized;
free_waiting_dir_move(sctx, dm);
@@ -3278,6 +3290,7 @@ static int apply_dir_move(struct send_ctx *sctx, struct pending_dir_move *pm)
dm = get_waiting_dir_move(sctx, pm->ino);
ASSERT(dm);
dm->rmdir_ino = rmdir_ino;
+ dm->rmdir_gen = rmdir_gen;
}
goto out;
}
@@ -3296,7 +3309,7 @@ static int apply_dir_move(struct send_ctx *sctx, struct pending_dir_move *pm)
struct orphan_dir_info *odi;
u64 gen;
- odi = get_orphan_dir_info(sctx, rmdir_ino);
+ odi = get_orphan_dir_info(sctx, rmdir_ino, rmdir_gen);
if (!odi) {
/* already deleted */
goto finish;
diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c
index 40f5b4dcb927..521f6c2091ad 100644
--- a/fs/btrfs/super.c
+++ b/fs/btrfs/super.c
@@ -1845,6 +1845,14 @@ static int btrfs_remount(struct super_block *sb, int *flags, char *data)
btrfs_scrub_cancel(fs_info);
btrfs_pause_balance(fs_info);
+ /*
+ * Pause the qgroup rescan worker if it is running. We don't want
+ * it to be still running after we are in RO mode, as after that,
+ * by the time we unmount, it might have left a transaction open,
+ * so we would leak the transaction and/or crash.
+ */
+ btrfs_qgroup_wait_for_completion(fs_info, false);
+
ret = btrfs_commit_super(fs_info);
if (ret)
goto restore;
diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c
index 95b6a4ea18f7..662711200eeb 100644
--- a/fs/btrfs/volumes.c
+++ b/fs/btrfs/volumes.c
@@ -4011,6 +4011,8 @@ int btrfs_recover_balance(struct btrfs_fs_info *fs_info)
btrfs_warn(fs_info,
"balance: cannot set exclusive op status, resume manually");
+ btrfs_release_path(path);
+
mutex_lock(&fs_info->balance_mutex);
BUG_ON(fs_info->balance_ctl);
spin_lock(&fs_info->balance_lock);
diff --git a/fs/crypto/hooks.c b/fs/crypto/hooks.c
index a3582f2ca376..f664370d39f3 100644
--- a/fs/crypto/hooks.c
+++ b/fs/crypto/hooks.c
@@ -59,8 +59,8 @@ int __fscrypt_prepare_link(struct inode *inode, struct inode *dir,
if (err)
return err;
- /* ... in case we looked up ciphertext name before key was added */
- if (dentry->d_flags & DCACHE_ENCRYPTED_NAME)
+ /* ... in case we looked up no-key name before key was added */
+ if (fscrypt_is_nokey_name(dentry))
return -ENOKEY;
if (!fscrypt_has_permitted_context(dir, inode))
@@ -84,9 +84,9 @@ int __fscrypt_prepare_rename(struct inode *old_dir, struct dentry *old_dentry,
if (err)
return err;
- /* ... in case we looked up ciphertext name(s) before key was added */
- if ((old_dentry->d_flags | new_dentry->d_flags) &
- DCACHE_ENCRYPTED_NAME)
+ /* ... in case we looked up no-key name(s) before key was added */
+ if (fscrypt_is_nokey_name(old_dentry) ||
+ fscrypt_is_nokey_name(new_dentry))
return -ENOKEY;
if (old_dir != new_dir) {
diff --git a/fs/exec.c b/fs/exec.c
index 17d7f8a66167..57c5933326c3 100644
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -1011,7 +1011,7 @@ static int exec_mmap(struct mm_struct *mm)
/* Notify parent that we're no longer interested in the old VM */
tsk = current;
old_mm = current->mm;
- mm_release(tsk, old_mm);
+ exec_mm_release(tsk, old_mm);
if (old_mm) {
sync_mm_rss(old_mm);
diff --git a/fs/ext4/ioctl.c b/fs/ext4/ioctl.c
index 692e0df72875..8c632208a251 100644
--- a/fs/ext4/ioctl.c
+++ b/fs/ext4/ioctl.c
@@ -1110,7 +1110,10 @@ resizefs_out:
err = ext4_journal_get_write_access(handle, sbi->s_sbh);
if (err)
goto pwsalt_err_journal;
+ lock_buffer(sbi->s_sbh);
generate_random_uuid(sbi->s_es->s_encrypt_pw_salt);
+ ext4_superblock_csum_set(sb);
+ unlock_buffer(sbi->s_sbh);
err = ext4_handle_dirty_metadata(handle, NULL,
sbi->s_sbh);
pwsalt_err_journal:
diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c
index ef6635131294..7c0532c55559 100644
--- a/fs/ext4/namei.c
+++ b/fs/ext4/namei.c
@@ -2310,6 +2310,9 @@ static int ext4_add_entry(handle_t *handle, struct dentry *dentry,
return -EINVAL;
#endif
+ if (fscrypt_is_nokey_name(dentry))
+ return -ENOKEY;
+
retval = ext4_fname_setup_filename(dir, &dentry->d_name, 0, &fname);
if (retval)
return retval;
@@ -3670,8 +3673,6 @@ static int ext4_setent(handle_t *handle, struct ext4_renament *ent,
return retval;
}
}
- brelse(ent->bh);
- ent->bh = NULL;
return 0;
}
@@ -3874,6 +3875,7 @@ static int ext4_rename(struct inode *old_dir, struct dentry *old_dentry,
}
}
+ old_file_type = old.de->file_type;
if (IS_DIRSYNC(old.dir) || IS_DIRSYNC(new.dir))
ext4_handle_sync(handle);
@@ -3901,7 +3903,6 @@ static int ext4_rename(struct inode *old_dir, struct dentry *old_dentry,
force_reread = (new.dir->i_ino == old.dir->i_ino &&
ext4_test_inode_flag(new.dir, EXT4_INODE_INLINE_DATA));
- old_file_type = old.de->file_type;
if (whiteout) {
/*
* Do this before adding a new entry, so the old entry is sure
@@ -3973,15 +3974,19 @@ static int ext4_rename(struct inode *old_dir, struct dentry *old_dentry,
retval = 0;
end_rename:
+ if (whiteout) {
+ if (retval) {
+ ext4_setent(handle, &old,
+ old.inode->i_ino, old_file_type);
+ drop_nlink(whiteout);
+ }
+ unlock_new_inode(whiteout);
+ iput(whiteout);
+
+ }
brelse(old.dir_bh);
brelse(old.bh);
brelse(new.bh);
- if (whiteout) {
- if (retval)
- drop_nlink(whiteout);
- unlock_new_inode(whiteout);
- iput(whiteout);
- }
if (handle)
ext4_journal_stop(handle);
return retval;
diff --git a/fs/ext4/super.c b/fs/ext4/super.c
index 9462b900f9a7..4f990006cbdb 100644
--- a/fs/ext4/super.c
+++ b/fs/ext4/super.c
@@ -455,19 +455,17 @@ static bool system_going_down(void)
static void ext4_handle_error(struct super_block *sb)
{
+ journal_t *journal = EXT4_SB(sb)->s_journal;
+
if (test_opt(sb, WARN_ON_ERROR))
WARN_ON_ONCE(1);
- if (sb_rdonly(sb))
+ if (sb_rdonly(sb) || test_opt(sb, ERRORS_CONT))
return;
- if (!test_opt(sb, ERRORS_CONT)) {
- journal_t *journal = EXT4_SB(sb)->s_journal;
-
- EXT4_SB(sb)->s_mount_flags |= EXT4_MF_FS_ABORTED;
- if (journal)
- jbd2_journal_abort(journal, -EIO);
- }
+ EXT4_SB(sb)->s_mount_flags |= EXT4_MF_FS_ABORTED;
+ if (journal)
+ jbd2_journal_abort(journal, -EIO);
/*
* We force ERRORS_RO behavior when system is rebooting. Otherwise we
* could panic during 'reboot -f' as the underlying device got already
diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h
index 07c443570207..70aa653c4aa9 100644
--- a/fs/f2fs/f2fs.h
+++ b/fs/f2fs/f2fs.h
@@ -3231,6 +3231,8 @@ bool f2fs_empty_dir(struct inode *dir);
static inline int f2fs_add_link(struct dentry *dentry, struct inode *inode)
{
+ if (fscrypt_is_nokey_name(dentry))
+ return -ENOKEY;
return f2fs_do_add_link(d_inode(dentry->d_parent), &dentry->d_name,
inode, inode->i_ino, inode->i_mode);
}
diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c
index afc055fb27a2..0d1a8fa80fe5 100644
--- a/fs/f2fs/super.c
+++ b/fs/f2fs/super.c
@@ -1731,6 +1731,9 @@ restore_flag:
static void f2fs_enable_checkpoint(struct f2fs_sb_info *sbi)
{
+ /* we should flush all the data to keep data consistency */
+ sync_inodes_sb(sbi->sb);
+
down_write(&sbi->gc_lock);
f2fs_dirty_to_prefree(sbi);
diff --git a/fs/fcntl.c b/fs/fcntl.c
index 4137d96534a6..e039af1872ab 100644
--- a/fs/fcntl.c
+++ b/fs/fcntl.c
@@ -779,9 +779,10 @@ void send_sigio(struct fown_struct *fown, int fd, int band)
{
struct task_struct *p;
enum pid_type type;
+ unsigned long flags;
struct pid *pid;
- read_lock(&fown->lock);
+ read_lock_irqsave(&fown->lock, flags);
type = fown->pid_type;
pid = fown->pid;
@@ -802,7 +803,7 @@ void send_sigio(struct fown_struct *fown, int fd, int band)
read_unlock(&tasklist_lock);
}
out_unlock_fown:
- read_unlock(&fown->lock);
+ read_unlock_irqrestore(&fown->lock, flags);
}
static void send_sigurg_to_task(struct task_struct *p,
@@ -817,9 +818,10 @@ int send_sigurg(struct fown_struct *fown)
struct task_struct *p;
enum pid_type type;
struct pid *pid;
+ unsigned long flags;
int ret = 0;
- read_lock(&fown->lock);
+ read_lock_irqsave(&fown->lock, flags);
type = fown->pid_type;
pid = fown->pid;
@@ -842,7 +844,7 @@ int send_sigurg(struct fown_struct *fown)
read_unlock(&tasklist_lock);
}
out_unlock_fown:
- read_unlock(&fown->lock);
+ read_unlock_irqrestore(&fown->lock, flags);
return ret;
}
diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c
index 96cdce0144ef..f2d0c4acb3cb 100644
--- a/fs/fs-writeback.c
+++ b/fs/fs-writeback.c
@@ -1392,22 +1392,26 @@ __writeback_single_inode(struct inode *inode, struct writeback_control *wbc)
ret = err;
}
+ /*
+ * If the inode has dirty timestamps and we need to write them, call
+ * mark_inode_dirty_sync() to notify the filesystem about it and to
+ * change I_DIRTY_TIME into I_DIRTY_SYNC.
+ */
+ if ((inode->i_state & I_DIRTY_TIME) &&
+ (wbc->sync_mode == WB_SYNC_ALL || wbc->for_sync ||
+ time_after(jiffies, inode->dirtied_time_when +
+ dirtytime_expire_interval * HZ))) {
+ trace_writeback_lazytime(inode);
+ mark_inode_dirty_sync(inode);
+ }
+
/*
* Some filesystems may redirty the inode during the writeback
* due to delalloc, clear dirty metadata flags right before
* write_inode()
*/
spin_lock(&inode->i_lock);
-
dirty = inode->i_state & I_DIRTY;
- if ((inode->i_state & I_DIRTY_TIME) &&
- ((dirty & I_DIRTY_INODE) ||
- wbc->sync_mode == WB_SYNC_ALL || wbc->for_sync ||
- time_after(jiffies, inode->dirtied_time_when +
- dirtytime_expire_interval * HZ))) {
- dirty |= I_DIRTY_TIME;
- trace_writeback_lazytime(inode);
- }
inode->i_state &= ~dirty;
/*
@@ -1428,8 +1432,6 @@ __writeback_single_inode(struct inode *inode, struct writeback_control *wbc)
spin_unlock(&inode->i_lock);
- if (dirty & I_DIRTY_TIME)
- mark_inode_dirty_sync(inode);
/* Don't write the inode if only I_DIRTY_PAGES was set */
if (dirty & ~I_DIRTY_PAGES) {
int err = write_inode(inode, wbc);
diff --git a/fs/incfs/Kconfig b/fs/incfs/Kconfig
index 8f7d5226266b..1ffe31a9c0ff 100644
--- a/fs/incfs/Kconfig
+++ b/fs/incfs/Kconfig
@@ -2,9 +2,13 @@ config INCREMENTAL_FS
tristate "Incremental file system support"
depends on BLOCK
select DECOMPRESS_LZ4
- select DECOMPRESS_ZSTD
- select CRYPTO_ZSTD
+ select CRC32
+ select CRYPTO
+ select CRYPTO_RSA
select CRYPTO_SHA256
+ select X509_CERTIFICATE_PARSER
+ select ASYMMETRIC_KEY_TYPE
+ select ASYMMETRIC_PUBLIC_KEY_SUBTYPE
help
Incremental FS is a read-only virtual file system that facilitates execution
of programs while their binaries are still being lazily downloaded over the
diff --git a/fs/incfs/Makefile b/fs/incfs/Makefile
index 0c3f6d348bec..8d734bf91ecd 100644
--- a/fs/incfs/Makefile
+++ b/fs/incfs/Makefile
@@ -6,5 +6,4 @@ incrementalfs-y := \
format.o \
integrity.o \
main.o \
- pseudo_files.o \
vfs.o
diff --git a/fs/incfs/data_mgmt.c b/fs/incfs/data_mgmt.c
index 66a809606fd6..074a733c7001 100644
--- a/fs/incfs/data_mgmt.c
+++ b/fs/incfs/data_mgmt.c
@@ -3,13 +3,11 @@
* Copyright 2019 Google LLC
*/
#include
-#include
#include
#include
#include
#include
#include
-#include
#include
#include
#include
@@ -19,8 +17,6 @@
#include "format.h"
#include "integrity.h"
-static int incfs_scan_metadata_chain(struct data_file *df);
-
static void log_wake_up_all(struct work_struct *work)
{
struct delayed_work *dw = container_of(work, struct delayed_work, work);
@@ -28,19 +24,6 @@ static void log_wake_up_all(struct work_struct *work)
wake_up_all(&rl->ml_notif_wq);
}
-static void zstd_free_workspace(struct work_struct *work)
-{
- struct delayed_work *dw = container_of(work, struct delayed_work, work);
- struct mount_info *mi =
- container_of(dw, struct mount_info, mi_zstd_cleanup_work);
-
- mutex_lock(&mi->mi_zstd_workspace_mutex);
- kvfree(mi->mi_zstd_workspace);
- mi->mi_zstd_workspace = NULL;
- mi->mi_zstd_stream = NULL;
- mutex_unlock(&mi->mi_zstd_workspace_mutex);
-}
-
struct mount_info *incfs_alloc_mount_info(struct super_block *sb,
struct mount_options *options,
struct path *backing_dir_path)
@@ -57,17 +40,12 @@ struct mount_info *incfs_alloc_mount_info(struct super_block *sb,
mi->mi_owner = get_current_cred();
path_get(&mi->mi_backing_dir_path);
mutex_init(&mi->mi_dir_struct_mutex);
+ mutex_init(&mi->mi_pending_reads_mutex);
init_waitqueue_head(&mi->mi_pending_reads_notif_wq);
init_waitqueue_head(&mi->mi_log.ml_notif_wq);
- init_waitqueue_head(&mi->mi_blocks_written_notif_wq);
- atomic_set(&mi->mi_blocks_written, 0);
INIT_DELAYED_WORK(&mi->mi_log.ml_wakeup_work, log_wake_up_all);
spin_lock_init(&mi->mi_log.rl_lock);
- spin_lock_init(&mi->pending_read_lock);
INIT_LIST_HEAD(&mi->mi_reads_list_head);
- spin_lock_init(&mi->mi_per_uid_read_timeouts_lock);
- mutex_init(&mi->mi_zstd_workspace_mutex);
- INIT_DELAYED_WORK(&mi->mi_zstd_cleanup_work, zstd_free_workspace);
error = incfs_realloc_mount_info(mi, options);
if (error)
@@ -127,106 +105,28 @@ void incfs_free_mount_info(struct mount_info *mi)
return;
flush_delayed_work(&mi->mi_log.ml_wakeup_work);
- flush_delayed_work(&mi->mi_zstd_cleanup_work);
dput(mi->mi_index_dir);
- dput(mi->mi_incomplete_dir);
path_put(&mi->mi_backing_dir_path);
mutex_destroy(&mi->mi_dir_struct_mutex);
- mutex_destroy(&mi->mi_zstd_workspace_mutex);
+ mutex_destroy(&mi->mi_pending_reads_mutex);
put_cred(mi->mi_owner);
kfree(mi->mi_log.rl_ring_buf);
kfree(mi->log_xattr);
kfree(mi->pending_read_xattr);
- kfree(mi->mi_per_uid_read_timeouts);
kfree(mi);
}
static void data_file_segment_init(struct data_file_segment *segment)
{
init_waitqueue_head(&segment->new_data_arrival_wq);
- init_rwsem(&segment->rwsem);
+ mutex_init(&segment->blockmap_mutex);
INIT_LIST_HEAD(&segment->reads_list_head);
}
-char *file_id_to_str(incfs_uuid_t id)
+static void data_file_segment_destroy(struct data_file_segment *segment)
{
- char *result = kmalloc(1 + sizeof(id.bytes) * 2, GFP_NOFS);
- char *end;
-
- if (!result)
- return NULL;
-
- end = bin2hex(result, id.bytes, sizeof(id.bytes));
- *end = 0;
- return result;
-}
-
-struct dentry *incfs_lookup_dentry(struct dentry *parent, const char *name)
-{
- struct inode *inode;
- struct dentry *result = NULL;
-
- if (!parent)
- return ERR_PTR(-EFAULT);
-
- inode = d_inode(parent);
- inode_lock_nested(inode, I_MUTEX_PARENT);
- result = lookup_one_len(name, parent, strlen(name));
- inode_unlock(inode);
-
- if (IS_ERR(result))
- pr_warn("%s err:%ld\n", __func__, PTR_ERR(result));
-
- return result;
-}
-
-static struct data_file *handle_mapped_file(struct mount_info *mi,
- struct data_file *df)
-{
- char *file_id_str;
- struct dentry *index_file_dentry;
- struct path path;
- struct file *bf;
- struct data_file *result = NULL;
-
- file_id_str = file_id_to_str(df->df_id);
- if (!file_id_str)
- return ERR_PTR(-ENOENT);
-
- index_file_dentry = incfs_lookup_dentry(mi->mi_index_dir,
- file_id_str);
- kfree(file_id_str);
- if (!index_file_dentry)
- return ERR_PTR(-ENOENT);
- if (IS_ERR(index_file_dentry))
- return (struct data_file *)index_file_dentry;
- if (!d_really_is_positive(index_file_dentry)) {
- result = ERR_PTR(-ENOENT);
- goto out;
- }
-
- path = (struct path) {
- .mnt = mi->mi_backing_dir_path.mnt,
- .dentry = index_file_dentry
- };
-
- bf = dentry_open(&path, O_RDWR | O_NOATIME | O_LARGEFILE, mi->mi_owner);
- if (IS_ERR(bf)) {
- result = (struct data_file *)bf;
- goto out;
- }
-
- result = incfs_open_data_file(mi, bf);
- fput(bf);
- if (IS_ERR(result))
- goto out;
-
- result->df_mapped_offset = df->df_metadata_off;
-
-out:
- dput(index_file_dentry);
- return result;
+ mutex_destroy(&segment->blockmap_mutex);
}
struct data_file *incfs_open_data_file(struct mount_info *mi, struct file *bf)
@@ -244,7 +144,7 @@ struct data_file *incfs_open_data_file(struct mount_info *mi, struct file *bf)
if (!S_ISREG(bf->f_inode->i_mode))
return ERR_PTR(-EBADF);
- bfc = incfs_alloc_bfc(mi, bf);
+ bfc = incfs_alloc_bfc(bf);
if (IS_ERR(bfc))
return ERR_CAST(bfc);
@@ -259,8 +159,12 @@ struct data_file *incfs_open_data_file(struct mount_info *mi, struct file *bf)
for (i = 0; i < ARRAY_SIZE(df->df_segments); i++)
data_file_segment_init(&df->df_segments[i]);
+ error = mutex_lock_interruptible(&bfc->bc_mutex);
+ if (error)
+ goto out;
error = incfs_read_file_header(bfc, &df->df_metadata_off, &df->df_id,
&size, &df->df_header_flags);
+ mutex_unlock(&bfc->bc_mutex);
if (error)
goto out;
@@ -269,13 +173,6 @@ struct data_file *incfs_open_data_file(struct mount_info *mi, struct file *bf)
if (size > 0)
df->df_data_block_count = get_blocks_count_for_size(size);
- if (df->df_header_flags & INCFS_FILE_MAPPED) {
- struct data_file *mapped_df = handle_mapped_file(mi, df);
-
- incfs_free_data_file(df);
- return mapped_df;
- }
-
md_records = incfs_scan_metadata_chain(df);
if (md_records < 0)
error = md_records;
@@ -293,36 +190,15 @@ out:
void incfs_free_data_file(struct data_file *df)
{
- u32 data_blocks_written, hash_blocks_written;
+ int i;
if (!df)
return;
- data_blocks_written = atomic_read(&df->df_data_blocks_written);
- hash_blocks_written = atomic_read(&df->df_hash_blocks_written);
-
- if (data_blocks_written != df->df_initial_data_blocks_written ||
- hash_blocks_written != df->df_initial_hash_blocks_written) {
- struct backing_file_context *bfc = df->df_backing_file_context;
- int error = -1;
-
- if (bfc && !mutex_lock_interruptible(&bfc->bc_mutex)) {
- error = incfs_write_status_to_backing_file(
- df->df_backing_file_context,
- df->df_status_offset,
- data_blocks_written,
- hash_blocks_written);
- mutex_unlock(&bfc->bc_mutex);
- }
-
- if (error)
- /* Nothing can be done, just warn */
- pr_warn("incfs: failed to write status to backing file\n");
- }
-
incfs_free_mtree(df->df_hash_tree);
+ for (i = 0; i < ARRAY_SIZE(df->df_segments); i++)
+ data_file_segment_destroy(&df->df_segments[i]);
incfs_free_bfc(df->df_backing_file_context);
- kfree(df->df_signature);
kfree(df);
}
@@ -375,73 +251,16 @@ void incfs_free_dir_file(struct dir_file *dir)
kfree(dir);
}
-static ssize_t zstd_decompress_safe(struct mount_info *mi,
- struct mem_range src, struct mem_range dst)
+static ssize_t decompress(struct mem_range src, struct mem_range dst)
{
- ssize_t result;
- ZSTD_inBuffer inbuf = {.src = src.data, .size = src.len};
- ZSTD_outBuffer outbuf = {.dst = dst.data, .size = dst.len};
+ int result = LZ4_decompress_safe(src.data, dst.data, src.len, dst.len);
- result = mutex_lock_interruptible(&mi->mi_zstd_workspace_mutex);
- if (result)
- return result;
+ if (result < 0)
+ return -EBADMSG;
- if (!mi->mi_zstd_stream) {
- unsigned int workspace_size = ZSTD_DStreamWorkspaceBound(
- INCFS_DATA_FILE_BLOCK_SIZE);
- void *workspace = kvmalloc(workspace_size, GFP_NOFS);
- ZSTD_DStream *stream;
-
- if (!workspace) {
- result = -ENOMEM;
- goto out;
- }
-
- stream = ZSTD_initDStream(INCFS_DATA_FILE_BLOCK_SIZE, workspace,
- workspace_size);
- if (!stream) {
- kvfree(workspace);
- result = -EIO;
- goto out;
- }
-
- mi->mi_zstd_workspace = workspace;
- mi->mi_zstd_stream = stream;
- }
-
- result = ZSTD_decompressStream(mi->mi_zstd_stream, &outbuf, &inbuf) ?
- -EBADMSG : outbuf.pos;
-
- mod_delayed_work(system_wq, &mi->mi_zstd_cleanup_work,
- msecs_to_jiffies(5000));
-
-out:
- mutex_unlock(&mi->mi_zstd_workspace_mutex);
return result;
}
-static ssize_t decompress(struct mount_info *mi,
- struct mem_range src, struct mem_range dst, int alg)
-{
- int result;
-
- switch (alg) {
- case INCFS_BLOCK_COMPRESSED_LZ4:
- result = LZ4_decompress_safe(src.data, dst.data, src.len,
- dst.len);
- if (result < 0)
- return -EBADMSG;
- return result;
-
- case INCFS_BLOCK_COMPRESSED_ZSTD:
- return zstd_decompress_safe(mi, src, dst);
-
- default:
- WARN_ON(true);
- return -EOPNOTSUPP;
- }
-}
-
static void log_read_one_record(struct read_log *rl, struct read_log_state *rs)
{
union log_record *record =
@@ -494,7 +313,6 @@ static void log_block_read(struct mount_info *mi, incfs_uuid_t *id,
s64 relative_us;
union log_record record;
size_t record_size;
- uid_t uid = current_uid().val;
/*
* This may read the old value, but it's OK to delay the logging start
@@ -516,14 +334,12 @@ static void log_block_read(struct mount_info *mi, incfs_uuid_t *id,
relative_us = now_us - head->base_record.absolute_ts_us;
if (memcmp(id, &head->base_record.file_id, sizeof(incfs_uuid_t)) ||
- relative_us >= 1ll << 32 ||
- uid != head->base_record.uid) {
+ relative_us >= 1ll << 32) {
record.full_record = (struct full_record){
.type = FULL,
.block_index = block_index,
.file_id = *id,
.absolute_ts_us = now_us,
- .uid = uid,
};
head->base_record.file_id = *id;
record_size = sizeof(struct full_record);
@@ -571,8 +387,8 @@ static void log_block_read(struct mount_info *mi, incfs_uuid_t *id,
schedule_delayed_work(&log->ml_wakeup_work, msecs_to_jiffies(16));
}
-static int validate_hash_tree(struct backing_file_context *bfc, struct file *f,
- int block_index, struct mem_range data, u8 *buf)
+static int validate_hash_tree(struct file *bf, struct file *f, int block_index,
+ struct mem_range data, u8 *buf)
{
struct data_file *df = get_incfs_data_file(f);
u8 stored_digest[INCFS_MAX_HASH_SIZE] = {};
@@ -629,7 +445,7 @@ static int validate_hash_tree(struct backing_file_context *bfc, struct file *f,
if (page)
put_page(page);
- res = incfs_kread(bfc, buf, INCFS_DATA_FILE_BLOCK_SIZE,
+ res = incfs_kread(bf, buf, INCFS_DATA_FILE_BLOCK_SIZE,
hash_block_offset[lvl] + sig->hash_offset);
if (res < 0)
return res;
@@ -711,7 +527,9 @@ static void convert_data_file_block(struct incfs_blockmap_entry *bme,
res_block->db_backing_file_data_offset |=
le32_to_cpu(bme->me_data_offset_lo);
res_block->db_stored_size = le16_to_cpu(bme->me_data_size);
- res_block->db_comp_alg = flags & INCFS_BLOCK_COMPRESSED_MASK;
+ res_block->db_comp_alg = (flags & INCFS_BLOCK_COMPRESSED_LZ4) ?
+ COMPRESSION_LZ4 :
+ COMPRESSION_NONE;
}
static int get_data_file_block(struct data_file *df, int index,
@@ -761,9 +579,36 @@ static int copy_one_range(struct incfs_filled_range *range, void __user *buffer,
return 0;
}
+static int update_file_header_flags(struct data_file *df, u32 bits_to_reset,
+ u32 bits_to_set)
+{
+ int result;
+ u32 new_flags;
+ struct backing_file_context *bfc;
+
+ if (!df)
+ return -EFAULT;
+ bfc = df->df_backing_file_context;
+ if (!bfc)
+ return -EFAULT;
+
+ result = mutex_lock_interruptible(&bfc->bc_mutex);
+ if (result)
+ return result;
+
+ new_flags = (df->df_header_flags & ~bits_to_reset) | bits_to_set;
+ if (new_flags != df->df_header_flags) {
+ df->df_header_flags = new_flags;
+ result = incfs_write_file_header_flags(bfc, new_flags);
+ }
+
+ mutex_unlock(&bfc->bc_mutex);
+
+ return result;
+}
+
#define READ_BLOCKMAP_ENTRIES 512
int incfs_get_filled_blocks(struct data_file *df,
- struct incfs_file_data *fd,
struct incfs_get_filled_blocks_args *arg)
{
int error = 0;
@@ -777,8 +622,6 @@ int incfs_get_filled_blocks(struct data_file *df,
int i = READ_BLOCKMAP_ENTRIES - 1;
int entries_read = 0;
struct incfs_blockmap_entry *bme;
- int data_blocks_filled = 0;
- int hash_blocks_filled = 0;
*size_out = 0;
if (end_index > df->df_total_block_count)
@@ -786,8 +629,7 @@ int incfs_get_filled_blocks(struct data_file *df,
arg->total_blocks_out = df->df_total_block_count;
arg->data_blocks_out = df->df_data_block_count;
- if (atomic_read(&df->df_data_blocks_written) ==
- df->df_data_block_count) {
+ if (df->df_header_flags & INCFS_FILE_COMPLETE) {
pr_debug("File marked full, fast get_filled_blocks");
if (arg->start_index > end_index) {
arg->index_out = arg->start_index;
@@ -840,13 +682,6 @@ int incfs_get_filled_blocks(struct data_file *df,
convert_data_file_block(bme + i, &dfb);
- if (is_data_block_present(&dfb)) {
- if (arg->index_out >= df->df_data_block_count)
- ++hash_blocks_filled;
- else
- ++data_blocks_filled;
- }
-
if (is_data_block_present(&dfb) == in_range)
continue;
@@ -876,28 +711,13 @@ int incfs_get_filled_blocks(struct data_file *df,
arg->index_out = range.begin;
}
- if (arg->start_index == 0) {
- fd->fd_get_block_pos = 0;
- fd->fd_filled_data_blocks = 0;
- fd->fd_filled_hash_blocks = 0;
- }
-
- if (arg->start_index == fd->fd_get_block_pos) {
- fd->fd_get_block_pos = arg->index_out + 1;
- fd->fd_filled_data_blocks += data_blocks_filled;
- fd->fd_filled_hash_blocks += hash_blocks_filled;
- }
-
- if (fd->fd_get_block_pos == df->df_total_block_count + 1) {
- if (fd->fd_filled_data_blocks >
- atomic_read(&df->df_data_blocks_written))
- atomic_set(&df->df_data_blocks_written,
- fd->fd_filled_data_blocks);
-
- if (fd->fd_filled_hash_blocks >
- atomic_read(&df->df_hash_blocks_written))
- atomic_set(&df->df_hash_blocks_written,
- fd->fd_filled_hash_blocks);
+ if (!error && in_range && arg->start_index == 0 &&
+ end_index == df->df_total_block_count &&
+ *size_out == sizeof(struct incfs_filled_range)) {
+ int result =
+ update_file_header_flags(df, 0, INCFS_FILE_COMPLETE);
+ /* Log failure only, since it's just a failed optimization */
+ pr_debug("Marked file full with result %d", result);
}
kfree(bme);
@@ -935,31 +755,20 @@ static struct pending_read *add_pending_read(struct data_file *df,
result->file_id = df->df_id;
result->block_index = block_index;
result->timestamp_us = ktime_to_us(ktime_get());
- result->uid = current_uid().val;
- spin_lock(&mi->pending_read_lock);
+ mutex_lock(&mi->mi_pending_reads_mutex);
result->serial_number = ++mi->mi_last_pending_read_number;
mi->mi_pending_reads_count++;
- list_add_rcu(&result->mi_reads_list, &mi->mi_reads_list_head);
- list_add_rcu(&result->segment_reads_list, &segment->reads_list_head);
-
- spin_unlock(&mi->pending_read_lock);
+ list_add(&result->mi_reads_list, &mi->mi_reads_list_head);
+ list_add(&result->segment_reads_list, &segment->reads_list_head);
+ mutex_unlock(&mi->mi_pending_reads_mutex);
wake_up_all(&mi->mi_pending_reads_notif_wq);
return result;
}
-static void free_pending_read_entry(struct rcu_head *entry)
-{
- struct pending_read *read;
-
- read = container_of(entry, struct pending_read, rcu);
-
- kfree(read);
-}
-
/* Notifies a given data file that pending read is completed. */
static void remove_pending_read(struct data_file *df, struct pending_read *read)
{
@@ -973,17 +782,14 @@ static void remove_pending_read(struct data_file *df, struct pending_read *read)
mi = df->df_mount_info;
- spin_lock(&mi->pending_read_lock);
-
- list_del_rcu(&read->mi_reads_list);
- list_del_rcu(&read->segment_reads_list);
+ mutex_lock(&mi->mi_pending_reads_mutex);
+ list_del(&read->mi_reads_list);
+ list_del(&read->segment_reads_list);
mi->mi_pending_reads_count--;
+ mutex_unlock(&mi->mi_pending_reads_mutex);
- spin_unlock(&mi->pending_read_lock);
-
- /* Don't free. Wait for readers */
- call_rcu(&read->rcu, free_pending_read_entry);
+ kfree(read);
}
static void notify_pending_reads(struct mount_info *mi,
@@ -993,38 +799,18 @@ static void notify_pending_reads(struct mount_info *mi,
struct pending_read *entry = NULL;
/* Notify pending reads waiting for this block. */
- rcu_read_lock();
- list_for_each_entry_rcu(entry, &segment->reads_list_head,
+ mutex_lock(&mi->mi_pending_reads_mutex);
+ list_for_each_entry(entry, &segment->reads_list_head,
segment_reads_list) {
if (entry->block_index == index)
set_read_done(entry);
}
- rcu_read_unlock();
+ mutex_unlock(&mi->mi_pending_reads_mutex);
wake_up_all(&segment->new_data_arrival_wq);
-
- atomic_inc(&mi->mi_blocks_written);
- wake_up_all(&mi->mi_blocks_written_notif_wq);
-}
-
-static int usleep_interruptible(u32 us)
-{
- /* See:
- * https://www.kernel.org/doc/Documentation/timers/timers-howto.txt
- * for explanation
- */
- if (us < 10) {
- udelay(us);
- return 0;
- } else if (us < 20000) {
- usleep_range(us, us + us / 10);
- return 0;
- } else
- return msleep_interruptible(us / 1000);
}
static int wait_for_data_block(struct data_file *df, int block_index,
- u32 min_time_us, u32 min_pending_time_us,
- u32 max_pending_time_us,
+ int timeout_ms,
struct data_file_block *res_block)
{
struct data_file_block block = {};
@@ -1033,7 +819,6 @@ static int wait_for_data_block(struct data_file *df, int block_index,
struct mount_info *mi = NULL;
int error = 0;
int wait_res = 0;
- u64 time;
if (!df || !res_block)
return -EFAULT;
@@ -1041,53 +826,50 @@ static int wait_for_data_block(struct data_file *df, int block_index,
if (block_index < 0 || block_index >= df->df_data_block_count)
return -EINVAL;
- if (df->df_blockmap_off <= 0 || !df->df_mount_info)
+ if (df->df_blockmap_off <= 0)
return -ENODATA;
- mi = df->df_mount_info;
segment = get_file_segment(df, block_index);
-
- error = down_read_killable(&segment->rwsem);
+ error = mutex_lock_interruptible(&segment->blockmap_mutex);
if (error)
return error;
/* Look up the given block */
error = get_data_file_block(df, block_index, &block);
- up_read(&segment->rwsem);
+ /* If it's not found, create a pending read */
+ if (!error && !is_data_block_present(&block) && timeout_ms != 0)
+ read = add_pending_read(df, block_index);
+ mutex_unlock(&segment->blockmap_mutex);
if (error)
return error;
/* If the block was found, just return it. No need to wait. */
if (is_data_block_present(&block)) {
- if (min_time_us)
- error = usleep_interruptible(min_time_us);
*res_block = block;
- return error;
- } else {
- /* If it's not found, create a pending read */
- if (max_pending_time_us != 0) {
- read = add_pending_read(df, block_index);
- if (!read)
- return -ENOMEM;
- } else {
- log_block_read(mi, &df->df_id, block_index);
- return -ETIME;
- }
+ return 0;
}
- if (min_pending_time_us)
- time = ktime_get_ns();
+ mi = df->df_mount_info;
+
+ if (timeout_ms == 0) {
+ log_block_read(mi, &df->df_id, block_index);
+ return -ETIME;
+ }
+
+ if (!read)
+ return -ENOMEM;
/* Wait for notifications about block's arrival */
wait_res =
wait_event_interruptible_timeout(segment->new_data_arrival_wq,
- (is_read_done(read)),
- usecs_to_jiffies(max_pending_time_us));
+ (is_read_done(read)),
+ msecs_to_jiffies(timeout_ms));
/* Woke up, the pending read is no longer needed. */
remove_pending_read(df, read);
+ read = NULL;
if (wait_res == 0) {
/* Wait has timed out */
@@ -1102,17 +884,7 @@ static int wait_for_data_block(struct data_file *df, int block_index,
return wait_res;
}
- if (min_pending_time_us) {
- time = div_u64(ktime_get_ns() - time, 1000);
- if (min_pending_time_us > time) {
- error = usleep_interruptible(
- min_pending_time_us - time);
- if (error)
- return error;
- }
- }
-
- error = down_read_killable(&segment->rwsem);
+ error = mutex_lock_interruptible(&segment->blockmap_mutex);
if (error)
return error;
@@ -1134,55 +906,52 @@ static int wait_for_data_block(struct data_file *df, int block_index,
}
}
- up_read(&segment->rwsem);
+ mutex_unlock(&segment->blockmap_mutex);
return error;
}
ssize_t incfs_read_data_file_block(struct mem_range dst, struct file *f,
- int index, u32 min_time_us,
- u32 min_pending_time_us, u32 max_pending_time_us,
- struct mem_range tmp)
+ int index, int timeout_ms,
+ struct mem_range tmp)
{
loff_t pos;
ssize_t result;
size_t bytes_to_read;
struct mount_info *mi = NULL;
- struct backing_file_context *bfc = NULL;
+ struct file *bf = NULL;
struct data_file_block block = {};
struct data_file *df = get_incfs_data_file(f);
- if (!dst.data || !df || !tmp.data)
+ if (!dst.data || !df)
return -EFAULT;
if (tmp.len < 2 * INCFS_DATA_FILE_BLOCK_SIZE)
return -ERANGE;
mi = df->df_mount_info;
- bfc = df->df_backing_file_context;
+ bf = df->df_backing_file_context->bc_file;
- result = wait_for_data_block(df, index, min_time_us,
- min_pending_time_us, max_pending_time_us, &block);
+ result = wait_for_data_block(df, index, timeout_ms, &block);
if (result < 0)
goto out;
pos = block.db_backing_file_data_offset;
if (block.db_comp_alg == COMPRESSION_NONE) {
bytes_to_read = min(dst.len, block.db_stored_size);
- result = incfs_kread(bfc, dst.data, bytes_to_read, pos);
+ result = incfs_kread(bf, dst.data, bytes_to_read, pos);
/* Some data was read, but not enough */
if (result >= 0 && result != bytes_to_read)
result = -EIO;
} else {
bytes_to_read = min(tmp.len, block.db_stored_size);
- result = incfs_kread(bfc, tmp.data, bytes_to_read, pos);
+ result = incfs_kread(bf, tmp.data, bytes_to_read, pos);
if (result == bytes_to_read) {
result =
- decompress(mi, range(tmp.data, bytes_to_read),
- dst, block.db_comp_alg);
+ decompress(range(tmp.data, bytes_to_read), dst);
if (result < 0) {
const char *name =
- bfc->bc_file->f_path.dentry->d_name.name;
+ bf->f_path.dentry->d_name.name;
pr_warn_once("incfs: Decompression error. %s",
name);
@@ -1194,7 +963,7 @@ ssize_t incfs_read_data_file_block(struct mem_range dst, struct file *f,
}
if (result > 0) {
- int err = validate_hash_tree(bfc, f, index, dst, tmp.data);
+ int err = validate_hash_tree(bf, f, index, dst, tmp.data);
if (err < 0)
result = err;
@@ -1229,33 +998,21 @@ int incfs_process_new_data_block(struct data_file *df,
segment = get_file_segment(df, block->block_index);
if (!segment)
return -EFAULT;
-
if (block->compression == COMPRESSION_LZ4)
flags |= INCFS_BLOCK_COMPRESSED_LZ4;
- else if (block->compression == COMPRESSION_ZSTD)
- flags |= INCFS_BLOCK_COMPRESSED_ZSTD;
- else if (block->compression)
- return -EINVAL;
- error = down_read_killable(&segment->rwsem);
+ error = mutex_lock_interruptible(&segment->blockmap_mutex);
if (error)
return error;
error = get_data_file_block(df, block->block_index, &existing_block);
-
- up_read(&segment->rwsem);
-
if (error)
- return error;
+ goto unlock;
if (is_data_block_present(&existing_block)) {
/* Block is already present, nothing to do here */
- return 0;
+ goto unlock;
}
- error = down_write_killable(&segment->rwsem);
- if (error)
- return error;
-
error = mutex_lock_interruptible(&bfc->bc_mutex);
if (!error) {
error = incfs_write_data_block_to_backing_file(
@@ -1263,21 +1020,20 @@ int incfs_process_new_data_block(struct data_file *df,
df->df_blockmap_off, flags);
mutex_unlock(&bfc->bc_mutex);
}
- if (!error) {
+ if (!error)
notify_pending_reads(mi, segment, block->block_index);
- atomic_inc(&df->df_data_blocks_written);
- }
-
- up_write(&segment->rwsem);
+unlock:
+ mutex_unlock(&segment->blockmap_mutex);
if (error)
- pr_debug("%d error: %d\n", block->block_index, error);
+ pr_debug("incfs: %s %d error: %d\n", __func__,
+ block->block_index, error);
return error;
}
int incfs_read_file_signature(struct data_file *df, struct mem_range dst)
{
- struct backing_file_context *bfc = df->df_backing_file_context;
+ struct file *bf = df->df_backing_file_context->bc_file;
struct incfs_df_signature *sig;
int read_res = 0;
@@ -1291,7 +1047,7 @@ int incfs_read_file_signature(struct data_file *df, struct mem_range dst)
if (dst.len < sig->sig_size)
return -E2BIG;
- read_res = incfs_kread(bfc, dst.data, sig->sig_size, sig->sig_offset);
+ read_res = incfs_kread(bf, dst.data, sig->sig_size, sig->sig_offset);
if (read_res < 0)
return read_res;
@@ -1345,9 +1101,6 @@ int incfs_process_new_hash_block(struct data_file *df,
hash_area_base, df->df_blockmap_off, df->df_size);
mutex_unlock(&bfc->bc_mutex);
}
- if (!error)
- atomic_inc(&df->df_hash_blocks_written);
-
return error;
}
@@ -1370,6 +1123,25 @@ static int process_blockmap_md(struct incfs_blockmap *bm,
return error;
}
+static int process_file_attr_md(struct incfs_file_attr *fa,
+ struct metadata_handler *handler)
+{
+ struct data_file *df = handler->context;
+ u16 attr_size = le16_to_cpu(fa->fa_size);
+
+ if (!df)
+ return -EFAULT;
+
+ if (attr_size > INCFS_MAX_FILE_ATTR_SIZE)
+ return -E2BIG;
+
+ df->n_attr.fa_value_offset = le64_to_cpu(fa->fa_offset);
+ df->n_attr.fa_value_size = attr_size;
+ df->n_attr.fa_crc = le32_to_cpu(fa->fa_crc);
+
+ return 0;
+}
+
static int process_file_signature_md(struct incfs_file_signature *sg,
struct metadata_handler *handler)
{
@@ -1401,7 +1173,7 @@ static int process_file_signature_md(struct incfs_file_signature *sg,
goto out;
}
- read = incfs_kread(df->df_backing_file_context, buf,
+ read = incfs_kread(df->df_backing_file_context->bc_file, buf,
signature->sig_size, signature->sig_offset);
if (read < 0) {
error = read;
@@ -1445,27 +1217,7 @@ out:
return error;
}
-static int process_status_md(struct incfs_status *is,
- struct metadata_handler *handler)
-{
- struct data_file *df = handler->context;
-
- df->df_initial_data_blocks_written =
- le32_to_cpu(is->is_data_blocks_written);
- atomic_set(&df->df_data_blocks_written,
- df->df_initial_data_blocks_written);
-
- df->df_initial_hash_blocks_written =
- le32_to_cpu(is->is_hash_blocks_written);
- atomic_set(&df->df_hash_blocks_written,
- df->df_initial_hash_blocks_written);
-
- df->df_status_offset = handler->md_record_offset;
-
- return 0;
-}
-
-static int incfs_scan_metadata_chain(struct data_file *df)
+int incfs_scan_metadata_chain(struct data_file *df)
{
struct metadata_handler *handler = NULL;
int result = 0;
@@ -1482,12 +1234,20 @@ static int incfs_scan_metadata_chain(struct data_file *df)
if (!handler)
return -ENOMEM;
+ /* No writing to the backing file while it's being scanned. */
+ error = mutex_lock_interruptible(&bfc->bc_mutex);
+ if (error)
+ goto out;
+
+ /* Reading superblock */
handler->md_record_offset = df->df_metadata_off;
handler->context = df;
handler->handle_blockmap = process_blockmap_md;
+ handler->handle_file_attr = process_file_attr_md;
handler->handle_signature = process_file_signature_md;
- handler->handle_status = process_status_md;
+ pr_debug("incfs: Starting reading incfs-metadata records at offset %lld\n",
+ handler->md_record_offset);
while (handler->md_record_offset > 0) {
error = incfs_read_next_metadata_record(bfc, handler);
if (error) {
@@ -1499,11 +1259,15 @@ static int incfs_scan_metadata_chain(struct data_file *df)
records_count++;
}
if (error) {
- pr_warn("incfs: Error %d after reading %d incfs-metadata records.\n",
+ pr_debug("incfs: Error %d after reading %d incfs-metadata records.\n",
-error, records_count);
result = error;
- } else
+ } else {
+ pr_debug("incfs: Finished reading %d incfs-metadata records.\n",
+ records_count);
result = records_count;
+ }
+ mutex_unlock(&bfc->bc_mutex);
if (df->df_hash_tree) {
int hash_block_count = get_blocks_count_for_size(
@@ -1515,6 +1279,7 @@ static int incfs_scan_metadata_chain(struct data_file *df)
} else if (df->df_data_block_count != df->df_total_block_count)
result = -EINVAL;
+out:
kfree(handler);
return result;
}
@@ -1527,17 +1292,16 @@ bool incfs_fresh_pending_reads_exist(struct mount_info *mi, int last_number)
{
bool result = false;
- spin_lock(&mi->pending_read_lock);
+ mutex_lock(&mi->mi_pending_reads_mutex);
result = (mi->mi_last_pending_read_number > last_number) &&
- (mi->mi_pending_reads_count > 0);
- spin_unlock(&mi->pending_read_lock);
+ (mi->mi_pending_reads_count > 0);
+ mutex_unlock(&mi->mi_pending_reads_mutex);
return result;
}
int incfs_collect_pending_reads(struct mount_info *mi, int sn_lowerbound,
struct incfs_pending_read_info *reads,
- struct incfs_pending_read_info2 *reads2,
- int reads_size, int *new_max_sn)
+ int reads_size)
{
int reported_reads = 0;
struct pending_read *entry = NULL;
@@ -1548,43 +1312,29 @@ int incfs_collect_pending_reads(struct mount_info *mi, int sn_lowerbound,
if (reads_size <= 0)
return 0;
- if (!incfs_fresh_pending_reads_exist(mi, sn_lowerbound))
- return 0;
+ mutex_lock(&mi->mi_pending_reads_mutex);
- rcu_read_lock();
+ if (mi->mi_last_pending_read_number <= sn_lowerbound
+ || mi->mi_pending_reads_count == 0)
+ goto unlock;
- list_for_each_entry_rcu(entry, &mi->mi_reads_list_head, mi_reads_list) {
+ list_for_each_entry(entry, &mi->mi_reads_list_head, mi_reads_list) {
if (entry->serial_number <= sn_lowerbound)
continue;
- if (reads) {
- reads[reported_reads].file_id = entry->file_id;
- reads[reported_reads].block_index = entry->block_index;
- reads[reported_reads].serial_number =
- entry->serial_number;
- reads[reported_reads].timestamp_us =
- entry->timestamp_us;
- }
-
- if (reads2) {
- reads2[reported_reads].file_id = entry->file_id;
- reads2[reported_reads].block_index = entry->block_index;
- reads2[reported_reads].serial_number =
- entry->serial_number;
- reads2[reported_reads].timestamp_us =
- entry->timestamp_us;
- reads2[reported_reads].uid = entry->uid;
- }
-
- if (entry->serial_number > *new_max_sn)
- *new_max_sn = entry->serial_number;
+ reads[reported_reads].file_id = entry->file_id;
+ reads[reported_reads].block_index = entry->block_index;
+ reads[reported_reads].serial_number = entry->serial_number;
+ reads[reported_reads].timestamp_us = entry->timestamp_us;
+ /* reads[reported_reads].kind = INCFS_READ_KIND_PENDING; */
reported_reads++;
if (reported_reads >= reads_size)
break;
}
- rcu_read_unlock();
+unlock:
+ mutex_unlock(&mi->mi_pending_reads_mutex);
return reported_reads;
}
@@ -1620,9 +1370,8 @@ int incfs_get_uncollected_logs_count(struct mount_info *mi,
}
int incfs_collect_logged_reads(struct mount_info *mi,
- struct read_log_state *state,
+ struct read_log_state *reader_state,
struct incfs_pending_read_info *reads,
- struct incfs_pending_read_info2 *reads2,
int reads_size)
{
int dst_idx;
@@ -1633,51 +1382,45 @@ int incfs_collect_logged_reads(struct mount_info *mi,
head = &log->rl_head;
tail = &log->rl_tail;
- if (state->generation_id != head->generation_id) {
+ if (reader_state->generation_id != head->generation_id) {
pr_debug("read ptr is wrong generation: %u/%u",
- state->generation_id, head->generation_id);
+ reader_state->generation_id, head->generation_id);
- *state = (struct read_log_state){
+ *reader_state = (struct read_log_state){
.generation_id = head->generation_id,
};
}
- if (state->current_record_no < tail->current_record_no) {
+ if (reader_state->current_record_no < tail->current_record_no) {
pr_debug("read ptr is behind, moving: %u/%u -> %u/%u\n",
- (u32)state->next_offset,
- (u32)state->current_pass_no,
+ (u32)reader_state->next_offset,
+ (u32)reader_state->current_pass_no,
(u32)tail->next_offset, (u32)tail->current_pass_no);
- *state = *tail;
+ *reader_state = *tail;
}
for (dst_idx = 0; dst_idx < reads_size; dst_idx++) {
- if (state->current_record_no == head->current_record_no)
+ if (reader_state->current_record_no == head->current_record_no)
break;
- log_read_one_record(log, state);
+ log_read_one_record(log, reader_state);
- if (reads)
- reads[dst_idx] = (struct incfs_pending_read_info) {
- .file_id = state->base_record.file_id,
- .block_index = state->base_record.block_index,
- .serial_number = state->current_record_no,
- .timestamp_us =
- state->base_record.absolute_ts_us,
- };
-
- if (reads2)
- reads2[dst_idx] = (struct incfs_pending_read_info2) {
- .file_id = state->base_record.file_id,
- .block_index = state->base_record.block_index,
- .serial_number = state->current_record_no,
- .timestamp_us =
- state->base_record.absolute_ts_us,
- .uid = state->base_record.uid,
- };
+ reads[dst_idx] = (struct incfs_pending_read_info){
+ .file_id = reader_state->base_record.file_id,
+ .block_index = reader_state->base_record.block_index,
+ .serial_number = reader_state->current_record_no,
+ .timestamp_us = reader_state->base_record.absolute_ts_us
+ };
}
spin_unlock(&log->rl_lock);
return dst_idx;
}
+bool incfs_equal_ranges(struct mem_range lhs, struct mem_range rhs)
+{
+ if (lhs.len != rhs.len)
+ return false;
+ return memcmp(lhs.data, rhs.data, lhs.len) == 0;
+}
diff --git a/fs/incfs/data_mgmt.h b/fs/incfs/data_mgmt.h
index 76b16999f854..2726867835a8 100644
--- a/fs/incfs/data_mgmt.h
+++ b/fs/incfs/data_mgmt.h
@@ -10,12 +10,9 @@
#include
#include
#include
-#include
#include
#include
-#include
#include
-#include
#include
@@ -35,14 +32,13 @@ struct full_record {
u32 block_index : 30;
incfs_uuid_t file_id;
u64 absolute_ts_us;
- uid_t uid;
} __packed; /* 28 bytes */
struct same_file_record {
enum LOG_RECORD_TYPE type : 2; /* SAME_FILE */
u32 block_index : 30;
u32 relative_ts_us; /* max 2^32 us ~= 1 hour (1:11:30) */
-} __packed; /* 8 bytes */
+} __packed; /* 12 bytes */
struct same_file_next_block {
enum LOG_RECORD_TYPE type : 2; /* SAME_FILE_NEXT_BLOCK */
@@ -103,7 +99,8 @@ struct mount_options {
unsigned int readahead_pages;
unsigned int read_log_pages;
unsigned int read_log_wakeup_count;
- bool report_uid;
+ bool no_backing_file_cache;
+ bool no_backing_file_readahead;
};
struct mount_info {
@@ -113,8 +110,6 @@ struct mount_info {
struct dentry *mi_index_dir;
- struct dentry *mi_incomplete_dir;
-
const struct cred *mi_owner;
struct mount_options mi_options;
@@ -128,13 +123,13 @@ struct mount_info {
wait_queue_head_t mi_pending_reads_notif_wq;
/*
- * Protects - RCU safe:
+ * Protects:
* - reads_list_head
* - mi_pending_reads_count
* - mi_last_pending_read_number
* - data_file_segment.reads_list_head
*/
- spinlock_t pending_read_lock;
+ struct mutex mi_pending_reads_mutex;
/* List of active pending_read objects */
struct list_head mi_reads_list_head;
@@ -156,23 +151,6 @@ struct mount_info {
void *pending_read_xattr;
size_t pending_read_xattr_size;
-
- /* A queue of waiters who want to be notified about blocks_written */
- wait_queue_head_t mi_blocks_written_notif_wq;
-
- /* Number of blocks written since mount */
- atomic_t mi_blocks_written;
-
- /* Per UID read timeouts */
- spinlock_t mi_per_uid_read_timeouts_lock;
- struct incfs_per_uid_read_timeouts *mi_per_uid_read_timeouts;
- int mi_per_uid_read_timeouts_size;
-
- /* zstd workspace */
- struct mutex mi_zstd_workspace_mutex;
- void *mi_zstd_workspace;
- ZSTD_DStream *mi_zstd_stream;
- struct delayed_work mi_zstd_cleanup_work;
};
struct data_file_block {
@@ -194,20 +172,17 @@ struct pending_read {
int serial_number;
- uid_t uid;
-
struct list_head mi_reads_list;
struct list_head segment_reads_list;
-
- struct rcu_head rcu;
};
struct data_file_segment {
wait_queue_head_t new_data_arrival_wq;
/* Protects reads and writes from the blockmap */
- struct rw_semaphore rwsem;
+ /* Good candidate for read/write mutex */
+ struct mutex blockmap_mutex;
/* List of active pending_read objects belonging to this segment */
/* Protected by mount_info.pending_reads_mutex */
@@ -257,23 +232,7 @@ struct data_file {
/* Total number of blocks, data + hash */
int df_total_block_count;
- /* For mapped files, the offset into the actual file */
- loff_t df_mapped_offset;
-
- /* Number of data blocks written to file */
- atomic_t df_data_blocks_written;
-
- /* Number of data blocks in the status block */
- u32 df_initial_data_blocks_written;
-
- /* Number of hash blocks written to file */
- atomic_t df_hash_blocks_written;
-
- /* Number of hash blocks in the status block */
- u32 df_initial_hash_blocks_written;
-
- /* Offset to status metadata header */
- loff_t df_status_offset;
+ struct file_attr n_attr;
struct mtree *df_hash_tree;
@@ -300,23 +259,6 @@ struct dentry_info {
struct path backing_path;
};
-enum FILL_PERMISSION {
- CANT_FILL = 0,
- CAN_FILL = 1,
-};
-
-struct incfs_file_data {
- /* Does this file handle have INCFS_IOC_FILL_BLOCKS permission */
- enum FILL_PERMISSION fd_fill_permission;
-
- /* If INCFS_IOC_GET_FILLED_BLOCKS has been called, where are we */
- int fd_get_block_pos;
-
- /* And how many filled blocks are there up to that point */
- int fd_filled_data_blocks;
- int fd_filled_hash_blocks;
-};
-
struct mount_info *incfs_alloc_mount_info(struct super_block *sb,
struct mount_options *options,
struct path *backing_dir_path);
@@ -326,21 +268,19 @@ int incfs_realloc_mount_info(struct mount_info *mi,
void incfs_free_mount_info(struct mount_info *mi);
-char *file_id_to_str(incfs_uuid_t id);
-struct dentry *incfs_lookup_dentry(struct dentry *parent, const char *name);
struct data_file *incfs_open_data_file(struct mount_info *mi, struct file *bf);
void incfs_free_data_file(struct data_file *df);
+int incfs_scan_metadata_chain(struct data_file *df);
+
struct dir_file *incfs_open_dir_file(struct mount_info *mi, struct file *bf);
void incfs_free_dir_file(struct dir_file *dir);
ssize_t incfs_read_data_file_block(struct mem_range dst, struct file *f,
- int index, u32 min_time_us,
- u32 min_pending_time_us, u32 max_pending_time_us,
- struct mem_range tmp);
+ int index, int timeout_ms,
+ struct mem_range tmp);
int incfs_get_filled_blocks(struct data_file *df,
- struct incfs_file_data *fd,
struct incfs_get_filled_blocks_args *arg);
int incfs_read_file_signature(struct data_file *df, struct mem_range dst);
@@ -360,13 +300,11 @@ bool incfs_fresh_pending_reads_exist(struct mount_info *mi, int last_number);
*/
int incfs_collect_pending_reads(struct mount_info *mi, int sn_lowerbound,
struct incfs_pending_read_info *reads,
- struct incfs_pending_read_info2 *reads2,
- int reads_size, int *new_max_sn);
+ int reads_size);
int incfs_collect_logged_reads(struct mount_info *mi,
struct read_log_state *start_state,
struct incfs_pending_read_info *reads,
- struct incfs_pending_read_info2 *reads2,
int reads_size);
struct read_log_state incfs_get_log_state(struct mount_info *mi);
int incfs_get_uncollected_logs_count(struct mount_info *mi,
@@ -451,4 +389,6 @@ static inline int get_blocks_count_for_size(u64 size)
return 1 + (size - 1) / INCFS_DATA_FILE_BLOCK_SIZE;
}
+bool incfs_equal_ranges(struct mem_range lhs, struct mem_range rhs);
+
#endif /* _INCFS_DATA_MGMT_H */
diff --git a/fs/incfs/format.c b/fs/incfs/format.c
index da9e7b723a41..c56e559b6893 100644
--- a/fs/incfs/format.c
+++ b/fs/incfs/format.c
@@ -15,8 +15,7 @@
#include "format.h"
#include "data_mgmt.h"
-struct backing_file_context *incfs_alloc_bfc(struct mount_info *mi,
- struct file *backing_file)
+struct backing_file_context *incfs_alloc_bfc(struct file *backing_file)
{
struct backing_file_context *result = NULL;
@@ -25,7 +24,6 @@ struct backing_file_context *incfs_alloc_bfc(struct mount_info *mi,
return ERR_PTR(-ENOMEM);
result->bc_file = get_file(backing_file);
- result->bc_cred = mi->mi_owner;
mutex_init(&result->bc_mutex);
return result;
}
@@ -42,7 +40,7 @@ void incfs_free_bfc(struct backing_file_context *bfc)
kfree(bfc);
}
-static loff_t incfs_get_end_offset(struct file *f)
+loff_t incfs_get_end_offset(struct file *f)
{
/*
* This function assumes that file size and the end-offset
@@ -91,42 +89,11 @@ static int truncate_backing_file(struct backing_file_context *bfc,
return result;
}
-static int write_to_bf(struct backing_file_context *bfc, const void *buf,
- size_t count, loff_t pos)
-{
- ssize_t res = incfs_kwrite(bfc, buf, count, pos);
-
- if (res < 0)
- return res;
- if (res != count)
- return -EIO;
- return 0;
-}
-
-static int append_zeros_no_fallocate(struct backing_file_context *bfc,
- size_t file_size, size_t len)
-{
- u8 buffer[256] = {};
- size_t i;
-
- for (i = 0; i < len; i += sizeof(buffer)) {
- int to_write = len - i > sizeof(buffer)
- ? sizeof(buffer) : len - i;
- int err = write_to_bf(bfc, buffer, to_write, file_size + i);
-
- if (err)
- return err;
- }
-
- return 0;
-}
-
/* Append a given number of zero bytes to the end of the backing file. */
static int append_zeros(struct backing_file_context *bfc, size_t len)
{
loff_t file_size = 0;
loff_t new_last_byte_offset = 0;
- int result;
if (!bfc)
return -EFAULT;
@@ -149,7 +116,7 @@ static int append_zeros(struct backing_file_context *bfc, size_t len)
static int write_to_bf(struct backing_file_context *bfc, const void *buf,
size_t count, loff_t pos)
{
- ssize_t res = incfs_kwrite(bfc, buf, count, pos);
+ ssize_t res = incfs_kwrite(bfc->bc_file, buf, count, pos);
if (res < 0)
return res;
@@ -175,7 +142,7 @@ static u32 calc_md_crc(struct incfs_md_header *record)
record->h_record_crc = saved_crc;
record->h_next_md_offset = saved_md_offset;
- return append_zeros_no_fallocate(bfc, file_size, len);
+ return result;
}
/*
@@ -201,7 +168,9 @@ static int append_md_to_backing_file(struct backing_file_context *bfc,
record_size = le16_to_cpu(record->h_record_size);
file_pos = incfs_get_end_offset(bfc->bc_file);
+ record->h_prev_md_offset = cpu_to_le64(bfc->bc_last_md_record_offset);
record->h_next_md_offset = 0;
+ record->h_record_crc = cpu_to_le32(calc_md_crc(record));
/* Write the metadata record to the end of the backing file */
record_offset = file_pos;
@@ -235,6 +204,16 @@ static int append_md_to_backing_file(struct backing_file_context *bfc,
return result;
}
+int incfs_write_file_header_flags(struct backing_file_context *bfc, u32 flags)
+{
+ if (!bfc)
+ return -EFAULT;
+
+ return write_to_bf(bfc, &flags, sizeof(flags),
+ offsetof(struct incfs_file_header,
+ fh_file_header_flags));
+}
+
/*
* Reserve 0-filled space for the blockmap body, and append
* incfs_blockmap metadata record pointing to it.
@@ -273,6 +252,49 @@ int incfs_write_blockmap_to_backing_file(struct backing_file_context *bfc,
return result;
}
+/*
+ * Write file attribute data and metadata record to the backing file.
+ */
+int incfs_write_file_attr_to_backing_file(struct backing_file_context *bfc,
+ struct mem_range value, struct incfs_file_attr *attr)
+{
+ struct incfs_file_attr file_attr = {};
+ int result = 0;
+ u32 crc = 0;
+ loff_t value_offset = 0;
+
+ if (!bfc)
+ return -EFAULT;
+
+ if (value.len > INCFS_MAX_FILE_ATTR_SIZE)
+ return -ENOSPC;
+
+ LOCK_REQUIRED(bfc->bc_mutex);
+
+ crc = crc32(0, value.data, value.len);
+ value_offset = incfs_get_end_offset(bfc->bc_file);
+ file_attr.fa_header.h_md_entry_type = INCFS_MD_FILE_ATTR;
+ file_attr.fa_header.h_record_size = cpu_to_le16(sizeof(file_attr));
+ file_attr.fa_header.h_next_md_offset = cpu_to_le64(0);
+ file_attr.fa_size = cpu_to_le16((u16)value.len);
+ file_attr.fa_offset = cpu_to_le64(value_offset);
+ file_attr.fa_crc = cpu_to_le32(crc);
+
+ result = write_to_bf(bfc, value.data, value.len, value_offset);
+ if (result)
+ return result;
+
+ result = append_md_to_backing_file(bfc, &file_attr.fa_header);
+ if (result) {
+ /* Error, rollback file changes */
+ truncate_backing_file(bfc, value_offset);
+ } else if (attr) {
+ *attr = file_attr;
+ }
+
+ return result;
+}
+
int incfs_write_signature_to_backing_file(struct backing_file_context *bfc,
struct mem_range sig, u32 tree_size)
{
@@ -337,62 +359,10 @@ err:
return result;
}
-static int write_new_status_to_backing_file(struct backing_file_context *bfc,
- u32 data_blocks_written,
- u32 hash_blocks_written)
-{
- int result;
- loff_t rollback_pos;
- struct incfs_status is = {
- .is_header = {
- .h_md_entry_type = INCFS_MD_STATUS,
- .h_record_size = cpu_to_le16(sizeof(is)),
- },
- .is_data_blocks_written = cpu_to_le32(data_blocks_written),
- .is_hash_blocks_written = cpu_to_le32(hash_blocks_written),
- };
-
- if (!bfc)
- return -EFAULT;
-
- LOCK_REQUIRED(bfc->bc_mutex);
- rollback_pos = incfs_get_end_offset(bfc->bc_file);
- result = append_md_to_backing_file(bfc, &is.is_header);
- if (result)
- truncate_backing_file(bfc, rollback_pos);
-
- return result;
-}
-
-int incfs_write_status_to_backing_file(struct backing_file_context *bfc,
- loff_t status_offset,
- u32 data_blocks_written,
- u32 hash_blocks_written)
-{
- struct incfs_status is;
- int result;
-
- if (status_offset == 0)
- return write_new_status_to_backing_file(bfc,
- data_blocks_written, hash_blocks_written);
-
- result = incfs_kread(bfc, &is, sizeof(is), status_offset);
- if (result != sizeof(is))
- return -EIO;
-
- is.is_data_blocks_written = cpu_to_le32(data_blocks_written);
- is.is_hash_blocks_written = cpu_to_le32(hash_blocks_written);
- result = incfs_kwrite(bfc, &is, sizeof(is), status_offset);
- if (result != sizeof(is))
- return -EIO;
-
- return 0;
-}
-
/*
* Write a backing file header
* It should always be called only on empty file.
- * fh.fh_first_md_offset is 0 for now, but will be updated
+ * incfs_super_block.s_first_md_offset is 0 for now, but will be updated
* once first metadata record is added.
*/
int incfs_write_fh_to_backing_file(struct backing_file_context *bfc,
@@ -422,38 +392,6 @@ int incfs_write_fh_to_backing_file(struct backing_file_context *bfc,
return write_to_bf(bfc, &fh, sizeof(fh), file_pos);
}
-/*
- * Write a backing file header for a mapping file
- * It should always be called only on empty file.
- */
-int incfs_write_mapping_fh_to_backing_file(struct backing_file_context *bfc,
- incfs_uuid_t *uuid, u64 file_size, u64 offset)
-{
- struct incfs_file_header fh = {};
- loff_t file_pos = 0;
-
- if (!bfc)
- return -EFAULT;
-
- fh.fh_magic = cpu_to_le64(INCFS_MAGIC_NUMBER);
- fh.fh_version = cpu_to_le64(INCFS_FORMAT_CURRENT_VER);
- fh.fh_header_size = cpu_to_le16(sizeof(fh));
- fh.fh_original_offset = cpu_to_le64(offset);
- fh.fh_data_block_size = cpu_to_le16(INCFS_DATA_FILE_BLOCK_SIZE);
-
- fh.fh_mapped_file_size = cpu_to_le64(file_size);
- fh.fh_original_uuid = *uuid;
- fh.fh_flags = cpu_to_le32(INCFS_FILE_MAPPED);
-
- LOCK_REQUIRED(bfc->bc_mutex);
-
- file_pos = incfs_get_end_offset(bfc->bc_file);
- if (file_pos != 0)
- return -EEXIST;
-
- return write_to_bf(bfc, &fh, sizeof(fh), file_pos);
-}
-
/* Write a given data block and update file's blockmap to point it. */
int incfs_write_data_block_to_backing_file(struct backing_file_context *bfc,
struct mem_range block, int block_index,
@@ -529,10 +467,34 @@ int incfs_write_hash_block_to_backing_file(struct backing_file_context *bfc,
bm_entry.me_data_offset_lo = cpu_to_le32((u32)data_offset);
bm_entry.me_data_offset_hi = cpu_to_le16((u16)(data_offset >> 32));
bm_entry.me_data_size = cpu_to_le16(INCFS_DATA_FILE_BLOCK_SIZE);
+ bm_entry.me_flags = cpu_to_le16(INCFS_BLOCK_HASH);
return write_to_bf(bfc, &bm_entry, sizeof(bm_entry), bm_entry_off);
}
+/* Initialize a new image in a given backing file. */
+int incfs_make_empty_backing_file(struct backing_file_context *bfc,
+ incfs_uuid_t *uuid, u64 file_size)
+{
+ int result = 0;
+
+ if (!bfc || !bfc->bc_file)
+ return -EFAULT;
+
+ result = mutex_lock_interruptible(&bfc->bc_mutex);
+ if (result)
+ goto out;
+
+ result = truncate_backing_file(bfc, 0);
+ if (result)
+ goto out;
+
+ result = incfs_write_fh_to_backing_file(bfc, uuid, file_size);
+out:
+ mutex_unlock(&bfc->bc_mutex);
+ return result;
+}
+
int incfs_read_blockmap_entry(struct backing_file_context *bfc, int block_index,
loff_t bm_base_off,
struct incfs_blockmap_entry *bm_entry)
@@ -569,7 +531,8 @@ int incfs_read_blockmap_entries(struct backing_file_context *bfc,
if (start_index < 0 || bm_base_off <= 0)
return -ENODATA;
- result = incfs_kread(bfc, entries, bytes_to_read, bm_entry_off);
+ result = incfs_kread(bfc->bc_file, entries, bytes_to_read,
+ bm_entry_off);
if (result < 0)
return result;
return result / sizeof(*entries);
@@ -585,7 +548,8 @@ int incfs_read_file_header(struct backing_file_context *bfc,
if (!bfc || !first_md_off)
return -EFAULT;
- bytes_read = incfs_kread(bfc, &fh, sizeof(fh), 0);
+ LOCK_REQUIRED(bfc->bc_mutex);
+ bytes_read = incfs_kread(bfc->bc_file, &fh, sizeof(fh), 0);
if (bytes_read < 0)
return bytes_read;
@@ -611,7 +575,7 @@ int incfs_read_file_header(struct backing_file_context *bfc,
if (file_size)
*file_size = le64_to_cpu(fh.fh_file_size);
if (flags)
- *flags = le32_to_cpu(fh.fh_flags);
+ *flags = le32_to_cpu(fh.fh_file_header_flags);
return 0;
}
@@ -626,18 +590,21 @@ int incfs_read_next_metadata_record(struct backing_file_context *bfc,
ssize_t bytes_read = 0;
size_t md_record_size = 0;
loff_t next_record = 0;
+ loff_t prev_record = 0;
int res = 0;
struct incfs_md_header *md_hdr = NULL;
if (!bfc || !handler)
return -EFAULT;
+ LOCK_REQUIRED(bfc->bc_mutex);
+
if (handler->md_record_offset == 0)
return -EPERM;
memset(&handler->md_buffer, 0, max_md_size);
- bytes_read = incfs_kread(bfc, &handler->md_buffer, max_md_size,
- handler->md_record_offset);
+ bytes_read = incfs_kread(bfc->bc_file, &handler->md_buffer,
+ max_md_size, handler->md_record_offset);
if (bytes_read < 0)
return bytes_read;
if (bytes_read < sizeof(*md_hdr))
@@ -645,6 +612,7 @@ int incfs_read_next_metadata_record(struct backing_file_context *bfc,
md_hdr = &handler->md_buffer.md_header;
next_record = le64_to_cpu(md_hdr->h_next_md_offset);
+ prev_record = le64_to_cpu(md_hdr->h_prev_md_offset);
md_record_size = le16_to_cpu(md_hdr->h_record_size);
if (md_record_size > max_md_size) {
@@ -664,6 +632,16 @@ int incfs_read_next_metadata_record(struct backing_file_context *bfc,
return -EBADMSG;
}
+ if (prev_record != handler->md_prev_record_offset) {
+ pr_warn("incfs: Metadata chain has been corrupted.");
+ return -EBADMSG;
+ }
+
+ if (le32_to_cpu(md_hdr->h_record_crc) != calc_md_crc(md_hdr)) {
+ pr_warn("incfs: Metadata CRC mismatch.");
+ return -EBADMSG;
+ }
+
switch (md_hdr->h_md_entry_type) {
case INCFS_MD_NONE:
break;
@@ -673,21 +651,15 @@ int incfs_read_next_metadata_record(struct backing_file_context *bfc,
&handler->md_buffer.blockmap, handler);
break;
case INCFS_MD_FILE_ATTR:
- /*
- * File attrs no longer supported, ignore section for
- * compatibility
- */
+ if (handler->handle_file_attr)
+ res = handler->handle_file_attr(
+ &handler->md_buffer.file_attr, handler);
break;
case INCFS_MD_SIGNATURE:
if (handler->handle_signature)
res = handler->handle_signature(
&handler->md_buffer.signature, handler);
break;
- case INCFS_MD_STATUS:
- if (handler->handle_status)
- res = handler->handle_status(
- &handler->md_buffer.status, handler);
- break;
default:
res = -ENOTSUPP;
break;
@@ -708,22 +680,12 @@ int incfs_read_next_metadata_record(struct backing_file_context *bfc,
return res;
}
-ssize_t incfs_kread(struct backing_file_context *bfc, void *buf, size_t size,
- loff_t pos)
+ssize_t incfs_kread(struct file *f, void *buf, size_t size, loff_t pos)
{
- const struct cred *old_cred = override_creds(bfc->bc_cred);
- int ret = kernel_read(bfc->bc_file, buf, size, &pos);
-
- revert_creds(old_cred);
- return ret;
+ return kernel_read(f, buf, size, &pos);
}
-ssize_t incfs_kwrite(struct backing_file_context *bfc, const void *buf,
- size_t size, loff_t pos)
+ssize_t incfs_kwrite(struct file *f, const void *buf, size_t size, loff_t pos)
{
- const struct cred *old_cred = override_creds(bfc->bc_cred);
- int ret = kernel_write(bfc->bc_file, buf, size, &pos);
-
- revert_creds(old_cred);
- return ret;
+ return kernel_write(f, buf, size, &pos);
}
diff --git a/fs/incfs/format.h b/fs/incfs/format.h
index 13a69abaf535..1a83349bb2eb 100644
--- a/fs/incfs/format.h
+++ b/fs/incfs/format.h
@@ -72,7 +72,7 @@
*
*
* +-------------------------------------------+
- * | incfs_file_header |]---+
+ * | incfs_super_block |]---+
* +-------------------------------------------+ |
* | metadata |<---+
* | incfs_file_signature |]---+
@@ -118,12 +118,11 @@ enum incfs_metadata_type {
INCFS_MD_NONE = 0,
INCFS_MD_BLOCK_MAP = 1,
INCFS_MD_FILE_ATTR = 2,
- INCFS_MD_SIGNATURE = 3,
- INCFS_MD_STATUS = 4,
+ INCFS_MD_SIGNATURE = 3
};
enum incfs_file_header_flags {
- INCFS_FILE_MAPPED = 1 << 1,
+ INCFS_FILE_COMPLETE = 1 << 0,
};
/* Header included at the beginning of all metadata records on the disk. */
@@ -137,16 +136,16 @@ struct incfs_md_header {
__le16 h_record_size;
/*
- * Was: CRC32 of the metadata record.
+ * CRC32 of the metadata record.
* (e.g. inode, dir entry etc) not just this struct.
*/
- __le32 h_unused1;
+ __le32 h_record_crc;
/* Offset of the next metadata entry if any */
__le64 h_next_md_offset;
- /* Was: Offset of the previous metadata entry if any */
- __le64 h_unused2;
+ /* Offset of the previous metadata entry if any */
+ __le64 h_prev_md_offset;
} __packed;
@@ -165,41 +164,25 @@ struct incfs_file_header {
__le16 fh_data_block_size;
/* File flags, from incfs_file_header_flags */
- __le32 fh_flags;
+ __le32 fh_file_header_flags;
- union {
- /* Standard incfs file */
- struct {
- /* Offset of the first metadata record */
- __le64 fh_first_md_offset;
+ /* Offset of the first metadata record */
+ __le64 fh_first_md_offset;
- /* Full size of the file's content */
- __le64 fh_file_size;
+ /*
+ * Put file specific information after this point
+ */
- /* File uuid */
- incfs_uuid_t fh_uuid;
- };
+ /* Full size of the file's content */
+ __le64 fh_file_size;
- /* Mapped file - INCFS_FILE_MAPPED set in fh_flags */
- struct {
- /* Offset in original file */
- __le64 fh_original_offset;
-
- /* Full size of the file's content */
- __le64 fh_mapped_file_size;
-
- /* Original file's uuid */
- incfs_uuid_t fh_original_uuid;
- };
- };
+ /* File uuid */
+ incfs_uuid_t fh_uuid;
} __packed;
enum incfs_block_map_entry_flags {
- INCFS_BLOCK_COMPRESSED_LZ4 = 1,
- INCFS_BLOCK_COMPRESSED_ZSTD = 2,
-
- /* Reserve 3 bits for compression alg */
- INCFS_BLOCK_COMPRESSED_MASK = 7,
+ INCFS_BLOCK_COMPRESSED_LZ4 = (1 << 0),
+ INCFS_BLOCK_HASH = (1 << 1),
};
/* Block map entry pointing to an actual location of the data block. */
@@ -228,6 +211,17 @@ struct incfs_blockmap {
__le32 m_block_count;
} __packed;
+/* Metadata record for file attribute. Type = INCFS_MD_FILE_ATTR */
+struct incfs_file_attr {
+ struct incfs_md_header fa_header;
+
+ __le64 fa_offset;
+
+ __le16 fa_size;
+
+ __le32 fa_crc;
+} __packed;
+
/* Metadata record for file signature. Type = INCFS_MD_SIGNATURE */
struct incfs_file_signature {
struct incfs_md_header sg_header;
@@ -249,16 +243,6 @@ struct incfs_df_signature {
u64 hash_offset;
};
-struct incfs_status {
- struct incfs_md_header is_header;
-
- __le32 is_data_blocks_written; /* Number of data blocks written */
-
- __le32 is_hash_blocks_written; /* Number of hash blocks written */
-
- __le32 is_dummy[6]; /* Spare fields */
-} __packed;
-
/* State of the backing file. */
struct backing_file_context {
/* Protects writes to bc_file */
@@ -272,13 +256,6 @@ struct backing_file_context {
* 0 means there are no metadata records.
*/
loff_t bc_last_md_record_offset;
-
- /*
- * Credentials to set before reads/writes
- * Note that this is a pointer to the mount_info mi_owner field so
- * there is no need to get/put the creds
- */
- const struct cred *bc_cred;
};
struct metadata_handler {
@@ -289,24 +266,24 @@ struct metadata_handler {
union {
struct incfs_md_header md_header;
struct incfs_blockmap blockmap;
+ struct incfs_file_attr file_attr;
struct incfs_file_signature signature;
- struct incfs_status status;
} md_buffer;
int (*handle_blockmap)(struct incfs_blockmap *bm,
struct metadata_handler *handler);
- int (*handle_signature)(struct incfs_file_signature *sig,
+ int (*handle_file_attr)(struct incfs_file_attr *fa,
struct metadata_handler *handler);
- int (*handle_status)(struct incfs_status *sig,
+ int (*handle_signature)(struct incfs_file_signature *sig,
struct metadata_handler *handler);
};
#define INCFS_MAX_METADATA_RECORD_SIZE \
FIELD_SIZEOF(struct metadata_handler, md_buffer)
+loff_t incfs_get_end_offset(struct file *f);
+
/* Backing file context management */
-struct mount_info;
-struct backing_file_context *incfs_alloc_bfc(struct mount_info *mi,
- struct file *backing_file);
+struct backing_file_context *incfs_alloc_bfc(struct file *backing_file);
void incfs_free_bfc(struct backing_file_context *bfc);
@@ -317,9 +294,6 @@ int incfs_write_blockmap_to_backing_file(struct backing_file_context *bfc,
int incfs_write_fh_to_backing_file(struct backing_file_context *bfc,
incfs_uuid_t *uuid, u64 file_size);
-int incfs_write_mapping_fh_to_backing_file(struct backing_file_context *bfc,
- incfs_uuid_t *uuid, u64 file_size, u64 offset);
-
int incfs_write_data_block_to_backing_file(struct backing_file_context *bfc,
struct mem_range block,
int block_index, loff_t bm_base_off,
@@ -332,13 +306,16 @@ int incfs_write_hash_block_to_backing_file(struct backing_file_context *bfc,
loff_t bm_base_off,
loff_t file_size);
+int incfs_write_file_attr_to_backing_file(struct backing_file_context *bfc,
+ struct mem_range value, struct incfs_file_attr *attr);
+
int incfs_write_signature_to_backing_file(struct backing_file_context *bfc,
struct mem_range sig, u32 tree_size);
-int incfs_write_status_to_backing_file(struct backing_file_context *bfc,
- loff_t status_offset,
- u32 data_blocks_written,
- u32 hash_blocks_written);
+int incfs_write_file_header_flags(struct backing_file_context *bfc, u32 flags);
+
+int incfs_make_empty_backing_file(struct backing_file_context *bfc,
+ incfs_uuid_t *uuid, u64 file_size);
/* Reading stuff */
int incfs_read_file_header(struct backing_file_context *bfc,
@@ -357,9 +334,7 @@ int incfs_read_blockmap_entries(struct backing_file_context *bfc,
int incfs_read_next_metadata_record(struct backing_file_context *bfc,
struct metadata_handler *handler);
-ssize_t incfs_kread(struct backing_file_context *bfc, void *buf, size_t size,
- loff_t pos);
-ssize_t incfs_kwrite(struct backing_file_context *bfc, const void *buf,
- size_t size, loff_t pos);
+ssize_t incfs_kread(struct file *f, void *buf, size_t size, loff_t pos);
+ssize_t incfs_kwrite(struct file *f, const void *buf, size_t size, loff_t pos);
#endif /* _INCFS_FORMAT_H */
diff --git a/fs/incfs/main.c b/fs/incfs/main.c
index a0a6733911f4..e65d0d895128 100644
--- a/fs/incfs/main.c
+++ b/fs/incfs/main.c
@@ -22,39 +22,16 @@ static struct file_system_type incfs_fs_type = {
static struct kobject *sysfs_root, *featurefs_root;
-static ssize_t supported(struct kobject *kobj,
- struct kobj_attribute *attr, char *buff)
+static ssize_t corefs_show(struct kobject *kobj,
+ struct kobj_attribute *attr, char *buff)
{
return snprintf(buff, PAGE_SIZE, "supported\n");
}
-typedef ssize_t (*const attr_show)(struct kobject *kobj,
- struct kobj_attribute *attr, char *buff);
-
-#define _DECLARE_FEATURE_FLAG(name) \
- static attr_show name##_show = supported; \
- static struct kobj_attribute name##_attr = __ATTR_RO(name)
-
-#define DECLARE_FEATURE_FLAG(name) _DECLARE_FEATURE_FLAG(name)
-
-DECLARE_FEATURE_FLAG(corefs);
-DECLARE_FEATURE_FLAG(zstd);
-DECLARE_FEATURE_FLAG(v2);
-
-static ssize_t mounter_context_for_backing_rw_show(struct kobject *kobj,
- struct kobj_attribute *attr, char *buff)
-{
- return scnprintf(buff, PAGE_SIZE, "%s", "supported\n");
-}
-
-static struct kobj_attribute mounter_context_for_backing_rw_attr =
- __ATTR_RO(mounter_context_for_backing_rw);
+static struct kobj_attribute corefs_attr = __ATTR_RO(corefs);
static struct attribute *attributes[] = {
&corefs_attr.attr,
- &mounter_context_for_backing_rw_attr.attr,
- &zstd_attr.attr,
- &v2_attr.attr,
NULL,
};
diff --git a/fs/incfs/pseudo_files.c b/fs/incfs/pseudo_files.c
deleted file mode 100644
index fccf06b06dc0..000000000000
--- a/fs/incfs/pseudo_files.c
+++ /dev/null
@@ -1,1280 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright 2020 Google LLC
- */
-
-#include
-#include
-#include
-#include
-#include
-
-#include
-
-#include "pseudo_files.h"
-
-#include "data_mgmt.h"
-#include "format.h"
-#include "integrity.h"
-#include "vfs.h"
-
-#define INCFS_PENDING_READS_INODE 2
-#define INCFS_LOG_INODE 3
-#define INCFS_BLOCKS_WRITTEN_INODE 4
-#define READ_WRITE_FILE_MODE 0666
-
-/*******************************************************************************
- * .log pseudo file definition
- ******************************************************************************/
-static const char log_file_name[] = INCFS_LOG_FILENAME;
-static const struct mem_range log_file_name_range = {
- .data = (u8 *)log_file_name,
- .len = ARRAY_SIZE(log_file_name) - 1
-};
-
-/* State of an open .log file, unique for each file descriptor. */
-struct log_file_state {
- struct read_log_state state;
-};
-
-static ssize_t log_read(struct file *f, char __user *buf, size_t len,
- loff_t *ppos)
-{
- struct log_file_state *log_state = f->private_data;
- struct mount_info *mi = get_mount_info(file_superblock(f));
- int total_reads_collected = 0;
- int rl_size;
- ssize_t result = 0;
- bool report_uid;
- unsigned long page = 0;
- struct incfs_pending_read_info *reads_buf = NULL;
- struct incfs_pending_read_info2 *reads_buf2 = NULL;
- size_t record_size;
- ssize_t reads_to_collect;
- ssize_t reads_per_page;
-
- if (!mi)
- return -EFAULT;
-
- report_uid = mi->mi_options.report_uid;
- record_size = report_uid ? sizeof(*reads_buf2) : sizeof(*reads_buf);
- reads_to_collect = len / record_size;
- reads_per_page = PAGE_SIZE / record_size;
-
- rl_size = READ_ONCE(mi->mi_log.rl_size);
- if (rl_size == 0)
- return 0;
-
- page = __get_free_page(GFP_NOFS);
- if (!page)
- return -ENOMEM;
-
- if (report_uid)
- reads_buf2 = (struct incfs_pending_read_info2 *) page;
- else
- reads_buf = (struct incfs_pending_read_info *) page;
-
- reads_to_collect = min_t(ssize_t, rl_size, reads_to_collect);
- while (reads_to_collect > 0) {
- struct read_log_state next_state;
- int reads_collected;
-
- memcpy(&next_state, &log_state->state, sizeof(next_state));
- reads_collected = incfs_collect_logged_reads(
- mi, &next_state, reads_buf, reads_buf2,
- min_t(ssize_t, reads_to_collect, reads_per_page));
- if (reads_collected <= 0) {
- result = total_reads_collected ?
- total_reads_collected * record_size :
- reads_collected;
- goto out;
- }
- if (copy_to_user(buf, (void *) page,
- reads_collected * record_size)) {
- result = total_reads_collected ?
- total_reads_collected * record_size :
- -EFAULT;
- goto out;
- }
-
- memcpy(&log_state->state, &next_state, sizeof(next_state));
- total_reads_collected += reads_collected;
- buf += reads_collected * record_size;
- reads_to_collect -= reads_collected;
- }
-
- result = total_reads_collected * record_size;
- *ppos = 0;
-out:
- free_page(page);
- return result;
-}
-
-static __poll_t log_poll(struct file *file, poll_table *wait)
-{
- struct log_file_state *log_state = file->private_data;
- struct mount_info *mi = get_mount_info(file_superblock(file));
- int count;
- __poll_t ret = 0;
-
- poll_wait(file, &mi->mi_log.ml_notif_wq, wait);
- count = incfs_get_uncollected_logs_count(mi, &log_state->state);
- if (count >= mi->mi_options.read_log_wakeup_count)
- ret = EPOLLIN | EPOLLRDNORM;
-
- return ret;
-}
-
-static int log_open(struct inode *inode, struct file *file)
-{
- struct log_file_state *log_state = NULL;
- struct mount_info *mi = get_mount_info(file_superblock(file));
-
- log_state = kzalloc(sizeof(*log_state), GFP_NOFS);
- if (!log_state)
- return -ENOMEM;
-
- log_state->state = incfs_get_log_state(mi);
- file->private_data = log_state;
- return 0;
-}
-
-static int log_release(struct inode *inode, struct file *file)
-{
- kfree(file->private_data);
- return 0;
-}
-
-static const struct file_operations incfs_log_file_ops = {
- .read = log_read,
- .poll = log_poll,
- .open = log_open,
- .release = log_release,
- .llseek = noop_llseek,
-};
-
-/*******************************************************************************
- * .pending_reads pseudo file definition
- ******************************************************************************/
-static const char pending_reads_file_name[] = INCFS_PENDING_READS_FILENAME;
-static const struct mem_range pending_reads_file_name_range = {
- .data = (u8 *)pending_reads_file_name,
- .len = ARRAY_SIZE(pending_reads_file_name) - 1
-};
-
-/* State of an open .pending_reads file, unique for each file descriptor. */
-struct pending_reads_state {
- /* A serial number of the last pending read obtained from this file. */
- int last_pending_read_sn;
-};
-
-static ssize_t pending_reads_read(struct file *f, char __user *buf, size_t len,
- loff_t *ppos)
-{
- struct pending_reads_state *pr_state = f->private_data;
- struct mount_info *mi = get_mount_info(file_superblock(f));
- bool report_uid;
- unsigned long page = 0;
- struct incfs_pending_read_info *reads_buf = NULL;
- struct incfs_pending_read_info2 *reads_buf2 = NULL;
- size_t record_size;
- size_t reads_to_collect;
- int last_known_read_sn = READ_ONCE(pr_state->last_pending_read_sn);
- int new_max_sn = last_known_read_sn;
- int reads_collected = 0;
- ssize_t result = 0;
-
- if (!mi)
- return -EFAULT;
-
- report_uid = mi->mi_options.report_uid;
- record_size = report_uid ? sizeof(*reads_buf2) : sizeof(*reads_buf);
- reads_to_collect = len / record_size;
-
- if (!incfs_fresh_pending_reads_exist(mi, last_known_read_sn))
- return 0;
-
- page = get_zeroed_page(GFP_NOFS);
- if (!page)
- return -ENOMEM;
-
- if (report_uid)
- reads_buf2 = (struct incfs_pending_read_info2 *) page;
- else
- reads_buf = (struct incfs_pending_read_info *) page;
-
- reads_to_collect =
- min_t(size_t, PAGE_SIZE / record_size, reads_to_collect);
-
- reads_collected = incfs_collect_pending_reads(mi, last_known_read_sn,
- reads_buf, reads_buf2, reads_to_collect,
- &new_max_sn);
-
- if (reads_collected < 0) {
- result = reads_collected;
- goto out;
- }
-
- /*
- * Just to make sure that we don't accidentally copy more data
- * to reads buffer than userspace can handle.
- */
- reads_collected = min_t(size_t, reads_collected, reads_to_collect);
- result = reads_collected * record_size;
-
- /* Copy reads info to the userspace buffer */
- if (copy_to_user(buf, (void *)page, result)) {
- result = -EFAULT;
- goto out;
- }
-
- WRITE_ONCE(pr_state->last_pending_read_sn, new_max_sn);
- *ppos = 0;
-
-out:
- free_page(page);
- return result;
-}
-
-static __poll_t pending_reads_poll(struct file *file, poll_table *wait)
-{
- struct pending_reads_state *state = file->private_data;
- struct mount_info *mi = get_mount_info(file_superblock(file));
- __poll_t ret = 0;
-
- poll_wait(file, &mi->mi_pending_reads_notif_wq, wait);
- if (incfs_fresh_pending_reads_exist(mi,
- state->last_pending_read_sn))
- ret = EPOLLIN | EPOLLRDNORM;
-
- return ret;
-}
-
-static int pending_reads_open(struct inode *inode, struct file *file)
-{
- struct pending_reads_state *state = NULL;
-
- state = kzalloc(sizeof(*state), GFP_NOFS);
- if (!state)
- return -ENOMEM;
-
- file->private_data = state;
- return 0;
-}
-
-static int pending_reads_release(struct inode *inode, struct file *file)
-{
- kfree(file->private_data);
- return 0;
-}
-
-static long ioctl_permit_fill(struct file *f, void __user *arg)
-{
- struct incfs_permit_fill __user *usr_permit_fill = arg;
- struct incfs_permit_fill permit_fill;
- long error = 0;
- struct file *file = NULL;
- struct incfs_file_data *fd;
-
- if (copy_from_user(&permit_fill, usr_permit_fill, sizeof(permit_fill)))
- return -EFAULT;
-
- file = fget(permit_fill.file_descriptor);
- if (IS_ERR(file))
- return PTR_ERR(file);
-
- if (file->f_op != &incfs_file_ops) {
- error = -EPERM;
- goto out;
- }
-
- if (file->f_inode->i_sb != f->f_inode->i_sb) {
- error = -EPERM;
- goto out;
- }
-
- fd = file->private_data;
-
- switch (fd->fd_fill_permission) {
- case CANT_FILL:
- fd->fd_fill_permission = CAN_FILL;
- break;
-
- case CAN_FILL:
- pr_debug("CAN_FILL already set");
- break;
-
- default:
- pr_warn("Invalid file private data");
- error = -EFAULT;
- goto out;
- }
-
-out:
- fput(file);
- return error;
-}
-
-static int chmod(struct dentry *dentry, umode_t mode)
-{
- struct inode *inode = dentry->d_inode;
- struct inode *delegated_inode = NULL;
- struct iattr newattrs;
- int error;
-
-retry_deleg:
- inode_lock(inode);
- newattrs.ia_mode = (mode & S_IALLUGO) | (inode->i_mode & ~S_IALLUGO);
- newattrs.ia_valid = ATTR_MODE | ATTR_CTIME;
- error = notify_change(dentry, &newattrs, &delegated_inode);
- inode_unlock(inode);
- if (delegated_inode) {
- error = break_deleg_wait(&delegated_inode);
- if (!error)
- goto retry_deleg;
- }
- return error;
-}
-
-static bool incfs_equal_ranges(struct mem_range lhs, struct mem_range rhs)
-{
- if (lhs.len != rhs.len)
- return false;
- return memcmp(lhs.data, rhs.data, lhs.len) == 0;
-}
-
-static bool is_pseudo_filename(struct mem_range name)
-{
- if (incfs_equal_ranges(pending_reads_file_name_range, name))
- return true;
- if (incfs_equal_ranges(log_file_name_range, name))
- return true;
-
- return false;
-}
-
-static int validate_name(char *file_name)
-{
- struct mem_range name = range(file_name, strlen(file_name));
- int i = 0;
-
- if (name.len > INCFS_MAX_NAME_LEN)
- return -ENAMETOOLONG;
-
- if (is_pseudo_filename(name))
- return -EINVAL;
-
- for (i = 0; i < name.len; i++)
- if (name.data[i] == '/')
- return -EINVAL;
-
- return 0;
-}
-
-static int dir_relative_path_resolve(
- struct mount_info *mi,
- const char __user *relative_path,
- struct path *result_path)
-{
- struct path *base_path = &mi->mi_backing_dir_path;
- int dir_fd = get_unused_fd_flags(0);
- struct file *dir_f = NULL;
- int error = 0;
-
- if (dir_fd < 0)
- return dir_fd;
-
- dir_f = dentry_open(base_path, O_RDONLY | O_NOATIME, mi->mi_owner);
-
- if (IS_ERR(dir_f)) {
- error = PTR_ERR(dir_f);
- goto out;
- }
- fd_install(dir_fd, dir_f);
-
- if (!relative_path) {
- /* No relative path given, just return the base dir. */
- *result_path = *base_path;
- path_get(result_path);
- goto out;
- }
-
- error = user_path_at_empty(dir_fd, relative_path,
- LOOKUP_FOLLOW | LOOKUP_DIRECTORY, result_path, NULL);
-
-out:
- ksys_close(dir_fd);
- if (error)
- pr_debug("incfs: %s %d\n", __func__, error);
- return error;
-}
-
-static struct mem_range incfs_copy_signature_info_from_user(u8 __user *original,
- u64 size)
-{
- u8 *result;
-
- if (!original)
- return range(NULL, 0);
-
- if (size > INCFS_MAX_SIGNATURE_SIZE)
- return range(ERR_PTR(-EFAULT), 0);
-
- result = kzalloc(size, GFP_NOFS | __GFP_COMP);
- if (!result)
- return range(ERR_PTR(-ENOMEM), 0);
-
- if (copy_from_user(result, original, size)) {
- kfree(result);
- return range(ERR_PTR(-EFAULT), 0);
- }
-
- return range(result, size);
-}
-
-static int init_new_file(struct mount_info *mi, struct dentry *dentry,
- incfs_uuid_t *uuid, u64 size, struct mem_range attr,
- u8 __user *user_signature_info, u64 signature_size)
-{
- struct path path = {};
- struct file *new_file;
- int error = 0;
- struct backing_file_context *bfc = NULL;
- u32 block_count;
- struct mem_range raw_signature = { NULL };
- struct mtree *hash_tree = NULL;
-
- if (!mi || !dentry || !uuid)
- return -EFAULT;
-
- /* Resize newly created file to its true size. */
- path = (struct path) {
- .mnt = mi->mi_backing_dir_path.mnt,
- .dentry = dentry
- };
- new_file = dentry_open(&path, O_RDWR | O_NOATIME | O_LARGEFILE,
- mi->mi_owner);
-
- if (IS_ERR(new_file)) {
- error = PTR_ERR(new_file);
- goto out;
- }
-
- bfc = incfs_alloc_bfc(mi, new_file);
- fput(new_file);
- if (IS_ERR(bfc)) {
- error = PTR_ERR(bfc);
- bfc = NULL;
- goto out;
- }
-
- mutex_lock(&bfc->bc_mutex);
- error = incfs_write_fh_to_backing_file(bfc, uuid, size);
- if (error)
- goto out;
-
- block_count = (u32)get_blocks_count_for_size(size);
-
- if (user_signature_info) {
- raw_signature = incfs_copy_signature_info_from_user(
- user_signature_info, signature_size);
-
- if (IS_ERR(raw_signature.data)) {
- error = PTR_ERR(raw_signature.data);
- raw_signature.data = NULL;
- goto out;
- }
-
- hash_tree = incfs_alloc_mtree(raw_signature, block_count);
- if (IS_ERR(hash_tree)) {
- error = PTR_ERR(hash_tree);
- hash_tree = NULL;
- goto out;
- }
-
- error = incfs_write_signature_to_backing_file(
- bfc, raw_signature, hash_tree->hash_tree_area_size);
- if (error)
- goto out;
-
- block_count += get_blocks_count_for_size(
- hash_tree->hash_tree_area_size);
- }
-
- if (block_count)
- error = incfs_write_blockmap_to_backing_file(bfc, block_count);
-
- if (error)
- goto out;
-out:
- if (bfc) {
- mutex_unlock(&bfc->bc_mutex);
- incfs_free_bfc(bfc);
- }
- incfs_free_mtree(hash_tree);
- kfree(raw_signature.data);
-
- if (error)
- pr_debug("incfs: %s error: %d\n", __func__, error);
- return error;
-}
-
-static long ioctl_create_file(struct mount_info *mi,
- struct incfs_new_file_args __user *usr_args)
-{
- struct incfs_new_file_args args;
- char *file_id_str = NULL;
- struct dentry *index_file_dentry = NULL;
- struct dentry *named_file_dentry = NULL;
- struct dentry *incomplete_file_dentry = NULL;
- struct path parent_dir_path = {};
- struct inode *index_dir_inode = NULL;
- __le64 size_attr_value = 0;
- char *file_name = NULL;
- char *attr_value = NULL;
- int error = 0;
- bool locked = false;
- bool index_linked = false;
- bool name_linked = false;
- bool incomplete_linked = false;
-
- if (!mi || !mi->mi_index_dir || !mi->mi_incomplete_dir) {
- error = -EFAULT;
- goto out;
- }
-
- if (copy_from_user(&args, usr_args, sizeof(args)) > 0) {
- error = -EFAULT;
- goto out;
- }
-
- file_name = strndup_user(u64_to_user_ptr(args.file_name), PATH_MAX);
- if (IS_ERR(file_name)) {
- error = PTR_ERR(file_name);
- file_name = NULL;
- goto out;
- }
-
- error = validate_name(file_name);
- if (error)
- goto out;
-
- file_id_str = file_id_to_str(args.file_id);
- if (!file_id_str) {
- error = -ENOMEM;
- goto out;
- }
-
- error = mutex_lock_interruptible(&mi->mi_dir_struct_mutex);
- if (error)
- goto out;
- locked = true;
-
- /* Find a directory to put the file into. */
- error = dir_relative_path_resolve(mi,
- u64_to_user_ptr(args.directory_path),
- &parent_dir_path);
- if (error)
- goto out;
-
- if (parent_dir_path.dentry == mi->mi_index_dir) {
- /* Can't create a file directly inside .index */
- error = -EBUSY;
- goto out;
- }
-
- if (parent_dir_path.dentry == mi->mi_incomplete_dir) {
- /* Can't create a file directly inside .incomplete */
- error = -EBUSY;
- goto out;
- }
-
- /* Look up a dentry in the parent dir. It should be negative. */
- named_file_dentry = incfs_lookup_dentry(parent_dir_path.dentry,
- file_name);
- if (!named_file_dentry) {
- error = -EFAULT;
- goto out;
- }
- if (IS_ERR(named_file_dentry)) {
- error = PTR_ERR(named_file_dentry);
- named_file_dentry = NULL;
- goto out;
- }
- if (d_really_is_positive(named_file_dentry)) {
- /* File with this path already exists. */
- error = -EEXIST;
- goto out;
- }
-
- /* Look up a dentry in the incomplete dir. It should be negative. */
- incomplete_file_dentry = incfs_lookup_dentry(mi->mi_incomplete_dir,
- file_id_str);
- if (!incomplete_file_dentry) {
- error = -EFAULT;
- goto out;
- }
- if (IS_ERR(incomplete_file_dentry)) {
- error = PTR_ERR(incomplete_file_dentry);
- incomplete_file_dentry = NULL;
- goto out;
- }
- if (d_really_is_positive(incomplete_file_dentry)) {
- /* File with this path already exists. */
- error = -EEXIST;
- goto out;
- }
-
- /* Look up a dentry in the .index dir. It should be negative. */
- index_file_dentry = incfs_lookup_dentry(mi->mi_index_dir, file_id_str);
- if (!index_file_dentry) {
- error = -EFAULT;
- goto out;
- }
- if (IS_ERR(index_file_dentry)) {
- error = PTR_ERR(index_file_dentry);
- index_file_dentry = NULL;
- goto out;
- }
- if (d_really_is_positive(index_file_dentry)) {
- /* File with this ID already exists in index. */
- error = -EEXIST;
- goto out;
- }
-
- /* Creating a file in the .index dir. */
- index_dir_inode = d_inode(mi->mi_index_dir);
- inode_lock_nested(index_dir_inode, I_MUTEX_PARENT);
- error = vfs_create(index_dir_inode, index_file_dentry, args.mode | 0222,
- true);
- inode_unlock(index_dir_inode);
-
- if (error)
- goto out;
- if (!d_really_is_positive(index_file_dentry)) {
- error = -EINVAL;
- goto out;
- }
-
- error = chmod(index_file_dentry, args.mode | 0222);
- if (error) {
- pr_debug("incfs: chmod err: %d\n", error);
- goto out;
- }
-
- /* Save the file's ID as an xattr for easy fetching in future. */
- error = vfs_setxattr(index_file_dentry, INCFS_XATTR_ID_NAME,
- file_id_str, strlen(file_id_str), XATTR_CREATE);
- if (error) {
- pr_debug("incfs: vfs_setxattr err:%d\n", error);
- goto out;
- }
-
- /* Save the file's size as an xattr for easy fetching in future. */
- size_attr_value = cpu_to_le64(args.size);
- error = vfs_setxattr(index_file_dentry, INCFS_XATTR_SIZE_NAME,
- (char *)&size_attr_value, sizeof(size_attr_value),
- XATTR_CREATE);
- if (error) {
- pr_debug("incfs: vfs_setxattr err:%d\n", error);
- goto out;
- }
-
- /* Save the file's attribute as an xattr */
- if (args.file_attr_len && args.file_attr) {
- if (args.file_attr_len > INCFS_MAX_FILE_ATTR_SIZE) {
- error = -E2BIG;
- goto out;
- }
-
- attr_value = kmalloc(args.file_attr_len, GFP_NOFS);
- if (!attr_value) {
- error = -ENOMEM;
- goto out;
- }
-
- if (copy_from_user(attr_value,
- u64_to_user_ptr(args.file_attr),
- args.file_attr_len) > 0) {
- error = -EFAULT;
- goto out;
- }
-
- error = vfs_setxattr(index_file_dentry,
- INCFS_XATTR_METADATA_NAME,
- attr_value, args.file_attr_len,
- XATTR_CREATE);
-
- if (error)
- goto out;
- }
-
- /* Initializing a newly created file. */
- error = init_new_file(mi, index_file_dentry, &args.file_id, args.size,
- range(attr_value, args.file_attr_len),
- (u8 __user *)args.signature_info,
- args.signature_size);
- if (error)
- goto out;
- index_linked = true;
-
- /* Linking a file with its real name from the requested dir. */
- error = incfs_link(index_file_dentry, named_file_dentry);
- if (error)
- goto out;
- name_linked = true;
-
- if (args.size) {
- /* Linking a file with its incomplete entry */
- error = incfs_link(index_file_dentry, incomplete_file_dentry);
- if (error)
- goto out;
- incomplete_linked = true;
- }
-
-out:
- if (error) {
- pr_debug("incfs: %s err:%d\n", __func__, error);
- if (index_linked)
- incfs_unlink(index_file_dentry);
- if (name_linked)
- incfs_unlink(named_file_dentry);
- if (incomplete_linked)
- incfs_unlink(incomplete_file_dentry);
- }
-
- kfree(file_id_str);
- kfree(file_name);
- kfree(attr_value);
- dput(named_file_dentry);
- dput(index_file_dentry);
- dput(incomplete_file_dentry);
- path_put(&parent_dir_path);
- if (locked)
- mutex_unlock(&mi->mi_dir_struct_mutex);
- return error;
-}
-
-static int init_new_mapped_file(struct mount_info *mi, struct dentry *dentry,
- incfs_uuid_t *uuid, u64 size, u64 offset)
-{
- struct path path = {};
- struct file *new_file;
- int error = 0;
- struct backing_file_context *bfc = NULL;
-
- if (!mi || !dentry || !uuid)
- return -EFAULT;
-
- /* Resize newly created file to its true size. */
- path = (struct path) {
- .mnt = mi->mi_backing_dir_path.mnt,
- .dentry = dentry
- };
- new_file = dentry_open(&path, O_RDWR | O_NOATIME | O_LARGEFILE,
- mi->mi_owner);
-
- if (IS_ERR(new_file)) {
- error = PTR_ERR(new_file);
- goto out;
- }
-
- bfc = incfs_alloc_bfc(mi, new_file);
- fput(new_file);
- if (IS_ERR(bfc)) {
- error = PTR_ERR(bfc);
- bfc = NULL;
- goto out;
- }
-
- mutex_lock(&bfc->bc_mutex);
- error = incfs_write_mapping_fh_to_backing_file(bfc, uuid, size, offset);
- if (error)
- goto out;
-
-out:
- if (bfc) {
- mutex_unlock(&bfc->bc_mutex);
- incfs_free_bfc(bfc);
- }
-
- if (error)
- pr_debug("incfs: %s error: %d\n", __func__, error);
- return error;
-}
-
-static long ioctl_create_mapped_file(struct mount_info *mi, void __user *arg)
-{
- struct incfs_create_mapped_file_args __user *args_usr_ptr = arg;
- struct incfs_create_mapped_file_args args = {};
- char *file_name;
- int error = 0;
- struct path parent_dir_path = {};
- char *source_file_name = NULL;
- struct dentry *source_file_dentry = NULL;
- u64 source_file_size;
- struct dentry *file_dentry = NULL;
- struct inode *parent_inode;
- __le64 size_attr_value;
-
- if (copy_from_user(&args, args_usr_ptr, sizeof(args)) > 0)
- return -EINVAL;
-
- file_name = strndup_user(u64_to_user_ptr(args.file_name), PATH_MAX);
- if (IS_ERR(file_name)) {
- error = PTR_ERR(file_name);
- file_name = NULL;
- goto out;
- }
-
- error = validate_name(file_name);
- if (error)
- goto out;
-
- if (args.source_offset % INCFS_DATA_FILE_BLOCK_SIZE) {
- error = -EINVAL;
- goto out;
- }
-
- /* Validate file mapping is in range */
- source_file_name = file_id_to_str(args.source_file_id);
- if (!source_file_name) {
- pr_warn("Failed to alloc source_file_name\n");
- error = -ENOMEM;
- goto out;
- }
-
- source_file_dentry = incfs_lookup_dentry(mi->mi_index_dir,
- source_file_name);
- if (!source_file_dentry) {
- pr_warn("Source file does not exist\n");
- error = -EINVAL;
- goto out;
- }
- if (IS_ERR(source_file_dentry)) {
- pr_warn("Error opening source file\n");
- error = PTR_ERR(source_file_dentry);
- source_file_dentry = NULL;
- goto out;
- }
- if (!d_really_is_positive(source_file_dentry)) {
- pr_warn("Source file dentry negative\n");
- error = -EINVAL;
- goto out;
- }
-
- error = vfs_getxattr(source_file_dentry, INCFS_XATTR_SIZE_NAME,
- (char *)&size_attr_value, sizeof(size_attr_value));
- if (error < 0)
- goto out;
-
- if (error != sizeof(size_attr_value)) {
- pr_warn("Mapped file has no size attr\n");
- error = -EINVAL;
- goto out;
- }
-
- source_file_size = le64_to_cpu(size_attr_value);
- if (args.source_offset + args.size > source_file_size) {
- pr_warn("Mapped file out of range\n");
- error = -EINVAL;
- goto out;
- }
-
- /* Find a directory to put the file into. */
- error = dir_relative_path_resolve(mi,
- u64_to_user_ptr(args.directory_path),
- &parent_dir_path);
- if (error)
- goto out;
-
- if (parent_dir_path.dentry == mi->mi_index_dir) {
- /* Can't create a file directly inside .index */
- error = -EBUSY;
- goto out;
- }
-
- /* Look up a dentry in the parent dir. It should be negative. */
- file_dentry = incfs_lookup_dentry(parent_dir_path.dentry,
- file_name);
- if (!file_dentry) {
- error = -EFAULT;
- goto out;
- }
- if (IS_ERR(file_dentry)) {
- error = PTR_ERR(file_dentry);
- file_dentry = NULL;
- goto out;
- }
- if (d_really_is_positive(file_dentry)) {
- error = -EEXIST;
- goto out;
- }
-
- parent_inode = d_inode(parent_dir_path.dentry);
- inode_lock_nested(parent_inode, I_MUTEX_PARENT);
- error = vfs_create(parent_inode, file_dentry, args.mode | 0222, true);
- inode_unlock(parent_inode);
- if (error)
- goto out;
-
- /* Save the file's size as an xattr for easy fetching in future. */
- size_attr_value = cpu_to_le64(args.size);
- error = vfs_setxattr(file_dentry, INCFS_XATTR_SIZE_NAME,
- (char *)&size_attr_value, sizeof(size_attr_value),
- XATTR_CREATE);
- if (error) {
- pr_debug("incfs: vfs_setxattr err:%d\n", error);
- goto delete_file;
- }
-
- error = init_new_mapped_file(mi, file_dentry, &args.source_file_id,
- args.size, args.source_offset);
- if (error)
- goto delete_file;
-
- goto out;
-
-delete_file:
- incfs_unlink(file_dentry);
-
-out:
- dput(file_dentry);
- dput(source_file_dentry);
- path_put(&parent_dir_path);
- kfree(file_name);
- kfree(source_file_name);
- return error;
-}
-
-static long ioctl_get_read_timeouts(struct mount_info *mi, void __user *arg)
-{
- struct incfs_get_read_timeouts_args __user *args_usr_ptr = arg;
- struct incfs_get_read_timeouts_args args = {};
- int error = 0;
- struct incfs_per_uid_read_timeouts *buffer;
- int size;
-
- if (copy_from_user(&args, args_usr_ptr, sizeof(args)))
- return -EINVAL;
-
- if (args.timeouts_array_size_out > INCFS_DATA_FILE_BLOCK_SIZE)
- return -EINVAL;
-
- buffer = kzalloc(args.timeouts_array_size_out, GFP_NOFS);
- if (!buffer)
- return -ENOMEM;
-
- spin_lock(&mi->mi_per_uid_read_timeouts_lock);
- size = mi->mi_per_uid_read_timeouts_size;
- if (args.timeouts_array_size < size)
- error = -E2BIG;
- else if (size)
- memcpy(buffer, mi->mi_per_uid_read_timeouts, size);
- spin_unlock(&mi->mi_per_uid_read_timeouts_lock);
-
- args.timeouts_array_size_out = size;
- if (!error && size)
- if (copy_to_user(u64_to_user_ptr(args.timeouts_array), buffer,
- size))
- error = -EFAULT;
-
- if (!error || error == -E2BIG)
- if (copy_to_user(args_usr_ptr, &args, sizeof(args)) > 0)
- error = -EFAULT;
-
- kfree(buffer);
- return error;
-}
-
-static long ioctl_set_read_timeouts(struct mount_info *mi, void __user *arg)
-{
- struct incfs_set_read_timeouts_args __user *args_usr_ptr = arg;
- struct incfs_set_read_timeouts_args args = {};
- int error = 0;
- int size;
- struct incfs_per_uid_read_timeouts *buffer = NULL, *tmp;
- int i;
-
- if (copy_from_user(&args, args_usr_ptr, sizeof(args)))
- return -EINVAL;
-
- size = args.timeouts_array_size;
- if (size) {
- if (size > INCFS_DATA_FILE_BLOCK_SIZE ||
- size % sizeof(*buffer) != 0)
- return -EINVAL;
-
- buffer = kzalloc(size, GFP_NOFS);
- if (!buffer)
- return -ENOMEM;
-
- if (copy_from_user(buffer, u64_to_user_ptr(args.timeouts_array),
- size)) {
- error = -EINVAL;
- goto out;
- }
-
- for (i = 0; i < size / sizeof(*buffer); ++i) {
- struct incfs_per_uid_read_timeouts *t = &buffer[i];
-
- if (t->min_pending_time_us > t->max_pending_time_us) {
- error = -EINVAL;
- goto out;
- }
- }
- }
-
- spin_lock(&mi->mi_per_uid_read_timeouts_lock);
- mi->mi_per_uid_read_timeouts_size = size;
- tmp = mi->mi_per_uid_read_timeouts;
- mi->mi_per_uid_read_timeouts = buffer;
- buffer = tmp;
- spin_unlock(&mi->mi_per_uid_read_timeouts_lock);
-
-out:
- kfree(buffer);
- return error;
-}
-
-static long pending_reads_dispatch_ioctl(struct file *f, unsigned int req,
- unsigned long arg)
-{
- struct mount_info *mi = get_mount_info(file_superblock(f));
-
- switch (req) {
- case INCFS_IOC_CREATE_FILE:
- return ioctl_create_file(mi, (void __user *)arg);
- case INCFS_IOC_PERMIT_FILL:
- return ioctl_permit_fill(f, (void __user *)arg);
- case INCFS_IOC_CREATE_MAPPED_FILE:
- return ioctl_create_mapped_file(mi, (void __user *)arg);
- case INCFS_IOC_GET_READ_TIMEOUTS:
- return ioctl_get_read_timeouts(mi, (void __user *)arg);
- case INCFS_IOC_SET_READ_TIMEOUTS:
- return ioctl_set_read_timeouts(mi, (void __user *)arg);
- default:
- return -EINVAL;
- }
-}
-
-static const struct file_operations incfs_pending_read_file_ops = {
- .read = pending_reads_read,
- .poll = pending_reads_poll,
- .open = pending_reads_open,
- .release = pending_reads_release,
- .llseek = noop_llseek,
- .unlocked_ioctl = pending_reads_dispatch_ioctl,
- .compat_ioctl = pending_reads_dispatch_ioctl
-};
-
-/*******************************************************************************
- * .blocks_written pseudo file definition
- ******************************************************************************/
-static const char blocks_written_file_name[] = INCFS_BLOCKS_WRITTEN_FILENAME;
-static const struct mem_range blocks_written_file_name_range = {
- .data = (u8 *)blocks_written_file_name,
- .len = ARRAY_SIZE(blocks_written_file_name) - 1
-};
-
-/* State of an open .blocks_written file, unique for each file descriptor. */
-struct blocks_written_file_state {
- unsigned long blocks_written;
-};
-
-static ssize_t blocks_written_read(struct file *f, char __user *buf, size_t len,
- loff_t *ppos)
-{
- struct mount_info *mi = get_mount_info(file_superblock(f));
- struct blocks_written_file_state *state = f->private_data;
- unsigned long blocks_written;
- char string[21];
- int result = 0;
-
- if (!mi)
- return -EFAULT;
-
- blocks_written = atomic_read(&mi->mi_blocks_written);
- if (state->blocks_written == blocks_written)
- return 0;
-
- result = snprintf(string, sizeof(string), "%lu", blocks_written);
- if (result > len)
- result = len;
- if (copy_to_user(buf, string, result))
- return -EFAULT;
-
- state->blocks_written = blocks_written;
- return result;
-}
-
-static __poll_t blocks_written_poll(struct file *f, poll_table *wait)
-{
- struct mount_info *mi = get_mount_info(file_superblock(f));
- struct blocks_written_file_state *state = f->private_data;
- unsigned long blocks_written;
-
- if (!mi)
- return 0;
-
- poll_wait(f, &mi->mi_blocks_written_notif_wq, wait);
- blocks_written = atomic_read(&mi->mi_blocks_written);
- if (state->blocks_written == blocks_written)
- return 0;
-
- return EPOLLIN | EPOLLRDNORM;
-}
-
-static int blocks_written_open(struct inode *inode, struct file *file)
-{
- struct blocks_written_file_state *state =
- kzalloc(sizeof(*state), GFP_NOFS);
-
- if (!state)
- return -ENOMEM;
-
- state->blocks_written = -1;
- file->private_data = state;
- return 0;
-}
-
-static int blocks_written_release(struct inode *inode, struct file *file)
-{
- kfree(file->private_data);
- return 0;
-}
-
-static const struct file_operations incfs_blocks_written_file_ops = {
- .read = blocks_written_read,
- .poll = blocks_written_poll,
- .open = blocks_written_open,
- .release = blocks_written_release,
- .llseek = noop_llseek,
-};
-
-/*******************************************************************************
- * Generic inode lookup functionality
- ******************************************************************************/
-static bool get_pseudo_inode(int ino, struct inode *inode)
-{
- inode->i_ctime = (struct timespec64){};
- inode->i_mtime = inode->i_ctime;
- inode->i_atime = inode->i_ctime;
- inode->i_size = 0;
- inode->i_ino = ino;
- inode->i_private = NULL;
- inode_init_owner(inode, NULL, S_IFREG | READ_WRITE_FILE_MODE);
- inode->i_op = &incfs_file_inode_ops;
-
- switch (ino) {
- case INCFS_PENDING_READS_INODE:
- inode->i_fop = &incfs_pending_read_file_ops;
- return true;
-
- case INCFS_LOG_INODE:
- inode->i_fop = &incfs_log_file_ops;
- return true;
-
- case INCFS_BLOCKS_WRITTEN_INODE:
- inode->i_fop = &incfs_blocks_written_file_ops;
- return true;
-
- default:
- return false;
- }
-}
-
-struct inode_search {
- unsigned long ino;
-};
-
-static int inode_test(struct inode *inode, void *opaque)
-{
- struct inode_search *search = opaque;
-
- return inode->i_ino == search->ino;
-}
-
-static int inode_set(struct inode *inode, void *opaque)
-{
- struct inode_search *search = opaque;
-
- if (get_pseudo_inode(search->ino, inode))
- return 0;
-
- /* Unknown inode requested. */
- return -EINVAL;
-}
-
-static struct inode *fetch_inode(struct super_block *sb, unsigned long ino)
-{
- struct inode_search search = {
- .ino = ino
- };
- struct inode *inode = iget5_locked(sb, search.ino, inode_test,
- inode_set, &search);
-
- if (!inode)
- return ERR_PTR(-ENOMEM);
-
- if (inode->i_state & I_NEW)
- unlock_new_inode(inode);
-
- return inode;
-}
-
-int dir_lookup_pseudo_files(struct super_block *sb, struct dentry *dentry)
-{
- struct mem_range name_range =
- range((u8 *)dentry->d_name.name, dentry->d_name.len);
- unsigned long ino;
- struct inode *inode;
-
- if (incfs_equal_ranges(pending_reads_file_name_range, name_range))
- ino = INCFS_PENDING_READS_INODE;
- else if (incfs_equal_ranges(log_file_name_range, name_range))
- ino = INCFS_LOG_INODE;
- else if (incfs_equal_ranges(blocks_written_file_name_range, name_range))
- ino = INCFS_BLOCKS_WRITTEN_INODE;
- else
- return -ENOENT;
-
- inode = fetch_inode(sb, ino);
- if (IS_ERR(inode))
- return PTR_ERR(inode);
-
- d_add(dentry, inode);
- return 0;
-}
-
-int emit_pseudo_files(struct dir_context *ctx)
-{
- if (ctx->pos == 0) {
- if (!dir_emit(ctx, pending_reads_file_name,
- ARRAY_SIZE(pending_reads_file_name) - 1,
- INCFS_PENDING_READS_INODE, DT_REG))
- return -EINVAL;
-
- ctx->pos++;
- }
-
- if (ctx->pos == 1) {
- if (!dir_emit(ctx, log_file_name,
- ARRAY_SIZE(log_file_name) - 1,
- INCFS_LOG_INODE, DT_REG))
- return -EINVAL;
-
- ctx->pos++;
- }
-
- if (ctx->pos == 2) {
- if (!dir_emit(ctx, blocks_written_file_name,
- ARRAY_SIZE(blocks_written_file_name) - 1,
- INCFS_BLOCKS_WRITTEN_INODE, DT_REG))
- return -EINVAL;
-
- ctx->pos++;
- }
-
- return 0;
-}
diff --git a/fs/incfs/pseudo_files.h b/fs/incfs/pseudo_files.h
deleted file mode 100644
index 358bcabfe49a..000000000000
--- a/fs/incfs/pseudo_files.h
+++ /dev/null
@@ -1,15 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright 2020 Google LLC
- */
-
-#ifndef _INCFS_PSEUDO_FILES_H
-#define _INCFS_PSEUDO_FILES_H
-
-#define PSEUDO_FILE_COUNT 3
-#define INCFS_START_INO_RANGE 10
-
-int dir_lookup_pseudo_files(struct super_block *sb, struct dentry *dentry);
-int emit_pseudo_files(struct dir_context *ctx);
-
-#endif
diff --git a/fs/incfs/vfs.c b/fs/incfs/vfs.c
index 62430da30708..045b83c798c8 100644
--- a/fs/incfs/vfs.c
+++ b/fs/incfs/vfs.c
@@ -4,21 +4,32 @@
*/
#include
+#include
+#include
#include
#include
#include
#include
#include
+#include
#include
+#include
+#include
#include
#include "vfs.h"
-
#include "data_mgmt.h"
#include "format.h"
+#include "integrity.h"
#include "internal.h"
-#include "pseudo_files.h"
+
+#define INCFS_PENDING_READS_INODE 2
+#define INCFS_LOG_INODE 3
+#define INCFS_START_INO_RANGE 10
+#define READ_FILE_MODE 0444
+#define READ_EXEC_FILE_MODE 0555
+#define READ_WRITE_FILE_MODE 0666
static int incfs_remount_fs(struct super_block *sb, int *flags, char *data);
@@ -41,6 +52,18 @@ static int file_release(struct inode *inode, struct file *file);
static int read_single_page(struct file *f, struct page *page);
static long dispatch_ioctl(struct file *f, unsigned int req, unsigned long arg);
+static ssize_t pending_reads_read(struct file *f, char __user *buf, size_t len,
+ loff_t *ppos);
+static __poll_t pending_reads_poll(struct file *file, poll_table *wait);
+static int pending_reads_open(struct inode *inode, struct file *file);
+static int pending_reads_release(struct inode *, struct file *);
+
+static ssize_t log_read(struct file *f, char __user *buf, size_t len,
+ loff_t *ppos);
+static __poll_t log_poll(struct file *file, poll_table *wait);
+static int log_open(struct inode *inode, struct file *file);
+static int log_release(struct inode *, struct file *);
+
static struct inode *alloc_inode(struct super_block *sb);
static void free_inode(struct inode *inode);
static void evict_inode(struct inode *inode);
@@ -86,6 +109,8 @@ static const struct file_operations incfs_dir_fops = {
.iterate = iterate_incfs_dir,
.open = file_open,
.release = file_release,
+ .unlocked_ioctl = dispatch_ioctl,
+ .compat_ioctl = dispatch_ioctl
};
static const struct dentry_operations incfs_dentry_ops = {
@@ -98,7 +123,7 @@ static const struct address_space_operations incfs_address_space_ops = {
/* .readpages = readpages */
};
-const struct file_operations incfs_file_ops = {
+static const struct file_operations incfs_file_ops = {
.open = file_open,
.release = file_release,
.read_iter = generic_file_read_iter,
@@ -109,7 +134,32 @@ const struct file_operations incfs_file_ops = {
.compat_ioctl = dispatch_ioctl
};
-const struct inode_operations incfs_file_inode_ops = {
+enum FILL_PERMISSION {
+ CANT_FILL = 0,
+ CAN_FILL = 1,
+};
+
+static const struct file_operations incfs_pending_read_file_ops = {
+ .read = pending_reads_read,
+ .poll = pending_reads_poll,
+ .open = pending_reads_open,
+ .release = pending_reads_release,
+ .llseek = noop_llseek,
+ .unlocked_ioctl = dispatch_ioctl,
+ .compat_ioctl = dispatch_ioctl
+};
+
+static const struct file_operations incfs_log_file_ops = {
+ .read = log_read,
+ .poll = log_poll,
+ .open = log_open,
+ .release = log_release,
+ .llseek = noop_llseek,
+ .unlocked_ioctl = dispatch_ioctl,
+ .compat_ioctl = dispatch_ioctl
+};
+
+static const struct inode_operations incfs_file_inode_ops = {
.setattr = incfs_setattr,
.getattr = simple_getattr,
.listxattr = incfs_listxattr
@@ -141,6 +191,17 @@ static const struct xattr_handler *incfs_xattr_ops[] = {
NULL,
};
+/* State of an open .pending_reads file, unique for each file descriptor. */
+struct pending_reads_state {
+ /* A serial number of the last pending read obtained from this file. */
+ int last_pending_read_sn;
+};
+
+/* State of an open .log file, unique for each file descriptor. */
+struct log_file_state {
+ struct read_log_state state;
+};
+
struct inode_search {
unsigned long ino;
@@ -152,18 +213,32 @@ struct inode_search {
enum parse_parameter {
Opt_read_timeout,
Opt_readahead_pages,
+ Opt_no_backing_file_cache,
+ Opt_no_backing_file_readahead,
Opt_rlog_pages,
Opt_rlog_wakeup_cnt,
- Opt_report_uid,
Opt_err
};
+static const char pending_reads_file_name[] = INCFS_PENDING_READS_FILENAME;
+static struct mem_range pending_reads_file_name_range = {
+ .data = (u8 *)pending_reads_file_name,
+ .len = ARRAY_SIZE(pending_reads_file_name) - 1
+};
+
+static const char log_file_name[] = INCFS_LOG_FILENAME;
+static struct mem_range log_file_name_range = {
+ .data = (u8 *)log_file_name,
+ .len = ARRAY_SIZE(log_file_name) - 1
+};
+
static const match_table_t option_tokens = {
{ Opt_read_timeout, "read_timeout_ms=%u" },
{ Opt_readahead_pages, "readahead=%u" },
+ { Opt_no_backing_file_cache, "no_bf_cache=%u" },
+ { Opt_no_backing_file_readahead, "no_bf_readahead=%u" },
{ Opt_rlog_pages, "rlog_pages=%u" },
{ Opt_rlog_wakeup_cnt, "rlog_wakeup_cnt=%u" },
- { Opt_report_uid, "report_uid" },
{ Opt_err, NULL }
};
@@ -176,13 +251,12 @@ static int parse_options(struct mount_options *opts, char *str)
if (opts == NULL)
return -EFAULT;
- *opts = (struct mount_options) {
- .read_timeout_ms = 1000, /* Default: 1s */
- .readahead_pages = 10,
- .read_log_pages = 2,
- .read_log_wakeup_count = 10,
- };
-
+ opts->read_timeout_ms = 1000; /* Default: 1s */
+ opts->readahead_pages = 10;
+ opts->read_log_pages = 2;
+ opts->read_log_wakeup_count = 10;
+ opts->no_backing_file_cache = false;
+ opts->no_backing_file_readahead = false;
if (str == NULL || *str == 0)
return 0;
@@ -198,8 +272,6 @@ static int parse_options(struct mount_options *opts, char *str)
case Opt_read_timeout:
if (match_int(&args[0], &value))
return -EINVAL;
- if (value > 3600000)
- return -EINVAL;
opts->read_timeout_ms = value;
break;
case Opt_readahead_pages:
@@ -207,6 +279,16 @@ static int parse_options(struct mount_options *opts, char *str)
return -EINVAL;
opts->readahead_pages = value;
break;
+ case Opt_no_backing_file_cache:
+ if (match_int(&args[0], &value))
+ return -EINVAL;
+ opts->no_backing_file_cache = (value != 0);
+ break;
+ case Opt_no_backing_file_readahead:
+ if (match_int(&args[0], &value))
+ return -EINVAL;
+ opts->no_backing_file_readahead = (value != 0);
+ break;
case Opt_rlog_pages:
if (match_int(&args[0], &value))
return -EINVAL;
@@ -217,9 +299,6 @@ static int parse_options(struct mount_options *opts, char *str)
return -EINVAL;
opts->read_log_wakeup_count = value;
break;
- case Opt_report_uid:
- opts->report_uid = true;
- break;
default:
return -EINVAL;
}
@@ -228,6 +307,21 @@ static int parse_options(struct mount_options *opts, char *str)
return 0;
}
+static struct super_block *file_superblock(struct file *f)
+{
+ struct inode *inode = file_inode(f);
+
+ return inode->i_sb;
+}
+
+static struct mount_info *get_mount_info(struct super_block *sb)
+{
+ struct mount_info *result = sb->s_fs_info;
+
+ WARN_ON(!result);
+ return result;
+}
+
/* Read file size from the attribute. Quicker than reading the header */
static u64 read_size_attr(struct dentry *backing_dentry)
{
@@ -247,53 +341,96 @@ static int inode_test(struct inode *inode, void *opaque)
{
struct inode_search *search = opaque;
struct inode_info *node = get_incfs_node(inode);
- struct inode *backing_inode = d_inode(search->backing_dentry);
if (!node)
return 0;
- return node->n_backing_inode == backing_inode &&
- inode->i_ino == search->ino;
+ if (search->backing_dentry) {
+ struct inode *backing_inode = d_inode(search->backing_dentry);
+
+ return (node->n_backing_inode == backing_inode) &&
+ inode->i_ino == search->ino;
+ } else
+ return inode->i_ino == search->ino;
}
static int inode_set(struct inode *inode, void *opaque)
{
struct inode_search *search = opaque;
struct inode_info *node = get_incfs_node(inode);
- struct dentry *backing_dentry = search->backing_dentry;
- struct inode *backing_inode = d_inode(backing_dentry);
- fsstack_copy_attr_all(inode, backing_inode);
- if (S_ISREG(inode->i_mode)) {
- u64 size = search->size;
+ if (search->backing_dentry) {
+ /* It's a regular inode that has corresponding backing inode */
+ struct dentry *backing_dentry = search->backing_dentry;
+ struct inode *backing_inode = d_inode(backing_dentry);
- inode->i_size = size;
- inode->i_blocks = get_blocks_count_for_size(size);
- inode->i_mapping->a_ops = &incfs_address_space_ops;
- inode->i_op = &incfs_file_inode_ops;
- inode->i_fop = &incfs_file_ops;
- inode->i_mode &= ~0222;
- } else if (S_ISDIR(inode->i_mode)) {
+ fsstack_copy_attr_all(inode, backing_inode);
+ if (S_ISREG(inode->i_mode)) {
+ u64 size = search->size;
+
+ inode->i_size = size;
+ inode->i_blocks = get_blocks_count_for_size(size);
+ inode->i_mapping->a_ops = &incfs_address_space_ops;
+ inode->i_op = &incfs_file_inode_ops;
+ inode->i_fop = &incfs_file_ops;
+ inode->i_mode &= ~0222;
+ } else if (S_ISDIR(inode->i_mode)) {
+ inode->i_size = 0;
+ inode->i_blocks = 1;
+ inode->i_mapping->a_ops = &incfs_address_space_ops;
+ inode->i_op = &incfs_dir_inode_ops;
+ inode->i_fop = &incfs_dir_fops;
+ } else {
+ pr_warn_once("incfs: Unexpected inode type\n");
+ return -EBADF;
+ }
+
+ ihold(backing_inode);
+ node->n_backing_inode = backing_inode;
+ node->n_mount_info = get_mount_info(inode->i_sb);
+ inode->i_ctime = backing_inode->i_ctime;
+ inode->i_mtime = backing_inode->i_mtime;
+ inode->i_atime = backing_inode->i_atime;
+ inode->i_ino = backing_inode->i_ino;
+ if (backing_inode->i_ino < INCFS_START_INO_RANGE) {
+ pr_warn("incfs: ino conflict with backing FS %ld\n",
+ backing_inode->i_ino);
+ }
+
+ return 0;
+ } else if (search->ino == INCFS_PENDING_READS_INODE) {
+ /* It's an inode for .pending_reads pseudo file. */
+
+ inode->i_ctime = (struct timespec64){};
+ inode->i_mtime = inode->i_ctime;
+ inode->i_atime = inode->i_ctime;
inode->i_size = 0;
- inode->i_blocks = 1;
- inode->i_mapping->a_ops = &incfs_address_space_ops;
- inode->i_op = &incfs_dir_inode_ops;
- inode->i_fop = &incfs_dir_fops;
- } else {
- pr_warn_once("incfs: Unexpected inode type\n");
- return -EBADF;
- }
+ inode->i_ino = INCFS_PENDING_READS_INODE;
+ inode->i_private = NULL;
- ihold(backing_inode);
- node->n_backing_inode = backing_inode;
- node->n_mount_info = get_mount_info(inode->i_sb);
- inode->i_ctime = backing_inode->i_ctime;
- inode->i_mtime = backing_inode->i_mtime;
- inode->i_atime = backing_inode->i_atime;
- inode->i_ino = backing_inode->i_ino;
- if (backing_inode->i_ino < INCFS_START_INO_RANGE) {
- pr_warn("incfs: ino conflict with backing FS %ld\n",
- backing_inode->i_ino);
+ inode_init_owner(inode, NULL, S_IFREG | READ_WRITE_FILE_MODE);
+
+ inode->i_op = &incfs_file_inode_ops;
+ inode->i_fop = &incfs_pending_read_file_ops;
+
+ } else if (search->ino == INCFS_LOG_INODE) {
+ /* It's an inode for .log pseudo file. */
+
+ inode->i_ctime = (struct timespec64){};
+ inode->i_mtime = inode->i_ctime;
+ inode->i_atime = inode->i_ctime;
+ inode->i_size = 0;
+ inode->i_ino = INCFS_LOG_INODE;
+ inode->i_private = NULL;
+
+ inode_init_owner(inode, NULL, S_IFREG | READ_WRITE_FILE_MODE);
+
+ inode->i_op = &incfs_file_inode_ops;
+ inode->i_fop = &incfs_log_file_ops;
+
+ } else {
+ /* Unknown inode requested. */
+ return -EINVAL;
}
return 0;
@@ -551,15 +688,29 @@ static int iterate_incfs_dir(struct file *file, struct dir_context *ctx)
root = dir->backing_dir->f_inode
== d_inode(mi->mi_backing_dir_path.dentry);
- if (root) {
- error = emit_pseudo_files(ctx);
- if (error)
+ if (root && ctx->pos == 0) {
+ if (!dir_emit(ctx, pending_reads_file_name,
+ ARRAY_SIZE(pending_reads_file_name) - 1,
+ INCFS_PENDING_READS_INODE, DT_REG)) {
+ error = -EINVAL;
goto out;
+ }
+ ctx->pos++;
}
- ctx->pos -= PSEUDO_FILE_COUNT;
+ if (root && ctx->pos == 1) {
+ if (!dir_emit(ctx, log_file_name,
+ ARRAY_SIZE(log_file_name) - 1,
+ INCFS_LOG_INODE, DT_REG)) {
+ error = -EINVAL;
+ goto out;
+ }
+ ctx->pos++;
+ }
+
+ ctx->pos -= 2;
error = iterate_dir(dir->backing_dir, ctx);
- ctx->pos += PSEUDO_FILE_COUNT;
+ ctx->pos += 2;
file->f_pos = dir->backing_dir->f_pos;
out:
if (error)
@@ -586,9 +737,29 @@ static int incfs_init_dentry(struct dentry *dentry, struct path *path)
return 0;
}
-static struct dentry *open_or_create_special_dir(struct dentry *backing_dir,
- const char *name)
+static struct dentry *incfs_lookup_dentry(struct dentry *parent,
+ const char *name)
{
+ struct inode *inode;
+ struct dentry *result = NULL;
+
+ if (!parent)
+ return ERR_PTR(-EFAULT);
+
+ inode = d_inode(parent);
+ inode_lock_nested(inode, I_MUTEX_PARENT);
+ result = lookup_one_len(name, parent, strlen(name));
+ inode_unlock(inode);
+
+ if (IS_ERR(result))
+ pr_warn("%s err:%ld\n", __func__, PTR_ERR(result));
+
+ return result;
+}
+
+static struct dentry *open_or_create_index_dir(struct dentry *backing_dir)
+{
+ static const char name[] = ".index";
struct dentry *index_dentry;
struct inode *backing_inode = d_inode(backing_dir);
int err = 0;
@@ -611,8 +782,7 @@ static struct dentry *open_or_create_special_dir(struct dentry *backing_dir,
if (err)
return ERR_PTR(err);
- if (!d_really_is_positive(index_dentry) ||
- unlikely(d_unhashed(index_dentry))) {
+ if (!d_really_is_positive(index_dentry)) {
dput(index_dentry);
return ERR_PTR(-EINVAL);
}
@@ -620,44 +790,6 @@ static struct dentry *open_or_create_special_dir(struct dentry *backing_dir,
return index_dentry;
}
-static int read_single_page_timeouts(struct data_file *df, struct file *f,
- int block_index, struct mem_range range,
- struct mem_range tmp)
-{
- struct mount_info *mi = df->df_mount_info;
- u32 min_time_us = 0;
- u32 min_pending_time_us = 0;
- u32 max_pending_time_us = U32_MAX;
- int uid = current_uid().val;
- int i;
-
- spin_lock(&mi->mi_per_uid_read_timeouts_lock);
- for (i = 0; i < mi->mi_per_uid_read_timeouts_size /
- sizeof(*mi->mi_per_uid_read_timeouts); ++i) {
- struct incfs_per_uid_read_timeouts *t =
- &mi->mi_per_uid_read_timeouts[i];
-
- if(t->uid == uid) {
- min_time_us = t->min_time_us;
- min_pending_time_us = t->min_pending_time_us;
- max_pending_time_us = t->max_pending_time_us;
- break;
- }
- }
- spin_unlock(&mi->mi_per_uid_read_timeouts_lock);
- if (max_pending_time_us == U32_MAX) {
- u64 read_timeout_us = (u64)mi->mi_options.read_timeout_ms *
- 1000;
-
- max_pending_time_us = read_timeout_us <= U32_MAX ?
- read_timeout_us : U32_MAX;
- }
-
- return incfs_read_data_file_block(range, f, block_index,
- min_time_us, min_pending_time_us, max_pending_time_us,
- tmp);
-}
-
static int read_single_page(struct file *f, struct page *page)
{
loff_t offset = 0;
@@ -666,34 +798,28 @@ static int read_single_page(struct file *f, struct page *page)
ssize_t read_result = 0;
struct data_file *df = get_incfs_data_file(f);
int result = 0;
- void *page_start;
+ void *page_start = kmap(page);
int block_index;
+ int timeout_ms;
- if (!df) {
- SetPageError(page);
- unlock_page(page);
+ if (!df)
return -EBADF;
- }
- page_start = kmap(page);
offset = page_offset(page);
- block_index = (offset + df->df_mapped_offset) /
- INCFS_DATA_FILE_BLOCK_SIZE;
+ block_index = offset / INCFS_DATA_FILE_BLOCK_SIZE;
size = df->df_size;
+ timeout_ms = df->df_mount_info->mi_options.read_timeout_ms;
if (offset < size) {
struct mem_range tmp = {
.len = 2 * INCFS_DATA_FILE_BLOCK_SIZE
};
- tmp.data = (u8 *)__get_free_pages(GFP_NOFS, get_order(tmp.len));
- if (!tmp.data) {
- read_result = -ENOMEM;
- goto err;
- }
- bytes_to_read = min_t(loff_t, size - offset, PAGE_SIZE);
- read_result = read_single_page_timeouts(df, f, block_index,
- range(page_start, bytes_to_read), tmp);
+ tmp.data = (u8 *)__get_free_pages(GFP_NOFS, get_order(tmp.len));
+ bytes_to_read = min_t(loff_t, size - offset, PAGE_SIZE);
+ read_result = incfs_read_data_file_block(
+ range(page_start, bytes_to_read), f, block_index,
+ timeout_ms, tmp);
free_pages((unsigned long)tmp.data, get_order(tmp.len));
} else {
@@ -701,7 +827,6 @@ static int read_single_page(struct file *f, struct page *page)
read_result = 0;
}
-err:
if (read_result < 0)
result = read_result;
else if (read_result < PAGE_SIZE)
@@ -775,14 +900,14 @@ static int init_new_file(struct mount_info *mi, struct dentry *dentry,
.dentry = dentry
};
new_file = dentry_open(&path, O_RDWR | O_NOATIME | O_LARGEFILE,
- current_cred());
+ mi->mi_owner);
if (IS_ERR(new_file)) {
error = PTR_ERR(new_file);
goto out;
}
- bfc = incfs_alloc_bfc(mi, new_file);
+ bfc = incfs_alloc_bfc(new_file);
fput(new_file);
if (IS_ERR(bfc)) {
error = PTR_ERR(bfc);
@@ -862,7 +987,7 @@ static int incfs_link(struct dentry *what, struct dentry *where)
return error;
}
-int incfs_unlink(struct dentry *dentry)
+static int incfs_unlink(struct dentry *dentry)
{
struct dentry *parent_dentry = dget_parent(dentry);
struct inode *pinode = d_inode(parent_dentry);
@@ -903,7 +1028,7 @@ static int dir_relative_path_resolve(
if (dir_fd < 0)
return dir_fd;
- dir_f = dentry_open(base_path, O_RDONLY | O_NOATIME, current_cred());
+ dir_f = dentry_open(base_path, O_RDONLY | O_NOATIME, mi->mi_owner);
if (IS_ERR(dir_f)) {
error = PTR_ERR(dir_f);
@@ -930,34 +1055,234 @@ out:
static int validate_name(char *file_name)
{
- char *file_id_str;
- struct dentry *incomplete_file_dentry;
+ struct mem_range name = range(file_name, strlen(file_name));
+ int i = 0;
- if (atomic_read(&df->df_data_blocks_written) < df->df_data_block_count)
- return;
+ if (name.len > INCFS_MAX_NAME_LEN)
+ return -ENAMETOOLONG;
- /* This is best effort - there is no useful action to take on failure */
- file_id_str = file_id_to_str(df->df_id);
- if (!file_id_str)
- return;
+ if (incfs_equal_ranges(pending_reads_file_name_range, name))
+ return -EINVAL;
- incomplete_file_dentry = incfs_lookup_dentry(
- df->df_mount_info->mi_incomplete_dir,
- file_id_str);
- if (!incomplete_file_dentry || IS_ERR(incomplete_file_dentry)) {
- incomplete_file_dentry = NULL;
+ for (i = 0; i < name.len; i++)
+ if (name.data[i] == '/')
+ return -EINVAL;
+
+ return 0;
+}
+
+static int chmod(struct dentry *dentry, umode_t mode)
+{
+ struct inode *inode = dentry->d_inode;
+ struct inode *delegated_inode = NULL;
+ struct iattr newattrs;
+ int error;
+
+retry_deleg:
+ inode_lock(inode);
+ newattrs.ia_mode = (mode & S_IALLUGO) | (inode->i_mode & ~S_IALLUGO);
+ newattrs.ia_valid = ATTR_MODE | ATTR_CTIME;
+ error = notify_change(dentry, &newattrs, &delegated_inode);
+ inode_unlock(inode);
+ if (delegated_inode) {
+ error = break_deleg_wait(&delegated_inode);
+ if (!error)
+ goto retry_deleg;
+ }
+ return error;
+}
+
+static long ioctl_create_file(struct mount_info *mi,
+ struct incfs_new_file_args __user *usr_args)
+{
+ struct incfs_new_file_args args;
+ char *file_id_str = NULL;
+ struct dentry *index_file_dentry = NULL;
+ struct dentry *named_file_dentry = NULL;
+ struct path parent_dir_path = {};
+ struct inode *index_dir_inode = NULL;
+ __le64 size_attr_value = 0;
+ char *file_name = NULL;
+ char *attr_value = NULL;
+ int error = 0;
+ bool locked = false;
+
+ if (!mi || !mi->mi_index_dir) {
+ error = -EFAULT;
goto out;
}
- if (!d_really_is_positive(incomplete_file_dentry))
+ if (copy_from_user(&args, usr_args, sizeof(args)) > 0) {
+ error = -EFAULT;
+ goto out;
+ }
+
+ file_name = strndup_user(u64_to_user_ptr(args.file_name), PATH_MAX);
+ if (IS_ERR(file_name)) {
+ error = PTR_ERR(file_name);
+ file_name = NULL;
+ goto out;
+ }
+
+ error = validate_name(file_name);
+ if (error)
goto out;
- vfs_fsync(df->df_backing_file_context->bc_file, 0);
- incfs_unlink(incomplete_file_dentry);
+ file_id_str = file_id_to_str(args.file_id);
+ if (!file_id_str) {
+ error = -ENOMEM;
+ goto out;
+ }
+
+ error = mutex_lock_interruptible(&mi->mi_dir_struct_mutex);
+ if (error)
+ goto out;
+ locked = true;
+
+ /* Find a directory to put the file into. */
+ error = dir_relative_path_resolve(mi,
+ u64_to_user_ptr(args.directory_path),
+ &parent_dir_path);
+ if (error)
+ goto out;
+
+ if (parent_dir_path.dentry == mi->mi_index_dir) {
+ /* Can't create a file directly inside .index */
+ error = -EBUSY;
+ goto out;
+ }
+
+ /* Look up a dentry in the parent dir. It should be negative. */
+ named_file_dentry = incfs_lookup_dentry(parent_dir_path.dentry,
+ file_name);
+ if (!named_file_dentry) {
+ error = -EFAULT;
+ goto out;
+ }
+ if (IS_ERR(named_file_dentry)) {
+ error = PTR_ERR(named_file_dentry);
+ named_file_dentry = NULL;
+ goto out;
+ }
+ if (d_really_is_positive(named_file_dentry)) {
+ /* File with this path already exists. */
+ error = -EEXIST;
+ goto out;
+ }
+ /* Look up a dentry in the .index dir. It should be negative. */
+ index_file_dentry = incfs_lookup_dentry(mi->mi_index_dir, file_id_str);
+ if (!index_file_dentry) {
+ error = -EFAULT;
+ goto out;
+ }
+ if (IS_ERR(index_file_dentry)) {
+ error = PTR_ERR(index_file_dentry);
+ index_file_dentry = NULL;
+ goto out;
+ }
+ if (d_really_is_positive(index_file_dentry)) {
+ /* File with this ID already exists in index. */
+ error = -EEXIST;
+ goto out;
+ }
+
+ /* Creating a file in the .index dir. */
+ index_dir_inode = d_inode(mi->mi_index_dir);
+ inode_lock_nested(index_dir_inode, I_MUTEX_PARENT);
+ error = vfs_create(index_dir_inode, index_file_dentry, args.mode | 0222,
+ true);
+ inode_unlock(index_dir_inode);
+
+ if (error)
+ goto out;
+ if (!d_really_is_positive(index_file_dentry)) {
+ error = -EINVAL;
+ goto out;
+ }
+
+ error = chmod(index_file_dentry, args.mode | 0222);
+ if (error) {
+ pr_debug("incfs: chmod err: %d\n", error);
+ goto delete_index_file;
+ }
+
+ /* Save the file's ID as an xattr for easy fetching in future. */
+ error = vfs_setxattr(index_file_dentry, INCFS_XATTR_ID_NAME,
+ file_id_str, strlen(file_id_str), XATTR_CREATE);
+ if (error) {
+ pr_debug("incfs: vfs_setxattr err:%d\n", error);
+ goto delete_index_file;
+ }
+
+ /* Save the file's size as an xattr for easy fetching in future. */
+ size_attr_value = cpu_to_le64(args.size);
+ error = vfs_setxattr(index_file_dentry, INCFS_XATTR_SIZE_NAME,
+ (char *)&size_attr_value, sizeof(size_attr_value),
+ XATTR_CREATE);
+ if (error) {
+ pr_debug("incfs: vfs_setxattr err:%d\n", error);
+ goto delete_index_file;
+ }
+
+ /* Save the file's attribute as an xattr */
+ if (args.file_attr_len && args.file_attr) {
+ if (args.file_attr_len > INCFS_MAX_FILE_ATTR_SIZE) {
+ error = -E2BIG;
+ goto delete_index_file;
+ }
+
+ attr_value = kmalloc(args.file_attr_len, GFP_NOFS);
+ if (!attr_value) {
+ error = -ENOMEM;
+ goto delete_index_file;
+ }
+
+ if (copy_from_user(attr_value,
+ u64_to_user_ptr(args.file_attr),
+ args.file_attr_len) > 0) {
+ error = -EFAULT;
+ goto delete_index_file;
+ }
+
+ error = vfs_setxattr(index_file_dentry,
+ INCFS_XATTR_METADATA_NAME,
+ attr_value, args.file_attr_len,
+ XATTR_CREATE);
+
+ if (error)
+ goto delete_index_file;
+ }
+
+ /* Initializing a newly created file. */
+ error = init_new_file(mi, index_file_dentry, &args.file_id, args.size,
+ range(attr_value, args.file_attr_len),
+ (u8 __user *)args.signature_info,
+ args.signature_size);
+ if (error)
+ goto delete_index_file;
+
+ /* Linking a file with it's real name from the requested dir. */
+ error = incfs_link(index_file_dentry, named_file_dentry);
+
+ if (!error)
+ goto out;
+
+delete_index_file:
+ incfs_unlink(index_file_dentry);
out:
- dput(incomplete_file_dentry);
+ if (error)
+ pr_debug("incfs: %s err:%d\n", __func__, error);
+
kfree(file_id_str);
+ kfree(file_name);
+ kfree(attr_value);
+ dput(named_file_dentry);
+ dput(index_file_dentry);
+ path_put(&parent_dir_path);
+ if (locked)
+ mutex_unlock(&mi->mi_dir_struct_mutex);
+ return error;
}
static long ioctl_fill_blocks(struct file *f, void __user *arg)
@@ -966,7 +1291,6 @@ static long ioctl_fill_blocks(struct file *f, void __user *arg)
struct incfs_fill_blocks fill_blocks;
struct incfs_fill_block __user *usr_fill_block_array;
struct data_file *df = get_incfs_data_file(f);
- struct incfs_file_data *fd = f->private_data;
const ssize_t data_buf_size = 2 * INCFS_DATA_FILE_BLOCK_SIZE;
u8 *data_buf = NULL;
ssize_t error = 0;
@@ -975,7 +1299,7 @@ static long ioctl_fill_blocks(struct file *f, void __user *arg)
if (!df)
return -EBADF;
- if (!fd || fd->fd_fill_permission != CAN_FILL)
+ if ((uintptr_t)f->private_data != CAN_FILL)
return -EPERM;
if (copy_from_user(&fill_blocks, usr_fill_blocks, sizeof(fill_blocks)))
@@ -1021,8 +1345,6 @@ static long ioctl_fill_blocks(struct file *f, void __user *arg)
if (data_buf)
free_pages((unsigned long)data_buf, get_order(data_buf_size));
- maybe_delete_incomplete_file(df);
-
/*
* Only report the error if no records were processed, otherwise
* just return how many were processed successfully.
@@ -1033,6 +1355,53 @@ static long ioctl_fill_blocks(struct file *f, void __user *arg)
return i;
}
+static long ioctl_permit_fill(struct file *f, void __user *arg)
+{
+ struct incfs_permit_fill __user *usr_permit_fill = arg;
+ struct incfs_permit_fill permit_fill;
+ long error = 0;
+ struct file *file = NULL;
+
+ if (f->f_op != &incfs_pending_read_file_ops)
+ return -EPERM;
+
+ if (copy_from_user(&permit_fill, usr_permit_fill, sizeof(permit_fill)))
+ return -EFAULT;
+
+ file = fget(permit_fill.file_descriptor);
+ if (IS_ERR(file))
+ return PTR_ERR(file);
+
+ if (file->f_op != &incfs_file_ops) {
+ error = -EPERM;
+ goto out;
+ }
+
+ if (file->f_inode->i_sb != f->f_inode->i_sb) {
+ error = -EPERM;
+ goto out;
+ }
+
+ switch ((uintptr_t)file->private_data) {
+ case CANT_FILL:
+ file->private_data = (void *)CAN_FILL;
+ break;
+
+ case CAN_FILL:
+ pr_debug("CAN_FILL already set");
+ break;
+
+ default:
+ pr_warn("Invalid file private data");
+ error = -EFAULT;
+ goto out;
+ }
+
+out:
+ fput(file);
+ return error;
+}
+
static long ioctl_read_file_signature(struct file *f, void __user *arg)
{
struct incfs_get_file_sig_args __user *args_usr_ptr = arg;
@@ -1086,19 +1455,18 @@ static long ioctl_get_filled_blocks(struct file *f, void __user *arg)
struct incfs_get_filled_blocks_args __user *args_usr_ptr = arg;
struct incfs_get_filled_blocks_args args = {};
struct data_file *df = get_incfs_data_file(f);
- struct incfs_file_data *fd = f->private_data;
int error;
- if (!df || !fd)
+ if (!df)
return -EINVAL;
- if (fd->fd_fill_permission != CAN_FILL)
+ if ((uintptr_t)f->private_data != CAN_FILL)
return -EPERM;
if (copy_from_user(&args, args_usr_ptr, sizeof(args)) > 0)
return -EINVAL;
- error = incfs_get_filled_blocks(df, fd, &args);
+ error = incfs_get_filled_blocks(df, &args);
if (copy_to_user(args_usr_ptr, &args, sizeof(args)))
return -EFAULT;
@@ -1106,38 +1474,21 @@ static long ioctl_get_filled_blocks(struct file *f, void __user *arg)
return error;
}
-static long ioctl_get_block_count(struct file *f, void __user *arg)
-{
- struct incfs_get_block_count_args __user *args_usr_ptr = arg;
- struct incfs_get_block_count_args args = {};
- struct data_file *df = get_incfs_data_file(f);
-
- if (!df)
- return -EINVAL;
-
- args.total_data_blocks_out = df->df_data_block_count;
- args.filled_data_blocks_out = atomic_read(&df->df_data_blocks_written);
- args.total_hash_blocks_out = df->df_total_block_count -
- df->df_data_block_count;
- args.filled_hash_blocks_out = atomic_read(&df->df_hash_blocks_written);
-
- if (copy_to_user(args_usr_ptr, &args, sizeof(args)))
- return -EFAULT;
-
- return 0;
-}
-
static long dispatch_ioctl(struct file *f, unsigned int req, unsigned long arg)
{
+ struct mount_info *mi = get_mount_info(file_superblock(f));
+
switch (req) {
+ case INCFS_IOC_CREATE_FILE:
+ return ioctl_create_file(mi, (void __user *)arg);
case INCFS_IOC_FILL_BLOCKS:
return ioctl_fill_blocks(f, (void __user *)arg);
+ case INCFS_IOC_PERMIT_FILL:
+ return ioctl_permit_fill(f, (void __user *)arg);
case INCFS_IOC_READ_FILE_SIGNATURE:
return ioctl_read_file_signature(f, (void __user *)arg);
case INCFS_IOC_GET_FILLED_BLOCKS:
return ioctl_get_filled_blocks(f, (void __user *)arg);
- case INCFS_IOC_GET_BLOCK_COUNT:
- return ioctl_get_block_count(f, (void __user *)arg);
default:
return -EINVAL;
}
@@ -1151,18 +1502,40 @@ static struct dentry *dir_lookup(struct inode *dir_inode, struct dentry *dentry,
struct dentry *backing_dentry = NULL;
struct path dir_backing_path = {};
struct inode_info *dir_info = get_incfs_node(dir_inode);
+ struct mem_range name_range =
+ range((u8 *)dentry->d_name.name, dentry->d_name.len);
int err = 0;
- if (!mi || !dir_info || !dir_info->n_backing_inode)
- return ERR_PTR(-EBADF);
-
if (d_inode(mi->mi_backing_dir_path.dentry) ==
dir_info->n_backing_inode) {
/* We do lookup in the FS root. Show pseudo files. */
- err = dir_lookup_pseudo_files(dir_inode->i_sb, dentry);
- if (err != -ENOENT)
+
+ if (incfs_equal_ranges(pending_reads_file_name_range,
+ name_range)) {
+ struct inode *inode = fetch_pending_reads_inode(
+ dir_inode->i_sb);
+
+ if (IS_ERR(inode)) {
+ err = PTR_ERR(inode);
+ goto out;
+ }
+
+ d_add(dentry, inode);
goto out;
- err = 0;
+ }
+
+ if (incfs_equal_ranges(log_file_name_range, name_range)) {
+ struct inode *inode = fetch_log_inode(
+ dir_inode->i_sb);
+
+ if (IS_ERR(inode)) {
+ err = PTR_ERR(inode);
+ goto out;
+ }
+
+ d_add(dentry, inode);
+ goto out;
+ }
}
dir_dentry = dget_parent(dentry);
@@ -1248,7 +1621,7 @@ static int dir_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
if (!backing_dentry) {
err = -EBADF;
- goto path_err;
+ goto out;
}
if (backing_dentry->d_parent == mi->mi_index_dir) {
@@ -1257,19 +1630,13 @@ static int dir_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
goto out;
}
- if (backing_dentry->d_parent == mi->mi_incomplete_dir) {
- /* Can't create a subdir inside .incomplete */
- err = -EBUSY;
- goto out;
- }
inode_lock_nested(dir_node->n_backing_inode, I_MUTEX_PARENT);
err = vfs_mkdir(dir_node->n_backing_inode, backing_dentry, mode | 0222);
inode_unlock(dir_node->n_backing_inode);
if (!err) {
struct inode *inode = NULL;
- if (d_really_is_negative(backing_dentry) ||
- unlikely(d_unhashed(backing_dentry))) {
+ if (d_really_is_negative(backing_dentry)) {
err = -EINVAL;
goto out;
}
@@ -1286,34 +1653,23 @@ out:
if (d_really_is_negative(dentry))
d_drop(dentry);
path_put(&backing_path);
-
-path_err:
mutex_unlock(&mi->mi_dir_struct_mutex);
if (err)
pr_debug("incfs: %s err:%d\n", __func__, err);
return err;
}
-/*
- * Delete file referenced by backing_dentry and if appropriate its hardlink
- * from .index and .incomplete
- */
-static int file_delete(struct mount_info *mi,
- struct dentry *backing_dentry,
- int nlink)
+/* Delete file referenced by backing_dentry and also its hardlink from .index */
+static int final_file_delete(struct mount_info *mi,
+ struct dentry *backing_dentry)
{
struct dentry *index_file_dentry = NULL;
- struct dentry *incomplete_file_dentry = NULL;
/* 2 chars per byte of file ID + 1 char for \0 */
char file_id_str[2 * sizeof(incfs_uuid_t) + 1] = {0};
ssize_t uuid_size = 0;
int error = 0;
WARN_ON(!mutex_is_locked(&mi->mi_dir_struct_mutex));
-
- if (nlink > 3)
- goto just_unlink;
-
uuid_size = vfs_getxattr(backing_dentry, INCFS_XATTR_ID_NAME,
file_id_str, 2 * sizeof(incfs_uuid_t));
if (uuid_size < 0) {
@@ -1329,46 +1685,17 @@ static int file_delete(struct mount_info *mi,
index_file_dentry = incfs_lookup_dentry(mi->mi_index_dir, file_id_str);
if (IS_ERR(index_file_dentry)) {
error = PTR_ERR(index_file_dentry);
- index_file_dentry = NULL;
goto out;
}
- if (d_really_is_positive(index_file_dentry) && nlink > 0)
- nlink--;
-
- if (nlink > 2)
- goto just_unlink;
-
- incomplete_file_dentry = incfs_lookup_dentry(mi->mi_incomplete_dir,
- file_id_str);
- if (IS_ERR(incomplete_file_dentry)) {
- error = PTR_ERR(incomplete_file_dentry);
- incomplete_file_dentry = NULL;
+ error = incfs_unlink(backing_dentry);
+ if (error)
goto out;
- }
-
- if (d_really_is_positive(incomplete_file_dentry) && nlink > 0)
- nlink--;
-
- if (nlink > 1)
- goto just_unlink;
if (d_really_is_positive(index_file_dentry))
error = incfs_unlink(index_file_dentry);
- if (error)
- goto out;
-
- if (d_really_is_positive(incomplete_file_dentry))
- error = incfs_unlink(incomplete_file_dentry);
- if (error)
- goto out;
-
-just_unlink:
- error = incfs_unlink(backing_dentry);
-
out:
dput(index_file_dentry);
- dput(incomplete_file_dentry);
if (error)
pr_debug("incfs: delete_file_from_index err:%d\n", error);
return error;
@@ -1381,9 +1708,6 @@ static int dir_unlink(struct inode *dir, struct dentry *dentry)
struct kstat stat;
int err = 0;
- if (!mi)
- return -EBADF;
-
err = mutex_lock_interruptible(&mi->mi_dir_struct_mutex);
if (err)
return err;
@@ -1391,7 +1715,7 @@ static int dir_unlink(struct inode *dir, struct dentry *dentry)
get_incfs_backing_path(dentry, &backing_path);
if (!backing_path.dentry) {
err = -EBADF;
- goto path_err;
+ goto out;
}
if (backing_path.dentry->d_parent == mi->mi_index_dir) {
@@ -1400,23 +1724,25 @@ static int dir_unlink(struct inode *dir, struct dentry *dentry)
goto out;
}
- if (backing_path.dentry->d_parent == mi->mi_incomplete_dir) {
- /* Direct unlink from .incomplete are not allowed. */
- err = -EBUSY;
- goto out;
- }
-
err = vfs_getattr(&backing_path, &stat, STATX_NLINK,
AT_STATX_SYNC_AS_STAT);
if (err)
goto out;
- err = file_delete(mi, backing_path.dentry, stat.nlink);
+ if (stat.nlink == 2) {
+ /*
+ * This is the last named link to this file. The only one left
+ * is in .index. Remove them both now.
+ */
+ err = final_file_delete(mi, backing_path.dentry);
+ } else {
+ /* There are other links to this file. Remove just this one. */
+ err = incfs_unlink(backing_path.dentry);
+ }
d_drop(dentry);
out:
path_put(&backing_path);
-path_err:
if (err)
pr_debug("incfs: %s err:%d\n", __func__, err);
mutex_unlock(&mi->mi_dir_struct_mutex);
@@ -1431,9 +1757,6 @@ static int dir_link(struct dentry *old_dentry, struct inode *dir,
struct path backing_new_path = {};
int error = 0;
- if (!mi)
- return -EBADF;
-
error = mutex_lock_interruptible(&mi->mi_dir_struct_mutex);
if (error)
return error;
@@ -1447,12 +1770,6 @@ static int dir_link(struct dentry *old_dentry, struct inode *dir,
goto out;
}
- if (backing_new_path.dentry->d_parent == mi->mi_incomplete_dir) {
- /* Can't link to .incomplete */
- error = -EBUSY;
- goto out;
- }
-
error = incfs_link(backing_old_path.dentry, backing_new_path.dentry);
if (!error) {
struct inode *inode = NULL;
@@ -1486,9 +1803,6 @@ static int dir_rmdir(struct inode *dir, struct dentry *dentry)
struct path backing_path = {};
int err = 0;
- if (!mi)
- return -EBADF;
-
err = mutex_lock_interruptible(&mi->mi_dir_struct_mutex);
if (err)
return err;
@@ -1496,7 +1810,7 @@ static int dir_rmdir(struct inode *dir, struct dentry *dentry)
get_incfs_backing_path(dentry, &backing_path);
if (!backing_path.dentry) {
err = -EBADF;
- goto path_err;
+ goto out;
}
if (backing_path.dentry == mi->mi_index_dir) {
@@ -1505,19 +1819,11 @@ static int dir_rmdir(struct inode *dir, struct dentry *dentry)
goto out;
}
- if (backing_path.dentry == mi->mi_incomplete_dir) {
- /* Can't delete .incomplete */
- err = -EBUSY;
- goto out;
- }
-
err = incfs_rmdir(backing_path.dentry);
if (!err)
d_drop(dentry);
out:
path_put(&backing_path);
-
-path_err:
if (err)
pr_debug("incfs: %s err:%d\n", __func__, err);
mutex_unlock(&mi->mi_dir_struct_mutex);
@@ -1541,14 +1847,6 @@ static int dir_rename(struct inode *old_dir, struct dentry *old_dentry,
return error;
backing_old_dentry = get_incfs_dentry(old_dentry)->backing_path.dentry;
-
- if (!backing_old_dentry || backing_old_dentry == mi->mi_index_dir ||
- backing_old_dentry == mi->mi_incomplete_dir) {
- /* Renaming .index or .incomplete not allowed */
- error = -EBUSY;
- goto exit;
- }
-
backing_new_dentry = get_incfs_dentry(new_dentry)->backing_path.dentry;
dget(backing_old_dentry);
dget(backing_new_dentry);
@@ -1557,9 +1855,8 @@ static int dir_rename(struct inode *old_dir, struct dentry *old_dentry,
backing_new_dir_dentry = dget_parent(backing_new_dentry);
target_inode = d_inode(new_dentry);
- if (backing_old_dir_dentry == mi->mi_index_dir ||
- backing_old_dir_dentry == mi->mi_incomplete_dir) {
- /* Direct moves from .index or .incomplete are not allowed. */
+ if (backing_old_dir_dentry == mi->mi_index_dir) {
+ /* Direct moves from .index are not allowed. */
error = -EBUSY;
goto out;
}
@@ -1596,7 +1893,6 @@ out:
dput(backing_new_dentry);
dput(backing_old_dentry);
-exit:
mutex_unlock(&mi->mi_dir_struct_mutex);
if (error)
pr_debug("incfs: %s err:%d\n", __func__, error);
@@ -1610,13 +1906,10 @@ static int file_open(struct inode *inode, struct file *file)
struct file *backing_file = NULL;
struct path backing_path = {};
int err = 0;
- const struct cred *old_cred;
get_incfs_backing_path(file->f_path.dentry, &backing_path);
- old_cred = override_creds(mi->mi_owner);
- backing_file = dentry_open(&backing_path,
- O_RDWR | O_NOATIME | O_LARGEFILE, current_cred());
- revert_creds(old_cred);
+ backing_file = dentry_open(
+ &backing_path, O_RDWR | O_NOATIME | O_LARGEFILE, mi->mi_owner);
path_put(&backing_path);
if (IS_ERR(backing_file)) {
@@ -1626,20 +1919,8 @@ static int file_open(struct inode *inode, struct file *file)
}
if (S_ISREG(inode->i_mode)) {
- struct incfs_file_data *fd = kzalloc(sizeof(*fd), GFP_NOFS);
-
- if (!fd) {
- err = -ENOMEM;
- goto out;
- }
-
- *fd = (struct incfs_file_data) {
- .fd_fill_permission = CANT_FILL,
- };
- file->private_data = fd;
-
err = make_inode_ready_for_data_ops(mi, inode, backing_file);
-
+ file->private_data = (void *)CANT_FILL;
} else if (S_ISDIR(inode->i_mode)) {
struct dir_file *dir = NULL;
@@ -1652,17 +1933,9 @@ static int file_open(struct inode *inode, struct file *file)
err = -EBADF;
out:
- if (err) {
- pr_debug("name:%s err: %d\n",
- file->f_path.dentry->d_name.name, err);
- if (S_ISREG(inode->i_mode))
- kfree(file->private_data);
- else if (S_ISDIR(inode->i_mode))
- incfs_free_dir_file(file->private_data);
-
- file->private_data = NULL;
- }
-
+ if (err)
+ pr_debug("incfs: %s name:%s err: %d\n", __func__,
+ file->f_path.dentry->d_name.name, err);
if (backing_file)
fput(backing_file);
return err;
@@ -1671,8 +1944,9 @@ out:
static int file_release(struct inode *inode, struct file *file)
{
if (S_ISREG(inode->i_mode)) {
- kfree(file->private_data);
- file->private_data = NULL;
+ /* Do nothing.
+ * data_file is released only by inode eviction.
+ */
} else if (S_ISDIR(inode->i_mode)) {
struct dir_file *dir = get_incfs_dir_file(file);
@@ -1891,13 +2165,10 @@ static ssize_t incfs_listxattr(struct dentry *d, char *list, size_t size)
struct dentry *incfs_mount_fs(struct file_system_type *type, int flags,
const char *dev_name, void *data)
{
- static const char index_name[] = ".index";
- static const char incomplete_name[] = ".incomplete";
struct mount_options options = {};
struct mount_info *mi = NULL;
struct path backing_dir_path = {};
- struct dentry *index_dir = NULL;
- struct dentry *incomplete_dir = NULL;
+ struct dentry *index_dir;
struct super_block *src_fs_sb = NULL;
struct inode *root_inode = NULL;
struct super_block *sb = sget(type, NULL, set_anon_super, flags, NULL);
@@ -1950,28 +2221,15 @@ struct dentry *incfs_mount_fs(struct file_system_type *type, int flags,
goto err;
}
- index_dir = open_or_create_special_dir(backing_dir_path.dentry,
- index_name);
+ index_dir = open_or_create_index_dir(backing_dir_path.dentry);
if (IS_ERR_OR_NULL(index_dir)) {
error = PTR_ERR(index_dir);
pr_err("incfs: Can't find or create .index dir in %s\n",
dev_name);
- /* No need to null index_dir since we don't put it */
goto err;
}
mi->mi_index_dir = index_dir;
- incomplete_dir = open_or_create_special_dir(backing_dir_path.dentry,
- incomplete_name);
- if (IS_ERR_OR_NULL(incomplete_dir)) {
- error = PTR_ERR(incomplete_dir);
- pr_err("incfs: Can't find or create .incomplete dir in %s\n",
- dev_name);
- /* No need to null incomplete_dir since we don't put it */
- goto err;
- }
- mi->mi_incomplete_dir = incomplete_dir;
-
sb->s_fs_info = mi;
root_inode = fetch_regular_inode(sb, backing_dir_path.dentry);
if (IS_ERR(root_inode)) {
@@ -2012,11 +2270,6 @@ static int incfs_remount_fs(struct super_block *sb, int *flags, char *data)
if (err)
return err;
- if (options.report_uid != mi->mi_options.report_uid) {
- pr_err("incfs: Can't change report_uid mount option on remount\n");
- return -EOPNOTSUPP;
- }
-
err = incfs_realloc_mount_info(mi, &options);
if (err)
return err;
@@ -2045,7 +2298,9 @@ static int show_options(struct seq_file *m, struct dentry *root)
seq_printf(m, ",rlog_wakeup_cnt=%u",
mi->mi_options.read_log_wakeup_count);
}
- if (mi->mi_options.report_uid)
- seq_puts(m, ",report_uid");
+ if (mi->mi_options.no_backing_file_cache)
+ seq_puts(m, ",no_bf_cache");
+ if (mi->mi_options.no_backing_file_readahead)
+ seq_puts(m, ",no_bf_readahead");
return 0;
}
diff --git a/fs/incfs/vfs.h b/fs/incfs/vfs.h
index 79fdf243733d..eaa490e19072 100644
--- a/fs/incfs/vfs.h
+++ b/fs/incfs/vfs.h
@@ -6,28 +6,8 @@
#ifndef _INCFS_VFS_H
#define _INCFS_VFS_H
-extern const struct file_operations incfs_file_ops;
-extern const struct inode_operations incfs_file_inode_ops;
-
void incfs_kill_sb(struct super_block *sb);
struct dentry *incfs_mount_fs(struct file_system_type *type, int flags,
const char *dev_name, void *data);
-int incfs_link(struct dentry *what, struct dentry *where);
-int incfs_unlink(struct dentry *dentry);
-
-static inline struct mount_info *get_mount_info(struct super_block *sb)
-{
- struct mount_info *result = sb->s_fs_info;
-
- WARN_ON(!result);
- return result;
-}
-
-static inline struct super_block *file_superblock(struct file *f)
-{
- struct inode *inode = file_inode(f);
-
- return inode->i_sb;
-}
#endif
diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h
index 8357ff69962f..cc07189a501f 100644
--- a/fs/nfs/internal.h
+++ b/fs/nfs/internal.h
@@ -575,12 +575,14 @@ extern int nfs4_test_session_trunk(struct rpc_clnt *,
static inline struct inode *nfs_igrab_and_active(struct inode *inode)
{
- inode = igrab(inode);
- if (inode != NULL && !nfs_sb_active(inode->i_sb)) {
- iput(inode);
- inode = NULL;
+ struct super_block *sb = inode->i_sb;
+
+ if (sb && nfs_sb_active(sb)) {
+ if (igrab(inode))
+ return inode;
+ nfs_sb_deactive(sb);
}
- return inode;
+ return NULL;
}
static inline void nfs_iput_and_deactive(struct inode *inode)
diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index 1a395647ae26..d89a815f7c31 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -6721,9 +6721,9 @@ static int _nfs4_do_setlk(struct nfs4_state *state, int cmd, struct file_lock *f
data->arg.new_lock_owner, ret);
} else
data->cancelled = true;
+ trace_nfs4_set_lock(fl, state, &data->res.stateid, cmd, ret);
rpc_put_task(task);
dprintk("%s: done, ret = %d!\n", __func__, ret);
- trace_nfs4_set_lock(fl, state, &data->res.stateid, cmd, ret);
return ret;
}
diff --git a/fs/nfs/nfs4super.c b/fs/nfs/nfs4super.c
index 6fb7cb6b3f4b..e7a10f5f5405 100644
--- a/fs/nfs/nfs4super.c
+++ b/fs/nfs/nfs4super.c
@@ -95,7 +95,7 @@ static void nfs4_evict_inode(struct inode *inode)
nfs_inode_return_delegation_noreclaim(inode);
/* Note that above delegreturn would trigger pnfs return-on-close */
pnfs_return_layout(inode);
- pnfs_destroy_layout(NFS_I(inode));
+ pnfs_destroy_layout_final(NFS_I(inode));
/* First call standard NFS clear_inode() code */
nfs_clear_inode(inode);
}
diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 2b9e139a2997..46ca5592b8b0 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -294,6 +294,7 @@ void
pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo)
{
struct inode *inode;
+ unsigned long i_state;
if (!lo)
return;
@@ -304,8 +305,12 @@ pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo)
if (!list_empty(&lo->plh_segs))
WARN_ONCE(1, "NFS: BUG unfreed layout segments.\n");
pnfs_detach_layout_hdr(lo);
+ i_state = inode->i_state;
spin_unlock(&inode->i_lock);
pnfs_free_layout_hdr(lo);
+ /* Notify pnfs_destroy_layout_final() that we're done */
+ if (i_state & (I_FREEING | I_CLEAR))
+ wake_up_var(lo);
}
}
@@ -713,8 +718,7 @@ pnfs_free_lseg_list(struct list_head *free_me)
}
}
-void
-pnfs_destroy_layout(struct nfs_inode *nfsi)
+static struct pnfs_layout_hdr *__pnfs_destroy_layout(struct nfs_inode *nfsi)
{
struct pnfs_layout_hdr *lo;
LIST_HEAD(tmp_list);
@@ -732,9 +736,34 @@ pnfs_destroy_layout(struct nfs_inode *nfsi)
pnfs_put_layout_hdr(lo);
} else
spin_unlock(&nfsi->vfs_inode.i_lock);
+ return lo;
+}
+
+void pnfs_destroy_layout(struct nfs_inode *nfsi)
+{
+ __pnfs_destroy_layout(nfsi);
}
EXPORT_SYMBOL_GPL(pnfs_destroy_layout);
+static bool pnfs_layout_removed(struct nfs_inode *nfsi,
+ struct pnfs_layout_hdr *lo)
+{
+ bool ret;
+
+ spin_lock(&nfsi->vfs_inode.i_lock);
+ ret = nfsi->layout != lo;
+ spin_unlock(&nfsi->vfs_inode.i_lock);
+ return ret;
+}
+
+void pnfs_destroy_layout_final(struct nfs_inode *nfsi)
+{
+ struct pnfs_layout_hdr *lo = __pnfs_destroy_layout(nfsi);
+
+ if (lo)
+ wait_var_event(lo, pnfs_layout_removed(nfsi, lo));
+}
+
static bool
pnfs_layout_add_bulk_destroy_list(struct inode *inode,
struct list_head *layout_list)
@@ -1431,12 +1460,18 @@ void pnfs_roc_release(struct nfs4_layoutreturn_args *args,
int ret)
{
struct pnfs_layout_hdr *lo = args->layout;
+ struct inode *inode = args->inode;
const nfs4_stateid *arg_stateid = NULL;
const nfs4_stateid *res_stateid = NULL;
struct nfs4_xdr_opaque_data *ld_private = args->ld_private;
switch (ret) {
case -NFS4ERR_NOMATCHING_LAYOUT:
+ spin_lock(&inode->i_lock);
+ if (pnfs_layout_is_valid(lo) &&
+ nfs4_stateid_match_other(&args->stateid, &lo->plh_stateid))
+ pnfs_set_plh_return_info(lo, args->range.iomode, 0);
+ spin_unlock(&inode->i_lock);
break;
case 0:
if (res->lrs_present)
@@ -2112,6 +2147,7 @@ static void _lgopen_prepare_attached(struct nfs4_opendata *data,
&rng, GFP_KERNEL);
if (!lgp) {
pnfs_clear_first_layoutget(lo);
+ nfs_layoutget_end(lo);
pnfs_put_layout_hdr(lo);
return;
}
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index 3ba44819a88a..80fafa29e567 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -254,6 +254,7 @@ struct pnfs_layout_segment *pnfs_layout_process(struct nfs4_layoutget *lgp);
void pnfs_layoutget_free(struct nfs4_layoutget *lgp);
void pnfs_free_lseg_list(struct list_head *tmp_list);
void pnfs_destroy_layout(struct nfs_inode *);
+void pnfs_destroy_layout_final(struct nfs_inode *);
void pnfs_destroy_all_layouts(struct nfs_client *);
int pnfs_destroy_layouts_byfsid(struct nfs_client *clp,
struct nfs_fsid *fsid,
@@ -645,6 +646,10 @@ static inline void pnfs_destroy_layout(struct nfs_inode *nfsi)
{
}
+static inline void pnfs_destroy_layout_final(struct nfs_inode *nfsi)
+{
+}
+
static inline struct pnfs_layout_segment *
pnfs_get_lseg(struct pnfs_layout_segment *lseg)
{
diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c
index 83919116d5cb..b90bea1c434e 100644
--- a/fs/nfsd/nfs3xdr.c
+++ b/fs/nfsd/nfs3xdr.c
@@ -844,9 +844,14 @@ compose_entry_fh(struct nfsd3_readdirres *cd, struct svc_fh *fhp,
if (isdotent(name, namlen)) {
if (namlen == 2) {
dchild = dget_parent(dparent);
- /* filesystem root - cannot return filehandle for ".." */
+ /*
+ * Don't return filehandle for ".." if we're at
+ * the filesystem or export root:
+ */
if (dchild == dparent)
goto out;
+ if (dparent == exp->ex_path.dentry)
+ goto out;
} else
dchild = dget(dparent);
} else
diff --git a/fs/proc/generic.c b/fs/proc/generic.c
index e39bac94dead..bab10368a04d 100644
--- a/fs/proc/generic.c
+++ b/fs/proc/generic.c
@@ -137,8 +137,12 @@ static int proc_getattr(const struct path *path, struct kstat *stat,
{
struct inode *inode = d_inode(path->dentry);
struct proc_dir_entry *de = PDE(inode);
- if (de && de->nlink)
- set_nlink(inode, de->nlink);
+ if (de) {
+ nlink_t nlink = READ_ONCE(de->nlink);
+ if (nlink > 0) {
+ set_nlink(inode, nlink);
+ }
+ }
generic_fillattr(inode, stat);
return 0;
@@ -337,6 +341,16 @@ static const struct file_operations proc_dir_operations = {
.iterate_shared = proc_readdir,
};
+static int proc_net_d_revalidate(struct dentry *dentry, unsigned int flags)
+{
+ return 0;
+}
+
+const struct dentry_operations proc_net_dentry_ops = {
+ .d_revalidate = proc_net_d_revalidate,
+ .d_delete = always_delete_dentry,
+};
+
/*
* proc directories can do almost nothing..
*/
@@ -361,6 +375,7 @@ struct proc_dir_entry *proc_register(struct proc_dir_entry *dir,
write_unlock(&proc_subdir_lock);
goto out_free_inum;
}
+ dir->nlink++;
write_unlock(&proc_subdir_lock);
return dp;
@@ -458,8 +473,8 @@ struct proc_dir_entry *proc_symlink(const char *name,
}
EXPORT_SYMBOL(proc_symlink);
-struct proc_dir_entry *proc_mkdir_data(const char *name, umode_t mode,
- struct proc_dir_entry *parent, void *data)
+struct proc_dir_entry *_proc_mkdir(const char *name, umode_t mode,
+ struct proc_dir_entry *parent, void *data, bool force_lookup)
{
struct proc_dir_entry *ent;
@@ -471,13 +486,20 @@ struct proc_dir_entry *proc_mkdir_data(const char *name, umode_t mode,
ent->data = data;
ent->proc_fops = &proc_dir_operations;
ent->proc_iops = &proc_dir_inode_operations;
- parent->nlink++;
+ if (force_lookup) {
+ pde_force_lookup(ent);
+ }
ent = proc_register(parent, ent);
- if (!ent)
- parent->nlink--;
}
return ent;
}
+EXPORT_SYMBOL_GPL(_proc_mkdir);
+
+struct proc_dir_entry *proc_mkdir_data(const char *name, umode_t mode,
+ struct proc_dir_entry *parent, void *data)
+{
+ return _proc_mkdir(name, mode, parent, data, false);
+}
EXPORT_SYMBOL_GPL(proc_mkdir_data);
struct proc_dir_entry *proc_mkdir_mode(const char *name, umode_t mode,
@@ -504,10 +526,7 @@ struct proc_dir_entry *proc_create_mount_point(const char *name)
ent->data = NULL;
ent->proc_fops = NULL;
ent->proc_iops = NULL;
- parent->nlink++;
ent = proc_register(parent, ent);
- if (!ent)
- parent->nlink--;
}
return ent;
}
@@ -665,8 +684,12 @@ void remove_proc_entry(const char *name, struct proc_dir_entry *parent)
len = strlen(fn);
de = pde_subdir_find(parent, fn, len);
- if (de)
+ if (de) {
rb_erase(&de->subdir_node, &parent->subdir);
+ if (S_ISDIR(de->mode)) {
+ parent->nlink--;
+ }
+ }
write_unlock(&proc_subdir_lock);
if (!de) {
WARN(1, "name '%s'\n", name);
@@ -675,9 +698,6 @@ void remove_proc_entry(const char *name, struct proc_dir_entry *parent)
proc_entry_rundown(de);
- if (S_ISDIR(de->mode))
- parent->nlink--;
- de->nlink = 0;
WARN(pde_subdir_first(de),
"%s: removing non-empty directory '%s/%s', leaking at least '%s'\n",
__func__, de->parent->name, de->name, pde_subdir_first(de)->name);
@@ -713,13 +733,12 @@ int remove_proc_subtree(const char *name, struct proc_dir_entry *parent)
de = next;
continue;
}
- write_unlock(&proc_subdir_lock);
-
- proc_entry_rundown(de);
next = de->parent;
if (S_ISDIR(de->mode))
next->nlink--;
- de->nlink = 0;
+ write_unlock(&proc_subdir_lock);
+
+ proc_entry_rundown(de);
if (de == root)
break;
pde_put(de);
diff --git a/fs/proc/internal.h b/fs/proc/internal.h
index db189d59ce1e..e05deaee80d9 100644
--- a/fs/proc/internal.h
+++ b/fs/proc/internal.h
@@ -314,3 +314,10 @@ extern unsigned long task_statm(struct mm_struct *,
unsigned long *, unsigned long *,
unsigned long *, unsigned long *);
extern void task_mem(struct seq_file *, struct mm_struct *);
+
+extern const struct dentry_operations proc_net_dentry_ops;
+static inline void pde_force_lookup(struct proc_dir_entry *pde)
+{
+ /* /proc/net/ entries can be changed under us by setns(CLONE_NEWNET) */
+ pde->proc_dops = &proc_net_dentry_ops;
+}
diff --git a/fs/proc/proc_net.c b/fs/proc/proc_net.c
index a7b12435519e..096bcc1e7a8a 100644
--- a/fs/proc/proc_net.c
+++ b/fs/proc/proc_net.c
@@ -38,22 +38,6 @@ static struct net *get_proc_net(const struct inode *inode)
return maybe_get_net(PDE_NET(PDE(inode)));
}
-static int proc_net_d_revalidate(struct dentry *dentry, unsigned int flags)
-{
- return 0;
-}
-
-static const struct dentry_operations proc_net_dentry_ops = {
- .d_revalidate = proc_net_d_revalidate,
- .d_delete = always_delete_dentry,
-};
-
-static void pde_force_lookup(struct proc_dir_entry *pde)
-{
- /* /proc/net/ entries can be changed under us by setns(CLONE_NEWNET) */
- pde->proc_dops = &proc_net_dentry_ops;
-}
-
static int seq_open_net(struct inode *inode, struct file *file)
{
unsigned int state_size = PDE(inode)->state_size;
diff --git a/fs/quota/quota_tree.c b/fs/quota/quota_tree.c
index bb3f59bcfcf5..656f9ff63edd 100644
--- a/fs/quota/quota_tree.c
+++ b/fs/quota/quota_tree.c
@@ -61,7 +61,7 @@ static ssize_t read_blk(struct qtree_mem_dqinfo *info, uint blk, char *buf)
memset(buf, 0, info->dqi_usable_bs);
return sb->s_op->quota_read(sb, info->dqi_type, buf,
- info->dqi_usable_bs, blk << info->dqi_blocksize_bits);
+ info->dqi_usable_bs, (loff_t)blk << info->dqi_blocksize_bits);
}
static ssize_t write_blk(struct qtree_mem_dqinfo *info, uint blk, char *buf)
@@ -70,7 +70,7 @@ static ssize_t write_blk(struct qtree_mem_dqinfo *info, uint blk, char *buf)
ssize_t ret;
ret = sb->s_op->quota_write(sb, info->dqi_type, buf,
- info->dqi_usable_bs, blk << info->dqi_blocksize_bits);
+ info->dqi_usable_bs, (loff_t)blk << info->dqi_blocksize_bits);
if (ret != info->dqi_usable_bs) {
quota_error(sb, "dquota write failed");
if (ret >= 0)
@@ -283,7 +283,7 @@ static uint find_free_dqentry(struct qtree_mem_dqinfo *info,
blk);
goto out_buf;
}
- dquot->dq_off = (blk << info->dqi_blocksize_bits) +
+ dquot->dq_off = ((loff_t)blk << info->dqi_blocksize_bits) +
sizeof(struct qt_disk_dqdbheader) +
i * info->dqi_entry_size;
kfree(buf);
@@ -558,7 +558,7 @@ static loff_t find_block_dqentry(struct qtree_mem_dqinfo *info,
ret = -EIO;
goto out_buf;
} else {
- ret = (blk << info->dqi_blocksize_bits) + sizeof(struct
+ ret = ((loff_t)blk << info->dqi_blocksize_bits) + sizeof(struct
qt_disk_dqdbheader) + i * info->dqi_entry_size;
}
out_buf:
diff --git a/fs/reiserfs/stree.c b/fs/reiserfs/stree.c
index 2946713cb00d..5229038852ca 100644
--- a/fs/reiserfs/stree.c
+++ b/fs/reiserfs/stree.c
@@ -454,6 +454,12 @@ static int is_leaf(char *buf, int blocksize, struct buffer_head *bh)
"(second one): %h", ih);
return 0;
}
+ if (is_direntry_le_ih(ih) && (ih_item_len(ih) < (ih_entry_count(ih) * IH_SIZE))) {
+ reiserfs_warning(NULL, "reiserfs-5093",
+ "item entry count seems wrong %h",
+ ih);
+ return 0;
+ }
prev_location = ih_location(ih);
}
diff --git a/fs/ubifs/dir.c b/fs/ubifs/dir.c
index 1a7c5c0754d4..cc2f10ec3839 100644
--- a/fs/ubifs/dir.c
+++ b/fs/ubifs/dir.c
@@ -290,6 +290,15 @@ done:
return d_splice_alias(inode, dentry);
}
+static int ubifs_prepare_create(struct inode *dir, struct dentry *dentry,
+ struct fscrypt_name *nm)
+{
+ if (fscrypt_is_nokey_name(dentry))
+ return -ENOKEY;
+
+ return fscrypt_setup_filename(dir, &dentry->d_name, 0, nm);
+}
+
static int ubifs_create(struct inode *dir, struct dentry *dentry, umode_t mode,
bool excl)
{
@@ -313,7 +322,7 @@ static int ubifs_create(struct inode *dir, struct dentry *dentry, umode_t mode,
if (err)
return err;
- err = fscrypt_setup_filename(dir, &dentry->d_name, 0, &nm);
+ err = ubifs_prepare_create(dir, dentry, &nm);
if (err)
goto out_budg;
@@ -977,7 +986,7 @@ static int ubifs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
if (err)
return err;
- err = fscrypt_setup_filename(dir, &dentry->d_name, 0, &nm);
+ err = ubifs_prepare_create(dir, dentry, &nm);
if (err)
goto out_budg;
@@ -1062,7 +1071,7 @@ static int ubifs_mknod(struct inode *dir, struct dentry *dentry,
return err;
}
- err = fscrypt_setup_filename(dir, &dentry->d_name, 0, &nm);
+ err = ubifs_prepare_create(dir, dentry, &nm);
if (err) {
kfree(dev);
goto out_budg;
@@ -1146,7 +1155,7 @@ static int ubifs_symlink(struct inode *dir, struct dentry *dentry,
if (err)
return err;
- err = fscrypt_setup_filename(dir, &dentry->d_name, 0, &nm);
+ err = ubifs_prepare_create(dir, dentry, &nm);
if (err)
goto out_budg;
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index 6a21d161f724..bef8b6d92a44 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -494,7 +494,10 @@
*/
#define TEXT_TEXT \
ALIGN_FUNCTION(); \
- *(.text.hot TEXT_MAIN .text.fixup .text.unlikely) \
+ *(.text.hot .text.hot.*) \
+ *(TEXT_MAIN .text.fixup) \
+ *(.text.unlikely .text.unlikely.*) \
+ *(.text.unknown .text.unknown.*) \
*(TEXT_CFI_MAIN) \
*(.text..refcount) \
*(.text..ftrace) \
diff --git a/include/linux/acpi.h b/include/linux/acpi.h
index cd412817654f..019468f072b7 100644
--- a/include/linux/acpi.h
+++ b/include/linux/acpi.h
@@ -812,6 +812,13 @@ static inline int acpi_device_modalias(struct device *dev,
return -ENODEV;
}
+static inline struct platform_device *
+acpi_create_platform_device(struct acpi_device *adev,
+ struct property_entry *properties)
+{
+ return NULL;
+}
+
static inline bool acpi_dma_supported(struct acpi_device *adev)
{
return false;
diff --git a/include/linux/compat.h b/include/linux/compat.h
index de0c13bdcd2c..189d0e111d57 100644
--- a/include/linux/compat.h
+++ b/include/linux/compat.h
@@ -445,8 +445,6 @@ struct compat_kexec_segment;
struct compat_mq_attr;
struct compat_msgbuf;
-extern void compat_exit_robust_list(struct task_struct *curr);
-
#define BITS_PER_COMPAT_LONG (8*sizeof(compat_long_t))
#define BITS_TO_COMPAT_LONGS(bits) DIV_ROUND_UP(bits, BITS_PER_COMPAT_LONG)
diff --git a/include/linux/compiler-gcc.h b/include/linux/compiler-gcc.h
index 89def28f7a5f..15aca9a97f60 100644
--- a/include/linux/compiler-gcc.h
+++ b/include/linux/compiler-gcc.h
@@ -12,6 +12,12 @@
#if GCC_VERSION < 40600
# error Sorry, your compiler is too old - please upgrade it.
+#elif defined(CONFIG_ARM64) && GCC_VERSION < 50100
+/*
+ * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63293
+ * https://lore.kernel.org/r/20210107111841.GN1551@shell.armlinux.org.uk
+ */
+# error Sorry, your version of GCC is too old - please use 5.1 or newer.
#endif
/*
diff --git a/include/linux/dm-bufio.h b/include/linux/dm-bufio.h
index 3c8b7d274bd9..45ba37aaf6b7 100644
--- a/include/linux/dm-bufio.h
+++ b/include/linux/dm-bufio.h
@@ -138,6 +138,7 @@ void dm_bufio_set_minimum_buffers(struct dm_bufio_client *c, unsigned n);
unsigned dm_bufio_get_block_size(struct dm_bufio_client *c);
sector_t dm_bufio_get_device_size(struct dm_bufio_client *c);
+struct dm_io_client *dm_bufio_get_dm_io_client(struct dm_bufio_client *c);
sector_t dm_bufio_get_block_number(struct dm_buffer *b);
void *dm_bufio_get_block_data(struct dm_buffer *b);
void *dm_bufio_get_aux_data(struct dm_buffer *b);
diff --git a/include/linux/fscrypt.h b/include/linux/fscrypt.h
index b60bf3d54423..93ea86f0f9a3 100644
--- a/include/linux/fscrypt.h
+++ b/include/linux/fscrypt.h
@@ -117,6 +117,35 @@ static inline void fscrypt_handle_d_move(struct dentry *dentry)
dentry->d_flags &= ~DCACHE_ENCRYPTED_NAME;
}
+/**
+ * fscrypt_is_nokey_name() - test whether a dentry is a no-key name
+ * @dentry: the dentry to check
+ *
+ * This returns true if the dentry is a no-key dentry. A no-key dentry is a
+ * dentry that was created in an encrypted directory that hasn't had its
+ * encryption key added yet. Such dentries may be either positive or negative.
+ *
+ * When a filesystem is asked to create a new filename in an encrypted directory
+ * and the new filename's dentry is a no-key dentry, it must fail the operation
+ * with ENOKEY. This includes ->create(), ->mkdir(), ->mknod(), ->symlink(),
+ * ->rename(), and ->link(). (However, ->rename() and ->link() are already
+ * handled by fscrypt_prepare_rename() and fscrypt_prepare_link().)
+ *
+ * This is necessary because creating a filename requires the directory's
+ * encryption key, but just checking for the key on the directory inode during
+ * the final filesystem operation doesn't guarantee that the key was available
+ * during the preceding dentry lookup. And the key must have already been
+ * available during the dentry lookup in order for it to have been checked
+ * whether the filename already exists in the directory and for the new file's
+ * dentry not to be invalidated due to it incorrectly having the no-key flag.
+ *
+ * Return: %true if the dentry is a no-key name
+ */
+static inline bool fscrypt_is_nokey_name(const struct dentry *dentry)
+{
+ return dentry->d_flags & DCACHE_ENCRYPTED_NAME;
+}
+
/* crypto.c */
void fscrypt_enqueue_decrypt_work(struct work_struct *);
@@ -253,6 +282,11 @@ static inline void fscrypt_handle_d_move(struct dentry *dentry)
{
}
+static inline bool fscrypt_is_nokey_name(const struct dentry *dentry)
+{
+ return false;
+}
+
/* crypto.c */
static inline void fscrypt_enqueue_decrypt_work(struct work_struct *work)
{
diff --git a/include/linux/futex.h b/include/linux/futex.h
index a61bf436dcf3..b70df27d7e85 100644
--- a/include/linux/futex.h
+++ b/include/linux/futex.h
@@ -2,7 +2,9 @@
#ifndef _LINUX_FUTEX_H
#define _LINUX_FUTEX_H
+#include
#include
+
#include
struct inode;
@@ -51,15 +53,35 @@ union futex_key {
#define FUTEX_KEY_INIT (union futex_key) { .both = { .ptr = 0ULL } }
#ifdef CONFIG_FUTEX
-extern void exit_robust_list(struct task_struct *curr);
+enum {
+ FUTEX_STATE_OK,
+ FUTEX_STATE_EXITING,
+ FUTEX_STATE_DEAD,
+};
+
+static inline void futex_init_task(struct task_struct *tsk)
+{
+ tsk->robust_list = NULL;
+#ifdef CONFIG_COMPAT
+ tsk->compat_robust_list = NULL;
+#endif
+ INIT_LIST_HEAD(&tsk->pi_state_list);
+ tsk->pi_state_cache = NULL;
+ tsk->futex_state = FUTEX_STATE_OK;
+ mutex_init(&tsk->futex_exit_mutex);
+}
+
+void futex_exit_recursive(struct task_struct *tsk);
+void futex_exit_release(struct task_struct *tsk);
+void futex_exec_release(struct task_struct *tsk);
long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout,
u32 __user *uaddr2, u32 val2, u32 val3);
#else
-static inline void exit_robust_list(struct task_struct *curr)
-{
-}
-
+static inline void futex_init_task(struct task_struct *tsk) { }
+static inline void futex_exit_recursive(struct task_struct *tsk) { }
+static inline void futex_exit_release(struct task_struct *tsk) { }
+static inline void futex_exec_release(struct task_struct *tsk) { }
static inline long do_futex(u32 __user *uaddr, int op, u32 val,
ktime_t *timeout, u32 __user *uaddr2,
u32 val2, u32 val3)
@@ -68,12 +90,4 @@ static inline long do_futex(u32 __user *uaddr, int op, u32 val,
}
#endif
-#ifdef CONFIG_FUTEX_PI
-extern void exit_pi_state_list(struct task_struct *curr);
-#else
-static inline void exit_pi_state_list(struct task_struct *curr)
-{
-}
-#endif
-
#endif
diff --git a/include/linux/kdev_t.h b/include/linux/kdev_t.h
index 85b5151911cf..4856706fbfeb 100644
--- a/include/linux/kdev_t.h
+++ b/include/linux/kdev_t.h
@@ -21,61 +21,61 @@
})
/* acceptable for old filesystems */
-static inline bool old_valid_dev(dev_t dev)
+static __always_inline bool old_valid_dev(dev_t dev)
{
return MAJOR(dev) < 256 && MINOR(dev) < 256;
}
-static inline u16 old_encode_dev(dev_t dev)
+static __always_inline u16 old_encode_dev(dev_t dev)
{
return (MAJOR(dev) << 8) | MINOR(dev);
}
-static inline dev_t old_decode_dev(u16 val)
+static __always_inline dev_t old_decode_dev(u16 val)
{
return MKDEV((val >> 8) & 255, val & 255);
}
-static inline u32 new_encode_dev(dev_t dev)
+static __always_inline u32 new_encode_dev(dev_t dev)
{
unsigned major = MAJOR(dev);
unsigned minor = MINOR(dev);
return (minor & 0xff) | (major << 8) | ((minor & ~0xff) << 12);
}
-static inline dev_t new_decode_dev(u32 dev)
+static __always_inline dev_t new_decode_dev(u32 dev)
{
unsigned major = (dev & 0xfff00) >> 8;
unsigned minor = (dev & 0xff) | ((dev >> 12) & 0xfff00);
return MKDEV(major, minor);
}
-static inline u64 huge_encode_dev(dev_t dev)
+static __always_inline u64 huge_encode_dev(dev_t dev)
{
return new_encode_dev(dev);
}
-static inline dev_t huge_decode_dev(u64 dev)
+static __always_inline dev_t huge_decode_dev(u64 dev)
{
return new_decode_dev(dev);
}
-static inline int sysv_valid_dev(dev_t dev)
+static __always_inline int sysv_valid_dev(dev_t dev)
{
return MAJOR(dev) < (1<<14) && MINOR(dev) < (1<<18);
}
-static inline u32 sysv_encode_dev(dev_t dev)
+static __always_inline u32 sysv_encode_dev(dev_t dev)
{
return MINOR(dev) | (MAJOR(dev) << 18);
}
-static inline unsigned sysv_major(u32 dev)
+static __always_inline unsigned sysv_major(u32 dev)
{
return (dev >> 18) & 0x3fff;
}
-static inline unsigned sysv_minor(u32 dev)
+static __always_inline unsigned sysv_minor(u32 dev)
{
return dev & 0x3ffff;
}
diff --git a/include/linux/of.h b/include/linux/of.h
index d4f14b0302b6..6429f00341d1 100644
--- a/include/linux/of.h
+++ b/include/linux/of.h
@@ -1258,6 +1258,7 @@ static inline int of_get_available_child_count(const struct device_node *np)
#define _OF_DECLARE(table, name, compat, fn, fn_type) \
static const struct of_device_id __of_table_##name \
__used __section(__##table##_of_table) \
+ __aligned(__alignof__(struct of_device_id)) \
= { .compatible = compat, \
.data = (fn == (fn_type)NULL) ? fn : fn }
#else
diff --git a/include/linux/proc_fs.h b/include/linux/proc_fs.h
index 0853d72a995e..c609d993511b 100644
--- a/include/linux/proc_fs.h
+++ b/include/linux/proc_fs.h
@@ -21,6 +21,7 @@ extern void proc_flush_task(struct task_struct *);
extern struct proc_dir_entry *proc_symlink(const char *,
struct proc_dir_entry *, const char *);
+struct proc_dir_entry *_proc_mkdir(const char *, umode_t, struct proc_dir_entry *, void *, bool);
extern struct proc_dir_entry *proc_mkdir(const char *, struct proc_dir_entry *);
extern struct proc_dir_entry *proc_mkdir_data(const char *, umode_t,
struct proc_dir_entry *, void *);
@@ -90,6 +91,11 @@ static inline struct proc_dir_entry *proc_symlink(const char *name,
static inline struct proc_dir_entry *proc_mkdir(const char *name,
struct proc_dir_entry *parent) {return NULL;}
static inline struct proc_dir_entry *proc_create_mount_point(const char *name) { return NULL; }
+static inline struct proc_dir_entry *_proc_mkdir(const char *name, umode_t mode,
+ struct proc_dir_entry *parent, void *data, bool force_lookup)
+{
+ return NULL;
+}
static inline struct proc_dir_entry *proc_mkdir_data(const char *name,
umode_t mode, struct proc_dir_entry *parent, void *data) { return NULL; }
static inline struct proc_dir_entry *proc_mkdir_mode(const char *name,
@@ -133,7 +139,7 @@ struct net;
static inline struct proc_dir_entry *proc_net_mkdir(
struct net *net, const char *name, struct proc_dir_entry *parent)
{
- return proc_mkdir_data(name, 0, parent, net);
+ return _proc_mkdir(name, 0, parent, net, true);
}
struct ns_common;
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 47d35cbf21ba..3f545c9cf403 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1481,6 +1481,27 @@ struct task_struct {
/* task is frozen/stopped (used by the cgroup freezer) */
ANDROID_KABI_USE(1, unsigned frozen:1);
+ /* 095444fad7e3 ("futex: Replace PF_EXITPIDONE with a state") */
+ ANDROID_KABI_USE(2, unsigned int futex_state);
+
+ /*
+ * f9b0c6c556db ("futex: Add mutex around futex exit")
+ * A struct mutex takes 32 bytes, or 4 64bit entries, so pick off
+ * 4 of the reserved members, and replace them with a struct mutex.
+ * Do the GENKSYMS hack to work around the CRC issues
+ */
+#ifdef __GENKSYMS__
+ ANDROID_KABI_RESERVE(3);
+ ANDROID_KABI_RESERVE(4);
+ ANDROID_KABI_RESERVE(5);
+ ANDROID_KABI_RESERVE(6);
+#else
+ struct mutex futex_exit_mutex;
+#endif
+
+ ANDROID_KABI_RESERVE(7);
+ ANDROID_KABI_RESERVE(8);
+
/*
* New fields for task_struct should be added above here, so that
* they are included in the randomized portion of task_struct.
@@ -1657,7 +1678,6 @@ extern struct pid *cad_pid;
*/
#define PF_IDLE 0x00000002 /* I am an IDLE thread */
#define PF_EXITING 0x00000004 /* Getting shut down */
-#define PF_EXITPIDONE 0x00000008 /* PI exit done on shut down */
#define PF_VCPU 0x00000010 /* I'm a virtual CPU */
#define PF_WQ_WORKER 0x00000020 /* I'm a workqueue worker */
#define PF_FORKNOEXEC 0x00000040 /* Forked but didn't exec */
diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h
index 766bbe813861..8d3b7e731b74 100644
--- a/include/linux/sched/mm.h
+++ b/include/linux/sched/mm.h
@@ -119,8 +119,10 @@ extern struct mm_struct *get_task_mm(struct task_struct *task);
* succeeds.
*/
extern struct mm_struct *mm_access(struct task_struct *task, unsigned int mode);
-/* Remove the current tasks stale references to the old mm_struct */
-extern void mm_release(struct task_struct *, struct mm_struct *);
+/* Remove the current tasks stale references to the old mm_struct on exit() */
+extern void exit_mm_release(struct task_struct *, struct mm_struct *);
+/* Remove the current tasks stale references to the old mm_struct on exec() */
+extern void exec_mm_release(struct task_struct *, struct mm_struct *);
#ifdef CONFIG_MEMCG
extern void mm_update_next_owner(struct mm_struct *mm);
diff --git a/include/net/red.h b/include/net/red.h
index 9665582c4687..e21e7fd4fe07 100644
--- a/include/net/red.h
+++ b/include/net/red.h
@@ -168,12 +168,14 @@ static inline void red_set_vars(struct red_vars *v)
v->qcount = -1;
}
-static inline bool red_check_params(u32 qth_min, u32 qth_max, u8 Wlog)
+static inline bool red_check_params(u32 qth_min, u32 qth_max, u8 Wlog, u8 Scell_log)
{
if (fls(qth_min) + Wlog > 32)
return false;
if (fls(qth_max) + Wlog > 32)
return false;
+ if (Scell_log >= 32)
+ return false;
if (qth_max < qth_min)
return false;
return true;
diff --git a/include/uapi/linux/const.h b/include/uapi/linux/const.h
index 5ed721ad5b19..af2a44c08683 100644
--- a/include/uapi/linux/const.h
+++ b/include/uapi/linux/const.h
@@ -28,4 +28,9 @@
#define _BITUL(x) (_UL(1) << (x))
#define _BITULL(x) (_ULL(1) << (x))
+#define __ALIGN_KERNEL(x, a) __ALIGN_KERNEL_MASK(x, (typeof(x))(a) - 1)
+#define __ALIGN_KERNEL_MASK(x, mask) (((x) + (mask)) & ~(mask))
+
+#define __KERNEL_DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
+
#endif /* _UAPI_LINUX_CONST_H */
diff --git a/include/uapi/linux/dm-user.h b/include/uapi/linux/dm-user.h
new file mode 100644
index 000000000000..6d8f535b3542
--- /dev/null
+++ b/include/uapi/linux/dm-user.h
@@ -0,0 +1,68 @@
+/* SPDX-License-Identifier: LGPL-2.0+ WITH Linux-syscall-note */
+/*
+ * Copyright (C) 2020 Google, Inc
+ * Copyright (C) 2020 Palmer Dabbelt
+ */
+
+#ifndef _LINUX_DM_USER_H
+#define _LINUX_DM_USER_H
+
+#include
+
+/*
+ * dm-user proxies device mapper ops between the kernel and userspace. It's
+ * essentially just an RPC mechanism: all kernel calls create a request,
+ * userspace handles that with a response. Userspace obtains requests via
+ * read() and provides responses via write().
+ *
+ * See Documentation/block/dm-user.rst for more information.
+ */
+
+#define DM_USER_REQ_MAP_READ 0
+#define DM_USER_REQ_MAP_WRITE 1
+#define DM_USER_REQ_MAP_FLUSH 2
+#define DM_USER_REQ_MAP_DISCARD 3
+#define DM_USER_REQ_MAP_SECURE_ERASE 4
+#define DM_USER_REQ_MAP_WRITE_SAME 5
+#define DM_USER_REQ_MAP_WRITE_ZEROES 6
+#define DM_USER_REQ_MAP_ZONE_OPEN 7
+#define DM_USER_REQ_MAP_ZONE_CLOSE 8
+#define DM_USER_REQ_MAP_ZONE_FINISH 9
+#define DM_USER_REQ_MAP_ZONE_APPEND 10
+#define DM_USER_REQ_MAP_ZONE_RESET 11
+#define DM_USER_REQ_MAP_ZONE_RESET_ALL 12
+
+#define DM_USER_REQ_MAP_FLAG_FAILFAST_DEV 0x00001
+#define DM_USER_REQ_MAP_FLAG_FAILFAST_TRANSPORT 0x00002
+#define DM_USER_REQ_MAP_FLAG_FAILFAST_DRIVER 0x00004
+#define DM_USER_REQ_MAP_FLAG_SYNC 0x00008
+#define DM_USER_REQ_MAP_FLAG_META 0x00010
+#define DM_USER_REQ_MAP_FLAG_PRIO 0x00020
+#define DM_USER_REQ_MAP_FLAG_NOMERGE 0x00040
+#define DM_USER_REQ_MAP_FLAG_IDLE 0x00080
+#define DM_USER_REQ_MAP_FLAG_INTEGRITY 0x00100
+#define DM_USER_REQ_MAP_FLAG_FUA 0x00200
+#define DM_USER_REQ_MAP_FLAG_PREFLUSH 0x00400
+#define DM_USER_REQ_MAP_FLAG_RAHEAD 0x00800
+#define DM_USER_REQ_MAP_FLAG_BACKGROUND 0x01000
+#define DM_USER_REQ_MAP_FLAG_NOWAIT 0x02000
+#define DM_USER_REQ_MAP_FLAG_CGROUP_PUNT 0x04000
+#define DM_USER_REQ_MAP_FLAG_NOUNMAP 0x08000
+#define DM_USER_REQ_MAP_FLAG_HIPRI 0x10000
+#define DM_USER_REQ_MAP_FLAG_DRV 0x20000
+#define DM_USER_REQ_MAP_FLAG_SWAP 0x40000
+
+#define DM_USER_RESP_SUCCESS 0
+#define DM_USER_RESP_ERROR 1
+#define DM_USER_RESP_UNSUPPORTED 2
+
+struct dm_user_message {
+ __u64 seq;
+ __u64 type;
+ __u64 flags;
+ __u64 sector;
+ __u64 len;
+ __u8 buf[];
+};
+
+#endif
diff --git a/include/uapi/linux/ethtool.h b/include/uapi/linux/ethtool.h
index dc69391d2bba..fc21d3726b59 100644
--- a/include/uapi/linux/ethtool.h
+++ b/include/uapi/linux/ethtool.h
@@ -14,7 +14,7 @@
#ifndef _UAPI_LINUX_ETHTOOL_H
#define _UAPI_LINUX_ETHTOOL_H
-#include
+#include
#include
#include
diff --git a/include/uapi/linux/incrementalfs.h b/include/uapi/linux/incrementalfs.h
index e59072c53fb9..13c3d5173e14 100644
--- a/include/uapi/linux/incrementalfs.h
+++ b/include/uapi/linux/incrementalfs.h
@@ -30,7 +30,6 @@
#define INCFS_PENDING_READS_FILENAME ".pending_reads"
#define INCFS_LOG_FILENAME ".log"
-#define INCFS_BLOCKS_WRITTEN_FILENAME ".blocks_written"
#define INCFS_XATTR_ID_NAME (XATTR_USER_PREFIX "incfs.id")
#define INCFS_XATTR_SIZE_NAME (XATTR_USER_PREFIX "incfs.size")
#define INCFS_XATTR_METADATA_NAME (XATTR_USER_PREFIX "incfs.metadata")
@@ -43,10 +42,7 @@
/* ===== ioctl requests on the command dir ===== */
-/*
- * Create a new file
- * May only be called on .pending_reads file
- */
+/* Create a new file */
#define INCFS_IOC_CREATE_FILE \
_IOWR(INCFS_IOCTL_BASE_CODE, 30, struct incfs_new_file_args)
@@ -96,66 +92,9 @@
#define INCFS_IOC_GET_FILLED_BLOCKS \
_IOR(INCFS_IOCTL_BASE_CODE, 34, struct incfs_get_filled_blocks_args)
-/*
- * Creates a new mapped file
- * May only be called on .pending_reads file
- */
-#define INCFS_IOC_CREATE_MAPPED_FILE \
- _IOWR(INCFS_IOCTL_BASE_CODE, 35, struct incfs_create_mapped_file_args)
-
-/*
- * Get number of blocks, total and filled
- * May only be called on .pending_reads file
- */
-#define INCFS_IOC_GET_BLOCK_COUNT \
- _IOR(INCFS_IOCTL_BASE_CODE, 36, struct incfs_get_block_count_args)
-
-/*
- * Get per UID read timeouts
- * May only be called on .pending_reads file
- */
-#define INCFS_IOC_GET_READ_TIMEOUTS \
- _IOR(INCFS_IOCTL_BASE_CODE, 37, struct incfs_get_read_timeouts_args)
-
-/*
- * Set per UID read timeouts
- * May only be called on .pending_reads file
- */
-#define INCFS_IOC_SET_READ_TIMEOUTS \
- _IOW(INCFS_IOCTL_BASE_CODE, 38, struct incfs_set_read_timeouts_args)
-
-/* ===== sysfs feature flags ===== */
-/*
- * Each flag is represented by a file in /sys/fs/incremental-fs/features
- * If the file exists the feature is supported
- * Also the file contents will be the line "supported"
- */
-
-/*
- * Basic flag stating that the core incfs file system is available
- */
-#define INCFS_FEATURE_FLAG_COREFS "corefs"
-
-/*
- * zstd compression support
- */
-#define INCFS_FEATURE_FLAG_ZSTD "zstd"
-
-/*
- * v2 feature set support. Covers:
- * INCFS_IOC_CREATE_MAPPED_FILE
- * INCFS_IOC_GET_BLOCK_COUNT
- * INCFS_IOC_GET_READ_TIMEOUTS/INCFS_IOC_SET_READ_TIMEOUTS
- * .blocks_written status file
- * .incomplete folder
- * report_uid mount option
- */
-#define INCFS_FEATURE_FLAG_V2 "v2"
-
enum incfs_compression_alg {
COMPRESSION_NONE = 0,
- COMPRESSION_LZ4 = 1,
- COMPRESSION_ZSTD = 2,
+ COMPRESSION_LZ4 = 1
};
enum incfs_block_flags {
@@ -170,8 +109,6 @@ typedef struct {
/*
* Description of a pending read. A pending read - a read call by
* a userspace program for which the filesystem currently doesn't have data.
- *
- * Reads from .pending_reads and .log return an array of these structure
*/
struct incfs_pending_read_info {
/* Id of a file that is being read from. */
@@ -187,32 +124,6 @@ struct incfs_pending_read_info {
__u32 serial_number;
};
-/*
- * Description of a pending read. A pending read - a read call by
- * a userspace program for which the filesystem currently doesn't have data.
- *
- * This version of incfs_pending_read_info is used whenever the file system is
- * mounted with the report_uid flag
- */
-struct incfs_pending_read_info2 {
- /* Id of a file that is being read from. */
- incfs_uuid_t file_id;
-
- /* A number of microseconds since system boot to the read. */
- __aligned_u64 timestamp_us;
-
- /* Index of a file block that is being read. */
- __u32 block_index;
-
- /* A serial number of this pending read. */
- __u32 serial_number;
-
- /* The UID of the reading process */
- __u32 uid;
-
- __u32 reserved;
-};
-
/*
* Description of a data or hash block to add to a data file.
*/
@@ -420,131 +331,4 @@ struct incfs_get_filled_blocks_args {
__u32 index_out;
};
-/*
- * Create a new mapped file
- * Argument for INCFS_IOC_CREATE_MAPPED_FILE
- */
-struct incfs_create_mapped_file_args {
- /*
- * Total size of the new file.
- */
- __aligned_u64 size;
-
- /*
- * File mode. Permissions and dir flag.
- */
- __u16 mode;
-
- __u16 reserved1;
-
- __u32 reserved2;
-
- /*
- * A pointer to a null-terminated relative path to the incfs mount
- * point
- * Max length: PATH_MAX
- *
- * Equivalent to: char *directory_path;
- */
- __aligned_u64 directory_path;
-
- /*
- * A pointer to a null-terminated file name.
- * Max length: PATH_MAX
- *
- * Equivalent to: char *file_name;
- */
- __aligned_u64 file_name;
-
- /* Id of source file to map. */
- incfs_uuid_t source_file_id;
-
- /*
- * Offset in source file to start mapping. Must be a multiple of
- * INCFS_DATA_FILE_BLOCK_SIZE
- */
- __aligned_u64 source_offset;
-};
-
-/*
- * Get information about the blocks in this file
- * Argument for INCFS_IOC_GET_BLOCK_COUNT
- */
-struct incfs_get_block_count_args {
- /* Total number of data blocks in the file */
- __u32 total_data_blocks_out;
-
- /* Number of filled data blocks in the file */
- __u32 filled_data_blocks_out;
-
- /* Total number of hash blocks in the file */
- __u32 total_hash_blocks_out;
-
- /* Number of filled hash blocks in the file */
- __u32 filled_hash_blocks_out;
-};
-
-/* Description of timeouts for one UID */
-struct incfs_per_uid_read_timeouts {
- /* UID to apply these timeouts to */
- __u32 uid;
-
- /*
- * Min time in microseconds to read any block. Note that this doesn't
- * apply to reads which are satisfied from the page cache.
- */
- __u32 min_time_us;
-
- /*
- * Min time in microseconds to satisfy a pending read. Any pending read
- * which is filled before this time will be delayed so that the total
- * read time >= this value.
- */
- __u32 min_pending_time_us;
-
- /*
- * Max time in microseconds to satisfy a pending read before the read
- * times out. If set to U32_MAX, defaults to mount options
- * read_timeout_ms * 1000. Must be >= min_pending_time_us
- */
- __u32 max_pending_time_us;
-};
-
-/*
- * Get the read timeouts array
- * Argument for INCFS_IOC_GET_READ_TIMEOUTS
- */
-struct incfs_get_read_timeouts_args {
- /*
- * A pointer to a buffer to fill with the current timeouts
- *
- * Equivalent to struct incfs_per_uid_read_timeouts *
- */
- __aligned_u64 timeouts_array;
-
- /* Size of above buffer in bytes */
- __u32 timeouts_array_size;
-
- /* Size used in bytes, or size needed if -ENOMEM returned */
- __u32 timeouts_array_size_out;
-};
-
-/*
- * Set the read timeouts array
- * Arguments for INCFS_IOC_SET_READ_TIMEOUTS
- */
-struct incfs_set_read_timeouts_args {
- /*
- * A pointer to an array containing the new timeouts
- * This will replace any existing timeouts
- *
- * Equivalent to struct incfs_per_uid_read_timeouts *
- */
- __aligned_u64 timeouts_array;
-
- /* Size of above array in bytes. Must be < 256 */
- __u32 timeouts_array_size;
-};
-
-
#endif /* _UAPI_LINUX_INCREMENTALFS_H */
diff --git a/include/uapi/linux/kernel.h b/include/uapi/linux/kernel.h
index 0ff8f7477847..fadf2db71fe8 100644
--- a/include/uapi/linux/kernel.h
+++ b/include/uapi/linux/kernel.h
@@ -3,13 +3,6 @@
#define _UAPI_LINUX_KERNEL_H
#include
-
-/*
- * 'kernel.h' contains some often-used function prototypes etc
- */
-#define __ALIGN_KERNEL(x, a) __ALIGN_KERNEL_MASK(x, (typeof(x))(a) - 1)
-#define __ALIGN_KERNEL_MASK(x, mask) (((x) + (mask)) & ~(mask))
-
-#define __KERNEL_DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
+#include
#endif /* _UAPI_LINUX_KERNEL_H */
diff --git a/include/uapi/linux/lightnvm.h b/include/uapi/linux/lightnvm.h
index f9a1be7fc696..ead2e72e5c88 100644
--- a/include/uapi/linux/lightnvm.h
+++ b/include/uapi/linux/lightnvm.h
@@ -21,7 +21,7 @@
#define _UAPI_LINUX_LIGHTNVM_H
#ifdef __KERNEL__
-#include
+#include
#include
#else /* __KERNEL__ */
#include
diff --git a/include/uapi/linux/mroute6.h b/include/uapi/linux/mroute6.h
index 9999cc006390..1617eb9949a5 100644
--- a/include/uapi/linux/mroute6.h
+++ b/include/uapi/linux/mroute6.h
@@ -2,7 +2,7 @@
#ifndef _UAPI__LINUX_MROUTE6_H
#define _UAPI__LINUX_MROUTE6_H
-#include
+#include
#include
#include
#include /* For struct sockaddr_in6. */
diff --git a/include/uapi/linux/netfilter/x_tables.h b/include/uapi/linux/netfilter/x_tables.h
index a8283f7dbc51..b8c6bb233ac1 100644
--- a/include/uapi/linux/netfilter/x_tables.h
+++ b/include/uapi/linux/netfilter/x_tables.h
@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
#ifndef _UAPI_X_TABLES_H
#define _UAPI_X_TABLES_H
-#include
+#include
#include
#define XT_FUNCTION_MAXNAMELEN 30
diff --git a/include/uapi/linux/netlink.h b/include/uapi/linux/netlink.h
index 5fa3fcc10128..d5ce61f86fd0 100644
--- a/include/uapi/linux/netlink.h
+++ b/include/uapi/linux/netlink.h
@@ -2,7 +2,7 @@
#ifndef _UAPI__LINUX_NETLINK_H
#define _UAPI__LINUX_NETLINK_H
-#include
+#include
#include /* for __kernel_sa_family_t */
#include
diff --git a/include/uapi/linux/sysctl.h b/include/uapi/linux/sysctl.h
index 4b9f32fb49c3..e5e038817292 100644
--- a/include/uapi/linux/sysctl.h
+++ b/include/uapi/linux/sysctl.h
@@ -23,7 +23,7 @@
#ifndef _UAPI_LINUX_SYSCTL_H
#define _UAPI_LINUX_SYSCTL_H
-#include
+#include
#include
#include
diff --git a/include/xen/xenbus.h b/include/xen/xenbus.h
index eba01ab5a55e..fe9a9fa2ebc4 100644
--- a/include/xen/xenbus.h
+++ b/include/xen/xenbus.h
@@ -187,7 +187,7 @@ void xs_suspend_cancel(void);
struct work_struct;
-void xenbus_probe(struct work_struct *);
+void xenbus_probe(void);
#define XENBUS_IS_ERR_READ(str) ({ \
if (!IS_ERR(str) && strlen(str) == 0) { \
diff --git a/kernel/exit.c b/kernel/exit.c
index 0ce250ee1d4c..81b8a3bc24db 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -498,7 +498,7 @@ static void exit_mm(void)
struct mm_struct *mm = current->mm;
struct core_state *core_state;
- mm_release(current, mm);
+ exit_mm_release(current, mm);
if (!mm)
return;
sync_mm_rss(mm);
@@ -819,33 +819,13 @@ void __noreturn do_exit(long code)
*/
if (unlikely(tsk->flags & PF_EXITING)) {
pr_alert("Fixing recursive fault but reboot is needed!\n");
- /*
- * We can do this unlocked here. The futex code uses
- * this flag just to verify whether the pi state
- * cleanup has been done or not. In the worst case it
- * loops once more. We pretend that the cleanup was
- * done as there is no way to return. Either the
- * OWNER_DIED bit is set by now or we push the blocked
- * task into the wait for ever nirwana as well.
- */
- tsk->flags |= PF_EXITPIDONE;
+ futex_exit_recursive(tsk);
set_current_state(TASK_UNINTERRUPTIBLE);
schedule();
}
exit_signals(tsk); /* sets PF_EXITING */
sched_exit(tsk);
- /*
- * Ensure that all new tsk->pi_lock acquisitions must observe
- * PF_EXITING. Serializes against futex.c:attach_to_pi_owner().
- */
- smp_mb();
- /*
- * Ensure that we must observe the pi_state in exit_mm() ->
- * mm_release() -> exit_pi_state_list().
- */
- raw_spin_lock_irq(&tsk->pi_lock);
- raw_spin_unlock_irq(&tsk->pi_lock);
/* sync mm's RSS info before statistics gathering */
if (tsk->mm)
@@ -920,12 +900,6 @@ void __noreturn do_exit(long code)
* Make sure we are holding no locks:
*/
debug_check_no_locks_held();
- /*
- * We can do this unlocked here. The futex code uses this flag
- * just to verify whether the pi state cleanup has been done
- * or not. In the worst case it loops once more.
- */
- tsk->flags |= PF_EXITPIDONE;
if (tsk->io_context)
exit_io_context(tsk);
diff --git a/kernel/fork.c b/kernel/fork.c
index f55889419f7c..6b1f9e7fdad6 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -1267,24 +1267,8 @@ static int wait_for_vfork_done(struct task_struct *child,
* restoring the old one. . .
* Eric Biederman 10 January 1998
*/
-void mm_release(struct task_struct *tsk, struct mm_struct *mm)
+static void mm_release(struct task_struct *tsk, struct mm_struct *mm)
{
- /* Get rid of any futexes when releasing the mm */
-#ifdef CONFIG_FUTEX
- if (unlikely(tsk->robust_list)) {
- exit_robust_list(tsk);
- tsk->robust_list = NULL;
- }
-#ifdef CONFIG_COMPAT
- if (unlikely(tsk->compat_robust_list)) {
- compat_exit_robust_list(tsk);
- tsk->compat_robust_list = NULL;
- }
-#endif
- if (unlikely(!list_empty(&tsk->pi_state_list)))
- exit_pi_state_list(tsk);
-#endif
-
uprobe_free_utask(tsk);
/* Get rid of any cached register state */
@@ -1317,6 +1301,18 @@ void mm_release(struct task_struct *tsk, struct mm_struct *mm)
complete_vfork_done(tsk);
}
+void exit_mm_release(struct task_struct *tsk, struct mm_struct *mm)
+{
+ futex_exit_release(tsk);
+ mm_release(tsk, mm);
+}
+
+void exec_mm_release(struct task_struct *tsk, struct mm_struct *mm)
+{
+ futex_exec_release(tsk);
+ mm_release(tsk, mm);
+}
+
/*
* Allocate a new mm structure and copy contents from the
* mm structure of the passed in task structure.
@@ -2101,14 +2097,8 @@ static __latent_entropy struct task_struct *copy_process(
#ifdef CONFIG_BLOCK
p->plug = NULL;
#endif
-#ifdef CONFIG_FUTEX
- p->robust_list = NULL;
-#ifdef CONFIG_COMPAT
- p->compat_robust_list = NULL;
-#endif
- INIT_LIST_HEAD(&p->pi_state_list);
- p->pi_state_cache = NULL;
-#endif
+ futex_init_task(p);
+
/*
* sigaltstack should be cleared when sharing the same VM
*/
diff --git a/kernel/futex.c b/kernel/futex.c
index 76b728ed6788..224adcdac6c1 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -341,6 +341,12 @@ static inline bool should_fail_futex(bool fshared)
}
#endif /* CONFIG_FAIL_FUTEX */
+#ifdef CONFIG_COMPAT
+static void compat_exit_robust_list(struct task_struct *curr);
+#else
+static inline void compat_exit_robust_list(struct task_struct *curr) { }
+#endif
+
static inline void futex_get_mm(union futex_key *key)
{
mmgrab(key->private.mm);
@@ -878,17 +884,11 @@ static void put_pi_state(struct futex_pi_state *pi_state)
* and has cleaned up the pi_state already
*/
if (pi_state->owner) {
- struct task_struct *owner;
unsigned long flags;
raw_spin_lock_irqsave(&pi_state->pi_mutex.wait_lock, flags);
- owner = pi_state->owner;
- if (owner) {
- raw_spin_lock(&owner->pi_lock);
- list_del_init(&pi_state->list);
- raw_spin_unlock(&owner->pi_lock);
- }
- rt_mutex_proxy_unlock(&pi_state->pi_mutex, owner);
+ pi_state_update_owner(pi_state, NULL);
+ rt_mutex_proxy_unlock(&pi_state->pi_mutex);
raw_spin_unlock_irqrestore(&pi_state->pi_mutex.wait_lock, flags);
}
@@ -913,7 +913,7 @@ static void put_pi_state(struct futex_pi_state *pi_state)
* Kernel cleans up PI-state, but userspace is likely hosed.
* (Robust-futex cleanup is separate and might save the day for userspace.)
*/
-void exit_pi_state_list(struct task_struct *curr)
+static void exit_pi_state_list(struct task_struct *curr)
{
struct list_head *next, *head = &curr->pi_state_list;
struct futex_pi_state *pi_state;
@@ -983,7 +983,8 @@ void exit_pi_state_list(struct task_struct *curr)
}
raw_spin_unlock_irq(&curr->pi_lock);
}
-
+#else
+static inline void exit_pi_state_list(struct task_struct *curr) { }
#endif
/*
@@ -1193,16 +1194,47 @@ out_error:
return ret;
}
+/**
+ * wait_for_owner_exiting - Block until the owner has exited
+ * @exiting: Pointer to the exiting task
+ *
+ * Caller must hold a refcount on @exiting.
+ */
+static void wait_for_owner_exiting(int ret, struct task_struct *exiting)
+{
+ if (ret != -EBUSY) {
+ WARN_ON_ONCE(exiting);
+ return;
+ }
+
+ if (WARN_ON_ONCE(ret == -EBUSY && !exiting))
+ return;
+
+ mutex_lock(&exiting->futex_exit_mutex);
+ /*
+ * No point in doing state checking here. If the waiter got here
+ * while the task was in exec()->exec_futex_release() then it can
+ * have any FUTEX_STATE_* value when the waiter has acquired the
+ * mutex. OK, if running, EXITING or DEAD if it reached exit()
+ * already. Highly unlikely and not a problem. Just one more round
+ * through the futex maze.
+ */
+ mutex_unlock(&exiting->futex_exit_mutex);
+
+ put_task_struct(exiting);
+}
+
static int handle_exit_race(u32 __user *uaddr, u32 uval,
struct task_struct *tsk)
{
u32 uval2;
/*
- * If PF_EXITPIDONE is not yet set, then try again.
+ * If the futex exit state is not yet FUTEX_STATE_DEAD, tell the
+ * caller that the alleged owner is busy.
*/
- if (tsk && !(tsk->flags & PF_EXITPIDONE))
- return -EAGAIN;
+ if (tsk && tsk->futex_state != FUTEX_STATE_DEAD)
+ return -EBUSY;
/*
* Reread the user space value to handle the following situation:
@@ -1220,8 +1252,9 @@ static int handle_exit_race(u32 __user *uaddr, u32 uval,
* *uaddr = 0xC0000000; tsk = get_task(PID);
* } if (!tsk->flags & PF_EXITING) {
* ... attach();
- * tsk->flags |= PF_EXITPIDONE; } else {
- * if (!(tsk->flags & PF_EXITPIDONE))
+ * tsk->futex_state = } else {
+ * FUTEX_STATE_DEAD; if (tsk->futex_state !=
+ * FUTEX_STATE_DEAD)
* return -EAGAIN;
* return -ESRCH; <--- FAIL
* }
@@ -1252,7 +1285,8 @@ static int handle_exit_race(u32 __user *uaddr, u32 uval,
* it after doing proper sanity checks.
*/
static int attach_to_pi_owner(u32 __user *uaddr, u32 uval, union futex_key *key,
- struct futex_pi_state **ps)
+ struct futex_pi_state **ps,
+ struct task_struct **exiting)
{
pid_t pid = uval & FUTEX_TID_MASK;
struct futex_pi_state *pi_state;
@@ -1277,22 +1311,33 @@ static int attach_to_pi_owner(u32 __user *uaddr, u32 uval, union futex_key *key,
}
/*
- * We need to look at the task state flags to figure out,
- * whether the task is exiting. To protect against the do_exit
- * change of the task flags, we do this protected by
- * p->pi_lock:
+ * We need to look at the task state to figure out, whether the
+ * task is exiting. To protect against the change of the task state
+ * in futex_exit_release(), we do this protected by p->pi_lock:
*/
raw_spin_lock_irq(&p->pi_lock);
- if (unlikely(p->flags & PF_EXITING)) {
+ if (unlikely(p->futex_state != FUTEX_STATE_OK)) {
/*
- * The task is on the way out. When PF_EXITPIDONE is
- * set, we know that the task has finished the
- * cleanup:
+ * The task is on the way out. When the futex state is
+ * FUTEX_STATE_DEAD, we know that the task has finished
+ * the cleanup:
*/
int ret = handle_exit_race(uaddr, uval, p);
raw_spin_unlock_irq(&p->pi_lock);
- put_task_struct(p);
+ /*
+ * If the owner task is between FUTEX_STATE_EXITING and
+ * FUTEX_STATE_DEAD then store the task pointer and keep
+ * the reference on the task struct. The calling code will
+ * drop all locks, wait for the task to reach
+ * FUTEX_STATE_DEAD and then drop the refcount. This is
+ * required to prevent a live lock when the current task
+ * preempted the exiting task between the two states.
+ */
+ if (ret == -EBUSY)
+ *exiting = p;
+ else
+ put_task_struct(p);
return ret;
}
@@ -1331,7 +1376,8 @@ static int attach_to_pi_owner(u32 __user *uaddr, u32 uval, union futex_key *key,
static int lookup_pi_state(u32 __user *uaddr, u32 uval,
struct futex_hash_bucket *hb,
- union futex_key *key, struct futex_pi_state **ps)
+ union futex_key *key, struct futex_pi_state **ps,
+ struct task_struct **exiting)
{
struct futex_q *top_waiter = futex_top_waiter(hb, key);
@@ -1346,7 +1392,7 @@ static int lookup_pi_state(u32 __user *uaddr, u32 uval,
* We are the first waiter - try to look up the owner based on
* @uval and attach to it.
*/
- return attach_to_pi_owner(uaddr, uval, key, ps);
+ return attach_to_pi_owner(uaddr, uval, key, ps, exiting);
}
static int lock_pi_update_atomic(u32 __user *uaddr, u32 uval, u32 newval)
@@ -1374,6 +1420,8 @@ static int lock_pi_update_atomic(u32 __user *uaddr, u32 uval, u32 newval)
* lookup
* @task: the task to perform the atomic lock work for. This will
* be "current" except in the case of requeue pi.
+ * @exiting: Pointer to store the task pointer of the owner task
+ * which is in the middle of exiting
* @set_waiters: force setting the FUTEX_WAITERS bit (1) or not (0)
*
* Return:
@@ -1382,11 +1430,17 @@ static int lock_pi_update_atomic(u32 __user *uaddr, u32 uval, u32 newval)
* - <0 - error
*
* The hb->lock and futex_key refs shall be held by the caller.
+ *
+ * @exiting is only set when the return value is -EBUSY. If so, this holds
+ * a refcount on the exiting task on return and the caller needs to drop it
+ * after waiting for the exit to complete.
*/
static int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb,
union futex_key *key,
struct futex_pi_state **ps,
- struct task_struct *task, int set_waiters)
+ struct task_struct *task,
+ struct task_struct **exiting,
+ int set_waiters)
{
u32 uval, newval, vpid = task_pid_vnr(task);
struct futex_q *top_waiter;
@@ -1456,7 +1510,7 @@ static int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb,
* attach to the owner. If that fails, no harm done, we only
* set the FUTEX_WAITERS bit in the user space variable.
*/
- return attach_to_pi_owner(uaddr, newval, key, ps);
+ return attach_to_pi_owner(uaddr, newval, key, ps, exiting);
}
/**
@@ -1866,6 +1920,8 @@ void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key,
* @key1: the from futex key
* @key2: the to futex key
* @ps: address to store the pi_state pointer
+ * @exiting: Pointer to store the task pointer of the owner task
+ * which is in the middle of exiting
* @set_waiters: force setting the FUTEX_WAITERS bit (1) or not (0)
*
* Try and get the lock on behalf of the top waiter if we can do it atomically.
@@ -1873,16 +1929,20 @@ void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key,
* then direct futex_lock_pi_atomic() to force setting the FUTEX_WAITERS bit.
* hb1 and hb2 must be held by the caller.
*
+ * @exiting is only set when the return value is -EBUSY. If so, this holds
+ * a refcount on the exiting task on return and the caller needs to drop it
+ * after waiting for the exit to complete.
+ *
* Return:
* - 0 - failed to acquire the lock atomically;
* - >0 - acquired the lock, return value is vpid of the top_waiter
* - <0 - error
*/
-static int futex_proxy_trylock_atomic(u32 __user *pifutex,
- struct futex_hash_bucket *hb1,
- struct futex_hash_bucket *hb2,
- union futex_key *key1, union futex_key *key2,
- struct futex_pi_state **ps, int set_waiters)
+static int
+futex_proxy_trylock_atomic(u32 __user *pifutex, struct futex_hash_bucket *hb1,
+ struct futex_hash_bucket *hb2, union futex_key *key1,
+ union futex_key *key2, struct futex_pi_state **ps,
+ struct task_struct **exiting, int set_waiters)
{
struct futex_q *top_waiter = NULL;
u32 curval;
@@ -1919,7 +1979,7 @@ static int futex_proxy_trylock_atomic(u32 __user *pifutex,
*/
vpid = task_pid_vnr(top_waiter->task);
ret = futex_lock_pi_atomic(pifutex, hb2, key2, ps, top_waiter->task,
- set_waiters);
+ exiting, set_waiters);
if (ret == 1) {
requeue_pi_wake_futex(top_waiter, key2, hb2);
return vpid;
@@ -2048,6 +2108,8 @@ retry_private:
}
if (requeue_pi && (task_count - nr_wake < nr_requeue)) {
+ struct task_struct *exiting = NULL;
+
/*
* Attempt to acquire uaddr2 and wake the top waiter. If we
* intend to requeue waiters, force setting the FUTEX_WAITERS
@@ -2055,7 +2117,8 @@ retry_private:
* faults rather in the requeue loop below.
*/
ret = futex_proxy_trylock_atomic(uaddr2, hb1, hb2, &key1,
- &key2, &pi_state, nr_requeue);
+ &key2, &pi_state,
+ &exiting, nr_requeue);
/*
* At this point the top_waiter has either taken uaddr2 or is
@@ -2082,7 +2145,8 @@ retry_private:
* If that call succeeds then we have pi_state and an
* initial refcount on it.
*/
- ret = lookup_pi_state(uaddr2, ret, hb2, &key2, &pi_state);
+ ret = lookup_pi_state(uaddr2, ret, hb2, &key2,
+ &pi_state, &exiting);
}
switch (ret) {
@@ -2100,17 +2164,24 @@ retry_private:
if (!ret)
goto retry;
goto out;
+ case -EBUSY:
case -EAGAIN:
/*
* Two reasons for this:
- * - Owner is exiting and we just wait for the
+ * - EBUSY: Owner is exiting and we just wait for the
* exit to complete.
- * - The user space value changed.
+ * - EAGAIN: The user space value changed.
*/
double_unlock_hb(hb1, hb2);
hb_waiters_dec(hb2);
put_futex_key(&key2);
put_futex_key(&key1);
+ /*
+ * Handle the case where the owner is in the middle of
+ * exiting. Wait for the exit to complete otherwise
+ * this task might loop forever, aka. live lock.
+ */
+ wait_for_owner_exiting(ret, exiting);
cond_resched();
goto retry;
default:
@@ -2833,6 +2904,7 @@ static int futex_lock_pi(u32 __user *uaddr, unsigned int flags,
ktime_t *time, int trylock)
{
struct hrtimer_sleeper timeout, *to = NULL;
+ struct task_struct *exiting = NULL;
struct rt_mutex_waiter rt_waiter;
struct futex_hash_bucket *hb;
struct futex_q q = futex_q_init;
@@ -2860,7 +2932,8 @@ retry:
retry_private:
hb = queue_lock(&q);
- ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current, 0);
+ ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current,
+ &exiting, 0);
if (unlikely(ret)) {
/*
* Atomic work succeeded and we got the lock,
@@ -2873,15 +2946,22 @@ retry_private:
goto out_unlock_put_key;
case -EFAULT:
goto uaddr_faulted;
+ case -EBUSY:
case -EAGAIN:
/*
* Two reasons for this:
- * - Task is exiting and we just wait for the
+ * - EBUSY: Task is exiting and we just wait for the
* exit to complete.
- * - The user space value changed.
+ * - EAGAIN: The user space value changed.
*/
queue_unlock(hb);
put_futex_key(&q.key);
+ /*
+ * Handle the case where the owner is in the middle of
+ * exiting. Wait for the exit to complete otherwise
+ * this task might loop forever, aka. live lock.
+ */
+ wait_for_owner_exiting(ret, exiting);
cond_resched();
goto retry;
default:
@@ -3604,7 +3684,7 @@ static inline int fetch_robust_entry(struct robust_list __user **entry,
*
* We silently return on any sign of list-walking problem.
*/
-void exit_robust_list(struct task_struct *curr)
+static void exit_robust_list(struct task_struct *curr)
{
struct robust_list_head __user *head = curr->robust_list;
struct robust_list __user *entry, *next_entry, *pending;
@@ -3669,6 +3749,114 @@ void exit_robust_list(struct task_struct *curr)
}
}
+static void futex_cleanup(struct task_struct *tsk)
+{
+ if (unlikely(tsk->robust_list)) {
+ exit_robust_list(tsk);
+ tsk->robust_list = NULL;
+ }
+
+#ifdef CONFIG_COMPAT
+ if (unlikely(tsk->compat_robust_list)) {
+ compat_exit_robust_list(tsk);
+ tsk->compat_robust_list = NULL;
+ }
+#endif
+
+ if (unlikely(!list_empty(&tsk->pi_state_list)))
+ exit_pi_state_list(tsk);
+}
+
+/**
+ * futex_exit_recursive - Set the tasks futex state to FUTEX_STATE_DEAD
+ * @tsk: task to set the state on
+ *
+ * Set the futex exit state of the task lockless. The futex waiter code
+ * observes that state when a task is exiting and loops until the task has
+ * actually finished the futex cleanup. The worst case for this is that the
+ * waiter runs through the wait loop until the state becomes visible.
+ *
+ * This is called from the recursive fault handling path in do_exit().
+ *
+ * This is best effort. Either the futex exit code has run already or
+ * not. If the OWNER_DIED bit has been set on the futex then the waiter can
+ * take it over. If not, the problem is pushed back to user space. If the
+ * futex exit code did not run yet, then an already queued waiter might
+ * block forever, but there is nothing which can be done about that.
+ */
+void futex_exit_recursive(struct task_struct *tsk)
+{
+ /* If the state is FUTEX_STATE_EXITING then futex_exit_mutex is held */
+ if (tsk->futex_state == FUTEX_STATE_EXITING)
+ mutex_unlock(&tsk->futex_exit_mutex);
+ tsk->futex_state = FUTEX_STATE_DEAD;
+}
+
+static void futex_cleanup_begin(struct task_struct *tsk)
+{
+ /*
+ * Prevent various race issues against a concurrent incoming waiter
+ * including live locks by forcing the waiter to block on
+ * tsk->futex_exit_mutex when it observes FUTEX_STATE_EXITING in
+ * attach_to_pi_owner().
+ */
+ mutex_lock(&tsk->futex_exit_mutex);
+
+ /*
+ * Switch the state to FUTEX_STATE_EXITING under tsk->pi_lock.
+ *
+ * This ensures that all subsequent checks of tsk->futex_state in
+ * attach_to_pi_owner() must observe FUTEX_STATE_EXITING with
+ * tsk->pi_lock held.
+ *
+ * It guarantees also that a pi_state which was queued right before
+ * the state change under tsk->pi_lock by a concurrent waiter must
+ * be observed in exit_pi_state_list().
+ */
+ raw_spin_lock_irq(&tsk->pi_lock);
+ tsk->futex_state = FUTEX_STATE_EXITING;
+ raw_spin_unlock_irq(&tsk->pi_lock);
+}
+
+static void futex_cleanup_end(struct task_struct *tsk, int state)
+{
+ /*
+ * Lockless store. The only side effect is that an observer might
+ * take another loop until it becomes visible.
+ */
+ tsk->futex_state = state;
+ /*
+ * Drop the exit protection. This unblocks waiters which observed
+ * FUTEX_STATE_EXITING to reevaluate the state.
+ */
+ mutex_unlock(&tsk->futex_exit_mutex);
+}
+
+void futex_exec_release(struct task_struct *tsk)
+{
+ /*
+ * The state handling is done for consistency, but in the case of
+ * exec() there is no way to prevent futher damage as the PID stays
+ * the same. But for the unlikely and arguably buggy case that a
+ * futex is held on exec(), this provides at least as much state
+ * consistency protection which is possible.
+ */
+ futex_cleanup_begin(tsk);
+ futex_cleanup(tsk);
+ /*
+ * Reset the state to FUTEX_STATE_OK. The task is alive and about
+ * exec a new binary.
+ */
+ futex_cleanup_end(tsk, FUTEX_STATE_OK);
+}
+
+void futex_exit_release(struct task_struct *tsk)
+{
+ futex_cleanup_begin(tsk);
+ futex_cleanup(tsk);
+ futex_cleanup_end(tsk, FUTEX_STATE_DEAD);
+}
+
long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout,
u32 __user *uaddr2, u32 val2, u32 val3)
{
@@ -3796,7 +3984,7 @@ static void __user *futex_uaddr(struct robust_list __user *entry,
*
* We silently return on any sign of list-walking problem.
*/
-void compat_exit_robust_list(struct task_struct *curr)
+static void compat_exit_robust_list(struct task_struct *curr)
{
struct compat_robust_list_head __user *head = curr->compat_robust_list;
struct robust_list __user *entry, *next_entry, *pending;
diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c
index 26814a14013c..d2496a5c8c48 100644
--- a/kernel/irq/irqdesc.c
+++ b/kernel/irq/irqdesc.c
@@ -962,3 +962,4 @@ unsigned int kstat_irqs_usr(unsigned int irq)
rcu_read_unlock();
return sum;
}
+EXPORT_SYMBOL_GPL(kstat_irqs_usr);
diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c
index 9562aaa2afdc..a5ec4f68527e 100644
--- a/kernel/locking/rtmutex.c
+++ b/kernel/locking/rtmutex.c
@@ -1719,8 +1719,7 @@ void rt_mutex_init_proxy_locked(struct rt_mutex *lock,
* possible because it belongs to the pi_state which is about to be freed
* and it is not longer visible to other tasks.
*/
-void rt_mutex_proxy_unlock(struct rt_mutex *lock,
- struct task_struct *proxy_owner)
+void rt_mutex_proxy_unlock(struct rt_mutex *lock)
{
debug_rt_mutex_proxy_unlock(lock);
rt_mutex_set_owner(lock, NULL);
diff --git a/kernel/locking/rtmutex_common.h b/kernel/locking/rtmutex_common.h
index d1d62f942be2..ca6fb489007b 100644
--- a/kernel/locking/rtmutex_common.h
+++ b/kernel/locking/rtmutex_common.h
@@ -133,8 +133,7 @@ enum rtmutex_chainwalk {
extern struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock);
extern void rt_mutex_init_proxy_locked(struct rt_mutex *lock,
struct task_struct *proxy_owner);
-extern void rt_mutex_proxy_unlock(struct rt_mutex *lock,
- struct task_struct *proxy_owner);
+extern void rt_mutex_proxy_unlock(struct rt_mutex *lock);
extern void rt_mutex_init_waiter(struct rt_mutex_waiter *waiter);
extern int __rt_mutex_start_proxy_lock(struct rt_mutex *lock,
struct rt_mutex_waiter *waiter,
diff --git a/kernel/module.c b/kernel/module.c
index 9dfc374bfeb4..20eb6d662d0a 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -1806,7 +1806,6 @@ static int mod_sysfs_init(struct module *mod)
if (err)
mod_kobject_put(mod);
- /* delay uevent until full sysfs population */
out:
return err;
}
@@ -1843,7 +1842,6 @@ static int mod_sysfs_setup(struct module *mod,
add_sect_attrs(mod, info);
add_notes_attrs(mod, info);
- kobject_uevent(&mod->mkobj.kobj, KOBJ_ADD);
return 0;
out_unreg_modinfo_attrs:
@@ -3514,6 +3512,9 @@ static noinline int do_init_module(struct module *mod)
blocking_notifier_call_chain(&module_notify_list,
MODULE_STATE_LIVE, mod);
+ /* Delay uevent until module has finished its init routine */
+ kobject_uevent(&mod->mkobj.kobj, KOBJ_ADD);
+
/*
* We need to finish all async code before the module init sequence
* is done. This has potential to deadlock. For example, a newly
@@ -3856,6 +3857,7 @@ static int load_module(struct load_info *info, const char __user *uargs,
MODULE_STATE_GOING, mod);
klp_module_going(mod);
bug_cleanup:
+ mod->state = MODULE_STATE_GOING;
/* module_bug_cleanup needs module_mutex protection */
mutex_lock(&module_mutex);
module_bug_cleanup(mod);
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index 75b31bc13823..cf01dd6ac1ae 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -504,7 +504,7 @@ config KPROBE_EVENTS
config KPROBE_EVENTS_ON_NOTRACE
bool "Do NOT protect notrace function from kprobe events"
depends on KPROBE_EVENTS
- depends on KPROBES_ON_FTRACE
+ depends on DYNAMIC_FTRACE
default n
help
This is only for the developers who want to debug ftrace itself
diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c
index c45b017bacd4..5c17f70c7f2d 100644
--- a/kernel/trace/trace_kprobe.c
+++ b/kernel/trace/trace_kprobe.c
@@ -517,7 +517,7 @@ disable_trace_kprobe(struct trace_kprobe *tk, struct trace_event_file *file)
return ret;
}
-#if defined(CONFIG_KPROBES_ON_FTRACE) && \
+#if defined(CONFIG_DYNAMIC_FTRACE) && \
!defined(CONFIG_KPROBE_EVENTS_ON_NOTRACE)
static bool __within_notrace_func(unsigned long addr)
{
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index f3ca7e0394b9..423778ab656a 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -3597,17 +3597,24 @@ static void pwq_adjust_max_active(struct pool_workqueue *pwq)
* is updated and visible.
*/
if (!freezable || !workqueue_freezing) {
+ bool kick = false;
+
pwq->max_active = wq->saved_max_active;
while (!list_empty(&pwq->delayed_works) &&
- pwq->nr_active < pwq->max_active)
+ pwq->nr_active < pwq->max_active) {
pwq_activate_first_delayed(pwq);
+ kick = true;
+ }
/*
* Need to kick a worker after thawed or an unbound wq's
- * max_active is bumped. It's a slow path. Do it always.
+ * max_active is bumped. In realtime scenarios, always kicking a
+ * worker will cause interference on the isolated cpu cores, so
+ * let's kick iff work items were activated.
*/
- wake_up_worker(pwq->pool);
+ if (kick)
+ wake_up_worker(pwq->pool);
} else {
pwq->max_active = 0;
}
diff --git a/lib/genalloc.c b/lib/genalloc.c
index 7e85d1e37a6e..0b8ee173cf3a 100644
--- a/lib/genalloc.c
+++ b/lib/genalloc.c
@@ -83,14 +83,14 @@ static int clear_bits_ll(unsigned long *addr, unsigned long mask_to_clear)
* users set the same bit, one user will return remain bits, otherwise
* return 0.
*/
-static int bitmap_set_ll(unsigned long *map, int start, int nr)
+static int bitmap_set_ll(unsigned long *map, unsigned long start, unsigned long nr)
{
unsigned long *p = map + BIT_WORD(start);
- const int size = start + nr;
+ const unsigned long size = start + nr;
int bits_to_set = BITS_PER_LONG - (start % BITS_PER_LONG);
unsigned long mask_to_set = BITMAP_FIRST_WORD_MASK(start);
- while (nr - bits_to_set >= 0) {
+ while (nr >= bits_to_set) {
if (set_bits_ll(p, mask_to_set))
return nr;
nr -= bits_to_set;
@@ -118,14 +118,15 @@ static int bitmap_set_ll(unsigned long *map, int start, int nr)
* users clear the same bit, one user will return remain bits,
* otherwise return 0.
*/
-static int bitmap_clear_ll(unsigned long *map, int start, int nr)
+static unsigned long
+bitmap_clear_ll(unsigned long *map, unsigned long start, unsigned long nr)
{
unsigned long *p = map + BIT_WORD(start);
- const int size = start + nr;
+ const unsigned long size = start + nr;
int bits_to_clear = BITS_PER_LONG - (start % BITS_PER_LONG);
unsigned long mask_to_clear = BITMAP_FIRST_WORD_MASK(start);
- while (nr - bits_to_clear >= 0) {
+ while (nr >= bits_to_clear) {
if (clear_bits_ll(p, mask_to_clear))
return nr;
nr -= bits_to_clear;
@@ -184,8 +185,8 @@ int gen_pool_add_virt(struct gen_pool *pool, unsigned long virt, phys_addr_t phy
size_t size, int nid)
{
struct gen_pool_chunk *chunk;
- int nbits = size >> pool->min_alloc_order;
- int nbytes = sizeof(struct gen_pool_chunk) +
+ unsigned long nbits = size >> pool->min_alloc_order;
+ unsigned long nbytes = sizeof(struct gen_pool_chunk) +
BITS_TO_LONGS(nbits) * sizeof(long);
chunk = vzalloc_node(nbytes, nid);
@@ -242,7 +243,7 @@ void gen_pool_destroy(struct gen_pool *pool)
struct list_head *_chunk, *_next_chunk;
struct gen_pool_chunk *chunk;
int order = pool->min_alloc_order;
- int bit, end_bit;
+ unsigned long bit, end_bit;
list_for_each_safe(_chunk, _next_chunk, &pool->chunks) {
chunk = list_entry(_chunk, struct gen_pool_chunk, next_chunk);
@@ -293,7 +294,7 @@ unsigned long gen_pool_alloc_algo(struct gen_pool *pool, size_t size,
struct gen_pool_chunk *chunk;
unsigned long addr = 0;
int order = pool->min_alloc_order;
- int nbits, start_bit, end_bit, remain;
+ unsigned long nbits, start_bit, end_bit, remain;
#ifndef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG
BUG_ON(in_nmi());
@@ -376,7 +377,7 @@ void gen_pool_free(struct gen_pool *pool, unsigned long addr, size_t size)
{
struct gen_pool_chunk *chunk;
int order = pool->min_alloc_order;
- int start_bit, nbits, remain;
+ unsigned long start_bit, nbits, remain;
#ifndef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG
BUG_ON(in_nmi());
@@ -638,7 +639,7 @@ unsigned long gen_pool_best_fit(unsigned long *map, unsigned long size,
index = bitmap_find_next_zero_area(map, size, start, nr, 0);
while (index < size) {
- int next_bit = find_next_bit(map, size, index + nr);
+ unsigned long next_bit = find_next_bit(map, size, index + nr);
if ((next_bit - index) < len) {
len = next_bit - index;
start_bit = index;
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index df155e1419f8..db68b758f865 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -3854,7 +3854,7 @@ retry:
* So we need to block hugepage fault by PG_hwpoison bit check.
*/
if (unlikely(PageHWPoison(page))) {
- ret = VM_FAULT_HWPOISON |
+ ret = VM_FAULT_HWPOISON_LARGE |
VM_FAULT_SET_HINDEX(hstate_index(h));
goto backout_unlocked;
}
diff --git a/mm/kasan/init.c b/mm/kasan/init.c
index 1f2d7d832c68..807b5416b375 100644
--- a/mm/kasan/init.c
+++ b/mm/kasan/init.c
@@ -383,9 +383,10 @@ static void kasan_remove_pmd_table(pmd_t *pmd, unsigned long addr,
if (kasan_pte_table(*pmd)) {
if (IS_ALIGNED(addr, PMD_SIZE) &&
- IS_ALIGNED(next, PMD_SIZE))
+ IS_ALIGNED(next, PMD_SIZE)) {
pmd_clear(pmd);
- continue;
+ continue;
+ }
}
pte = pte_offset_kernel(pmd, addr);
kasan_remove_pte_table(pte, addr, next);
@@ -408,9 +409,10 @@ static void kasan_remove_pud_table(pud_t *pud, unsigned long addr,
if (kasan_pmd_table(*pud)) {
if (IS_ALIGNED(addr, PUD_SIZE) &&
- IS_ALIGNED(next, PUD_SIZE))
+ IS_ALIGNED(next, PUD_SIZE)) {
pud_clear(pud);
- continue;
+ continue;
+ }
}
pmd = pmd_offset(pud, addr);
pmd_base = pmd_offset(pud, 0);
@@ -434,9 +436,10 @@ static void kasan_remove_p4d_table(p4d_t *p4d, unsigned long addr,
if (kasan_pud_table(*p4d)) {
if (IS_ALIGNED(addr, P4D_SIZE) &&
- IS_ALIGNED(next, P4D_SIZE))
+ IS_ALIGNED(next, P4D_SIZE)) {
p4d_clear(p4d);
- continue;
+ continue;
+ }
}
pud = pud_offset(p4d, addr);
kasan_remove_pud_table(pud, addr, next);
@@ -468,9 +471,10 @@ void kasan_remove_zero_shadow(void *start, unsigned long size)
if (kasan_p4d_table(*pgd)) {
if (IS_ALIGNED(addr, PGDIR_SIZE) &&
- IS_ALIGNED(next, PGDIR_SIZE))
+ IS_ALIGNED(next, PGDIR_SIZE)) {
pgd_clear(pgd);
- continue;
+ continue;
+ }
}
p4d = p4d_offset(pgd, addr);
@@ -494,7 +498,6 @@ int kasan_add_zero_shadow(void *start, unsigned long size)
ret = kasan_populate_early_shadow(shadow_start, shadow_end);
if (ret)
- kasan_remove_zero_shadow(shadow_start,
- size >> KASAN_SHADOW_SCALE_SHIFT);
+ kasan_remove_zero_shadow(start, size);
return ret;
}
diff --git a/mm/slub.c b/mm/slub.c
index dc50ed7f3a88..99d5ac8cabbb 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1899,7 +1899,7 @@ static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,
t = acquire_slab(s, n, page, object == NULL, &objects);
if (!t)
- break;
+ continue; /* cmpxchg raced */
available += objects;
if (!object) {
@@ -5860,10 +5860,8 @@ static int sysfs_slab_add(struct kmem_cache *s)
s->kobj.kset = kset;
err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name);
- if (err) {
- kobject_put(&s->kobj);
+ if (err)
goto out;
- }
err = sysfs_create_group(&s->kobj, &slab_attr_group);
if (err)
diff --git a/net/8021q/vlan.c b/net/8021q/vlan.c
index 5e9950453955..512ada90657b 100644
--- a/net/8021q/vlan.c
+++ b/net/8021q/vlan.c
@@ -277,7 +277,8 @@ static int register_vlan_device(struct net_device *real_dev, u16 vlan_id)
return 0;
out_free_newdev:
- if (new_dev->reg_state == NETREG_UNINITIALIZED)
+ if (new_dev->reg_state == NETREG_UNINITIALIZED ||
+ new_dev->reg_state == NETREG_UNREGISTERED)
free_netdev(new_dev);
return err;
}
diff --git a/net/core/dev.c b/net/core/dev.c
index c7ee9db77b1e..85ceb00271b0 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -8376,6 +8376,11 @@ static netdev_features_t netdev_fix_features(struct net_device *dev,
}
}
+ if ((features & NETIF_F_HW_TLS_RX) && !(features & NETIF_F_RXCSUM)) {
+ netdev_dbg(dev, "Dropping TLS RX HW offload feature since no RXCSUM feature.\n");
+ features &= ~NETIF_F_HW_TLS_RX;
+ }
+
return features;
}
diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c
index 001d7f07e780..fe0d255d66c8 100644
--- a/net/core/net-sysfs.c
+++ b/net/core/net-sysfs.c
@@ -1244,8 +1244,8 @@ static const struct attribute_group dql_group = {
static ssize_t xps_cpus_show(struct netdev_queue *queue,
char *buf)
{
+ int cpu, len, ret, num_tc = 1, tc = 0;
struct net_device *dev = queue->dev;
- int cpu, len, num_tc = 1, tc = 0;
struct xps_dev_maps *dev_maps;
cpumask_var_t mask;
unsigned long index;
@@ -1255,22 +1255,31 @@ static ssize_t xps_cpus_show(struct netdev_queue *queue,
index = get_netdev_queue_index(queue);
+ if (!rtnl_trylock())
+ return restart_syscall();
+
if (dev->num_tc) {
/* Do not allow XPS on subordinate device directly */
num_tc = dev->num_tc;
- if (num_tc < 0)
- return -EINVAL;
+ if (num_tc < 0) {
+ ret = -EINVAL;
+ goto err_rtnl_unlock;
+ }
/* If queue belongs to subordinate dev use its map */
dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev;
tc = netdev_txq_to_tc(dev, index);
- if (tc < 0)
- return -EINVAL;
+ if (tc < 0) {
+ ret = -EINVAL;
+ goto err_rtnl_unlock;
+ }
}
- if (!zalloc_cpumask_var(&mask, GFP_KERNEL))
- return -ENOMEM;
+ if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
+ ret = -ENOMEM;
+ goto err_rtnl_unlock;
+ }
rcu_read_lock();
dev_maps = rcu_dereference(dev->xps_cpus_map);
@@ -1293,9 +1302,15 @@ static ssize_t xps_cpus_show(struct netdev_queue *queue,
}
rcu_read_unlock();
+ rtnl_unlock();
+
len = snprintf(buf, PAGE_SIZE, "%*pb\n", cpumask_pr_args(mask));
free_cpumask_var(mask);
return len < PAGE_SIZE ? len : -EINVAL;
+
+err_rtnl_unlock:
+ rtnl_unlock();
+ return ret;
}
static ssize_t xps_cpus_store(struct netdev_queue *queue,
@@ -1323,7 +1338,13 @@ static ssize_t xps_cpus_store(struct netdev_queue *queue,
return err;
}
+ if (!rtnl_trylock()) {
+ free_cpumask_var(mask);
+ return restart_syscall();
+ }
+
err = netif_set_xps_queue(dev, mask, index);
+ rtnl_unlock();
free_cpumask_var(mask);
@@ -1335,23 +1356,30 @@ static struct netdev_queue_attribute xps_cpus_attribute __ro_after_init
static ssize_t xps_rxqs_show(struct netdev_queue *queue, char *buf)
{
+ int j, len, ret, num_tc = 1, tc = 0;
struct net_device *dev = queue->dev;
struct xps_dev_maps *dev_maps;
unsigned long *mask, index;
- int j, len, num_tc = 1, tc = 0;
index = get_netdev_queue_index(queue);
+ if (!rtnl_trylock())
+ return restart_syscall();
+
if (dev->num_tc) {
num_tc = dev->num_tc;
tc = netdev_txq_to_tc(dev, index);
- if (tc < 0)
- return -EINVAL;
+ if (tc < 0) {
+ ret = -EINVAL;
+ goto err_rtnl_unlock;
+ }
}
mask = kcalloc(BITS_TO_LONGS(dev->num_rx_queues), sizeof(long),
GFP_KERNEL);
- if (!mask)
- return -ENOMEM;
+ if (!mask) {
+ ret = -ENOMEM;
+ goto err_rtnl_unlock;
+ }
rcu_read_lock();
dev_maps = rcu_dereference(dev->xps_rxqs_map);
@@ -1377,10 +1405,16 @@ static ssize_t xps_rxqs_show(struct netdev_queue *queue, char *buf)
out_no_maps:
rcu_read_unlock();
+ rtnl_unlock();
+
len = bitmap_print_to_pagebuf(false, buf, mask, dev->num_rx_queues);
kfree(mask);
return len < PAGE_SIZE ? len : -EINVAL;
+
+err_rtnl_unlock:
+ rtnl_unlock();
+ return ret;
}
static ssize_t xps_rxqs_store(struct netdev_queue *queue, const char *buf,
@@ -1407,10 +1441,17 @@ static ssize_t xps_rxqs_store(struct netdev_queue *queue, const char *buf,
return err;
}
+ if (!rtnl_trylock()) {
+ bitmap_free(mask);
+ return restart_syscall();
+ }
+
cpus_read_lock();
err = __netif_set_xps_queue(dev, mask, index, true);
cpus_read_unlock();
+ rtnl_unlock();
+
kfree(mask);
return err ? : len;
}
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index b9f2c7ec2f91..157dc97f9948 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -401,7 +401,11 @@ struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len,
len += NET_SKB_PAD;
- if ((len > SKB_WITH_OVERHEAD(PAGE_SIZE)) ||
+ /* If requested length is either too small or too big,
+ * we use kmalloc() for skb->head allocation.
+ */
+ if (len <= SKB_WITH_OVERHEAD(1024) ||
+ len > SKB_WITH_OVERHEAD(PAGE_SIZE) ||
(gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
if (!skb)
@@ -462,13 +466,17 @@ EXPORT_SYMBOL(__netdev_alloc_skb);
struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len,
gfp_t gfp_mask)
{
- struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
+ struct napi_alloc_cache *nc;
struct sk_buff *skb;
void *data;
len += NET_SKB_PAD + NET_IP_ALIGN;
- if ((len > SKB_WITH_OVERHEAD(PAGE_SIZE)) ||
+ /* If requested length is either too small or too big,
+ * we use kmalloc() for skb->head allocation.
+ */
+ if (len <= SKB_WITH_OVERHEAD(1024) ||
+ len > SKB_WITH_OVERHEAD(PAGE_SIZE) ||
(gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
if (!skb)
@@ -476,6 +484,7 @@ struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len,
goto skb_success;
}
+ nc = this_cpu_ptr(&napi_alloc_cache);
len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
len = SKB_DATA_ALIGN(len);
@@ -1856,6 +1865,12 @@ int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len)
skb->csum = csum_block_sub(skb->csum,
skb_checksum(skb, len, delta, 0),
len);
+ } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
+ int hdlen = (len > skb_headlen(skb)) ? skb_headlen(skb) : len;
+ int offset = skb_checksum_start_offset(skb) + skb->csum_offset;
+
+ if (offset + sizeof(__sum16) > hdlen)
+ return -EINVAL;
}
return __pskb_trim(skb, len);
}
diff --git a/net/core/sock_reuseport.c b/net/core/sock_reuseport.c
index 9c85ef2b7e1d..375a3bbe6485 100644
--- a/net/core/sock_reuseport.c
+++ b/net/core/sock_reuseport.c
@@ -299,7 +299,7 @@ select_by_hash:
i = j = reciprocal_scale(hash, socks);
while (reuse->socks[i]->sk_state == TCP_ESTABLISHED) {
i++;
- if (i >= reuse->num_socks)
+ if (i >= socks)
i = 0;
if (i == j)
goto out;
diff --git a/net/dcb/dcbnl.c b/net/dcb/dcbnl.c
index 5ee6b94131b2..33684f1818a8 100644
--- a/net/dcb/dcbnl.c
+++ b/net/dcb/dcbnl.c
@@ -1756,6 +1756,8 @@ static int dcb_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
fn = &reply_funcs[dcb->cmd];
if (!fn->cb)
return -EOPNOTSUPP;
+ if (fn->type == RTM_SETDCB && !netlink_capable(skb, CAP_NET_ADMIN))
+ return -EPERM;
if (!tb[DCB_ATTR_IFNAME])
return -EINVAL;
diff --git a/net/ipv4/esp4.c b/net/ipv4/esp4.c
index 114f9def1ec5..0792a9e2a555 100644
--- a/net/ipv4/esp4.c
+++ b/net/ipv4/esp4.c
@@ -270,7 +270,6 @@ static int esp_output_udp_encap(struct xfrm_state *x, struct sk_buff *skb, struc
int esp_output_head(struct xfrm_state *x, struct sk_buff *skb, struct esp_info *esp)
{
u8 *tail;
- u8 *vaddr;
int nfrags;
int esph_offset;
struct page *page;
@@ -312,14 +311,10 @@ int esp_output_head(struct xfrm_state *x, struct sk_buff *skb, struct esp_info *
page = pfrag->page;
get_page(page);
- vaddr = kmap_atomic(page);
-
- tail = vaddr + pfrag->offset;
+ tail = page_address(page) + pfrag->offset;
esp_output_fill_trailer(tail, esp->tfclen, esp->plen, esp->proto);
- kunmap_atomic(vaddr);
-
nfrags = skb_shinfo(skb)->nr_frags;
__skb_fill_page_desc(skb, nfrags, page, pfrag->offset,
diff --git a/net/ipv4/fib_frontend.c b/net/ipv4/fib_frontend.c
index 7f4ec36e5f70..b96aa88087be 100644
--- a/net/ipv4/fib_frontend.c
+++ b/net/ipv4/fib_frontend.c
@@ -302,7 +302,7 @@ __be32 fib_compute_spec_dst(struct sk_buff *skb)
.flowi4_iif = LOOPBACK_IFINDEX,
.flowi4_oif = l3mdev_master_ifindex_rcu(dev),
.daddr = ip_hdr(skb)->saddr,
- .flowi4_tos = RT_TOS(ip_hdr(skb)->tos),
+ .flowi4_tos = ip_hdr(skb)->tos & IPTOS_RT_MASK,
.flowi4_scope = scope,
.flowi4_mark = vmark ? skb->mark : 0,
};
diff --git a/net/ipv4/gre_demux.c b/net/ipv4/gre_demux.c
index ad9ea82daeb3..9376b30cf626 100644
--- a/net/ipv4/gre_demux.c
+++ b/net/ipv4/gre_demux.c
@@ -133,7 +133,7 @@ int gre_parse_header(struct sk_buff *skb, struct tnl_ptk_info *tpi,
* to 0 and sets the configured key in the
* inner erspan header field
*/
- if (greh->protocol == htons(ETH_P_ERSPAN) ||
+ if ((greh->protocol == htons(ETH_P_ERSPAN) && hdr_len != 4) ||
greh->protocol == htons(ETH_P_ERSPAN2)) {
struct erspan_base_hdr *ershdr;
diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c
index f0faf1193dd8..e411c42d8428 100644
--- a/net/ipv4/ip_output.c
+++ b/net/ipv4/ip_output.c
@@ -312,7 +312,7 @@ static int ip_finish_output(struct net *net, struct sock *sk, struct sk_buff *sk
if (skb_is_gso(skb))
return ip_finish_output_gso(net, sk, skb, mtu);
- if (skb->len > mtu || (IPCB(skb)->flags & IPSKB_FRAG_PMTU))
+ if (skb->len > mtu || IPCB(skb)->frag_max_size)
return ip_fragment(net, sk, skb, mtu, ip_finish_output2);
return ip_finish_output2(net, sk, skb);
diff --git a/net/ipv4/ip_tunnel.c b/net/ipv4/ip_tunnel.c
index 375d0e516d85..1cad731039c3 100644
--- a/net/ipv4/ip_tunnel.c
+++ b/net/ipv4/ip_tunnel.c
@@ -736,7 +736,11 @@ void ip_tunnel_xmit(struct sk_buff *skb, struct net_device *dev,
goto tx_error;
}
- if (tnl_update_pmtu(dev, skb, rt, tnl_params->frag_off, inner_iph)) {
+ df = tnl_params->frag_off;
+ if (skb->protocol == htons(ETH_P_IP) && !tunnel->ignore_df)
+ df |= (inner_iph->frag_off & htons(IP_DF));
+
+ if (tnl_update_pmtu(dev, skb, rt, df, inner_iph)) {
ip_rt_put(rt);
goto tx_error;
}
@@ -764,10 +768,6 @@ void ip_tunnel_xmit(struct sk_buff *skb, struct net_device *dev,
ttl = ip4_dst_hoplimit(&rt->dst);
}
- df = tnl_params->frag_off;
- if (skb->protocol == htons(ETH_P_IP) && !tunnel->ignore_df)
- df |= (inner_iph->frag_off&htons(IP_DF));
-
max_headroom = LL_RESERVED_SPACE(rt->dst.dev) + sizeof(struct iphdr)
+ rt->dst.header_len + ip_encap_hlen(&tunnel->encap);
if (max_headroom > dev->needed_headroom)
diff --git a/net/ipv4/netfilter/arp_tables.c b/net/ipv4/netfilter/arp_tables.c
index ac0762ddd4e5..c187369f6eeb 100644
--- a/net/ipv4/netfilter/arp_tables.c
+++ b/net/ipv4/netfilter/arp_tables.c
@@ -1404,7 +1404,7 @@ static int compat_get_entries(struct net *net,
xt_compat_lock(NFPROTO_ARP);
t = xt_find_table_lock(net, NFPROTO_ARP, get.name);
if (!IS_ERR(t)) {
- const struct xt_table_info *private = t->private;
+ const struct xt_table_info *private = xt_table_get_private_protected(t);
struct xt_table_info info;
ret = compat_table_info(private, &info);
diff --git a/net/ipv4/netfilter/ip_tables.c b/net/ipv4/netfilter/ip_tables.c
index 520aedb73cd3..3c3caa6ff0d1 100644
--- a/net/ipv4/netfilter/ip_tables.c
+++ b/net/ipv4/netfilter/ip_tables.c
@@ -1618,7 +1618,7 @@ compat_get_entries(struct net *net, struct compat_ipt_get_entries __user *uptr,
xt_compat_lock(AF_INET);
t = xt_find_table_lock(net, AF_INET, get.name);
if (!IS_ERR(t)) {
- const struct xt_table_info *private = t->private;
+ const struct xt_table_info *private = xt_table_get_private_protected(t);
struct xt_table_info info;
ret = compat_table_info(private, &info);
if (!ret && get.size == info.size)
diff --git a/net/ipv4/netfilter/ipt_rpfilter.c b/net/ipv4/netfilter/ipt_rpfilter.c
index 74b19a5c572e..088320ce77a1 100644
--- a/net/ipv4/netfilter/ipt_rpfilter.c
+++ b/net/ipv4/netfilter/ipt_rpfilter.c
@@ -94,7 +94,7 @@ static bool rpfilter_mt(const struct sk_buff *skb, struct xt_action_param *par)
flow.daddr = iph->saddr;
flow.saddr = rpfilter_get_saddr(iph->daddr);
flow.flowi4_mark = info->flags & XT_RPFILTER_VALID_MARK ? skb->mark : 0;
- flow.flowi4_tos = RT_TOS(iph->tos);
+ flow.flowi4_tos = iph->tos & IPTOS_RT_MASK;
flow.flowi4_scope = RT_SCOPE_UNIVERSE;
flow.flowi4_oif = l3mdev_master_ifindex_rcu(xt_in(par));
diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index c59a2deeee57..af085a16b203 100644
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -2449,7 +2449,8 @@ int udp_v4_early_demux(struct sk_buff *skb)
*/
if (!inet_sk(sk)->inet_daddr && in_dev)
return ip_mc_validate_source(skb, iph->daddr,
- iph->saddr, iph->tos,
+ iph->saddr,
+ iph->tos & IPTOS_RT_MASK,
skb->dev, in_dev, &itag);
}
return 0;
diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c
index f6975a633074..cd46eaeb4e61 100644
--- a/net/ipv6/addrconf.c
+++ b/net/ipv6/addrconf.c
@@ -2426,8 +2426,9 @@ static void addrconf_add_mroute(struct net_device *dev)
.fc_ifindex = dev->ifindex,
.fc_dst_len = 8,
.fc_flags = RTF_UP,
- .fc_type = RTN_UNICAST,
+ .fc_type = RTN_MULTICAST,
.fc_nlinfo.nl_net = dev_net(dev),
+ .fc_protocol = RTPROT_KERNEL,
};
ipv6_addr_set(&cfg.fc_dst, htonl(0xFF000000), 0, 0, 0);
diff --git a/net/ipv6/esp6.c b/net/ipv6/esp6.c
index a7d996148eed..25317d5ccf2c 100644
--- a/net/ipv6/esp6.c
+++ b/net/ipv6/esp6.c
@@ -237,7 +237,6 @@ static void esp_output_fill_trailer(u8 *tail, int tfclen, int plen, __u8 proto)
int esp6_output_head(struct xfrm_state *x, struct sk_buff *skb, struct esp_info *esp)
{
u8 *tail;
- u8 *vaddr;
int nfrags;
struct page *page;
struct sk_buff *trailer;
@@ -270,14 +269,10 @@ int esp6_output_head(struct xfrm_state *x, struct sk_buff *skb, struct esp_info
page = pfrag->page;
get_page(page);
- vaddr = kmap_atomic(page);
-
- tail = vaddr + pfrag->offset;
+ tail = page_address(page) + pfrag->offset;
esp_output_fill_trailer(tail, esp->tfclen, esp->plen, esp->proto);
- kunmap_atomic(vaddr);
-
nfrags = skb_shinfo(skb)->nr_frags;
__skb_fill_page_desc(skb, nfrags, page, pfrag->offset,
diff --git a/net/ipv6/ip6_fib.c b/net/ipv6/ip6_fib.c
index 8b5459b34bc4..e0e464b72c1f 100644
--- a/net/ipv6/ip6_fib.c
+++ b/net/ipv6/ip6_fib.c
@@ -906,6 +906,8 @@ static void fib6_purge_rt(struct fib6_info *rt, struct fib6_node *fn,
{
struct fib6_table *table = rt->fib6_table;
+ /* Flush all cached dst in exception table */
+ rt6_flush_exceptions(rt);
if (rt->rt6i_pcpu)
fib6_drop_pcpu_from(rt, table);
@@ -1756,9 +1758,6 @@ static void fib6_del_route(struct fib6_table *table, struct fib6_node *fn,
net->ipv6.rt6_stats->fib_rt_entries--;
net->ipv6.rt6_stats->fib_discarded_routes++;
- /* Flush all cached dst in exception table */
- rt6_flush_exceptions(rt);
-
/* Reset round-robin state, if necessary */
if (rcu_access_pointer(fn->rr_ptr) == rt)
fn->rr_ptr = NULL;
diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c
index 22665e3638ac..e1bb7db88483 100644
--- a/net/ipv6/ip6_output.c
+++ b/net/ipv6/ip6_output.c
@@ -128,8 +128,42 @@ static int ip6_finish_output2(struct net *net, struct sock *sk, struct sk_buff *
return -EINVAL;
}
+static int
+ip6_finish_output_gso_slowpath_drop(struct net *net, struct sock *sk,
+ struct sk_buff *skb, unsigned int mtu)
+{
+ struct sk_buff *segs, *nskb;
+ netdev_features_t features;
+ int ret = 0;
+
+ /* Please see corresponding comment in ip_finish_output_gso
+ * describing the cases where GSO segment length exceeds the
+ * egress MTU.
+ */
+ features = netif_skb_features(skb);
+ segs = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK);
+ if (IS_ERR_OR_NULL(segs)) {
+ kfree_skb(skb);
+ return -ENOMEM;
+ }
+
+ consume_skb(skb);
+
+ skb_list_walk_safe(segs, segs, nskb) {
+ int err;
+
+ skb_mark_not_on_list(segs);
+ err = ip6_fragment(net, sk, segs, ip6_finish_output2);
+ if (err && ret == 0)
+ ret = err;
+ }
+
+ return ret;
+}
+
static int ip6_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb)
{
+ unsigned int mtu;
int ret;
ret = BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb);
@@ -146,7 +180,11 @@ static int ip6_finish_output(struct net *net, struct sock *sk, struct sk_buff *s
}
#endif
- if ((skb->len > ip6_skb_dst_mtu(skb) && !skb_is_gso(skb)) ||
+ mtu = ip6_skb_dst_mtu(skb);
+ if (skb_is_gso(skb) && !skb_gso_validate_network_len(skb, mtu))
+ return ip6_finish_output_gso_slowpath_drop(net, sk, skb, mtu);
+
+ if ((skb->len > mtu && !skb_is_gso(skb)) ||
dst_allfrag(skb_dst(skb)) ||
(IP6CB(skb)->frag_max_size && skb->len > IP6CB(skb)->frag_max_size))
return ip6_fragment(net, sk, skb, ip6_finish_output2);
diff --git a/net/ipv6/netfilter/ip6_tables.c b/net/ipv6/netfilter/ip6_tables.c
index b66451e4bf8e..84429cd5f860 100644
--- a/net/ipv6/netfilter/ip6_tables.c
+++ b/net/ipv6/netfilter/ip6_tables.c
@@ -1627,7 +1627,7 @@ compat_get_entries(struct net *net, struct compat_ip6t_get_entries __user *uptr,
xt_compat_lock(AF_INET6);
t = xt_find_table_lock(net, AF_INET6, get.name);
if (!IS_ERR(t)) {
- const struct xt_table_info *private = t->private;
+ const struct xt_table_info *private = xt_table_get_private_protected(t);
struct xt_table_info info;
ret = compat_table_info(private, &info);
if (!ret && get.size == info.size)
diff --git a/net/ipv6/sit.c b/net/ipv6/sit.c
index 98c108baf35e..bcf29201f87b 100644
--- a/net/ipv6/sit.c
+++ b/net/ipv6/sit.c
@@ -1596,8 +1596,11 @@ static int ipip6_newlink(struct net *src_net, struct net_device *dev,
}
#ifdef CONFIG_IPV6_SIT_6RD
- if (ipip6_netlink_6rd_parms(data, &ip6rd))
+ if (ipip6_netlink_6rd_parms(data, &ip6rd)) {
err = ipip6_tunnel_update_6rd(nt, &ip6rd);
+ if (err < 0)
+ unregister_netdevice_queue(dev, NULL);
+ }
#endif
return err;
diff --git a/net/ncsi/ncsi-rsp.c b/net/ncsi/ncsi-rsp.c
index 930c1d3796f0..a43c9a44f870 100644
--- a/net/ncsi/ncsi-rsp.c
+++ b/net/ncsi/ncsi-rsp.c
@@ -949,7 +949,7 @@ int ncsi_rcv_rsp(struct sk_buff *skb, struct net_device *dev,
int payload, i, ret;
/* Find the NCSI device */
- nd = ncsi_find_dev(dev);
+ nd = ncsi_find_dev(orig_dev);
ndp = nd ? TO_NCSI_DEV_PRIV(nd) : NULL;
if (!ndp)
return -ENODEV;
diff --git a/net/netfilter/ipset/ip_set_hash_gen.h b/net/netfilter/ipset/ip_set_hash_gen.h
index ddfe06d7530b..154b40f2f2a8 100644
--- a/net/netfilter/ipset/ip_set_hash_gen.h
+++ b/net/netfilter/ipset/ip_set_hash_gen.h
@@ -115,20 +115,6 @@ htable_size(u8 hbits)
return hsize * sizeof(struct hbucket *) + sizeof(struct htable);
}
-/* Compute htable_bits from the user input parameter hashsize */
-static u8
-htable_bits(u32 hashsize)
-{
- /* Assume that hashsize == 2^htable_bits */
- u8 bits = fls(hashsize - 1);
-
- if (jhash_size(bits) != hashsize)
- /* Round up to the first 2^n value */
- bits = fls(hashsize);
-
- return bits;
-}
-
#ifdef IP_SET_HASH_WITH_NETS
#if IPSET_NET_COUNT > 1
#define __CIDR(cidr, i) (cidr[i])
@@ -1287,7 +1273,11 @@ IPSET_TOKEN(HTYPE, _create)(struct net *net, struct ip_set *set,
if (!h)
return -ENOMEM;
- hbits = htable_bits(hashsize);
+ /* Compute htable_bits from the user input parameter hashsize.
+ * Assume that hashsize == 2^htable_bits,
+ * otherwise round up to the first 2^n value.
+ */
+ hbits = fls(hashsize - 1);
hsize = htable_size(hbits);
if (hsize == 0) {
kfree(h);
diff --git a/net/netfilter/nf_conntrack_standalone.c b/net/netfilter/nf_conntrack_standalone.c
index 4c94f3ba2ae4..dcd8e7922951 100644
--- a/net/netfilter/nf_conntrack_standalone.c
+++ b/net/netfilter/nf_conntrack_standalone.c
@@ -500,6 +500,9 @@ nf_conntrack_hash_sysctl(struct ctl_table *table, int write,
{
int ret;
+ /* module_param hashsize could have changed value */
+ nf_conntrack_htable_size_user = nf_conntrack_htable_size;
+
ret = proc_dointvec(table, write, buffer, lenp, ppos);
if (ret < 0 || !write)
return ret;
diff --git a/net/netfilter/nf_nat_core.c b/net/netfilter/nf_nat_core.c
index 2268b10a9dcf..c31df6a76504 100644
--- a/net/netfilter/nf_nat_core.c
+++ b/net/netfilter/nf_nat_core.c
@@ -1068,6 +1068,7 @@ static int __init nf_nat_init(void)
ret = register_pernet_subsys(&nat_net_ops);
if (ret < 0) {
nf_ct_extend_unregister(&nat_extend);
+ kvfree(nf_nat_bysource);
return ret;
}
diff --git a/net/netfilter/xt_RATEEST.c b/net/netfilter/xt_RATEEST.c
index 9e05c86ba5c4..932c0ae99bdc 100644
--- a/net/netfilter/xt_RATEEST.c
+++ b/net/netfilter/xt_RATEEST.c
@@ -118,6 +118,9 @@ static int xt_rateest_tg_checkentry(const struct xt_tgchk_param *par)
} cfg;
int ret;
+ if (strnlen(info->name, sizeof(est->name)) >= sizeof(est->name))
+ return -ENAMETOOLONG;
+
net_get_random_once(&jhash_rnd, sizeof(jhash_rnd));
mutex_lock(&xn->hash_lock);
diff --git a/net/rxrpc/input.c b/net/rxrpc/input.c
index 58bd558a277a..40711f410828 100644
--- a/net/rxrpc/input.c
+++ b/net/rxrpc/input.c
@@ -446,7 +446,7 @@ static void rxrpc_input_data(struct rxrpc_call *call, struct sk_buff *skb,
if (state >= RXRPC_CALL_COMPLETE)
return;
- if (call->state == RXRPC_CALL_SERVER_RECV_REQUEST) {
+ if (state == RXRPC_CALL_SERVER_RECV_REQUEST) {
unsigned long timo = READ_ONCE(call->next_req_timo);
unsigned long now, expect_req_by;
diff --git a/net/rxrpc/key.c b/net/rxrpc/key.c
index 2fe2add62a8e..9be6b35fd9b2 100644
--- a/net/rxrpc/key.c
+++ b/net/rxrpc/key.c
@@ -1112,7 +1112,7 @@ static long rxrpc_read(const struct key *key,
default: /* we have a ticket we can't encode */
pr_err("Unsupported key token type (%u)\n",
token->security_index);
- continue;
+ return -ENOPKG;
}
_debug("token[%u]: toksize=%u", ntoks, toksize);
@@ -1227,7 +1227,9 @@ static long rxrpc_read(const struct key *key,
break;
default:
- break;
+ pr_err("Unsupported key token type (%u)\n",
+ token->security_index);
+ return -ENOPKG;
}
ASSERTCMP((unsigned long)xdr - (unsigned long)oldxdr, ==,
diff --git a/net/sched/cls_tcindex.c b/net/sched/cls_tcindex.c
index 0d7a0aac8dbb..e41bc5ecaa09 100644
--- a/net/sched/cls_tcindex.c
+++ b/net/sched/cls_tcindex.c
@@ -339,9 +339,13 @@ tcindex_set_parms(struct net *net, struct tcf_proto *tp, unsigned long base,
if (tb[TCA_TCINDEX_MASK])
cp->mask = nla_get_u16(tb[TCA_TCINDEX_MASK]);
- if (tb[TCA_TCINDEX_SHIFT])
+ if (tb[TCA_TCINDEX_SHIFT]) {
cp->shift = nla_get_u32(tb[TCA_TCINDEX_SHIFT]);
-
+ if (cp->shift > 16) {
+ err = -EINVAL;
+ goto errout;
+ }
+ }
if (!cp->hash) {
/* Hash not specified, use perfect hash if the upper limit
* of the hashing index is below the threshold.
diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c
index 8af781054a52..1f54eece3b77 100644
--- a/net/sched/sch_api.c
+++ b/net/sched/sch_api.c
@@ -398,7 +398,8 @@ struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r,
{
struct qdisc_rate_table *rtab;
- if (tab == NULL || r->rate == 0 || r->cell_log == 0 ||
+ if (tab == NULL || r->rate == 0 ||
+ r->cell_log == 0 || r->cell_log >= 32 ||
nla_len(tab) != TC_RTAB_SIZE) {
NL_SET_ERR_MSG(extack, "Invalid rate table parameters for searching");
return NULL;
diff --git a/net/sched/sch_choke.c b/net/sched/sch_choke.c
index 63bfceeb8e3c..d058397d284b 100644
--- a/net/sched/sch_choke.c
+++ b/net/sched/sch_choke.c
@@ -371,7 +371,7 @@ static int choke_change(struct Qdisc *sch, struct nlattr *opt,
ctl = nla_data(tb[TCA_CHOKE_PARMS]);
- if (!red_check_params(ctl->qth_min, ctl->qth_max, ctl->Wlog))
+ if (!red_check_params(ctl->qth_min, ctl->qth_max, ctl->Wlog, ctl->Scell_log))
return -EINVAL;
if (ctl->limit > CHOKE_MAX_QUEUE)
diff --git a/net/sched/sch_gred.c b/net/sched/sch_gred.c
index 4a042abf844c..6f94bca75520 100644
--- a/net/sched/sch_gred.c
+++ b/net/sched/sch_gred.c
@@ -357,7 +357,7 @@ static inline int gred_change_vq(struct Qdisc *sch, int dp,
struct gred_sched *table = qdisc_priv(sch);
struct gred_sched_data *q = table->tab[dp];
- if (!red_check_params(ctl->qth_min, ctl->qth_max, ctl->Wlog))
+ if (!red_check_params(ctl->qth_min, ctl->qth_max, ctl->Wlog, ctl->Scell_log))
return -EINVAL;
if (!q) {
diff --git a/net/sched/sch_red.c b/net/sched/sch_red.c
index 56c181c3feeb..a3dc2118539b 100644
--- a/net/sched/sch_red.c
+++ b/net/sched/sch_red.c
@@ -214,7 +214,7 @@ static int red_change(struct Qdisc *sch, struct nlattr *opt,
max_P = tb[TCA_RED_MAX_P] ? nla_get_u32(tb[TCA_RED_MAX_P]) : 0;
ctl = nla_data(tb[TCA_RED_PARMS]);
- if (!red_check_params(ctl->qth_min, ctl->qth_max, ctl->Wlog))
+ if (!red_check_params(ctl->qth_min, ctl->qth_max, ctl->Wlog, ctl->Scell_log))
return -EINVAL;
if (ctl->limit > 0) {
diff --git a/net/sched/sch_sfq.c b/net/sched/sch_sfq.c
index b89cf0971d3d..74f697c4a4d3 100644
--- a/net/sched/sch_sfq.c
+++ b/net/sched/sch_sfq.c
@@ -651,7 +651,7 @@ static int sfq_change(struct Qdisc *sch, struct nlattr *opt)
}
if (ctl_v1 && !red_check_params(ctl_v1->qth_min, ctl_v1->qth_max,
- ctl_v1->Wlog))
+ ctl_v1->Wlog, ctl_v1->Scell_log))
return -EINVAL;
if (ctl_v1 && ctl_v1->qth_min) {
p = kmalloc(sizeof(*p), GFP_KERNEL);
diff --git a/net/sunrpc/addr.c b/net/sunrpc/addr.c
index 8391c2785550..7404f02702a1 100644
--- a/net/sunrpc/addr.c
+++ b/net/sunrpc/addr.c
@@ -184,7 +184,7 @@ static int rpc_parse_scope_id(struct net *net, const char *buf,
scope_id = dev->ifindex;
dev_put(dev);
} else {
- if (kstrtou32(p, 10, &scope_id) == 0) {
+ if (kstrtou32(p, 10, &scope_id) != 0) {
kfree(p);
return 0;
}
diff --git a/net/tipc/link.c b/net/tipc/link.c
index f756b721f93e..bd28ac7f2195 100644
--- a/net/tipc/link.c
+++ b/net/tipc/link.c
@@ -914,9 +914,7 @@ void tipc_link_reset(struct tipc_link *l)
int tipc_link_xmit(struct tipc_link *l, struct sk_buff_head *list,
struct sk_buff_head *xmitq)
{
- struct tipc_msg *hdr = buf_msg(skb_peek(list));
unsigned int maxwin = l->window;
- int imp = msg_importance(hdr);
unsigned int mtu = l->mtu;
u16 ack = l->rcv_nxt - 1;
u16 seqno = l->snd_nxt;
@@ -925,13 +923,20 @@ int tipc_link_xmit(struct tipc_link *l, struct sk_buff_head *list,
struct sk_buff_head *backlogq = &l->backlogq;
struct sk_buff *skb, *_skb, **tskb;
int pkt_cnt = skb_queue_len(list);
+ struct tipc_msg *hdr;
int rc = 0;
+ int imp;
+ if (pkt_cnt <= 0)
+ return 0;
+
+ hdr = buf_msg(skb_peek(list));
if (unlikely(msg_size(hdr) > mtu)) {
__skb_queue_purge(list);
return -EMSGSIZE;
}
+ imp = msg_importance(hdr);
/* Allow oversubscription of one data msg per source at congestion */
if (unlikely(l->backlog[imp].len >= l->backlog[imp].limit)) {
if (imp == TIPC_SYSTEM_IMPORTANCE) {
diff --git a/scripts/depmod.sh b/scripts/depmod.sh
index e083bcae343f..3643b4f896ed 100755
--- a/scripts/depmod.sh
+++ b/scripts/depmod.sh
@@ -15,6 +15,8 @@ if ! test -r System.map ; then
exit 0
fi
+# legacy behavior: "depmod" in /sbin, no /sbin in PATH
+PATH="$PATH:/sbin"
if [ -z $(command -v $DEPMOD) ]; then
echo "Warning: 'make modules_install' requires $DEPMOD. Please install it." >&2
echo "This is probably in the kmod package." >&2
diff --git a/security/integrity/ima/ima_crypto.c b/security/integrity/ima/ima_crypto.c
index 5596ea8f339a..f0b244989734 100644
--- a/security/integrity/ima/ima_crypto.c
+++ b/security/integrity/ima/ima_crypto.c
@@ -641,7 +641,7 @@ int ima_calc_buffer_hash(const void *buf, loff_t len,
return calc_buffer_shash(buf, len, hash);
}
-static void __init ima_pcrread(int idx, u8 *pcr)
+static void ima_pcrread(int idx, u8 *pcr)
{
if (!ima_tpm_chip)
return;
diff --git a/security/lsm_audit.c b/security/lsm_audit.c
index 33028c098ef3..d6fea68d22ad 100644
--- a/security/lsm_audit.c
+++ b/security/lsm_audit.c
@@ -277,7 +277,9 @@ static void dump_common_audit_data(struct audit_buffer *ab,
struct inode *inode;
audit_log_format(ab, " name=");
+ spin_lock(&a->u.dentry->d_lock);
audit_log_untrustedstring(ab, a->u.dentry->d_name.name);
+ spin_unlock(&a->u.dentry->d_lock);
inode = d_backing_inode(a->u.dentry);
if (inode) {
@@ -295,8 +297,9 @@ static void dump_common_audit_data(struct audit_buffer *ab,
dentry = d_find_alias(inode);
if (dentry) {
audit_log_format(ab, " name=");
- audit_log_untrustedstring(ab,
- dentry->d_name.name);
+ spin_lock(&dentry->d_lock);
+ audit_log_untrustedstring(ab, dentry->d_name.name);
+ spin_unlock(&dentry->d_lock);
dput(dentry);
}
audit_log_format(ab, " dev=");
diff --git a/sound/core/pcm_native.c b/sound/core/pcm_native.c
index ab173b4a8076..9895c59c1632 100644
--- a/sound/core/pcm_native.c
+++ b/sound/core/pcm_native.c
@@ -755,8 +755,13 @@ static int snd_pcm_hw_params(struct snd_pcm_substream *substream,
runtime->boundary *= 2;
/* clear the buffer for avoiding possible kernel info leaks */
- if (runtime->dma_area && !substream->ops->copy_user)
- memset(runtime->dma_area, 0, runtime->dma_bytes);
+ if (runtime->dma_area && !substream->ops->copy_user) {
+ size_t size = runtime->dma_bytes;
+
+ if (runtime->info & SNDRV_PCM_INFO_MMAP)
+ size = PAGE_ALIGN(size);
+ memset(runtime->dma_area, 0, size);
+ }
snd_pcm_timer_resolution_change(substream);
snd_pcm_set_state(substream, SNDRV_PCM_STATE_SETUP);
diff --git a/sound/core/rawmidi.c b/sound/core/rawmidi.c
index 9b26973fe697..f4f855d7a791 100644
--- a/sound/core/rawmidi.c
+++ b/sound/core/rawmidi.c
@@ -87,11 +87,21 @@ static inline unsigned short snd_rawmidi_file_flags(struct file *file)
}
}
-static inline int snd_rawmidi_ready(struct snd_rawmidi_substream *substream)
+static inline bool __snd_rawmidi_ready(struct snd_rawmidi_runtime *runtime)
+{
+ return runtime->avail >= runtime->avail_min;
+}
+
+static bool snd_rawmidi_ready(struct snd_rawmidi_substream *substream)
{
struct snd_rawmidi_runtime *runtime = substream->runtime;
+ unsigned long flags;
+ bool ready;
- return runtime->avail >= runtime->avail_min;
+ spin_lock_irqsave(&runtime->lock, flags);
+ ready = __snd_rawmidi_ready(runtime);
+ spin_unlock_irqrestore(&runtime->lock, flags);
+ return ready;
}
static inline int snd_rawmidi_ready_append(struct snd_rawmidi_substream *substream,
@@ -960,7 +970,7 @@ int snd_rawmidi_receive(struct snd_rawmidi_substream *substream,
if (result > 0) {
if (runtime->event)
schedule_work(&runtime->event_work);
- else if (snd_rawmidi_ready(substream))
+ else if (__snd_rawmidi_ready(runtime))
wake_up(&runtime->sleep);
}
spin_unlock_irqrestore(&runtime->lock, flags);
@@ -1039,7 +1049,7 @@ static ssize_t snd_rawmidi_read(struct file *file, char __user *buf, size_t coun
result = 0;
while (count > 0) {
spin_lock_irq(&runtime->lock);
- while (!snd_rawmidi_ready(substream)) {
+ while (!__snd_rawmidi_ready(runtime)) {
wait_queue_entry_t wait;
if ((file->f_flags & O_NONBLOCK) != 0 || result > 0) {
@@ -1056,9 +1066,11 @@ static ssize_t snd_rawmidi_read(struct file *file, char __user *buf, size_t coun
return -ENODEV;
if (signal_pending(current))
return result > 0 ? result : -ERESTARTSYS;
- if (!runtime->avail)
- return result > 0 ? result : -EIO;
spin_lock_irq(&runtime->lock);
+ if (!runtime->avail) {
+ spin_unlock_irq(&runtime->lock);
+ return result > 0 ? result : -EIO;
+ }
}
spin_unlock_irq(&runtime->lock);
count1 = snd_rawmidi_kernel_read1(substream,
@@ -1196,7 +1208,7 @@ int __snd_rawmidi_transmit_ack(struct snd_rawmidi_substream *substream, int coun
runtime->avail += count;
substream->bytes += count;
if (count > 0) {
- if (runtime->drain || snd_rawmidi_ready(substream))
+ if (runtime->drain || __snd_rawmidi_ready(runtime))
wake_up(&runtime->sleep);
}
return count;
@@ -1363,9 +1375,11 @@ static ssize_t snd_rawmidi_write(struct file *file, const char __user *buf,
return -ENODEV;
if (signal_pending(current))
return result > 0 ? result : -ERESTARTSYS;
- if (!runtime->avail && !timeout)
- return result > 0 ? result : -EIO;
spin_lock_irq(&runtime->lock);
+ if (!runtime->avail && !timeout) {
+ spin_unlock_irq(&runtime->lock);
+ return result > 0 ? result : -EIO;
+ }
}
spin_unlock_irq(&runtime->lock);
count1 = snd_rawmidi_kernel_write1(substream, buf, NULL, count);
@@ -1445,6 +1459,7 @@ static void snd_rawmidi_proc_info_read(struct snd_info_entry *entry,
struct snd_rawmidi *rmidi;
struct snd_rawmidi_substream *substream;
struct snd_rawmidi_runtime *runtime;
+ unsigned long buffer_size, avail, xruns;
rmidi = entry->private_data;
snd_iprintf(buffer, "%s\n\n", rmidi->name);
@@ -1463,13 +1478,16 @@ static void snd_rawmidi_proc_info_read(struct snd_info_entry *entry,
" Owner PID : %d\n",
pid_vnr(substream->pid));
runtime = substream->runtime;
+ spin_lock_irq(&runtime->lock);
+ buffer_size = runtime->buffer_size;
+ avail = runtime->avail;
+ spin_unlock_irq(&runtime->lock);
snd_iprintf(buffer,
" Mode : %s\n"
" Buffer size : %lu\n"
" Avail : %lu\n",
runtime->oss ? "OSS compatible" : "native",
- (unsigned long) runtime->buffer_size,
- (unsigned long) runtime->avail);
+ buffer_size, avail);
}
}
}
@@ -1487,13 +1505,16 @@ static void snd_rawmidi_proc_info_read(struct snd_info_entry *entry,
" Owner PID : %d\n",
pid_vnr(substream->pid));
runtime = substream->runtime;
+ spin_lock_irq(&runtime->lock);
+ buffer_size = runtime->buffer_size;
+ avail = runtime->avail;
+ xruns = runtime->xruns;
+ spin_unlock_irq(&runtime->lock);
snd_iprintf(buffer,
" Buffer size : %lu\n"
" Avail : %lu\n"
" Overruns : %lu\n",
- (unsigned long) runtime->buffer_size,
- (unsigned long) runtime->avail,
- (unsigned long) runtime->xruns);
+ buffer_size, avail, xruns);
}
}
}
diff --git a/sound/core/seq/oss/seq_oss_synth.c b/sound/core/seq/oss/seq_oss_synth.c
index c93945917235..247b68790a52 100644
--- a/sound/core/seq/oss/seq_oss_synth.c
+++ b/sound/core/seq/oss/seq_oss_synth.c
@@ -624,7 +624,8 @@ snd_seq_oss_synth_make_info(struct seq_oss_devinfo *dp, int dev, struct synth_in
if (info->is_midi) {
struct midi_info minf;
- snd_seq_oss_midi_make_info(dp, info->midi_mapped, &minf);
+ if (snd_seq_oss_midi_make_info(dp, info->midi_mapped, &minf))
+ return -ENXIO;
inf->synth_type = SYNTH_TYPE_MIDI;
inf->synth_subtype = 0;
inf->nr_voices = 16;
diff --git a/sound/core/seq/seq_queue.h b/sound/core/seq/seq_queue.h
index e006fc8e3a36..6b634cfb42ed 100644
--- a/sound/core/seq/seq_queue.h
+++ b/sound/core/seq/seq_queue.h
@@ -40,10 +40,10 @@ struct snd_seq_queue {
struct snd_seq_timer *timer; /* time keeper for this queue */
int owner; /* client that 'owns' the timer */
- unsigned int locked:1, /* timer is only accesibble by owner if set */
- klocked:1, /* kernel lock (after START) */
- check_again:1,
- check_blocked:1;
+ bool locked; /* timer is only accesibble by owner if set */
+ bool klocked; /* kernel lock (after START) */
+ bool check_again; /* concurrent access happened during check */
+ bool check_blocked; /* queue being checked */
unsigned int flags; /* status flags */
unsigned int info_flags; /* info for sync */
diff --git a/sound/firewire/fireface/ff-transaction.c b/sound/firewire/fireface/ff-transaction.c
index 332b29f8ed75..47280ae50682 100644
--- a/sound/firewire/fireface/ff-transaction.c
+++ b/sound/firewire/fireface/ff-transaction.c
@@ -99,7 +99,7 @@ static void transmit_midi_msg(struct snd_ff *ff, unsigned int port)
/* Set interval to next transaction. */
ff->next_ktime[port] = ktime_add_ns(ktime_get(),
- len * 8 * NSEC_PER_SEC / 31250);
+ len * 8 * (NSEC_PER_SEC / 31250));
ff->rx_bytes[port] = len;
/*
diff --git a/sound/firewire/tascam/tascam-transaction.c b/sound/firewire/tascam/tascam-transaction.c
index 2ad692dd4b13..92f7e74d5847 100644
--- a/sound/firewire/tascam/tascam-transaction.c
+++ b/sound/firewire/tascam/tascam-transaction.c
@@ -210,7 +210,7 @@ static void midi_port_work(struct work_struct *work)
/* Set interval to next transaction. */
port->next_ktime = ktime_add_ns(ktime_get(),
- port->consume_bytes * 8 * NSEC_PER_SEC / 31250);
+ port->consume_bytes * 8 * (NSEC_PER_SEC / 31250));
/* Start this transaction. */
port->idling = false;
diff --git a/sound/pci/hda/hda_intel.c b/sound/pci/hda/hda_intel.c
index d43245937db7..2cd8bfd5293b 100644
--- a/sound/pci/hda/hda_intel.c
+++ b/sound/pci/hda/hda_intel.c
@@ -2328,8 +2328,6 @@ static struct snd_pci_quirk power_save_blacklist[] = {
SND_PCI_QUIRK(0x1849, 0x7662, "Asrock H81M-HDS", 0),
/* https://bugzilla.redhat.com/show_bug.cgi?id=1525104 */
SND_PCI_QUIRK(0x1043, 0x8733, "Asus Prime X370-Pro", 0),
- /* https://bugzilla.redhat.com/show_bug.cgi?id=1581607 */
- SND_PCI_QUIRK(0x1558, 0x3501, "Clevo W35xSS_370SS", 0),
/* https://bugzilla.redhat.com/show_bug.cgi?id=1525104 */
SND_PCI_QUIRK(0x1558, 0x6504, "Clevo W65_67SB", 0),
/* https://bugzilla.redhat.com/show_bug.cgi?id=1525104 */
diff --git a/sound/pci/hda/patch_conexant.c b/sound/pci/hda/patch_conexant.c
index 78bb96263bc2..7d471ecc1ca0 100644
--- a/sound/pci/hda/patch_conexant.c
+++ b/sound/pci/hda/patch_conexant.c
@@ -1088,6 +1088,7 @@ static int patch_conexant_auto(struct hda_codec *codec)
static const struct hda_device_id snd_hda_id_conexant[] = {
HDA_CODEC_ENTRY(0x14f11f86, "CX8070", patch_conexant_auto),
HDA_CODEC_ENTRY(0x14f12008, "CX8200", patch_conexant_auto),
+ HDA_CODEC_ENTRY(0x14f120d0, "CX11970", patch_conexant_auto),
HDA_CODEC_ENTRY(0x14f15045, "CX20549 (Venice)", patch_conexant_auto),
HDA_CODEC_ENTRY(0x14f15047, "CX20551 (Waikiki)", patch_conexant_auto),
HDA_CODEC_ENTRY(0x14f15051, "CX20561 (Hermosa)", patch_conexant_auto),
diff --git a/sound/pci/hda/patch_realtek.c b/sound/pci/hda/patch_realtek.c
index 37b2bcdb3d65..9f0b05bcbd86 100644
--- a/sound/pci/hda/patch_realtek.c
+++ b/sound/pci/hda/patch_realtek.c
@@ -5774,6 +5774,7 @@ enum {
ALC221_FIXUP_HP_FRONT_MIC,
ALC292_FIXUP_TPT460,
ALC298_FIXUP_SPK_VOLUME,
+ ALC298_FIXUP_LENOVO_SPK_VOLUME,
ALC256_FIXUP_DELL_INSPIRON_7559_SUBWOOFER,
ALC269_FIXUP_ATIV_BOOK_8,
ALC221_FIXUP_HP_MIC_NO_PRESENCE,
@@ -6545,6 +6546,10 @@ static const struct hda_fixup alc269_fixups[] = {
.chained = true,
.chain_id = ALC298_FIXUP_DELL_AIO_MIC_NO_PRESENCE,
},
+ [ALC298_FIXUP_LENOVO_SPK_VOLUME] = {
+ .type = HDA_FIXUP_FUNC,
+ .v.func = alc298_fixup_speaker_volume,
+ },
[ALC295_FIXUP_DISABLE_DAC3] = {
.type = HDA_FIXUP_FUNC,
.v.func = alc295_fixup_disable_dac3,
@@ -7220,6 +7225,7 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = {
SND_PCI_QUIRK(0x17aa, 0x3151, "ThinkCentre Station", ALC283_FIXUP_HEADSET_MIC),
SND_PCI_QUIRK(0x17aa, 0x3176, "ThinkCentre Station", ALC283_FIXUP_HEADSET_MIC),
SND_PCI_QUIRK(0x17aa, 0x3178, "ThinkCentre Station", ALC283_FIXUP_HEADSET_MIC),
+ SND_PCI_QUIRK(0x17aa, 0x3818, "Lenovo C940", ALC298_FIXUP_LENOVO_SPK_VOLUME),
SND_PCI_QUIRK(0x17aa, 0x3902, "Lenovo E50-80", ALC269_FIXUP_DMIC_THINKPAD_ACPI),
SND_PCI_QUIRK(0x17aa, 0x3977, "IdeaPad S210", ALC283_FIXUP_INT_MIC),
SND_PCI_QUIRK(0x17aa, 0x3978, "Lenovo B50-70", ALC269_FIXUP_DMIC_THINKPAD_ACPI),
diff --git a/sound/pci/hda/patch_via.c b/sound/pci/hda/patch_via.c
index 6b9617aee0e6..efba9057b2b6 100644
--- a/sound/pci/hda/patch_via.c
+++ b/sound/pci/hda/patch_via.c
@@ -126,6 +126,7 @@ static struct via_spec *via_new_spec(struct hda_codec *codec)
spec->codec_type = VT1708S;
spec->gen.indep_hp = 1;
spec->gen.keep_eapd_on = 1;
+ spec->gen.dac_min_mute = 1;
spec->gen.pcm_playback_hook = via_playback_pcm_hook;
spec->gen.add_stereo_mix_input = HDA_HINT_STEREO_MIX_AUTO;
codec->power_save_node = 1;
@@ -1015,6 +1016,7 @@ static const struct hda_verb vt1802_init_verbs[] = {
enum {
VIA_FIXUP_INTMIC_BOOST,
VIA_FIXUP_ASUS_G75,
+ VIA_FIXUP_POWER_SAVE,
};
static void via_fixup_intmic_boost(struct hda_codec *codec,
@@ -1024,6 +1026,13 @@ static void via_fixup_intmic_boost(struct hda_codec *codec,
override_mic_boost(codec, 0x30, 0, 2, 40);
}
+static void via_fixup_power_save(struct hda_codec *codec,
+ const struct hda_fixup *fix, int action)
+{
+ if (action == HDA_FIXUP_ACT_PRE_PROBE)
+ codec->power_save_node = 0;
+}
+
static const struct hda_fixup via_fixups[] = {
[VIA_FIXUP_INTMIC_BOOST] = {
.type = HDA_FIXUP_FUNC,
@@ -1038,11 +1047,16 @@ static const struct hda_fixup via_fixups[] = {
{ }
}
},
+ [VIA_FIXUP_POWER_SAVE] = {
+ .type = HDA_FIXUP_FUNC,
+ .v.func = via_fixup_power_save,
+ },
};
static const struct snd_pci_quirk vt2002p_fixups[] = {
SND_PCI_QUIRK(0x1043, 0x1487, "Asus G75", VIA_FIXUP_ASUS_G75),
SND_PCI_QUIRK(0x1043, 0x8532, "Asus X202E", VIA_FIXUP_INTMIC_BOOST),
+ SND_PCI_QUIRK(0x1558, 0x3501, "Clevo W35xSS_370SS", VIA_FIXUP_POWER_SAVE),
{}
};
diff --git a/sound/soc/intel/boards/haswell.c b/sound/soc/intel/boards/haswell.c
index a4022983a7ce..67eb4a446c3c 100644
--- a/sound/soc/intel/boards/haswell.c
+++ b/sound/soc/intel/boards/haswell.c
@@ -198,6 +198,7 @@ static struct platform_driver haswell_audio = {
.probe = haswell_audio_probe,
.driver = {
.name = "haswell-audio",
+ .pm = &snd_soc_pm_ops,
},
};
diff --git a/sound/soc/intel/skylake/cnl-sst.c b/sound/soc/intel/skylake/cnl-sst.c
index 245df1067ba8..7b429c0fb4e5 100644
--- a/sound/soc/intel/skylake/cnl-sst.c
+++ b/sound/soc/intel/skylake/cnl-sst.c
@@ -212,6 +212,7 @@ static int cnl_set_dsp_D0(struct sst_dsp *ctx, unsigned int core_id)
"dsp boot timeout, status=%#x error=%#x\n",
sst_dsp_shim_read(ctx, CNL_ADSP_FW_STATUS),
sst_dsp_shim_read(ctx, CNL_ADSP_ERROR_CODE));
+ ret = -ETIMEDOUT;
goto err;
}
} else {
diff --git a/sound/soc/meson/axg-tdm-interface.c b/sound/soc/meson/axg-tdm-interface.c
index 5c055d8de8c7..01cc551a8e3f 100644
--- a/sound/soc/meson/axg-tdm-interface.c
+++ b/sound/soc/meson/axg-tdm-interface.c
@@ -459,8 +459,20 @@ static int axg_tdm_iface_set_bias_level(struct snd_soc_component *component,
return ret;
}
+static const struct snd_soc_dapm_widget axg_tdm_iface_dapm_widgets[] = {
+ SND_SOC_DAPM_SIGGEN("Playback Signal"),
+};
+
+static const struct snd_soc_dapm_route axg_tdm_iface_dapm_routes[] = {
+ { "Loopback", NULL, "Playback Signal" },
+};
+
static const struct snd_soc_component_driver axg_tdm_iface_component_drv = {
- .set_bias_level = axg_tdm_iface_set_bias_level,
+ .dapm_widgets = axg_tdm_iface_dapm_widgets,
+ .num_dapm_widgets = ARRAY_SIZE(axg_tdm_iface_dapm_widgets),
+ .dapm_routes = axg_tdm_iface_dapm_routes,
+ .num_dapm_routes = ARRAY_SIZE(axg_tdm_iface_dapm_routes),
+ .set_bias_level = axg_tdm_iface_set_bias_level,
};
static const struct of_device_id axg_tdm_iface_of_match[] = {
diff --git a/sound/soc/soc-dapm.c b/sound/soc/soc-dapm.c
index 4e21ce136e3e..f6987d9dd7ec 100644
--- a/sound/soc/soc-dapm.c
+++ b/sound/soc/soc-dapm.c
@@ -2465,6 +2465,7 @@ void snd_soc_dapm_free_widget(struct snd_soc_dapm_widget *w)
enum snd_soc_dapm_direction dir;
list_del(&w->list);
+ list_del(&w->dirty);
/*
* remove source and sink paths associated to this widget.
* While removing the path, remove reference to it from both
diff --git a/sound/usb/midi.c b/sound/usb/midi.c
index 137e1e8718d6..26548f760bc1 100644
--- a/sound/usb/midi.c
+++ b/sound/usb/midi.c
@@ -1890,6 +1890,8 @@ static int snd_usbmidi_get_ms_info(struct snd_usb_midi *umidi,
ms_ep = find_usb_ms_endpoint_descriptor(hostep);
if (!ms_ep)
continue;
+ if (ms_ep->bNumEmbMIDIJack > 0x10)
+ continue;
if (usb_endpoint_dir_out(ep)) {
if (endpoints[epidx].out_ep) {
if (++epidx >= MIDI_MAX_ENDPOINTS) {
@@ -2142,6 +2144,8 @@ static int snd_usbmidi_detect_roland(struct snd_usb_midi *umidi,
cs_desc[1] == USB_DT_CS_INTERFACE &&
cs_desc[2] == 0xf1 &&
cs_desc[3] == 0x02) {
+ if (cs_desc[4] > 0x10 || cs_desc[5] > 0x10)
+ continue;
endpoint->in_cables = (1 << cs_desc[4]) - 1;
endpoint->out_cables = (1 << cs_desc[5]) - 1;
return snd_usbmidi_detect_endpoints(umidi, endpoint, 1);
diff --git a/tools/build/Makefile b/tools/build/Makefile
index 727050c40f09..8a55378e8b7c 100644
--- a/tools/build/Makefile
+++ b/tools/build/Makefile
@@ -15,10 +15,6 @@ endef
$(call allow-override,CC,$(CROSS_COMPILE)gcc)
$(call allow-override,LD,$(CROSS_COMPILE)ld)
-HOSTCC ?= gcc
-HOSTLD ?= ld
-HOSTAR ?= ar
-
export HOSTCC HOSTLD HOSTAR
ifeq ($(V),1)
diff --git a/tools/objtool/Makefile b/tools/objtool/Makefile
index baa92279c137..15f32f67cf34 100644
--- a/tools/objtool/Makefile
+++ b/tools/objtool/Makefile
@@ -7,15 +7,6 @@ ARCH := x86
endif
# always use the host compiler
-ifneq ($(LLVM),)
-HOSTAR ?= llvm-ar
-HOSTCC ?= clang
-HOSTLD ?= ld.lld
-else
-HOSTAR ?= ar
-HOSTCC ?= gcc
-HOSTLD ?= ld
-endif
AR = $(HOSTAR)
CC = $(HOSTCC)
LD = $(HOSTLD)
diff --git a/tools/perf/Makefile.perf b/tools/perf/Makefile.perf
index 0be411695379..678aa7feb84d 100644
--- a/tools/perf/Makefile.perf
+++ b/tools/perf/Makefile.perf
@@ -148,10 +148,6 @@ endef
LD += $(EXTRA_LDFLAGS)
-HOSTCC ?= gcc
-HOSTLD ?= ld
-HOSTAR ?= ar
-
PKG_CONFIG = $(CROSS_COMPILE)pkg-config
LLVM_CONFIG ?= llvm-config
diff --git a/tools/power/acpi/Makefile.config b/tools/power/acpi/Makefile.config
index fc116c060b98..32ff7baf39df 100644
--- a/tools/power/acpi/Makefile.config
+++ b/tools/power/acpi/Makefile.config
@@ -57,7 +57,6 @@ INSTALL_SCRIPT = ${INSTALL_PROGRAM}
CROSS = #/usr/i386-linux-uclibc/usr/bin/i386-uclibc-
CROSS_COMPILE ?= $(CROSS)
LD = $(CC)
-HOSTCC = gcc
# check if compiler option is supported
cc-supports = ${shell if $(CC) ${1} -S -o /dev/null -x c /dev/null > /dev/null 2>&1; then echo "$(1)"; fi;}
diff --git a/tools/scripts/Makefile.include b/tools/scripts/Makefile.include
index 8fc6b1ca47dc..42dbe05b1807 100644
--- a/tools/scripts/Makefile.include
+++ b/tools/scripts/Makefile.include
@@ -60,6 +60,16 @@ $(call allow-override,LD,$(CROSS_COMPILE)ld)
$(call allow-override,CXX,$(CROSS_COMPILE)g++)
$(call allow-override,STRIP,$(CROSS_COMPILE)strip)
+ifneq ($(LLVM),)
+HOSTAR ?= llvm-ar
+HOSTCC ?= clang
+HOSTLD ?= ld.lld
+else
+HOSTAR ?= ar
+HOSTCC ?= gcc
+HOSTLD ?= ld
+endif
+
ifeq ($(CC_NO_CLANG), 1)
EXTRA_WARNINGS += -Wstrict-aliasing=3
endif
diff --git a/tools/testing/selftests/filesystems/incfs/.gitignore b/tools/testing/selftests/filesystems/incfs/.gitignore
index f0e3cd99d4ac..4cba9c219a92 100644
--- a/tools/testing/selftests/filesystems/incfs/.gitignore
+++ b/tools/testing/selftests/filesystems/incfs/.gitignore
@@ -1,3 +1 @@
-incfs_test
-incfs_stress
-incfs_perf
+incfs_test
\ No newline at end of file
diff --git a/tools/testing/selftests/filesystems/incfs/Makefile b/tools/testing/selftests/filesystems/incfs/Makefile
index f3798029247a..5b2e627ce883 100644
--- a/tools/testing/selftests/filesystems/incfs/Makefile
+++ b/tools/testing/selftests/filesystems/incfs/Makefile
@@ -1,11 +1,11 @@
# SPDX-License-Identifier: GPL-2.0
-CFLAGS += -D_FILE_OFFSET_BITS=64 -Wall -Werror -I../.. -I../../../../..
-LDLIBS := -llz4 -lzstd -lcrypto -lpthread
-TEST_GEN_PROGS := incfs_test incfs_stress incfs_perf
+CFLAGS += -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall
+CFLAGS += -I../.. -I../../../../..
+
+LDLIBS := -llz4 -lcrypto
+EXTRA_SOURCES := utils.c
+TEST_GEN_PROGS := incfs_test
+
+$(TEST_GEN_PROGS): $(EXTRA_SOURCES)
include ../../lib.mk
-
-# Put after include ../../lib.mk since that changes $(TEST_GEN_PROGS)
-# Otherwise you get multiple targets, this becomes the default, and it's a mess
-EXTRA_SOURCES := utils.c
-$(TEST_GEN_PROGS) : $(EXTRA_SOURCES)
diff --git a/tools/testing/selftests/filesystems/incfs/incfs_perf.c b/tools/testing/selftests/filesystems/incfs/incfs_perf.c
deleted file mode 100644
index ed36bbd774e3..000000000000
--- a/tools/testing/selftests/filesystems/incfs/incfs_perf.c
+++ /dev/null
@@ -1,717 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright 2020 Google LLC
- */
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-
-#include "utils.h"
-
-#define err_msg(...) \
- do { \
- fprintf(stderr, "%s: (%d) ", TAG, __LINE__); \
- fprintf(stderr, __VA_ARGS__); \
- fprintf(stderr, " (%s)\n", strerror(errno)); \
- } while (false)
-
-#define TAG "incfs_perf"
-
-struct options {
- int blocks; /* -b number of diff block sizes */
- bool no_cleanup; /* -c don't clean up after */
- const char *test_dir; /* -d working directory */
- const char *file_types; /* -f sScCvV */
- bool no_native; /* -n don't test native files */
- bool no_random; /* -r don't do random reads*/
- bool no_linear; /* -R random reads only */
- size_t size; /* -s file size as power of 2 */
- int tries; /* -t times to run test*/
-};
-
-enum flags {
- SHUFFLE = 1,
- COMPRESS = 2,
- VERIFY = 4,
- LAST_FLAG = 8,
-};
-
-void print_help(void)
-{
- puts(
- "incfs_perf. Performance test tool for incfs\n"
- "\tTests read performance of incfs by creating files of various types\n"
- "\tflushing caches and then reading them back.\n"
- "\tEach file is read with different block sizes and average\n"
- "\tthroughput in megabytes/second and memory usage are reported for\n"
- "\teach block size\n"
- "\tNative files are tested for comparison\n"
- "\tNative files are created in native folder, incfs files are created\n"
- "\tin src folder which is mounted on dst folder\n"
- "\n"
- "\t-bn (default 8) number of different block sizes, starting at 4096\n"
- "\t and doubling\n"
- "\t-c don't Clean up - leave files and mount point\n"
- "\t-d dir create directories in dir\n"
- "\t-fs|Sc|Cv|V restrict which files are created.\n"
- "\t s blocks not shuffled, S blocks shuffled\n"
- "\t c blocks not compress, C blocks compressed\n"
- "\t v files not verified, V files verified\n"
- "\t If a letter is omitted, both options are tested\n"
- "\t If no letter are given, incfs is not tested\n"
- "\t-n Don't test native files\n"
- "\t-r No random reads (sequential only)\n"
- "\t-R Random reads only (no sequential)\n"
- "\t-sn (default 30)File size as power of 2\n"
- "\t-tn (default 5) Number of tries per file. Results are averaged\n"
- );
-}
-
-int parse_options(int argc, char *const *argv, struct options *options)
-{
- signed char c;
-
- /* Set defaults here */
- *options = (struct options){
- .blocks = 8,
- .test_dir = ".",
- .tries = 5,
- .size = 30,
- };
-
- /* Load options from command line here */
- while ((c = getopt(argc, argv, "b:cd:f::hnrRs:t:")) != -1) {
- switch (c) {
- case 'b':
- options->blocks = strtol(optarg, NULL, 10);
- break;
-
- case 'c':
- options->no_cleanup = true;
- break;
-
- case 'd':
- options->test_dir = optarg;
- break;
-
- case 'f':
- if (optarg)
- options->file_types = optarg;
- else
- options->file_types = "sS";
- break;
-
- case 'h':
- print_help();
- exit(0);
-
- case 'n':
- options->no_native = true;
- break;
-
- case 'r':
- options->no_random = true;
- break;
-
- case 'R':
- options->no_linear = true;
- break;
-
- case 's':
- options->size = strtol(optarg, NULL, 10);
- break;
-
- case 't':
- options->tries = strtol(optarg, NULL, 10);
- break;
-
- default:
- print_help();
- return -EINVAL;
- }
- }
-
- options->size = 1L << options->size;
-
- return 0;
-}
-
-void shuffle(size_t *buffer, size_t size)
-{
- size_t i;
-
- for (i = 0; i < size; ++i) {
- size_t j = random() * (size - i - 1) / RAND_MAX;
- size_t temp = buffer[i];
-
- buffer[i] = buffer[j];
- buffer[j] = temp;
- }
-}
-
-int get_free_memory(void)
-{
- FILE *meminfo = fopen("/proc/meminfo", "re");
- char field[256];
- char value[256] = {};
-
- if (!meminfo)
- return -ENOENT;
-
- while (fscanf(meminfo, "%[^:]: %s kB\n", field, value) == 2) {
- if (!strcmp(field, "MemFree"))
- break;
- *value = 0;
- }
-
- fclose(meminfo);
-
- if (!*value)
- return -ENOENT;
-
- return strtol(value, NULL, 10);
-}
-
-int write_data(int cmd_fd, int dir_fd, const char *name, size_t size, int flags)
-{
- int fd = openat(dir_fd, name, O_RDWR | O_CLOEXEC);
- struct incfs_permit_fill permit_fill = {
- .file_descriptor = fd,
- };
- int block_count = 1 + (size - 1) / INCFS_DATA_FILE_BLOCK_SIZE;
- size_t *blocks = malloc(sizeof(size_t) * block_count);
- int error = 0;
- size_t i;
- uint8_t data[INCFS_DATA_FILE_BLOCK_SIZE] = {};
- uint8_t compressed_data[INCFS_DATA_FILE_BLOCK_SIZE] = {};
- struct incfs_fill_block fill_block = {
- .compression = COMPRESSION_NONE,
- .data_len = sizeof(data),
- .data = ptr_to_u64(data),
- };
-
- if (!blocks) {
- err_msg("Out of memory");
- error = -errno;
- goto out;
- }
-
- if (fd == -1) {
- err_msg("Could not open file for writing %s", name);
- error = -errno;
- goto out;
- }
-
- if (ioctl(cmd_fd, INCFS_IOC_PERMIT_FILL, &permit_fill)) {
- err_msg("Failed to call PERMIT_FILL");
- error = -errno;
- goto out;
- }
-
- for (i = 0; i < block_count; ++i)
- blocks[i] = i;
-
- if (flags & SHUFFLE)
- shuffle(blocks, block_count);
-
- if (flags & COMPRESS) {
- size_t comp_size = LZ4_compress_default(
- (char *)data, (char *)compressed_data, sizeof(data),
- ARRAY_SIZE(compressed_data));
-
- if (comp_size <= 0) {
- error = -EBADMSG;
- goto out;
- }
- fill_block.compression = COMPRESSION_LZ4;
- fill_block.data = ptr_to_u64(compressed_data);
- fill_block.data_len = comp_size;
- }
-
- for (i = 0; i < block_count; ++i) {
- struct incfs_fill_blocks fill_blocks = {
- .count = 1,
- .fill_blocks = ptr_to_u64(&fill_block),
- };
-
- fill_block.block_index = blocks[i];
- int written = ioctl(fd, INCFS_IOC_FILL_BLOCKS, &fill_blocks);
-
- if (written != 1) {
- error = -errno;
- err_msg("Failed to write block %lu in file %s", i,
- name);
- break;
- }
- }
-
-out:
- free(blocks);
- close(fd);
- sync();
- return error;
-}
-
-int measure_read_throughput_internal(const char *tag, int dir, const char *name,
- const struct options *options, bool random)
-{
- int block;
-
- if (random)
- printf("%32s(random)", tag);
- else
- printf("%40s", tag);
-
- for (block = 0; block < options->blocks; ++block) {
- size_t buffer_size;
- char *buffer;
- int try;
- double time = 0;
- double throughput;
- int memory = 0;
-
- buffer_size = 1 << (block + 12);
- buffer = malloc(buffer_size);
-
- for (try = 0; try < options->tries; ++try) {
- int err;
- struct timespec start_time, end_time;
- off_t i;
- int fd;
- size_t offsets_size = options->size / buffer_size;
- size_t *offsets =
- malloc(offsets_size * sizeof(*offsets));
- int start_memory, end_memory;
-
- if (!offsets) {
- err_msg("Not enough memory");
- return -ENOMEM;
- }
-
- for (i = 0; i < offsets_size; ++i)
- offsets[i] = i * buffer_size;
-
- if (random)
- shuffle(offsets, offsets_size);
-
- err = drop_caches();
- if (err) {
- err_msg("Failed to drop caches");
- return err;
- }
-
- start_memory = get_free_memory();
- if (start_memory < 0) {
- err_msg("Failed to get start memory");
- return start_memory;
- }
-
- fd = openat(dir, name, O_RDONLY | O_CLOEXEC);
- if (fd == -1) {
- err_msg("Failed to open file");
- return err;
- }
-
- err = clock_gettime(CLOCK_MONOTONIC, &start_time);
- if (err) {
- err_msg("Failed to get start time");
- return err;
- }
-
- for (i = 0; i < offsets_size; ++i)
- if (pread(fd, buffer, buffer_size,
- offsets[i]) != buffer_size) {
- err_msg("Failed to read file");
- err = -errno;
- goto fail;
- }
-
- err = clock_gettime(CLOCK_MONOTONIC, &end_time);
- if (err) {
- err_msg("Failed to get start time");
- goto fail;
- }
-
- end_memory = get_free_memory();
- if (end_memory < 0) {
- err_msg("Failed to get end memory");
- return end_memory;
- }
-
- time += end_time.tv_sec - start_time.tv_sec;
- time += (end_time.tv_nsec - start_time.tv_nsec) / 1e9;
-
- close(fd);
- fd = -1;
- memory += start_memory - end_memory;
-
-fail:
- free(offsets);
- close(fd);
- if (err)
- return err;
- }
-
- throughput = options->size * options->tries / time;
- printf("%10.3e %10d", throughput, memory / options->tries);
- free(buffer);
- }
-
- printf("\n");
- return 0;
-}
-
-int measure_read_throughput(const char *tag, int dir, const char *name,
- const struct options *options)
-{
- int err = 0;
-
- if (!options->no_linear)
- err = measure_read_throughput_internal(tag, dir, name, options,
- false);
-
- if (!err && !options->no_random)
- err = measure_read_throughput_internal(tag, dir, name, options,
- true);
- return err;
-}
-
-int test_native_file(int dir, const struct options *options)
-{
- const char *name = "file";
- int fd;
- char buffer[4096] = {};
- off_t i;
- int err;
-
- fd = openat(dir, name, O_CREAT | O_WRONLY | O_CLOEXEC, 0600);
- if (fd == -1) {
- err_msg("Could not open native file");
- return -errno;
- }
-
- for (i = 0; i < options->size; i += sizeof(buffer))
- if (pwrite(fd, buffer, sizeof(buffer), i) != sizeof(buffer)) {
- err_msg("Failed to write file");
- err = -errno;
- goto fail;
- }
-
- close(fd);
- sync();
- fd = -1;
-
- err = measure_read_throughput("native", dir, name, options);
-
-fail:
- close(fd);
- return err;
-}
-
-struct hash_block {
- char data[INCFS_DATA_FILE_BLOCK_SIZE];
-};
-
-static struct hash_block *build_mtree(size_t size, char *root_hash,
- int *mtree_block_count)
-{
- char data[INCFS_DATA_FILE_BLOCK_SIZE] = {};
- const int digest_size = SHA256_DIGEST_SIZE;
- const int hash_per_block = INCFS_DATA_FILE_BLOCK_SIZE / digest_size;
- int block_count = 0;
- int hash_block_count = 0;
- int total_tree_block_count = 0;
- int tree_lvl_index[INCFS_MAX_MTREE_LEVELS] = {};
- int tree_lvl_count[INCFS_MAX_MTREE_LEVELS] = {};
- int levels_count = 0;
- int i, level;
- struct hash_block *mtree;
-
- if (size == 0)
- return 0;
-
- block_count = 1 + (size - 1) / INCFS_DATA_FILE_BLOCK_SIZE;
- hash_block_count = block_count;
- for (i = 0; hash_block_count > 1; i++) {
- hash_block_count = (hash_block_count + hash_per_block - 1) /
- hash_per_block;
- tree_lvl_count[i] = hash_block_count;
- total_tree_block_count += hash_block_count;
- }
- levels_count = i;
-
- for (i = 0; i < levels_count; i++) {
- int prev_lvl_base = (i == 0) ? total_tree_block_count :
- tree_lvl_index[i - 1];
-
- tree_lvl_index[i] = prev_lvl_base - tree_lvl_count[i];
- }
-
- *mtree_block_count = total_tree_block_count;
- mtree = calloc(total_tree_block_count, sizeof(*mtree));
- /* Build level 0 hashes. */
- for (i = 0; i < block_count; i++) {
- int block_index = tree_lvl_index[0] + i / hash_per_block;
- int block_off = (i % hash_per_block) * digest_size;
- char *hash_ptr = mtree[block_index].data + block_off;
-
- sha256(data, INCFS_DATA_FILE_BLOCK_SIZE, hash_ptr);
- }
-
- /* Build higher levels of hash tree. */
- for (level = 1; level < levels_count; level++) {
- int prev_lvl_base = tree_lvl_index[level - 1];
- int prev_lvl_count = tree_lvl_count[level - 1];
-
- for (i = 0; i < prev_lvl_count; i++) {
- int block_index =
- i / hash_per_block + tree_lvl_index[level];
- int block_off = (i % hash_per_block) * digest_size;
- char *hash_ptr = mtree[block_index].data + block_off;
-
- sha256(mtree[i + prev_lvl_base].data,
- INCFS_DATA_FILE_BLOCK_SIZE, hash_ptr);
- }
- }
-
- /* Calculate root hash from the top block */
- sha256(mtree[0].data, INCFS_DATA_FILE_BLOCK_SIZE, root_hash);
-
- return mtree;
-}
-
-static int load_hash_tree(int cmd_fd, int dir, const char *name,
- struct hash_block *mtree, int mtree_block_count)
-{
- int err;
- int i;
- int fd;
- struct incfs_fill_block *fill_block_array =
- calloc(mtree_block_count, sizeof(struct incfs_fill_block));
- struct incfs_fill_blocks fill_blocks = {
- .count = mtree_block_count,
- .fill_blocks = ptr_to_u64(fill_block_array),
- };
- struct incfs_permit_fill permit_fill;
-
- if (!fill_block_array)
- return -ENOMEM;
-
- for (i = 0; i < fill_blocks.count; i++) {
- fill_block_array[i] = (struct incfs_fill_block){
- .block_index = i,
- .data_len = INCFS_DATA_FILE_BLOCK_SIZE,
- .data = ptr_to_u64(mtree[i].data),
- .flags = INCFS_BLOCK_FLAGS_HASH
- };
- }
-
- fd = openat(dir, name, O_RDONLY | O_CLOEXEC);
- if (fd < 0) {
- err = errno;
- goto failure;
- }
-
- permit_fill.file_descriptor = fd;
- if (ioctl(cmd_fd, INCFS_IOC_PERMIT_FILL, &permit_fill)) {
- err_msg("Failed to call PERMIT_FILL");
- err = -errno;
- goto failure;
- }
-
- err = ioctl(fd, INCFS_IOC_FILL_BLOCKS, &fill_blocks);
- close(fd);
- if (err < fill_blocks.count)
- err = errno;
- else
- err = 0;
-
-failure:
- free(fill_block_array);
- return err;
-}
-
-int test_incfs_file(int dst_dir, const struct options *options, int flags)
-{
- int cmd_file = openat(dst_dir, INCFS_PENDING_READS_FILENAME,
- O_RDONLY | O_CLOEXEC);
- int err;
- char name[4];
- incfs_uuid_t id;
- char tag[256];
-
- snprintf(name, sizeof(name), "%c%c%c",
- flags & SHUFFLE ? 'S' : 's',
- flags & COMPRESS ? 'C' : 'c',
- flags & VERIFY ? 'V' : 'v');
-
- if (cmd_file == -1) {
- err_msg("Could not open command file");
- return -errno;
- }
-
- if (flags & VERIFY) {
- char root_hash[INCFS_MAX_HASH_SIZE];
- int mtree_block_count;
- struct hash_block *mtree = build_mtree(options->size, root_hash,
- &mtree_block_count);
-
- if (!mtree) {
- err_msg("Failed to build hash tree");
- err = -ENOMEM;
- goto fail;
- }
-
- err = crypto_emit_file(cmd_file, NULL, name, &id, options->size,
- root_hash, "add_data");
-
- if (!err)
- err = load_hash_tree(cmd_file, dst_dir, name, mtree,
- mtree_block_count);
-
- free(mtree);
- } else
- err = emit_file(cmd_file, NULL, name, &id, options->size, NULL);
-
- if (err) {
- err_msg("Failed to create file %s", name);
- goto fail;
- }
-
- if (write_data(cmd_file, dst_dir, name, options->size, flags))
- goto fail;
-
- snprintf(tag, sizeof(tag), "incfs%s%s%s",
- flags & SHUFFLE ? "(shuffle)" : "",
- flags & COMPRESS ? "(compress)" : "",
- flags & VERIFY ? "(verify)" : "");
-
- err = measure_read_throughput(tag, dst_dir, name, options);
-
-fail:
- close(cmd_file);
- return err;
-}
-
-bool skip(struct options const *options, int flag, char c)
-{
- if (!options->file_types)
- return false;
-
- if (flag && strchr(options->file_types, tolower(c)))
- return true;
-
- if (!flag && strchr(options->file_types, toupper(c)))
- return true;
-
- return false;
-}
-
-int main(int argc, char *const *argv)
-{
- struct options options;
- int err;
- const char *native_dir = "native";
- const char *src_dir = "src";
- const char *dst_dir = "dst";
- int native_dir_fd = -1;
- int src_dir_fd = -1;
- int dst_dir_fd = -1;
- int block;
- int flags;
-
- err = parse_options(argc, argv, &options);
- if (err)
- return err;
-
- err = chdir(options.test_dir);
- if (err) {
- err_msg("Failed to change to %s", options.test_dir);
- return -errno;
- }
-
- /* Clean up any interrupted previous runs */
- while (!umount(dst_dir))
- ;
-
- err = remove_dir(native_dir) || remove_dir(src_dir) ||
- remove_dir(dst_dir);
- if (err)
- return err;
-
- err = mkdir(native_dir, 0700);
- if (err) {
- err_msg("Failed to make directory %s", src_dir);
- err = -errno;
- goto cleanup;
- }
-
- err = mkdir(src_dir, 0700);
- if (err) {
- err_msg("Failed to make directory %s", src_dir);
- err = -errno;
- goto cleanup;
- }
-
- err = mkdir(dst_dir, 0700);
- if (err) {
- err_msg("Failed to make directory %s", src_dir);
- err = -errno;
- goto cleanup;
- }
-
- err = mount_fs_opt(dst_dir, src_dir, "readahead=0,rlog_pages=0", 0);
- if (err) {
- err_msg("Failed to mount incfs");
- goto cleanup;
- }
-
- native_dir_fd = open(native_dir, O_RDONLY | O_CLOEXEC);
- src_dir_fd = open(src_dir, O_RDONLY | O_CLOEXEC);
- dst_dir_fd = open(dst_dir, O_RDONLY | O_CLOEXEC);
- if (native_dir_fd == -1 || src_dir_fd == -1 || dst_dir_fd == -1) {
- err_msg("Failed to open native, src or dst dir");
- err = -errno;
- goto cleanup;
- }
-
- printf("%40s", "");
- for (block = 0; block < options.blocks; ++block)
- printf("%21d", 1 << (block + 12));
- printf("\n");
-
- if (!err && !options.no_native)
- err = test_native_file(native_dir_fd, &options);
-
- for (flags = 0; flags < LAST_FLAG && !err; ++flags) {
- if (skip(&options, flags & SHUFFLE, 's') ||
- skip(&options, flags & COMPRESS, 'c') ||
- skip(&options, flags & VERIFY, 'v'))
- continue;
- err = test_incfs_file(dst_dir_fd, &options, flags);
- }
-
-cleanup:
- close(native_dir_fd);
- close(src_dir_fd);
- close(dst_dir_fd);
- if (!options.no_cleanup) {
- umount(dst_dir);
- remove_dir(native_dir);
- remove_dir(dst_dir);
- remove_dir(src_dir);
- }
-
- return err;
-}
diff --git a/tools/testing/selftests/filesystems/incfs/incfs_stress.c b/tools/testing/selftests/filesystems/incfs/incfs_stress.c
deleted file mode 100644
index a1d491755832..000000000000
--- a/tools/testing/selftests/filesystems/incfs/incfs_stress.c
+++ /dev/null
@@ -1,322 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright 2020 Google LLC
- */
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-
-#include "utils.h"
-
-#define err_msg(...) \
- do { \
- fprintf(stderr, "%s: (%d) ", TAG, __LINE__); \
- fprintf(stderr, __VA_ARGS__); \
- fprintf(stderr, " (%s)\n", strerror(errno)); \
- } while (false)
-
-#define TAG "incfs_stress"
-
-struct options {
- bool no_cleanup; /* -c */
- const char *test_dir; /* -d */
- unsigned int rng_seed; /* -g */
- int num_reads; /* -n */
- int readers; /* -r */
- int size; /* -s */
- int timeout; /* -t */
-};
-
-struct read_data {
- const char *filename;
- int dir_fd;
- size_t filesize;
- int num_reads;
- unsigned int rng_seed;
-};
-
-int cancel_threads;
-
-int parse_options(int argc, char *const *argv, struct options *options)
-{
- signed char c;
-
- /* Set defaults here */
- *options = (struct options){
- .test_dir = ".",
- .num_reads = 1000,
- .readers = 10,
- .size = 10,
- };
-
- /* Load options from command line here */
- while ((c = getopt(argc, argv, "cd:g:n:r:s:t:")) != -1) {
- switch (c) {
- case 'c':
- options->no_cleanup = true;
- break;
-
- case 'd':
- options->test_dir = optarg;
- break;
-
- case 'g':
- options->rng_seed = strtol(optarg, NULL, 10);
- break;
-
- case 'n':
- options->num_reads = strtol(optarg, NULL, 10);
- break;
-
- case 'r':
- options->readers = strtol(optarg, NULL, 10);
- break;
-
- case 's':
- options->size = strtol(optarg, NULL, 10);
- break;
-
- case 't':
- options->timeout = strtol(optarg, NULL, 10);
- break;
- }
- }
-
- return 0;
-}
-
-void *reader(void *data)
-{
- struct read_data *read_data = (struct read_data *)data;
- int i;
- int fd = -1;
- void *buffer = malloc(read_data->filesize);
-
- if (!buffer) {
- err_msg("Failed to alloc read buffer");
- goto out;
- }
-
- fd = openat(read_data->dir_fd, read_data->filename,
- O_RDONLY | O_CLOEXEC);
- if (fd == -1) {
- err_msg("Failed to open file");
- goto out;
- }
-
- for (i = 0; i < read_data->num_reads && !cancel_threads; ++i) {
- off_t offset = rnd(read_data->filesize, &read_data->rng_seed);
- size_t count =
- rnd(read_data->filesize - offset, &read_data->rng_seed);
- ssize_t err = pread(fd, buffer, count, offset);
-
- if (err != count)
- err_msg("failed to read with value %lu", err);
- }
-
-out:
- close(fd);
- free(read_data);
- free(buffer);
- return NULL;
-}
-
-int write_data(int cmd_fd, int dir_fd, const char *name, size_t size)
-{
- int fd = openat(dir_fd, name, O_RDWR | O_CLOEXEC);
- struct incfs_permit_fill permit_fill = {
- .file_descriptor = fd,
- };
- int error = 0;
- int i;
- int block_count = 1 + (size - 1) / INCFS_DATA_FILE_BLOCK_SIZE;
-
- if (fd == -1) {
- err_msg("Could not open file for writing %s", name);
- return -errno;
- }
-
- if (ioctl(cmd_fd, INCFS_IOC_PERMIT_FILL, &permit_fill)) {
- err_msg("Failed to call PERMIT_FILL");
- error = -errno;
- goto out;
- }
-
- for (i = 0; i < block_count; ++i) {
- uint8_t data[INCFS_DATA_FILE_BLOCK_SIZE] = {};
- size_t block_size =
- size > i * INCFS_DATA_FILE_BLOCK_SIZE ?
- INCFS_DATA_FILE_BLOCK_SIZE :
- size - (i * INCFS_DATA_FILE_BLOCK_SIZE);
- struct incfs_fill_block fill_block = {
- .compression = COMPRESSION_NONE,
- .block_index = i,
- .data_len = block_size,
- .data = ptr_to_u64(data),
- };
- struct incfs_fill_blocks fill_blocks = {
- .count = 1,
- .fill_blocks = ptr_to_u64(&fill_block),
- };
- int written = ioctl(fd, INCFS_IOC_FILL_BLOCKS, &fill_blocks);
-
- if (written != 1) {
- error = -errno;
- err_msg("Failed to write block %d in file %s", i, name);
- break;
- }
- }
-out:
- close(fd);
- return error;
-}
-
-int test_files(int src_dir, int dst_dir, struct options const *options)
-{
- unsigned int seed = options->rng_seed;
- int cmd_file = openat(dst_dir, INCFS_PENDING_READS_FILENAME,
- O_RDONLY | O_CLOEXEC);
- int err;
- const char *name = "001";
- incfs_uuid_t id;
- size_t size;
- int i;
- pthread_t *threads = NULL;
-
- size = 1 << (rnd(options->size, &seed) + 12);
- size += rnd(size, &seed);
-
- if (cmd_file == -1) {
- err_msg("Could not open command file");
- return -errno;
- }
-
- err = emit_file(cmd_file, NULL, name, &id, size, NULL);
- if (err) {
- err_msg("Failed to create file %s", name);
- return err;
- }
-
- threads = malloc(sizeof(pthread_t) * options->readers);
- if (!threads) {
- err_msg("Could not allocate memory for threads");
- return -ENOMEM;
- }
-
- for (i = 0; i < options->readers; ++i) {
- struct read_data *read_data = malloc(sizeof(*read_data));
-
- if (!read_data) {
- err_msg("Failed to allocate read_data");
- err = -ENOMEM;
- break;
- }
-
- *read_data = (struct read_data){
- .filename = name,
- .dir_fd = dst_dir,
- .filesize = size,
- .num_reads = options->num_reads,
- .rng_seed = seed,
- };
-
- rnd(0, &seed);
-
- err = pthread_create(threads + i, 0, reader, read_data);
- if (err) {
- err_msg("Failed to create thread");
- free(read_data);
- break;
- }
- }
-
- if (err)
- cancel_threads = 1;
- else
- err = write_data(cmd_file, dst_dir, name, size);
-
- for (; i > 0; --i) {
- if (pthread_join(threads[i - 1], NULL)) {
- err_msg("FATAL: failed to join thread");
- exit(-errno);
- }
- }
-
- free(threads);
- close(cmd_file);
- return err;
-}
-
-int main(int argc, char *const *argv)
-{
- struct options options;
- int err;
- const char *src_dir = "src";
- const char *dst_dir = "dst";
- int src_dir_fd = -1;
- int dst_dir_fd = -1;
-
- err = parse_options(argc, argv, &options);
- if (err)
- return err;
-
- err = chdir(options.test_dir);
- if (err) {
- err_msg("Failed to change to %s", options.test_dir);
- return -errno;
- }
-
- err = remove_dir(src_dir) || remove_dir(dst_dir);
- if (err)
- return err;
-
- err = mkdir(src_dir, 0700);
- if (err) {
- err_msg("Failed to make directory %s", src_dir);
- err = -errno;
- goto cleanup;
- }
-
- err = mkdir(dst_dir, 0700);
- if (err) {
- err_msg("Failed to make directory %s", src_dir);
- err = -errno;
- goto cleanup;
- }
-
- err = mount_fs(dst_dir, src_dir, options.timeout);
- if (err) {
- err_msg("Failed to mount incfs");
- goto cleanup;
- }
-
- src_dir_fd = open(src_dir, O_RDONLY | O_CLOEXEC);
- dst_dir_fd = open(dst_dir, O_RDONLY | O_CLOEXEC);
- if (src_dir_fd == -1 || dst_dir_fd == -1) {
- err_msg("Failed to open src or dst dir");
- err = -errno;
- goto cleanup;
- }
-
- err = test_files(src_dir_fd, dst_dir_fd, &options);
-
-cleanup:
- close(src_dir_fd);
- close(dst_dir_fd);
- if (!options.no_cleanup) {
- umount(dst_dir);
- remove_dir(dst_dir);
- remove_dir(src_dir);
- }
-
- return err;
-}
diff --git a/tools/testing/selftests/filesystems/incfs/incfs_test.c b/tools/testing/selftests/filesystems/incfs/incfs_test.c
index 4d8c70f606b7..6809399eac97 100644
--- a/tools/testing/selftests/filesystems/incfs/incfs_test.c
+++ b/tools/testing/selftests/filesystems/incfs/incfs_test.c
@@ -7,15 +7,12 @@
#include
#include
#include
-#include
#include
#include
#include
#include
#include
-#include
#include
-#include
#include
#include
@@ -32,46 +29,10 @@
#define TEST_FAILURE 1
#define TEST_SUCCESS 0
+#define INCFS_MAX_MTREE_LEVELS 8
#define INCFS_ROOT_INODE 0
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-#define le16_to_cpu(x) (x)
-#define le32_to_cpu(x) (x)
-#define le64_to_cpu(x) (x)
-#else
-#error Big endian not supported!
-#endif
-
-struct {
- int file;
- int test;
- bool verbose;
-} options;
-
-#define TESTCOND(condition) \
- do { \
- if (!(condition)) { \
- ksft_print_msg("%s failed %d\n", \
- __func__, __LINE__); \
- goto out; \
- } else if (options.verbose) \
- ksft_print_msg("%s succeeded %d\n", \
- __func__, __LINE__); \
- } while (false)
-
-#define TEST(statement, condition) \
- do { \
- statement; \
- TESTCOND(condition); \
- } while (false)
-
-#define TESTEQUAL(statement, res) \
- TESTCOND((statement) == (res))
-
-#define TESTNE(statement, res) \
- TESTCOND((statement) != (res))
-
struct hash_block {
char data[INCFS_DATA_FILE_BLOCK_SIZE];
};
@@ -142,13 +103,6 @@ struct test_files_set get_test_files_set(void)
.size = 900 * INCFS_DATA_FILE_BLOCK_SIZE + 7 },
{ .index = 10, .name = "file_big", .size = 500 * 1024 * 1024 }
};
-
- if (options.file)
- return (struct test_files_set) {
- .files = files + options.file - 1,
- .files_count = 1,
- };
-
return (struct test_files_set){ .files = files,
.files_count = ARRAY_SIZE(files) };
}
@@ -252,17 +206,6 @@ static char *get_index_filename(const char *mnt_dir, incfs_uuid_t id)
return strdup(path);
}
-static char *get_incomplete_filename(const char *mnt_dir, incfs_uuid_t id)
-{
- char path[FILENAME_MAX];
- char str_id[1 + 2 * sizeof(id)];
-
- bin2hex(str_id, id.bytes, sizeof(id.bytes));
- snprintf(path, ARRAY_SIZE(path), "%s/.incomplete/%s", mnt_dir, str_id);
-
- return strdup(path);
-}
-
int open_file_by_id(const char *mnt_dir, incfs_uuid_t id, bool use_ioctl)
{
char *path = get_index_filename(mnt_dir, id);
@@ -285,7 +228,8 @@ int open_file_by_id(const char *mnt_dir, incfs_uuid_t id, bool use_ioctl)
goto out;
}
- if (ioctl(fd, INCFS_IOC_PERMIT_FILL, &permit_fill) != -1) {
+ if (ioctl(fd, INCFS_IOC_PERMIT_FILL, &permit_fill) != -1 ||
+ errno != EPERM) {
print_error(
"Successfully called PERMIT_FILL on non pending_read file");
return -errno;
@@ -304,7 +248,7 @@ out:
return fd;
}
-int get_file_attr(const char *mnt_dir, incfs_uuid_t id, char *value, int size)
+int get_file_attr(char *mnt_dir, incfs_uuid_t id, char *value, int size)
{
char *path = get_index_filename(mnt_dir, id);
int res;
@@ -322,13 +266,7 @@ static bool same_id(incfs_uuid_t *id1, incfs_uuid_t *id2)
return !memcmp(id1->bytes, id2->bytes, sizeof(id1->bytes));
}
-ssize_t ZSTD_compress_default(char *data, char *comp_data, size_t data_size,
- size_t comp_size)
-{
- return ZSTD_compress(comp_data, comp_size, data, data_size, 1);
-}
-
-static int emit_test_blocks(const char *mnt_dir, struct test_file *file,
+static int emit_test_blocks(char *mnt_dir, struct test_file *file,
int blocks[], int count)
{
uint8_t data[INCFS_DATA_FILE_BLOCK_SIZE];
@@ -352,8 +290,7 @@ static int emit_test_blocks(const char *mnt_dir, struct test_file *file,
for (i = 0; i < block_count; i++) {
int block_index = blocks[i];
- bool compress_zstd = (file->index + block_index) % 4 == 2;
- bool compress_lz4 = (file->index + block_index) % 4 == 0;
+ bool compress = (file->index + block_index) % 2 == 0;
int seed = get_file_block_seed(file->index, block_index);
off_t block_offset =
((off_t)block_index) * INCFS_DATA_FILE_BLOCK_SIZE;
@@ -370,10 +307,10 @@ static int emit_test_blocks(const char *mnt_dir, struct test_file *file,
block_size = file->size - block_offset;
rnd_buf(data, block_size, seed);
- if (compress_lz4) {
- size_t comp_size = LZ4_compress_default((char *)data,
- (char *)comp_data, block_size,
- ARRAY_SIZE(comp_data));
+ if (compress) {
+ size_t comp_size = LZ4_compress_default(
+ (char *)data, (char *)comp_data, block_size,
+ ARRAY_SIZE(comp_data));
if (comp_size <= 0) {
error = -EBADMSG;
@@ -386,22 +323,6 @@ static int emit_test_blocks(const char *mnt_dir, struct test_file *file,
memcpy(current_data, comp_data, comp_size);
block_size = comp_size;
block_buf[i].compression = COMPRESSION_LZ4;
- } else if (compress_zstd) {
- size_t comp_size = ZSTD_compress(comp_data,
- ARRAY_SIZE(comp_data), data, block_size,
- 1);
-
- if (comp_size <= 0) {
- error = -EBADMSG;
- break;
- }
- if (current_data + comp_size > data_end) {
- error = -ENOMEM;
- break;
- }
- memcpy(current_data, comp_data, comp_size);
- block_size = comp_size;
- block_buf[i].compression = COMPRESSION_ZSTD;
} else {
if (current_data + block_size > data_end) {
error = -ENOMEM;
@@ -455,7 +376,7 @@ out:
return (error < 0) ? error : blocks_written;
}
-static int emit_test_block(const char *mnt_dir, struct test_file *file,
+static int emit_test_block(char *mnt_dir, struct test_file *file,
int block_index)
{
int res = emit_test_blocks(mnt_dir, file, &block_index, 1);
@@ -485,7 +406,7 @@ static void shuffle(int array[], int count, unsigned int seed)
}
}
-static int emit_test_file_data(const char *mount_dir, struct test_file *file)
+static int emit_test_file_data(char *mount_dir, struct test_file *file)
{
int i;
int block_cnt = 1 + (file->size - 1) / INCFS_DATA_FILE_BLOCK_SIZE;
@@ -518,7 +439,7 @@ out:
return result;
}
-static loff_t read_whole_file(const char *filename)
+static loff_t read_whole_file(char *filename)
{
int fd = -1;
loff_t result;
@@ -584,7 +505,7 @@ cleanup:
return result;
}
-static char *create_backing_dir(const char *mount_dir)
+static char *create_backing_dir(char *mount_dir)
{
struct stat st;
char backing_dir_name[255];
@@ -620,7 +541,7 @@ static char *create_backing_dir(const char *mount_dir)
return strdup(backing_dir_name);
}
-static int validate_test_file_content_with_seed(const char *mount_dir,
+static int validate_test_file_content_with_seed(char *mount_dir,
struct test_file *file,
unsigned int shuffle_seed)
{
@@ -682,13 +603,12 @@ failure:
return error;
}
-static int validate_test_file_content(const char *mount_dir,
- struct test_file *file)
+static int validate_test_file_content(char *mount_dir, struct test_file *file)
{
return validate_test_file_content_with_seed(mount_dir, file, 0);
}
-static int data_producer(const char *mount_dir, struct test_files_set *test_set)
+static int data_producer(char *mount_dir, struct test_files_set *test_set)
{
int ret = 0;
int timeout_ms = 1000;
@@ -736,55 +656,6 @@ static int data_producer(const char *mount_dir, struct test_files_set *test_set)
return ret;
}
-static int data_producer2(const char *mount_dir,
- struct test_files_set *test_set)
-{
- int ret = 0;
- int timeout_ms = 1000;
- struct incfs_pending_read_info2 prs[100] = {};
- int prs_size = ARRAY_SIZE(prs);
- int fd = open_commands_file(mount_dir);
-
- if (fd < 0)
- return -errno;
-
- while ((ret = wait_for_pending_reads2(fd, timeout_ms, prs, prs_size)) >
- 0) {
- int read_count = ret;
- int i;
-
- for (i = 0; i < read_count; i++) {
- int j = 0;
- struct test_file *file = NULL;
-
- for (j = 0; j < test_set->files_count; j++) {
- bool same = same_id(&(test_set->files[j].id),
- &(prs[i].file_id));
-
- if (same) {
- file = &test_set->files[j];
- break;
- }
- }
- if (!file) {
- ksft_print_msg(
- "Unknown file in pending reads.\n");
- break;
- }
-
- ret = emit_test_block(mount_dir, file,
- prs[i].block_index);
- if (ret < 0) {
- ksft_print_msg("Emitting test data error: %s\n",
- strerror(-ret));
- break;
- }
- }
- }
- close(fd);
- return ret;
-}
-
static int build_mtree(struct test_file *file)
{
char data[INCFS_DATA_FILE_BLOCK_SIZE] = {};
@@ -932,7 +803,7 @@ failure:
return err;
}
-static int cant_touch_index_test(const char *mount_dir)
+static int cant_touch_index_test(char *mount_dir)
{
char *file_name = "test_file";
int file_size = 123;
@@ -966,12 +837,6 @@ static int cant_touch_index_test(const char *mount_dir)
goto failure;
}
- err = rmdir(index_path);
- if (err == 0 || errno != EBUSY) {
- print_error(".index directory should not be removed\n");
- goto failure;
- }
-
err = emit_file(cmd_fd, ".index", file_name, &file_id,
file_size, NULL);
if (err != -EBUSY) {
@@ -1006,12 +871,6 @@ static int cant_touch_index_test(const char *mount_dir)
goto failure;
}
- err = rename(index_path, dst_name);
- if (err == 0 || errno != EBUSY) {
- print_error("Shouldn't rename .index directory\n");
- goto failure;
- }
-
close(cmd_fd);
free(subdir);
free(index_path);
@@ -1034,8 +893,7 @@ failure:
return TEST_FAILURE;
}
-static bool iterate_directory(const char *dir_to_iterate, bool root,
- int file_count)
+static bool iterate_directory(char *dir_to_iterate, bool root, int file_count)
{
struct expected_name {
const char *name;
@@ -1044,9 +902,7 @@ static bool iterate_directory(const char *dir_to_iterate, bool root,
} names[] = {
{INCFS_LOG_FILENAME, true, false},
{INCFS_PENDING_READS_FILENAME, true, false},
- {INCFS_BLOCKS_WRITTEN_FILENAME, true, false},
{".index", true, false},
- {".incomplete", true, false},
{"..", false, false},
{".", false, false},
};
@@ -1142,7 +998,7 @@ failure:
return pass;
}
-static int basic_file_ops_test(const char *mount_dir)
+static int basic_file_ops_test(char *mount_dir)
{
struct test_files_set test = get_test_files_set();
const int file_num = test.files_count;
@@ -1308,7 +1164,7 @@ failure:
return TEST_FAILURE;
}
-static int dynamic_files_and_data_test(const char *mount_dir)
+static int dynamic_files_and_data_test(char *mount_dir)
{
struct test_files_set test = get_test_files_set();
const int file_num = test.files_count;
@@ -1414,7 +1270,7 @@ failure:
return TEST_FAILURE;
}
-static int concurrent_reads_and_writes_test(const char *mount_dir)
+static int concurrent_reads_and_writes_test(char *mount_dir)
{
struct test_files_set test = get_test_files_set();
const int file_num = test.files_count;
@@ -1536,7 +1392,7 @@ failure:
return TEST_FAILURE;
}
-static int work_after_remount_test(const char *mount_dir)
+static int work_after_remount_test(char *mount_dir)
{
struct test_files_set test = get_test_files_set();
const int file_num = test.files_count;
@@ -1686,7 +1542,7 @@ failure:
return TEST_FAILURE;
}
-static int attribute_test(const char *mount_dir)
+static int attribute_test(char *mount_dir)
{
char file_attr[] = "metadata123123";
char attr_buf[INCFS_MAX_FILE_ATTR_SIZE] = {};
@@ -1769,7 +1625,7 @@ failure:
return TEST_FAILURE;
}
-static int child_procs_waiting_for_data_test(const char *mount_dir)
+static int child_procs_waiting_for_data_test(char *mount_dir)
{
struct test_files_set test = get_test_files_set();
const int file_num = test.files_count;
@@ -1861,7 +1717,7 @@ failure:
return TEST_FAILURE;
}
-static int multiple_providers_test(const char *mount_dir)
+static int multiple_providers_test(char *mount_dir)
{
struct test_files_set test = get_test_files_set();
const int file_num = test.files_count;
@@ -1877,8 +1733,7 @@ static int multiple_providers_test(const char *mount_dir)
goto failure;
/* Mount FS and release the backing file. (10s wait time) */
- if (mount_fs_opt(mount_dir, backing_dir,
- "read_timeout_ms=10000,report_uid", false) != 0)
+ if (mount_fs(mount_dir, backing_dir, 10000) != 0)
goto failure;
cmd_fd = open_commands_file(mount_dir);
@@ -1905,7 +1760,7 @@ static int multiple_providers_test(const char *mount_dir)
* pending reads.
*/
- ret = data_producer2(mount_dir, &test);
+ ret = data_producer(mount_dir, &test);
exit(-ret);
} else if (producer_pid > 0) {
producer_pids[i] = producer_pid;
@@ -1957,7 +1812,7 @@ failure:
return TEST_FAILURE;
}
-static int hash_tree_test(const char *mount_dir)
+static int hash_tree_test(char *mount_dir)
{
char *backing_dir;
struct test_files_set test = get_test_files_set();
@@ -2079,234 +1934,301 @@ failure:
enum expected_log { FULL_LOG, NO_LOG, PARTIAL_LOG };
-static int validate_logs(const char *mount_dir, int log_fd,
- struct test_file *file,
- enum expected_log expected_log,
- bool report_uid, bool expect_data)
+static int validate_logs(char *mount_dir, int log_fd, struct test_file *file,
+ enum expected_log expected_log)
{
- int result = TEST_FAILURE;
uint8_t data[INCFS_DATA_FILE_BLOCK_SIZE];
struct incfs_pending_read_info prs[2048] = {};
- struct incfs_pending_read_info2 prs2[2048] = {};
- struct incfs_pending_read_info *previous_record = NULL;
int prs_size = ARRAY_SIZE(prs);
- int block_count = 1 + (file->size - 1) / INCFS_DATA_FILE_BLOCK_SIZE;
- int expected_read_count, read_count, block_index, read_index;
- char *filename = NULL;
- int fd = -1;
+ int block_cnt = 1 + (file->size - 1) / INCFS_DATA_FILE_BLOCK_SIZE;
+ int expected_read_block_cnt;
+ int res;
+ int read_count;
+ int i, j;
+ char *filename = concat_file_name(mount_dir, file->name);
+ int fd;
- TEST(filename = concat_file_name(mount_dir, file->name), filename);
- TEST(fd = open(filename, O_RDONLY | O_CLOEXEC), fd != -1);
+ fd = open(filename, O_RDONLY | O_CLOEXEC);
+ free(filename);
+ if (fd <= 0)
+ return TEST_FAILURE;
- if (block_count > prs_size)
- block_count = prs_size;
- expected_read_count = block_count;
+ if (block_cnt > prs_size)
+ block_cnt = prs_size;
+ expected_read_block_cnt = block_cnt;
- for (block_index = 0; block_index < block_count; block_index++) {
- int result = pread(fd, data, sizeof(data),
- INCFS_DATA_FILE_BLOCK_SIZE * block_index);
+ for (i = 0; i < block_cnt; i++) {
+ res = pread(fd, data, sizeof(data),
+ INCFS_DATA_FILE_BLOCK_SIZE * i);
/* Make some read logs of type SAME_FILE_NEXT_BLOCK */
- if (block_index % 100 == 10)
+ if (i % 10 == 0)
usleep(20000);
/* Skip some blocks to make logs of type SAME_FILE */
- if (block_index % 10 == 5) {
- ++block_index;
- --expected_read_count;
+ if (i % 10 == 5) {
+ ++i;
+ --expected_read_block_cnt;
}
- if (expect_data)
- TESTCOND(result > 0);
-
- if (!expect_data)
- TESTEQUAL(result, -1);
+ if (res <= 0)
+ goto failure;
}
- if (report_uid)
- read_count = wait_for_pending_reads2(log_fd,
- expected_log == NO_LOG ? 10 : 0,
- prs2, prs_size);
- else
- read_count = wait_for_pending_reads(log_fd,
- expected_log == NO_LOG ? 10 : 0,
- prs, prs_size);
-
- if (expected_log == NO_LOG)
- TESTEQUAL(read_count, 0);
-
- if (expected_log == PARTIAL_LOG)
- TESTCOND(read_count > 0 &&
- read_count <= expected_read_count);
-
- if (expected_log == FULL_LOG)
- TESTEQUAL(read_count, expected_read_count);
-
- /* If read less than expected, advance block_index appropriately */
- for (block_index = 0, read_index = 0;
- read_index < expected_read_count - read_count;
- block_index++, read_index++)
- if (block_index % 10 == 5)
- ++block_index;
-
- for (read_index = 0; read_index < read_count;
- block_index++, read_index++) {
- struct incfs_pending_read_info *record = report_uid ?
- (struct incfs_pending_read_info *) &prs2[read_index] :
- &prs[read_index];
-
- TESTCOND(same_id(&record->file_id, &file->id));
- TESTEQUAL(record->block_index, block_index);
- TESTNE(record->timestamp_us, 0);
- if (previous_record)
- TESTEQUAL(record->serial_number,
- previous_record->serial_number + 1);
-
- previous_record = record;
- if (block_index % 10 == 5)
- ++block_index;
+ read_count = wait_for_pending_reads(
+ log_fd, expected_log == NO_LOG ? 10 : 0, prs, prs_size);
+ if (expected_log == NO_LOG) {
+ if (read_count == 0)
+ goto success;
+ if (read_count < 0)
+ ksft_print_msg("Error reading logged reads %s.\n",
+ strerror(-read_count));
+ else
+ ksft_print_msg("Somehow read empty logs.\n");
+ goto failure;
}
- result = TEST_SUCCESS;
-out:
+ if (read_count < 0) {
+ ksft_print_msg("Error reading logged reads %s.\n",
+ strerror(-read_count));
+ goto failure;
+ }
+
+ i = 0;
+ if (expected_log == PARTIAL_LOG) {
+ if (read_count == 0) {
+ ksft_print_msg("No logs %s.\n", file->name);
+ goto failure;
+ }
+
+ for (i = 0, j = 0; j < expected_read_block_cnt - read_count;
+ i++, j++)
+ if (i % 10 == 5)
+ ++i;
+
+ } else if (read_count != expected_read_block_cnt) {
+ ksft_print_msg("Bad log read count %s %d %d.\n", file->name,
+ read_count, expected_read_block_cnt);
+ goto failure;
+ }
+
+ for (j = 0; j < read_count; i++, j++) {
+ struct incfs_pending_read_info *read = &prs[j];
+
+ if (!same_id(&read->file_id, &file->id)) {
+ ksft_print_msg("Bad log read ino %s\n", file->name);
+ goto failure;
+ }
+
+ if (read->block_index != i) {
+ ksft_print_msg("Bad log read ino %s %d %d.\n",
+ file->name, read->block_index, i);
+ goto failure;
+ }
+
+ if (j != 0) {
+ unsigned long psn = prs[j - 1].serial_number;
+
+ if (read->serial_number != psn + 1) {
+ ksft_print_msg("Bad log read sn %s %d %d.\n",
+ file->name, read->serial_number,
+ psn);
+ goto failure;
+ }
+ }
+
+ if (read->timestamp_us == 0) {
+ ksft_print_msg("Bad log read timestamp %s.\n",
+ file->name);
+ goto failure;
+ }
+
+ if (i % 10 == 5)
+ ++i;
+ }
+
+success:
close(fd);
- free(filename);
- return result;
+ return TEST_SUCCESS;
+
+failure:
+ close(fd);
+ return TEST_FAILURE;
}
-static int read_log_test(const char *mount_dir)
+static int read_log_test(char *mount_dir)
{
- int result = TEST_FAILURE;
struct test_files_set test = get_test_files_set();
const int file_num = test.files_count;
int i = 0;
- int cmd_fd = -1, log_fd = -1;
- char *backing_dir = NULL;
+ int cmd_fd = -1, log_fd = -1, drop_caches = -1;
+ char *backing_dir;
- /* Create files */
- TEST(backing_dir = create_backing_dir(mount_dir), backing_dir);
- TESTEQUAL(mount_fs_opt(mount_dir, backing_dir,
- "readahead=0,report_uid,read_timeout_ms=0",
- false), 0);
- TEST(cmd_fd = open_commands_file(mount_dir), cmd_fd != -1);
+ backing_dir = create_backing_dir(mount_dir);
+ if (!backing_dir)
+ goto failure;
+
+ if (mount_fs_opt(mount_dir, backing_dir, "readahead=0", false) != 0)
+ goto failure;
+
+ cmd_fd = open_commands_file(mount_dir);
+ if (cmd_fd < 0)
+ goto failure;
+
+ log_fd = open_log_file(mount_dir);
+ if (log_fd < 0)
+ ksft_print_msg("Can't open log file.\n");
+
+ /* Write data. */
for (i = 0; i < file_num; i++) {
struct test_file *file = &test.files[i];
- TESTEQUAL(emit_file(cmd_fd, NULL, file->name, &file->id,
- file->size, NULL), 0);
+ if (emit_file(cmd_fd, NULL, file->name, &file->id,
+ file->size, NULL))
+ goto failure;
+
+ if (emit_test_file_data(mount_dir, file))
+ goto failure;
}
- close(cmd_fd);
- cmd_fd = -1;
- /* Validate logs */
- TEST(log_fd = open_log_file(mount_dir), log_fd != -1);
- for (i = 0; i < file_num; i++)
- TESTEQUAL(validate_logs(mount_dir, log_fd, &test.files[i],
- FULL_LOG, true, false), 0);
-
- /* Unmount and mount again without report_uid */
- close(log_fd);
- log_fd = -1;
- TESTEQUAL(umount(mount_dir), 0);
- TESTEQUAL(mount_fs_opt(mount_dir, backing_dir,
- "readahead=0,read_timeout_ms=0", false), 0);
-
- TEST(log_fd = open_log_file(mount_dir), log_fd != -1);
- for (i = 0; i < file_num; i++)
- TESTEQUAL(validate_logs(mount_dir, log_fd, &test.files[i],
- FULL_LOG, false, false), 0);
-
- /* No read log to make sure poll doesn't crash */
- close(log_fd);
- log_fd = -1;
- TESTEQUAL(umount(mount_dir), 0);
- TESTEQUAL(mount_fs_opt(mount_dir, backing_dir,
- "readahead=0,rlog_pages=0,read_timeout_ms=0",
- false), 0);
-
- TEST(log_fd = open_log_file(mount_dir), log_fd != -1);
- for (i = 0; i < file_num; i++)
- TESTEQUAL(validate_logs(mount_dir, log_fd, &test.files[i],
- NO_LOG, false, false), 0);
-
- /* Remount and check that logs start working again */
- TESTEQUAL(mount_fs_opt(mount_dir, backing_dir,
- "readahead=0,rlog_pages=1,read_timeout_ms=0",
- true), 0);
- for (i = 0; i < file_num; i++)
- TESTEQUAL(validate_logs(mount_dir, log_fd, &test.files[i],
- PARTIAL_LOG, false, false), 0);
-
- /* Remount and check that logs continue working */
- TESTEQUAL(mount_fs_opt(mount_dir, backing_dir,
- "readahead=0,rlog_pages=4,read_timeout_ms=0",
- true), 0);
- for (i = 0; i < file_num; i++)
- TESTEQUAL(validate_logs(mount_dir, log_fd, &test.files[i],
- FULL_LOG, false, false), 0);
-
- /* Check logs work with data */
+ /* Validate data */
for (i = 0; i < file_num; i++) {
- TESTEQUAL(emit_test_file_data(mount_dir, &test.files[i]), 0);
- TESTEQUAL(validate_logs(mount_dir, log_fd, &test.files[i],
- FULL_LOG, false, true), 0);
+ struct test_file *file = &test.files[i];
+
+ if (validate_logs(mount_dir, log_fd, file, FULL_LOG))
+ goto failure;
+ }
+
+ /* Unmount and mount again, to see that logs work after remount. */
+ close(cmd_fd);
+ close(log_fd);
+ cmd_fd = -1;
+ if (umount(mount_dir) != 0) {
+ print_error("Can't unmout FS");
+ goto failure;
+ }
+
+ if (mount_fs_opt(mount_dir, backing_dir, "readahead=0", false) != 0)
+ goto failure;
+
+ cmd_fd = open_commands_file(mount_dir);
+ if (cmd_fd < 0)
+ goto failure;
+
+ log_fd = open_log_file(mount_dir);
+ if (log_fd < 0)
+ ksft_print_msg("Can't open log file.\n");
+
+ /* Validate data again */
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+
+ if (validate_logs(mount_dir, log_fd, file, FULL_LOG))
+ goto failure;
+ }
+
+ /*
+ * Unmount and mount again with no read log to make sure poll
+ * doesn't crash
+ */
+ close(cmd_fd);
+ close(log_fd);
+ if (umount(mount_dir) != 0) {
+ print_error("Can't unmout FS");
+ goto failure;
+ }
+
+ if (mount_fs_opt(mount_dir, backing_dir, "readahead=0,rlog_pages=0",
+ false) != 0)
+ goto failure;
+
+ log_fd = open_log_file(mount_dir);
+ if (log_fd < 0)
+ ksft_print_msg("Can't open log file.\n");
+
+ /* Validate data again - note should fail this time */
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+
+ if (validate_logs(mount_dir, log_fd, file, NO_LOG))
+ goto failure;
+ }
+
+ /*
+ * Remount and check that logs start working again
+ */
+ drop_caches = open("/proc/sys/vm/drop_caches", O_WRONLY | O_CLOEXEC);
+ if (drop_caches == -1)
+ goto failure;
+ i = write(drop_caches, "3", 1);
+ close(drop_caches);
+ if (i != 1)
+ goto failure;
+
+ if (mount_fs_opt(mount_dir, backing_dir, "readahead=0,rlog_pages=1",
+ true) != 0)
+ goto failure;
+
+ /* Validate data again */
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+
+ if (validate_logs(mount_dir, log_fd, file, PARTIAL_LOG))
+ goto failure;
+ }
+
+ /*
+ * Remount and check that logs start working again
+ */
+ drop_caches = open("/proc/sys/vm/drop_caches", O_WRONLY | O_CLOEXEC);
+ if (drop_caches == -1)
+ goto failure;
+ i = write(drop_caches, "3", 1);
+ close(drop_caches);
+ if (i != 1)
+ goto failure;
+
+ if (mount_fs_opt(mount_dir, backing_dir, "readahead=0,rlog_pages=4",
+ true) != 0)
+ goto failure;
+
+ /* Validate data again */
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+
+ if (validate_logs(mount_dir, log_fd, file, FULL_LOG))
+ goto failure;
}
/* Final unmount */
close(log_fd);
- log_fd = -1;
- TESTEQUAL(umount(mount_dir), 0);
+ free(backing_dir);
+ if (umount(mount_dir) != 0) {
+ print_error("Can't unmout FS");
+ goto failure;
+ }
- result = TEST_SUCCESS;
-out:
+ return TEST_SUCCESS;
+
+failure:
close(cmd_fd);
close(log_fd);
free(backing_dir);
umount(mount_dir);
- return result;
+ return TEST_FAILURE;
}
-static int emit_partial_test_file_data(const char *mount_dir,
- struct test_file *file)
+static int emit_partial_test_file_data(char *mount_dir, struct test_file *file)
{
int i, j;
int block_cnt = 1 + (file->size - 1) / INCFS_DATA_FILE_BLOCK_SIZE;
int *block_indexes = NULL;
int result = 0;
int blocks_written = 0;
- int bw_fd = -1;
- char buffer[20];
- struct pollfd pollfd;
- long blocks_written_total, blocks_written_new_total;
if (file->size == 0)
return 0;
- bw_fd = open_blocks_written_file(mount_dir);
- if (bw_fd == -1)
- return -errno;
-
- result = read(bw_fd, buffer, sizeof(buffer));
- if (result <= 0) {
- result = -EIO;
- goto out;
- }
-
- buffer[result] = 0;
- blocks_written_total = strtol(buffer, NULL, 10);
- result = 0;
-
- pollfd = (struct pollfd) {
- .fd = bw_fd,
- .events = POLLIN,
- };
-
- result = poll(&pollfd, 1, 0);
- if (result) {
- result = -EIO;
- goto out;
- }
-
/* Emit 2 blocks, skip 2 blocks etc*/
block_indexes = calloc(block_cnt, sizeof(*block_indexes));
for (i = 0, j = 0; i < block_cnt; ++i)
@@ -2326,29 +2248,9 @@ static int emit_partial_test_file_data(const char *mount_dir,
result = -EIO;
goto out;
}
-
- result = poll(&pollfd, 1, 0);
- if (result != 1 || pollfd.revents != POLLIN) {
- result = -EIO;
- goto out;
- }
-
- result = read(bw_fd, buffer, sizeof(buffer));
- buffer[result] = 0;
- blocks_written_new_total = strtol(buffer, NULL, 10);
-
- if (blocks_written_new_total - blocks_written_total
- != blocks_written) {
- result = -EIO;
- goto out;
- }
-
- blocks_written_total = blocks_written_new_total;
- result = 0;
}
out:
free(block_indexes);
- close(bw_fd);
return result;
}
@@ -2482,7 +2384,7 @@ out:
return error;
}
-static int get_blocks_test(const char *mount_dir)
+static int get_blocks_test(char *mount_dir)
{
char *backing_dir;
int cmd_fd = -1;
@@ -2539,8 +2441,7 @@ failure:
return TEST_FAILURE;
}
-static int emit_partial_test_file_hash(const char *mount_dir,
- struct test_file *file)
+static int emit_partial_test_file_hash(char *mount_dir, struct test_file *file)
{
int err;
int fd;
@@ -2554,6 +2455,9 @@ static int emit_partial_test_file_hash(const char *mount_dir,
if (file->size <= 4096 / 32 * 4096)
return 0;
+ if (fill_blocks.count == 0)
+ return 0;
+
if (!fill_block_array)
return -ENOMEM;
fill_blocks.fill_blocks = ptr_to_u64(fill_block_array);
@@ -2655,7 +2559,7 @@ out:
return error;
}
-static int get_hash_blocks_test(const char *mount_dir)
+static int get_hash_blocks_test(char *mount_dir)
{
char *backing_dir;
int cmd_fd = -1;
@@ -2705,7 +2609,7 @@ failure:
return TEST_FAILURE;
}
-static int large_file_test(const char *mount_dir)
+static int large_file(char *mount_dir)
{
char *backing_dir;
int cmd_fd = -1;
@@ -2720,7 +2624,7 @@ static int large_file_test(const char *mount_dir)
.fill_blocks = ptr_to_u64(block_buf),
};
incfs_uuid_t id;
- int fd = -1;
+ int fd;
backing_dir = create_backing_dir(mount_dir);
if (!backing_dir)
@@ -2761,714 +2665,6 @@ static int large_file_test(const char *mount_dir)
failure:
close(fd);
close(cmd_fd);
- umount(mount_dir);
- free(backing_dir);
- return result;
-}
-
-static int validate_mapped_file(const char *orig_name, const char *name,
- size_t size, size_t offset)
-{
- struct stat st;
- int orig_fd = -1, fd = -1;
- size_t block;
- int result = TEST_FAILURE;
-
- if (stat(name, &st)) {
- ksft_print_msg("Failed to stat %s with error %s\n",
- name, strerror(errno));
- goto failure;
- }
-
- if (size != st.st_size) {
- ksft_print_msg("Mismatched file sizes for file %s - expected %llu, got %llu\n",
- name, size, st.st_size);
- goto failure;
- }
-
- fd = open(name, O_RDONLY | O_CLOEXEC);
- if (fd == -1) {
- ksft_print_msg("Failed to open %s with error %s\n", name,
- strerror(errno));
- goto failure;
- }
-
- orig_fd = open(orig_name, O_RDONLY | O_CLOEXEC);
- if (orig_fd == -1) {
- ksft_print_msg("Failed to open %s with error %s\n", orig_name,
- strerror(errno));
- goto failure;
- }
-
- for (block = 0; block < size; block += INCFS_DATA_FILE_BLOCK_SIZE) {
- uint8_t orig_data[INCFS_DATA_FILE_BLOCK_SIZE];
- uint8_t data[INCFS_DATA_FILE_BLOCK_SIZE];
- ssize_t orig_read, mapped_read;
-
- orig_read = pread(orig_fd, orig_data,
- INCFS_DATA_FILE_BLOCK_SIZE, block + offset);
- mapped_read = pread(fd, data, INCFS_DATA_FILE_BLOCK_SIZE,
- block);
-
- if (orig_read < mapped_read ||
- mapped_read != min(size - block,
- INCFS_DATA_FILE_BLOCK_SIZE)) {
- ksft_print_msg("Failed to read enough data: %llu %llu %llu %lld %lld\n",
- block, size, offset, orig_read,
- mapped_read);
- goto failure;
- }
-
- if (memcmp(orig_data, data, mapped_read)) {
- ksft_print_msg("Data doesn't match: %llu %llu %llu %lld %lld\n",
- block, size, offset, orig_read,
- mapped_read);
- goto failure;
- }
- }
-
- result = TEST_SUCCESS;
-
-failure:
- close(orig_fd);
- close(fd);
- return result;
-}
-
-static int mapped_file_test(const char *mount_dir)
-{
- char *backing_dir;
- int result = TEST_FAILURE;
- int cmd_fd = -1;
- int i;
- struct test_files_set test = get_test_files_set();
- const int file_num = test.files_count;
-
- backing_dir = create_backing_dir(mount_dir);
- if (!backing_dir)
- goto failure;
-
- if (mount_fs_opt(mount_dir, backing_dir, "readahead=0", false) != 0)
- goto failure;
-
- cmd_fd = open_commands_file(mount_dir);
- if (cmd_fd < 0)
- goto failure;
-
- for (i = 0; i < file_num; ++i) {
- struct test_file *file = &test.files[i];
- size_t blocks = file->size / INCFS_DATA_FILE_BLOCK_SIZE;
- size_t mapped_offset = blocks / 4 *
- INCFS_DATA_FILE_BLOCK_SIZE;
- size_t mapped_size = file->size / 4 * 3 - mapped_offset;
- struct incfs_create_mapped_file_args mfa;
- char mapped_file_name[FILENAME_MAX];
- char orig_file_path[PATH_MAX];
- char mapped_file_path[PATH_MAX];
-
- if (emit_file(cmd_fd, NULL, file->name, &file->id, file->size,
- NULL) < 0)
- goto failure;
-
- if (emit_test_file_data(mount_dir, file))
- goto failure;
-
- if (snprintf(mapped_file_name, ARRAY_SIZE(mapped_file_name),
- "%s.mapped", file->name) < 0)
- goto failure;
-
- mfa = (struct incfs_create_mapped_file_args) {
- .size = mapped_size,
- .mode = 0664,
- .file_name = ptr_to_u64(mapped_file_name),
- .source_file_id = file->id,
- .source_offset = mapped_offset,
- };
-
- result = ioctl(cmd_fd, INCFS_IOC_CREATE_MAPPED_FILE, &mfa);
- if (result) {
- ksft_print_msg(
- "Failed to create mapped file with error %d\n",
- result);
- goto failure;
- }
-
- result = snprintf(orig_file_path,
- ARRAY_SIZE(orig_file_path), "%s/%s",
- mount_dir, file->name);
-
- if (result < 0 || result >= ARRAY_SIZE(mapped_file_path)) {
- result = TEST_FAILURE;
- goto failure;
- }
-
- result = snprintf(mapped_file_path,
- ARRAY_SIZE(mapped_file_path), "%s/%s",
- mount_dir, mapped_file_name);
-
- if (result < 0 || result >= ARRAY_SIZE(mapped_file_path)) {
- result = TEST_FAILURE;
- goto failure;
- }
-
- result = validate_mapped_file(orig_file_path, mapped_file_path,
- mapped_size, mapped_offset);
- if (result)
- goto failure;
- }
-
-failure:
- close(cmd_fd);
- umount(mount_dir);
- free(backing_dir);
- return result;
-}
-
-static const char v1_file[] = {
- /* Header */
- /* 0x00: Magic number */
- 0x49, 0x4e, 0x43, 0x46, 0x53, 0x00, 0x00, 0x00,
- /* 0x08: Version */
- 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- /* 0x10: Header size */
- 0x38, 0x00,
- /* 0x12: Block size */
- 0x00, 0x10,
- /* 0x14: Flags */
- 0x00, 0x00, 0x00, 0x00,
- /* 0x18: First md offset */
- 0x46, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- /* 0x20: File size */
- 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- /* 0x28: UUID */
- 0x8c, 0x7d, 0xd9, 0x22, 0xad, 0x47, 0x49, 0x4f,
- 0xc0, 0x2c, 0x38, 0x8e, 0x12, 0xc0, 0x0e, 0xac,
-
- /* 0x38: Attribute */
- 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
- 0x31, 0x32, 0x33, 0x31, 0x32, 0x33,
-
- /* Attribute md record */
- /* 0x46: Type */
- 0x02,
- /* 0x47: Size */
- 0x25, 0x00,
- /* 0x49: CRC */
- 0x9a, 0xef, 0xef, 0x72,
- /* 0x4d: Next md offset */
- 0x75, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- /* 0x55: Prev md offset */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- /* 0x5d: fa_offset */
- 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- /* 0x65: fa_size */
- 0x0e, 0x00,
- /* 0x67: fa_crc */
- 0xfb, 0x5e, 0x72, 0x89,
-
- /* Blockmap table */
- /* 0x6b: First 10-byte entry */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-
- /* Blockmap md record */
- /* 0x75: Type */
- 0x01,
- /* 0x76: Size */
- 0x23, 0x00,
- /* 0x78: CRC */
- 0x74, 0x45, 0xd3, 0xb9,
- /* 0x7c: Next md offset */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- /* 0x84: Prev md offset */
- 0x46, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- /* 0x8c: blockmap offset */
- 0x6b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- /* 0x94: blockmap count */
- 0x01, 0x00, 0x00, 0x00,
-};
-
-static int compatibility_test(const char *mount_dir)
-{
- static const char *name = "file";
- int result = TEST_FAILURE;
- char *backing_dir = NULL;
- char *filename = NULL;
- int fd = -1;
- uint64_t size = 0x0c;
-
- TEST(backing_dir = create_backing_dir(mount_dir), backing_dir);
- TEST(filename = concat_file_name(backing_dir, name), filename);
- TEST(fd = open(filename, O_CREAT | O_WRONLY | O_CLOEXEC, 0777),
- fd != -1);
- TESTEQUAL(write(fd, v1_file, sizeof(v1_file)), sizeof(v1_file));
- TESTEQUAL(fsetxattr(fd, INCFS_XATTR_SIZE_NAME, &size, sizeof(size), 0),
- 0);
- TESTEQUAL(mount_fs(mount_dir, backing_dir, 50), 0);
- free(filename);
- TEST(filename = concat_file_name(mount_dir, name), filename);
- close(fd);
- TEST(fd = open(filename, O_RDONLY | O_CLOEXEC), fd != -1);
-
- result = TEST_SUCCESS;
-out:
- close(fd);
- umount(mount_dir);
- free(backing_dir);
- free(filename);
- return result;
-}
-
-static int zero_blocks_written_count(int fd, uint32_t data_blocks_written,
- uint32_t hash_blocks_written)
-{
- int test_result = TEST_FAILURE;
- uint64_t offset;
- uint8_t type;
- uint32_t bw;
-
- /* Get first md record */
- TESTEQUAL(pread(fd, &offset, sizeof(offset), 24), sizeof(offset));
-
- /* Find status md record */
- for (;;) {
- TESTNE(offset, 0);
- TESTEQUAL(pread(fd, &type, sizeof(type), le64_to_cpu(offset)),
- sizeof(type));
- if (type == 4)
- break;
- TESTEQUAL(pread(fd, &offset, sizeof(offset),
- le64_to_cpu(offset) + 7),
- sizeof(offset));
- }
-
- /* Read blocks_written */
- offset = le64_to_cpu(offset);
- TESTEQUAL(pread(fd, &bw, sizeof(bw), offset + 23), sizeof(bw));
- TESTEQUAL(le32_to_cpu(bw), data_blocks_written);
- TESTEQUAL(pread(fd, &bw, sizeof(bw), offset + 27), sizeof(bw));
- TESTEQUAL(le32_to_cpu(bw), hash_blocks_written);
-
- /* Write out zero */
- bw = 0;
- TESTEQUAL(pwrite(fd, &bw, sizeof(bw), offset + 23), sizeof(bw));
- TESTEQUAL(pwrite(fd, &bw, sizeof(bw), offset + 27), sizeof(bw));
-
- test_result = TEST_SUCCESS;
-out:
- return test_result;
-}
-
-static int validate_block_count(const char *mount_dir, const char *backing_dir,
- struct test_file *file,
- int total_data_blocks, int filled_data_blocks,
- int total_hash_blocks, int filled_hash_blocks)
-{
- char *filename = NULL;
- char *backing_filename = NULL;
- int fd = -1;
- struct incfs_get_block_count_args bca = {};
- int test_result = TEST_FAILURE;
- struct incfs_filled_range ranges[128];
- struct incfs_get_filled_blocks_args fba = {
- .range_buffer = ptr_to_u64(ranges),
- .range_buffer_size = sizeof(ranges),
- };
- int cmd_fd = -1;
- struct incfs_permit_fill permit_fill;
-
- TEST(filename = concat_file_name(mount_dir, file->name), filename);
- TEST(backing_filename = concat_file_name(backing_dir, file->name),
- backing_filename);
- TEST(fd = open(filename, O_RDONLY | O_CLOEXEC), fd != -1);
-
- TESTEQUAL(ioctl(fd, INCFS_IOC_GET_BLOCK_COUNT, &bca), 0);
- TESTEQUAL(bca.total_data_blocks_out, total_data_blocks);
- TESTEQUAL(bca.filled_data_blocks_out, filled_data_blocks);
- TESTEQUAL(bca.total_hash_blocks_out, total_hash_blocks);
- TESTEQUAL(bca.filled_hash_blocks_out, filled_hash_blocks);
-
- close(fd);
- TESTEQUAL(umount(mount_dir), 0);
- TEST(fd = open(backing_filename, O_RDWR | O_CLOEXEC), fd != -1);
- TESTEQUAL(zero_blocks_written_count(fd, filled_data_blocks,
- filled_hash_blocks),
- TEST_SUCCESS);
- close(fd);
- fd = -1;
- TESTEQUAL(mount_fs_opt(mount_dir, backing_dir, "readahead=0", false),
- 0);
- TEST(fd = open(filename, O_RDONLY | O_CLOEXEC), fd != -1);
-
- TESTEQUAL(ioctl(fd, INCFS_IOC_GET_BLOCK_COUNT, &bca), 0);
- TESTEQUAL(bca.total_data_blocks_out, total_data_blocks);
- TESTEQUAL(bca.filled_data_blocks_out, 0);
- TESTEQUAL(bca.total_hash_blocks_out, total_hash_blocks);
- TESTEQUAL(bca.filled_hash_blocks_out, 0);
-
- TEST(cmd_fd = open_commands_file(mount_dir), cmd_fd != -1);
- permit_fill.file_descriptor = fd;
- TESTEQUAL(ioctl(cmd_fd, INCFS_IOC_PERMIT_FILL, &permit_fill), 0);
- do {
- ioctl(fd, INCFS_IOC_GET_FILLED_BLOCKS, &fba);
- fba.start_index = fba.index_out + 1;
- } while (fba.index_out < fba.total_blocks_out);
-
- TESTEQUAL(ioctl(fd, INCFS_IOC_GET_BLOCK_COUNT, &bca), 0);
- TESTEQUAL(bca.total_data_blocks_out, total_data_blocks);
- TESTEQUAL(bca.filled_data_blocks_out, filled_data_blocks);
- TESTEQUAL(bca.total_hash_blocks_out, total_hash_blocks);
- TESTEQUAL(bca.filled_hash_blocks_out, filled_hash_blocks);
-
- test_result = TEST_SUCCESS;
-out:
- close(cmd_fd);
- close(fd);
- free(filename);
- free(backing_filename);
- return test_result;
-}
-
-
-
-static int validate_data_block_count(const char *mount_dir,
- const char *backing_dir,
- struct test_file *file)
-{
- const int total_data_blocks = 1 + (file->size - 1) /
- INCFS_DATA_FILE_BLOCK_SIZE;
- const int filled_data_blocks = (total_data_blocks + 1) / 2;
-
- int test_result = TEST_FAILURE;
- char *filename = NULL;
- char *incomplete_filename = NULL;
- struct stat stat_buf_incomplete, stat_buf_file;
- int fd = -1;
- struct incfs_get_block_count_args bca = {};
- int i;
-
- TEST(filename = concat_file_name(mount_dir, file->name), filename);
- TEST(incomplete_filename = get_incomplete_filename(mount_dir, file->id),
- incomplete_filename);
-
- TESTEQUAL(stat(filename, &stat_buf_file), 0);
- TESTEQUAL(stat(incomplete_filename, &stat_buf_incomplete), 0);
- TESTEQUAL(stat_buf_file.st_ino, stat_buf_incomplete.st_ino);
- TESTEQUAL(stat_buf_file.st_nlink, 3);
-
- TEST(fd = open(filename, O_RDONLY | O_CLOEXEC), fd != -1);
- TESTEQUAL(ioctl(fd, INCFS_IOC_GET_BLOCK_COUNT, &bca), 0);
- TESTEQUAL(bca.total_data_blocks_out, total_data_blocks);
- TESTEQUAL(bca.filled_data_blocks_out, 0);
- TESTEQUAL(bca.total_hash_blocks_out, 0);
- TESTEQUAL(bca.filled_hash_blocks_out, 0);
-
- for (i = 0; i < total_data_blocks; i += 2)
- TESTEQUAL(emit_test_block(mount_dir, file, i), 0);
-
- TESTEQUAL(ioctl(fd, INCFS_IOC_GET_BLOCK_COUNT, &bca), 0);
- TESTEQUAL(bca.total_data_blocks_out, total_data_blocks);
- TESTEQUAL(bca.filled_data_blocks_out, filled_data_blocks);
- TESTEQUAL(bca.total_hash_blocks_out, 0);
- TESTEQUAL(bca.filled_hash_blocks_out, 0);
- close(fd);
- fd = -1;
-
- TESTEQUAL(validate_block_count(mount_dir, backing_dir, file,
- total_data_blocks, filled_data_blocks,
- 0, 0),
- 0);
-
- for (i = 1; i < total_data_blocks; i += 2)
- TESTEQUAL(emit_test_block(mount_dir, file, i), 0);
-
- TESTEQUAL(stat(incomplete_filename, &stat_buf_incomplete), -1);
- TESTEQUAL(errno, ENOENT);
-
- test_result = TEST_SUCCESS;
-out:
- close(fd);
- free(incomplete_filename);
- free(filename);
- return test_result;
-}
-
-static int data_block_count_test(const char *mount_dir)
-{
- int result = TEST_FAILURE;
- char *backing_dir;
- int cmd_fd = -1;
- int i;
- struct test_files_set test = get_test_files_set();
-
- TEST(backing_dir = create_backing_dir(mount_dir), backing_dir);
- TESTEQUAL(mount_fs_opt(mount_dir, backing_dir, "readahead=0", false),
- 0);
-
- for (i = 0; i < test.files_count; ++i) {
- struct test_file *file = &test.files[i];
-
- TEST(cmd_fd = open_commands_file(mount_dir), cmd_fd != -1);
- TESTEQUAL(emit_file(cmd_fd, NULL, file->name, &file->id,
- file->size, NULL),
- 0);
- close(cmd_fd);
- cmd_fd = -1;
-
- TESTEQUAL(validate_data_block_count(mount_dir, backing_dir,
- file),
- 0);
- }
-
- result = TEST_SUCCESS;
-out:
- close(cmd_fd);
- umount(mount_dir);
- free(backing_dir);
- return result;
-}
-
-static int validate_hash_block_count(const char *mount_dir,
- const char *backing_dir,
- struct test_file *file)
-{
- const int digest_size = SHA256_DIGEST_SIZE;
- const int hash_per_block = INCFS_DATA_FILE_BLOCK_SIZE / digest_size;
- const int total_data_blocks = 1 + (file->size - 1) /
- INCFS_DATA_FILE_BLOCK_SIZE;
-
- int result = TEST_FAILURE;
- int hash_layer = total_data_blocks;
- int total_hash_blocks = 0;
- int filled_hash_blocks;
- char *filename = NULL;
- int fd = -1;
- struct incfs_get_block_count_args bca = {};
-
- while (hash_layer > 1) {
- hash_layer = (hash_layer + hash_per_block - 1) / hash_per_block;
- total_hash_blocks += hash_layer;
- }
- filled_hash_blocks = total_hash_blocks > 1 ? 1 : 0;
-
- TEST(filename = concat_file_name(mount_dir, file->name), filename);
- TEST(fd = open(filename, O_RDONLY | O_CLOEXEC), fd != -1);
-
- TESTEQUAL(ioctl(fd, INCFS_IOC_GET_BLOCK_COUNT, &bca), 0);
- TESTEQUAL(bca.total_data_blocks_out, total_data_blocks);
- TESTEQUAL(bca.filled_data_blocks_out, 0);
- TESTEQUAL(bca.total_hash_blocks_out, total_hash_blocks);
- TESTEQUAL(bca.filled_hash_blocks_out, 0);
-
- TESTEQUAL(emit_partial_test_file_hash(mount_dir, file), 0);
-
- TESTEQUAL(ioctl(fd, INCFS_IOC_GET_BLOCK_COUNT, &bca), 0);
- TESTEQUAL(bca.total_data_blocks_out, total_data_blocks);
- TESTEQUAL(bca.filled_data_blocks_out, 0);
- TESTEQUAL(bca.total_hash_blocks_out, total_hash_blocks);
- TESTEQUAL(bca.filled_hash_blocks_out, filled_hash_blocks);
- close(fd);
- fd = -1;
-
- if (filled_hash_blocks)
- TESTEQUAL(validate_block_count(mount_dir, backing_dir, file,
- total_data_blocks, 0,
- total_hash_blocks, filled_hash_blocks),
- 0);
-
- result = TEST_SUCCESS;
-out:
- close(fd);
- free(filename);
- return result;
-}
-
-static int hash_block_count_test(const char *mount_dir)
-{
- int result = TEST_FAILURE;
- char *backing_dir;
- int cmd_fd = -1;
- int i;
- struct test_files_set test = get_test_files_set();
-
- TEST(backing_dir = create_backing_dir(mount_dir), backing_dir);
- TESTEQUAL(mount_fs_opt(mount_dir, backing_dir, "readahead=0", false),
- 0);
-
- for (i = 0; i < test.files_count; i++) {
- struct test_file *file = &test.files[i];
-
- TEST(cmd_fd = open_commands_file(mount_dir), cmd_fd != -1);
- TESTEQUAL(crypto_emit_file(cmd_fd, NULL, file->name, &file->id,
- file->size, file->root_hash,
- file->sig.add_data),
- 0);
- close(cmd_fd);
- cmd_fd = -1;
-
- TESTEQUAL(validate_hash_block_count(mount_dir, backing_dir,
- &test.files[i]),
- 0);
- }
-
- result = TEST_SUCCESS;
-out:
- close(cmd_fd);
- umount(mount_dir);
- free(backing_dir);
- return result;
-}
-
-static int is_close(struct timespec *start, int expected_ms)
-{
- const int allowed_variance = 100;
- int result = TEST_FAILURE;
- struct timespec finish;
- int diff;
-
- TESTEQUAL(clock_gettime(CLOCK_MONOTONIC, &finish), 0);
- diff = (finish.tv_sec - start->tv_sec) * 1000 +
- (finish.tv_nsec - start->tv_nsec) / 1000000;
-
- TESTCOND(diff >= expected_ms - allowed_variance);
- TESTCOND(diff <= expected_ms + allowed_variance);
- result = TEST_SUCCESS;
-out:
- return result;
-}
-
-static int per_uid_read_timeouts_test(const char *mount_dir)
-{
- struct test_file file = {
- .name = "file",
- .size = 16 * INCFS_DATA_FILE_BLOCK_SIZE
- };
-
- int result = TEST_FAILURE;
- char *backing_dir = NULL;
- int pid = -1;
- int cmd_fd = -1;
- char *filename = NULL;
- int fd = -1;
- struct timespec start;
- char buffer[4096];
- struct incfs_per_uid_read_timeouts purt_get[1];
- struct incfs_get_read_timeouts_args grt = {
- ptr_to_u64(purt_get),
- sizeof(purt_get)
- };
- struct incfs_per_uid_read_timeouts purt_set[] = {
- {
- .uid = 0,
- .min_time_us = 1000000,
- .min_pending_time_us = 2000000,
- .max_pending_time_us = 3000000,
- },
- };
- struct incfs_set_read_timeouts_args srt = {
- ptr_to_u64(purt_set),
- sizeof(purt_set)
- };
- int status;
-
- TEST(backing_dir = create_backing_dir(mount_dir), backing_dir);
- TESTEQUAL(mount_fs_opt(mount_dir, backing_dir,
- "read_timeout_ms=1000,readahead=0", false), 0);
-
- TEST(cmd_fd = open_commands_file(mount_dir), cmd_fd != -1);
- TESTEQUAL(emit_file(cmd_fd, NULL, file.name, &file.id, file.size,
- NULL), 0);
-
- TEST(filename = concat_file_name(mount_dir, file.name), filename);
- TEST(fd = open(filename, O_RDONLY | O_CLOEXEC), fd != -1);
- TESTEQUAL(fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC), 0);
-
- /* Default mount options read failure is 1000 */
- TESTEQUAL(clock_gettime(CLOCK_MONOTONIC, &start), 0);
- TESTEQUAL(pread(fd, buffer, sizeof(buffer), 0), -1);
- TESTEQUAL(is_close(&start, 1000), 0);
-
- grt.timeouts_array_size = 0;
- TESTEQUAL(ioctl(cmd_fd, INCFS_IOC_GET_READ_TIMEOUTS, &grt), 0);
- TESTEQUAL(grt.timeouts_array_size_out, 0);
-
- /* Set it to 3000 */
- TESTEQUAL(ioctl(cmd_fd, INCFS_IOC_SET_READ_TIMEOUTS, &srt), 0);
- TESTEQUAL(clock_gettime(CLOCK_MONOTONIC, &start), 0);
- TESTEQUAL(pread(fd, buffer, sizeof(buffer), 0), -1);
- TESTEQUAL(is_close(&start, 3000), 0);
- TESTEQUAL(ioctl(cmd_fd, INCFS_IOC_GET_READ_TIMEOUTS, &grt), -1);
- TESTEQUAL(errno, E2BIG);
- TESTEQUAL(grt.timeouts_array_size_out, sizeof(purt_get));
- grt.timeouts_array_size = sizeof(purt_get);
- TESTEQUAL(ioctl(cmd_fd, INCFS_IOC_GET_READ_TIMEOUTS, &grt), 0);
- TESTEQUAL(grt.timeouts_array_size_out, sizeof(purt_get));
- TESTEQUAL(purt_get[0].uid, purt_set[0].uid);
- TESTEQUAL(purt_get[0].min_time_us, purt_set[0].min_time_us);
- TESTEQUAL(purt_get[0].min_pending_time_us,
- purt_set[0].min_pending_time_us);
- TESTEQUAL(purt_get[0].max_pending_time_us,
- purt_set[0].max_pending_time_us);
-
- /* Still 1000 in UID 2 */
- TESTEQUAL(clock_gettime(CLOCK_MONOTONIC, &start), 0);
- TEST(pid = fork(), pid != -1);
- if (pid == 0) {
- TESTEQUAL(setuid(2), 0);
- TESTEQUAL(pread(fd, buffer, sizeof(buffer), 0), -1);
- exit(0);
- }
- TESTNE(wait(&status), -1);
- TESTEQUAL(WEXITSTATUS(status), 0);
- TESTEQUAL(is_close(&start, 1000), 0);
-
- /* Set it to default */
- purt_set[0].max_pending_time_us = UINT32_MAX;
- TESTEQUAL(ioctl(cmd_fd, INCFS_IOC_SET_READ_TIMEOUTS, &srt), 0);
- TESTEQUAL(clock_gettime(CLOCK_MONOTONIC, &start), 0);
- TESTEQUAL(pread(fd, buffer, sizeof(buffer), 0), -1);
- TESTEQUAL(is_close(&start, 1000), 0);
-
- /* Test min read time */
- TESTEQUAL(emit_test_block(mount_dir, &file, 0), 0);
- TESTEQUAL(clock_gettime(CLOCK_MONOTONIC, &start), 0);
- TESTEQUAL(pread(fd, buffer, sizeof(buffer), 0), sizeof(buffer));
- TESTEQUAL(is_close(&start, 1000), 0);
-
- /* Test min pending time */
- purt_set[0].uid = 2;
- TESTEQUAL(ioctl(cmd_fd, INCFS_IOC_SET_READ_TIMEOUTS, &srt), 0);
- TESTEQUAL(clock_gettime(CLOCK_MONOTONIC, &start), 0);
- TEST(pid = fork(), pid != -1);
- if (pid == 0) {
- TESTEQUAL(setuid(2), 0);
- TESTEQUAL(pread(fd, buffer, sizeof(buffer), sizeof(buffer)),
- sizeof(buffer));
- exit(0);
- }
- sleep(1);
- TESTEQUAL(emit_test_block(mount_dir, &file, 1), 0);
- TESTNE(wait(&status), -1);
- TESTEQUAL(WEXITSTATUS(status), 0);
- TESTEQUAL(is_close(&start, 2000), 0);
-
- /* Clear timeouts */
- srt.timeouts_array_size = 0;
- TESTEQUAL(ioctl(cmd_fd, INCFS_IOC_SET_READ_TIMEOUTS, &srt), 0);
- grt.timeouts_array_size = 0;
- TESTEQUAL(ioctl(cmd_fd, INCFS_IOC_GET_READ_TIMEOUTS, &grt), 0);
- TESTEQUAL(grt.timeouts_array_size_out, 0);
-
- result = TEST_SUCCESS;
-out:
- close(fd);
-
- if (pid == 0)
- exit(result);
-
- free(filename);
- close(cmd_fd);
- umount(backing_dir);
- free(backing_dir);
return result;
}
@@ -3495,54 +2691,13 @@ static char *setup_mount_dir()
return mount_dir;
}
-int parse_options(int argc, char *const *argv)
-{
- signed char c;
-
- while ((c = getopt(argc, argv, "f:t:v")) != -1)
- switch (c) {
- case 'f':
- options.file = strtol(optarg, NULL, 10);
- break;
-
- case 't':
- options.test = strtol(optarg, NULL, 10);
- break;
-
- case 'v':
- options.verbose = true;
- break;
-
- default:
- return -EINVAL;
- }
-
- return 0;
-}
-
-struct test_case {
- int (*pfunc)(const char *dir);
- const char *name;
-};
-
-void run_one_test(const char *mount_dir, struct test_case *test_case)
-{
- ksft_print_msg("Running %s\n", test_case->name);
- if (test_case->pfunc(mount_dir) == TEST_SUCCESS)
- ksft_test_result_pass("%s\n", test_case->name);
- else
- ksft_test_result_fail("%s\n", test_case->name);
-}
-
int main(int argc, char *argv[])
{
char *mount_dir = NULL;
+ int fails = 0;
int i;
int fd, count;
- if (parse_options(argc, argv))
- ksft_exit_fail_msg("Bad options\n");
-
// Seed randomness pool for testing on QEMU
// NOTE - this abuses the concept of randomness - do *not* ever do this
// on a machine for production use - the device will think it has good
@@ -3566,7 +2721,10 @@ int main(int argc, char *argv[])
{ \
test, #test \
}
- struct test_case cases[] = {
+ struct {
+ int (*pfunc)(char *dir);
+ const char *name;
+ } cases[] = {
MAKE_TEST(basic_file_ops_test),
MAKE_TEST(cant_touch_index_test),
MAKE_TEST(dynamic_files_and_data_test),
@@ -3579,25 +2737,29 @@ int main(int argc, char *argv[])
MAKE_TEST(read_log_test),
MAKE_TEST(get_blocks_test),
MAKE_TEST(get_hash_blocks_test),
- MAKE_TEST(large_file_test),
- MAKE_TEST(mapped_file_test),
- MAKE_TEST(compatibility_test),
- MAKE_TEST(data_block_count_test),
- MAKE_TEST(hash_block_count_test),
- MAKE_TEST(per_uid_read_timeouts_test),
+ MAKE_TEST(large_file),
};
#undef MAKE_TEST
- if (options.test) {
- if (options.test <= 0 || options.test > ARRAY_SIZE(cases))
- ksft_exit_fail_msg("Invalid test\n");
+ /* Bring back for kernel 5.x */
+ /* ksft_set_plan(ARRAY_SIZE(cases)); */
- run_one_test(mount_dir, &cases[options.test - 1]);
- } else
- for (i = 0; i < ARRAY_SIZE(cases); ++i)
- run_one_test(mount_dir, &cases[i]);
+ for (i = 0; i < ARRAY_SIZE(cases); ++i) {
+ ksft_print_msg("Running %s\n", cases[i].name);
+ if (cases[i].pfunc(mount_dir) == TEST_SUCCESS)
+ ksft_test_result_pass("%s\n", cases[i].name);
+ else {
+ ksft_test_result_fail("%s\n", cases[i].name);
+ fails++;
+ }
+ }
umount2(mount_dir, MNT_FORCE);
rmdir(mount_dir);
- return !ksft_get_fail_cnt() ? ksft_exit_pass() : ksft_exit_fail();
+
+ if (fails > 0)
+ ksft_exit_fail();
+ else
+ ksft_exit_pass();
+ return 0;
}
diff --git a/tools/testing/selftests/filesystems/incfs/utils.c b/tools/testing/selftests/filesystems/incfs/utils.c
index e60b0d36f49d..e194f63ba922 100644
--- a/tools/testing/selftests/filesystems/incfs/utils.c
+++ b/tools/testing/selftests/filesystems/incfs/utils.c
@@ -25,45 +25,6 @@
#define __S_IFREG S_IFREG
#endif
-unsigned int rnd(unsigned int max, unsigned int *seed)
-{
- return rand_r(seed) * ((uint64_t)max + 1) / RAND_MAX;
-}
-
-int remove_dir(const char *dir)
-{
- int err = rmdir(dir);
-
- if (err && errno == ENOTEMPTY) {
- err = delete_dir_tree(dir);
- if (err)
- return err;
- return 0;
- }
-
- if (err && errno != ENOENT)
- return -errno;
-
- return 0;
-}
-
-int drop_caches(void)
-{
- int drop_caches =
- open("/proc/sys/vm/drop_caches", O_WRONLY | O_CLOEXEC);
- int i;
-
- if (drop_caches == -1)
- return -errno;
- i = write(drop_caches, "3", 1);
- close(drop_caches);
-
- if (i != 1)
- return -errno;
-
- return 0;
-}
-
int mount_fs(const char *mount_dir, const char *backing_dir,
int read_timeout_ms)
{
@@ -234,28 +195,14 @@ int open_commands_file(const char *mount_dir)
int open_log_file(const char *mount_dir)
{
- char file[255];
- int fd;
+ char cmd_file[255];
+ int cmd_fd;
- snprintf(file, ARRAY_SIZE(file), "%s/.log", mount_dir);
- fd = open(file, O_RDWR | O_CLOEXEC);
- if (fd < 0)
+ snprintf(cmd_file, ARRAY_SIZE(cmd_file), "%s/.log", mount_dir);
+ cmd_fd = open(cmd_file, O_RDWR | O_CLOEXEC);
+ if (cmd_fd < 0)
perror("Can't open log file");
- return fd;
-}
-
-int open_blocks_written_file(const char *mount_dir)
-{
- char file[255];
- int fd;
-
- snprintf(file, ARRAY_SIZE(file),
- "%s/%s", mount_dir, INCFS_BLOCKS_WRITTEN_FILENAME);
- fd = open(file, O_RDONLY | O_CLOEXEC);
-
- if (fd < 0)
- perror("Can't open blocks_written file");
- return fd;
+ return cmd_fd;
}
int wait_for_pending_reads(int fd, int timeout_ms,
@@ -286,35 +233,7 @@ int wait_for_pending_reads(int fd, int timeout_ms,
return read_res / sizeof(*prs);
}
-int wait_for_pending_reads2(int fd, int timeout_ms,
- struct incfs_pending_read_info2 *prs, int prs_count)
-{
- ssize_t read_res = 0;
-
- if (timeout_ms > 0) {
- int poll_res = 0;
- struct pollfd pollfd = {
- .fd = fd,
- .events = POLLIN
- };
-
- poll_res = poll(&pollfd, 1, timeout_ms);
- if (poll_res < 0)
- return -errno;
- if (poll_res == 0)
- return 0;
- if (!(pollfd.revents | POLLIN))
- return 0;
- }
-
- read_res = read(fd, prs, prs_count * sizeof(*prs));
- if (read_res < 0)
- return -errno;
-
- return read_res / sizeof(*prs);
-}
-
-char *concat_file_name(const char *dir, const char *file)
+char *concat_file_name(const char *dir, char *file)
{
char full_name[FILENAME_MAX] = "";
diff --git a/tools/testing/selftests/filesystems/incfs/utils.h b/tools/testing/selftests/filesystems/incfs/utils.h
index f5ed8dc5f0ff..9af63e4e922c 100644
--- a/tools/testing/selftests/filesystems/incfs/utils.h
+++ b/tools/testing/selftests/filesystems/incfs/utils.h
@@ -18,13 +18,6 @@
#endif
#define SHA256_DIGEST_SIZE 32
-#define INCFS_MAX_MTREE_LEVELS 8
-
-unsigned int rnd(unsigned int max, unsigned int *seed);
-
-int remove_dir(const char *dir);
-
-int drop_caches(void);
int mount_fs(const char *mount_dir, const char *backing_dir,
int read_timeout_ms);
@@ -52,15 +45,10 @@ int open_commands_file(const char *mount_dir);
int open_log_file(const char *mount_dir);
-int open_blocks_written_file(const char *mount_dir);
-
int wait_for_pending_reads(int fd, int timeout_ms,
struct incfs_pending_read_info *prs, int prs_count);
-int wait_for_pending_reads2(int fd, int timeout_ms,
- struct incfs_pending_read_info2 *prs, int prs_count);
-
-char *concat_file_name(const char *dir, const char *file);
+char *concat_file_name(const char *dir, char *file);
void sha256(const char *data, size_t dsize, char *hash);
diff --git a/tools/testing/selftests/net/fib_tests.sh b/tools/testing/selftests/net/fib_tests.sh
index 67048f922ff2..a5ba149761bf 100755
--- a/tools/testing/selftests/net/fib_tests.sh
+++ b/tools/testing/selftests/net/fib_tests.sh
@@ -987,7 +987,6 @@ ipv6_addr_metric_test()
check_route6 "2001:db8:104::1 dev dummy2 proto kernel metric 260"
log_test $? 0 "Set metric with peer route on local side"
- log_test $? 0 "User specified metric on local address"
check_route6 "2001:db8:104::2 dev dummy2 proto kernel metric 260"
log_test $? 0 "Set metric with peer route on peer side"