[chromium-freeworld/f33] Enable aarch64 build

commit ddd3ec6a040e089e856c3ea4b1391af6f9e5e457 Author: Leigh Scott <leigh123linux(a)gmail.com&gt; Date: Fri Sep 3 12:50:19 2021 +0100 Enable aarch64 build chromium-93-ffmpeg-4.4.patch | 36 + chromium-freeworld.spec | 17 +- chromium-update-highway-0.12.2.patch | 4199 ++++++++++++++++++++++++++++ chromium_revert_crap_upstream_commit.patch | 474 ---- 4 files changed, 4246 insertions(+), 480 deletions(-) --- diff --git a/chromium-93-ffmpeg-4.4.patch b/chromium-93-ffmpeg-4.4.patch new file mode 100644 index 0000000..f0ec736 --- /dev/null +++ b/chromium-93-ffmpeg-4.4.patch @@ -0,0 +1,36 @@ +diff --git a/media/filters/ffmpeg_demuxer.cc b/media/filters/ffmpeg_demuxer.cc +index ac4713b07268..492a9a37d096 100644 +--- a/media/filters/ffmpeg_demuxer.cc ++++ b/media/filters/ffmpeg_demuxer.cc +@@ -427,11 +427,11 @@ void FFmpegDemuxerStream::EnqueuePacket(ScopedAVPacket packet) { + scoped_refptr<DecoderBuffer> buffer; + + if (type() == DemuxerStream::TEXT) { +- size_t id_size = 0; ++ int id_size = 0; + uint8_t* id_data = av_packet_get_side_data( + packet.get(), AV_PKT_DATA_WEBVTT_IDENTIFIER, &id_size); + +- size_t settings_size = 0; ++ int settings_size = 0; + uint8_t* settings_data = av_packet_get_side_data( + packet.get(), AV_PKT_DATA_WEBVTT_SETTINGS, &settings_size); + +@@ -443,7 +443,7 @@ void FFmpegDemuxerStream::EnqueuePacket(ScopedAVPacket packet) { + buffer = DecoderBuffer::CopyFrom(packet->data, packet->size, + side_data.data(), side_data.size()); + } else { +- size_t side_data_size = 0; ++ int side_data_size = 0; + uint8_t* side_data = av_packet_get_side_data( + packet.get(), AV_PKT_DATA_MATROSKA_BLOCKADDITIONAL, &side_data_size); + +@@ -504,7 +504,7 @@ void FFmpegDemuxerStream::EnqueuePacket(ScopedAVPacket packet) { + packet->size - data_offset); + } + +- size_t skip_samples_size = 0; ++ int skip_samples_size = 0; + const uint32_t* skip_samples_ptr = + reinterpret_cast<const uint32_t*>(av_packet_get_side_data( + packet.get(), AV_PKT_DATA_SKIP_SAMPLES, &skip_samples_size)); diff --git a/chromium-freeworld.spec b/chromium-freeworld.spec index 228538d..037b0c5 100644 --- a/chromium-freeworld.spec +++ b/chromium-freeworld.spec @@ -25,7 +25,7 @@ ##############################Package Definitions###################################### Name: chromium-freeworld Version: 93.0.4577.63 -Release: 1%{?dist} +Release: 2%{?dist} Summary: Chromium built with all freeworld codecs and VA-API support License: BSD and LGPLv2+ and ASL 2.0 and IJG and MIT and GPLv2+ and ISC and OpenSSL and (MPLv1.1 or GPLv2 or LGPLv2) URL: https://www.chromium.org/Home @@ -147,15 +147,17 @@ Obsoletes: chromium-vaapi < %{version}-%{release} Recommends: libva-utils %global debug_package %{nil} # This build should be only available to amd64 -ExclusiveArch: x86_64 +ExclusiveArch: x86_64 aarch64 # Gentoo patches: Patch201: chromium-93-EnumTable-crash.patch +# Arch Linux patches: +Patch226: chromium-93-ffmpeg-4.4.patch + # Upstream patches: Patch251: chromium-sandbox-syscall-broker-use-struct-kernel_stat.patch Patch252: chromium-sandbox-fix-fstatat-crash.patch -Patch1101: chromium_revert_crap_upstream_commit.patch # Fedora patches: @@ -163,6 +165,7 @@ Patch300: chromium-py3-bootstrap.patch Patch301: chromium-gcc11.patch Patch302: chromium-py3-fixes.patch Patch303: chromium-java-only-allowed-in-android-builds.patch +Patch304: chromium-update-highway-0.12.2.patch Patch1303: chromium-rawhide-gcc-std-max-fix.patch # RPM Fusion patches [free/chromium-freeworld]: @@ -190,6 +193,7 @@ Patch1406: chromium-rpm-fusion-brand.patch %patchset_apply chromium-78-protobuf-RepeatedPtrField-export.patch %patchset_apply chromium-90-ruy-include.patch +%patchset_apply chromium-91-libyuv-aarch64.patch %patchset_apply chromium-93-BluetoothLowEnergyScanFilter-include.patch %patchset_apply chromium-93-ClassProperty-include.patch %patchset_apply chromium-93-ContextSet-permissive.patch @@ -203,8 +207,6 @@ Patch1406: chromium-rpm-fusion-brand.patch # Apply patches up to #1000 from this spec. %autopatch -M1000 -p1 -%patch1101 -R -p1 - # Manually apply patches that need an ifdef %if 0%{?fedora} >= 35 %patch1303 -p1 @@ -679,7 +681,7 @@ appstream-util validate-relax --nonet "%{buildroot}%{_metainfodir}/%{name}.appda %{_datadir}/icons/hicolor/128x128/apps/%{name}.png %{_datadir}/icons/hicolor/256x256/apps/%{name}.png %{_datadir}/icons/hicolor/symbolic/apps/%{name}-symbolic.svg -%{_mandir}/man1/%{name}.1.gz +%{_mandir}/man1/%{name}.1.* %dir %{chromiumdir} %{chromiumdir}/%{name} %{chromiumdir}/chrome-sandbox @@ -704,6 +706,9 @@ appstream-util validate-relax --nonet "%{buildroot}%{_metainfodir}/%{name}.appda %{chromiumdir}/swiftshader/libGLESv2.so #########################################changelogs################################################# %changelog +* Fri Sep 03 2021 Leigh Scott <leigh123linux(a)gmail.com&gt; - 93.0.4577.63-2 +- Enable aarch64 build + * Wed Sep 01 2021 Leigh Scott <leigh123linux(a)gmail.com&gt; - 93.0.4577.63-1 - Update to 93.0.4577.63 diff --git a/chromium-update-highway-0.12.2.patch b/chromium-update-highway-0.12.2.patch new file mode 100644 index 0000000..fca308b --- /dev/null +++ b/chromium-update-highway-0.12.2.patch @@ -0,0 +1,4199 @@ +diff -up chromium-92.0.4515.107/third_party/highway/src/CMakeLists.txt.in.update-highway-0.12.2 chromium-92.0.4515.107/third_party/highway/src/CMakeLists.txt.in +diff -up chromium-92.0.4515.107/third_party/highway/src/CMakeLists.txt.update-highway-0.12.2 chromium-92.0.4515.107/third_party/highway/src/CMakeLists.txt +--- chromium-92.0.4515.107/third_party/highway/src/CMakeLists.txt.update-highway-0.12.2 2021-07-19 14:47:23.000000000 -0400 ++++ chromium-92.0.4515.107/third_party/highway/src/CMakeLists.txt 2021-07-26 17:13:36.158002603 -0400 +@@ -19,7 +19,7 @@ if(POLICY CMP0083) + cmake_policy(SET CMP0083 NEW) + endif() + +-project(hwy VERSION 0.1) ++project(hwy VERSION 0.12.2) # Keep in sync with highway.h version + + set(CMAKE_CXX_STANDARD 11) + set(CMAKE_CXX_EXTENSIONS OFF) +@@ -40,6 +40,8 @@ if (NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE RelWithDebInfo) + endif() + ++set(HWY_CMAKE_ARM7 OFF CACHE BOOL "Set copts for ARMv7 with NEON?") ++ + include(CheckCXXSourceCompiles) + check_cxx_source_compiles( + "int main() { +@@ -51,10 +53,13 @@ check_cxx_source_compiles( + HWY_EMSCRIPTEN + ) + ++set(HWY_CONTRIB_SOURCES ++ hwy/contrib/image/image.cc ++ hwy/contrib/image/image.h ++ hwy/contrib/math/math-inl.h ++) ++ + set(HWY_SOURCES +- contrib/image/image.cc +- contrib/image/image.h +- contrib/math/math-inl.h + hwy/aligned_allocator.cc + hwy/aligned_allocator.h + hwy/base.h +@@ -64,6 +69,7 @@ set(HWY_SOURCES + hwy/nanobenchmark.cc + hwy/nanobenchmark.h + hwy/ops/arm_neon-inl.h ++ hwy/ops/arm_sve-inl.h + hwy/ops/scalar-inl.h + hwy/ops/set_macros-inl.h + hwy/ops/shared-inl.h +@@ -146,13 +152,28 @@ else() + -fno-exceptions + ) + endif() +-endif() ++ ++ if (HWY_CMAKE_ARM7) ++ list(APPEND HWY_FLAGS ++ -march=armv7-a ++ -mfpu=neon-vfpv4 ++ -mfloat-abi=hard # must match the toolchain specified as CXX= ++ -mfp16-format=ieee # required for vcvt_f32_f16 ++ ) ++ endif() # HWY_CMAKE_ARM7 ++ ++endif() # !MSVC + + add_library(hwy STATIC ${HWY_SOURCES}) + target_compile_options(hwy PRIVATE ${HWY_FLAGS}) + set_property(TARGET hwy PROPERTY POSITION_INDEPENDENT_CODE ON) + target_include_directories(hwy PUBLIC ${CMAKE_CURRENT_LIST_DIR}) + ++add_library(hwy_contrib STATIC ${HWY_CONTRIB_SOURCES}) ++target_compile_options(hwy_contrib PRIVATE ${HWY_FLAGS}) ++set_property(TARGET hwy_contrib PROPERTY POSITION_INDEPENDENT_CODE ON) ++target_include_directories(hwy_contrib PUBLIC ${CMAKE_CURRENT_LIST_DIR}) ++ + # -------------------------------------------------------- install library + install(TARGETS hwy + DESTINATION "${CMAKE_INSTALL_LIBDIR}") +@@ -166,9 +187,21 @@ foreach (source ${HWY_SOURCES}) + endif() + endforeach() + +-# Add a pkg-config file for libhwy and the test library. ++install(TARGETS hwy_contrib ++ DESTINATION "${CMAKE_INSTALL_LIBDIR}") ++# Install all the headers keeping the relative path to the current directory ++# when installing them. ++foreach (source ${HWY_CONTRIB_SOURCES}) ++ if ("${source}" MATCHES "\.h$") ++ get_filename_component(dirname "${source}" DIRECTORY) ++ install(FILES "${source}" ++ DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/${dirname}") ++ endif() ++endforeach() ++ ++# Add a pkg-config file for libhwy and the contrib/test libraries. + set(HWY_LIBRARY_VERSION "${CMAKE_PROJECT_VERSION}") +-foreach (pc libhwy.pc libhwy-test.pc) ++foreach (pc libhwy.pc libhwy-contrib.pc libhwy-test.pc) + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/${pc}.in" "${pc}" @ONLY) + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${pc}" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") +@@ -251,8 +284,8 @@ endif() + endif() # HWY_SYSTEM_GTEST + + set(HWY_TEST_FILES +- contrib/image/image_test.cc +- # contrib/math/math_test.cc ++ hwy/contrib/image/image_test.cc ++ # hwy/contrib/math/math_test.cc + hwy/aligned_allocator_test.cc + hwy/base_test.cc + hwy/highway_test.cc +@@ -274,11 +307,16 @@ foreach (TESTFILE IN LISTS HWY_TEST_FILE + get_filename_component(TESTNAME ${TESTFILE} NAME_WE) + add_executable(${TESTNAME} ${TESTFILE}) + target_compile_options(${TESTNAME} PRIVATE ${HWY_FLAGS}) ++ # Test all targets, not just the best/baseline. This changes the default ++ # policy to all-attainable; note that setting -DHWY_COMPILE_* directly can ++ # cause compile errors because only one may be set, and other CMakeLists.txt ++ # that include us may set them. ++ target_compile_options(${TESTNAME} PRIVATE -DHWY_IS_TEST=1) + + if(HWY_SYSTEM_GTEST) +- target_link_libraries(${TESTNAME} hwy GTest::GTest GTest::Main) ++ target_link_libraries(${TESTNAME} hwy hwy_contrib GTest::GTest GTest::Main) + else() +- target_link_libraries(${TESTNAME} hwy gtest gtest_main) ++ target_link_libraries(${TESTNAME} hwy hwy_contrib gtest gtest_main) + endif() + # Output test targets in the test directory. + set_target_properties(${TESTNAME} PROPERTIES PREFIX "tests/") +diff -up chromium-92.0.4515.107/third_party/highway/src/debian/changelog.update-highway-0.12.2 chromium-92.0.4515.107/third_party/highway/src/debian/changelog +diff -up chromium-92.0.4515.107/third_party/highway/src/hwy/aligned_allocator.h.update-highway-0.12.2 chromium-92.0.4515.107/third_party/highway/src/hwy/aligned_allocator.h +--- chromium-92.0.4515.107/third_party/highway/src/hwy/aligned_allocator.h.update-highway-0.12.2 2021-07-19 14:47:23.000000000 -0400 ++++ chromium-92.0.4515.107/third_party/highway/src/hwy/aligned_allocator.h 2021-07-26 17:15:37.281847484 -0400 +@@ -111,6 +111,32 @@ AlignedUniquePtr<T> MakeUniqueAligned(Ar + new (ptr) T(std::forward<Args>(args)...), AlignedDeleter()); + } + ++// Helpers for array allocators (avoids overflow) ++namespace detail { ++ ++// Returns x such that 1u << x == n (if n is a power of two). ++static inline constexpr size_t ShiftCount(size_t n) { ++ return (n <= 1) ? 0 : 1 + ShiftCount(n / 2); ++} ++ ++template <typename T> ++T* AllocateAlignedItems(size_t items, AllocPtr alloc_ptr, void* opaque_ptr) { ++ constexpr size_t size = sizeof(T); ++ ++ constexpr bool is_pow2 = (size & (size - 1)) == 0; ++ constexpr size_t bits = ShiftCount(size); ++ static_assert(!is_pow2 || (1ull << bits) == size, "ShiftCount is incorrect"); ++ ++ const size_t bytes = is_pow2 ? items << bits : items * size; ++ const size_t check = is_pow2 ? bytes >> bits : bytes / size; ++ if (check != items) { ++ return nullptr; // overflowed ++ } ++ return static_cast<T*>(AllocateAlignedBytes(bytes, alloc_ptr, opaque_ptr)); ++} ++ ++} // namespace detail ++ + // Aligned memory equivalent of make_unique<T[]> for array types using the + // custom allocators alloc/free. This function calls the constructor with the + // passed Args... on every created item. The destructor of each element will be +@@ -118,10 +144,11 @@ AlignedUniquePtr<T> MakeUniqueAligned(Ar + template <typename T, typename... Args> + AlignedUniquePtr<T[]> MakeUniqueAlignedArrayWithAlloc( + size_t items, AllocPtr alloc, FreePtr free, void* opaque, Args&&... args) { +- T* ptr = +- static_cast<T*>(AllocateAlignedBytes(items * sizeof(T), alloc, opaque)); +- for (size_t i = 0; i < items; i++) { +- new (ptr + i) T(std::forward<Args>(args)...); ++ T* ptr = detail::AllocateAlignedItems<T>(items, alloc, opaque); ++ if (ptr != nullptr) { ++ for (size_t i = 0; i < items; i++) { ++ new (ptr + i) T(std::forward<Args>(args)...); ++ } + } + return AlignedUniquePtr<T[]>(ptr, AlignedDeleter(free, opaque)); + } +@@ -165,7 +192,7 @@ template <typename T> + AlignedFreeUniquePtr<T[]> AllocateAligned(const size_t items, AllocPtr alloc, + FreePtr free, void* opaque) { + return AlignedFreeUniquePtr<T[]>( +- static_cast<T*>(AllocateAlignedBytes(items * sizeof(T), alloc, opaque)), ++ detail::AllocateAlignedItems<T>(items, alloc, opaque), + AlignedFreer(free, opaque)); + } + +diff -up chromium-92.0.4515.107/third_party/highway/src/hwy/aligned_allocator_test.cc.update-highway-0.12.2 chromium-92.0.4515.107/third_party/highway/src/hwy/aligned_allocator_test.cc +--- chromium-92.0.4515.107/third_party/highway/src/hwy/aligned_allocator_test.cc.update-highway-0.12.2 2021-07-19 14:47:23.000000000 -0400 ++++ chromium-92.0.4515.107/third_party/highway/src/hwy/aligned_allocator_test.cc 2021-07-26 17:16:43.672858709 -0400 +@@ -16,6 +16,7 @@ + + #include <stddef.h> + ++#include <array> + #include <new> + #include <random> + #include <vector> +@@ -87,6 +88,32 @@ TEST(AlignedAllocatorTest, FreeNullptr) + /*opaque_ptr=*/nullptr); + } + ++TEST(AlignedAllocatorTest, Log2) { ++ EXPECT_EQ(0u, detail::ShiftCount(1)); ++ EXPECT_EQ(1u, detail::ShiftCount(2)); ++ EXPECT_EQ(3u, detail::ShiftCount(8)); ++} ++ ++// Allocator returns null when it detects overflow of items * sizeof(T). ++TEST(AlignedAllocatorTest, Overflow) { ++ constexpr size_t max = ~size_t(0); ++ constexpr size_t msb = (max >> 1) + 1; ++ using Size5 = std::array<uint8_t, 5>; ++ using Size10 = std::array<uint8_t, 10>; ++ EXPECT_EQ(nullptr, ++ detail::AllocateAlignedItems<uint32_t>(max / 2, nullptr, nullptr)); ++ EXPECT_EQ(nullptr, ++ detail::AllocateAlignedItems<uint32_t>(max / 3, nullptr, nullptr)); ++ EXPECT_EQ(nullptr, ++ detail::AllocateAlignedItems<Size5>(max / 4, nullptr, nullptr)); ++ EXPECT_EQ(nullptr, ++ detail::AllocateAlignedItems<uint16_t>(msb, nullptr, nullptr)); ++ EXPECT_EQ(nullptr, ++ detail::AllocateAlignedItems<double>(msb + 1, nullptr, nullptr)); ++ EXPECT_EQ(nullptr, ++ detail::AllocateAlignedItems<Size10>(msb / 4, nullptr, nullptr)); ++} ++ + TEST(AlignedAllocatorTest, AllocDefaultPointers) { + const size_t kSize = 7777; + void* ptr = AllocateAlignedBytes(kSize, /*alloc_ptr=*/nullptr, +@@ -215,7 +242,8 @@ TEST(AlignedAllocatorTest, MakeUniqueAli + auto arr = MakeUniqueAlignedArrayWithAlloc<SampleObject<24>>( + 7, FakeAllocator::StaticAlloc, FakeAllocator::StaticFree, &fake_alloc, + &counter); +- // An array shold still only call a single allocation. ++ ASSERT_NE(nullptr, arr.get()); ++ // An array should still only call a single allocation. + EXPECT_EQ(1u, fake_alloc.PendingAllocs()); + EXPECT_EQ(7, counter); + for (size_t i = 0; i < 7; i++) { +diff -up chromium-92.0.4515.107/third_party/highway/src/hwy/base.h.update-highway-0.12.2 chromium-92.0.4515.107/third_party/highway/src/hwy/base.h +--- chromium-92.0.4515.107/third_party/highway/src/hwy/base.h.update-highway-0.12.2 2021-07-19 14:47:23.000000000 -0400 ++++ chromium-92.0.4515.107/third_party/highway/src/hwy/base.h 2021-07-26 17:16:04.753265910 -0400 +@@ -203,6 +203,10 @@ + #define HWY_ARCH_X86_64 0 + #endif + ++#if HWY_ARCH_X86_32 && HWY_ARCH_X86_64 ++#error "Cannot have both x86-32 and x86-64" ++#endif ++ + #if HWY_ARCH_X86_32 || HWY_ARCH_X86_64 + #define HWY_ARCH_X86 1 + #else +@@ -249,9 +253,11 @@ + #define HWY_ARCH_RVV 0 + #endif + ++// It is an error to detect multiple architectures at the same time, but OK to ++// detect none of the above. + #if (HWY_ARCH_X86 + HWY_ARCH_PPC + HWY_ARCH_ARM + HWY_ARCH_WASM + \ +- HWY_ARCH_RVV) != 1 +-#error "Must detect exactly one platform" ++ HWY_ARCH_RVV) > 1 ++#error "Must not detect more than one architecture" + #endif + + //------------------------------------------------------------------------------ +@@ -328,6 +334,12 @@ static constexpr HWY_MAYBE_UNUSED size_t + + // RVV already has a builtin type and the GCC intrinsics require it. + #if HWY_ARCH_RVV && HWY_COMPILER_GCC ++#define HWY_NATIVE_FLOAT16 1 ++#else ++#define HWY_NATIVE_FLOAT16 0 ++#endif ++ ++#if HWY_NATIVE_FLOAT16 + using float16_t = __fp16; + // Clang does not allow __fp16 arguments, but scalar.h requires LaneType + // arguments, so use a wrapper. +@@ -597,7 +609,7 @@ HWY_API size_t PopCount(uint64_t x) { + return static_cast<size_t>(__builtin_popcountll(x)); + #elif HWY_COMPILER_MSVC && HWY_ARCH_X86_64 + return _mm_popcnt_u64(x); +-#elif HWY_COMPILER_MSVC ++#elif HWY_COMPILER_MSVC && HWY_ARCH_X86_32 + return _mm_popcnt_u32(uint32_t(x)) + _mm_popcnt_u32(uint32_t(x >> 32)); + #else + x -= ((x >> 1) & 0x55555555U); +diff -up chromium-92.0.4515.107/third_party/highway/src/hwy/cache_control.h.update-highway-0.12.2 chromium-92.0.4515.107/third_party/highway/src/hwy/cache_control.h +--- chromium-92.0.4515.107/third_party/highway/src/hwy/cache_control.h.update-highway-0.12.2 2021-07-19 14:47:23.000000000 -0400 ++++ chromium-92.0.4515.107/third_party/highway/src/hwy/cache_control.h 2021-07-26 17:16:26.004589594 -0400 +@@ -32,6 +32,14 @@ + #include <emmintrin.h> // SSE2 + #endif + ++// Windows.h #defines these, which causes infinite recursion. Temporarily ++// undefine them in this header; these functions are anyway deprecated. ++// TODO(janwas): remove when these functions are removed. ++#pragma push_macro("LoadFence") ++#pragma push_macro("StoreFence") ++#undef LoadFence ++#undef StoreFence ++ + namespace hwy { + + // Even if N*sizeof(T) is smaller, Stream may write a multiple of this size. +@@ -83,6 +91,17 @@ HWY_INLINE HWY_ATTR_CACHE void FlushCach + #endif + } + ++// Reduces power consumption in spin-loops. No effect on non-x86. ++HWY_INLINE HWY_ATTR_CACHE void Pause() { ++#if HWY_ARCH_X86 && !defined(HWY_DISABLE_CACHE_CONTROL) ++ _mm_pause(); ++#endif ++} ++ + } // namespace hwy + ++// TODO(janwas): remove when these functions are removed. (See above.) ++#pragma pop_macro("StoreFence") ++#pragma pop_macro("LoadFence") ++ + #endif // HIGHWAY_HWY_CACHE_CONTROL_H_ +diff -up chromium-92.0.4515.107/third_party/highway/src/hwy/examples/skeleton.cc.update-highway-0.12.2 chromium-92.0.4515.107/third_party/highway/src/hwy/examples/skeleton.cc +diff -up chromium-92.0.4515.107/third_party/highway/src/hwy/examples/skeleton_test.cc.update-highway-0.12.2 chromium-92.0.4515.107/third_party/highway/src/hwy/examples/skeleton_test.cc +diff -up chromium-92.0.4515.107/third_party/highway/src/hwy/highway.h.update-highway-0.12.2 chromium-92.0.4515.107/third_party/highway/src/hwy/highway.h +--- chromium-92.0.4515.107/third_party/highway/src/hwy/highway.h.update-highway-0.12.2 2021-07-19 14:47:23.000000000 -0400 ++++ chromium-92.0.4515.107/third_party/highway/src/hwy/highway.h 2021-07-26 17:16:58.109078590 -0400 +@@ -25,10 +25,10 @@ + + namespace hwy { + +-// API version (https://semver.org/) ++// API version (https://semver.org/); keep in sync with CMakeLists.txt. + #define HWY_MAJOR 0 + #define HWY_MINOR 12 +-#define HWY_PATCH 0 ++#define HWY_PATCH 2 + + //------------------------------------------------------------------------------ + // Shorthand for descriptors (defined in shared-inl.h) used to select overloads. +@@ -49,7 +49,7 @@ namespace hwy { + HWY_FULL_RECOMPOSER((__VA_ARGS__, HWY_FULL2, HWY_FULL1, )) + #define HWY_FULL(...) HWY_CHOOSE_FULL(__VA_ARGS__())(__VA_ARGS__) + +-// Vector of up to MAX_N lanes. ++// Vector of up to MAX_N lanes. Discouraged, when possible, use Half<> instead. + #define HWY_CAPPED(T, MAX_N) \ + hwy::HWY_NAMESPACE::Simd<T, HWY_MIN(MAX_N, HWY_LANES(T))> + +@@ -75,6 +75,10 @@ namespace hwy { + #define HWY_STATIC_DISPATCH(FUNC_NAME) N_WASM::FUNC_NAME + #elif HWY_STATIC_TARGET == HWY_NEON + #define HWY_STATIC_DISPATCH(FUNC_NAME) N_NEON::FUNC_NAME ++#elif HWY_STATIC_TARGET == HWY_SVE ++#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SVE::FUNC_NAME ++#elif HWY_STATIC_TARGET == HWY_SVE2 ++#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SVE2::FUNC_NAME + #elif HWY_STATIC_TARGET == HWY_PPC8 + #define HWY_STATIC_DISPATCH(FUNC_NAME) N_PPC8::FUNC_NAME + #elif HWY_STATIC_TARGET == HWY_SSE4 +@@ -143,6 +147,18 @@ FunctionCache<RetType, Args...> Function + #define HWY_CHOOSE_NEON(FUNC_NAME) nullptr + #endif + ++#if HWY_TARGETS & HWY_SVE ++#define HWY_CHOOSE_SVE(FUNC_NAME) &N_SVE::FUNC_NAME ++#else ++#define HWY_CHOOSE_SVE(FUNC_NAME) nullptr ++#endif ++ ++#if HWY_TARGETS & HWY_SVE2 ++#define HWY_CHOOSE_SVE2(FUNC_NAME) &N_SVE2::FUNC_NAME ++#else ++#define HWY_CHOOSE_SVE2(FUNC_NAME) nullptr ++#endif ++ + #if HWY_TARGETS & HWY_PPC8 + #define HWY_CHOOSE_PCC8(FUNC_NAME) &N_PPC8::FUNC_NAME + #else +@@ -261,8 +277,11 @@ FunctionCache<RetType, Args...> Function + #elif HWY_TARGET == HWY_AVX3 + #include "hwy/ops/x86_512-inl.h" + #elif HWY_TARGET == HWY_PPC8 ++#error "PPC is not yet supported" + #elif HWY_TARGET == HWY_NEON + #include "hwy/ops/arm_neon-inl.h" ++#elif HWY_TARGET == HWY_SVE || HWY_TARGET == HWY_SVE2 ++#include "hwy/ops/arm_sve-inl.h" + #elif HWY_TARGET == HWY_WASM + #include "hwy/ops/wasm_128-inl.h" + #elif HWY_TARGET == HWY_RVV +diff -up chromium-92.0.4515.107/third_party/highway/src/hwy/nanobenchmark.cc.update-highway-0.12.2 chromium-92.0.4515.107/third_party/highway/src/hwy/nanobenchmark.cc +--- chromium-92.0.4515.107/third_party/highway/src/hwy/nanobenchmark.cc.update-highway-0.12.2 2021-07-19 14:47:23.000000000 -0400 ++++ chromium-92.0.4515.107/third_party/highway/src/hwy/nanobenchmark.cc 2021-07-26 17:17:12.094291603 -0400 +@@ -29,6 +29,22 @@ + #include <string> + #include <vector> + ++#if defined(_WIN32) || defined(_WIN64) ++#ifndef NOMINMAX ++#define NOMINMAX ++#endif // NOMINMAX ++#include <windows.h> ++#endif ++ ++#if defined(__MACH__) ++#include <mach/mach.h> ++#include <mach/mach_time.h> ++#endif ++ ++#if defined(__HAIKU__) ++#include <OS.h> ++#endif ++ + #include "hwy/base.h" + #if HWY_ARCH_PPC + #include <sys/platform/ppc.h> // NOLINT __ppc_get_timebase_freq +@@ -43,114 +59,13 @@ + #endif // HWY_ARCH_X86 + + namespace hwy { +-namespace platform { + namespace { +- +-#if HWY_ARCH_X86 +- +-void Cpuid(const uint32_t level, const uint32_t count, +- uint32_t* HWY_RESTRICT abcd) { +-#if HWY_COMPILER_MSVC +- int regs[4]; +- __cpuidex(regs, level, count); +- for (int i = 0; i < 4; ++i) { +- abcd[i] = regs[i]; +- } +-#else +- uint32_t a; +- uint32_t b; +- uint32_t c; +- uint32_t d; +- __cpuid_count(level, count, a, b, c, d); +- abcd[0] = a; +- abcd[1] = b; +- abcd[2] = c; +- abcd[3] = d; +-#endif +-} +- +-std::string BrandString() { +- char brand_string[49]; +- std::array<uint32_t, 4> abcd; +- +- // Check if brand string is supported (it is on all reasonable Intel/AMD) +- Cpuid(0x80000000U, 0, abcd.data()); +- if (abcd[0] < 0x80000004U) { +- return std::string(); +- } +- +- for (size_t i = 0; i < 3; ++i) { +- Cpuid(static_cast<uint32_t>(0x80000002U + i), 0, abcd.data()); +- memcpy(brand_string + i * 16, abcd.data(), sizeof(abcd)); +- } +- brand_string[48] = 0; +- return brand_string; +-} +- +-// Returns the frequency quoted inside the brand string. This does not +-// account for throttling nor Turbo Boost. +-double NominalClockRate() { +- const std::string& brand_string = BrandString(); +- // Brand strings include the maximum configured frequency. These prefixes are +- // defined by Intel CPUID documentation. +- const char* prefixes[3] = {"MHz", "GHz", "THz"}; +- const double multipliers[3] = {1E6, 1E9, 1E12}; +- for (size_t i = 0; i < 3; ++i) { +- const size_t pos_prefix = brand_string.find(prefixes[i]); +- if (pos_prefix != std::string::npos) { +- const size_t pos_space = brand_string.rfind(' ', pos_prefix - 1); +- if (pos_space != std::string::npos) { +- const std::string digits = +- brand_string.substr(pos_space + 1, pos_prefix - pos_space - 1); +- return std::stod(digits) * multipliers[i]; +- } +- } +- } +- +- return 0.0; +-} +- +-#endif // HWY_ARCH_X86 +- +-} // namespace +- +-// Returns tick rate. Invariant means the tick counter frequency is independent +-// of CPU throttling or sleep. May be expensive, caller should cache the result. +-double InvariantTicksPerSecond() { +-#if HWY_ARCH_PPC +- return __ppc_get_timebase_freq(); +-#elif HWY_ARCH_X86 +- // We assume the TSC is invariant; it is on all recent Intel/AMD CPUs. +- return NominalClockRate(); +-#else +- // Fall back to clock_gettime nanoseconds. +- return 1E9; +-#endif +-} +- +-} // namespace platform +-namespace { +- +-// Prevents the compiler from eliding the computations that led to "output". +-template <class T> +-inline void PreventElision(T&& output) { +-#if HWY_COMPILER_MSVC == 0 +- // Works by indicating to the compiler that "output" is being read and +- // modified. The +r constraint avoids unnecessary writes to memory, but only +- // works for built-in types (typically FuncOutput). +- asm volatile("" : "+r"(output) : : "memory"); +-#else +- // MSVC does not support inline assembly anymore (and never supported GCC's +- // RTL constraints). Self-assignment with #pragma optimize("off") might be +- // expected to prevent elision, but it does not with MSVC 2015. Type-punning +- // with volatile pointers generates inefficient code on MSVC 2017. +- static std::atomic<T> dummy(T{}); +- dummy.store(output, std::memory_order_relaxed); +-#endif +-} +- + namespace timer { + ++// Ticks := platform-specific timer values (CPU cycles on x86). Must be ++// unsigned to guarantee wraparound on overflow. ++using Ticks = uint64_t; ++ + // Start/Stop return absolute timestamps and must be placed immediately before + // and after the region to measure. We provide separate Start/Stop functions + // because they use different fences. +@@ -202,8 +117,8 @@ namespace timer { + + // Returns a 64-bit timestamp in unit of 'ticks'; to convert to seconds, + // divide by InvariantTicksPerSecond. +-inline uint64_t Start64() { +- uint64_t t; ++inline Ticks Start() { ++ Ticks t; + #if HWY_ARCH_PPC + asm volatile("mfspr %0, %1" : "=r"(t) : "i"(268)); + #elif HWY_ARCH_X86 && HWY_COMPILER_MSVC +@@ -228,8 +143,15 @@ inline uint64_t Start64() { + : "rdx", "memory", "cc"); + #elif HWY_ARCH_RVV + asm volatile("rdcycle %0" : "=r"(t)); +-#else +- // Fall back to OS - unsure how to reliably query cntvct_el0 frequency. ++#elif defined(_WIN32) || defined(_WIN64) ++ LARGE_INTEGER counter; ++ (void)QueryPerformanceCounter(&counter); ++ t = counter.QuadPart; ++#elif defined(__MACH__) ++ t = mach_absolute_time(); ++#elif defined(__HAIKU__) ++ t = system_time_nsecs(); // since boot ++#else // POSIX + timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + t = ts.tv_sec * 1000000000LL + ts.tv_nsec; +@@ -237,7 +159,7 @@ inline uint64_t Start64() { + return t; + } + +-inline uint64_t Stop64() { ++inline Ticks Stop() { + uint64_t t; + #if HWY_ARCH_PPC + asm volatile("mfspr %0, %1" : "=r"(t) : "i"(268)); +@@ -261,61 +183,7 @@ inline uint64_t Stop64() { + // "cc" = flags modified by SHL. + : "rcx", "rdx", "memory", "cc"); + #else +- t = Start64(); +-#endif +- return t; +-} +- +-// Returns a 32-bit timestamp with about 4 cycles less overhead than +-// Start64. Only suitable for measuring very short regions because the +-// timestamp overflows about once a second. +-inline uint32_t Start32() { +- uint32_t t; +-#if HWY_ARCH_X86 && HWY_COMPILER_MSVC +- _ReadWriteBarrier(); +- _mm_lfence(); +- _ReadWriteBarrier(); +- t = static_cast<uint32_t>(__rdtsc()); +- _ReadWriteBarrier(); +- _mm_lfence(); +- _ReadWriteBarrier(); +-#elif HWY_ARCH_X86_64 +- asm volatile( +- "lfence\n\t" +- "rdtsc\n\t" +- "lfence" +- : "=a"(t) +- : +- // "memory" avoids reordering. rdx = TSC >> 32. +- : "rdx", "memory"); +-#elif HWY_ARCH_RVV +- asm volatile("rdcycle %0" : "=r"(t)); +-#else +- t = static_cast<uint32_t>(Start64()); +-#endif +- return t; +-} +- +-inline uint32_t Stop32() { +- uint32_t t; +-#if HWY_ARCH_X86 && HWY_COMPILER_MSVC +- _ReadWriteBarrier(); +- unsigned aux; +- t = static_cast<uint32_t>(__rdtscp(&aux)); +- _ReadWriteBarrier(); +- _mm_lfence(); +- _ReadWriteBarrier(); +-#elif HWY_ARCH_X86_64 +- // Use inline asm because __rdtscp generates code to store TSC_AUX (ecx). +- asm volatile( +- "rdtscp\n\t" +- "lfence" +- : "=a"(t) +- : +- // "memory" avoids reordering. rcx = TSC_AUX. rdx = TSC >> 32. +- : "rcx", "rdx", "memory"); +-#else +- t = static_cast<uint32_t>(Stop64()); ++ t = Start(); + #endif + return t; + } +@@ -440,21 +308,130 @@ T MedianAbsoluteDeviation(const T* value + } + + } // namespace robust_statistics ++} // namespace ++namespace platform { ++namespace { + +-// Ticks := platform-specific timer values (CPU cycles on x86). Must be +-// unsigned to guarantee wraparound on overflow. 32 bit timers are faster to +-// read than 64 bit. +-using Ticks = uint32_t; ++// Prevents the compiler from eliding the computations that led to "output". ++template <class T> ++inline void PreventElision(T&& output) { ++#if HWY_COMPILER_MSVC == 0 ++ // Works by indicating to the compiler that "output" is being read and ++ // modified. The +r constraint avoids unnecessary writes to memory, but only ++ // works for built-in types (typically FuncOutput). ++ asm volatile("" : "+r"(output) : : "memory"); ++#else ++ // MSVC does not support inline assembly anymore (and never supported GCC's ++ // RTL constraints). Self-assignment with #pragma optimize("off") might be ++ // expected to prevent elision, but it does not with MSVC 2015. Type-punning ++ // with volatile pointers generates inefficient code on MSVC 2017. ++ static std::atomic<T> dummy(T{}); ++ dummy.store(output, std::memory_order_relaxed); ++#endif ++} ++ ++#if HWY_ARCH_X86 ++ ++void Cpuid(const uint32_t level, const uint32_t count, ++ uint32_t* HWY_RESTRICT abcd) { ++#if HWY_COMPILER_MSVC ++ int regs[4]; ++ __cpuidex(regs, level, count); ++ for (int i = 0; i < 4; ++i) { ++ abcd[i] = regs[i]; ++ } ++#else ++ uint32_t a; ++ uint32_t b; ++ uint32_t c; ++ uint32_t d; ++ __cpuid_count(level, count, a, b, c, d); ++ abcd[0] = a; ++ abcd[1] = b; ++ abcd[2] = c; ++ abcd[3] = d; ++#endif ++} ++ ++std::string BrandString() { ++ char brand_string[49]; ++ std::array<uint32_t, 4> abcd; ++ ++ // Check if brand string is supported (it is on all reasonable Intel/AMD) ++ Cpuid(0x80000000U, 0, abcd.data()); ++ if (abcd[0] < 0x80000004U) { ++ return std::string(); ++ } ++ ++ for (size_t i = 0; i < 3; ++i) { ++ Cpuid(static_cast<uint32_t>(0x80000002U + i), 0, abcd.data()); ++ memcpy(brand_string + i * 16, abcd.data(), sizeof(abcd)); ++ } ++ brand_string[48] = 0; ++ return brand_string; ++} ++ ++// Returns the frequency quoted inside the brand string. This does not ++// account for throttling nor Turbo Boost. ++double NominalClockRate() { ++ const std::string& brand_string = BrandString(); ++ // Brand strings include the maximum configured frequency. These prefixes are ++ // defined by Intel CPUID documentation. ++ const char* prefixes[3] = {"MHz", "GHz", "THz"}; ++ const double multipliers[3] = {1E6, 1E9, 1E12}; ++ for (size_t i = 0; i < 3; ++i) { ++ const size_t pos_prefix = brand_string.find(prefixes[i]); ++ if (pos_prefix != std::string::npos) { ++ const size_t pos_space = brand_string.rfind(' ', pos_prefix - 1); ++ if (pos_space != std::string::npos) { ++ const std::string digits = ++ brand_string.substr(pos_space + 1, pos_prefix - pos_space - 1); ++ return std::stod(digits) * multipliers[i]; ++ } ++ } ++ } ++ ++ return 0.0; ++} ++ ++#endif // HWY_ARCH_X86 ++ ++} // namespace ++ ++double InvariantTicksPerSecond() { ++#if HWY_ARCH_PPC ++ return __ppc_get_timebase_freq(); ++#elif HWY_ARCH_X86 ++ // We assume the TSC is invariant; it is on all recent Intel/AMD CPUs. ++ return NominalClockRate(); ++#elif defined(_WIN32) || defined(_WIN64) ++ LARGE_INTEGER freq; ++ (void)QueryPerformanceFrequency(&freq); ++ return double(freq.QuadPart); ++#elif defined(__MACH__) ++ // https://developer.apple.com/library/mac/qa/qa1398/_index.html ++ mach_timebase_info_data_t timebase; ++ (void)mach_timebase_info(&timebase); ++ return double(timebase.denom) / timebase.numer * 1E9; ++#else ++ // TODO(janwas): ARM? Unclear how to reliably query cntvct_el0 frequency. ++ return 1E9; // Haiku and clock_gettime return nanoseconds. ++#endif ++} + +-// Returns timer overhead / minimum measurable difference. +-Ticks TimerResolution() { ++double Now() { ++ static const double mul = 1.0 / InvariantTicksPerSecond(); ++ return static_cast<double>(timer::Start()) * mul; ++} ++ ++uint64_t TimerResolution() { + // Nested loop avoids exceeding stack/L1 capacity. +- Ticks repetitions[Params::kTimerSamples]; ++ timer::Ticks repetitions[Params::kTimerSamples]; + for (size_t rep = 0; rep < Params::kTimerSamples; ++rep) { +- Ticks samples[Params::kTimerSamples]; ++ timer::Ticks samples[Params::kTimerSamples]; + for (size_t i = 0; i < Params::kTimerSamples; ++i) { +- const Ticks t0 = timer::Start32(); +- const Ticks t1 = timer::Stop32(); ++ const timer::Ticks t0 = timer::Start(); ++ const timer::Ticks t1 = timer::Stop(); + samples[i] = t1 - t0; + } + repetitions[rep] = robust_statistics::Mode(samples); +@@ -462,18 +439,21 @@ Ticks TimerResolution() { + return robust_statistics::Mode(repetitions); + } + +-static const Ticks timer_resolution = TimerResolution(); ++} // namespace platform ++namespace { ++ ++static const timer::Ticks timer_resolution = platform::TimerResolution(); + + // Estimates the expected value of "lambda" values with a variable number of + // samples until the variability "rel_mad" is less than "max_rel_mad". + template <class Lambda> +-Ticks SampleUntilStable(const double max_rel_mad, double* rel_mad, +- const Params& p, const Lambda& lambda) { ++timer::Ticks SampleUntilStable(const double max_rel_mad, double* rel_mad, ++ const Params& p, const Lambda& lambda) { + // Choose initial samples_per_eval based on a single estimated duration. +- Ticks t0 = timer::Start32(); ++ timer::Ticks t0 = timer::Start(); + lambda(); +- Ticks t1 = timer::Stop32(); +- Ticks est = t1 - t0; ++ timer::Ticks t1 = timer::Stop(); ++ timer::Ticks est = t1 - t0; + static const double ticks_per_second = platform::InvariantTicksPerSecond(); + const size_t ticks_per_eval = + static_cast<size_t>(ticks_per_second * p.seconds_per_eval); +@@ -481,21 +461,21 @@ Ticks SampleUntilStable(const double max + est == 0 ? p.min_samples_per_eval : ticks_per_eval / est; + samples_per_eval = std::max(samples_per_eval, p.min_samples_per_eval); + +- std::vector<Ticks> samples; ++ std::vector<timer::Ticks> samples; + samples.reserve(1 + samples_per_eval); + samples.push_back(est); + + // Percentage is too strict for tiny differences, so also allow a small + // absolute "median absolute deviation". +- const Ticks max_abs_mad = (timer_resolution + 99) / 100; ++ const timer::Ticks max_abs_mad = (timer_resolution + 99) / 100; + *rel_mad = 0.0; // ensure initialized + + for (size_t eval = 0; eval < p.max_evals; ++eval, samples_per_eval *= 2) { + samples.reserve(samples.size() + samples_per_eval); + for (size_t i = 0; i < samples_per_eval; ++i) { +- t0 = timer::Start32(); ++ t0 = timer::Start(); + lambda(); +- t1 = timer::Stop32(); ++ t1 = timer::Stop(); + samples.push_back(t1 - t0); + } + +@@ -508,14 +488,14 @@ Ticks SampleUntilStable(const double max + NANOBENCHMARK_CHECK(est != 0); + + // Median absolute deviation (mad) is a robust measure of 'variability'. +- const Ticks abs_mad = robust_statistics::MedianAbsoluteDeviation( ++ const timer::Ticks abs_mad = robust_statistics::MedianAbsoluteDeviation( + samples.data(), samples.size(), est); +- *rel_mad = static_cast<double>(int(abs_mad)) / est; ++ *rel_mad = static_cast<double>(abs_mad) / static_cast<double>(est); + + if (*rel_mad <= max_rel_mad || abs_mad <= max_abs_mad) { + if (p.verbose) { +- printf("%6zu samples => %5u (abs_mad=%4u, rel_mad=%4.2f%%)\n", +- samples.size(), est, abs_mad, *rel_mad * 100.0); ++ printf("%6zu samples => %5zu (abs_mad=%4zu, rel_mad=%4.2f%%)\n", ++ samples.size(), size_t(est), size_t(abs_mad), *rel_mad * 100.0); + } + return est; + } +@@ -539,29 +519,17 @@ InputVec UniqueInputs(const FuncInput* i + return unique; + } + +-// Returns how often we need to call func for sufficient precision, or zero +-// on failure (e.g. the elapsed time is too long for a 32-bit tick count). ++// Returns how often we need to call func for sufficient precision. + size_t NumSkip(const Func func, const uint8_t* arg, const InputVec& unique, + const Params& p) { + // Min elapsed ticks for any input. +- Ticks min_duration = ~0u; ++ timer::Ticks min_duration = ~timer::Ticks(0); + + for (const FuncInput input : unique) { +- // Make sure a 32-bit timer is sufficient. +- const uint64_t t0 = timer::Start64(); +- PreventElision(func(arg, input)); +- const uint64_t t1 = timer::Stop64(); +- const uint64_t elapsed = t1 - t0; +- if (elapsed >= (1ULL << 30)) { +- fprintf(stderr, "Measurement failed: need 64-bit timer for input=%zu\n", +- input); +- return 0; +- } +- + double rel_mad; +- const Ticks total = SampleUntilStable( ++ const timer::Ticks total = SampleUntilStable( + p.target_rel_mad, &rel_mad, p, +- [func, arg, input]() { PreventElision(func(arg, input)); }); ++ [func, arg, input]() { platform::PreventElision(func(arg, input)); }); + min_duration = std::min(min_duration, total - timer_resolution); + } + +@@ -571,8 +539,8 @@ size_t NumSkip(const Func func, const ui + const size_t num_skip = + min_duration == 0 ? 0 : (max_skip + min_duration - 1) / min_duration; + if (p.verbose) { +- printf("res=%u max_skip=%zu min_dur=%u num_skip=%zu\n", timer_resolution, +- max_skip, min_duration, num_skip); ++ printf("res=%zu max_skip=%zu min_dur=%zu num_skip=%zu\n", ++ size_t(timer_resolution), max_skip, size_t(min_duration), num_skip); + } + return num_skip; + } +@@ -637,13 +605,14 @@ void FillSubset(const InputVec& full, co + } + + // Returns total ticks elapsed for all inputs. +-Ticks TotalDuration(const Func func, const uint8_t* arg, const InputVec* inputs, +- const Params& p, double* max_rel_mad) { ++timer::Ticks TotalDuration(const Func func, const uint8_t* arg, ++ const InputVec* inputs, const Params& p, ++ double* max_rel_mad) { + double rel_mad; +- const Ticks duration = ++ const timer::Ticks duration = + SampleUntilStable(p.target_rel_mad, &rel_mad, p, [func, arg, inputs]() { + for (const FuncInput input : *inputs) { +- PreventElision(func(arg, input)); ++ platform::PreventElision(func(arg, input)); + } + }); + *max_rel_mad = std::max(*max_rel_mad, rel_mad); +@@ -657,19 +626,20 @@ HWY_NOINLINE FuncOutput EmptyFunc(const + + // Returns overhead of accessing inputs[] and calling a function; this will + // be deducted from future TotalDuration return values. +-Ticks Overhead(const uint8_t* arg, const InputVec* inputs, const Params& p) { ++timer::Ticks Overhead(const uint8_t* arg, const InputVec* inputs, ++ const Params& p) { + double rel_mad; + // Zero tolerance because repeatability is crucial and EmptyFunc is fast. + return SampleUntilStable(0.0, &rel_mad, p, [arg, inputs]() { + for (const FuncInput input : *inputs) { +- PreventElision(EmptyFunc(arg, input)); ++ platform::PreventElision(EmptyFunc(arg, input)); + } + }); + } + + } // namespace + +-int Unpredictable1() { return timer::Start64() != ~0ULL; } ++int Unpredictable1() { return timer::Start() != ~0ULL; } + + size_t Measure(const Func func, const uint8_t* arg, const FuncInput* inputs, + const size_t num_inputs, Result* results, const Params& p) { +@@ -685,32 +655,35 @@ size_t Measure(const Func func, const ui + ReplicateInputs(inputs, num_inputs, unique.size(), num_skip, p); + InputVec subset(full.size() - num_skip); + +- const Ticks overhead = Overhead(arg, &full, p); +- const Ticks overhead_skip = Overhead(arg, &subset, p); ++ const timer::Ticks overhead = Overhead(arg, &full, p); ++ const timer::Ticks overhead_skip = Overhead(arg, &subset, p); + if (overhead < overhead_skip) { +- fprintf(stderr, "Measurement failed: overhead %u < %u\n", overhead, +- overhead_skip); ++ fprintf(stderr, "Measurement failed: overhead %zu < %zu\n", ++ size_t(overhead), size_t(overhead_skip)); + return 0; + } + + if (p.verbose) { +- printf("#inputs=%5zu,%5zu overhead=%5u,%5u\n", full.size(), subset.size(), +- overhead, overhead_skip); ++ printf("#inputs=%5zu,%5zu overhead=%5zu,%5zu\n", full.size(), subset.size(), ++ size_t(overhead), size_t(overhead_skip)); + } + + double max_rel_mad = 0.0; +- const Ticks total = TotalDuration(func, arg, &full, p, &max_rel_mad); ++ const timer::Ticks total = TotalDuration(func, arg, &full, p, &max_rel_mad); + + for (size_t i = 0; i < unique.size(); ++i) { + FillSubset(full, unique[i], num_skip, &subset); +- const Ticks total_skip = TotalDuration(func, arg, &subset, p, &max_rel_mad); ++ const timer::Ticks total_skip = ++ TotalDuration(func, arg, &subset, p, &max_rel_mad); + + if (total < total_skip) { +- fprintf(stderr, "Measurement failed: total %u < %u\n", total, total_skip); ++ fprintf(stderr, "Measurement failed: total %zu < %zu\n", size_t(total), ++ size_t(total_skip)); + return 0; + } + +- const Ticks duration = (total - overhead) - (total_skip - overhead_skip); ++ const timer::Ticks duration = ++ (total - overhead) - (total_skip - overhead_skip); + results[i].input = unique[i]; + results[i].ticks = static_cast<float>(duration) * mul; + results[i].variability = static_cast<float>(max_rel_mad); +diff -up chromium-92.0.4515.107/third_party/highway/src/hwy/nanobenchmark.h.update-highway-0.12.2 chromium-92.0.4515.107/third_party/highway/src/hwy/nanobenchmark.h +--- chromium-92.0.4515.107/third_party/highway/src/hwy/nanobenchmark.h.update-highway-0.12.2 2021-07-19 14:47:23.000000000 -0400 ++++ chromium-92.0.4515.107/third_party/highway/src/hwy/nanobenchmark.h 2021-07-26 17:17:12.094291603 -0400 +@@ -44,11 +44,6 @@ + // central tendency of the measurement samples with the "half sample mode", + // which is more robust to outliers and skewed data than the mean or median. + +-// WARNING if included from multiple translation units compiled with distinct +-// flags: this header requires textual inclusion and a predefined NB_NAMESPACE +-// macro that is unique to the current compile flags. We must also avoid +-// standard library headers such as vector and functional that define functions. +- + #include <stddef.h> + #include <stdint.h> + +@@ -79,6 +74,16 @@ namespace platform { + // This call may be expensive, callers should cache the result. + double InvariantTicksPerSecond(); + ++// Returns current timestamp [in seconds] relative to an unspecified origin. ++// Features: monotonic (no negative elapsed time), steady (unaffected by system ++// time changes), high-resolution (on the order of microseconds). ++double Now(); ++ ++// Returns ticks elapsed in back to back timer calls, i.e. a function of the ++// timer resolution (minimum measurable difference) and overhead. ++// This call is expensive, callers should cache the result. ++uint64_t TimerResolution(); ++ + } // namespace platform + + // Returns 1, but without the compiler knowing what the value is. This prevents +diff -up chromium-92.0.4515.107/third_party/highway/src/hwy/nanobenchmark_test.cc.update-highway-0.12.2 chromium-92.0.4515.107/third_party/highway/src/hwy/nanobenchmark_test.cc +--- chromium-92.0.4515.107/third_party/highway/src/hwy/nanobenchmark_test.cc.update-highway-0.12.2 2021-07-19 14:47:23.000000000 -0400 ++++ chromium-92.0.4515.107/third_party/highway/src/hwy/nanobenchmark_test.cc 2021-07-26 17:10:30.283171481 -0400 +@@ -15,11 +15,11 @@ + #include "hwy/nanobenchmark.h" + + #include <stdio.h> +-#include <stdlib.h> // strtol +-#include <unistd.h> // sleep + + #include <random> + ++#include "hwy/tests/test_util-inl.h" ++ + namespace hwy { + namespace { + +@@ -31,6 +31,7 @@ FuncOutput Div(const void*, FuncInput in + + template <size_t N> + void MeasureDiv(const FuncInput (&inputs)[N]) { ++ printf("Measuring integer division (output on final two lines)\n"); + Result results[N]; + Params params; + params.max_evals = 4; // avoid test timeout +@@ -66,39 +67,14 @@ void MeasureRandom(const FuncInput (&inp + } + } + +-template <size_t N> +-void EnsureLongMeasurementFails(const FuncInput (&inputs)[N]) { +- printf("Expect a 'measurement failed' below:\n"); +- Result results[N]; +- +- const size_t num_results = Measure( +- [](const void*, const FuncInput input) -> FuncOutput { +- // Loop until the sleep succeeds (not interrupted by signal). We assume +- // >= 512 MHz, so 2 seconds will exceed the 1 << 30 tick safety limit. +- while (sleep(2) != 0) { +- } +- return input; +- }, +- nullptr, inputs, N, results); +- NANOBENCHMARK_CHECK(num_results == 0); +- (void)num_results; +-} +- +-void RunAll(const int argc, char** /*argv*/) { +- // unpredictable == 1 but the compiler doesn't know that. +- const int unpredictable = argc != 999; ++TEST(NanobenchmarkTest, RunAll) { ++ const int unpredictable = Unpredictable1(); // == 1, unknown to compiler. + static const FuncInput inputs[] = {static_cast<FuncInput>(unpredictable) + 2, + static_cast<FuncInput>(unpredictable + 9)}; + + MeasureDiv(inputs); + MeasureRandom(inputs); +- EnsureLongMeasurementFails(inputs); + } + + } // namespace + } // namespace hwy +- +-int main(int argc, char* argv[]) { +- hwy::RunAll(argc, argv); +- return 0; +-} +diff -up chromium-92.0.4515.107/third_party/highway/src/hwy/ops/arm_neon-inl.h.update-highway-0.12.2 chromium-92.0.4515.107/third_party/highway/src/hwy/ops/arm_neon-inl.h +--- chromium-92.0.4515.107/third_party/highway/src/hwy/ops/arm_neon-inl.h.update-highway-0.12.2 2021-07-19 14:47:23.000000000 -0400 ++++ chromium-92.0.4515.107/third_party/highway/src/hwy/ops/arm_neon-inl.h 2021-07-26 17:20:19.294142914 -0400 +@@ -26,6 +26,8 @@ HWY_BEFORE_NAMESPACE(); + namespace hwy { + namespace HWY_NAMESPACE { + ++namespace detail { // for code folding and Raw128 ++ + // Macros used to define single and double function calls for multiple types + // for full and half vectors. These macros are undefined at the end of the file. + +@@ -437,12 +439,14 @@ struct Raw128<int8_t, 1> { + using type = int8x8_t; + }; + ++} // namespace detail ++ + template <typename T> + using Full128 = Simd<T, 16 / sizeof(T)>; + + template <typename T, size_t N = 16 / sizeof(T)> + class Vec128 { +- using Raw = typename Raw128<T, N>::type; ++ using Raw = typename detail::Raw128<T, N>::type; + + public: + HWY_INLINE Vec128() {} +@@ -480,7 +484,8 @@ class Vec128 { + // FF..FF or 0, also for floating-point - see README. + template <typename T, size_t N = 16 / sizeof(T)> + class Mask128 { +- using Raw = typename Raw128<T, N>::type; ++ // ARM C Language Extensions return and expect unsigned type. ++ using Raw = typename detail::Raw128<MakeUnsigned<T>, N>::type; + + public: + HWY_INLINE Mask128() {} +@@ -664,15 +669,25 @@ template <typename T, size_t N> + HWY_INLINE Vec128<T, N> Undefined(Simd<T, N> /*d*/) { + HWY_DIAGNOSTICS(push) + HWY_DIAGNOSTICS_OFF(disable : 4701, ignored "-Wuninitialized") +- typename Raw128<T, N>::type a; ++ typename detail::Raw128<T, N>::type a; + return Vec128<T, N>(a); + HWY_DIAGNOSTICS(pop) + } + +-// ------------------------------ Extract lane ++// Returns a vector with lane i=[0, N) set to "first" + i. ++template <typename T, size_t N, typename T2> ++Vec128<T, N> Iota(const Simd<T, N> d, const T2 first) { ++ HWY_ALIGN T lanes[16 / sizeof(T)]; ++ for (size_t i = 0; i < 16 / sizeof(T); ++i) { ++ lanes[i] = static_cast<T>(first + static_cast<T2>(i)); ++ } ++ return Load(d, lanes); ++} ++ ++// ------------------------------ GetLane + + HWY_INLINE uint8_t GetLane(const Vec128<uint8_t, 16> v) { +- return vget_lane_u8(vget_low_u8(v.raw), 0); ++ return vgetq_lane_u8(v.raw, 0); + } + template <size_t N> + HWY_INLINE uint8_t GetLane(const Vec128<uint8_t, N> v) { +@@ -680,7 +695,7 @@ HWY_INLINE uint8_t GetLane(const Vec128< + } + + HWY_INLINE int8_t GetLane(const Vec128<int8_t, 16> v) { +- return vget_lane_s8(vget_low_s8(v.raw), 0); ++ return vgetq_lane_s8(v.raw, 0); + } + template <size_t N> + HWY_INLINE int8_t GetLane(const Vec128<int8_t, N> v) { +@@ -688,7 +703,7 @@ HWY_INLINE int8_t GetLane(const Vec128<i + } + + HWY_INLINE uint16_t GetLane(const Vec128<uint16_t, 8> v) { +- return vget_lane_u16(vget_low_u16(v.raw), 0); ++ return vgetq_lane_u16(v.raw, 0); + } + template <size_t N> + HWY_INLINE uint16_t GetLane(const Vec128<uint16_t, N> v) { +@@ -696,7 +711,7 @@ HWY_INLINE uint16_t GetLane(const Vec128 + } + + HWY_INLINE int16_t GetLane(const Vec128<int16_t, 8> v) { +- return vget_lane_s16(vget_low_s16(v.raw), 0); ++ return vgetq_lane_s16(v.raw, 0); + } + template <size_t N> + HWY_INLINE int16_t GetLane(const Vec128<int16_t, N> v) { +@@ -704,7 +719,7 @@ HWY_INLINE int16_t GetLane(const Vec128< + } + + HWY_INLINE uint32_t GetLane(const Vec128<uint32_t, 4> v) { +- return vget_lane_u32(vget_low_u32(v.raw), 0); ++ return vgetq_lane_u32(v.raw, 0); + } + template <size_t N> + HWY_INLINE uint32_t GetLane(const Vec128<uint32_t, N> v) { +@@ -712,7 +727,7 @@ HWY_INLINE uint32_t GetLane(const Vec128 + } + + HWY_INLINE int32_t GetLane(const Vec128<int32_t, 4> v) { +- return vget_lane_s32(vget_low_s32(v.raw), 0); ++ return vgetq_lane_s32(v.raw, 0); + } + template <size_t N> + HWY_INLINE int32_t GetLane(const Vec128<int32_t, N> v) { +@@ -720,20 +735,20 @@ HWY_INLINE int32_t GetLane(const Vec128< + } + + HWY_INLINE uint64_t GetLane(const Vec128<uint64_t, 2> v) { +- return vget_lane_u64(vget_low_u64(v.raw), 0); ++ return vgetq_lane_u64(v.raw, 0); + } + HWY_INLINE uint64_t GetLane(const Vec128<uint64_t, 1> v) { + return vget_lane_u64(v.raw, 0); + } + HWY_INLINE int64_t GetLane(const Vec128<int64_t, 2> v) { +- return vget_lane_s64(vget_low_s64(v.raw), 0); ++ return vgetq_lane_s64(v.raw, 0); + } + HWY_INLINE int64_t GetLane(const Vec128<int64_t, 1> v) { + return vget_lane_s64(v.raw, 0); + } + + HWY_INLINE float GetLane(const Vec128<float, 4> v) { +- return vget_lane_f32(vget_low_f32(v.raw), 0); ++ return vgetq_lane_f32(v.raw, 0); + } + HWY_INLINE float GetLane(const Vec128<float, 2> v) { + return vget_lane_f32(v.raw, 0); +@@ -743,7 +758,7 @@ HWY_INLINE float GetLane(const Vec128<fl + } + #if HWY_ARCH_ARM_A64 + HWY_INLINE double GetLane(const Vec128<double, 2> v) { +- return vget_lane_f64(vget_low_f64(v.raw), 0); ++ return vgetq_lane_f64(v.raw, 0); + } + HWY_INLINE double GetLane(const Vec128<double, 1> v) { + return vget_lane_f64(v.raw, 0); +@@ -785,8 +800,6 @@ HWY_NEON_DEF_FUNCTION_INT_64(SaturatedSu + // ------------------------------ Average + + // Returns (a + b + 1) / 2 +- +-// Unsigned + HWY_NEON_DEF_FUNCTION_UINT_8(AverageRound, vrhadd, _, 2) + HWY_NEON_DEF_FUNCTION_UINT_16(AverageRound, vrhadd, _, 2) + +@@ -802,6 +815,7 @@ HWY_INLINE Vec128<int16_t> Abs(const Vec + HWY_INLINE Vec128<int32_t> Abs(const Vec128<int32_t> v) { + return Vec128<int32_t>(vabsq_s32(v.raw)); + } ++// i64 is implemented after BroadcastSignBit. + HWY_INLINE Vec128<float> Abs(const Vec128<float> v) { + return Vec128<float>(vabsq_f32(v.raw)); + } +@@ -1184,21 +1198,34 @@ HWY_INLINE Vec128<float, N> ApproximateR + #if HWY_ARCH_ARM_A64 + HWY_NEON_DEF_FUNCTION_ALL_FLOATS(operator/, vdiv, _, 2) + #else +-// Emulated with approx reciprocal + Newton-Raphson + mul ++// Not defined on armv7: approximate ++namespace detail { ++ ++HWY_INLINE Vec128<float> ReciprocalNewtonRaphsonStep( ++ const Vec128<float> recip, const Vec128<float> divisor) { ++ return Vec128<float>(vrecpsq_f32(recip.raw, divisor.raw)); ++} ++template <size_t N> ++HWY_INLINE Vec128<float, N> ReciprocalNewtonRaphsonStep( ++ const Vec128<float, N> recip, Vec128<float, N> divisor) { ++ return Vec128<float, N>(vrecps_f32(recip.raw, divisor.raw)); ++} ++ ++} // namespace detail ++ + template <size_t N> + HWY_INLINE Vec128<float, N> operator/(const Vec128<float, N> a, + const Vec128<float, N> b) { + auto x = ApproximateReciprocal(b); +- // Newton-Raphson on 1/x - b +- const auto two = Set(Simd<float, N>(), 2); +- x = x * (two - b * x); +- x = x * (two - b * x); +- x = x * (two - b * x); ++ x *= detail::ReciprocalNewtonRaphsonStep(x, b); ++ x *= detail::ReciprocalNewtonRaphsonStep(x, b); ++ x *= detail::ReciprocalNewtonRaphsonStep(x, b); + return a * x; + } + #endif + +-// Absolute value of difference. ++// ------------------------------ Absolute value of difference. ++ + HWY_INLINE Vec128<float> AbsDiff(const Vec128<float> a, const Vec128<float> b) { + return Vec128<float>(vabdq_f32(a.raw, b.raw)); + } +@@ -1312,7 +1339,7 @@ HWY_INLINE Vec128<double, N> NegMulSub(c + } + #endif + +-// ------------------------------ Floating-point square root ++// ------------------------------ Floating-point square root (IfThenZeroElse) + + // Approximate reciprocal square root + HWY_INLINE Vec128<float> ApproximateReciprocalSqrt(const Vec128<float> v) { +@@ -1328,77 +1355,33 @@ HWY_INLINE Vec128<float, N> ApproximateR + #if HWY_ARCH_ARM_A64 + HWY_NEON_DEF_FUNCTION_ALL_FLOATS(Sqrt, vsqrt, _, 1) + #else +-// Not defined on armv7: emulate with approx reciprocal sqrt + Goldschmidt. +-template <size_t N> +-HWY_INLINE Vec128<float, N> Sqrt(const Vec128<float, N> v) { +- auto b = v; +- auto Y = ApproximateReciprocalSqrt(v); +- auto x = v * Y; +- const auto half = Set(Simd<float, N>(), 0.5); +- const auto oneandhalf = Set(Simd<float, N>(), 1.5); +- for (size_t i = 0; i < 3; i++) { +- b = b * Y * Y; +- Y = oneandhalf - half * b; +- x = x * Y; +- } +- return IfThenZeroElse(v == Zero(Simd<float, N>()), x); +-} +-#endif +- +-// ================================================== COMPARE +- +-// Comparisons fill a lane with 1-bits if the condition is true, else 0. ++namespace detail { + +-template <typename TFrom, typename TTo, size_t N> +-HWY_API Mask128<TTo, N> RebindMask(Simd<TTo, N> /*tag*/, Mask128<TFrom, N> m) { +- static_assert(sizeof(TFrom) == sizeof(TTo), "Must have same size"); +- return Mask128<TTo, N>{m.raw}; ++HWY_INLINE Vec128<float> ReciprocalSqrtStep(const Vec128<float> root, ++ const Vec128<float> recip) { ++ return Vec128<float>(vrsqrtsq_f32(root.raw, recip.raw)); ++} ++template <size_t N> ++HWY_INLINE Vec128<float, N> ReciprocalSqrtStep(const Vec128<float, N> root, ++ Vec128<float, N> recip) { ++ return Vec128<float, N>(vrsqrts_f32(root.raw, recip.raw)); + } + +-#define HWY_NEON_BUILD_TPL_HWY_COMPARE +-#define HWY_NEON_BUILD_RET_HWY_COMPARE(type, size) Mask128<type, size> +-#define HWY_NEON_BUILD_PARAM_HWY_COMPARE(type, size) \ +- const Vec128<type, size> a, const Vec128<type, size> b +-#define HWY_NEON_BUILD_ARG_HWY_COMPARE a.raw, b.raw +- +-// ------------------------------ Equality +-HWY_NEON_DEF_FUNCTION_ALL_FLOATS(operator==, vceq, _, HWY_COMPARE) +-#if HWY_ARCH_ARM_A64 +-HWY_NEON_DEF_FUNCTION_INTS_UINTS(operator==, vceq, _, HWY_COMPARE) +-#else +-// No 64-bit comparisons on armv7: emulate them below, after Shuffle2301. +-HWY_NEON_DEF_FUNCTION_INT_8_16_32(operator==, vceq, _, HWY_COMPARE) +-HWY_NEON_DEF_FUNCTION_UINT_8_16_32(operator==, vceq, _, HWY_COMPARE) +-#endif ++} // namespace detail + +-// ------------------------------ Strict inequality ++// Not defined on armv7: approximate ++template <size_t N> ++HWY_INLINE Vec128<float, N> Sqrt(const Vec128<float, N> v) { ++ auto recip = ApproximateReciprocalSqrt(v); + +-// Signed/float < (no unsigned) +-#if HWY_ARCH_ARM_A64 +-HWY_NEON_DEF_FUNCTION_INTS(operator<, vclt, _, HWY_COMPARE) +-#else +-HWY_NEON_DEF_FUNCTION_INT_8_16_32(operator<, vclt, _, HWY_COMPARE) +-#endif +-HWY_NEON_DEF_FUNCTION_ALL_FLOATS(operator<, vclt, _, HWY_COMPARE) ++ recip *= detail::ReciprocalSqrtStep(v * recip, recip); ++ recip *= detail::ReciprocalSqrtStep(v * recip, recip); ++ recip *= detail::ReciprocalSqrtStep(v * recip, recip); + +-// Signed/float > (no unsigned) +-#if HWY_ARCH_ARM_A64 +-HWY_NEON_DEF_FUNCTION_INTS(operator>, vcgt, _, HWY_COMPARE) +-#else +-HWY_NEON_DEF_FUNCTION_INT_8_16_32(operator>, vcgt, _, HWY_COMPARE) ++ const auto root = v * recip; ++ return IfThenZeroElse(v == Zero(Simd<float, N>()), root); ++} + #endif +-HWY_NEON_DEF_FUNCTION_ALL_FLOATS(operator>, vcgt, _, HWY_COMPARE) +- +-// ------------------------------ Weak inequality +- +-// Float <= >= +-HWY_NEON_DEF_FUNCTION_ALL_FLOATS(operator<=, vcle, _, HWY_COMPARE) +-HWY_NEON_DEF_FUNCTION_ALL_FLOATS(operator>=, vcge, _, HWY_COMPARE) +- +-#undef HWY_NEON_BUILD_TPL_HWY_COMPARE +-#undef HWY_NEON_BUILD_RET_HWY_COMPARE +-#undef HWY_NEON_BUILD_PARAM_HWY_COMPARE +-#undef HWY_NEON_BUILD_ARG_HWY_COMPARE + + // ================================================== LOGICAL + +@@ -1407,13 +1390,16 @@ HWY_NEON_DEF_FUNCTION_ALL_FLOATS(operato + // There is no 64-bit vmvn, so cast instead of using HWY_NEON_DEF_FUNCTION. + template <typename T> + HWY_INLINE Vec128<T> Not(const Vec128<T> v) { +- const Full128<uint8_t> d8; +- return Vec128<T>(vmvnq_u8(BitCast(d8, v).raw)); ++ const Full128<T> d; ++ const Repartition<uint8_t, decltype(d)> d8; ++ return BitCast(d, Vec128<uint8_t>(vmvnq_u8(BitCast(d8, v).raw))); + } + template <typename T, size_t N, HWY_IF_LE64(T, N)> + HWY_INLINE Vec128<T, N> Not(const Vec128<T, N> v) { +- const Repartition<uint8_t, Simd<T, N>> d8; +- return Vec128<T, N>(vmvn_u8(BitCast(d8, v).raw)); ++ const Simd<T, N> d; ++ const Repartition<uint8_t, decltype(d)> d8; ++ using V8 = decltype(Zero(d8)); ++ return BitCast(d, V8(vmvn_u8(BitCast(d8, v).raw))); + } + + // ------------------------------ And +@@ -1513,33 +1499,38 @@ HWY_API Vec128<T, N> BroadcastSignBit(co + return ShiftRight<sizeof(T) * 8 - 1>(v); + } + +-// ------------------------------ Make mask ++// ================================================== MASK + +-template <typename T, size_t N> +-HWY_INLINE Mask128<T, N> TestBit(Vec128<T, N> v, Vec128<T, N> bit) { +- static_assert(!hwy::IsFloat<T>(), "Only integer vectors supported"); +- return (v & bit) == bit; +-} ++// ------------------------------ To/from vector + +-// Mask and Vec are the same (true = FF..FF). ++// Mask and Vec have the same representation (true = FF..FF). + template <typename T, size_t N> + HWY_INLINE Mask128<T, N> MaskFromVec(const Vec128<T, N> v) { +- return Mask128<T, N>(v.raw); ++ const Simd<MakeUnsigned<T>, N> du; ++ return Mask128<T, N>(BitCast(du, v).raw); + } + ++// DEPRECATED + template <typename T, size_t N> + HWY_INLINE Vec128<T, N> VecFromMask(const Mask128<T, N> v) { +- return Vec128<T, N>(v.raw); ++ return BitCast(Simd<T, N>(), Vec128<MakeUnsigned<T>, N>(v.raw)); + } + + template <typename T, size_t N> +-HWY_INLINE Vec128<T, N> VecFromMask(Simd<T, N> /* tag */, +- const Mask128<T, N> v) { +- return Vec128<T, N>(v.raw); ++HWY_INLINE Vec128<T, N> VecFromMask(Simd<T, N> d, const Mask128<T, N> v) { ++ return BitCast(d, Vec128<MakeUnsigned<T>, N>(v.raw)); ++} ++ ++// ------------------------------ RebindMask ++ ++template <typename TFrom, typename TTo, size_t N> ++HWY_API Mask128<TTo, N> RebindMask(Simd<TTo, N> dto, Mask128<TFrom, N> m) { ++ static_assert(sizeof(TFrom) == sizeof(TTo), "Must have same size"); ++ return MaskFromVec(BitCast(dto, VecFromMask(Simd<TFrom, N>(), m))); + } + +-// IfThenElse(mask, yes, no) +-// Returns mask ? b : a. ++// ------------------------------ IfThenElse(mask, yes, no) = mask ? b : a. ++ + #define HWY_NEON_BUILD_TPL_HWY_IF + #define HWY_NEON_BUILD_RET_HWY_IF(type, size) Vec128<type, size> + #define HWY_NEON_BUILD_PARAM_HWY_IF(type, size) \ +@@ -1574,7 +1565,6 @@ HWY_INLINE Vec128<T, N> ZeroIfNegative(V + return Max(zero, v); + } + +- + // ------------------------------ Mask logical + + template <typename T, size_t N> +@@ -1607,30 +1597,183 @@ HWY_API Mask128<T, N> Xor(const Mask128< + return MaskFromVec(Xor(VecFromMask(d, a), VecFromMask(d, b))); + } + +-// ------------------------------ Min (IfThenElse, BroadcastSignBit) ++// ================================================== COMPARE + +-namespace detail { ++// Comparisons fill a lane with 1-bits if the condition is true, else 0. ++ ++// ------------------------------ Shuffle2301 (for i64 compares) ++ ++// Swap 32-bit halves in 64-bits ++HWY_INLINE Vec128<uint32_t, 2> Shuffle2301(const Vec128<uint32_t, 2> v) { ++ return Vec128<uint32_t, 2>(vrev64_u32(v.raw)); ++} ++HWY_INLINE Vec128<int32_t, 2> Shuffle2301(const Vec128<int32_t, 2> v) { ++ return Vec128<int32_t, 2>(vrev64_s32(v.raw)); ++} ++HWY_INLINE Vec128<float, 2> Shuffle2301(const Vec128<float, 2> v) { ++ return Vec128<float, 2>(vrev64_f32(v.raw)); ++} ++HWY_INLINE Vec128<uint32_t> Shuffle2301(const Vec128<uint32_t> v) { ++ return Vec128<uint32_t>(vrev64q_u32(v.raw)); ++} ++HWY_INLINE Vec128<int32_t> Shuffle2301(const Vec128<int32_t> v) { ++ return Vec128<int32_t>(vrev64q_s32(v.raw)); ++} ++HWY_INLINE Vec128<float> Shuffle2301(const Vec128<float> v) { ++ return Vec128<float>(vrev64q_f32(v.raw)); ++} ++ ++#define HWY_NEON_BUILD_TPL_HWY_COMPARE ++#define HWY_NEON_BUILD_RET_HWY_COMPARE(type, size) Mask128<type, size> ++#define HWY_NEON_BUILD_PARAM_HWY_COMPARE(type, size) \ ++ const Vec128<type, size> a, const Vec128<type, size> b ++#define HWY_NEON_BUILD_ARG_HWY_COMPARE a.raw, b.raw + ++// ------------------------------ Equality ++HWY_NEON_DEF_FUNCTION_ALL_FLOATS(operator==, vceq, _, HWY_COMPARE) + #if HWY_ARCH_ARM_A64 ++HWY_NEON_DEF_FUNCTION_INTS_UINTS(operator==, vceq, _, HWY_COMPARE) ++#else ++// No 64-bit comparisons on armv7: emulate them below, after Shuffle2301. ++HWY_NEON_DEF_FUNCTION_INT_8_16_32(operator==, vceq, _, HWY_COMPARE) ++HWY_NEON_DEF_FUNCTION_UINT_8_16_32(operator==, vceq, _, HWY_COMPARE) ++#endif + +-HWY_INLINE Vec128<uint64_t> Gt(Vec128<uint64_t> a, Vec128<uint64_t> b) { +- return Vec128<uint64_t>(vcgtq_u64(a.raw, b.raw)); ++// ------------------------------ Strict inequality (signed, float) ++#if HWY_ARCH_ARM_A64 ++HWY_NEON_DEF_FUNCTION_INTS(operator<, vclt, _, HWY_COMPARE) ++#else ++HWY_NEON_DEF_FUNCTION_INT_8_16_32(operator<, vclt, _, HWY_COMPARE) ++#endif ++HWY_NEON_DEF_FUNCTION_ALL_FLOATS(operator<, vclt, _, HWY_COMPARE) ++ ++// ------------------------------ Weak inequality (float) ++HWY_NEON_DEF_FUNCTION_ALL_FLOATS(operator<=, vcle, _, HWY_COMPARE) ++ ++#undef HWY_NEON_BUILD_TPL_HWY_COMPARE ++#undef HWY_NEON_BUILD_RET_HWY_COMPARE ++#undef HWY_NEON_BUILD_PARAM_HWY_COMPARE ++#undef HWY_NEON_BUILD_ARG_HWY_COMPARE ++ ++// ------------------------------ ARMv7 i64 compare (Shuffle2301, Eq) ++ ++#if HWY_ARCH_ARM_V7 ++ ++template <size_t N> ++HWY_INLINE Mask128<int64_t, N> operator==(const Vec128<int64_t, N> a, ++ const Vec128<int64_t, N> b) { ++ const Simd<int32_t, N * 2> d32; ++ const Simd<int64_t, N> d64; ++ const auto cmp32 = VecFromMask(d32, Eq(BitCast(d32, a), BitCast(d32, b))); ++ const auto cmp64 = cmp32 & Shuffle2301(cmp32); ++ return MaskFromVec(BitCast(d64, cmp64)); + } +-HWY_INLINE Vec128<uint64_t, 1> Gt(Vec128<uint64_t, 1> a, +- Vec128<uint64_t, 1> b) { +- return Vec128<uint64_t, 1>(vcgt_u64(a.raw, b.raw)); ++ ++template <size_t N> ++HWY_INLINE Mask128<uint64_t, N> operator==(const Vec128<uint64_t, N> a, ++ const Vec128<uint64_t, N> b) { ++ const Simd<uint32_t, N * 2> d32; ++ const Simd<uint64_t, N> d64; ++ const auto cmp32 = VecFromMask(d32, Eq(BitCast(d32, a), BitCast(d32, b))); ++ const auto cmp64 = cmp32 & Shuffle2301(cmp32); ++ return MaskFromVec(BitCast(d64, cmp64)); + } + +-HWY_INLINE Vec128<int64_t> Gt(Vec128<int64_t> a, Vec128<int64_t> b) { +- return Vec128<int64_t>(vcgtq_s64(a.raw, b.raw)); ++HWY_INLINE Mask128<int64_t> operator<(const Vec128<int64_t> a, ++ const Vec128<int64_t> b) { ++ const int64x2_t sub = vqsubq_s64(a.raw, b.raw); ++ return MaskFromVec(BroadcastSignBit(Vec128<int64_t>(sub))); + } +-HWY_INLINE Vec128<int64_t, 1> Gt(Vec128<int64_t, 1> a, Vec128<int64_t, 1> b) { +- return Vec128<int64_t, 1>(vcgt_s64(a.raw, b.raw)); ++HWY_INLINE Mask128<int64_t, 1> operator<(const Vec128<int64_t, 1> a, ++ const Vec128<int64_t, 1> b) { ++ const int64x1_t sub = vqsub_s64(a.raw, b.raw); ++ return MaskFromVec(BroadcastSignBit(Vec128<int64_t, 1>(sub))); + } + + #endif + +-} // namespace detail ++// ------------------------------ Reversed comparisons ++ ++template <typename T, size_t N> ++HWY_API Mask128<T, N> operator>(Vec128<T, N> a, Vec128<T, N> b) { ++ return operator<(b, a); ++} ++template <typename T, size_t N> ++HWY_API Mask128<T, N> operator>=(Vec128<T, N> a, Vec128<T, N> b) { ++ return operator<=(b, a); ++} ++ ++// ------------------------------ FirstN (Iota, Lt) ++ ++template <typename T, size_t N> ++HWY_API Mask128<T, N> FirstN(const Simd<T, N> d, size_t num) { ++ const RebindToSigned<decltype(d)> di; // Signed comparisons are cheaper. ++ return RebindMask(d, Iota(di, 0) < Set(di, static_cast<MakeSigned<T>>(num))); ++} ++ ++// ------------------------------ TestBit (Eq) ++ ++#define HWY_NEON_BUILD_TPL_HWY_TESTBIT ++#define HWY_NEON_BUILD_RET_HWY_TESTBIT(type, size) Mask128<type, size> ++#define HWY_NEON_BUILD_PARAM_HWY_TESTBIT(type, size) \ ++ Vec128<type, size> v, Vec128<type, size> bit ++#define HWY_NEON_BUILD_ARG_HWY_TESTBIT v.raw, bit.raw ++ ++#if HWY_ARCH_ARM_A64 ++HWY_NEON_DEF_FUNCTION_INTS_UINTS(TestBit, vtst, _, HWY_TESTBIT) ++#else ++// No 64-bit versions on armv7 ++HWY_NEON_DEF_FUNCTION_UINT_8_16_32(TestBit, vtst, _, HWY_TESTBIT) ++HWY_NEON_DEF_FUNCTION_INT_8_16_32(TestBit, vtst, _, HWY_TESTBIT) ++ ++template <size_t N> ++HWY_INLINE Mask128<uint64_t, N> TestBit(Vec128<uint64_t, N> v, ++ Vec128<uint64_t, N> bit) { ++ return (v & bit) == bit; ++} ++template <size_t N> ++HWY_INLINE Mask128<int64_t, N> TestBit(Vec128<int64_t, N> v, ++ Vec128<int64_t, N> bit) { ++ return (v & bit) == bit; ++} ++ ++#endif ++#undef HWY_NEON_BUILD_TPL_HWY_TESTBIT ++#undef HWY_NEON_BUILD_RET_HWY_TESTBIT ++#undef HWY_NEON_BUILD_PARAM_HWY_TESTBIT ++#undef HWY_NEON_BUILD_ARG_HWY_TESTBIT ++ ++// ------------------------------ Abs i64 (IfThenElse, BroadcastSignBit) ++HWY_INLINE Vec128<int64_t> Abs(const Vec128<int64_t> v) { ++#if HWY_ARCH_ARM_A64 ++ return Vec128<int64_t>(vabsq_s64(v.raw)); ++#else ++ const auto zero = Zero(Full128<int64_t>()); ++ return IfThenElse(MaskFromVec(BroadcastSignBit(v)), zero - v, v); ++#endif ++} ++HWY_INLINE Vec128<int64_t, 1> Abs(const Vec128<int64_t, 1> v) { ++#if HWY_ARCH_ARM_A64 ++ return Vec128<int64_t, 1>(vabs_s64(v.raw)); ++#else ++ const auto zero = Zero(Simd<int64_t, 1>()); ++ return IfThenElse(MaskFromVec(BroadcastSignBit(v)), zero - v, v); ++#endif ++} ++ ++// ------------------------------ Min (IfThenElse, BroadcastSignBit) ++ ++#if HWY_ARCH_ARM_A64 ++ ++HWY_INLINE Mask128<uint64_t> operator<(Vec128<uint64_t> a, Vec128<uint64_t> b) { ++ return Mask128<uint64_t>(vcltq_u64(a.raw, b.raw)); ++} ++HWY_INLINE Mask128<uint64_t, 1> operator<(Vec128<uint64_t, 1> a, ++ Vec128<uint64_t, 1> b) { ++ return Mask128<uint64_t, 1>(vclt_u64(a.raw, b.raw)); ++} ++ ++#endif + + // Unsigned + HWY_NEON_DEF_FUNCTION_UINT_8_16_32(Min, vmin, _, 2) +@@ -1639,7 +1782,7 @@ template <size_t N> + HWY_INLINE Vec128<uint64_t, N> Min(const Vec128<uint64_t, N> a, + const Vec128<uint64_t, N> b) { + #if HWY_ARCH_ARM_A64 +- return IfThenElse(MaskFromVec(detail::Gt(a, b)), b, a); ++ return IfThenElse(b < a, b, a); + #else + const Simd<uint64_t, N> du; + const Simd<int64_t, N> di; +@@ -1654,7 +1797,7 @@ template <size_t N> + HWY_INLINE Vec128<int64_t, N> Min(const Vec128<int64_t, N> a, + const Vec128<int64_t, N> b) { + #if HWY_ARCH_ARM_A64 +- return IfThenElse(MaskFromVec(detail::Gt(a, b)), b, a); ++ return IfThenElse(b < a, b, a); + #else + const Vec128<int64_t, N> sign = detail::SaturatedSub(a, b); + return IfThenElse(MaskFromVec(BroadcastSignBit(sign)), a, b); +@@ -1677,7 +1820,7 @@ template <size_t N> + HWY_INLINE Vec128<uint64_t, N> Max(const Vec128<uint64_t, N> a, + const Vec128<uint64_t, N> b) { + #if HWY_ARCH_ARM_A64 +- return IfThenElse(MaskFromVec(detail::Gt(a, b)), a, b); ++ return IfThenElse(b < a, a, b); + #else + const Simd<uint64_t, N> du; + const Simd<int64_t, N> di; +@@ -1692,7 +1835,7 @@ template <size_t N> + HWY_INLINE Vec128<int64_t, N> Max(const Vec128<int64_t, N> a, + const Vec128<int64_t, N> b) { + #if HWY_ARCH_ARM_A64 +- return IfThenElse(MaskFromVec(detail::Gt(a, b)), a, b); ++ return IfThenElse(b < a, a, b); + #else + const Vec128<int64_t, N> sign = detail::SaturatedSub(a, b); + return IfThenElse(MaskFromVec(BroadcastSignBit(sign)), b, a); +@@ -1805,73 +1948,72 @@ HWY_INLINE Vec128<double, 1> LoadU(Simd< + // we don't actually care what is in it, and we don't want + // to introduce extra overhead by initializing it to something. + +-HWY_INLINE Vec128<uint8_t, 4> LoadU(Simd<uint8_t, 4> d, ++HWY_INLINE Vec128<uint8_t, 4> LoadU(Simd<uint8_t, 4> /*tag*/, + const uint8_t* HWY_RESTRICT p) { +- uint32x2_t a = Undefined(d).raw; ++ uint32x2_t a = Undefined(Simd<uint32_t, 2>()).raw; + uint32x2_t b = vld1_lane_u32(reinterpret_cast<const uint32_t*>(p), a, 0); + return Vec128<uint8_t, 4>(vreinterpret_u8_u32(b)); + } +-HWY_INLINE Vec128<uint16_t, 2> LoadU(Simd<uint16_t, 2> d, ++HWY_INLINE Vec128<uint16_t, 2> LoadU(Simd<uint16_t, 2> /*tag*/, + const uint16_t* HWY_RESTRICT p) { +- uint32x2_t a = Undefined(d).raw; ++ uint32x2_t a = Undefined(Simd<uint32_t, 2>()).raw; + uint32x2_t b = vld1_lane_u32(reinterpret_cast<const uint32_t*>(p), a, 0); + return Vec128<uint16_t, 2>(vreinterpret_u16_u32(b)); + } +-HWY_INLINE Vec128<uint32_t, 1> LoadU(Simd<uint32_t, 1> d, ++HWY_INLINE Vec128<uint32_t, 1> LoadU(Simd<uint32_t, 1> /*tag*/, + const uint32_t* HWY_RESTRICT p) { +- uint32x2_t a = Undefined(d).raw; ++ uint32x2_t a = Undefined(Simd<uint32_t, 2>()).raw; + uint32x2_t b = vld1_lane_u32(p, a, 0); + return Vec128<uint32_t, 1>(b); + } +-HWY_INLINE Vec128<int8_t, 4> LoadU(Simd<int8_t, 4> d, ++HWY_INLINE Vec128<int8_t, 4> LoadU(Simd<int8_t, 4> /*tag*/, + const int8_t* HWY_RESTRICT p) { +- int32x2_t a = Undefined(d).raw; ++ int32x2_t a = Undefined(Simd<int32_t, 2>()).raw; + int32x2_t b = vld1_lane_s32(reinterpret_cast<const int32_t*>(p), a, 0); + return Vec128<int8_t, 4>(vreinterpret_s8_s32(b)); + } +-HWY_INLINE Vec128<int16_t, 2> LoadU(Simd<int16_t, 2> d, ++HWY_INLINE Vec128<int16_t, 2> LoadU(Simd<int16_t, 2> /*tag*/, + const int16_t* HWY_RESTRICT p) { +- int32x2_t a = Undefined(d).raw; ++ int32x2_t a = Undefined(Simd<int32_t, 2>()).raw; + int32x2_t b = vld1_lane_s32(reinterpret_cast<const int32_t*>(p), a, 0); + return Vec128<int16_t, 2>(vreinterpret_s16_s32(b)); + } +-HWY_INLINE Vec128<int32_t, 1> LoadU(Simd<int32_t, 1> d, ++HWY_INLINE Vec128<int32_t, 1> LoadU(Simd<int32_t, 1> /*tag*/, + const int32_t* HWY_RESTRICT p) { +- int32x2_t a = Undefined(d).raw; ++ int32x2_t a = Undefined(Simd<int32_t, 2>()).raw; + int32x2_t b = vld1_lane_s32(p, a, 0); + return Vec128<int32_t, 1>(b); + } +-HWY_INLINE Vec128<float, 1> LoadU(Simd<float, 1> d, ++HWY_INLINE Vec128<float, 1> LoadU(Simd<float, 1> /*tag*/, + const float* HWY_RESTRICT p) { +- float32x2_t a = Undefined(d).raw; ++ float32x2_t a = Undefined(Simd<float, 2>()).raw; + float32x2_t b = vld1_lane_f32(p, a, 0); + return Vec128<float, 1>(b); + } + + // ------------------------------ Load 16 + +-HWY_INLINE Vec128<uint8_t, 2> LoadU(Simd<uint8_t, 2> d, ++HWY_INLINE Vec128<uint8_t, 2> LoadU(Simd<uint8_t, 2> /*tag*/, + const uint8_t* HWY_RESTRICT p) { +- uint16x4_t a = Undefined(d).raw; ++ uint16x4_t a = Undefined(Simd<uint16_t, 4>()).raw; + uint16x4_t b = vld1_lane_u16(reinterpret_cast<const uint16_t*>(p), a, 0); + return Vec128<uint8_t, 2>(vreinterpret_u8_u16(b)); + } +-HWY_INLINE Vec128<uint16_t, 1> LoadU(Simd<uint16_t, 1> d, ++HWY_INLINE Vec128<uint16_t, 1> LoadU(Simd<uint16_t, 1> /*tag*/, + const uint16_t* HWY_RESTRICT p) { +- uint16x4_t a = Undefined(d).raw; ++ uint16x4_t a = Undefined(Simd<uint16_t, 4>()).raw; + uint16x4_t b = vld1_lane_u16(p, a, 0); + return Vec128<uint16_t, 1>(b); + } +- +-HWY_INLINE Vec128<int8_t, 2> LoadU(Simd<int8_t, 2> d, ++HWY_INLINE Vec128<int8_t, 2> LoadU(Simd<int8_t, 2> /*tag*/, + const int8_t* HWY_RESTRICT p) { +- int16x4_t a = Undefined(d).raw; ++ int16x4_t a = Undefined(Simd<int16_t, 4>()).raw; + int16x4_t b = vld1_lane_s16(reinterpret_cast<const int16_t*>(p), a, 0); + return Vec128<int8_t, 2>(vreinterpret_s8_s16(b)); + } +-HWY_INLINE Vec128<int16_t, 1> LoadU(Simd<int16_t, 1> d, ++HWY_INLINE Vec128<int16_t, 1> LoadU(Simd<int16_t, 1> /*tag*/, + const int16_t* HWY_RESTRICT p) { +- int16x4_t a = Undefined(d).raw; ++ int16x4_t a = Undefined(Simd<int16_t, 4>()).raw; + int16x4_t b = vld1_lane_s16(p, a, 0); + return Vec128<int16_t, 1>(b); + } +@@ -2009,12 +2151,12 @@ HWY_INLINE void StoreU(const Vec128<doub + HWY_INLINE void StoreU(const Vec128<uint8_t, 4> v, Simd<uint8_t, 4>, + uint8_t* HWY_RESTRICT p) { + uint32x2_t a = vreinterpret_u32_u8(v.raw); +- vst1_lane_u32(p, a, 0); ++ vst1_lane_u32(reinterpret_cast<uint32_t*>(p), a, 0); + } + HWY_INLINE void StoreU(const Vec128<uint16_t, 2> v, Simd<uint16_t, 2>, + uint16_t* HWY_RESTRICT p) { + uint32x2_t a = vreinterpret_u32_u16(v.raw); +- vst1_lane_u32(p, a, 0); ++ vst1_lane_u32(reinterpret_cast<uint32_t*>(p), a, 0); + } + HWY_INLINE void StoreU(const Vec128<uint32_t, 1> v, Simd<uint32_t, 1>, + uint32_t* HWY_RESTRICT p) { +@@ -2023,12 +2165,12 @@ HWY_INLINE void StoreU(const Vec128<uint + HWY_INLINE void StoreU(const Vec128<int8_t, 4> v, Simd<int8_t, 4>, + int8_t* HWY_RESTRICT p) { + int32x2_t a = vreinterpret_s32_s8(v.raw); +- vst1_lane_s32(p, a, 0); ++ vst1_lane_s32(reinterpret_cast<int32_t*>(p), a, 0); + } + HWY_INLINE void StoreU(const Vec128<int16_t, 2> v, Simd<int16_t, 2>, + int16_t* HWY_RESTRICT p) { + int32x2_t a = vreinterpret_s32_s16(v.raw); +- vst1_lane_s32(p, a, 0); ++ vst1_lane_s32(reinterpret_cast<int32_t*>(p), a, 0); + } + HWY_INLINE void StoreU(const Vec128<int32_t, 1> v, Simd<int32_t, 1>, + int32_t* HWY_RESTRICT p) { +@@ -2044,7 +2186,7 @@ HWY_INLINE void StoreU(const Vec128<floa + HWY_INLINE void StoreU(const Vec128<uint8_t, 2> v, Simd<uint8_t, 2>, + uint8_t* HWY_RESTRICT p) { + uint16x4_t a = vreinterpret_u16_u8(v.raw); +- vst1_lane_u16(p, a, 0); ++ vst1_lane_u16(reinterpret_cast<uint16_t*>(p), a, 0); + } + HWY_INLINE void StoreU(const Vec128<uint16_t, 1> v, Simd<uint16_t, 1>, + uint16_t* HWY_RESTRICT p) { +@@ -2053,7 +2195,7 @@ HWY_INLINE void StoreU(const Vec128<uint + HWY_INLINE void StoreU(const Vec128<int8_t, 2> v, Simd<int8_t, 2>, + int8_t* HWY_RESTRICT p) { + int16x4_t a = vreinterpret_s16_s8(v.raw); +- vst1_lane_s16(p, a, 0); ++ vst1_lane_s16(reinterpret_cast<int16_t*>(p), a, 0); + } + HWY_INLINE void StoreU(const Vec128<int16_t, 1> v, Simd<int16_t, 1>, + int16_t* HWY_RESTRICT p) { +@@ -2118,18 +2260,18 @@ HWY_INLINE Vec128<uint64_t> PromoteTo(Fu + const Vec128<uint32_t, 2> v) { + return Vec128<uint64_t>(vmovl_u32(v.raw)); + } +-HWY_INLINE Vec128<int16_t> PromoteTo(Full128<int16_t> /* tag */, ++HWY_INLINE Vec128<int16_t> PromoteTo(Full128<int16_t> d, + const Vec128<uint8_t, 8> v) { +- return Vec128<int16_t>(vmovl_u8(v.raw)); ++ return BitCast(d, Vec128<uint16_t>(vmovl_u8(v.raw))); + } +-HWY_INLINE Vec128<int32_t> PromoteTo(Full128<int32_t> /* tag */, ++HWY_INLINE Vec128<int32_t> PromoteTo(Full128<int32_t> d, + const Vec128<uint8_t, 4> v) { + uint16x8_t a = vmovl_u8(v.raw); +- return Vec128<int32_t>(vreinterpretq_s32_u16(vmovl_u16(vget_low_u16(a)))); ++ return BitCast(d, Vec128<uint32_t>(vmovl_u16(vget_low_u16(a)))); + } +-HWY_INLINE Vec128<int32_t> PromoteTo(Full128<int32_t> /* tag */, ++HWY_INLINE Vec128<int32_t> PromoteTo(Full128<int32_t> d, + const Vec128<uint16_t, 4> v) { +- return Vec128<int32_t>(vmovl_u16(v.raw)); ++ return BitCast(d, Vec128<uint32_t>(vmovl_u16(v.raw))); + } + + // Unsigned: zero-extend to half vector. +@@ -2155,9 +2297,9 @@ HWY_INLINE Vec128<uint64_t, N> PromoteTo + return Vec128<uint64_t, N>(vget_low_u64(vmovl_u32(v.raw))); + } + template <size_t N, HWY_IF_LE64(int16_t, N)> +-HWY_INLINE Vec128<int16_t, N> PromoteTo(Simd<int16_t, N> /* tag */, ++HWY_INLINE Vec128<int16_t, N> PromoteTo(Simd<int16_t, N> d, + const Vec128<uint8_t, N> v) { +- return Vec128<int16_t, N>(vget_low_s16(vmovl_u8(v.raw))); ++ return BitCast(d, Vec128<uint16_t, N>(vget_low_u16(vmovl_u8(v.raw)))); + } + template <size_t N, HWY_IF_LE64(int32_t, N)> + HWY_INLINE Vec128<int32_t, N> PromoteTo(Simd<int32_t, N> /* tag */, +@@ -2220,12 +2362,14 @@ HWY_INLINE Vec128<int64_t, N> PromoteTo( + + HWY_INLINE Vec128<float> PromoteTo(Full128<float> /* tag */, + const Vec128<float16_t, 4> v) { +- return Vec128<float>(vcvt_f32_f16(vreinterpret_f16_u16(v.raw))); ++ const float32x4_t f32 = vcvt_f32_f16(vreinterpret_f16_u16(v.raw)); ++ return Vec128<float>(f32); + } + template <size_t N> + HWY_INLINE Vec128<float, N> PromoteTo(Simd<float, N> /* tag */, + const Vec128<float16_t, N> v) { +- return Vec128<float, N>(vget_low_f32(vcvt_f32_f16(v.raw))); ++ const float32x4_t f32 = vcvt_f32_f16(vreinterpret_f16_u16(v.raw)); ++ return Vec128<float, N>(vget_low_f32(f32)); + } + + #else +@@ -2353,7 +2497,8 @@ HWY_INLINE Vec128<float16_t, 4> DemoteTo + template <size_t N> + HWY_INLINE Vec128<float16_t, N> DemoteTo(Simd<float16_t, N> /* tag */, + const Vec128<float, N> v) { +- return Vec128<float16_t, N>{vcvt_f16_f32(vcombine_f32(v.raw, v.raw))}; ++ const float16x4_t f16 = vcvt_f16_f32(vcombine_f32(v.raw, v.raw)); ++ return Vec128<float16_t, N>(vreinterpret_u16_f16(f16)); + } + + #else +@@ -2965,33 +3110,58 @@ HWY_INLINE Vec128<T, N> TableLookupBytes + BitCast(d8, from).raw))); + } + +-// ------------------------------ Hard-coded shuffles ++// ------------------------------ TableLookupLanes + +-// Notation: let Vec128<int32_t> have lanes 3,2,1,0 (0 is least-significant). +-// Shuffle0321 rotates one lane to the right (the previous least-significant +-// lane is now most-significant). These could also be implemented via +-// CombineShiftRightBytes but the shuffle_abcd notation is more convenient. ++// Returned by SetTableIndices for use by TableLookupLanes. ++template <typename T, size_t N> ++struct Indices128 { ++ typename detail::Raw128<T, N>::type raw; ++}; + +-// Swap 32-bit halves in 64-bits +-HWY_INLINE Vec128<uint32_t, 2> Shuffle2301(const Vec128<uint32_t, 2> v) { +- return Vec128<uint32_t, 2>(vrev64_u32(v.raw)); +-} +-HWY_INLINE Vec128<int32_t, 2> Shuffle2301(const Vec128<int32_t, 2> v) { +- return Vec128<int32_t, 2>(vrev64_s32(v.raw)); +-} +-HWY_INLINE Vec128<float, 2> Shuffle2301(const Vec128<float, 2> v) { +- return Vec128<float, 2>(vrev64_f32(v.raw)); ++template <typename T, size_t N, HWY_IF_LE128(T, N)> ++HWY_INLINE Indices128<T, N> SetTableIndices(Simd<T, N> d, const int32_t* idx) { ++#if !defined(NDEBUG) || defined(ADDRESS_SANITIZER) ++ for (size_t i = 0; i < N; ++i) { ++ HWY_DASSERT(0 <= idx[i] && idx[i] < static_cast<int32_t>(N)); ++ } ++#endif ++ ++ const Repartition<uint8_t, decltype(d)> d8; ++ alignas(16) uint8_t control[16] = {0}; ++ for (size_t idx_lane = 0; idx_lane < N; ++idx_lane) { ++ for (size_t idx_byte = 0; idx_byte < sizeof(T); ++idx_byte) { ++ control[idx_lane * sizeof(T) + idx_byte] = ++ static_cast<uint8_t>(idx[idx_lane] * sizeof(T) + idx_byte); ++ } ++ } ++ return Indices128<T, N>{BitCast(d, Load(d8, control)).raw}; + } +-HWY_INLINE Vec128<uint32_t> Shuffle2301(const Vec128<uint32_t> v) { +- return Vec128<uint32_t>(vrev64q_u32(v.raw)); ++ ++template <size_t N> ++HWY_INLINE Vec128<uint32_t, N> TableLookupLanes( ++ const Vec128<uint32_t, N> v, const Indices128<uint32_t, N> idx) { ++ return TableLookupBytes(v, Vec128<uint32_t, N>{idx.raw}); + } +-HWY_INLINE Vec128<int32_t> Shuffle2301(const Vec128<int32_t> v) { +- return Vec128<int32_t>(vrev64q_s32(v.raw)); ++template <size_t N> ++HWY_INLINE Vec128<int32_t, N> TableLookupLanes( ++ const Vec128<int32_t, N> v, const Indices128<int32_t, N> idx) { ++ return TableLookupBytes(v, Vec128<int32_t, N>{idx.raw}); + } +-HWY_INLINE Vec128<float> Shuffle2301(const Vec128<float> v) { +- return Vec128<float>(vrev64q_f32(v.raw)); ++template <size_t N> ++HWY_INLINE Vec128<float, N> TableLookupLanes(const Vec128<float, N> v, ++ const Indices128<float, N> idx) { ++ const Simd<int32_t, N> di; ++ const auto idx_i = BitCast(di, Vec128<float, N>{idx.raw}); ++ return BitCast(Simd<float, N>(), TableLookupBytes(BitCast(di, v), idx_i)); + } + ++// ------------------------------ Other shuffles (TableLookupBytes) ++ ++// Notation: let Vec128<int32_t> have lanes 3,2,1,0 (0 is least-significant). ++// Shuffle0321 rotates one lane to the right (the previous least-significant ++// lane is now most-significant). These could also be implemented via ++// CombineShiftRightBytes but the shuffle_abcd notation is more convenient. ++ + // Swap 64-bit halves + template <typename T> + HWY_INLINE Vec128<T> Shuffle1032(const Vec128<T> v) { +@@ -3029,49 +3199,6 @@ HWY_INLINE Vec128<T> Shuffle0123(const V + return TableLookupBytes(v, BitCast(d, Load(d8, bytes))); + } + +-// ------------------------------ TableLookupLanes +- +-// Returned by SetTableIndices for use by TableLookupLanes. +-template <typename T> +-struct Indices128 { +- typename Raw128<T, 16 / sizeof(T)>::type raw; +-}; +- +-template <typename T> +-HWY_INLINE Indices128<T> SetTableIndices(const Full128<T>, const int32_t* idx) { +-#if !defined(NDEBUG) || defined(ADDRESS_SANITIZER) +- const size_t N = 16 / sizeof(T); +- for (size_t i = 0; i < N; ++i) { +- HWY_DASSERT(0 <= idx[i] && idx[i] < static_cast<int32_t>(N)); +- } +-#endif +- +- const Full128<uint8_t> d8; +- alignas(16) uint8_t control[16]; +- for (size_t idx_byte = 0; idx_byte < 16; ++idx_byte) { +- const size_t idx_lane = idx_byte / sizeof(T); +- const size_t mod = idx_byte % sizeof(T); +- control[idx_byte] = idx[idx_lane] * sizeof(T) + mod; +- } +- return Indices128<T>{BitCast(Full128<T>(), Load(d8, control)).raw}; +-} +- +-HWY_INLINE Vec128<uint32_t> TableLookupLanes(const Vec128<uint32_t> v, +- const Indices128<uint32_t> idx) { +- return TableLookupBytes(v, Vec128<uint32_t>(idx.raw)); +-} +-HWY_INLINE Vec128<int32_t> TableLookupLanes(const Vec128<int32_t> v, +- const Indices128<int32_t> idx) { +- return TableLookupBytes(v, Vec128<int32_t>(idx.raw)); +-} +-HWY_INLINE Vec128<float> TableLookupLanes(const Vec128<float> v, +- const Indices128<float> idx) { +- const Full128<int32_t> di; +- const Full128<float> df; +- return BitCast(df, +- TableLookupBytes(BitCast(di, v), Vec128<int32_t>(idx.raw))); +-} +- + // ------------------------------ Interleave lanes + + // Interleaves lanes from halves of the 128-bit blocks of "a" (which provides +@@ -3334,16 +3461,6 @@ HWY_INLINE Vec128<T> OddEven(const Vec12 + + // ================================================== MISC + +-// Returns a vector with lane i=[0, N) set to "first" + i. +-template <typename T, size_t N, typename T2> +-Vec128<T, N> Iota(const Simd<T, N> d, const T2 first) { +- HWY_ALIGN T lanes[16 / sizeof(T)]; +- for (size_t i = 0; i < 16 / sizeof(T); ++i) { +- lanes[i] = static_cast<T>(first + static_cast<T2>(i)); +- } +- return Load(d, lanes); +-} +- + // ------------------------------ Scatter (Store) + + template <typename T, size_t N, typename Offset, HWY_IF_LE128(T, N)> +@@ -3413,52 +3530,44 @@ HWY_API Vec128<T, N> GatherIndex(const S + return Load(d, lanes); + } + +-// ------------------------------ ARMv7 int64 comparisons (requires Shuffle2301) ++// ------------------------------ Reductions + +-#if HWY_ARCH_ARM_V7 ++namespace detail { + +-template <size_t N> +-HWY_INLINE Mask128<int64_t, N> operator==(const Vec128<int64_t, N> a, +- const Vec128<int64_t, N> b) { +- const Simd<int32_t, N * 2> d32; +- const Simd<int64_t, N> d64; +- const auto cmp32 = VecFromMask(d32, BitCast(d32, a) == BitCast(d32, b)); +- const auto cmp64 = cmp32 & Shuffle2301(cmp32); +- return MaskFromVec(BitCast(d64, cmp64)); ++// N=1 for any T: no-op ++template <typename T> ++HWY_API Vec128<T, 1> SumOfLanes(const Vec128<T, 1> v) { ++ return v; + } +- +-template <size_t N> +-HWY_INLINE Mask128<uint64_t, N> operator==(const Vec128<uint64_t, N> a, +- const Vec128<uint64_t, N> b) { +- const Simd<uint32_t, N * 2> d32; +- const Simd<uint64_t, N> d64; +- const auto cmp32 = VecFromMask(d32, BitCast(d32, a) == BitCast(d32, b)); +- const auto cmp64 = cmp32 & Shuffle2301(cmp32); +- return MaskFromVec(BitCast(d64, cmp64)); ++template <typename T> ++HWY_API Vec128<T, 1> MinOfLanes(hwy::SizeTag<sizeof(T)> /* tag */, ++ const Vec128<T, 1> v) { ++ return v; ++} ++template <typename T> ++HWY_API Vec128<T, 1> MaxOfLanes(hwy::SizeTag<sizeof(T)> /* tag */, ++ const Vec128<T, 1> v) { ++ return v; + } + +-HWY_INLINE Mask128<int64_t> operator<(const Vec128<int64_t> a, +- const Vec128<int64_t> b) { +- const int64x2_t sub = vqsubq_s64(a.raw, b.raw); +- return MaskFromVec(BroadcastSignBit(Vec128<int64_t>(sub))); ++// u32/i32/f32: N=2 ++template <typename T, HWY_IF_LANE_SIZE(T, 4)> ++HWY_API Vec128<T, 2> SumOfLanes(const Vec128<T, 2> v10) { ++ return v10 + Shuffle2301(v10); + } +-HWY_INLINE Mask128<int64_t, 1> operator<(const Vec128<int64_t, 1> a, +- const Vec128<int64_t, 1> b) { +- const int64x1_t sub = vqsub_s64(a.raw, b.raw); +- return MaskFromVec(BroadcastSignBit(Vec128<int64_t, 1>(sub))); ++template <typename T> ++HWY_API Vec128<T, 2> MinOfLanes(hwy::SizeTag<4> /* tag */, ++ const Vec128<T, 2> v10) { ++ return Min(v10, Shuffle2301(v10)); + } +- +-template <size_t N> +-HWY_INLINE Mask128<int64_t, N> operator>(const Vec128<int64_t, N> a, +- const Vec128<int64_t, N> b) { +- return b < a; ++template <typename T> ++HWY_API Vec128<T, 2> MaxOfLanes(hwy::SizeTag<4> /* tag */, ++ const Vec128<T, 2> v10) { ++ return Max(v10, Shuffle2301(v10)); + } +-#endif +- +-// ------------------------------ Reductions + ++// full vectors + #if HWY_ARCH_ARM_A64 +-// Supported for 32b and 64b vector types. Returns the sum in each lane. + HWY_INLINE Vec128<uint32_t> SumOfLanes(const Vec128<uint32_t> v) { + return Vec128<uint32_t>(vdupq_n_u32(vaddvq_u32(v.raw))); + } +@@ -3505,20 +3614,15 @@ HWY_INLINE Vec128<int64_t> SumOfLanes(co + } + #endif + +-namespace detail { +- +-// For u32/i32/f32. +-template <typename T, size_t N> +-HWY_API Vec128<T, N> MinOfLanes(hwy::SizeTag<4> /* tag */, +- const Vec128<T, N> v3210) { ++template <typename T> ++HWY_API Vec128<T> MinOfLanes(hwy::SizeTag<4> /* tag */, const Vec128<T> v3210) { + const Vec128<T> v1032 = Shuffle1032(v3210); + const Vec128<T> v31_20_31_20 = Min(v3210, v1032); + const Vec128<T> v20_31_20_31 = Shuffle0321(v31_20_31_20); + return Min(v20_31_20_31, v31_20_31_20); + } +-template <typename T, size_t N> +-HWY_API Vec128<T, N> MaxOfLanes(hwy::SizeTag<4> /* tag */, +- const Vec128<T, N> v3210) { ++template <typename T> ++HWY_API Vec128<T> MaxOfLanes(hwy::SizeTag<4> /* tag */, const Vec128<T> v3210) { + const Vec128<T> v1032 = Shuffle1032(v3210); + const Vec128<T> v31_20_31_20 = Max(v3210, v1032); + const Vec128<T> v20_31_20_31 = Shuffle0321(v31_20_31_20); +@@ -3526,15 +3630,13 @@ HWY_API Vec128<T, N> MaxOfLanes(hwy::Siz + } + + // For u64/i64[/f64]. +-template <typename T, size_t N> +-HWY_API Vec128<T, N> MinOfLanes(hwy::SizeTag<8> /* tag */, +- const Vec128<T, N> v10) { ++template <typename T> ++HWY_API Vec128<T> MinOfLanes(hwy::SizeTag<8> /* tag */, const Vec128<T> v10) { + const Vec128<T> v01 = Shuffle01(v10); + return Min(v10, v01); + } +-template <typename T, size_t N> +-HWY_API Vec128<T, N> MaxOfLanes(hwy::SizeTag<8> /* tag */, +- const Vec128<T, N> v10) { ++template <typename T> ++HWY_API Vec128<T> MaxOfLanes(hwy::SizeTag<8> /* tag */, const Vec128<T> v10) { + const Vec128<T> v01 = Shuffle01(v10); + return Max(v10, v01); + } +@@ -3542,6 +3644,10 @@ HWY_API Vec128<T, N> MaxOfLanes(hwy::Siz + } // namespace detail + + template <typename T, size_t N> ++HWY_API Vec128<T, N> SumOfLanes(const Vec128<T, N> v) { ++ return detail::SumOfLanes(v); ++} ++template <typename T, size_t N> + HWY_API Vec128<T, N> MinOfLanes(const Vec128<T, N> v) { + return detail::MinOfLanes(hwy::SizeTag<sizeof(T)>(), v); + } +@@ -3569,13 +3675,13 @@ HWY_INLINE uint64_t BitsFromMask(hwy::Si + const uint8x8_t x2 = vget_low_u8(vpaddq_u8(values.raw, values.raw)); + const uint8x8_t x4 = vpadd_u8(x2, x2); + const uint8x8_t x8 = vpadd_u8(x4, x4); +- return vreinterpret_u16_u8(x8)[0]; ++ return vget_lane_u64(vreinterpret_u64_u8(x8), 0); + #else + // Don't have vpaddq, so keep doubling lane size. + const uint16x8_t x2 = vpaddlq_u8(values.raw); + const uint32x4_t x4 = vpaddlq_u16(x2); + const uint64x2_t x8 = vpaddlq_u32(x4); +- return (uint64_t(x8[1]) << 8) | x8[0]; ++ return (vgetq_lane_u64(x8, 1) << 8) | vgetq_lane_u64(x8, 0); + #endif + } + +@@ -3725,7 +3831,7 @@ HWY_INLINE size_t CountTrue(hwy::SizeTag + const int16x8_t x2 = vpaddlq_s8(ones); + const int32x4_t x4 = vpaddlq_s16(x2); + const int64x2_t x8 = vpaddlq_s32(x4); +- return x8[0] + x8[1]; ++ return vgetq_lane_s64(x8, 0) + vgetq_lane_s64(x8, 1); + #endif + } + template <typename T> +@@ -3739,7 +3845,7 @@ HWY_INLINE size_t CountTrue(hwy::SizeTag + #else + const int32x4_t x2 = vpaddlq_s16(ones); + const int64x2_t x4 = vpaddlq_s32(x2); +- return x4[0] + x4[1]; ++ return vgetq_lane_s64(x4, 0) + vgetq_lane_s64(x4, 1); + #endif + } + +@@ -3753,7 +3859,7 @@ HWY_INLINE size_t CountTrue(hwy::SizeTag + return vaddvq_s32(ones); + #else + const int64x2_t x2 = vpaddlq_s32(ones); +- return x2[0] + x2[1]; ++ return vgetq_lane_s64(x2, 0) + vgetq_lane_s64(x2, 1); + #endif + } + +@@ -3765,10 +3871,10 @@ HWY_INLINE size_t CountTrue(hwy::SizeTag + vnegq_s64(BitCast(di, VecFromMask(Full128<T>(), mask)).raw); + return vaddvq_s64(ones); + #else +- const Full128<int64_t> di; +- const int64x2_t ones = +- vshrq_n_u64(BitCast(di, VecFromMask(Full128<T>(), mask)).raw, 63); +- return ones[0] + ones[1]; ++ const Full128<uint64_t> du; ++ const auto mask_u = VecFromMask(du, RebindMask(du, mask)); ++ const uint64x2_t ones = vshrq_n_u64(mask_u.raw, 63); ++ return vgetq_lane_u64(ones, 0) + vgetq_lane_u64(ones, 1); + #endif + } + +@@ -3798,11 +3904,13 @@ HWY_INLINE size_t StoreMaskBits(const Ma + template <typename T> + HWY_INLINE bool AllFalse(const Mask128<T> m) { + #if HWY_ARCH_ARM_A64 +- return (vmaxvq_u32(m.raw) == 0); ++ const Full128<uint32_t> d32; ++ const auto m32 = MaskFromVec(BitCast(d32, VecFromMask(Full128<T>(), m))); ++ return (vmaxvq_u32(m32.raw) == 0); + #else + const auto v64 = BitCast(Full128<uint64_t>(), VecFromMask(Full128<T>(), m)); + uint32x2_t a = vqmovn_u64(v64.raw); +- return vreinterpret_u64_u32(a)[0] == 0; ++ return vget_lane_u64(vreinterpret_u64_u32(a), 0) == 0; + #endif + } + +@@ -4178,6 +4286,7 @@ HWY_API auto Le(V a, V b) -> decltype(a + return a <= b; + } + ++namespace detail { // for code folding + #if HWY_ARCH_ARM_V7 + #undef vuzp1_s8 + #undef vuzp1_u8 +@@ -4265,6 +4374,7 @@ HWY_API auto Le(V a, V b) -> decltype(a + #undef HWY_NEON_DEF_FUNCTION_UINT_8_16_32 + #undef HWY_NEON_DEF_FUNCTION_UINTS + #undef HWY_NEON_EVAL ++} // namespace detail + + // NOLINTNEXTLINE(google-readability-namespace-comments) + } // namespace HWY_NAMESPACE +diff -up chromium-92.0.4515.107/third_party/highway/src/hwy/ops/rvv-inl.h.update-highway-0.12.2 chromium-92.0.4515.107/third_party/highway/src/hwy/ops/rvv-inl.h +--- chromium-92.0.4515.107/third_party/highway/src/hwy/ops/rvv-inl.h.update-highway-0.12.2 2021-07-19 14:47:23.000000000 -0400 ++++ chromium-92.0.4515.107/third_party/highway/src/hwy/ops/rvv-inl.h 2021-07-26 17:10:30.290171587 -0400 +@@ -39,6 +39,11 @@ using TFromV = TFromD<DFromV<V>>; + hwy::EnableIf<IsSigned<TFromV<V>>() && !IsFloat<TFromV<V>>()>* = nullptr + #define HWY_IF_FLOAT_V(V) hwy::EnableIf<IsFloat<TFromV<V>>()>* = nullptr + ++// kShift = log2 of multiplier: 0 for m1, 1 for m2, -2 for mf4 ++template <typename T, int kShift = 0> ++using Full = Simd<T, (kShift < 0) ? (HWY_LANES(T) >> (-kShift)) ++ : (HWY_LANES(T) << kShift)>; ++ + // ================================================== MACROS + + // Generate specializations and function definitions using X macros. Although +@@ -58,29 +63,30 @@ namespace detail { // for code folding + + // For given SEW, iterate over all LMUL. Precompute SEW/LMUL => MLEN because the + // preprocessor cannot easily do it. +-#define HWY_RVV_FOREACH_08(X_MACRO, BASE, CHAR, NAME, OP) \ +- X_MACRO(BASE, CHAR, 8, 1, 8, NAME, OP) \ +- X_MACRO(BASE, CHAR, 8, 2, 4, NAME, OP) \ +- X_MACRO(BASE, CHAR, 8, 4, 2, NAME, OP) \ +- X_MACRO(BASE, CHAR, 8, 8, 1, NAME, OP) +- +-#define HWY_RVV_FOREACH_16(X_MACRO, BASE, CHAR, NAME, OP) \ +- X_MACRO(BASE, CHAR, 16, 1, 16, NAME, OP) \ +- X_MACRO(BASE, CHAR, 16, 2, 8, NAME, OP) \ +- X_MACRO(BASE, CHAR, 16, 4, 4, NAME, OP) \ +- X_MACRO(BASE, CHAR, 16, 8, 2, NAME, OP) +- +-#define HWY_RVV_FOREACH_32(X_MACRO, BASE, CHAR, NAME, OP) \ +- X_MACRO(BASE, CHAR, 32, 1, 32, NAME, OP) \ +- X_MACRO(BASE, CHAR, 32, 2, 16, NAME, OP) \ +- X_MACRO(BASE, CHAR, 32, 4, 8, NAME, OP) \ +- X_MACRO(BASE, CHAR, 32, 8, 4, NAME, OP) +- +-#define HWY_RVV_FOREACH_64(X_MACRO, BASE, CHAR, NAME, OP) \ +- X_MACRO(BASE, CHAR, 64, 1, 64, NAME, OP) \ +- X_MACRO(BASE, CHAR, 64, 2, 32, NAME, OP) \ +- X_MACRO(BASE, CHAR, 64, 4, 16, NAME, OP) \ +- X_MACRO(BASE, CHAR, 64, 8, 8, NAME, OP) ++// TODO(janwas): GCC does not yet support fractional LMUL ++#define HWY_RVV_FOREACH_08(X_MACRO, BASE, CHAR, NAME, OP) \ ++ X_MACRO(BASE, CHAR, 8, m1, /*kShift=*/0, /*MLEN=*/8, NAME, OP) \ ++ X_MACRO(BASE, CHAR, 8, m2, /*kShift=*/1, /*MLEN=*/4, NAME, OP) \ ++ X_MACRO(BASE, CHAR, 8, m4, /*kShift=*/2, /*MLEN=*/2, NAME, OP) \ ++ X_MACRO(BASE, CHAR, 8, m8, /*kShift=*/3, /*MLEN=*/1, NAME, OP) ++ ++#define HWY_RVV_FOREACH_16(X_MACRO, BASE, CHAR, NAME, OP) \ ++ X_MACRO(BASE, CHAR, 16, m1, /*kShift=*/0, /*MLEN=*/16, NAME, OP) \ ++ X_MACRO(BASE, CHAR, 16, m2, /*kShift=*/1, /*MLEN=*/8, NAME, OP) \ ++ X_MACRO(BASE, CHAR, 16, m4, /*kShift=*/2, /*MLEN=*/4, NAME, OP) \ ++ X_MACRO(BASE, CHAR, 16, m8, /*kShift=*/3, /*MLEN=*/2, NAME, OP) ++ ++#define HWY_RVV_FOREACH_32(X_MACRO, BASE, CHAR, NAME, OP) \ ++ X_MACRO(BASE, CHAR, 32, m1, /*kShift=*/0, /*MLEN=*/32, NAME, OP) \ ++ X_MACRO(BASE, CHAR, 32, m2, /*kShift=*/1, /*MLEN=*/16, NAME, OP) \ ++ X_MACRO(BASE, CHAR, 32, m4, /*kShift=*/2, /*MLEN=*/8, NAME, OP) \ ++ X_MACRO(BASE, CHAR, 32, m8, /*kShift=*/3, /*MLEN=*/4, NAME, OP) ++ ++#define HWY_RVV_FOREACH_64(X_MACRO, BASE, CHAR, NAME, OP) \ ++ X_MACRO(BASE, CHAR, 64, m1, /*kShift=*/0, /*MLEN=*/64, NAME, OP) \ ++ X_MACRO(BASE, CHAR, 64, m2, /*kShift=*/1, /*MLEN=*/32, NAME, OP) \ ++ X_MACRO(BASE, CHAR, 64, m4, /*kShift=*/2, /*MLEN=*/16, NAME, OP) \ ++ X_MACRO(BASE, CHAR, 64, m8, /*kShift=*/3, /*MLEN=*/8, NAME, OP) + + // SEW for unsigned: + #define HWY_RVV_FOREACH_U08(X_MACRO, NAME, OP) \ +@@ -153,63 +159,61 @@ namespace detail { // for code folding + + // Assemble types for use in x-macros + #define HWY_RVV_T(BASE, SEW) BASE##SEW##_t +-#define HWY_RVV_D(CHAR, SEW, LMUL) D##CHAR##SEW##m##LMUL +-#define HWY_RVV_V(BASE, SEW, LMUL) v##BASE##SEW##m##LMUL##_t ++#define HWY_RVV_D(CHAR, SEW, LMUL) D##CHAR##SEW##LMUL ++#define HWY_RVV_V(BASE, SEW, LMUL) v##BASE##SEW##LMUL##_t + #define HWY_RVV_M(MLEN) vbool##MLEN##_t + + } // namespace detail + + // TODO(janwas): remove typedefs and only use HWY_RVV_V etc. directly + +-// TODO(janwas): do we want fractional LMUL? (can encode as negative) +-// Mixed-precision code can use LMUL 1..8 and that should be enough unless they +-// need many registers. +-#define HWY_SPECIALIZE(BASE, CHAR, SEW, LMUL, MLEN, NAME, OP) \ +- using HWY_RVV_D(CHAR, SEW, LMUL) = \ +- Simd<HWY_RVV_T(BASE, SEW), HWY_LANES(HWY_RVV_T(BASE, SEW)) * LMUL>; \ +- using V##CHAR##SEW##m##LMUL = HWY_RVV_V(BASE, SEW, LMUL); \ +- template <> \ +- struct DFromV_t<HWY_RVV_V(BASE, SEW, LMUL)> { \ +- using Lane = HWY_RVV_T(BASE, SEW); \ +- using type = Simd<Lane, HWY_LANES(Lane) * LMUL>; \ ++// Until we have full intrinsic support for fractional LMUL, mixed-precision ++// code can use LMUL 1..8 (adequate unless they need many registers). ++#define HWY_SPECIALIZE(BASE, CHAR, SEW, LMUL, SHIFT, MLEN, NAME, OP) \ ++ using HWY_RVV_D(CHAR, SEW, LMUL) = Full<HWY_RVV_T(BASE, SEW), SHIFT>; \ ++ using V##CHAR##SEW##LMUL = HWY_RVV_V(BASE, SEW, LMUL); \ ++ template <> \ ++ struct DFromV_t<HWY_RVV_V(BASE, SEW, LMUL)> { \ ++ using Lane = HWY_RVV_T(BASE, SEW); \ ++ using type = Full<Lane, SHIFT>; \ + }; + using Vf16m1 = vfloat16m1_t; + using Vf16m2 = vfloat16m2_t; + using Vf16m4 = vfloat16m4_t; + using Vf16m8 = vfloat16m8_t; +-using Df16m1 = Simd<float16_t, HWY_LANES(uint16_t) * 1>; +-using Df16m2 = Simd<float16_t, HWY_LANES(uint16_t) * 2>; +-using Df16m4 = Simd<float16_t, HWY_LANES(uint16_t) * 4>; +-using Df16m8 = Simd<float16_t, HWY_LANES(uint16_t) * 8>; ++using Df16m1 = Full<float16_t, 0>; ++using Df16m2 = Full<float16_t, 1>; ++using Df16m4 = Full<float16_t, 2>; ++using Df16m8 = Full<float16_t, 3>; + + HWY_RVV_FOREACH(HWY_SPECIALIZE, _, _) + #undef HWY_SPECIALIZE + + // vector = f(d), e.g. Zero +-#define HWY_RVV_RETV_ARGD(BASE, CHAR, SEW, LMUL, MLEN, NAME, OP) \ ++#define HWY_RVV_RETV_ARGD(BASE, CHAR, SEW, LMUL, SHIFT, MLEN, NAME, OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) NAME(HWY_RVV_D(CHAR, SEW, LMUL) d) { \ + (void)Lanes(d); \ +- return v##OP##_##CHAR##SEW##m##LMUL(); \ ++ return v##OP##_##CHAR##SEW##LMUL(); \ + } + + // vector = f(vector), e.g. Not +-#define HWY_RVV_RETV_ARGV(BASE, CHAR, SEW, LMUL, MLEN, NAME, OP) \ ++#define HWY_RVV_RETV_ARGV(BASE, CHAR, SEW, LMUL, SHIFT, MLEN, NAME, OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) NAME(HWY_RVV_V(BASE, SEW, LMUL) v) { \ +- return v##OP##_v_##CHAR##SEW##m##LMUL(v); \ ++ return v##OP##_v_##CHAR##SEW##LMUL(v); \ + } + + // vector = f(vector, scalar), e.g. detail::Add +-#define HWY_RVV_RETV_ARGVS(BASE, CHAR, SEW, LMUL, MLEN, NAME, OP) \ +- HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ +- NAME(HWY_RVV_V(BASE, SEW, LMUL) a, HWY_RVV_T(BASE, SEW) b) { \ +- return v##OP##_##CHAR##SEW##m##LMUL(a, b); \ ++#define HWY_RVV_RETV_ARGVS(BASE, CHAR, SEW, LMUL, SHIFT, MLEN, NAME, OP) \ ++ HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ ++ NAME(HWY_RVV_V(BASE, SEW, LMUL) a, HWY_RVV_T(BASE, SEW) b) { \ ++ return v##OP##_##CHAR##SEW##LMUL(a, b); \ + } + + // vector = f(vector, vector), e.g. Add +-#define HWY_RVV_RETV_ARGVV(BASE, CHAR, SEW, LMUL, MLEN, NAME, OP) \ ++#define HWY_RVV_RETV_ARGVV(BASE, CHAR, SEW, LMUL, SHIFT, MLEN, NAME, OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ + NAME(HWY_RVV_V(BASE, SEW, LMUL) a, HWY_RVV_V(BASE, SEW, LMUL) b) { \ +- return v##OP##_vv_##CHAR##SEW##m##LMUL(a, b); \ ++ return v##OP##_vv_##CHAR##SEW##LMUL(a, b); \ + } + + // ================================================== INIT +@@ -218,9 +222,9 @@ HWY_RVV_FOREACH(HWY_SPECIALIZE, _, _) + + // WARNING: we want to query VLMAX/sizeof(T), but this actually changes VL! + // vlenb is not exposed through intrinsics and vreadvl is not VLMAX. +-#define HWY_RVV_LANES(BASE, CHAR, SEW, LMUL, MLEN, NAME, OP) \ +- HWY_API size_t NAME(HWY_RVV_D(CHAR, SEW, LMUL) /* d */) { \ +- return v##OP##SEW##m##LMUL(); \ ++#define HWY_RVV_LANES(BASE, CHAR, SEW, LMUL, SHIFT, MLEN, NAME, OP) \ ++ HWY_API size_t NAME(HWY_RVV_D(CHAR, SEW, LMUL) /* d */) { \ ++ return v##OP##SEW##LMUL(); \ + } + + HWY_RVV_FOREACH(HWY_RVV_LANES, Lanes, setvlmax_e) +@@ -233,19 +237,31 @@ HWY_RVV_FOREACH(HWY_RVV_RETV_ARGD, Zero, + template <class D> + using VFromD = decltype(Zero(D())); + ++// Partial ++template <typename T, size_t N, HWY_IF_LE128(T, N)> ++HWY_API VFromD<Full<T>> Zero(Simd<T, N> /*tag*/) { ++ return Zero(Full<T>()); ++} ++ + // ------------------------------ Set + // vector = f(d, scalar), e.g. Set +-#define HWY_RVV_SET(BASE, CHAR, SEW, LMUL, MLEN, NAME, OP) \ ++#define HWY_RVV_SET(BASE, CHAR, SEW, LMUL, SHIFT, MLEN, NAME, OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ + NAME(HWY_RVV_D(CHAR, SEW, LMUL) d, HWY_RVV_T(BASE, SEW) arg) { \ + (void)Lanes(d); \ +- return v##OP##_##CHAR##SEW##m##LMUL(arg); \ ++ return v##OP##_##CHAR##SEW##LMUL(arg); \ + } + + HWY_RVV_FOREACH_UI(HWY_RVV_SET, Set, mv_v_x) + HWY_RVV_FOREACH_F(HWY_RVV_SET, Set, fmv_v_f) + #undef HWY_RVV_SET + ++// Partial vectors ++template <typename T, size_t N, HWY_IF_LE128(T, N)> ++HWY_API VFromD<Simd<T, N>> Set(Simd<T, N> /*tag*/, T arg) { ++ return Set(Full<T>(), arg); ++} ++ + // ------------------------------ Undefined + + // RVV vundefined is 'poisoned' such that even XORing a _variable_ initialized +@@ -265,7 +281,7 @@ HWY_API VFromD<D> Undefined(D d) { + namespace detail { + + // u8: no change +-#define HWY_RVV_CAST_NOP(BASE, CHAR, SEW, LMUL, MLEN, NAME, OP) \ ++#define HWY_RVV_CAST_NOP(BASE, CHAR, SEW, LMUL, SHIFT, MLEN, NAME, OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ + BitCastToByte(HWY_RVV_V(BASE, SEW, LMUL) v) { \ + return v; \ +@@ -276,25 +292,25 @@ namespace detail { + } + + // Other integers +-#define HWY_RVV_CAST_UI(BASE, CHAR, SEW, LMUL, MLEN, NAME, OP) \ +- HWY_API vuint8m##LMUL##_t BitCastToByte(HWY_RVV_V(BASE, SEW, LMUL) v) { \ +- return v##OP##_v_##CHAR##SEW##m##LMUL##_u8m##LMUL(v); \ +- } \ +- HWY_API HWY_RVV_V(BASE, SEW, LMUL) BitCastFromByte( \ +- HWY_RVV_D(CHAR, SEW, LMUL) /* d */, vuint8m##LMUL##_t v) { \ +- return v##OP##_v_u8m##LMUL##_##CHAR##SEW##m##LMUL(v); \ ++#define HWY_RVV_CAST_UI(BASE, CHAR, SEW, LMUL, SHIFT, MLEN, NAME, OP) \ ++ HWY_API vuint8##LMUL##_t BitCastToByte(HWY_RVV_V(BASE, SEW, LMUL) v) { \ ++ return v##OP##_v_##CHAR##SEW##LMUL##_u8##LMUL(v); \ ++ } \ ++ HWY_API HWY_RVV_V(BASE, SEW, LMUL) BitCastFromByte( \ ++ HWY_RVV_D(CHAR, SEW, LMUL) /* d */, vuint8##LMUL##_t v) { \ ++ return v##OP##_v_u8##LMUL##_##CHAR##SEW##LMUL(v); \ + } + + // Float: first cast to/from unsigned +-#define HWY_RVV_CAST_F(BASE, CHAR, SEW, LMUL, MLEN, NAME, OP) \ +- HWY_API vuint8m##LMUL##_t BitCastToByte(HWY_RVV_V(BASE, SEW, LMUL) v) { \ +- return v##OP##_v_u##SEW##m##LMUL##_u8m##LMUL( \ +- v##OP##_v_f##SEW##m##LMUL##_u##SEW##m##LMUL(v)); \ +- } \ +- HWY_API HWY_RVV_V(BASE, SEW, LMUL) BitCastFromByte( \ +- HWY_RVV_D(CHAR, SEW, LMUL) /* d */, vuint8m##LMUL##_t v) { \ +- return v##OP##_v_u##SEW##m##LMUL##_f##SEW##m##LMUL( \ +- v##OP##_v_u8m##LMUL##_u##SEW##m##LMUL(v)); \ ++#define HWY_RVV_CAST_F(BASE, CHAR, SEW, LMUL, SHIFT, MLEN, NAME, OP) \ ++ HWY_API vuint8##LMUL##_t BitCastToByte(HWY_RVV_V(BASE, SEW, LMUL) v) { \ ++ return v##OP##_v_u##SEW##LMUL##_u8##LMUL( \ ++ v##OP##_v_f##SEW##LMUL##_u##SEW##LMUL(v)); \ ++ } \ ++ HWY_API HWY_RVV_V(BASE, SEW, LMUL) BitCastFromByte( \ ++ HWY_RVV_D(CHAR, SEW, LMUL) /* d */, vuint8##LMUL##_t v) { \ ++ return v##OP##_v_u##SEW##LMUL##_f##SEW##LMUL( \ ++ v##OP##_v_u8##LMUL##_u##SEW##LMUL(v)); \ + } + + HWY_RVV_FOREACH_U08(HWY_RVV_CAST_NOP, _, _) +@@ -315,6 +331,12 @@ HWY_API VFromD<D> BitCast(D d, FromV v) + return detail::BitCastFromByte(d, detail::BitCastToByte(v)); + } + ++// Partial ++template <typename T, size_t N, class FromV, HWY_IF_LE128(T, N)> ++HWY_API VFromD<Simd<T, N>> BitCast(Simd<T, N> /*tag*/, FromV v) { ++ return BitCast(Full<T>(), v); ++} ++ + namespace detail { + + template <class V, class DU = RebindToUnsigned<DFromV<V>>> +@@ -336,6 +358,12 @@ HWY_API VFromD<DU> Iota0(const D /*d*/) + return BitCastToUnsigned(Iota0(DU())); + } + ++// Partial ++template <typename T, size_t N, HWY_IF_LE128(T, N)> ++HWY_API VFromD<Simd<T, N>> Iota0(Simd<T, N> /*tag*/) { ++ return Iota0(Full<T>()); ++} ++ + } // namespace detail + + // ================================================== LOGICAL +@@ -370,11 +398,11 @@ HWY_API V And(const V a, const V b) { + // ------------------------------ Or + + // Scalar argument plus mask. Used by VecFromMask. +-#define HWY_RVV_OR_MASK(BASE, CHAR, SEW, LMUL, MLEN, NAME, OP) \ ++#define HWY_RVV_OR_MASK(BASE, CHAR, SEW, LMUL, SHIFT, MLEN, NAME, OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ + NAME(HWY_RVV_V(BASE, SEW, LMUL) v, HWY_RVV_T(BASE, SEW) imm, \ + HWY_RVV_M(MLEN) mask, HWY_RVV_V(BASE, SEW, LMUL) maskedoff) { \ +- return v##OP##_##CHAR##SEW##m##LMUL##_m(mask, maskedoff, v, imm); \ ++ return v##OP##_##CHAR##SEW##LMUL##_m(mask, maskedoff, v, imm); \ + } + + namespace detail { +@@ -466,14 +494,14 @@ HWY_RVV_FOREACH_U16(HWY_RVV_RETV_ARGVV, + // ------------------------------ ShiftLeft[Same] + + // Intrinsics do not define .vi forms, so use .vx instead. +-#define HWY_RVV_SHIFT(BASE, CHAR, SEW, LMUL, MLEN, NAME, OP) \ +- template <int kBits> \ +- HWY_API HWY_RVV_V(BASE, SEW, LMUL) NAME(HWY_RVV_V(BASE, SEW, LMUL) v) { \ +- return v##OP##_vx_##CHAR##SEW##m##LMUL(v, kBits); \ +- } \ +- HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ +- NAME##Same(HWY_RVV_V(BASE, SEW, LMUL) v, int bits) { \ +- return v##OP##_vx_##CHAR##SEW##m##LMUL(v, static_cast<uint8_t>(bits)); \ ++#define HWY_RVV_SHIFT(BASE, CHAR, SEW, LMUL, SHIFT, MLEN, NAME, OP) \ ++ template <int kBits> \ ++ HWY_API HWY_RVV_V(BASE, SEW, LMUL) NAME(HWY_RVV_V(BASE, SEW, LMUL) v) { \ ++ return v##OP##_vx_##CHAR##SEW##LMUL(v, kBits); \ ++ } \ ++ HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ ++ NAME##Same(HWY_RVV_V(BASE, SEW, LMUL) v, int bits) { \ ++ return v##OP##_vx_##CHAR##SEW##LMUL(v, static_cast<uint8_t>(bits)); \ + } + + HWY_RVV_FOREACH_UI(HWY_RVV_SHIFT, ShiftLeft, sll) +@@ -486,19 +514,18 @@ HWY_RVV_FOREACH_I(HWY_RVV_SHIFT, ShiftRi + #undef HWY_RVV_SHIFT + + // ------------------------------ Shl +-#define HWY_RVV_SHIFT_VV(BASE, CHAR, SEW, LMUL, MLEN, NAME, OP) \ ++#define HWY_RVV_SHIFT_VV(BASE, CHAR, SEW, LMUL, SHIFT, MLEN, NAME, OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ + NAME(HWY_RVV_V(BASE, SEW, LMUL) v, HWY_RVV_V(BASE, SEW, LMUL) bits) { \ +- return v##OP##_vv_##CHAR##SEW##m##LMUL(v, bits); \ ++ return v##OP##_vv_##CHAR##SEW##LMUL(v, bits); \ + } + + HWY_RVV_FOREACH_U(HWY_RVV_SHIFT_VV, Shl, sll) + +-#define HWY_RVV_SHIFT_II(BASE, CHAR, SEW, LMUL, MLEN, NAME, OP) \ ++#define HWY_RVV_SHIFT_II(BASE, CHAR, SEW, LMUL, SHIFT, MLEN, NAME, OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ + NAME(HWY_RVV_V(BASE, SEW, LMUL) v, HWY_RVV_V(BASE, SEW, LMUL) bits) { \ +- return v##OP##_vv_##CHAR##SEW##m##LMUL(v, \ +- detail::BitCastToUnsigned(bits)); \ ++ return v##OP##_vv_##CHAR##SEW##LMUL(v, detail::BitCastToUnsigned(bits)); \ + } + + HWY_RVV_FOREACH_I(HWY_RVV_SHIFT_II, Shl, sll) +@@ -569,11 +596,11 @@ HWY_API V ApproximateReciprocalSqrt(cons + + // ------------------------------ MulAdd + // Note: op is still named vv, not vvv. +-#define HWY_RVV_FMA(BASE, CHAR, SEW, LMUL, MLEN, NAME, OP) \ ++#define HWY_RVV_FMA(BASE, CHAR, SEW, LMUL, SHIFT, MLEN, NAME, OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ + NAME(HWY_RVV_V(BASE, SEW, LMUL) mul, HWY_RVV_V(BASE, SEW, LMUL) x, \ + HWY_RVV_V(BASE, SEW, LMUL) add) { \ +- return v##OP##_vv_##CHAR##SEW##m##LMUL(add, mul, x); \ ++ return v##OP##_vv_##CHAR##SEW##LMUL(add, mul, x); \ + } + + HWY_RVV_FOREACH_F(HWY_RVV_FMA, MulAdd, fmacc) +@@ -596,11 +623,11 @@ HWY_RVV_FOREACH_F(HWY_RVV_FMA, NegMulSub + // of all bits; SLEN 8 / LMUL 4 = half of all bits. + + // mask = f(vector, vector) +-#define HWY_RVV_RETM_ARGVV(BASE, CHAR, SEW, LMUL, MLEN, NAME, OP) \ ++#define HWY_RVV_RETM_ARGVV(BASE, CHAR, SEW, LMUL, SHIFT, MLEN, NAME, OP) \ + HWY_API HWY_RVV_M(MLEN) \ + NAME(HWY_RVV_V(BASE, SEW, LMUL) a, HWY_RVV_V(BASE, SEW, LMUL) b) { \ + (void)Lanes(DFromV<decltype(a)>()); \ +- return v##OP##_vv_##CHAR##SEW##m##LMUL##_b##MLEN(a, b); \ ++ return v##OP##_vv_##CHAR##SEW##LMUL##_b##MLEN(a, b); \ + } + + // ------------------------------ Eq +@@ -675,11 +702,11 @@ HWY_RVV_FOREACH_B(HWY_RVV_RETM_ARGMM, Xo + #undef HWY_RVV_RETM_ARGMM + + // ------------------------------ IfThenElse +-#define HWY_RVV_IF_THEN_ELSE(BASE, CHAR, SEW, LMUL, MLEN, NAME, OP) \ +- HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ +- NAME(HWY_RVV_M(MLEN) m, HWY_RVV_V(BASE, SEW, LMUL) yes, \ +- HWY_RVV_V(BASE, SEW, LMUL) no) { \ +- return v##OP##_vvm_##CHAR##SEW##m##LMUL(m, no, yes); \ ++#define HWY_RVV_IF_THEN_ELSE(BASE, CHAR, SEW, LMUL, SHIFT, MLEN, NAME, OP) \ ++ HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ ++ NAME(HWY_RVV_M(MLEN) m, HWY_RVV_V(BASE, SEW, LMUL) yes, \ ++ HWY_RVV_V(BASE, SEW, LMUL) no) { \ ++ return v##OP##_vvm_##CHAR##SEW##LMUL(m, no, yes); \ + } + + HWY_RVV_FOREACH(HWY_RVV_IF_THEN_ELSE, IfThenElse, merge) +@@ -774,17 +801,17 @@ HWY_RVV_FOREACH_B(HWY_RVV_COUNT_TRUE, _, + + // ------------------------------ Load + +-#define HWY_RVV_LOAD(BASE, CHAR, SEW, LMUL, MLEN, NAME, OP) \ +- HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ +- NAME(HWY_RVV_D(CHAR, SEW, LMUL) d, \ +- const HWY_RVV_T(BASE, SEW) * HWY_RESTRICT p) { \ +- (void)Lanes(d); \ +- return v##OP##SEW##_v_##CHAR##SEW##m##LMUL(p); \ ++#define HWY_RVV_LOAD(BASE, CHAR, SEW, LMUL, SHIFT, MLEN, NAME, OP) \ ++ HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ ++ NAME(HWY_RVV_D(CHAR, SEW, LMUL) d, \ ++ const HWY_RVV_T(BASE, SEW) * HWY_RESTRICT p) { \ ++ (void)Lanes(d); \ ++ return v##OP##SEW##_v_##CHAR##SEW##LMUL(p); \ + } + HWY_RVV_FOREACH(HWY_RVV_LOAD, Load, le) + #undef HWY_RVV_LOAD + +-// Partial load ++// Partial + template <typename T, size_t N, HWY_IF_LE128(T, N)> + HWY_API VFromD<Simd<T, N>> Load(Simd<T, N> d, const T* HWY_RESTRICT p) { + return Load(d, p); +@@ -800,16 +827,22 @@ HWY_API VFromD<D> LoadU(D d, const TFrom + + // ------------------------------ Store + +-#define HWY_RVV_RET_ARGVDP(BASE, CHAR, SEW, LMUL, MLEN, NAME, OP) \ +- HWY_API void NAME(HWY_RVV_V(BASE, SEW, LMUL) v, \ +- HWY_RVV_D(CHAR, SEW, LMUL) d, \ +- HWY_RVV_T(BASE, SEW) * HWY_RESTRICT p) { \ +- (void)Lanes(d); \ +- return v##OP##SEW##_v_##CHAR##SEW##m##LMUL(p, v); \ ++#define HWY_RVV_RET_ARGVDP(BASE, CHAR, SEW, LMUL, SHIFT, MLEN, NAME, OP) \ ++ HWY_API void NAME(HWY_RVV_V(BASE, SEW, LMUL) v, \ ++ HWY_RVV_D(CHAR, SEW, LMUL) d, \ ++ HWY_RVV_T(BASE, SEW) * HWY_RESTRICT p) { \ ++ (void)Lanes(d); \ ++ return v##OP##SEW##_v_##CHAR##SEW##LMUL(p, v); \ + } + HWY_RVV_FOREACH(HWY_RVV_RET_ARGVDP, Store, se) + #undef HWY_RVV_RET_ARGVDP + ++// Partial ++template <typename T, size_t N, HWY_IF_LE128(T, N)> ++HWY_API void Store(VFromD<Simd<T, N>> v, Simd<T, N> d, T* HWY_RESTRICT p) { ++ return Store(v, Full<T>(), p); ++} ++ + // ------------------------------ StoreU + + // RVV only requires lane alignment, not natural alignment of the entire vector. +@@ -963,67 +996,6 @@ HWY_API VFromD<Simd<int32_t, N>> Promote + return BitCast(d, PromoteTo(Simd<uint32_t, N>(), v)); + } + +-// ------------------------------ PromoteTo I +- +-HWY_API Vi16m2 PromoteTo(Di16m2 /* d */, Vi8m1 v) { return vsext_vf2_i16m2(v); } +-HWY_API Vi16m4 PromoteTo(Di16m4 /* d */, Vi8m2 v) { return vsext_vf2_i16m4(v); } +-HWY_API Vi16m8 PromoteTo(Di16m8 /* d */, Vi8m4 v) { return vsext_vf2_i16m8(v); } +- +-HWY_API Vi32m4 PromoteTo(Di32m4 /* d */, Vi8m1 v) { return vsext_vf4_i32m4(v); } +-HWY_API Vi32m8 PromoteTo(Di32m8 /* d */, Vi8m2 v) { return vsext_vf4_i32m8(v); } +- +-HWY_API Vi32m2 PromoteTo(Di32m2 /* d */, const Vi16m1 v) { +- return vsext_vf2_i32m2(v); +-} +-HWY_API Vi32m4 PromoteTo(Di32m4 /* d */, const Vi16m2 v) { +- return vsext_vf2_i32m4(v); +-} +-HWY_API Vi32m8 PromoteTo(Di32m8 /* d */, const Vi16m4 v) { +- return vsext_vf2_i32m8(v); +-} +- +-HWY_API Vi64m2 PromoteTo(Di64m2 /* d */, const Vi32m1 v) { +- return vsext_vf2_i64m2(v); +-} +-HWY_API Vi64m4 PromoteTo(Di64m4 /* d */, const Vi32m2 v) { +- return vsext_vf2_i64m4(v); +-} +-HWY_API Vi64m8 PromoteTo(Di64m8 /* d */, const Vi32m4 v) { +- return vsext_vf2_i64m8(v); +-} +- +-// ------------------------------ PromoteTo F +- +-HWY_API Vf32m2 PromoteTo(Df32m2 /* d */, const Vf16m1 v) { +- return vfwcvt_f_f_v_f32m2(v); +-} +-HWY_API Vf32m4 PromoteTo(Df32m4 /* d */, const Vf16m2 v) { +- return vfwcvt_f_f_v_f32m4(v); +-} +-HWY_API Vf32m8 PromoteTo(Df32m8 /* d */, const Vf16m4 v) { +- return vfwcvt_f_f_v_f32m8(v); +-} +- +-HWY_API Vf64m2 PromoteTo(Df64m2 /* d */, const Vf32m1 v) { +- return vfwcvt_f_f_v_f64m2(v); +-} +-HWY_API Vf64m4 PromoteTo(Df64m4 /* d */, const Vf32m2 v) { +- return vfwcvt_f_f_v_f64m4(v); +-} +-HWY_API Vf64m8 PromoteTo(Df64m8 /* d */, const Vf32m4 v) { +- return vfwcvt_f_f_v_f64m8(v); +-} +- +-HWY_API Vf64m2 PromoteTo(Df64m2 /* d */, const Vi32m1 v) { +- return vfwcvt_f_x_v_f64m2(v); +-} +-HWY_API Vf64m4 PromoteTo(Df64m4 /* d */, const Vi32m2 v) { +- return vfwcvt_f_x_v_f64m4(v); +-} +-HWY_API Vf64m8 PromoteTo(Df64m8 /* d */, const Vi32m4 v) { +- return vfwcvt_f_x_v_f64m8(v); +-} +- + // ------------------------------ DemoteTo U + + // First clamp negative numbers to zero to match x86 packus. +@@ -1124,19 +1096,19 @@ HWY_API Vi32m4 DemoteTo(Di32m4 /* d */, + + // ------------------------------ ConvertTo F + +-#define HWY_RVV_CONVERT(BASE, CHAR, SEW, LMUL, MLEN, NAME, OP) \ ++#define HWY_RVV_CONVERT(BASE, CHAR, SEW, LMUL, SHIFT, MLEN, NAME, OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) ConvertTo( \ + HWY_RVV_D(CHAR, SEW, LMUL) /* d */, HWY_RVV_V(int, SEW, LMUL) v) { \ +- return vfcvt_f_x_v_f##SEW##m##LMUL(v); \ ++ return vfcvt_f_x_v_f##SEW##LMUL(v); \ + } \ + /* Truncates (rounds toward zero). */ \ + HWY_API HWY_RVV_V(int, SEW, LMUL) ConvertTo(HWY_RVV_D(i, SEW, LMUL) /* d */, \ + HWY_RVV_V(BASE, SEW, LMUL) v) { \ +- return vfcvt_rtz_x_f_v_i##SEW##m##LMUL(v); \ ++ return vfcvt_rtz_x_f_v_i##SEW##LMUL(v); \ + } \ + /* Uses default rounding mode. */ \ + HWY_API HWY_RVV_V(int, SEW, LMUL) NearestInt(HWY_RVV_V(BASE, SEW, LMUL) v) { \ +- return vfcvt_x_f_v_i##SEW##m##LMUL(v); \ ++ return vfcvt_x_f_v_i##SEW##LMUL(v); \ + } + + // API only requires f32 but we provide f64 for internal use (otherwise, it +@@ -1184,10 +1156,10 @@ HWY_API VFromD<DU> SetTableIndices(D d, + + // <32bit are not part of Highway API, but used in Broadcast. This limits VLMAX + // to 2048! We could instead use vrgatherei16. +-#define HWY_RVV_TABLE(BASE, CHAR, SEW, LMUL, MLEN, NAME, OP) \ ++#define HWY_RVV_TABLE(BASE, CHAR, SEW, LMUL, SHIFT, MLEN, NAME, OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ + NAME(HWY_RVV_V(BASE, SEW, LMUL) v, HWY_RVV_V(uint, SEW, LMUL) idx) { \ +- return v##OP##_vv_##CHAR##SEW##m##LMUL(v, idx); \ ++ return v##OP##_vv_##CHAR##SEW##LMUL(v, idx); \ + } + + HWY_RVV_FOREACH(HWY_RVV_TABLE, TableLookupLanes, rgather) +@@ -1279,7 +1251,6 @@ HWY_API V OffsetsOf128BitBlocks(const D + using T = MakeUnsigned<TFromD<D>>; + return detail::And(iota0, static_cast<T>(~(LanesPerBlock(d) - 1))); + } +- + } // namespace detail + + template <class V> +@@ -1307,9 +1278,9 @@ HWY_API V Broadcast(const V v) { + + // ------------------------------ GetLane + +-#define HWY_RVV_GET_LANE(BASE, CHAR, SEW, LMUL, MLEN, NAME, OP) \ +- HWY_API HWY_RVV_T(BASE, SEW) NAME(HWY_RVV_V(BASE, SEW, LMUL) v) { \ +- return v##OP##_s_##CHAR##SEW##m##LMUL##_##CHAR##SEW(v); \ ++#define HWY_RVV_GET_LANE(BASE, CHAR, SEW, LMUL, SHIFT, MLEN, NAME, OP) \ ++ HWY_API HWY_RVV_T(BASE, SEW) NAME(HWY_RVV_V(BASE, SEW, LMUL) v) { \ ++ return v##OP##_s_##CHAR##SEW##LMUL##_##CHAR##SEW(v); \ + } + + HWY_RVV_FOREACH_UI(HWY_RVV_GET_LANE, GetLane, mv_x) +@@ -1318,11 +1289,12 @@ HWY_RVV_FOREACH_F(HWY_RVV_GET_LANE, GetL + + // ------------------------------ ShiftLeftLanes + +-// vector = f(vector, size_t) +-#define HWY_RVV_SLIDE(BASE, CHAR, SEW, LMUL, MLEN, NAME, OP) \ +- HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ +- NAME(HWY_RVV_V(BASE, SEW, LMUL) v, size_t lanes) { \ +- return v##OP##_vx_##CHAR##SEW##m##LMUL(v, v, lanes); \ ++// vector = f(vector, vector, size_t) ++#define HWY_RVV_SLIDE(BASE, CHAR, SEW, LMUL, SHIFT, MLEN, NAME, OP) \ ++ HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ ++ NAME(HWY_RVV_V(BASE, SEW, LMUL) dst, HWY_RVV_V(BASE, SEW, LMUL) src, \ ++ size_t lanes) { \ ++ return v##OP##_vx_##CHAR##SEW##LMUL(dst, src, lanes); \ + } + + namespace detail { +@@ -1333,7 +1305,7 @@ template <size_t kLanes, class V> + HWY_API V ShiftLeftLanes(const V v) { + using D = DFromV<V>; + const RebindToSigned<D> di; +- const auto shifted = detail::SlideUp(v, kLanes); ++ const auto shifted = detail::SlideUp(v, v, kLanes); + // Match x86 semantics by zeroing lower lanes in 128-bit blocks + constexpr size_t kLanesPerBlock = detail::LanesPerBlock(di); + const auto idx_mod = detail::And(detail::Iota0(di), kLanesPerBlock - 1); +@@ -1363,7 +1335,7 @@ template <size_t kLanes, class V> + HWY_API V ShiftRightLanes(const V v) { + using D = DFromV<V>; + const RebindToSigned<D> di; +- const auto shifted = detail::SlideDown(v, kLanes); ++ const auto shifted = detail::SlideDown(v, v, kLanes); + // Match x86 semantics by zeroing upper lanes in 128-bit blocks + constexpr size_t kLanesPerBlock = detail::LanesPerBlock(di); + const auto idx_mod = detail::And(detail::Iota0(di), kLanesPerBlock - 1); +@@ -1405,7 +1377,7 @@ HWY_API V ConcatUpperLower(const V hi, c + template <class V> + HWY_API V ConcatLowerLower(const V hi, const V lo) { + // Move lower half into upper +- const auto hi_up = detail::SlideUp(hi, Lanes(DFromV<V>()) / 2); ++ const auto hi_up = detail::SlideUp(hi, hi, Lanes(DFromV<V>()) / 2); + return ConcatUpperLower(hi_up, lo); + } + +@@ -1414,7 +1386,7 @@ HWY_API V ConcatLowerLower(const V hi, c + template <class V> + HWY_API V ConcatUpperUpper(const V hi, const V lo) { + // Move upper half into lower +- const auto lo_down = detail::SlideDown(lo, Lanes(DFromV<V>()) / 2); ++ const auto lo_down = detail::SlideDown(lo, lo, Lanes(DFromV<V>()) / 2); + return ConcatUpperLower(hi, lo_down); + } + +@@ -1423,8 +1395,8 @@ HWY_API V ConcatUpperUpper(const V hi, c + template <class V> + HWY_API V ConcatLowerUpper(const V hi, const V lo) { + // Move half of both inputs to the other half +- const auto hi_up = detail::SlideUp(hi, Lanes(DFromV<V>()) / 2); +- const auto lo_down = detail::SlideDown(lo, Lanes(DFromV<V>()) / 2); ++ const auto hi_up = detail::SlideUp(hi, hi, Lanes(DFromV<V>()) / 2); ++ const auto lo_down = detail::SlideDown(lo, lo, Lanes(DFromV<V>()) / 2); + return ConcatUpperLower(hi_up, lo_down); + } + +@@ -1491,61 +1463,55 @@ HWY_API V Combine(const V a, const V b) + // ================================================== REDUCE + + // vector = f(vector, zero_m1) +-#define HWY_RVV_REDUCE(BASE, CHAR, SEW, LMUL, MLEN, NAME, OP) \ +- HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ +- NAME(HWY_RVV_V(BASE, SEW, LMUL) v, HWY_RVV_V(BASE, SEW, 1) v0) { \ +- vsetvlmax_e##SEW##m##LMUL(); \ +- return Set(HWY_RVV_D(CHAR, SEW, LMUL)(), \ +- GetLane(v##OP##_vs_##CHAR##SEW##m##LMUL##_##CHAR##SEW##m1( \ +- v0, v, v0))); \ ++#define HWY_RVV_REDUCE(BASE, CHAR, SEW, LMUL, SHIFT, MLEN, NAME, OP) \ ++ HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ ++ NAME(HWY_RVV_V(BASE, SEW, LMUL) v, HWY_RVV_V(BASE, SEW, m1) v0) { \ ++ vsetvlmax_e##SEW##LMUL(); \ ++ return Set( \ ++ HWY_RVV_D(CHAR, SEW, LMUL)(), \ ++ GetLane(v##OP##_vs_##CHAR##SEW##LMUL##_##CHAR##SEW##m1(v0, v, v0))); \ + } + + // ------------------------------ SumOfLanes + + namespace detail { +- + HWY_RVV_FOREACH_UI(HWY_RVV_REDUCE, RedSum, redsum) + HWY_RVV_FOREACH_F(HWY_RVV_REDUCE, RedSum, fredsum) +- + } // namespace detail + + template <class V> + HWY_API V SumOfLanes(const V v) { + using T = TFromV<V>; +- const auto v0 = Zero(Simd<T, HWY_LANES(T)>()); // always m1 ++ const auto v0 = Zero(Full<T>()); // always m1 + return detail::RedSum(v, v0); + } + + // ------------------------------ MinOfLanes + namespace detail { +- + HWY_RVV_FOREACH_U(HWY_RVV_REDUCE, RedMin, redminu) + HWY_RVV_FOREACH_I(HWY_RVV_REDUCE, RedMin, redmin) + HWY_RVV_FOREACH_F(HWY_RVV_REDUCE, RedMin, fredmin) +- + } // namespace detail + + template <class V> + HWY_API V MinOfLanes(const V v) { + using T = TFromV<V>; +- const Simd<T, HWY_LANES(T)> d1; // always m1 ++ const Full<T> d1; // always m1 + const auto neutral = Set(d1, HighestValue<T>()); + return detail::RedMin(v, neutral); + } + + // ------------------------------ MaxOfLanes + namespace detail { +- + HWY_RVV_FOREACH_U(HWY_RVV_REDUCE, RedMax, redmaxu) + HWY_RVV_FOREACH_I(HWY_RVV_REDUCE, RedMax, redmax) + HWY_RVV_FOREACH_F(HWY_RVV_REDUCE, RedMax, fredmax) +- + } // namespace detail + + template <class V> + HWY_API V MaxOfLanes(const V v) { + using T = TFromV<V>; +- const Simd<T, HWY_LANES(T)> d1; // always m1 ++ const Full<T> d1; // always m1 + const auto neutral = Set(d1, LowestValue<T>()); + return detail::RedMax(v, neutral); + } +@@ -1570,7 +1536,7 @@ HWY_API VFromD<D> LoadDup128(D d, const + #define HWY_RVV_STORE_MASK_BITS(MLEN, NAME, OP) \ + HWY_API size_t StoreMaskBits(HWY_RVV_M(MLEN) m, uint8_t* p) { \ + /* LMUL=1 is always enough */ \ +- Simd<uint8_t, HWY_LANES(uint8_t)> d8; \ ++ Full<uint8_t> d8; \ + const size_t num_bytes = (Lanes(d8) + MLEN - 1) / MLEN; \ + /* TODO(janwas): how to convert vbool* to vuint?*/ \ + /*Store(m, d8, p);*/ \ +@@ -1581,6 +1547,22 @@ HWY_API VFromD<D> LoadDup128(D d, const + HWY_RVV_FOREACH_B(HWY_RVV_STORE_MASK_BITS, _, _) + #undef HWY_RVV_STORE_MASK_BITS + ++// ------------------------------ FirstN (Iota0, Lt, RebindMask, SlideUp) ++ ++// Disallow for 8-bit because Iota is likely to overflow. ++template <class D, HWY_IF_NOT_LANE_SIZE_D(D, 1)> ++HWY_API MFromD<D> FirstN(const D d, const size_t n) { ++ const RebindToSigned<D> di; ++ return RebindMask(d, Lt(BitCast(di, detail::Iota0(d)), Set(di, n))); ++} ++ ++template <class D, HWY_IF_LANE_SIZE_D(D, 1)> ++HWY_API MFromD<D> FirstN(const D d, const size_t n) { ++ const auto zero = Zero(d); ++ const auto one = Set(d, 1); ++ return Eq(detail::SlideUp(one, zero, n), one); ++} ++ + // ------------------------------ Neg + + template <class V, HWY_IF_SIGNED_V(V)> +@@ -1589,9 +1571,9 @@ HWY_API V Neg(const V v) { + } + + // vector = f(vector), but argument is repeated +-#define HWY_RVV_RETV_ARGV2(BASE, CHAR, SEW, LMUL, MLEN, NAME, OP) \ ++#define HWY_RVV_RETV_ARGV2(BASE, CHAR, SEW, LMUL, SHIFT, MLEN, NAME, OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) NAME(HWY_RVV_V(BASE, SEW, LMUL) v) { \ +- return v##OP##_vv_##CHAR##SEW##m##LMUL(v, v); \ ++ return v##OP##_vv_##CHAR##SEW##LMUL(v, v); \ + } + + HWY_RVV_FOREACH_F(HWY_RVV_RETV_ARGV2, Neg, fsgnjn) +@@ -1628,7 +1610,6 @@ template <class V> + HWY_API auto UseInt(const V v) -> decltype(MaskFromVec(v)) { + return Lt(Abs(v), Set(DFromV<V>(), MantissaEnd<TFromV<V>>())); + } +- + } // namespace detail + + template <class V> +@@ -1699,10 +1680,8 @@ HWY_API VFromD<D> Iota(const D d, TFromD + // Using vwmul does not work for m8, so use mulh instead. Highway only provides + // MulHigh for 16-bit, so use a private wrapper. + namespace detail { +- + HWY_RVV_FOREACH_U32(HWY_RVV_RETV_ARGVV, MulHigh, mulhu) + HWY_RVV_FOREACH_I32(HWY_RVV_RETV_ARGVV, MulHigh, mulh) +- + } // namespace detail + + template <class V> +@@ -1712,7 +1691,7 @@ HWY_API VFromD<RepartitionToWide<DFromV< + const auto lo = Mul(a, b); + const auto hi = detail::MulHigh(a, b); + const RepartitionToWide<DFromV<V>> dw; +- return BitCast(dw, OddEven(detail::SlideUp(hi, 1), lo)); ++ return BitCast(dw, OddEven(detail::SlideUp(hi, hi, 1), lo)); + } + + // ================================================== END MACROS +diff -up chromium-92.0.4515.107/third_party/highway/src/hwy/ops/x86_128-inl.h.update-highway-0.12.2 chromium-92.0.4515.107/third_party/highway/src/hwy/ops/x86_128-inl.h +--- chromium-92.0.4515.107/third_party/highway/src/hwy/ops/x86_128-inl.h.update-highway-0.12.2 2021-07-19 14:47:23.000000000 -0400 ++++ chromium-92.0.4515.107/third_party/highway/src/hwy/ops/x86_128-inl.h 2021-07-26 17:19:52.153729522 -0400 +@@ -154,27 +154,28 @@ HWY_API Vec128<double, N> Zero(Simd<doub + // Returns a vector/part with all lanes set to "t". + template <size_t N, HWY_IF_LE128(uint8_t, N)> + HWY_API Vec128<uint8_t, N> Set(Simd<uint8_t, N> /* tag */, const uint8_t t) { +- return Vec128<uint8_t, N>{_mm_set1_epi8(t)}; ++ return Vec128<uint8_t, N>{_mm_set1_epi8(static_cast<char>(t))}; // NOLINT + } + template <size_t N, HWY_IF_LE128(uint16_t, N)> + HWY_API Vec128<uint16_t, N> Set(Simd<uint16_t, N> /* tag */, const uint16_t t) { +- return Vec128<uint16_t, N>{_mm_set1_epi16(t)}; ++ return Vec128<uint16_t, N>{_mm_set1_epi16(static_cast<short>(t))}; // NOLINT + } + template <size_t N, HWY_IF_LE128(uint32_t, N)> + HWY_API Vec128<uint32_t, N> Set(Simd<uint32_t, N> /* tag */, const uint32_t t) { +- return Vec128<uint32_t, N>{_mm_set1_epi32(t)}; ++ return Vec128<uint32_t, N>{_mm_set1_epi32(static_cast<int>(t))}; + } + template <size_t N, HWY_IF_LE128(uint64_t, N)> + HWY_API Vec128<uint64_t, N> Set(Simd<uint64_t, N> /* tag */, const uint64_t t) { +- return Vec128<uint64_t, N>{_mm_set1_epi64x(t)}; ++ return Vec128<uint64_t, N>{ ++ _mm_set1_epi64x(static_cast<long long>(t))}; // NOLINT + } + template <size_t N, HWY_IF_LE128(int8_t, N)> + HWY_API Vec128<int8_t, N> Set(Simd<int8_t, N> /* tag */, const int8_t t) { +- return Vec128<int8_t, N>{_mm_set1_epi8(t)}; ++ return Vec128<int8_t, N>{_mm_set1_epi8(static_cast<char>(t))}; // NOLINT + } + template <size_t N, HWY_IF_LE128(int16_t, N)> + HWY_API Vec128<int16_t, N> Set(Simd<int16_t, N> /* tag */, const int16_t t) { +- return Vec128<int16_t, N>{_mm_set1_epi16(t)}; ++ return Vec128<int16_t, N>{_mm_set1_epi16(static_cast<short>(t))}; // NOLINT + } + template <size_t N, HWY_IF_LE128(int32_t, N)> + HWY_API Vec128<int32_t, N> Set(Simd<int32_t, N> /* tag */, const int32_t t) { +@@ -182,7 +183,8 @@ HWY_API Vec128<int32_t, N> Set(Simd<int3 + } + template <size_t N, HWY_IF_LE128(int64_t, N)> + HWY_API Vec128<int64_t, N> Set(Simd<int64_t, N> /* tag */, const int64_t t) { +- return Vec128<int64_t, N>{_mm_set1_epi64x(t)}; ++ return Vec128<int64_t, N>{ ++ _mm_set1_epi64x(static_cast<long long>(t))}; // NOLINT + } + template <size_t N, HWY_IF_LE128(float, N)> + HWY_API Vec128<float, N> Set(Simd<float, N> /* tag */, const float t) { +@@ -684,6 +686,14 @@ HWY_API Mask128<double, N> operator>=(co + return Mask128<double, N>{_mm_cmpge_pd(a.raw, b.raw)}; + } + ++// ------------------------------ FirstN (Iota, Lt) ++ ++template <typename T, size_t N, HWY_IF_LE128(T, N)> ++HWY_API Mask128<T, N> FirstN(const Simd<T, N> d, size_t num) { ++ const RebindToSigned<decltype(d)> di; // Signed comparisons are cheaper. ++ return RebindMask(d, Iota(di, 0) < Set(di, static_cast<MakeSigned<T>>(num))); ++} ++ + // ================================================== ARITHMETIC + + // ------------------------------ Addition +@@ -895,7 +905,7 @@ template <size_t N> + HWY_API Vec128<int32_t, N> Abs(const Vec128<int32_t, N> v) { + return Vec128<int32_t, N>{_mm_abs_epi32(v.raw)}; + } +- ++// i64 is implemented after BroadcastSignBit. + template <size_t N> + HWY_API Vec128<float, N> Abs(const Vec128<float, N> v) { + const Vec128<int32_t, N> mask{_mm_set1_epi32(0x7FFFFFFF)}; +@@ -1067,15 +1077,24 @@ HWY_API Vec128<int64_t, N> BroadcastSign + return VecFromMask(v < Zero(Simd<int64_t, N>())); + #else + // Efficient Gt() requires SSE4.2 but we only have SSE4.1. BLENDVPD requires +- // two constants and domain crossing. 32-bit compare only requires Zero() +- // plus a shuffle to replicate the upper 32 bits. ++ // two constants and domain crossing. 32-bit shift avoids generating a zero. + const Simd<int32_t, N * 2> d32; +- const auto sign = BitCast(d32, v) < Zero(d32); ++ const auto sign = ShiftRight<31>(BitCast(d32, v)); + return Vec128<int64_t, N>{ + _mm_shuffle_epi32(sign.raw, _MM_SHUFFLE(3, 3, 1, 1))}; + #endif + } + ++template <size_t N> ++HWY_API Vec128<int64_t, N> Abs(const Vec128<int64_t, N> v) { ++#if HWY_TARGET == HWY_AVX3 ++ return Vec128<int64_t, N>{_mm_abs_epi64(v.raw)}; ++#else ++ const auto zero = Zero(Simd<int64_t,N>()); ++ return IfThenElse(MaskFromVec(BroadcastSignBit(v)), zero - v, v); ++#endif ++} ++ + template <int kBits, size_t N> + HWY_API Vec128<int64_t, N> ShiftRight(const Vec128<int64_t, N> v) { + #if HWY_TARGET == HWY_AVX3 +@@ -1787,6 +1806,10 @@ HWY_API void Stream(const Vec128<double, + + // ------------------------------ Scatter + ++// Work around warnings in the intrinsic definitions (passing -1 as a mask). ++HWY_DIAGNOSTICS(push) ++HWY_DIAGNOSTICS_OFF(disable : 4245 4365, ignored "-Wsign-conversion") ++ + // Unfortunately the GCC/Clang intrinsics do not accept int64_t*. + using GatherIndex64 = long long int; // NOLINT(google-runtime-int) + static_assert(sizeof(GatherIndex64) == 8, "Must be 64-bit type"); +@@ -2048,6 +2071,8 @@ HWY_API Vec128<double, N> GatherIndex(Si + + #endif // HWY_TARGET != HWY_SSE4 + ++HWY_DIAGNOSTICS(pop) ++ + // ================================================== SWIZZLE + + // ------------------------------ Extract half +@@ -2075,10 +2100,10 @@ HWY_INLINE Vec128<double, 1> UpperHalf(V + // ------------------------------ Shift vector by constant #bytes + + // 0x01..0F, kBytes = 1 => 0x02..0F00 +-template <int kBytes, typename T> +-HWY_API Vec128<T> ShiftLeftBytes(const Vec128<T> v) { ++template <int kBytes, typename T, size_t N> ++HWY_API Vec128<T, N> ShiftLeftBytes(const Vec128<T, N> v) { + static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes"); +- return Vec128<T>{_mm_slli_si128(v.raw, kBytes)}; ++ return Vec128<T, N>{_mm_slli_si128(v.raw, kBytes)}; + } + + template <int kLanes, typename T, size_t N> +@@ -2089,10 +2114,10 @@ HWY_API Vec128<T, N> ShiftLeftLanes(cons + } + + // 0x01..0F, kBytes = 1 => 0x0001..0E +-template <int kBytes, typename T> +-HWY_API Vec128<T> ShiftRightBytes(const Vec128<T> v) { ++template <int kBytes, typename T, size_t N> ++HWY_API Vec128<T, N> ShiftRightBytes(const Vec128<T, N> v) { + static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes"); +- return Vec128<T>{_mm_srli_si128(v.raw, kBytes)}; ++ return Vec128<T, N>{_mm_srli_si128(v.raw, kBytes)}; + } + + template <int kLanes, typename T, size_t N> +@@ -2257,44 +2282,47 @@ HWY_API Vec128<float> Shuffle0123(const + // ------------------------------ TableLookupLanes + + // Returned by SetTableIndices for use by TableLookupLanes. +-template <typename T> ++template <typename T, size_t N> + struct Indices128 { + __m128i raw; + }; + +-template <typename T> +-HWY_API Indices128<T> SetTableIndices(Full128<T>, const int32_t* idx) { ++template <typename T, size_t N, HWY_IF_LE128(T, N)> ++HWY_API Indices128<T, N> SetTableIndices(Simd<T, N> d, const int32_t* idx) { + #if !defined(NDEBUG) || defined(ADDRESS_SANITIZER) +- const size_t N = 16 / sizeof(T); + for (size_t i = 0; i < N; ++i) { + HWY_DASSERT(0 <= idx[i] && idx[i] < static_cast<int32_t>(N)); + } + #endif + +- const Full128<uint8_t> d8; +- alignas(16) uint8_t control[16]; +- for (size_t idx_byte = 0; idx_byte < 16; ++idx_byte) { +- const size_t idx_lane = idx_byte / sizeof(T); +- const size_t mod = idx_byte % sizeof(T); +- control[idx_byte] = static_cast<uint8_t>(idx[idx_lane] * sizeof(T) + mod); ++ const Repartition<uint8_t, decltype(d)> d8; ++ alignas(16) uint8_t control[16] = {0}; ++ for (size_t idx_lane = 0; idx_lane < N; ++idx_lane) { ++ for (size_t idx_byte = 0; idx_byte < sizeof(T); ++idx_byte) { ++ control[idx_lane * sizeof(T) + idx_byte] = ++ static_cast<uint8_t>(idx[idx_lane] * sizeof(T) + idx_byte); ++ } + } +- return Indices128<T>{Load(d8, control).raw}; ++ return Indices128<T, N>{Load(d8, control).raw}; + } + +-HWY_API Vec128<uint32_t> TableLookupLanes(const Vec128<uint32_t> v, +- const Indices128<uint32_t> idx) { +- return TableLookupBytes(v, Vec128<uint32_t>{idx.raw}); ++template <size_t N> ++HWY_API Vec128<uint32_t, N> TableLookupLanes( ++ const Vec128<uint32_t, N> v, const Indices128<uint32_t, N> idx) { ++ return TableLookupBytes(v, Vec128<uint32_t, N>{idx.raw}); + } +-HWY_API Vec128<int32_t> TableLookupLanes(const Vec128<int32_t> v, +- const Indices128<int32_t> idx) { +- return TableLookupBytes(v, Vec128<int32_t>{idx.raw}); ++template <size_t N> ++HWY_API Vec128<int32_t, N> TableLookupLanes(const Vec128<int32_t, N> v, ++ const Indices128<int32_t, N> idx) { ++ return TableLookupBytes(v, Vec128<int32_t, N>{idx.raw}); + } +-HWY_API Vec128<float> TableLookupLanes(const Vec128<float> v, +- const Indices128<float> idx) { +- const Full128<int32_t> di; +- const Full128<float> df; ++template <size_t N> ++HWY_API Vec128<float, N> TableLookupLanes(const Vec128<float, N> v, ++ const Indices128<float, N> idx) { ++ const Simd<int32_t, N> di; ++ const Simd<float, N> df; + return BitCast(df, +- TableLookupBytes(BitCast(di, v), Vec128<int32_t>{idx.raw})); ++ TableLookupBytes(BitCast(di, v), Vec128<int32_t, N>{idx.raw})); + } + + // ------------------------------ Interleave lanes +@@ -2502,47 +2530,47 @@ HWY_INLINE Vec128<double> ConcatUpperLow + + namespace detail { + +-template <typename T> +-HWY_API Vec128<T> OddEven(hwy::SizeTag<1> /* tag */, const Vec128<T> a, +- const Vec128<T> b) { +- const Full128<T> d; +- const Full128<uint8_t> d8; ++template <typename T, size_t N> ++HWY_API Vec128<T, N> OddEven(hwy::SizeTag<1> /* tag */, const Vec128<T, N> a, ++ const Vec128<T, N> b) { ++ const Simd<T, N> d; ++ const Repartition<uint8_t, decltype(d)> d8; + alignas(16) constexpr uint8_t mask[16] = {0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, + 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0}; + return IfThenElse(MaskFromVec(BitCast(d, Load(d8, mask))), b, a); + } +-template <typename T> +-HWY_API Vec128<T> OddEven(hwy::SizeTag<2> /* tag */, const Vec128<T> a, +- const Vec128<T> b) { +- return Vec128<T>{_mm_blend_epi16(a.raw, b.raw, 0x55)}; ++template <typename T, size_t N> ++HWY_API Vec128<T, N> OddEven(hwy::SizeTag<2> /* tag */, const Vec128<T, N> a, ++ const Vec128<T, N> b) { ++ return Vec128<T, N>{_mm_blend_epi16(a.raw, b.raw, 0x55)}; + } +-template <typename T> +-HWY_API Vec128<T> OddEven(hwy::SizeTag<4> /* tag */, const Vec128<T> a, +- const Vec128<T> b) { +- return Vec128<T>{_mm_blend_epi16(a.raw, b.raw, 0x33)}; ++template <typename T, size_t N> ++HWY_API Vec128<T, N> OddEven(hwy::SizeTag<4> /* tag */, const Vec128<T, N> a, ++ const Vec128<T, N> b) { ++ return Vec128<T, N>{_mm_blend_epi16(a.raw, b.raw, 0x33)}; + } +-template <typename T> +-HWY_API Vec128<T> OddEven(hwy::SizeTag<8> /* tag */, const Vec128<T> a, +- const Vec128<T> b) { +- return Vec128<T>{_mm_blend_epi16(a.raw, b.raw, 0x0F)}; ++template <typename T, size_t N> ++HWY_API Vec128<T, N> OddEven(hwy::SizeTag<8> /* tag */, const Vec128<T, N> a, ++ const Vec128<T, N> b) { ++ return Vec128<T, N>{_mm_blend_epi16(a.raw, b.raw, 0x0F)}; + } + + } // namespace detail + +-template <typename T> +-HWY_API Vec128<T> OddEven(const Vec128<T> a, const Vec128<T> b) { ++template <typename T, size_t N> ++HWY_API Vec128<T, N> OddEven(const Vec128<T, N> a, const Vec128<T, N> b) { + return detail::OddEven(hwy::SizeTag<sizeof(T)>(), a, b); + } +-template <> +-HWY_INLINE Vec128<float> OddEven<float>(const Vec128<float> a, +- const Vec128<float> b) { +- return Vec128<float>{_mm_blend_ps(a.raw, b.raw, 5)}; ++template <size_t N> ++HWY_INLINE Vec128<float, N> OddEven(const Vec128<float, N> a, ++ const Vec128<float, N> b) { ++ return Vec128<float, N>{_mm_blend_ps(a.raw, b.raw, 5)}; + } + +-template <> +-HWY_INLINE Vec128<double> OddEven<double>(const Vec128<double> a, +- const Vec128<double> b) { +- return Vec128<double>{_mm_blend_pd(a.raw, b.raw, 1)}; ++template <size_t N> ++HWY_INLINE Vec128<double, N> OddEven(const Vec128<double, N> a, ++ const Vec128<double, N> b) { ++ return Vec128<double, N>{_mm_blend_pd(a.raw, b.raw, 1)}; + } + + // ------------------------------ Shl (ZipLower, Mul) +@@ -2980,7 +3008,7 @@ HWY_API Vec128<uint8_t, N> U8FromU32(con + return LowerHalf(LowerHalf(BitCast(d8, quad))); + } + +-// ------------------------------ Convert integer <=> floating point ++// ------------------------------ Integer <=> fp (ShiftRight, OddEven) + + template <size_t N> + HWY_API Vec128<float, N> ConvertTo(Simd<float, N> /* tag */, +@@ -2995,13 +3023,20 @@ HWY_API Vec128<double, N> ConvertTo(Simd + (void)dd; + return Vec128<double, N>{_mm_cvtepi64_pd(v.raw)}; + #else +- alignas(16) int64_t lanes_i[2]; +- Store(v, Simd<int64_t, N>(), lanes_i); +- alignas(16) double lanes_d[2]; +- for (size_t i = 0; i < N; ++i) { +- lanes_d[i] = static_cast<double>(lanes_i[i]); +- } +- return Load(dd, lanes_d); ++ // Based on wim's approach (https://stackoverflow.com/questions/41144668/) ++ const Repartition<uint32_t, decltype(dd)> d32; ++ const Repartition<uint64_t, decltype(dd)> d64; ++ ++ // Toggle MSB of lower 32-bits and insert exponent for 2^84 + 2^63 ++ const auto k84_63 = Set(d64, 0x4530000080000000ULL); ++ const auto v_upper = BitCast(dd, ShiftRight<32>(BitCast(d64, v)) ^ k84_63); ++ ++ // Exponent is 2^52, lower 32 bits from v (=> 32-bit OddEven) ++ const auto k52 = Set(d32, 0x43300000); ++ const auto v_lower = BitCast(dd, OddEven(k52, BitCast(d32, v))); ++ ++ const auto k84_63_52 = BitCast(dd, Set(d64, 0x4530000080100000ULL)); ++ return (v_upper - k84_63_52) + v_lower; // order matters! + #endif + } + +@@ -3572,55 +3607,87 @@ HWY_API void StoreInterleaved4(const Vec + + namespace detail { + +-// For u32/i32/f32. +-template <typename T, size_t N> +-HWY_API Vec128<T, N> SumOfLanes(hwy::SizeTag<4> /* tag */, +- const Vec128<T, N> v3210) { ++// N=1 for any T: no-op ++template <typename T> ++HWY_API Vec128<T, 1> SumOfLanes(hwy::SizeTag<sizeof(T)> /* tag */, ++ const Vec128<T, 1> v) { ++ return v; ++} ++template <typename T> ++HWY_API Vec128<T, 1> MinOfLanes(hwy::SizeTag<sizeof(T)> /* tag */, ++ const Vec128<T, 1> v) { ++ return v; ++} ++template <typename T> ++HWY_API Vec128<T, 1> MaxOfLanes(hwy::SizeTag<sizeof(T)> /* tag */, ++ const Vec128<T, 1> v) { ++ return v; ++} ++ ++// u32/i32/f32: ++ ++// N=2 ++template <typename T> ++HWY_API Vec128<T, 2> SumOfLanes(hwy::SizeTag<4> /* tag */, ++ const Vec128<T, 2> v10) { ++ return v10 + Vec128<T, 2>{Shuffle2301(Vec128<T>{v10.raw}).raw}; ++} ++template <typename T> ++HWY_API Vec128<T, 2> MinOfLanes(hwy::SizeTag<4> /* tag */, ++ const Vec128<T, 2> v10) { ++ return Min(v10, Vec128<T, 2>{Shuffle2301(Vec128<T>{v10.raw}).raw}); ++} ++template <typename T> ++HWY_API Vec128<T, 2> MaxOfLanes(hwy::SizeTag<4> /* tag */, ++ const Vec128<T, 2> v10) { ++ return Max(v10, Vec128<T, 2>{Shuffle2301(Vec128<T>{v10.raw}).raw}); ++} ++ ++// N=4 (full) ++template <typename T> ++HWY_API Vec128<T> SumOfLanes(hwy::SizeTag<4> /* tag */, const Vec128<T> v3210) { + const Vec128<T> v1032 = Shuffle1032(v3210); + const Vec128<T> v31_20_31_20 = v3210 + v1032; + const Vec128<T> v20_31_20_31 = Shuffle0321(v31_20_31_20); + return v20_31_20_31 + v31_20_31_20; + } +-template <typename T, size_t N> +-HWY_API Vec128<T, N> MinOfLanes(hwy::SizeTag<4> /* tag */, +- const Vec128<T, N> v3210) { ++template <typename T> ++HWY_API Vec128<T> MinOfLanes(hwy::SizeTag<4> /* tag */, const Vec128<T> v3210) { + const Vec128<T> v1032 = Shuffle1032(v3210); + const Vec128<T> v31_20_31_20 = Min(v3210, v1032); + const Vec128<T> v20_31_20_31 = Shuffle0321(v31_20_31_20); + return Min(v20_31_20_31, v31_20_31_20); + } +-template <typename T, size_t N> +-HWY_API Vec128<T, N> MaxOfLanes(hwy::SizeTag<4> /* tag */, +- const Vec128<T, N> v3210) { ++template <typename T> ++HWY_API Vec128<T> MaxOfLanes(hwy::SizeTag<4> /* tag */, const Vec128<T> v3210) { + const Vec128<T> v1032 = Shuffle1032(v3210); + const Vec128<T> v31_20_31_20 = Max(v3210, v1032); + const Vec128<T> v20_31_20_31 = Shuffle0321(v31_20_31_20); + return Max(v20_31_20_31, v31_20_31_20); + } + +-// For u64/i64/f64. +-template <typename T, size_t N> +-HWY_API Vec128<T, N> SumOfLanes(hwy::SizeTag<8> /* tag */, +- const Vec128<T, N> v10) { ++// u64/i64/f64: ++ ++// N=2 (full) ++template <typename T> ++HWY_API Vec128<T> SumOfLanes(hwy::SizeTag<8> /* tag */, const Vec128<T> v10) { + const Vec128<T> v01 = Shuffle01(v10); + return v10 + v01; + } +-template <typename T, size_t N> +-HWY_API Vec128<T, N> MinOfLanes(hwy::SizeTag<8> /* tag */, +- const Vec128<T, N> v10) { ++template <typename T> ++HWY_API Vec128<T> MinOfLanes(hwy::SizeTag<8> /* tag */, const Vec128<T> v10) { + const Vec128<T> v01 = Shuffle01(v10); + return Min(v10, v01); + } +-template <typename T, size_t N> +-HWY_API Vec128<T, N> MaxOfLanes(hwy::SizeTag<8> /* tag */, +- const Vec128<T, N> v10) { ++template <typename T> ++HWY_API Vec128<T> MaxOfLanes(hwy::SizeTag<8> /* tag */, const Vec128<T> v10) { + const Vec128<T> v01 = Shuffle01(v10); + return Max(v10, v01); + } + + } // namespace detail + +-// Supported for u/i/f 32/64. Returns the sum in each lane. ++// Supported for u/i/f 32/64. Returns the same value in each lane. + template <typename T, size_t N> + HWY_API Vec128<T, N> SumOfLanes(const Vec128<T, N> v) { + return detail::SumOfLanes(hwy::SizeTag<sizeof(T)>(), v); +diff -up chromium-92.0.4515.107/third_party/highway/src/hwy/ops/x86_256-inl.h.update-highway-0.12.2 chromium-92.0.4515.107/third_party/highway/src/hwy/ops/x86_256-inl.h +--- chromium-92.0.4515.107/third_party/highway/src/hwy/ops/x86_256-inl.h.update-highway-0.12.2 2021-07-19 14:47:23.000000000 -0400 ++++ chromium-92.0.4515.107/third_party/highway/src/hwy/ops/x86_256-inl.h 2021-07-26 17:19:30.740403369 -0400 +@@ -20,15 +20,18 @@ + // particular, "Broadcast", pack and zip behavior may be surprising. + + #include <immintrin.h> // AVX2+ ++ + #if defined(_MSC_VER) && defined(__clang__) + // Including <immintrin.h> should be enough, but Clang's headers helpfully skip + // including these headers when _MSC_VER is defined, like when using clang-cl. + // Include these directly here. +-#include <smmintrin.h> + #include <avxintrin.h> ++// avxintrin defines __m256i and must come before avx2intrin. + #include <avx2intrin.h> ++#include <bmi2intrin.h> // _pext_u64 + #include <f16cintrin.h> + #include <fmaintrin.h> ++#include <smmintrin.h> + #endif + + #include <stddef.h> +@@ -159,23 +162,24 @@ HWY_API Vec256<uint16_t> Set(Full256<uin + return Vec256<uint16_t>{_mm256_set1_epi16(static_cast<short>(t))}; // NOLINT + } + HWY_API Vec256<uint32_t> Set(Full256<uint32_t> /* tag */, const uint32_t t) { +- return Vec256<uint32_t>{_mm256_set1_epi32(static_cast<int>(t))}; // NOLINT ++ return Vec256<uint32_t>{_mm256_set1_epi32(static_cast<int>(t))}; + } + HWY_API Vec256<uint64_t> Set(Full256<uint64_t> /* tag */, const uint64_t t) { + return Vec256<uint64_t>{ + _mm256_set1_epi64x(static_cast<long long>(t))}; // NOLINT + } + HWY_API Vec256<int8_t> Set(Full256<int8_t> /* tag */, const int8_t t) { +- return Vec256<int8_t>{_mm256_set1_epi8(t)}; ++ return Vec256<int8_t>{_mm256_set1_epi8(static_cast<char>(t))}; // NOLINT + } + HWY_API Vec256<int16_t> Set(Full256<int16_t> /* tag */, const int16_t t) { +- return Vec256<int16_t>{_mm256_set1_epi16(t)}; ++ return Vec256<int16_t>{_mm256_set1_epi16(static_cast<short>(t))}; // NOLINT + } + HWY_API Vec256<int32_t> Set(Full256<int32_t> /* tag */, const int32_t t) { + return Vec256<int32_t>{_mm256_set1_epi32(t)}; + } + HWY_API Vec256<int64_t> Set(Full256<int64_t> /* tag */, const int64_t t) { +- return Vec256<int64_t>{_mm256_set1_epi64x(t)}; ++ return Vec256<int64_t>{ ++ _mm256_set1_epi64x(static_cast<long long>(t))}; // NOLINT + } + HWY_API Vec256<float> Set(Full256<float> /* tag */, const float t) { + return Vec256<float>{_mm256_set1_ps(t)}; +@@ -351,6 +355,8 @@ HWY_API Vec256<T> VecFromMask(Full256<T> + return Vec256<T>{v.raw}; + } + ++// ------------------------------ IfThenElse ++ + // mask ? yes : no + template <typename T> + HWY_API Vec256<T> IfThenElse(const Mask256<T> mask, const Vec256<T> yes, +@@ -681,6 +687,14 @@ HWY_API Vec256<double> Max(const Vec256< + return Vec256<double>{_mm256_max_pd(a.raw, b.raw)}; + } + ++// ------------------------------ FirstN (Iota, Lt) ++ ++template <typename T> ++HWY_API Mask256<T> FirstN(const Full256<T> d, size_t n) { ++ const RebindToSigned<decltype(d)> di; // Signed comparisons are cheaper. ++ return RebindMask(d, Iota(di, 0) < Set(di, static_cast<MakeSigned<T>>(n))); ++} ++ + // ================================================== ARITHMETIC + + // ------------------------------ Addition +@@ -843,7 +857,13 @@ HWY_API Vec256<uint16_t> AverageRound(co + + // Returns absolute value, except that LimitsMin() maps to LimitsMax() + 1. + HWY_API Vec256<int8_t> Abs(const Vec256<int8_t> v) { ++#if HWY_COMPILER_MSVC ++ // Workaround for incorrect codegen? (wrong result) ++ const auto zero = Zero(Full256<int8_t>()); ++ return Vec256<int8_t>{_mm256_max_epi8(v.raw, (zero - v).raw)}; ++#else + return Vec256<int8_t>{_mm256_abs_epi8(v.raw)}; ++#endif + } + HWY_API Vec256<int16_t> Abs(const Vec256<int16_t> v) { + return Vec256<int16_t>{_mm256_abs_epi16(v.raw)}; +@@ -851,6 +871,7 @@ HWY_API Vec256<int16_t> Abs(const Vec256 + HWY_API Vec256<int32_t> Abs(const Vec256<int32_t> v) { + return Vec256<int32_t>{_mm256_abs_epi32(v.raw)}; + } ++// i64 is implemented after BroadcastSignBit. + + HWY_API Vec256<float> Abs(const Vec256<float> v) { + const Vec256<int32_t> mask{_mm256_set1_epi32(0x7FFFFFFF)}; +@@ -1027,6 +1048,15 @@ HWY_API Vec256<int64_t> ShiftRight(const + #endif + } + ++HWY_API Vec256<int64_t> Abs(const Vec256<int64_t> v) { ++#if HWY_TARGET == HWY_AVX3 ++ return Vec256<int64_t>{_mm256_abs_epi64(v.raw)}; ++#else ++ const auto zero = Zero(Full256<int64_t>()); ++ return IfThenElse(MaskFromVec(BroadcastSignBit(v)), zero - v, v); ++#endif ++} ++ + // ------------------------------ ShiftLeftSame + + HWY_API Vec256<uint16_t> ShiftLeftSame(const Vec256<uint16_t> v, +@@ -1398,6 +1428,10 @@ HWY_API void Stream(const Vec256<double> + + // ------------------------------ Scatter + ++// Work around warnings in the intrinsic definitions (passing -1 as a mask). ++HWY_DIAGNOSTICS(push) ++HWY_DIAGNOSTICS_OFF(disable : 4245 4365, ignored "-Wsign-conversion") ++ + #if HWY_TARGET == HWY_AVX3 + namespace detail { + +@@ -1584,6 +1618,8 @@ HWY_INLINE Vec256<double> GatherIndex<do + return Vec256<double>{_mm256_i64gather_pd(base, index.raw, 8)}; + } + ++HWY_DIAGNOSTICS(pop) ++ + // ================================================== SWIZZLE + + template <typename T> +@@ -2379,11 +2415,18 @@ HWY_API Vec128<int8_t> DemoteTo(Full128< + _mm256_castsi256_si128(_mm256_permute4x64_epi64(i8, 0x88))}; + } + ++ // Avoid "value of intrinsic immediate argument '8' is out of range '0 - 7'". ++ // 8 is the correct value of _MM_FROUND_NO_EXC, which is allowed here. ++HWY_DIAGNOSTICS(push) ++HWY_DIAGNOSTICS_OFF(disable : 4556, ignored "-Wsign-conversion") ++ + HWY_API Vec128<float16_t> DemoteTo(Full128<float16_t> /* tag */, + const Vec256<float> v) { + return Vec128<float16_t>{_mm256_cvtps_ph(v.raw, _MM_FROUND_NO_EXC)}; + } + ++HWY_DIAGNOSTICS(pop) ++ + HWY_API Vec128<float> DemoteTo(Full128<float> /* tag */, + const Vec256<double> v) { + return Vec128<float>{_mm256_cvtpd_ps(v.raw)}; +@@ -2409,7 +2452,7 @@ HWY_API Vec128<uint8_t, 8> U8FromU32(con + return BitCast(Simd<uint8_t, 8>(), pair); + } + +-// ------------------------------ Convert integer <=> floating point ++// ------------------------------ Integer <=> fp (ShiftRight, OddEven) + + HWY_API Vec256<float> ConvertTo(Full256<float> /* tag */, + const Vec256<int32_t> v) { +@@ -2421,13 +2464,20 @@ HWY_API Vec256<double> ConvertTo(Full256 + (void)dd; + return Vec256<double>{_mm256_cvtepi64_pd(v.raw)}; + #else +- alignas(32) int64_t lanes_i[4]; +- Store(v, Full256<int64_t>(), lanes_i); +- alignas(32) double lanes_d[4]; +- for (size_t i = 0; i < 4; ++i) { +- lanes_d[i] = static_cast<double>(lanes_i[i]); +- } +- return Load(dd, lanes_d); ++ // Based on wim's approach (https://stackoverflow.com/questions/41144668/) ++ const Repartition<uint32_t, decltype(dd)> d32; ++ const Repartition<uint64_t, decltype(dd)> d64; ++ ++ // Toggle MSB of lower 32-bits and insert exponent for 2^84 + 2^63 ++ const auto k84_63 = Set(d64, 0x4530000080000000ULL); ++ const auto v_upper = BitCast(dd, ShiftRight<32>(BitCast(d64, v)) ^ k84_63); ++ ++ // Exponent is 2^52, lower 32 bits from v (=> 32-bit OddEven) ++ const auto k52 = Set(d32, 0x43300000); ++ const auto v_lower = BitCast(dd, OddEven(k52, BitCast(d32, v))); ++ ++ const auto k84_63_52 = BitCast(dd, Set(d64, 0x4530000080100000ULL)); ++ return (v_upper - k84_63_52) + v_lower; // order matters! + #endif + } + +@@ -2502,8 +2552,7 @@ HWY_API uint64_t BitsFromMask(hwy::SizeT + const auto compressed = + _mm256_permute4x64_epi64(sign_bits, _MM_SHUFFLE(3, 1, 2, 0)); + return static_cast<unsigned>(_mm256_movemask_epi8(compressed)); +- +-#endif ++#endif // HWY_ARCH_X86_64 + } + + template <typename T> +diff -up chromium-92.0.4515.107/third_party/highway/src/hwy/targets.cc.update-highway-0.12.2 chromium-92.0.4515.107/third_party/highway/src/hwy/targets.cc +--- chromium-92.0.4515.107/third_party/highway/src/hwy/targets.cc.update-highway-0.12.2 2021-07-19 14:47:23.000000000 -0400 ++++ chromium-92.0.4515.107/third_party/highway/src/hwy/targets.cc 2021-07-26 17:17:24.610482240 -0400 +@@ -32,8 +32,8 @@ + #include <intrin.h> + #else // HWY_COMPILER_MSVC + #include <cpuid.h> +-#endif // HWY_COMPILER_MSVC +-#endif ++#endif // HWY_COMPILER_MSVC ++#endif // HWY_ARCH_X86 + + namespace hwy { + namespace { +@@ -126,7 +126,7 @@ constexpr uint32_t kAVX512VL = 1u << 13; + constexpr uint32_t kAVX512DQ = 1u << 14; + constexpr uint32_t kAVX512BW = 1u << 15; + constexpr uint32_t kGroupAVX3 = kAVX512F | kAVX512VL | kAVX512DQ | kAVX512BW; +-#endif ++#endif // HWY_ARCH_X86 + + } // namespace + +diff -up chromium-92.0.4515.107/third_party/highway/src/hwy/targets.h.update-highway-0.12.2 chromium-92.0.4515.107/third_party/highway/src/hwy/targets.h +--- chromium-92.0.4515.107/third_party/highway/src/hwy/targets.h.update-highway-0.12.2 2021-07-19 14:47:23.000000000 -0400 ++++ chromium-92.0.4515.107/third_party/highway/src/hwy/targets.h 2021-07-26 17:17:24.610482240 -0400 +@@ -65,7 +65,9 @@ + // HWY_MAX_DYNAMIC_TARGETS in total. + #define HWY_HIGHEST_TARGET_BIT_X86 9 + +-// 0x400, 0x800, 0x1000 reserved for SVE, SVE2, Helium ++#define HWY_SVE2 0x400 ++#define HWY_SVE 0x800 ++// 0x1000 reserved for Helium + #define HWY_NEON 0x2000 + + #define HWY_HIGHEST_TARGET_BIT_ARM 13 +@@ -90,6 +92,9 @@ + // 0x2000000, 0x4000000, 0x8000000, 0x10000000 reserved + + #define HWY_SCALAR 0x20000000 ++ ++#define HWY_HIGHEST_TARGET_BIT_SCALAR 29 ++ + // Cannot use higher values, otherwise HWY_TARGETS computation might overflow. + + //------------------------------------------------------------------------------ +@@ -106,25 +111,26 @@ + #ifndef HWY_BROKEN_TARGETS + + // x86 clang-6: we saw multiple AVX2/3 compile errors and in one case invalid +-// SSE4 codegen (msan failure), so disable all those targets. ++// SSE4 codegen (possibly only for msan), so disable all those targets. + #if HWY_ARCH_X86 && (HWY_COMPILER_CLANG != 0 && HWY_COMPILER_CLANG < 700) +-// TODO: Disable all non-scalar targets for every build target once we have +-// clang-7 enabled in our builders. +-#ifdef MEMORY_SANITIZER + #define HWY_BROKEN_TARGETS (HWY_SSE4 | HWY_AVX2 | HWY_AVX3) +-#else +-#define HWY_BROKEN_TARGETS 0 +-#endif + // This entails a major speed reduction, so warn unless the user explicitly + // opts in to scalar-only. + #if !defined(HWY_COMPILE_ONLY_SCALAR) + #pragma message("x86 Clang <= 6: define HWY_COMPILE_ONLY_SCALAR or upgrade.") + #endif + +-// MSVC, or 32-bit may fail to compile AVX2/3. +-#elif HWY_COMPILER_MSVC != 0 || HWY_ARCH_X86_32 ++// 32-bit may fail to compile AVX2/3. ++#elif HWY_ARCH_X86_32 + #define HWY_BROKEN_TARGETS (HWY_AVX2 | HWY_AVX3) +-#pragma message("Disabling AVX2/3 due to known issues with MSVC/32-bit builds") ++ ++// MSVC AVX3 support is buggy: https://github.com/Mysticial/Flops/issues/16 ++#elif HWY_COMPILER_MSVC != 0 ++#define HWY_BROKEN_TARGETS (HWY_AVX3) ++ ++// armv7be has not been tested and is not yet supported. ++#elif HWY_ARCH_ARM_V7 && (defined(__ARM_BIG_ENDIAN) || defined(__BIG_ENDIAN)) ++#define HWY_BROKEN_TARGETS (HWY_NEON) + + #else + #define HWY_BROKEN_TARGETS 0 +@@ -145,53 +151,74 @@ + // user to override this without any guarantee of success. + #ifndef HWY_BASELINE_TARGETS + +-#ifdef __wasm_simd128__ ++// Also check HWY_ARCH to ensure that simulating unknown platforms ends up with ++// HWY_TARGET == HWY_SCALAR. ++ ++#if HWY_ARCH_WASM && defined(__wasm_simd128__) + #define HWY_BASELINE_WASM HWY_WASM + #else + #define HWY_BASELINE_WASM 0 + #endif + +-#ifdef __VSX__ ++// Avoid choosing the PPC target until we have an implementation. ++#if HWY_ARCH_PPC && defined(__VSX__) && 0 + #define HWY_BASELINE_PPC8 HWY_PPC8 + #else + #define HWY_BASELINE_PPC8 0 + #endif + +-// GCC 4.5.4 only defines the former; 5.4 defines both. +-#if defined(__ARM_NEON__) || defined(__ARM_NEON) ++// Avoid choosing the SVE[2] targets the implementation is ready. ++#if HWY_ARCH_ARM && defined(__ARM_FEATURE_SVE2) && 0 ++#define HWY_BASELINE_SVE2 HWY_SVE2 ++#else ++#define HWY_BASELINE_SVE2 0 ++#endif ++ ++#if HWY_ARCH_ARM && defined(__ARM_FEATURE_SVE) && 0 ++#define HWY_BASELINE_SVE HWY_SVE ++#else ++#define HWY_BASELINE_SVE 0 ++#endif ++ ++// GCC 4.5.4 only defines __ARM_NEON__; 5.4 defines both. ++#if HWY_ARCH_ARM && (defined(__ARM_NEON__) || defined(__ARM_NEON)) + #define HWY_BASELINE_NEON HWY_NEON + #else + #define HWY_BASELINE_NEON 0 + #endif + +-#ifdef __SSE4_1__ ++// MSVC does not set SSE4_1, but it does set AVX; checking for the latter means ++// we at least get SSE4 on machines supporting AVX but not AVX2. ++// https://stackoverflow.com/questions/18563978/ ++#if HWY_ARCH_X86 && \ ++ (defined(__SSE4_1__) || (HWY_COMPILER_MSVC != 0 && defined(__AVX__))) + #define HWY_BASELINE_SSE4 HWY_SSE4 + #else + #define HWY_BASELINE_SSE4 0 + #endif + +-#ifdef __AVX2__ ++#if HWY_ARCH_X86 && defined(__AVX2__) + #define HWY_BASELINE_AVX2 HWY_AVX2 + #else + #define HWY_BASELINE_AVX2 0 + #endif + +-#ifdef __AVX512F__ ++#if HWY_ARCH_X86 && defined(__AVX512F__) + #define HWY_BASELINE_AVX3 HWY_AVX3 + #else + #define HWY_BASELINE_AVX3 0 + #endif + +-#ifdef __riscv_vector ++#if HWY_ARCH_RVV && defined(__riscv_vector) + #define HWY_BASELINE_RVV HWY_RVV + #else + #define HWY_BASELINE_RVV 0 + #endif + + #define HWY_BASELINE_TARGETS \ +- (HWY_SCALAR | HWY_BASELINE_WASM | HWY_BASELINE_PPC8 | HWY_BASELINE_NEON | \ +- HWY_BASELINE_SSE4 | HWY_BASELINE_AVX2 | HWY_BASELINE_AVX3 | \ +- HWY_BASELINE_RVV) ++ (HWY_SCALAR | HWY_BASELINE_WASM | HWY_BASELINE_PPC8 | HWY_BASELINE_SVE2 | \ ++ HWY_BASELINE_SVE | HWY_BASELINE_NEON | HWY_BASELINE_SSE4 | \ ++ HWY_BASELINE_AVX2 | HWY_BASELINE_AVX3 | HWY_BASELINE_RVV) + + #endif // HWY_BASELINE_TARGETS + +@@ -242,13 +269,12 @@ + #define HWY_TARGETS HWY_STATIC_TARGET + + // 3) For tests: include all attainable targets (in particular: scalar) +-#elif defined(HWY_COMPILE_ALL_ATTAINABLE) ++#elif defined(HWY_COMPILE_ALL_ATTAINABLE) || defined(HWY_IS_TEST) + #define HWY_TARGETS HWY_ATTAINABLE_TARGETS + + // 4) Default: attainable WITHOUT non-best baseline. This reduces code size by + // excluding superseded targets, in particular scalar. + #else +- + #define HWY_TARGETS (HWY_ATTAINABLE_TARGETS & (2 * HWY_STATIC_TARGET - 1)) + + #endif // target policy +@@ -323,6 +349,10 @@ static inline HWY_MAYBE_UNUSED const cha + #endif + + #if HWY_ARCH_ARM ++ case HWY_SVE2: ++ return "SVE2"; ++ case HWY_SVE: ++ return "SVE"; + case HWY_NEON: + return "Neon"; + #endif +@@ -346,7 +376,7 @@ static inline HWY_MAYBE_UNUSED const cha + return "Scalar"; + + default: +- return "?"; ++ return "Unknown"; // must satisfy gtest IsValidParamName() + } + } + +@@ -405,21 +435,17 @@ static inline HWY_MAYBE_UNUSED const cha + nullptr, /* SSE3 */ \ + nullptr /* SSE2 */ + +-#endif // HWY_ARCH_X86 +- +-#if HWY_ARCH_ARM ++#elif HWY_ARCH_ARM + // See HWY_ARCH_X86 above for details. + #define HWY_MAX_DYNAMIC_TARGETS 4 + #define HWY_HIGHEST_TARGET_BIT HWY_HIGHEST_TARGET_BIT_ARM + #define HWY_CHOOSE_TARGET_LIST(func_name) \ +- nullptr, /* reserved */ \ +- nullptr, /* reserved */ \ ++ HWY_CHOOSE_SVE2(func_name), /* SVE2 */ \ ++ HWY_CHOOSE_SVE(func_name), /* SVE */ \ + nullptr, /* reserved */ \ + HWY_CHOOSE_NEON(func_name) /* NEON */ + +-#endif // HWY_ARCH_ARM +- +-#if HWY_ARCH_PPC ++#elif HWY_ARCH_PPC + // See HWY_ARCH_X86 above for details. + #define HWY_MAX_DYNAMIC_TARGETS 5 + #define HWY_HIGHEST_TARGET_BIT HWY_HIGHEST_TARGET_BIT_PPC +@@ -430,9 +456,7 @@ static inline HWY_MAYBE_UNUSED const cha + nullptr, /* VSX */ \ + nullptr /* AltiVec */ + +-#endif // HWY_ARCH_PPC +- +-#if HWY_ARCH_WASM ++#elif HWY_ARCH_WASM + // See HWY_ARCH_X86 above for details. + #define HWY_MAX_DYNAMIC_TARGETS 4 + #define HWY_HIGHEST_TARGET_BIT HWY_HIGHEST_TARGET_BIT_WASM +@@ -442,9 +466,7 @@ static inline HWY_MAYBE_UNUSED const cha + nullptr, /* reserved */ \ + HWY_CHOOSE_WASM(func_name) /* WASM */ + +-#endif // HWY_ARCH_WASM +- +-#if HWY_ARCH_RVV ++#elif HWY_ARCH_RVV + // See HWY_ARCH_X86 above for details. + #define HWY_MAX_DYNAMIC_TARGETS 4 + #define HWY_HIGHEST_TARGET_BIT HWY_HIGHEST_TARGET_BIT_RVV +@@ -454,7 +476,12 @@ static inline HWY_MAYBE_UNUSED const cha + nullptr, /* reserved */ \ + HWY_CHOOSE_RVV(func_name) /* RVV */ + +-#endif // HWY_ARCH_RVV ++#else ++// Unknown architecture, will use HWY_SCALAR without dynamic dispatch, though ++// still creating single-entry tables in HWY_EXPORT to ensure portability. ++#define HWY_MAX_DYNAMIC_TARGETS 1 ++#define HWY_HIGHEST_TARGET_BIT HWY_HIGHEST_TARGET_BIT_SCALAR ++#endif + + struct ChosenTarget { + public: +diff -up chromium-92.0.4515.107/third_party/highway/src/hwy/tests/memory_test.cc.update-highway-0.12.2 chromium-92.0.4515.107/third_party/highway/src/hwy/tests/memory_test.cc +--- chromium-92.0.4515.107/third_party/highway/src/hwy/tests/memory_test.cc.update-highway-0.12.2 2021-07-19 14:47:23.000000000 -0400 ++++ chromium-92.0.4515.107/third_party/highway/src/hwy/tests/memory_test.cc 2021-07-26 17:10:40.022319820 -0400 +@@ -12,6 +12,12 @@ + // See the License for the specific language governing permissions and + // limitations under the License. + ++// Ensure incompabilities with Windows macros (e.g. #define StoreFence) are ++// detected. Must come before Highway headers. ++#if defined(_WIN32) || defined(_WIN64) ++#include <Windows.h> ++#endif ++ + #include <stddef.h> + #include <stdint.h> + +@@ -199,13 +205,14 @@ struct TestLoadDup128 { + for (size_t i = 0; i < N128; ++i) { + lanes[i] = static_cast<T>(1 + i); + } +- const auto v = LoadDup128(d, lanes); ++ + const size_t N = Lanes(d); +- auto out = AllocateAligned<T>(N); +- Store(v, d, out.get()); ++ auto expected = AllocateAligned<T>(N); + for (size_t i = 0; i < N; ++i) { +- HWY_ASSERT_EQ(T(i % N128 + 1), out[i]); ++ expected[i] = static_cast<T>(i % N128 + 1); + } ++ ++ HWY_ASSERT_VEC_EQ(d, expected.get(), LoadDup128(d, lanes)); + #else + (void)d; + #endif +@@ -327,6 +334,84 @@ HWY_NOINLINE void TestAllScatter() { + ForFloatTypes(test); + } + ++// Assumes little-endian byte order! ++struct TestScatter { ++ template <class T, class D> ++ HWY_NOINLINE void operator()(T /*unused*/, D d) { ++ using Offset = MakeSigned<T>; ++ ++ const size_t N = Lanes(d); ++ const size_t range = 4 * N; // number of items to scatter ++ const size_t max_bytes = range * sizeof(T); // upper bound on offset ++ ++ RandomState rng; ++ ++ // Data to be scattered ++ auto bytes = AllocateAligned<uint8_t>(max_bytes); ++ for (size_t i = 0; i < max_bytes; ++i) { ++ bytes[i] = static_cast<uint8_t>(Random32(&rng) & 0xFF); ++ } ++ const auto data = Load(d, reinterpret_cast<const T*>(bytes.get())); ++ ++ // Scatter into these regions, ensure vector results match scalar ++ auto expected = AllocateAligned<T>(range); ++ auto actual = AllocateAligned<T>(range); ++ ++ const Rebind<Offset, D> d_offsets; ++ auto offsets = AllocateAligned<Offset>(N); // or indices ++ ++ for (size_t rep = 0; rep < 100; ++rep) { ++ // Byte offsets ++ std::fill(expected.get(), expected.get() + range, T(0)); ++ std::fill(actual.get(), actual.get() + range, T(0)); ++ for (size_t i = 0; i < N; ++i) { ++ offsets[i] = ++ static_cast<Offset>(Random32(&rng) % (max_bytes - sizeof(T))); ++ CopyBytes<sizeof(T)>( ++ bytes.get() + i * sizeof(T), ++ reinterpret_cast<uint8_t*>(expected.get()) + offsets[i]); ++ } ++ const auto voffsets = Load(d_offsets, offsets.get()); ++ ScatterOffset(data, d, actual.get(), voffsets); ++ if (!BytesEqual(expected.get(), actual.get(), max_bytes)) { ++ Print(d, "Data", data); ++ Print(d_offsets, "Offsets", voffsets); ++ HWY_ASSERT(false); ++ } ++ ++ // Indices ++ std::fill(expected.get(), expected.get() + range, T(0)); ++ std::fill(actual.get(), actual.get() + range, T(0)); ++ for (size_t i = 0; i < N; ++i) { ++ offsets[i] = static_cast<Offset>(Random32(&rng) % range); ++ CopyBytes<sizeof(T)>(bytes.get() + i * sizeof(T), ++ &expected[offsets[i]]); ++ } ++ const auto vindices = Load(d_offsets, offsets.get()); ++ ScatterIndex(data, d, actual.get(), vindices); ++ if (!BytesEqual(expected.get(), actual.get(), max_bytes)) { ++ Print(d, "Data", data); ++ Print(d_offsets, "Indices", vindices); ++ HWY_ASSERT(false); ++ } ++ } ++ } ++}; ++ ++HWY_NOINLINE void TestAllScatter() { ++ // No u8,u16,i8,i16. ++ const ForPartialVectors<TestScatter> test; ++ test(uint32_t()); ++ test(int32_t()); ++ ++#if HWY_CAP_INTEGER64 ++ test(uint64_t()); ++ test(int64_t()); ++#endif ++ ++ ForFloatTypes(test); ++} ++ + struct TestGather { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { +@@ -391,6 +476,7 @@ HWY_NOINLINE void TestAllCache() { + int test = 0; + Prefetch(&test); + FlushCacheline(&test); ++ Pause(); + } + + // NOLINTNEXTLINE(google-readability-namespace-comments) +diff -up chromium-92.0.4515.107/third_party/highway/src/hwy/tests/swizzle_test.cc.update-highway-0.12.2 chromium-92.0.4515.107/third_party/highway/src/hwy/tests/swizzle_test.cc +--- chromium-92.0.4515.107/third_party/highway/src/hwy/tests/swizzle_test.cc.update-highway-0.12.2 2021-07-19 14:47:23.000000000 -0400 ++++ chromium-92.0.4515.107/third_party/highway/src/hwy/tests/swizzle_test.cc 2021-07-26 17:10:40.023319835 -0400 +@@ -223,6 +223,7 @@ struct TestTableLookupBytes { + HWY_NOINLINE void TestAllTableLookupBytes() { + ForIntegerTypes(ForPartialVectors<TestTableLookupBytes>()); + } ++ + struct TestTableLookupLanes { + #if HWY_TARGET == HWY_RVV + using Index = uint32_t; +@@ -242,12 +243,13 @@ struct TestTableLookupLanes { + if (N <= 8) { // Test all permutations + for (size_t i0 = 0; i0 < N; ++i0) { + idx[0] = static_cast<Index>(i0); ++ + for (size_t i1 = 0; i1 < N; ++i1) { +- idx[1] = static_cast<Index>(i1); ++ if (N >= 2) idx[1] = static_cast<Index>(i1); + for (size_t i2 = 0; i2 < N; ++i2) { +- idx[2] = static_cast<Index>(i2); ++ if (N >= 4) idx[2] = static_cast<Index>(i2); + for (size_t i3 = 0; i3 < N; ++i3) { +- idx[3] = static_cast<Index>(i3); ++ if (N >= 4) idx[3] = static_cast<Index>(i3); + + for (size_t i = 0; i < N; ++i) { + expected[i] = static_cast<T>(idx[i] + 1); // == v[idx[i]] +@@ -286,7 +288,7 @@ struct TestTableLookupLanes { + }; + + HWY_NOINLINE void TestAllTableLookupLanes() { +- const ForFullVectors<TestTableLookupLanes> test; ++ const ForPartialVectors<TestTableLookupLanes> test; + test(uint32_t()); + test(int32_t()); + test(float()); +diff -up chromium-92.0.4515.107/third_party/highway/src/README.md.update-highway-0.12.2 chromium-92.0.4515.107/third_party/highway/src/README.md +--- chromium-92.0.4515.107/third_party/highway/src/README.md.update-highway-0.12.2 2021-07-26 17:10:40.838332249 -0400 ++++ chromium-92.0.4515.107/third_party/highway/src/README.md 2021-07-26 17:15:00.832292309 -0400 +@@ -15,7 +15,7 @@ applying the same operation to 'lanes'. + ## Current status + + Supported targets: scalar, SSE4, AVX2, AVX-512, NEON (ARMv7 and v8), WASM SIMD. +-A port to RVV is in progress. ++Ports to RVV and SVE/SVE2 are in progress. + + Version 0.11 is considered stable enough to use in other projects, and is + expected to remain backwards compatible unless serious issues are discovered +@@ -23,8 +23,11 @@ while implementing SVE/RVV targets. Afte + reach version 1.0. + + Continuous integration tests build with a recent version of Clang (running on +-x86 and QEMU for ARM) and MSVC from VS2015 (running on x86). Also periodically +-tested on x86 with Clang 7-11 and GCC 8, 9 and 10.2.1. ++x86 and QEMU for ARM) and MSVC from VS2015 (running on x86). ++ ++Before releases, we also test on x86 with Clang and GCC, and ARMv7/8 via ++GCC cross-compile and QEMU. See the ++[testing process](g3doc/release_testing_process.md) for details. + + The `contrib` directory contains SIMD-related utilities: an image class with + aligned rows, and a math library (16 functions already implemented, mostly +@@ -63,6 +66,8 @@ To test on all the attainable targets fo + default configuration skips baseline targets (e.g. scalar) that are superseded + by another baseline target. + ++Bazel is also supported for building, but it is not as widely used/tested. ++ + ## Quick start + + You can use the `benchmark` inside examples/ as a starting point. +diff -up chromium-92.0.4515.107/third_party/highway/src/run_tests.bat.update-highway-0.12.2 chromium-92.0.4515.107/third_party/highway/src/run_tests.bat +--- chromium-92.0.4515.107/third_party/highway/src/run_tests.bat.update-highway-0.12.2 2021-07-19 14:47:23.000000000 -0400 ++++ chromium-92.0.4515.107/third_party/highway/src/run_tests.bat 2021-07-26 17:14:47.466088723 -0400 +@@ -2,9 +2,9 @@ + REM Switch directory of this batch file + cd %~dp0 + +-if not exist build mkdir build ++if not exist build_win mkdir build_win + +-cd build ++cd build_win + cmake .. -G Ninja || goto error + ninja || goto error + ctest -j || goto error