chore: add vendor dependencies for kauma build
This commit is contained in:
parent
7c94e5d8fb
commit
067ef6141c
1758 changed files with 398473 additions and 0 deletions
1352
vendor/memchr/src/arch/x86_64/avx2/memchr.rs
vendored
Normal file
1352
vendor/memchr/src/arch/x86_64/avx2/memchr.rs
vendored
Normal file
File diff suppressed because it is too large
Load diff
6
vendor/memchr/src/arch/x86_64/avx2/mod.rs
vendored
Normal file
6
vendor/memchr/src/arch/x86_64/avx2/mod.rs
vendored
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
/*!
|
||||
Algorithms for the `x86_64` target using 256-bit vectors via AVX2.
|
||||
*/
|
||||
|
||||
pub mod memchr;
|
||||
pub mod packedpair;
|
||||
272
vendor/memchr/src/arch/x86_64/avx2/packedpair.rs
vendored
Normal file
272
vendor/memchr/src/arch/x86_64/avx2/packedpair.rs
vendored
Normal file
|
|
@ -0,0 +1,272 @@
|
|||
/*!
|
||||
A 256-bit vector implementation of the "packed pair" SIMD algorithm.
|
||||
|
||||
The "packed pair" algorithm is based on the [generic SIMD] algorithm. The main
|
||||
difference is that it (by default) uses a background distribution of byte
|
||||
frequencies to heuristically select the pair of bytes to search for.
|
||||
|
||||
[generic SIMD]: http://0x80.pl/articles/simd-strfind.html#first-and-last
|
||||
*/
|
||||
|
||||
use core::arch::x86_64::{__m128i, __m256i};
|
||||
|
||||
use crate::arch::{all::packedpair::Pair, generic::packedpair};
|
||||
|
||||
/// A "packed pair" finder that uses 256-bit vector operations.
|
||||
///
|
||||
/// This finder picks two bytes that it believes have high predictive power
|
||||
/// for indicating an overall match of a needle. Depending on whether
|
||||
/// `Finder::find` or `Finder::find_prefilter` is used, it reports offsets
|
||||
/// where the needle matches or could match. In the prefilter case, candidates
|
||||
/// are reported whenever the [`Pair`] of bytes given matches.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Finder {
|
||||
sse2: packedpair::Finder<__m128i>,
|
||||
avx2: packedpair::Finder<__m256i>,
|
||||
}
|
||||
|
||||
impl Finder {
|
||||
/// Create a new pair searcher. The searcher returned can either report
|
||||
/// exact matches of `needle` or act as a prefilter and report candidate
|
||||
/// positions of `needle`.
|
||||
///
|
||||
/// If AVX2 is unavailable in the current environment or if a [`Pair`]
|
||||
/// could not be constructed from the needle given, then `None` is
|
||||
/// returned.
|
||||
#[inline]
|
||||
pub fn new(needle: &[u8]) -> Option<Finder> {
|
||||
Finder::with_pair(needle, Pair::new(needle)?)
|
||||
}
|
||||
|
||||
/// Create a new "packed pair" finder using the pair of bytes given.
|
||||
///
|
||||
/// This constructor permits callers to control precisely which pair of
|
||||
/// bytes is used as a predicate.
|
||||
///
|
||||
/// If AVX2 is unavailable in the current environment, then `None` is
|
||||
/// returned.
|
||||
#[inline]
|
||||
pub fn with_pair(needle: &[u8], pair: Pair) -> Option<Finder> {
|
||||
if Finder::is_available() {
|
||||
// SAFETY: we check that sse2/avx2 is available above. We are also
|
||||
// guaranteed to have needle.len() > 1 because we have a valid
|
||||
// Pair.
|
||||
unsafe { Some(Finder::with_pair_impl(needle, pair)) }
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new `Finder` specific to SSE2 vectors and routines.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// Same as the safety for `packedpair::Finder::new`, and callers must also
|
||||
/// ensure that both SSE2 and AVX2 are available.
|
||||
#[target_feature(enable = "sse2", enable = "avx2")]
|
||||
#[inline]
|
||||
unsafe fn with_pair_impl(needle: &[u8], pair: Pair) -> Finder {
|
||||
let sse2 = packedpair::Finder::<__m128i>::new(needle, pair);
|
||||
let avx2 = packedpair::Finder::<__m256i>::new(needle, pair);
|
||||
Finder { sse2, avx2 }
|
||||
}
|
||||
|
||||
/// Returns true when this implementation is available in the current
|
||||
/// environment.
|
||||
///
|
||||
/// When this is true, it is guaranteed that [`Finder::with_pair`] will
|
||||
/// return a `Some` value. Similarly, when it is false, it is guaranteed
|
||||
/// that `Finder::with_pair` will return a `None` value. Notice that this
|
||||
/// does not guarantee that [`Finder::new`] will return a `Finder`. Namely,
|
||||
/// even when `Finder::is_available` is true, it is not guaranteed that a
|
||||
/// valid [`Pair`] can be found from the needle given.
|
||||
///
|
||||
/// Note also that for the lifetime of a single program, if this returns
|
||||
/// true then it will always return true.
|
||||
#[inline]
|
||||
pub fn is_available() -> bool {
|
||||
#[cfg(not(target_feature = "sse2"))]
|
||||
{
|
||||
false
|
||||
}
|
||||
#[cfg(target_feature = "sse2")]
|
||||
{
|
||||
#[cfg(target_feature = "avx2")]
|
||||
{
|
||||
true
|
||||
}
|
||||
#[cfg(not(target_feature = "avx2"))]
|
||||
{
|
||||
#[cfg(feature = "std")]
|
||||
{
|
||||
std::is_x86_feature_detected!("avx2")
|
||||
}
|
||||
#[cfg(not(feature = "std"))]
|
||||
{
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Execute a search using AVX2 vectors and routines.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// When `haystack.len()` is less than [`Finder::min_haystack_len`].
|
||||
#[inline]
|
||||
pub fn find(&self, haystack: &[u8], needle: &[u8]) -> Option<usize> {
|
||||
// SAFETY: Building a `Finder` means it's safe to call 'sse2' routines.
|
||||
unsafe { self.find_impl(haystack, needle) }
|
||||
}
|
||||
|
||||
/// Run this finder on the given haystack as a prefilter.
|
||||
///
|
||||
/// If a candidate match is found, then an offset where the needle *could*
|
||||
/// begin in the haystack is returned.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// When `haystack.len()` is less than [`Finder::min_haystack_len`].
|
||||
#[inline]
|
||||
pub fn find_prefilter(&self, haystack: &[u8]) -> Option<usize> {
|
||||
// SAFETY: Building a `Finder` means it's safe to call 'sse2' routines.
|
||||
unsafe { self.find_prefilter_impl(haystack) }
|
||||
}
|
||||
|
||||
/// Execute a search using AVX2 vectors and routines.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// When `haystack.len()` is less than [`Finder::min_haystack_len`].
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// (The target feature safety obligation is automatically fulfilled by
|
||||
/// virtue of being a method on `Finder`, which can only be constructed
|
||||
/// when it is safe to call `sse2` and `avx2` routines.)
|
||||
#[target_feature(enable = "sse2", enable = "avx2")]
|
||||
#[inline]
|
||||
unsafe fn find_impl(
|
||||
&self,
|
||||
haystack: &[u8],
|
||||
needle: &[u8],
|
||||
) -> Option<usize> {
|
||||
if haystack.len() < self.avx2.min_haystack_len() {
|
||||
self.sse2.find(haystack, needle)
|
||||
} else {
|
||||
self.avx2.find(haystack, needle)
|
||||
}
|
||||
}
|
||||
|
||||
/// Execute a prefilter search using AVX2 vectors and routines.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// When `haystack.len()` is less than [`Finder::min_haystack_len`].
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// (The target feature safety obligation is automatically fulfilled by
|
||||
/// virtue of being a method on `Finder`, which can only be constructed
|
||||
/// when it is safe to call `sse2` and `avx2` routines.)
|
||||
#[target_feature(enable = "sse2", enable = "avx2")]
|
||||
#[inline]
|
||||
unsafe fn find_prefilter_impl(&self, haystack: &[u8]) -> Option<usize> {
|
||||
if haystack.len() < self.avx2.min_haystack_len() {
|
||||
self.sse2.find_prefilter(haystack)
|
||||
} else {
|
||||
self.avx2.find_prefilter(haystack)
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the pair of offsets (into the needle) used to check as a
|
||||
/// predicate before confirming whether a needle exists at a particular
|
||||
/// position.
|
||||
#[inline]
|
||||
pub fn pair(&self) -> &Pair {
|
||||
self.avx2.pair()
|
||||
}
|
||||
|
||||
/// Returns the minimum haystack length that this `Finder` can search.
|
||||
///
|
||||
/// Using a haystack with length smaller than this in a search will result
|
||||
/// in a panic. The reason for this restriction is that this finder is
|
||||
/// meant to be a low-level component that is part of a larger substring
|
||||
/// strategy. In that sense, it avoids trying to handle all cases and
|
||||
/// instead only handles the cases that it can handle very well.
|
||||
#[inline]
|
||||
pub fn min_haystack_len(&self) -> usize {
|
||||
// The caller doesn't need to care about AVX2's min_haystack_len
|
||||
// since this implementation will automatically switch to the SSE2
|
||||
// implementation if the haystack is too short for AVX2. Therefore, the
|
||||
// caller only needs to care about SSE2's min_haystack_len.
|
||||
//
|
||||
// This does assume that SSE2's min_haystack_len is less than or
|
||||
// equal to AVX2's min_haystack_len. In practice, this is true and
|
||||
// there is no way it could be false based on how this Finder is
|
||||
// implemented. Namely, both SSE2 and AVX2 use the same `Pair`. If
|
||||
// they used different pairs, then it's possible (although perhaps
|
||||
// pathological) for SSE2's min_haystack_len to be bigger than AVX2's.
|
||||
self.sse2.min_haystack_len()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn find(haystack: &[u8], needle: &[u8]) -> Option<Option<usize>> {
|
||||
let f = Finder::new(needle)?;
|
||||
if haystack.len() < f.min_haystack_len() {
|
||||
return None;
|
||||
}
|
||||
Some(f.find(haystack, needle))
|
||||
}
|
||||
|
||||
define_substring_forward_quickcheck!(find);
|
||||
|
||||
#[test]
|
||||
fn forward_substring() {
|
||||
crate::tests::substring::Runner::new().fwd(find).run()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn forward_packedpair() {
|
||||
fn find(
|
||||
haystack: &[u8],
|
||||
needle: &[u8],
|
||||
index1: u8,
|
||||
index2: u8,
|
||||
) -> Option<Option<usize>> {
|
||||
let pair = Pair::with_indices(needle, index1, index2)?;
|
||||
let f = Finder::with_pair(needle, pair)?;
|
||||
if haystack.len() < f.min_haystack_len() {
|
||||
return None;
|
||||
}
|
||||
Some(f.find(haystack, needle))
|
||||
}
|
||||
crate::tests::packedpair::Runner::new().fwd(find).run()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn forward_packedpair_prefilter() {
|
||||
fn find(
|
||||
haystack: &[u8],
|
||||
needle: &[u8],
|
||||
index1: u8,
|
||||
index2: u8,
|
||||
) -> Option<Option<usize>> {
|
||||
if !cfg!(target_feature = "sse2") {
|
||||
return None;
|
||||
}
|
||||
let pair = Pair::with_indices(needle, index1, index2)?;
|
||||
let f = Finder::with_pair(needle, pair)?;
|
||||
if haystack.len() < f.min_haystack_len() {
|
||||
return None;
|
||||
}
|
||||
Some(f.find_prefilter(haystack))
|
||||
}
|
||||
crate::tests::packedpair::Runner::new().fwd(find).run()
|
||||
}
|
||||
}
|
||||
335
vendor/memchr/src/arch/x86_64/memchr.rs
vendored
Normal file
335
vendor/memchr/src/arch/x86_64/memchr.rs
vendored
Normal file
|
|
@ -0,0 +1,335 @@
|
|||
/*!
|
||||
Wrapper routines for `memchr` and friends.
|
||||
|
||||
These routines efficiently dispatch to the best implementation based on what
|
||||
the CPU supports.
|
||||
*/
|
||||
|
||||
/// Provides a way to run a memchr-like function while amortizing the cost of
|
||||
/// runtime CPU feature detection.
|
||||
///
|
||||
/// This works by loading a function pointer from an atomic global. Initially,
|
||||
/// this global is set to a function that does CPU feature detection. For
|
||||
/// example, if AVX2 is enabled, then the AVX2 implementation is used.
|
||||
/// Otherwise, at least on x86_64, the SSE2 implementation is used. (And
|
||||
/// in some niche cases, if SSE2 isn't available, then the architecture
|
||||
/// independent fallback implementation is used.)
|
||||
///
|
||||
/// After the first call to this function, the atomic global is replaced with
|
||||
/// the specific AVX2, SSE2 or fallback routine chosen. Subsequent calls then
|
||||
/// will directly call the chosen routine instead of needing to go through the
|
||||
/// CPU feature detection branching again.
|
||||
///
|
||||
/// This particular macro is specifically written to provide the implementation
|
||||
/// of functions with the following signature:
|
||||
///
|
||||
/// ```ignore
|
||||
/// fn memchr(needle1: u8, start: *const u8, end: *const u8) -> Option<usize>;
|
||||
/// ```
|
||||
///
|
||||
/// Where you can also have `memchr2` and `memchr3`, but with `needle2` and
|
||||
/// `needle3`, respectively. The `start` and `end` parameters correspond to the
|
||||
/// start and end of the haystack, respectively.
|
||||
///
|
||||
/// We use raw pointers here instead of the more obvious `haystack: &[u8]` so
|
||||
/// that the function is compatible with our lower level iterator logic that
|
||||
/// operates on raw pointers. We use this macro to implement "raw" memchr
|
||||
/// routines with the signature above, and then define memchr routines using
|
||||
/// regular slices on top of them.
|
||||
///
|
||||
/// Note that we use `#[cfg(target_feature = "sse2")]` below even though
|
||||
/// it shouldn't be strictly necessary because without it, it seems to
|
||||
/// cause the compiler to blow up. I guess it can't handle a function
|
||||
/// pointer being created with a sse target feature? Dunno. See the
|
||||
/// `build-for-x86-64-but-non-sse-target` CI job if you want to experiment with
|
||||
/// this.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// Primarily callers must that `$fnty` is a correct function pointer type and
|
||||
/// not something else.
|
||||
///
|
||||
/// Callers must also ensure that `$memchrty::$memchrfind` corresponds to a
|
||||
/// routine that returns a valid function pointer when a match is found. That
|
||||
/// is, a pointer that is `>= start` and `< end`.
|
||||
///
|
||||
/// Callers must also ensure that the `$hay_start` and `$hay_end` identifiers
|
||||
/// correspond to valid pointers.
|
||||
macro_rules! unsafe_ifunc {
|
||||
(
|
||||
$memchrty:ident,
|
||||
$memchrfind:ident,
|
||||
$fnty:ty,
|
||||
$retty:ty,
|
||||
$hay_start:ident,
|
||||
$hay_end:ident,
|
||||
$($needle:ident),+
|
||||
) => {{
|
||||
#![allow(unused_unsafe)]
|
||||
|
||||
use core::sync::atomic::{AtomicPtr, Ordering};
|
||||
|
||||
type Fn = *mut ();
|
||||
type RealFn = $fnty;
|
||||
static FN: AtomicPtr<()> = AtomicPtr::new(detect as Fn);
|
||||
|
||||
#[cfg(target_feature = "sse2")]
|
||||
#[target_feature(enable = "sse2", enable = "avx2")]
|
||||
unsafe fn find_avx2(
|
||||
$($needle: u8),+,
|
||||
$hay_start: *const u8,
|
||||
$hay_end: *const u8,
|
||||
) -> $retty {
|
||||
use crate::arch::x86_64::avx2::memchr::$memchrty;
|
||||
$memchrty::new_unchecked($($needle),+)
|
||||
.$memchrfind($hay_start, $hay_end)
|
||||
}
|
||||
|
||||
#[cfg(target_feature = "sse2")]
|
||||
#[target_feature(enable = "sse2")]
|
||||
unsafe fn find_sse2(
|
||||
$($needle: u8),+,
|
||||
$hay_start: *const u8,
|
||||
$hay_end: *const u8,
|
||||
) -> $retty {
|
||||
use crate::arch::x86_64::sse2::memchr::$memchrty;
|
||||
$memchrty::new_unchecked($($needle),+)
|
||||
.$memchrfind($hay_start, $hay_end)
|
||||
}
|
||||
|
||||
unsafe fn find_fallback(
|
||||
$($needle: u8),+,
|
||||
$hay_start: *const u8,
|
||||
$hay_end: *const u8,
|
||||
) -> $retty {
|
||||
use crate::arch::all::memchr::$memchrty;
|
||||
$memchrty::new($($needle),+).$memchrfind($hay_start, $hay_end)
|
||||
}
|
||||
|
||||
unsafe fn detect(
|
||||
$($needle: u8),+,
|
||||
$hay_start: *const u8,
|
||||
$hay_end: *const u8,
|
||||
) -> $retty {
|
||||
let fun = {
|
||||
#[cfg(not(target_feature = "sse2"))]
|
||||
{
|
||||
debug!(
|
||||
"no sse2 feature available, using fallback for {}",
|
||||
stringify!($memchrty),
|
||||
);
|
||||
find_fallback as RealFn
|
||||
}
|
||||
#[cfg(target_feature = "sse2")]
|
||||
{
|
||||
use crate::arch::x86_64::{sse2, avx2};
|
||||
if avx2::memchr::$memchrty::is_available() {
|
||||
debug!("chose AVX2 for {}", stringify!($memchrty));
|
||||
find_avx2 as RealFn
|
||||
} else if sse2::memchr::$memchrty::is_available() {
|
||||
debug!("chose SSE2 for {}", stringify!($memchrty));
|
||||
find_sse2 as RealFn
|
||||
} else {
|
||||
debug!("chose fallback for {}", stringify!($memchrty));
|
||||
find_fallback as RealFn
|
||||
}
|
||||
}
|
||||
};
|
||||
FN.store(fun as Fn, Ordering::Relaxed);
|
||||
// SAFETY: The only thing we need to uphold here is the
|
||||
// `#[target_feature]` requirements. Since we check is_available
|
||||
// above before using the corresponding implementation, we are
|
||||
// guaranteed to only call code that is supported on the current
|
||||
// CPU.
|
||||
fun($($needle),+, $hay_start, $hay_end)
|
||||
}
|
||||
|
||||
// SAFETY: By virtue of the caller contract, RealFn is a function
|
||||
// pointer, which is always safe to transmute with a *mut (). Also,
|
||||
// since we use $memchrty::is_available, it is guaranteed to be safe
|
||||
// to call $memchrty::$memchrfind.
|
||||
unsafe {
|
||||
let fun = FN.load(Ordering::Relaxed);
|
||||
core::mem::transmute::<Fn, RealFn>(fun)(
|
||||
$($needle),+,
|
||||
$hay_start,
|
||||
$hay_end,
|
||||
)
|
||||
}
|
||||
}};
|
||||
}
|
||||
|
||||
// The routines below dispatch to AVX2, SSE2 or a fallback routine based on
|
||||
// what's available in the current environment. The secret sauce here is that
|
||||
// we only check for which one to use approximately once, and then "cache" that
|
||||
// choice into a global function pointer. Subsequent invocations then just call
|
||||
// the appropriate function directly.
|
||||
|
||||
/// memchr, but using raw pointers to represent the haystack.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// Pointers must be valid. See `One::find_raw`.
|
||||
#[inline(always)]
|
||||
pub(crate) fn memchr_raw(
|
||||
n1: u8,
|
||||
start: *const u8,
|
||||
end: *const u8,
|
||||
) -> Option<*const u8> {
|
||||
// SAFETY: We provide a valid function pointer type.
|
||||
unsafe_ifunc!(
|
||||
One,
|
||||
find_raw,
|
||||
unsafe fn(u8, *const u8, *const u8) -> Option<*const u8>,
|
||||
Option<*const u8>,
|
||||
start,
|
||||
end,
|
||||
n1
|
||||
)
|
||||
}
|
||||
|
||||
/// memrchr, but using raw pointers to represent the haystack.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// Pointers must be valid. See `One::rfind_raw`.
|
||||
#[inline(always)]
|
||||
pub(crate) fn memrchr_raw(
|
||||
n1: u8,
|
||||
start: *const u8,
|
||||
end: *const u8,
|
||||
) -> Option<*const u8> {
|
||||
// SAFETY: We provide a valid function pointer type.
|
||||
unsafe_ifunc!(
|
||||
One,
|
||||
rfind_raw,
|
||||
unsafe fn(u8, *const u8, *const u8) -> Option<*const u8>,
|
||||
Option<*const u8>,
|
||||
start,
|
||||
end,
|
||||
n1
|
||||
)
|
||||
}
|
||||
|
||||
/// memchr2, but using raw pointers to represent the haystack.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// Pointers must be valid. See `Two::find_raw`.
|
||||
#[inline(always)]
|
||||
pub(crate) fn memchr2_raw(
|
||||
n1: u8,
|
||||
n2: u8,
|
||||
start: *const u8,
|
||||
end: *const u8,
|
||||
) -> Option<*const u8> {
|
||||
// SAFETY: We provide a valid function pointer type.
|
||||
unsafe_ifunc!(
|
||||
Two,
|
||||
find_raw,
|
||||
unsafe fn(u8, u8, *const u8, *const u8) -> Option<*const u8>,
|
||||
Option<*const u8>,
|
||||
start,
|
||||
end,
|
||||
n1,
|
||||
n2
|
||||
)
|
||||
}
|
||||
|
||||
/// memrchr2, but using raw pointers to represent the haystack.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// Pointers must be valid. See `Two::rfind_raw`.
|
||||
#[inline(always)]
|
||||
pub(crate) fn memrchr2_raw(
|
||||
n1: u8,
|
||||
n2: u8,
|
||||
start: *const u8,
|
||||
end: *const u8,
|
||||
) -> Option<*const u8> {
|
||||
// SAFETY: We provide a valid function pointer type.
|
||||
unsafe_ifunc!(
|
||||
Two,
|
||||
rfind_raw,
|
||||
unsafe fn(u8, u8, *const u8, *const u8) -> Option<*const u8>,
|
||||
Option<*const u8>,
|
||||
start,
|
||||
end,
|
||||
n1,
|
||||
n2
|
||||
)
|
||||
}
|
||||
|
||||
/// memchr3, but using raw pointers to represent the haystack.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// Pointers must be valid. See `Three::find_raw`.
|
||||
#[inline(always)]
|
||||
pub(crate) fn memchr3_raw(
|
||||
n1: u8,
|
||||
n2: u8,
|
||||
n3: u8,
|
||||
start: *const u8,
|
||||
end: *const u8,
|
||||
) -> Option<*const u8> {
|
||||
// SAFETY: We provide a valid function pointer type.
|
||||
unsafe_ifunc!(
|
||||
Three,
|
||||
find_raw,
|
||||
unsafe fn(u8, u8, u8, *const u8, *const u8) -> Option<*const u8>,
|
||||
Option<*const u8>,
|
||||
start,
|
||||
end,
|
||||
n1,
|
||||
n2,
|
||||
n3
|
||||
)
|
||||
}
|
||||
|
||||
/// memrchr3, but using raw pointers to represent the haystack.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// Pointers must be valid. See `Three::rfind_raw`.
|
||||
#[inline(always)]
|
||||
pub(crate) fn memrchr3_raw(
|
||||
n1: u8,
|
||||
n2: u8,
|
||||
n3: u8,
|
||||
start: *const u8,
|
||||
end: *const u8,
|
||||
) -> Option<*const u8> {
|
||||
// SAFETY: We provide a valid function pointer type.
|
||||
unsafe_ifunc!(
|
||||
Three,
|
||||
rfind_raw,
|
||||
unsafe fn(u8, u8, u8, *const u8, *const u8) -> Option<*const u8>,
|
||||
Option<*const u8>,
|
||||
start,
|
||||
end,
|
||||
n1,
|
||||
n2,
|
||||
n3
|
||||
)
|
||||
}
|
||||
|
||||
/// Count all matching bytes, but using raw pointers to represent the haystack.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// Pointers must be valid. See `One::count_raw`.
|
||||
#[inline(always)]
|
||||
pub(crate) fn count_raw(n1: u8, start: *const u8, end: *const u8) -> usize {
|
||||
// SAFETY: We provide a valid function pointer type.
|
||||
unsafe_ifunc!(
|
||||
One,
|
||||
count_raw,
|
||||
unsafe fn(u8, *const u8, *const u8) -> usize,
|
||||
usize,
|
||||
start,
|
||||
end,
|
||||
n1
|
||||
)
|
||||
}
|
||||
8
vendor/memchr/src/arch/x86_64/mod.rs
vendored
Normal file
8
vendor/memchr/src/arch/x86_64/mod.rs
vendored
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
/*!
|
||||
Vector algorithms for the `x86_64` target.
|
||||
*/
|
||||
|
||||
pub mod avx2;
|
||||
pub mod sse2;
|
||||
|
||||
pub(crate) mod memchr;
|
||||
1077
vendor/memchr/src/arch/x86_64/sse2/memchr.rs
vendored
Normal file
1077
vendor/memchr/src/arch/x86_64/sse2/memchr.rs
vendored
Normal file
File diff suppressed because it is too large
Load diff
6
vendor/memchr/src/arch/x86_64/sse2/mod.rs
vendored
Normal file
6
vendor/memchr/src/arch/x86_64/sse2/mod.rs
vendored
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
/*!
|
||||
Algorithms for the `x86_64` target using 128-bit vectors via SSE2.
|
||||
*/
|
||||
|
||||
pub mod memchr;
|
||||
pub mod packedpair;
|
||||
232
vendor/memchr/src/arch/x86_64/sse2/packedpair.rs
vendored
Normal file
232
vendor/memchr/src/arch/x86_64/sse2/packedpair.rs
vendored
Normal file
|
|
@ -0,0 +1,232 @@
|
|||
/*!
|
||||
A 128-bit vector implementation of the "packed pair" SIMD algorithm.
|
||||
|
||||
The "packed pair" algorithm is based on the [generic SIMD] algorithm. The main
|
||||
difference is that it (by default) uses a background distribution of byte
|
||||
frequencies to heuristically select the pair of bytes to search for.
|
||||
|
||||
[generic SIMD]: http://0x80.pl/articles/simd-strfind.html#first-and-last
|
||||
*/
|
||||
|
||||
use core::arch::x86_64::__m128i;
|
||||
|
||||
use crate::arch::{all::packedpair::Pair, generic::packedpair};
|
||||
|
||||
/// A "packed pair" finder that uses 128-bit vector operations.
|
||||
///
|
||||
/// This finder picks two bytes that it believes have high predictive power
|
||||
/// for indicating an overall match of a needle. Depending on whether
|
||||
/// `Finder::find` or `Finder::find_prefilter` is used, it reports offsets
|
||||
/// where the needle matches or could match. In the prefilter case, candidates
|
||||
/// are reported whenever the [`Pair`] of bytes given matches.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Finder(packedpair::Finder<__m128i>);
|
||||
|
||||
impl Finder {
|
||||
/// Create a new pair searcher. The searcher returned can either report
|
||||
/// exact matches of `needle` or act as a prefilter and report candidate
|
||||
/// positions of `needle`.
|
||||
///
|
||||
/// If SSE2 is unavailable in the current environment or if a [`Pair`]
|
||||
/// could not be constructed from the needle given, then `None` is
|
||||
/// returned.
|
||||
#[inline]
|
||||
pub fn new(needle: &[u8]) -> Option<Finder> {
|
||||
Finder::with_pair(needle, Pair::new(needle)?)
|
||||
}
|
||||
|
||||
/// Create a new "packed pair" finder using the pair of bytes given.
|
||||
///
|
||||
/// This constructor permits callers to control precisely which pair of
|
||||
/// bytes is used as a predicate.
|
||||
///
|
||||
/// If SSE2 is unavailable in the current environment, then `None` is
|
||||
/// returned.
|
||||
#[inline]
|
||||
pub fn with_pair(needle: &[u8], pair: Pair) -> Option<Finder> {
|
||||
if Finder::is_available() {
|
||||
// SAFETY: we check that sse2 is available above. We are also
|
||||
// guaranteed to have needle.len() > 1 because we have a valid
|
||||
// Pair.
|
||||
unsafe { Some(Finder::with_pair_impl(needle, pair)) }
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new `Finder` specific to SSE2 vectors and routines.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// Same as the safety for `packedpair::Finder::new`, and callers must also
|
||||
/// ensure that SSE2 is available.
|
||||
#[target_feature(enable = "sse2")]
|
||||
#[inline]
|
||||
unsafe fn with_pair_impl(needle: &[u8], pair: Pair) -> Finder {
|
||||
let finder = packedpair::Finder::<__m128i>::new(needle, pair);
|
||||
Finder(finder)
|
||||
}
|
||||
|
||||
/// Returns true when this implementation is available in the current
|
||||
/// environment.
|
||||
///
|
||||
/// When this is true, it is guaranteed that [`Finder::with_pair`] will
|
||||
/// return a `Some` value. Similarly, when it is false, it is guaranteed
|
||||
/// that `Finder::with_pair` will return a `None` value. Notice that this
|
||||
/// does not guarantee that [`Finder::new`] will return a `Finder`. Namely,
|
||||
/// even when `Finder::is_available` is true, it is not guaranteed that a
|
||||
/// valid [`Pair`] can be found from the needle given.
|
||||
///
|
||||
/// Note also that for the lifetime of a single program, if this returns
|
||||
/// true then it will always return true.
|
||||
#[inline]
|
||||
pub fn is_available() -> bool {
|
||||
#[cfg(not(target_feature = "sse2"))]
|
||||
{
|
||||
false
|
||||
}
|
||||
#[cfg(target_feature = "sse2")]
|
||||
{
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
/// Execute a search using SSE2 vectors and routines.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// When `haystack.len()` is less than [`Finder::min_haystack_len`].
|
||||
#[inline]
|
||||
pub fn find(&self, haystack: &[u8], needle: &[u8]) -> Option<usize> {
|
||||
// SAFETY: Building a `Finder` means it's safe to call 'sse2' routines.
|
||||
unsafe { self.find_impl(haystack, needle) }
|
||||
}
|
||||
|
||||
/// Run this finder on the given haystack as a prefilter.
|
||||
///
|
||||
/// If a candidate match is found, then an offset where the needle *could*
|
||||
/// begin in the haystack is returned.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// When `haystack.len()` is less than [`Finder::min_haystack_len`].
|
||||
#[inline]
|
||||
pub fn find_prefilter(&self, haystack: &[u8]) -> Option<usize> {
|
||||
// SAFETY: Building a `Finder` means it's safe to call 'sse2' routines.
|
||||
unsafe { self.find_prefilter_impl(haystack) }
|
||||
}
|
||||
|
||||
/// Execute a search using SSE2 vectors and routines.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// When `haystack.len()` is less than [`Finder::min_haystack_len`].
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// (The target feature safety obligation is automatically fulfilled by
|
||||
/// virtue of being a method on `Finder`, which can only be constructed
|
||||
/// when it is safe to call `sse2` routines.)
|
||||
#[target_feature(enable = "sse2")]
|
||||
#[inline]
|
||||
unsafe fn find_impl(
|
||||
&self,
|
||||
haystack: &[u8],
|
||||
needle: &[u8],
|
||||
) -> Option<usize> {
|
||||
self.0.find(haystack, needle)
|
||||
}
|
||||
|
||||
/// Execute a prefilter search using SSE2 vectors and routines.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// When `haystack.len()` is less than [`Finder::min_haystack_len`].
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// (The target feature safety obligation is automatically fulfilled by
|
||||
/// virtue of being a method on `Finder`, which can only be constructed
|
||||
/// when it is safe to call `sse2` routines.)
|
||||
#[target_feature(enable = "sse2")]
|
||||
#[inline]
|
||||
unsafe fn find_prefilter_impl(&self, haystack: &[u8]) -> Option<usize> {
|
||||
self.0.find_prefilter(haystack)
|
||||
}
|
||||
|
||||
/// Returns the pair of offsets (into the needle) used to check as a
|
||||
/// predicate before confirming whether a needle exists at a particular
|
||||
/// position.
|
||||
#[inline]
|
||||
pub fn pair(&self) -> &Pair {
|
||||
self.0.pair()
|
||||
}
|
||||
|
||||
/// Returns the minimum haystack length that this `Finder` can search.
|
||||
///
|
||||
/// Using a haystack with length smaller than this in a search will result
|
||||
/// in a panic. The reason for this restriction is that this finder is
|
||||
/// meant to be a low-level component that is part of a larger substring
|
||||
/// strategy. In that sense, it avoids trying to handle all cases and
|
||||
/// instead only handles the cases that it can handle very well.
|
||||
#[inline]
|
||||
pub fn min_haystack_len(&self) -> usize {
|
||||
self.0.min_haystack_len()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn find(haystack: &[u8], needle: &[u8]) -> Option<Option<usize>> {
|
||||
let f = Finder::new(needle)?;
|
||||
if haystack.len() < f.min_haystack_len() {
|
||||
return None;
|
||||
}
|
||||
Some(f.find(haystack, needle))
|
||||
}
|
||||
|
||||
define_substring_forward_quickcheck!(find);
|
||||
|
||||
#[test]
|
||||
fn forward_substring() {
|
||||
crate::tests::substring::Runner::new().fwd(find).run()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn forward_packedpair() {
|
||||
fn find(
|
||||
haystack: &[u8],
|
||||
needle: &[u8],
|
||||
index1: u8,
|
||||
index2: u8,
|
||||
) -> Option<Option<usize>> {
|
||||
let pair = Pair::with_indices(needle, index1, index2)?;
|
||||
let f = Finder::with_pair(needle, pair)?;
|
||||
if haystack.len() < f.min_haystack_len() {
|
||||
return None;
|
||||
}
|
||||
Some(f.find(haystack, needle))
|
||||
}
|
||||
crate::tests::packedpair::Runner::new().fwd(find).run()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn forward_packedpair_prefilter() {
|
||||
fn find(
|
||||
haystack: &[u8],
|
||||
needle: &[u8],
|
||||
index1: u8,
|
||||
index2: u8,
|
||||
) -> Option<Option<usize>> {
|
||||
let pair = Pair::with_indices(needle, index1, index2)?;
|
||||
let f = Finder::with_pair(needle, pair)?;
|
||||
if haystack.len() < f.min_haystack_len() {
|
||||
return None;
|
||||
}
|
||||
Some(f.find_prefilter(haystack))
|
||||
}
|
||||
crate::tests::packedpair::Runner::new().fwd(find).run()
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue