diff --git a/.cargo/config.toml b/.cargo/config.toml index 0eba3eb..ad9095c 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,3 +1,3 @@ -[unstable] -profile-rustflags = true -trim-paths = true +# [unstable] +# profile-rustflags = true +# trim-paths = true diff --git a/.env.example b/.env.example index aac6f3b..e49dc85 100644 --- a/.env.example +++ b/.env.example @@ -122,8 +122,15 @@ DEBUG_LOG_FILE=debug.log # 日志储存条数(最大值100000)(为0则无日志,为100000则无限制,但日志文件上限8EB=8192PB=8388608TB,以防你看不懂,前提是你内存多大) REQUEST_LOGS_LIMIT=100 -# TCP保活时间(秒)(最大值600) -TCP_KEEPALIVE=90 +# 连接空闲多久后开始发送探测包(秒)(最大值600) +TCP_KEEPALIVE=15 + +# 探测包发送间隔(秒)(最大值600) +# 按理不应大于 TCP_KEEPALIVE +TCP_KEEPALIVE_INTERVAL=15 + +# 探测失败后的最大重试次数(最大值20) +TCP_KEEPALIVE_RETRIES=3 # 服务请求超时(秒)(最大值600) SERVICE_TIMEOUT=30 @@ -152,8 +159,8 @@ GENERAL_TIMEZONE=Asia/Shanghai # 禁用HTTP2 # DISABLE_HTTP2=false -# Cursor客户端版本 -CURSOR_CLIENT_VERSION=2.0.0 +# Cursor客户端版本(已弃用) +# CURSOR_CLIENT_VERSION=2.0.0 # 思考标签(已弃用) # THINKING_TAG=think diff --git a/Cargo.toml b/Cargo.toml index 5253120..03dbcb4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,11 +9,12 @@ members = [ "crates/byte_str", "crates/atomic_enum", "crates/proto-value", + "__build", ] default-members = ["."] [workspace.package] -version = "0.4.0-pre.17" +version = "0.4.0-pre.19" edition = "2024" authors = ["wisdgod "] description = "A format compatibility layer for the Cursor API" @@ -42,7 +43,6 @@ codegen-units = 256 [profile.fast] inherits = "dev" opt-level = 1 -trim-paths = "all" # ===== 性能测试配置(接近 release 但编译更快)===== [profile.bench] @@ -52,21 +52,16 @@ codegen-units = 16 # ===== 发布配置(性能最大化)===== [profile.release] -opt-level = 3 -lto = "fat" +lto = true codegen-units = 1 panic = "abort" strip = true -debug = false -overflow-checks = false -incremental = false -trim-paths = "all" [patch.crates-io] h2 = { path = "patch/h2-0.4.10" } reqwest = { path = "patch/reqwest-0.12.18" } rustls = { path = "patch/rustls-0.23.35" } -chrono = { path = "patch/chrono-0.4.42" } +chrono = { path = "patch/chrono-0.4.43" } dotenvy = { path = "patch/dotenvy-0.15.7" } prost = { path = "patch/prost-0.14.1" } # prost-derive = { path = "patch/prost-derive" } @@ -95,7 +90,9 @@ path = "src/main.rs" # path = "tools/rkyv_adapter/src/main.rs" [build-dependencies] -chrono = { version = "0.4", default-features = false, features = ["alloc"] } +__build = { package = "cursor-api-build", path = "__build" } + +# chrono = { version = "0.4", default-features = false, features = ["alloc"] } # prost-build = { version = "0.14", optional = true } sha2 = { version = "0", default-features = false } serde_json = "1" @@ -134,7 +131,7 @@ chrono = { version = "0.4", default-features = false, features = [ "serde", "rkyv-64", ] } -chrono-tz = { version = "0.10", features = ["serde"] } +chrono-tz = { version = "0.11.0", features = ["serde"], git = "https://github.com/chronotope/chrono-tz.git" } # crossbeam = { version = "0.8.4", features = ["nightly"] } # dashmap = { version = "7.0.0-rc2", features = ["inline-more"] } dotenvy = "0.15" diff --git a/VERSION b/VERSION index b5045cc..b393560 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -21 \ No newline at end of file +23 \ No newline at end of file diff --git a/__build/Cargo.toml b/__build/Cargo.toml new file mode 100644 index 0000000..b95402e --- /dev/null +++ b/__build/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "cursor-api-build" +version.workspace = true +edition.workspace = true +authors.workspace = true +description.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +chrono = { version = "0.4.43", default-features = false, features = ["alloc"] } diff --git a/__build/rustfmt.toml b/__build/rustfmt.toml new file mode 100644 index 0000000..fd8a930 --- /dev/null +++ b/__build/rustfmt.toml @@ -0,0 +1,8 @@ +style_edition = "2024" +use_small_heuristics = "Max" +merge_derives = false +group_imports = "One" +imports_granularity = "Module" +use_field_init_shorthand = true +tab_spaces = 2 +where_single_line = true diff --git a/__build/src/_marco.rs b/__build/src/_marco.rs new file mode 100644 index 0000000..3ab354b --- /dev/null +++ b/__build/src/_marco.rs @@ -0,0 +1,61 @@ +macro_rules! generate_static_variable { + ( + $variable_name: ident + $variable_type: ty + ) => { + pub(self) static $variable_name: $crate::_marco::Variable<$variable_type> = + $crate::_marco::uninit_variable(); + #[inline] + pub(crate) unsafe fn get_unchecked() -> &'static $variable_type { + unsafe { (&*$variable_name.0.get()).assume_init_ref() } + } + #[inline] + pub(crate) unsafe fn initialize() { + unsafe { (&mut *$variable_name.0.get()).write(_initialize()) }; + } + }; +} +pub(crate) struct Variable(pub(crate) ::core::cell::UnsafeCell<::core::mem::MaybeUninit>); +unsafe impl Send for Variable {} +unsafe impl Sync for Variable {} +#[inline(always)] +pub(crate) const fn uninit_variable() -> Variable { + Variable(::core::cell::UnsafeCell::new(::core::mem::MaybeUninit::uninit())) +} + +macro_rules! generate_variable_get { + ( + $variable_name: ident + $fn_result: ty + |$x:ident| $map:expr + ) => { + pub(crate) mod $variable_name; + #[inline] + pub fn $variable_name() -> $fn_result { + if !$crate::once::initialized() { + $crate::variables::initialize(); + } + let $x = unsafe { $variable_name::get_unchecked() }; + $map + } + }; +} + +macro_rules! reexport_info_types { + { + $( + $info_name: ident + $($info_type: ty)+, + )* + } => { + $( + mod $info_name; + )* + #[allow(unused_braces)] + pub(crate) mod prelude { + $( + pub use super::$info_name::{$($info_type,)+}; + )* + } + }; +} diff --git a/__build/src/infos/build.rs b/__build/src/infos/build.rs new file mode 100644 index 0000000..a3727d0 --- /dev/null +++ b/__build/src/infos/build.rs @@ -0,0 +1,53 @@ +use super::*; + +pub struct BuildInfo; + +impl BuildInfo { + pub fn write_to(self, mut writer: W) -> io::Result<()> { + write_generated(&mut writer)?; + writer.write_all(b"use crate::app::model::version::{Version, ReleaseStage::*};\n\n")?; + let version_number = version_number(); + if version_number != 0 { + writeln!(writer, "pub const BUILD_VERSION: u32 = {version_number};")?; + } + let build_timestamp = build_timestamp(); + writeln!( + writer, + "pub const BUILD_TIMESTAMP: &'static str = {:?};", + chrono::DateTime::from_timestamp_secs(build_timestamp as i64) + .unwrap() + .to_rfc3339_opts(chrono::SecondsFormat::Secs, true) + )?; + writeln!( + writer, + "/// pub const VERSION_STR: &'static str = \"{version}\";\npub const VERSION: Version = {version:?};", + version = pkg_version() + )?; + let is_preview = is_preview(); + let is_debug = cfg!(debug_assertions); + write!( + writer, + "pub const IS_PRERELEASE: bool = {is_preview};\npub const IS_DEBUG: bool = {is_debug};\n\n" + )?; + write!( + writer, + r#"#[cfg(unix)] +pub const BUILD_EPOCH: std::time::SystemTime = + unsafe {{ ::core::intrinsics::transmute(({build_timestamp}i64, 0u32)) }}; + +#[cfg(windows)] +pub const BUILD_EPOCH: std::time::SystemTime = unsafe {{ + const INTERVALS_PER_SEC: u64 = 10_000_000; + const INTERVALS_TO_UNIX_EPOCH: u64 = 11_644_473_600 * INTERVALS_PER_SEC; + const TARGET_INTERVALS: u64 = INTERVALS_TO_UNIX_EPOCH + {build_timestamp} * INTERVALS_PER_SEC; + + ::core::intrinsics::transmute(( + TARGET_INTERVALS as u32, + (TARGET_INTERVALS >> 32) as u32, + )) +}}; +"# + )?; + Ok(()) + } +} diff --git a/__build/src/infos/platform.rs b/__build/src/infos/platform.rs new file mode 100644 index 0000000..26b2abd --- /dev/null +++ b/__build/src/infos/platform.rs @@ -0,0 +1,63 @@ +use super::*; +use std::fs; + +#[allow(non_camel_case_types)] +#[derive(Clone, Copy)] +pub enum PlatformType { + Windows, + macOS, + Linux, + Android, + FreeBSD, + Unknown, +} + +impl PlatformType { + #[inline] + pub const fn as_str(self) -> &'static str { + match self { + PlatformType::Windows => "Windows", + PlatformType::macOS => "macOS", + PlatformType::Linux => "Linux", + PlatformType::Android => "Android", + PlatformType::FreeBSD => "FreeBSD", + PlatformType::Unknown => "Unknown", + } + } + #[inline] + pub const fn or_default(self) -> Self { + match self { + PlatformType::Windows | PlatformType::macOS | PlatformType::Linux => self, + _ => PlatformType::Windows, + } + } +} + +pub const CURRENT: PlatformType = cfg_select! { + target_os = "windows" => {PlatformType::Windows} + target_os = "macos" => {PlatformType::macOS} + target_os = "linux" => {PlatformType::Linux} + target_os = "android" => {PlatformType::Android} + target_os = "freebsd" => {PlatformType::FreeBSD} + _ => {PlatformType::Unknown} +}; + +pub struct PlatformInfo; + +impl PlatformInfo { + pub fn write_to(self, mut writer: W) -> io::Result<()> { + write_generated(&mut writer)?; + writer + .write_all(b"use crate::app::model::platform::PlatformType;\n\n")?; + let default = CURRENT.or_default(); + writeln!(writer, "pub const DEFAULT: PlatformType = PlatformType::{};", default.as_str())?; + writeln!( + writer, + "pub const CONFIG_EXAMPLE: &'static str = {:?};", + fs::read_to_string(manifest_dir().join("config.example.toml")) + .unwrap() + .replace("{DEFAULT_PLATFORM}", default.as_str()) + )?; + Ok(()) + } +} diff --git a/__build/src/lib.rs b/__build/src/lib.rs new file mode 100644 index 0000000..25f2f5e --- /dev/null +++ b/__build/src/lib.rs @@ -0,0 +1,53 @@ +// style +#![allow(clippy::redundant_static_lifetimes, clippy::needless_pub_self)] +#![feature(cfg_select)] + +#[macro_use] +mod _marco; +mod once; +mod version; + +pub mod variables { + use crate::version::Version; + use std::path::Path; + + #[cold] + fn initialize() { + crate::once::initialize(|| unsafe { + out_dir::initialize(); + cfg_feature::initialize(); + manifest_dir::initialize(); + build_timestamp::initialize(); + + // after cfg_feature + is_preview::initialize(); + + // after manifest_dir is_preview + version_number::initialize(); + // after version_number + pkg_version::initialize(); + }) + } + generate_variable_get!(out_dir &'static Path |x| x.as_path()); + generate_variable_get!(build_timestamp u64 |x| *x); + generate_variable_get!(version_number u16 |x| *x); + generate_variable_get!(manifest_dir &'static Path |x| x.as_path()); + generate_variable_get!(pkg_version Version |x| *x); + generate_variable_get!(cfg_feature &'static str |x| x.as_str()); + generate_variable_get!(is_preview bool |x| *x); +} + +mod infos { + use crate::variables::*; + use std::io::{self, Write}; + + fn write_generated(writer: &mut W) -> io::Result<()> { + writer + .write_all(b"// This file is automatically @generated by build.rs. Do not edit manually.\n") + } + reexport_info_types!( + build BuildInfo, + platform PlatformType CURRENT PlatformInfo, + ); +} +pub use infos::prelude::*; diff --git a/__build/src/once.rs b/__build/src/once.rs new file mode 100644 index 0000000..39e73ab --- /dev/null +++ b/__build/src/once.rs @@ -0,0 +1,14 @@ +use std::sync::Once; + +static ONCE: Once = Once::new(); + +#[cold] +pub fn initialize(f: F) +where F: FnOnce() { + ONCE.call_once_force(|_| f()); +} + +#[inline] +pub fn initialized() -> bool { + ONCE.is_completed() +} diff --git a/__build/src/variables/build_timestamp.rs b/__build/src/variables/build_timestamp.rs new file mode 100644 index 0000000..4b2d823 --- /dev/null +++ b/__build/src/variables/build_timestamp.rs @@ -0,0 +1,12 @@ +use std::env; +use std::time::{SystemTime, UNIX_EPOCH}; + +generate_static_variable!(BUILD_TIMESTAMP u64); + +fn _initialize() -> u64 { + if let Some(s) = env::var_os("BUILD_TIMESTAMP") { + s.to_str().unwrap().parse().unwrap() + } else { + SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() + } +} diff --git a/__build/src/variables/cfg_feature.rs b/__build/src/variables/cfg_feature.rs new file mode 100644 index 0000000..6ec41d7 --- /dev/null +++ b/__build/src/variables/cfg_feature.rs @@ -0,0 +1,9 @@ +use std::env; + +generate_static_variable!(CFG_FEATURE String); + +fn _initialize() -> String { + let s = env::var("CARGO_CFG_FEATURE").unwrap_or_default(); + println!("{s}"); + s +} diff --git a/__build/src/variables/is_preview.rs b/__build/src/variables/is_preview.rs new file mode 100644 index 0000000..92240cd --- /dev/null +++ b/__build/src/variables/is_preview.rs @@ -0,0 +1,5 @@ +generate_static_variable!(IS_PREVIEW bool); + +fn _initialize() -> bool { + unsafe { super::cfg_feature::get_unchecked() }.contains("__preview") +} diff --git a/__build/src/variables/manifest_dir.rs b/__build/src/variables/manifest_dir.rs new file mode 100644 index 0000000..7ef7b22 --- /dev/null +++ b/__build/src/variables/manifest_dir.rs @@ -0,0 +1,8 @@ +use std::env; +use std::path::PathBuf; + +generate_static_variable!(CARGO_MANIFEST_DIR PathBuf); + +fn _initialize() -> PathBuf { + env::var_os("CARGO_MANIFEST_DIR").unwrap().into() +} diff --git a/__build/src/variables/out_dir.rs b/__build/src/variables/out_dir.rs new file mode 100644 index 0000000..7ec413e --- /dev/null +++ b/__build/src/variables/out_dir.rs @@ -0,0 +1,8 @@ +use std::env; +use std::path::PathBuf; + +generate_static_variable!(OUT_DIR PathBuf); + +fn _initialize() -> PathBuf { + env::var_os("OUT_DIR").unwrap().into() +} diff --git a/__build/src/variables/pkg_version.rs b/__build/src/variables/pkg_version.rs new file mode 100644 index 0000000..9d8ec6c --- /dev/null +++ b/__build/src/variables/pkg_version.rs @@ -0,0 +1,14 @@ +use std::env; + +use crate::version::ReleaseStage::*; +use crate::version::Version; + +generate_static_variable!(PKG_VERSION Version); + +fn _initialize() -> Version { + let mut ver: Version = env::var("CARGO_PKG_VERSION").unwrap().parse().unwrap(); + if let Preview { ref mut build, .. } = ver.stage { + *build = Some(*unsafe { super::version_number::get_unchecked() }); + } + ver +} diff --git a/__build/src/variables/version_number.rs b/__build/src/variables/version_number.rs new file mode 100644 index 0000000..f49d4b7 --- /dev/null +++ b/__build/src/variables/version_number.rs @@ -0,0 +1,19 @@ +use std::{env, fs}; + +generate_static_variable!(VERSION_NUMBER u16); + +fn _initialize() -> u16 { + if unsafe { *super::is_preview::get_unchecked() } { + if let Some(s) = env::var_os("VERSION_NUMBER") { + s.to_str().unwrap().trim().parse().unwrap() + } else { + fs::read_to_string(unsafe { super::manifest_dir::get_unchecked() }.join("VERSION")) + .unwrap() + .trim() + .parse() + .unwrap() + } + } else { + 0 + } +} diff --git a/__build/src/version.rs b/__build/src/version.rs new file mode 100644 index 0000000..ecb4ee7 --- /dev/null +++ b/__build/src/version.rs @@ -0,0 +1,144 @@ +/// 版本发布阶段 +#[derive(Debug, Clone, Copy)] +pub enum ReleaseStage { + /// 正式发布版本 + Release, + /// 预览版本,格式如 `-pre.6` 或 `-pre.6+build.8` + Preview { + /// 预览版本号 + version: u16, + /// 构建号(可选) + build: Option, + }, +} + +impl core::fmt::Display for ReleaseStage { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + ReleaseStage::Release => Ok(()), + ReleaseStage::Preview { version, build: None } => { + write!(f, "-pre.{version}") + } + ReleaseStage::Preview { version, build: Some(build) } => { + write!(f, "-pre.{version}+build.{build}") + } + } + } +} + +/// 遵循格式:v0.4.0-pre.6+build.8 +#[derive(Debug, Clone, Copy)] +pub struct Version { + pub major: u16, + pub minor: u16, + pub patch: u16, + pub stage: ReleaseStage, +} + +impl core::fmt::Display for Version { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{}.{}.{}", self.major, self.minor, self.patch)?; + self.stage.fmt(f) + } +} + +/// 版本字符串解析错误 +#[allow(clippy::enum_variant_names)] +#[derive(Debug)] +pub enum ParseError { + /// 整体格式错误(如缺少必需部分) + InvalidFormat, + /// 数字解析失败 + InvalidNumber, + /// pre 部分格式错误 + InvalidPreRelease, + /// build 部分格式错误 + InvalidBuild, + // /// 正式版不能带 build 标识 + // BuildWithoutPreview, +} + +impl core::fmt::Display for ParseError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + ParseError::InvalidFormat => write!(f, "invalid version format"), + ParseError::InvalidNumber => write!(f, "invalid number in version"), + ParseError::InvalidPreRelease => write!(f, "invalid pre-release format"), + ParseError::InvalidBuild => write!(f, "invalid build format"), + // ParseError::BuildWithoutPreview => { + // write!(f, "build metadata cannot exist without pre-release version") + // } + } + } +} + +impl std::error::Error for ParseError {} + +impl core::str::FromStr for Version { + type Err = ParseError; + fn from_str(s: &str) -> core::result::Result { + // 按 '-' 分割基础版本号和扩展部分 + let (base, extension) = match s.split_once('-') { + Some((base, ext)) => (base, Some(ext)), + None => (s, None), + }; + + // 解析基础版本号 major.minor.patch + let mut parts: [u16; 3] = [0, 0, 0]; + let mut parsed_count = 0; + for (i, s) in base.split('.').enumerate() { + if i >= parts.len() { + return Err(ParseError::InvalidFormat); + } + parts[i] = s.parse().map_err(|_| ParseError::InvalidNumber)?; + parsed_count += 1; + } + if parsed_count != 3 { + return Err(ParseError::InvalidFormat); + } + + let major = parts[0]; + let minor = parts[1]; + let patch = parts[2]; + + // 解析扩展部分(如果存在) + let stage = + if let Some(ext) = extension { parse_extension(ext)? } else { ReleaseStage::Release }; + + Ok(Version { major, minor, patch, stage }) + } +} + +/// 解析扩展部分:pre.X 或 pre.X+build.Y +fn parse_extension(s: &str) -> core::result::Result { + // 检查是否以 "pre." 开头 + // 移除 "pre." 前缀 + let Some(after_pre) = s.strip_prefix("pre.") else { + return Err(ParseError::InvalidPreRelease); + }; + + // 按 '+' 分割 version 和 build 部分 + let (version_str, build_str) = match after_pre.split_once('+') { + Some((ver, build_part)) => (ver, Some(build_part)), + None => (after_pre, None), + }; + + // 解析 pre 版本号 + let version = version_str.parse().map_err(|_| ParseError::InvalidPreRelease)?; + + // 解析 build 号(如果存在) + let build = if let Some(build_part) = build_str { + // 检查格式是否为 "build.X" + let Some(build_num_str) = build_part.strip_prefix("build.") else { + return Err(ParseError::InvalidBuild); + }; + + let build_num = build_num_str.parse().map_err(|_| ParseError::InvalidBuild)?; + + Some(build_num) + } else { + None + }; + + Ok(ReleaseStage::Preview { version, build }) +} diff --git a/build.rs b/build.rs index 254db20..0e3d6b5 100644 --- a/build.rs +++ b/build.rs @@ -4,18 +4,15 @@ use sha2::{Digest, Sha256}; use std::collections::HashMap; #[cfg(not(feature = "use-minified"))] use std::fs; -#[cfg(not(debug_assertions))] -#[cfg(feature = "__preview")] -use std::fs::File; -use std::io::Result; -#[cfg(not(debug_assertions))] -#[cfg(feature = "__preview")] +#[cfg(all(not(debug_assertions), not(feature = "__preview_locked"), feature = "__preview"))] use std::io::{Read, Write}; +#[cfg(not(feature = "use-minified"))] use std::path::Path; #[cfg(not(feature = "use-minified"))] use std::path::PathBuf; #[cfg(not(feature = "use-minified"))] use std::process::Command; +use std::{fs::File, io::Result}; // 支持的文件类型 // #[cfg(not(feature = "use-minified"))] @@ -187,7 +184,15 @@ fn minify_assets() -> Result<()> { Ok(()) } -include!("build_info.rs"); +fn generate_build_info() -> Result<()> { + let file = File::create(__build::variables::out_dir().join("build_info.rs"))?; + __build::BuildInfo.write_to(file) +} + +fn generate_platform_info() -> Result<()> { + let file = File::create(__build::variables::out_dir().join("platform_info.rs"))?; + __build::PlatformInfo.write_to(file) +} // #[cfg(feature = "__protoc")] // macro_rules! proto_attributes { @@ -202,7 +207,7 @@ include!("build_info.rs"); fn main() -> Result<()> { // 更新版本号 - 只在 release 构建时执行 - #[cfg(all(not(debug_assertions), feature = "__preview"))] + #[cfg(all(not(debug_assertions), not(feature = "__preview_locked"), feature = "__preview"))] update_version()?; // #[cfg(feature = "__protoc")] @@ -345,6 +350,7 @@ fn main() -> Result<()> { // 生成构建信息文件 generate_build_info()?; + generate_platform_info()?; Ok(()) } diff --git a/config.example.toml b/config.example.toml index 534586a..ebe3e8c 100644 --- a/config.example.toml +++ b/config.example.toml @@ -41,8 +41,18 @@ dynamic_key_secret = "" web_references_included = false # 模型数据获取模式 -# - 可选值: -# - truncate - 覆盖模式(默认): 完全使用新获取的模型列表,替换所有现有模型 -# - append:truncate - 智能合并模式: 保留现有模型中不在新列表中的,同时添加或更新新模型 -# - append - 纯追加模式: 只添加不存在的新模型,已有模型保持不变 +# 可选值: +# - truncate - 覆盖模式(默认): 完全使用新获取的模型列表,替换所有现有模型 +# - append:truncate - 智能合并模式: 保留现有模型中不在新列表中的,同时添加或更新新模型 +# - append - 纯追加模式: 只添加不存在的新模型,已有模型保持不变 raw_model_fetch_mode = "truncate" + +# 模拟平台(默认{DEFAULT_PLATFORM}) +# 可选值: +# - Windows +# - macOS +# - Linux +emulated_platform = "{DEFAULT_PLATFORM}" + +# Cursor客户端版本 +cursor_client_version = "2.0.0" diff --git a/crates/byte_str/Cargo.toml b/crates/byte_str/Cargo.toml index bed66e1..0612d20 100644 --- a/crates/byte_str/Cargo.toml +++ b/crates/byte_str/Cargo.toml @@ -10,10 +10,10 @@ license = "MIT" any_all_workaround = { version = "0.1", optional = true } bytes = { version = "1", default-features = false } cfg-if = "1" -serde = { version = "1", default-features = false, optional = true } +serde_core = { version = "1", default-features = false, optional = true } [features] default = ["nightly"] std = ["bytes/std"] -serde = ["dep:serde"] +serde = ["dep:serde_core"] nightly = ["dep:any_all_workaround"] diff --git a/crates/byte_str/src/lib.rs b/crates/byte_str/src/lib.rs index daf0ebc..38b9759 100644 --- a/crates/byte_str/src/lib.rs +++ b/crates/byte_str/src/lib.rs @@ -20,7 +20,7 @@ extern crate alloc; extern crate bytes; #[cfg(feature = "serde")] -extern crate serde; +extern crate serde_core; #[macro_use] extern crate cfg_if; @@ -207,18 +207,13 @@ impl core::hash::Hash for ByteStr { #[inline] fn hash(&self, state: &mut H) where H: core::hash::Hasher { - self.bytes.hash(state) + ops::Deref::deref(self).hash(state) } } impl const Borrow for ByteStr { #[inline] - fn borrow(&self) -> &str { &**self } -} - -impl const Borrow<[u8]> for ByteStr { - #[inline] - fn borrow(&self) -> &[u8] { self.as_ref() } + fn borrow(&self) -> &str { self } } impl PartialEq for ByteStr { diff --git a/crates/byte_str/src/serde_impls.rs b/crates/byte_str/src/serde_impls.rs index de7defa..e3a98d5 100644 --- a/crates/byte_str/src/serde_impls.rs +++ b/crates/byte_str/src/serde_impls.rs @@ -3,17 +3,17 @@ use alloc::vec::Vec; use super::*; -impl serde::Serialize for ByteStr { +impl serde_core::Serialize for ByteStr { #[inline] fn serialize(&self, serializer: S) -> Result - where S: serde::Serializer { - serializer.serialize_str(&**self) + where S: serde_core::Serializer { + serializer.serialize_str(self) } } struct ByteStrVisitor; -impl<'de> serde::de::Visitor<'de> for ByteStrVisitor { +impl<'de> serde_core::de::Visitor<'de> for ByteStrVisitor { type Value = ByteStr; fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -22,19 +22,19 @@ impl<'de> serde::de::Visitor<'de> for ByteStrVisitor { #[inline] fn visit_str(self, v: &str) -> Result - where E: serde::de::Error { + where E: serde_core::de::Error { Ok(ByteStr::from(v)) } #[inline] fn visit_string(self, v: String) -> Result - where E: serde::de::Error { + where E: serde_core::de::Error { Ok(ByteStr::from(v)) } #[inline] fn visit_bytes(self, v: &[u8]) -> Result - where E: serde::de::Error { + where E: serde_core::de::Error { match str::from_utf8(v) { Ok(s) => Ok(ByteStr::from(s)), Err(e) => Err(E::custom(format_args!("invalid UTF-8: {e}"))), @@ -43,7 +43,7 @@ impl<'de> serde::de::Visitor<'de> for ByteStrVisitor { #[inline] fn visit_byte_buf(self, v: Vec) -> Result - where E: serde::de::Error { + where E: serde_core::de::Error { match String::from_utf8(v) { Ok(s) => Ok(ByteStr::from(s)), Err(e) => Err(E::custom(format_args!("invalid UTF-8: {}", e.utf8_error()))), @@ -52,8 +52,8 @@ impl<'de> serde::de::Visitor<'de> for ByteStrVisitor { #[inline] fn visit_seq(self, mut seq: V) -> Result - where V: serde::de::SeqAccess<'de> { - use serde::de::Error as _; + where V: serde_core::de::SeqAccess<'de> { + use serde_core::de::Error as _; let len = core::cmp::min(seq.size_hint().unwrap_or(0), 4096); let mut bytes: Vec = Vec::with_capacity(len); @@ -68,10 +68,10 @@ impl<'de> serde::de::Visitor<'de> for ByteStrVisitor { } } -impl<'de> serde::Deserialize<'de> for ByteStr { +impl<'de> serde_core::Deserialize<'de> for ByteStr { #[inline] fn deserialize(deserializer: D) -> Result - where D: serde::Deserializer<'de> { + where D: serde_core::Deserializer<'de> { deserializer.deserialize_string(ByteStrVisitor) } } diff --git a/crates/interned/Cargo.toml b/crates/interned/Cargo.toml index 99dc2ff..fcfccca 100644 --- a/crates/interned/Cargo.toml +++ b/crates/interned/Cargo.toml @@ -36,12 +36,12 @@ ahash = { version = "0.8", default-features = false, features = [ manually_init.workspace = true -serde = { version = "1.0", default-features = false, optional = true } +serde_core = { version = "1", default-features = false, optional = true } [features] default = ["serde"] nightly = [] -serde = ["dep:serde"] +serde = ["dep:serde_core"] #[profile.release] #opt-level = 3 diff --git a/crates/interned/src/arc_str.rs b/crates/interned/src/arc_str.rs index 0824e52..c1c26b8 100644 --- a/crates/interned/src/arc_str.rs +++ b/crates/interned/src/arc_str.rs @@ -58,7 +58,7 @@ use scc::{Equivalent, HashMap}; /// # 设计目标 /// /// - **内存去重**:相同内容的字符串共享同一内存地址 -/// - **零拷贝克隆**:clone() 只涉及原子递增操作 +/// - **零拷贝克隆**:`clone()` 只涉及原子递增操作 /// - **线程安全**:支持多线程环境下的安全使用 /// - **高性能查找**:使用预计算哈希值优化池查找 /// @@ -115,8 +115,8 @@ impl ArcStr { /// /// # 性能特征 /// - /// - **池命中**:O(1) HashMap 查找 + 原子递增 - /// - **池缺失**:O(1) 内存分配 + O(1) HashMap 插入 + /// - **池命中**:O(1) `HashMap` 查找 + 原子递增 + /// - **池缺失**:O(1) 内存分配 + O(1) `HashMap` 插入 /// - **哈希计算**:使用 ahash 的高性能哈希算法 /// /// # Examples @@ -150,9 +150,8 @@ impl ArcStr { // 进入这里说明需要创建新的字符串实例 let pool = ARC_STR_POOL.get(); - use scc::hash_map::RawEntry; match pool.raw_entry().from_key_hashed_nocheck_sync(hash, string) { - RawEntry::Occupied(entry) => { + scc::hash_map::RawEntry::Occupied(entry) => { // 双重检查:在获取写锁的过程中,其他线程可能已经创建了相同的字符串 let ptr = entry.key().0; @@ -162,13 +161,13 @@ impl ArcStr { Self { ptr, _marker: PhantomData } } - RawEntry::Vacant(entry) => { + scc::hash_map::RawEntry::Vacant(entry) => { // 确认需要创建新实例:分配内存并初始化 let layout = ArcStrInner::layout_for_string(string.len()); // SAFETY: layout_for_string 确保布局有效且大小合理 let ptr = unsafe { - let alloc = alloc::alloc::alloc(layout) as *mut ArcStrInner; + let alloc: *mut ArcStrInner = alloc::alloc::alloc(layout).cast(); if alloc.is_null() { hint::cold_path(); @@ -197,7 +196,8 @@ impl ArcStr { /// /// 这是一个 `const fn`,在编译时就能确定偏移量, /// 运行时仅需要一次内存解引用。 - #[inline(always)] + #[must_use] + #[inline] pub const fn as_str(&self) -> &str { // SAFETY: ptr 在 ArcStr 生命周期内始终指向有效的 ArcStrInner, // 且字符串数据保证是有效的 UTF-8 @@ -207,28 +207,32 @@ impl ArcStr { /// 获取字符串的字节切片 /// /// 提供对底层字节数据的直接访问。 - #[inline(always)] + #[must_use] + #[inline] pub const fn as_bytes(&self) -> &[u8] { // SAFETY: ptr 始终指向有效的 ArcStrInner unsafe { self.ptr.as_ref().as_bytes() } } /// 获取字符串长度(字节数) - #[inline(always)] + #[must_use] + #[inline] pub const fn len(&self) -> usize { // SAFETY: ptr 始终指向有效的 ArcStrInner unsafe { self.ptr.as_ref().string_len } } /// 检查字符串是否为空 - #[inline(always)] + #[must_use] + #[inline] pub const fn is_empty(&self) -> bool { self.len() == 0 } /// 获取当前引用计数 /// /// 注意:由于并发访问,返回的值可能在返回后立即发生变化。 /// 此方法主要用于调试和测试。 - #[inline(always)] + #[must_use] + #[inline] pub fn ref_count(&self) -> usize { // SAFETY: ptr 始终指向有效的 ArcStrInner unsafe { self.ptr.as_ref().strong_count() } @@ -237,7 +241,8 @@ impl ArcStr { /// 获取字符串数据的内存地址(用于调试和测试) /// /// 返回字符串内容的起始地址,可用于验证字符串是否共享内存。 - #[inline(always)] + #[must_use] + #[inline] pub const fn as_ptr(&self) -> *const u8 { // SAFETY: ptr 始终指向有效的 ArcStrInner unsafe { self.ptr.as_ref().string_ptr() } @@ -257,7 +262,8 @@ impl ArcStr { /// # 返回值 /// /// 如果找到匹配的字符串,返回增加引用计数后的 `ArcStr`;否则返回 `None`。 - #[inline(always)] + #[must_use] + #[inline] fn try_find_existing(pool: &PtrMap, hash: u64, string: &str) -> Option { // 使用 hashbrown 的 from_key_hashed_nocheck API // 这利用了 Equivalent trait 来进行高效比较 @@ -349,10 +355,10 @@ impl Drop for ArcStr { // 第二层:标准库集成 // ═══════════════════════════════════════════════════════════════════════════ -/// # 基础 Trait 实现 -/// -/// 这些实现确保 `ArcStr` 能够与 Rust 的标准库类型无缝集成, -/// 提供符合直觉的比较、格式化和访问接口。 +// # 基础 Trait 实现 +// +// 这些实现确保 `ArcStr` 能够与 Rust 的标准库类型无缝集成, +// 提供符合直觉的比较、格式化和访问接口。 impl PartialEq for ArcStr { /// 基于指针的快速相等比较 @@ -423,10 +429,10 @@ impl const core::ops::Deref for ArcStr { fn deref(&self) -> &Self::Target { self.as_str() } } -/// # 与其他字符串类型的互操作性 -/// -/// 这些实现使得 `ArcStr` 可以与 Rust 生态系统中的各种字符串类型 -/// 进行直接比较,提供良好的开发体验。 +// # 与其他字符串类型的互操作性 +// +// 这些实现使得 `ArcStr` 可以与 Rust 生态系统中的各种字符串类型 +// 进行直接比较,提供良好的开发体验。 impl const PartialEq for ArcStr { #[inline] @@ -470,10 +476,10 @@ impl PartialOrd for ArcStr { } } -/// # 类型转换实现 -/// -/// 提供从各种字符串类型到 `ArcStr` 的便捷转换, -/// 以及从 `ArcStr` 到其他类型的转换。 +// # 类型转换实现 +// +// 提供从各种字符串类型到 `ArcStr` 的便捷转换, +// 以及从 `ArcStr` 到其他类型的转换。 impl<'a> From<&'a str> for ArcStr { #[inline] @@ -523,8 +529,8 @@ impl str::FromStr for ArcStr { /// 序列化时输出字符串内容,反序列化时重新建立池化引用。 #[cfg(feature = "serde")] mod serde_impls { - use super::*; - use serde::{Deserialize, Deserializer, Serialize, Serializer}; + use super::ArcStr; + use serde_core::{Deserialize, Deserializer, Serialize, Serializer}; impl Serialize for ArcStr { #[inline] @@ -547,10 +553,10 @@ mod serde_impls { // 第三层:核心实现机制 // ═══════════════════════════════════════════════════════════════════════════ -/// # 内存布局与数据结构设计 -/// -/// 这个模块包含了 `ArcStr` 的底层数据结构定义和内存布局管理。 -/// 理解这部分有助于深入了解性能优化的原理。 +// # 内存布局与数据结构设计 +// +// 这个模块包含了 `ArcStr` 的底层数据结构定义和内存布局管理。 +// 理解这部分有助于深入了解性能优化的原理。 /// 字符串内容的内部表示(DST 头部) /// @@ -593,15 +599,15 @@ struct ArcStrInner { /// 预计算的内容哈希值 /// /// 这个哈希值在多个场景中被复用: - /// - 全局池的HashMap键 - /// - Drop时的快速查找 + /// - 全局池的 `HashMap` 键 + /// - `Drop` 时的快速查找 /// - 避免重复哈希计算的性能优化 hash: u64, /// 原子引用计数 /// /// 使用原生原子类型确保最佳性能。 - /// 计数范围:[1, isize::MAX],超出时触发abort。 + /// 计数范围:[1, `isize::MAX`],超出时触发abort。 count: AtomicUsize, /// 字符串的字节长度(UTF-8编码) @@ -627,10 +633,11 @@ impl ArcStrInner { /// - `self` 必须是指向有效 `ArcStrInner` 的指针 /// - 必须确保字符串数据已经被正确初始化 /// - 调用者负责确保返回的指针在使用期间保持有效 - #[inline(always)] + #[must_use] + #[inline] const unsafe fn string_ptr(&self) -> *const u8 { // SAFETY: repr(C) 保证字符串数据位于结构体末尾的固定偏移处 - (self as *const Self).add(1).cast() + core::ptr::from_ref(self).add(1).cast() } /// 获取字符串的字节切片 @@ -641,7 +648,8 @@ impl ArcStrInner { /// - 字符串数据必须已经被正确初始化 /// - `string_len` 必须准确反映实际字符串长度 /// - 字符串数据必须在返回的切片生命周期内保持有效 - #[inline(always)] + #[must_use] + #[inline] const unsafe fn as_bytes(&self) -> &[u8] { let ptr = self.string_ptr(); // SAFETY: 调用者保证 ptr 指向有效的 string_len 字节数据 @@ -656,7 +664,8 @@ impl ArcStrInner { /// - 字符串数据必须是有效的 UTF-8 编码 /// - `string_len` 必须准确反映实际字符串长度 /// - 字符串数据必须在返回的切片生命周期内保持有效 - #[inline(always)] + #[must_use] + #[inline] const unsafe fn as_str(&self) -> &str { // SAFETY: 调用者保证字符串数据是有效的 UTF-8 core::str::from_utf8_unchecked(self.as_bytes()) @@ -729,7 +738,7 @@ impl ArcStrInner { // - string_ptr() 计算出的地址位于已分配内存范围内 // - string.len() 与分配时的长度一致 // - string.as_ptr() 指向有效的 UTF-8 数据 - let string_ptr = (*inner).string_ptr() as *mut u8; + let string_ptr = (*inner).string_ptr().cast_mut(); core::ptr::copy_nonoverlapping(string.as_ptr(), string_ptr, string.len()); } @@ -783,14 +792,14 @@ impl ArcStrInner { fn strong_count(&self) -> usize { self.count.load(Relaxed) } } -/// # 全局字符串池的设计与实现 -/// -/// 全局池是整个系统的核心,负责去重和生命周期管理。 +// # 全局字符串池的设计与实现 +// +// 全局池是整个系统的核心,负责去重和生命周期管理。 /// 线程安全的内部指针包装 /// /// 这个类型解决了在 `HashMap` 中存储 `NonNull` 的问题: -/// - 提供必要的 trait 实现(Hash, PartialEq, Send, Sync) +/// - 提供必要的 trait 实现( `Hash`, `PartialEq`, `Send`, `Sync` ) /// - 封装指针的线程安全语义 /// - 支持基于内容的查找(通过 Equivalent trait) /// @@ -811,7 +820,7 @@ unsafe impl Sync for ThreadSafePtr {} impl const core::ops::Deref for ThreadSafePtr { type Target = NonNull; - #[inline(always)] + #[inline] fn deref(&self) -> &Self::Target { &self.0 } } @@ -826,7 +835,7 @@ impl Hash for ThreadSafePtr { // SAFETY: ThreadSafePtr 保证指针在池生命周期内始终有效 unsafe { let inner = self.0.as_ref(); - state.write_u64(inner.hash) + state.write_u64(inner.hash); } } } @@ -871,11 +880,11 @@ impl Equivalent for str { } } -/// # 哈希算法选择与池类型定义 +// # 哈希算法选择与池类型定义 /// 透传哈希器,用于全局池内部 /// -/// 由于我们在 `ArcStrInner` 中预存了哈希值,池内部的 HashMap +/// 由于我们在 `ArcStrInner` 中预存了哈希值,池内部的 `HashMap` /// 不需要重新计算哈希。`IdentityHasher` 直接透传 u64 值。 /// /// # 工作原理 @@ -883,7 +892,7 @@ impl Equivalent for str { /// 1. `ThreadSafePtr::hash()` 调用 `hasher.write_u64(stored_hash)` /// 2. `IdentityHasher::write_u64()` 直接存储这个值 /// 3. `IdentityHasher::finish()` 返回存储的值 -/// 4. HashMap 使用这个哈希值进行桶分配和查找 +/// 4. `HashMap` 使用这个哈希值进行桶分配和查找 /// /// 这避免了重复的哈希计算,将池操作的哈希开销降到最低。 #[derive(Default, Clone, Copy)] @@ -894,10 +903,10 @@ impl Hasher for IdentityHasher { unreachable!("IdentityHasher usage error"); } - #[inline(always)] - fn write_u64(&mut self, id: u64) { self.0 = id; } + #[inline] + fn write_u64(&mut self, id: u64) { self.0 = id } - #[inline(always)] + #[inline] fn finish(&self) -> u64 { self.0 } } @@ -912,7 +921,7 @@ type PtrMap = HashMap; /// /// # 为什么使用 ahash? /// -/// - 高性能:比标准库的 DefaultHasher 更快 +/// - 高性能:比标准库的 `DefaultHasher` 更快 /// - 安全性:抗哈希洪水攻击 /// - 质量:分布均匀,减少哈希冲突 static CONTENT_HASHER: ManuallyInit = ManuallyInit::new(); @@ -949,6 +958,8 @@ static ARC_STR_POOL: ManuallyInit = ManuallyInit::new(); /// 虽然这个函数本身不是线程安全的,但它应该在单线程环境下 /// (如 main 函数开始或静态初始化时)被调用一次。 #[inline(always)] +// 只调用一次 +#[allow(clippy::inline_always)] pub(crate) fn __init() { CONTENT_HASHER.init(ahash::RandomState::new()); ARC_STR_POOL.init(PtrMap::with_capacity_and_hasher(128, PoolHasher::default())); diff --git a/crates/interned/src/str.rs b/crates/interned/src/str.rs index 35feb63..ac0944b 100644 --- a/crates/interned/src/str.rs +++ b/crates/interned/src/str.rs @@ -29,8 +29,8 @@ //! | 创建 | 0 ns | ~100 ns (首次) / ~20 ns (池命中) | //! | Clone | ~1 ns | ~5 ns (atomic inc) | //! | Drop | 0 ns | ~5 ns (atomic dec) + 可能的清理 | -//! | as_str() | 0 ns | 0 ns (直接访问) | -//! | len() | 0 ns | 0 ns (直接读字段) | +//! | `as_str()` | 0 ns | 0 ns (直接访问) | +//! | `len()` | 0 ns | 0 ns (直接读字段) | //! //! # 使用场景 //! @@ -220,6 +220,7 @@ impl Str { /// assert_eq!(s.as_static(), Some("constant")); /// assert_eq!(s.ref_count(), None); /// ``` + #[must_use] #[inline] pub const fn from_static(s: &'static str) -> Self { Self::Static(s) } @@ -229,8 +230,8 @@ impl Str { /// /// # Performance /// - /// - **首次创建**:堆分配 + HashMap 插入 ≈ 100-200ns - /// - **池命中**:HashMap 查找 + 引用计数递增 ≈ 10-20ns + /// - **首次创建**:堆分配 + `HashMap` 插入 ≈ 100-200ns + /// - **池命中**:`HashMap` 查找 + 引用计数递增 ≈ 10-20ns /// /// # Thread Safety /// @@ -304,6 +305,7 @@ impl Str { /// } /// # fn register_constant(_: &'static str) {} /// ``` + #[must_use] #[inline] pub const fn is_static(&self) -> bool { matches!(self, Self::Static(_)) } @@ -330,6 +332,7 @@ impl Str { /// assert_eq!(s2.ref_count(), Some(2)); /// assert_eq!(s3.ref_count(), Some(2)); /// ``` + #[must_use] #[inline] pub fn ref_count(&self) -> Option { match self { @@ -374,6 +377,7 @@ impl Str { /// eprintln!("warning: not a static string"); /// } /// ``` + #[must_use] #[inline] pub const fn as_static(&self) -> Option<&'static str> { match self { @@ -397,6 +401,7 @@ impl Str { /// assert!(s1.as_arc_str().is_none()); /// assert!(s2.as_arc_str().is_some()); /// ``` + #[must_use] #[inline] pub const fn as_arc_str(&self) -> Option<&ArcStr> { match self { @@ -421,6 +426,7 @@ impl Str { /// assert!(s1.into_arc_str().is_some()); /// assert!(s2.into_arc_str().is_none()); /// ``` + #[must_use] #[inline] pub fn into_arc_str(self) -> Option { match self { @@ -454,7 +460,8 @@ impl Str { /// let s = Str::new("hello"); /// assert_eq!(s.as_str(), "hello"); /// ``` - #[inline(always)] + #[must_use] + #[inline] pub const fn as_str(&self) -> &str { match self { Self::Static(s) => s, @@ -474,7 +481,8 @@ impl Str { /// let s = Str::new("hello"); /// assert_eq!(s.as_bytes(), b"hello"); /// ``` - #[inline(always)] + #[must_use] + #[inline] pub const fn as_bytes(&self) -> &[u8] { match self { Self::Static(s) => s.as_bytes(), @@ -499,7 +507,8 @@ impl Str { /// let s = Str::new("hello"); /// assert_eq!(s.len(), 5); /// ``` - #[inline(always)] + #[must_use] + #[inline] pub const fn len(&self) -> usize { match self { Self::Static(s) => s.len(), @@ -522,7 +531,8 @@ impl Str { /// assert!(s1.is_empty()); /// assert!(!s2.is_empty()); /// ``` - #[inline(always)] + #[must_use] + #[inline] pub const fn is_empty(&self) -> bool { match self { Self::Static(s) => s.is_empty(), @@ -541,7 +551,8 @@ impl Str { /// let ptr = s.as_ptr(); /// assert!(!ptr.is_null()); /// ``` - #[inline(always)] + #[must_use] + #[inline] pub const fn as_ptr(&self) -> *const u8 { match self { Self::Static(s) => s.as_ptr(), @@ -686,7 +697,7 @@ impl From for alloc::boxed::Box { fn from(s: Str) -> Self { s.as_str().into() } } -impl<'a> From for Cow<'a, str> { +impl From for Cow<'_, str> { /// 转换为 `Cow` /// /// - **Static 变体**:转换为 `Cow::Borrowed`(零成本) @@ -1057,8 +1068,8 @@ impl const Default for Str { #[cfg(feature = "serde")] mod serde_impls { - use super::*; - use serde::{Deserialize, Deserializer, Serialize, Serializer}; + use super::Str; + use serde_core::{Deserialize, Deserializer, Serialize, Serializer}; impl Serialize for Str { /// 序列化为普通字符串,丢失变体信息 diff --git a/crates/proto-value/Cargo.toml b/crates/proto-value/Cargo.toml index 1bef760..268cbfe 100644 --- a/crates/proto-value/Cargo.toml +++ b/crates/proto-value/Cargo.toml @@ -18,12 +18,12 @@ byte_str = { version = "0.1.0", path = "../byte_str", default-features = false, bytes = { version = "1.11.0", default-features = false, optional = true } indexmap = { version = "2.12.1", default-features = false, optional = true } itoa = { version = "1.0.17", optional = true } -serde = { version = "1.0.228", default-features = false, optional = true } +serde_core = { version = "1.0", default-features = false, optional = true } [features] default = ["bytes", "byte_str"] alloc = [] -serde = ["dep:serde", "dep:base64-simd", "dep:itoa"] +serde = ["dep:serde_core", "dep:base64-simd", "dep:itoa"] std = ["alloc"] indexmap = ["dep:indexmap"] bytes = ["dep:bytes"] diff --git a/crates/proto-value/src/bytes_value.rs b/crates/proto-value/src/bytes_value.rs index dca0d1c..8335e7d 100644 --- a/crates/proto-value/src/bytes_value.rs +++ b/crates/proto-value/src/bytes_value.rs @@ -129,8 +129,8 @@ mod serde_impls { use core::fmt; use base64_simd::{forgiving_decode_to_vec, STANDARD}; - use serde::{Deserialize, Deserializer, Serialize, Serializer}; - use serde::de::{self, Unexpected, Visitor}; + use serde_core::{Deserialize, Deserializer, Serialize, Serializer}; + use serde_core::de::{self, Unexpected, Visitor}; use super::Bytes; diff --git a/crates/proto-value/src/enum_value.rs b/crates/proto-value/src/enum_value.rs index b426dd9..68a87d3 100644 --- a/crates/proto-value/src/enum_value.rs +++ b/crates/proto-value/src/enum_value.rs @@ -41,7 +41,7 @@ impl> const From for Enum { mod serde_impls { use super::Enum; use core::{fmt, marker::PhantomData}; - use serde::{ + use serde_core::{ Deserialize, Deserializer, Serialize, Serializer, de::{self, Unexpected, Visitor, value::StrDeserializer}, }; diff --git a/crates/proto-value/src/lib.rs b/crates/proto-value/src/lib.rs index 949b5c3..c700051 100644 --- a/crates/proto-value/src/lib.rs +++ b/crates/proto-value/src/lib.rs @@ -11,7 +11,7 @@ extern crate alloc; #[cfg(feature = "serde")] -extern crate serde; +extern crate serde_core; mod bytes_value; mod enum_value; diff --git a/crates/proto-value/src/stringify.rs b/crates/proto-value/src/stringify.rs index d1563d2..d9a7406 100644 --- a/crates/proto-value/src/stringify.rs +++ b/crates/proto-value/src/stringify.rs @@ -6,7 +6,7 @@ //! 特别适用于与 JavaScript 交互时避免精度损失的场景。 use core::{fmt, marker::PhantomData}; -use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; +use serde_core::{Deserialize, Deserializer, Serialize, Serializer, de}; /// 密封特征,限制可以被字符串化的类型 mod private { diff --git a/patch/chrono-0.4.42/src/offset/local/win_bindings.txt b/patch/chrono-0.4.42/src/offset/local/win_bindings.txt deleted file mode 100644 index 2efa431..0000000 --- a/patch/chrono-0.4.42/src/offset/local/win_bindings.txt +++ /dev/null @@ -1,7 +0,0 @@ ---out src/offset/local/win_bindings.rs ---flat --sys --no-comment ---filter - GetTimeZoneInformationForYear - SystemTimeToFileTime - SystemTimeToTzSpecificLocalTime - TzSpecificLocalTimeToSystemTime diff --git a/patch/chrono-0.4.42/CITATION.cff b/patch/chrono-0.4.43/CITATION.cff similarity index 94% rename from patch/chrono-0.4.42/CITATION.cff rename to patch/chrono-0.4.43/CITATION.cff index 19ae4f2..b1d4b97 100644 --- a/patch/chrono-0.4.42/CITATION.cff +++ b/patch/chrono-0.4.43/CITATION.cff @@ -3,8 +3,8 @@ cff-version: 1.2.0 message: Please cite this crate using these information. # Version information. -date-released: 2025-02-26 -version: 0.4.41 +date-released: 2026-01-09 +version: 0.4.43 # Project information. abstract: Date and time library for Rust diff --git a/patch/chrono-0.4.42/Cargo.toml b/patch/chrono-0.4.43/Cargo.toml similarity index 88% rename from patch/chrono-0.4.42/Cargo.toml rename to patch/chrono-0.4.43/Cargo.toml index 507b61a..27cbd2b 100644 --- a/patch/chrono-0.4.42/Cargo.toml +++ b/patch/chrono-0.4.43/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "chrono" -version = "0.4.42" +version = "0.4.43" description = "Date and time library for Rust" homepage = "https://github.com/chronotope/chrono" documentation = "https://docs.rs/chrono/" @@ -20,6 +20,7 @@ name = "chrono" # Don't forget to adjust `ALL_NON_EXCLUSIVE_FEATURES` in CI scripts when adding a feature or an optional dependency. default = ["clock", "std", "oldtime", "wasmbind"] alloc = [] +defmt = ["dep:defmt", "pure-rust-locales?/defmt"] libc = [] winapi = ["windows-link"] std = ["alloc"] @@ -41,9 +42,10 @@ __internal_bench = [] [dependencies] num-traits = { version = "0.2", default-features = false } serde = { version = "1.0.99", default-features = false, optional = true } -pure-rust-locales = { version = "0.8", optional = true } -rkyv = { version = "0.8", optional = true, default-features = false } +pure-rust-locales = { version = "0.8.2", optional = true } +rkyv = { version = "0.8.14", optional = true, default-features = false } arbitrary = { version = "1.0.0", features = ["derive"], optional = true } +defmt = { version = "1.0.1", optional = true } [target.'cfg(all(target_arch = "wasm32", not(any(target_os = "emscripten", target_os = "wasi"))))'.dependencies] wasm-bindgen = { version = "0.2", optional = true } @@ -52,9 +54,6 @@ js-sys = { version = "0.3", optional = true } # contains FFI bindings for [target.'cfg(windows)'.dependencies] windows-link = { version = "0.2", optional = true } -[target.'cfg(windows)'.dev-dependencies] -windows-bindgen = { version = "0.63" } # MSRV is 1.74 - [target.'cfg(unix)'.dependencies] iana-time-zone = { version = "0.1.45", optional = true, features = ["fallback"] } @@ -63,6 +62,7 @@ serde_json = { version = "1" } serde_derive = { version = "1", default-features = false } similar-asserts = { version = "1.6.1" } bincode = { version = "1.3.0" } +windows-bindgen = { version = "0.66" } # MSRV is 1.74 [target.'cfg(all(target_arch = "wasm32", not(any(target_os = "emscripten", target_os = "wasi"))))'.dev-dependencies] wasm-bindgen-test = "0.3" diff --git a/patch/chrono-0.4.42/LICENSE.txt b/patch/chrono-0.4.43/LICENSE.txt similarity index 99% rename from patch/chrono-0.4.42/LICENSE.txt rename to patch/chrono-0.4.43/LICENSE.txt index 0f9e3cf..480fdf7 100644 --- a/patch/chrono-0.4.42/LICENSE.txt +++ b/patch/chrono-0.4.43/LICENSE.txt @@ -1,5 +1,5 @@ Rust-chrono is dual-licensed under The MIT License [1] and -Apache 2.0 License [2]. Copyright (c) 2014--2025, Kang Seonghoon and +Apache 2.0 License [2]. Copyright (c) 2014--2026, Kang Seonghoon and contributors. Nota Bene: This is same as the Rust Project's own license. diff --git a/patch/chrono-0.4.42/README.md b/patch/chrono-0.4.43/README.md similarity index 100% rename from patch/chrono-0.4.42/README.md rename to patch/chrono-0.4.43/README.md diff --git a/patch/chrono-0.4.42/src/date.rs b/patch/chrono-0.4.43/src/date.rs similarity index 98% rename from patch/chrono-0.4.42/src/date.rs rename to patch/chrono-0.4.43/src/date.rs index a66882c..59cef0e 100644 --- a/patch/chrono-0.4.42/src/date.rs +++ b/patch/chrono-0.4.43/src/date.rs @@ -551,6 +551,16 @@ where } } +#[cfg(feature = "defmt")] +impl defmt::Format for Date +where + Tz::Offset: defmt::Format, +{ + fn format(&self, fmt: defmt::Formatter) { + defmt::write!(fmt, "{}{}", self.naive_local(), self.offset); + } +} + // Note that implementation of Arbitrary cannot be automatically derived for Date, due to // the nontrivial bound ::Offset: Arbitrary. #[cfg(all(feature = "arbitrary", feature = "std"))] diff --git a/patch/chrono-0.4.42/src/datetime/mod.rs b/patch/chrono-0.4.43/src/datetime/mod.rs similarity index 99% rename from patch/chrono-0.4.42/src/datetime/mod.rs rename to patch/chrono-0.4.43/src/datetime/mod.rs index 023bede..d416f65 100644 --- a/patch/chrono-0.4.42/src/datetime/mod.rs +++ b/patch/chrono-0.4.43/src/datetime/mod.rs @@ -21,8 +21,8 @@ use crate::format::Locale; #[cfg(feature = "alloc")] use crate::format::{DelayedFormat, SecondsFormat, write_rfc2822, write_rfc3339}; use crate::format::{ - Fixed, Item, ParseError, ParseResult, Parsed, StrftimeItems, TOO_LONG, parse, - parse_and_remainder, parse_rfc3339, + Fixed, Item, ParseError, ParseResult, Parsed, StrftimeItems, parse, parse_and_remainder, + parse_rfc3339, }; use crate::naive::{Days, IsoWeek, NaiveDate, NaiveDateTime, NaiveTime}; #[cfg(feature = "clock")] @@ -414,7 +414,7 @@ impl DateTime { } /// Fix the offset from UTC to its current value, dropping the associated timezone information. - /// This it useful for converting a generic `DateTime` to `DateTime`. + /// This is useful for converting a generic `DateTime` to `DateTime`. #[inline] #[must_use] pub fn fixed_offset(&self) -> DateTime { @@ -1068,12 +1068,7 @@ impl DateTime { /// also simultaneously valid RFC 3339 values, but not all RFC 3339 values are valid ISO 8601 /// values (or the other way around). pub fn parse_from_rfc3339(s: &str) -> ParseResult> { - let mut parsed = Parsed::new(); - let (s, _) = parse_rfc3339(&mut parsed, s)?; - if !s.is_empty() { - return Err(TOO_LONG); - } - parsed.to_datetime() + parse_rfc3339(s) } /// Parses a string from a user-specified format into a `DateTime` value. @@ -1523,7 +1518,7 @@ impl hash::Hash for DateTime { /// Add `TimeDelta` to `DateTime`. /// /// As a part of Chrono's [leap second handling], the addition assumes that **there is no leap -/// second ever**, except when the `NaiveDateTime` itself represents a leap second in which case +/// second ever**, except when the `NaiveDateTime` itself represents a leap second in which case /// the assumption becomes that **there is exactly a single leap second ever**. /// /// # Panics @@ -1542,7 +1537,7 @@ impl Add for DateTime { /// Add `std::time::Duration` to `DateTime`. /// /// As a part of Chrono's [leap second handling], the addition assumes that **there is no leap -/// second ever**, except when the `NaiveDateTime` itself represents a leap second in which case +/// second ever**, except when the `NaiveDateTime` itself represents a leap second in which case /// the assumption becomes that **there is exactly a single leap second ever**. /// /// # Panics @@ -1563,7 +1558,7 @@ impl Add for DateTime { /// Add-assign `chrono::Duration` to `DateTime`. /// /// As a part of Chrono's [leap second handling], the addition assumes that **there is no leap -/// second ever**, except when the `NaiveDateTime` itself represents a leap second in which case +/// second ever**, except when the `NaiveDateTime` itself represents a leap second in which case /// the assumption becomes that **there is exactly a single leap second ever**. /// /// # Panics @@ -1583,7 +1578,7 @@ impl AddAssign for DateTime { /// Add-assign `std::time::Duration` to `DateTime`. /// /// As a part of Chrono's [leap second handling], the addition assumes that **there is no leap -/// second ever**, except when the `NaiveDateTime` itself represents a leap second in which case +/// second ever**, except when the `NaiveDateTime` itself represents a leap second in which case /// the assumption becomes that **there is exactly a single leap second ever**. /// /// # Panics @@ -1683,7 +1678,7 @@ impl Sub for DateTime { /// This is the same as the addition with a negated `TimeDelta`. /// /// As a part of Chrono's [leap second handling], the addition assumes that **there is no leap -/// second ever**, except when the `DateTime` itself represents a leap second in which case +/// second ever**, except when the `DateTime` itself represents a leap second in which case /// the assumption becomes that **there is exactly a single leap second ever**. /// /// # Panics @@ -1703,7 +1698,7 @@ impl SubAssign for DateTime { /// Subtract-assign `std::time::Duration` from `DateTime`. /// /// As a part of Chrono's [leap second handling], the addition assumes that **there is no leap -/// second ever**, except when the `DateTime` itself represents a leap second in which case +/// second ever**, except when the `DateTime` itself represents a leap second in which case /// the assumption becomes that **there is exactly a single leap second ever**. /// /// # Panics @@ -1817,6 +1812,16 @@ impl fmt::Debug for DateTime { } } +#[cfg(feature = "defmt")] +impl defmt::Format for DateTime +where + Tz::Offset: defmt::Format, +{ + fn format(&self, fmt: defmt::Formatter) { + defmt::write!(fmt, "{}{}", self.overflowing_naive_local(), self.offset); + } +} + // `fmt::Debug` is hand implemented for the `rkyv::Archive` variant of `DateTime` because // deriving a trait recursively does not propagate trait defined associated types with their own // constraints: diff --git a/patch/chrono-0.4.42/src/datetime/serde.rs b/patch/chrono-0.4.43/src/datetime/serde.rs similarity index 99% rename from patch/chrono-0.4.42/src/datetime/serde.rs rename to patch/chrono-0.4.43/src/datetime/serde.rs index b08e4e1..694fe53 100644 --- a/patch/chrono-0.4.42/src/datetime/serde.rs +++ b/patch/chrono-0.4.43/src/datetime/serde.rs @@ -50,9 +50,7 @@ impl ser::Serialize for DateTime { } } -#[allow(missing_docs)] -#[allow(missing_debug_implementations)] -pub struct DateTimeVisitor; +struct DateTimeVisitor; impl de::Visitor<'_> for DateTimeVisitor { type Value = DateTime; @@ -76,6 +74,7 @@ impl de::Visitor<'_> for DateTimeVisitor { /// /// See [the `serde` module](crate::serde) for alternate deserialization formats. impl<'de> de::Deserialize<'de> for DateTime { + #[inline] fn deserialize(deserializer: D) -> Result where D: de::Deserializer<'de>, diff --git a/patch/chrono-0.4.42/src/datetime/tests.rs b/patch/chrono-0.4.43/src/datetime/tests.rs similarity index 100% rename from patch/chrono-0.4.42/src/datetime/tests.rs rename to patch/chrono-0.4.43/src/datetime/tests.rs diff --git a/patch/chrono-0.4.42/src/format/formatting.rs b/patch/chrono-0.4.43/src/format/formatting.rs similarity index 99% rename from patch/chrono-0.4.42/src/format/formatting.rs rename to patch/chrono-0.4.43/src/format/formatting.rs index 79ed694..a298eda 100644 --- a/patch/chrono-0.4.42/src/format/formatting.rs +++ b/patch/chrono-0.4.43/src/format/formatting.rs @@ -474,6 +474,7 @@ impl OffsetFormat { /// /// See the `TimeZone::to_rfc3339_opts` function for usage. #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +#[cfg_attr(feature = "defmt", derive(defmt::Format))] #[allow(clippy::manual_non_exhaustive)] pub enum SecondsFormat { /// Format whole seconds only, with no decimal point nor subseconds. diff --git a/patch/chrono-0.4.42/src/format/locales.rs b/patch/chrono-0.4.43/src/format/locales.rs similarity index 100% rename from patch/chrono-0.4.42/src/format/locales.rs rename to patch/chrono-0.4.43/src/format/locales.rs diff --git a/patch/chrono-0.4.42/src/format/mod.rs b/patch/chrono-0.4.43/src/format/mod.rs similarity index 93% rename from patch/chrono-0.4.42/src/format/mod.rs rename to patch/chrono-0.4.43/src/format/mod.rs index f90314a..dab4705 100644 --- a/patch/chrono-0.4.42/src/format/mod.rs +++ b/patch/chrono-0.4.43/src/format/mod.rs @@ -78,6 +78,7 @@ enum Void {} /// Padding characters for numeric items. #[derive(Copy, Clone, PartialEq, Eq, Debug, Hash)] +#[cfg_attr(feature = "defmt", derive(defmt::Format))] pub enum Pad { /// No padding. None, @@ -102,6 +103,7 @@ pub enum Pad { /// parsed with the same formatting items. #[non_exhaustive] #[derive(Clone, PartialEq, Eq, Debug, Hash)] +#[cfg_attr(feature = "defmt", derive(defmt::Format))] pub enum Numeric { /// Full Gregorian year (FW=4, PW=∞). /// May accept years before 1 BCE or after 9999 CE, given an initial sign (+/-). @@ -170,12 +172,20 @@ impl fmt::Debug for InternalNumeric { } } +#[cfg(feature = "defmt")] +impl defmt::Format for InternalNumeric { + fn format(&self, f: defmt::Formatter) { + defmt::write!(f, "") + } +} + /// Fixed-format item types. /// /// They have their own rules of formatting and parsing. /// Otherwise noted, they print in the specified cases but parse case-insensitively. #[non_exhaustive] #[derive(Clone, PartialEq, Eq, Debug, Hash)] +#[cfg_attr(feature = "defmt", derive(defmt::Format))] pub enum Fixed { /// Abbreviated month names. /// @@ -260,11 +270,13 @@ pub enum Fixed { /// An opaque type representing fixed-format item types for internal uses only. #[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "defmt", derive(defmt::Format))] pub struct InternalFixed { val: InternalInternal, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "defmt", derive(defmt::Format))] enum InternalInternal { /// Same as [`TimezoneOffsetColonZ`](#variant.TimezoneOffsetColonZ), but /// allows missing minutes (per [ISO 8601][iso8601]). @@ -285,6 +297,7 @@ enum InternalInternal { /// Type for specifying the format of UTC offsets. #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "defmt", derive(defmt::Format))] pub struct OffsetFormat { /// See `OffsetPrecision`. pub precision: OffsetPrecision, @@ -298,6 +311,7 @@ pub struct OffsetFormat { /// The precision of an offset from UTC formatting item. #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "defmt", derive(defmt::Format))] pub enum OffsetPrecision { /// Format offset from UTC as only hours. Not recommended, it is not uncommon for timezones to /// have an offset of 30 minutes, 15 minutes, etc. @@ -319,6 +333,7 @@ pub enum OffsetPrecision { /// The separator between hours and minutes in an offset. #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "defmt", derive(defmt::Format))] pub enum Colons { /// No separator None, @@ -350,6 +365,23 @@ pub enum Item<'a> { Error, } +#[cfg(feature = "defmt")] +impl<'a> defmt::Format for Item<'a> { + fn format(&self, f: defmt::Formatter) { + match self { + Item::Literal(v) => defmt::write!(f, "Literal {{ {} }}", v), + #[cfg(feature = "alloc")] + Item::OwnedLiteral(_) => {} + Item::Space(v) => defmt::write!(f, "Space {{ {} }}", v), + #[cfg(feature = "alloc")] + Item::OwnedSpace(_) => {} + Item::Numeric(u, v) => defmt::write!(f, "Numeric {{ {}, {} }}", u, v), + Item::Fixed(v) => defmt::write!(f, "Fixed {{ {} }}", v), + Item::Error => defmt::write!(f, "Error"), + } + } +} + const fn num(numeric: Numeric) -> Item<'static> { Item::Numeric(numeric, Pad::None) } @@ -388,6 +420,7 @@ impl Item<'_> { /// An error from the `parse` function. #[derive(Debug, Clone, PartialEq, Eq, Copy, Hash)] +#[cfg_attr(feature = "defmt", derive(defmt::Format))] pub struct ParseError(ParseErrorKind); impl ParseError { @@ -400,6 +433,7 @@ impl ParseError { /// The category of parse error #[allow(clippy::manual_non_exhaustive)] #[derive(Debug, Clone, PartialEq, Eq, Copy, Hash)] +#[cfg_attr(feature = "defmt", derive(defmt::Format))] pub enum ParseErrorKind { /// Given field is out of permitted range. OutOfRange, diff --git a/patch/chrono-0.4.42/src/format/parse.rs b/patch/chrono-0.4.43/src/format/parse.rs similarity index 95% rename from patch/chrono-0.4.42/src/format/parse.rs rename to patch/chrono-0.4.43/src/format/parse.rs index 2e3b298..18857a2 100644 --- a/patch/chrono-0.4.42/src/format/parse.rs +++ b/patch/chrono-0.4.43/src/format/parse.rs @@ -11,7 +11,7 @@ use super::scan; use super::{BAD_FORMAT, INVALID, OUT_OF_RANGE, TOO_LONG, TOO_SHORT}; use super::{Fixed, InternalFixed, InternalInternal, Item, Numeric, Pad, Parsed}; use super::{ParseError, ParseResult}; -use crate::{DateTime, FixedOffset, Weekday}; +use crate::{DateTime, FixedOffset, MappedLocalTime, NaiveDate, NaiveTime, Weekday}; fn set_weekday_with_num_days_from_sunday(p: &mut Parsed, v: i64) -> ParseResult<()> { p.set_weekday(match v { @@ -151,7 +151,7 @@ fn parse_rfc2822<'a>(parsed: &mut Parsed, mut s: &'a str) -> ParseResult<(&'a st Ok((s, ())) } -pub(crate) fn parse_rfc3339<'a>(parsed: &mut Parsed, mut s: &'a str) -> ParseResult<(&'a str, ())> { +pub(crate) fn parse_rfc3339(mut s: &str) -> ParseResult> { macro_rules! try_consume { ($e:expr) => {{ let (s_, v) = $e?; @@ -189,40 +189,81 @@ pub(crate) fn parse_rfc3339<'a>(parsed: &mut Parsed, mut s: &'a str) -> ParseRes // // - For readability a full-date and a full-time may be separated by a space character. - parsed.set_year(try_consume!(scan::number(s, 4, 4)))?; - s = scan::char(s, b'-')?; - parsed.set_month(try_consume!(scan::number(s, 2, 2)))?; - s = scan::char(s, b'-')?; - parsed.set_day(try_consume!(scan::number(s, 2, 2)))?; + let bytes = s.as_bytes(); + if bytes.len() < 19 { + return Err(TOO_SHORT); + } - s = match s.as_bytes().first() { - Some(&b't' | &b'T' | &b' ') => &s[1..], - Some(_) => return Err(INVALID), - None => return Err(TOO_SHORT), + let fixed = <&[u8; 19]>::try_from(&bytes[..19]).unwrap(); // we just checked the length + let year = digit(fixed, 0)? as u16 * 1000 + + digit(fixed, 1)? as u16 * 100 + + digit(fixed, 2)? as u16 * 10 + + digit(fixed, 3)? as u16; + if bytes.get(4) != Some(&b'-') { + return Err(INVALID); + } + + let month = digit(fixed, 5)? * 10 + digit(fixed, 6)?; + if bytes.get(7) != Some(&b'-') { + return Err(INVALID); + } + + let day = digit(fixed, 8)? * 10 + digit(fixed, 9)?; + let date = + NaiveDate::from_ymd_opt(year as i32, month as u32, day as u32).ok_or(OUT_OF_RANGE)?; + + if !matches!(bytes.get(10), Some(&b't' | &b'T' | &b' ')) { + return Err(INVALID); + } + + let hour = digit(fixed, 11)? * 10 + digit(fixed, 12)?; + if bytes.get(13) != Some(&b':') { + return Err(INVALID); + } + + let min = digit(fixed, 14)? * 10 + digit(fixed, 15)?; + if bytes.get(16) != Some(&b':') { + return Err(INVALID); + } + + let sec = digit(fixed, 17)? * 10 + digit(fixed, 18)?; + let (sec, extra_nanos) = match sec { + 60 => (59, 1_000_000_000), // rfc3339 allows leap seconds + _ => (sec, 0), }; - parsed.set_hour(try_consume!(scan::number(s, 2, 2)))?; - s = scan::char(s, b':')?; - parsed.set_minute(try_consume!(scan::number(s, 2, 2)))?; - s = scan::char(s, b':')?; - parsed.set_second(try_consume!(scan::number(s, 2, 2)))?; - if s.starts_with('.') { - let nanosecond = try_consume!(scan::nanosecond(&s[1..])); - parsed.set_nanosecond(nanosecond)?; - } + let nano = if bytes.get(19) == Some(&b'.') { + let nanosecond = try_consume!(scan::nanosecond(&s[20..])); + extra_nanos + nanosecond + } else { + s = &s[19..]; + extra_nanos + }; + + let time = NaiveTime::from_hms_nano_opt(hour as u32, min as u32, sec as u32, nano) + .ok_or(OUT_OF_RANGE)?; - let offset = try_consume!(scan::timezone_offset(s, |s| scan::char(s, b':'), true, false, true)); - // This range check is similar to the one in `FixedOffset::east_opt`, so it would be redundant. - // But it is possible to read the offset directly from `Parsed`. We want to only successfully - // populate `Parsed` if the input is fully valid RFC 3339. // Max for the hours field is `23`, and for the minutes field `59`. - const MAX_RFC3339_OFFSET: i32 = (23 * 60 + 59) * 60; - if !(-MAX_RFC3339_OFFSET..=MAX_RFC3339_OFFSET).contains(&offset) { - return Err(OUT_OF_RANGE); + let offset = try_consume!(scan::timezone_offset(s, |s| scan::char(s, b':'), true, false, true)); + if !s.is_empty() { + return Err(TOO_LONG); } - parsed.set_offset(i64::from(offset))?; - Ok((s, ())) + let tz = FixedOffset::east_opt(offset).ok_or(OUT_OF_RANGE)?; + Ok(match date.and_time(time).and_local_timezone(tz) { + MappedLocalTime::Single(dt) => dt, + // `FixedOffset::with_ymd_and_hms` doesn't return `MappedLocalTime::Ambiguous` + // and returns `MappedLocalTime::None` on invalid data + MappedLocalTime::Ambiguous(_, _) | MappedLocalTime::None => unreachable!(), + }) +} + +#[inline] +fn digit(bytes: &[u8; 19], index: usize) -> ParseResult { + match bytes[index].is_ascii_digit() { + true => Ok(bytes[index] - b'0'), + false => Err(INVALID), + } } /// Tries to parse given string into `parsed` with given formatting items. @@ -420,7 +461,7 @@ where &Nanosecond => { if s.starts_with('.') { let nano = try_consume!(scan::nanosecond(&s[1..])); - parsed.set_nanosecond(nano)?; + parsed.set_nanosecond(nano as i64)?; } } @@ -1830,29 +1871,34 @@ mod tests { "2015-01-20T17:35:20.000000000452−08:00", Ok(ymd_hmsn(2015, 1, 20, 17, 35, 20, 0, -8)), ), // too small with MINUS SIGN (U+2212) + ("2023-11-05T01:30:00-04:00", Ok(ymd_hmsn(2023, 11, 5, 1, 30, 0, 0, -4))), // ambiguous timestamp ("2015-01-20 17:35:20-08:00", Ok(ymd_hmsn(2015, 1, 20, 17, 35, 20, 0, -8))), // without 'T' - ("2015/01/20T17:35:20.001-08:00", Err(INVALID)), // wrong separator char YMD - ("2015-01-20T17-35-20.001-08:00", Err(INVALID)), // wrong separator char HMS - ("-01-20T17:35:20-08:00", Err(INVALID)), // missing year - ("99-01-20T17:35:20-08:00", Err(INVALID)), // bad year format - ("99999-01-20T17:35:20-08:00", Err(INVALID)), // bad year value - ("-2000-01-20T17:35:20-08:00", Err(INVALID)), // bad year value + ("2015-01-20_17:35:20-08:00", Err(INVALID)), // wrong date time separator + ("2015/01/20T17:35:20.001-08:00", Err(INVALID)), // wrong separator char YM + ("2015-01/20T17:35:20.001-08:00", Err(INVALID)), // wrong separator char MD + ("2015-01-20T17-35-20.001-08:00", Err(INVALID)), // wrong separator char HM + ("2015-01-20T17-35:20.001-08:00", Err(INVALID)), // wrong separator char MS + ("-01-20T17:35:20-08:00", Err(INVALID)), // missing year + ("99-01-20T17:35:20-08:00", Err(INVALID)), // bad year format + ("99999-01-20T17:35:20-08:00", Err(INVALID)), // bad year value + ("-2000-01-20T17:35:20-08:00", Err(INVALID)), // bad year value + ("2015-00-30T17:35:20-08:00", Err(OUT_OF_RANGE)), // bad month value ("2015-02-30T17:35:20-08:00", Err(OUT_OF_RANGE)), // bad day of month value ("2015-01-20T25:35:20-08:00", Err(OUT_OF_RANGE)), // bad hour value ("2015-01-20T17:65:20-08:00", Err(OUT_OF_RANGE)), // bad minute value ("2015-01-20T17:35:90-08:00", Err(OUT_OF_RANGE)), // bad second value ("2015-01-20T17:35:20-24:00", Err(OUT_OF_RANGE)), // bad offset value - ("15-01-20T17:35:20-08:00", Err(INVALID)), // bad year format - ("15-01-20T17:35:20-08:00:00", Err(INVALID)), // bad year format, bad offset format - ("2015-01-20T17:35:2008:00", Err(INVALID)), // missing offset sign - ("2015-01-20T17:35:20 08:00", Err(INVALID)), // missing offset sign - ("2015-01-20T17:35:20Zulu", Err(TOO_LONG)), // bad offset format - ("2015-01-20T17:35:20 Zulu", Err(INVALID)), // bad offset format - ("2015-01-20T17:35:20GMT", Err(INVALID)), // bad offset format - ("2015-01-20T17:35:20 GMT", Err(INVALID)), // bad offset format - ("2015-01-20T17:35:20+GMT", Err(INVALID)), // bad offset format - ("2015-01-20T17:35:20++08:00", Err(INVALID)), // bad offset format - ("2015-01-20T17:35:20--08:00", Err(INVALID)), // bad offset format + ("15-01-20T17:35:20-08:00", Err(INVALID)), // bad year format + ("15-01-20T17:35:20-08:00:00", Err(INVALID)), // bad year format, bad offset format + ("2015-01-20T17:35:2008:00", Err(INVALID)), // missing offset sign + ("2015-01-20T17:35:20 08:00", Err(INVALID)), // missing offset sign + ("2015-01-20T17:35:20Zulu", Err(TOO_LONG)), // bad offset format + ("2015-01-20T17:35:20 Zulu", Err(INVALID)), // bad offset format + ("2015-01-20T17:35:20GMT", Err(INVALID)), // bad offset format + ("2015-01-20T17:35:20 GMT", Err(INVALID)), // bad offset format + ("2015-01-20T17:35:20+GMT", Err(INVALID)), // bad offset format + ("2015-01-20T17:35:20++08:00", Err(INVALID)), // bad offset format + ("2015-01-20T17:35:20--08:00", Err(INVALID)), // bad offset format ("2015-01-20T17:35:20−−08:00", Err(INVALID)), // bad offset format with MINUS SIGN (U+2212) ("2015-01-20T17:35:20±08:00", Err(INVALID)), // bad offset sign ("2015-01-20T17:35:20-08-00", Err(INVALID)), // bad offset separator diff --git a/patch/chrono-0.4.42/src/format/parsed.rs b/patch/chrono-0.4.43/src/format/parsed.rs similarity index 99% rename from patch/chrono-0.4.42/src/format/parsed.rs rename to patch/chrono-0.4.43/src/format/parsed.rs index 8524daa..3746ba4 100644 --- a/patch/chrono-0.4.42/src/format/parsed.rs +++ b/patch/chrono-0.4.43/src/format/parsed.rs @@ -126,6 +126,7 @@ use crate::{DateTime, Datelike, TimeDelta, Timelike, Weekday}; /// ``` #[allow(clippy::manual_non_exhaustive)] #[derive(Clone, PartialEq, Eq, Debug, Default, Hash)] +#[cfg_attr(feature = "defmt", derive(defmt::Format))] pub struct Parsed { #[doc(hidden)] pub year: Option, diff --git a/patch/chrono-0.4.42/src/format/scan.rs b/patch/chrono-0.4.43/src/format/scan.rs similarity index 98% rename from patch/chrono-0.4.42/src/format/scan.rs rename to patch/chrono-0.4.43/src/format/scan.rs index 49a4f12..30dbc67 100644 --- a/patch/chrono-0.4.42/src/format/scan.rs +++ b/patch/chrono-0.4.43/src/format/scan.rs @@ -47,14 +47,15 @@ pub(super) fn number(s: &str, min: usize, max: usize) -> ParseResult<(&str, i64) /// Tries to consume at least one digits as a fractional second. /// Returns the number of whole nanoseconds (0--999,999,999). -pub(super) fn nanosecond(s: &str) -> ParseResult<(&str, i64)> { +pub(super) fn nanosecond(s: &str) -> ParseResult<(&str, u32)> { // record the number of digits consumed for later scaling. let origlen = s.len(); let (s, v) = number(s, 1, 9)?; + let v = u32::try_from(v).expect("999,999,999 should fit u32"); let consumed = origlen - s.len(); // scale the number accordingly. - static SCALE: [i64; 10] = + const SCALE: [u32; 10] = [0, 100_000_000, 10_000_000, 1_000_000, 100_000, 10_000, 1_000, 100, 10, 1]; let v = v.checked_mul(SCALE[consumed]).ok_or(OUT_OF_RANGE)?; diff --git a/patch/chrono-0.4.42/src/format/strftime.rs b/patch/chrono-0.4.43/src/format/strftime.rs similarity index 99% rename from patch/chrono-0.4.42/src/format/strftime.rs rename to patch/chrono-0.4.43/src/format/strftime.rs index 5dc1180..d93ef81 100644 --- a/patch/chrono-0.4.42/src/format/strftime.rs +++ b/patch/chrono-0.4.43/src/format/strftime.rs @@ -189,6 +189,7 @@ use alloc::vec::Vec; /// [`DateTime`]: crate::DateTime /// [`format::parse()`]: crate::format::parse() #[derive(Clone, Debug)] +#[cfg_attr(feature = "defmt", derive(defmt::Format))] pub struct StrftimeItems<'a> { /// Remaining portion of the string. remainder: &'a str, diff --git a/patch/chrono-0.4.42/src/lib.rs b/patch/chrono-0.4.43/src/lib.rs similarity index 99% rename from patch/chrono-0.4.42/src/lib.rs rename to patch/chrono-0.4.43/src/lib.rs index 32a1b82..0e942e0 100644 --- a/patch/chrono-0.4.42/src/lib.rs +++ b/patch/chrono-0.4.43/src/lib.rs @@ -501,12 +501,10 @@ //! [chrono#1095]: https://github.com/chronotope/chrono/pull/1095 #![doc(html_root_url = "https://docs.rs/chrono/latest/", test(attr(deny(warnings))))] -#![deny(missing_docs)] -#![deny(missing_debug_implementations)] #![warn(unreachable_pub)] -#![deny(clippy::tests_outside_test_module)] +#![warn(clippy::tests_outside_test_module)] #![cfg_attr(not(any(feature = "std", test)), no_std)] -#![cfg_attr(docsrs, feature(doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #[cfg(feature = "alloc")] extern crate alloc; @@ -687,6 +685,13 @@ impl fmt::Debug for OutOfRange { } } +#[cfg(feature = "defmt")] +impl defmt::Format for OutOfRange { + fn format(&self, fmt: defmt::Formatter) { + defmt::write!(fmt, "out of range"); + } +} + #[cfg(feature = "std")] impl std::error::Error for OutOfRange {} diff --git a/patch/chrono-0.4.42/src/month.rs b/patch/chrono-0.4.43/src/month.rs similarity index 97% rename from patch/chrono-0.4.42/src/month.rs rename to patch/chrono-0.4.43/src/month.rs index 3d12115..944e300 100644 --- a/patch/chrono-0.4.42/src/month.rs +++ b/patch/chrono-0.4.43/src/month.rs @@ -34,10 +34,11 @@ use crate::naive::NaiveDate; any(feature = "rkyv", feature = "rkyv-16", feature = "rkyv-32", feature = "rkyv-64"), derive(Archive, Deserialize, Serialize), rkyv(compare(PartialEq, PartialOrd)), - rkyv(attr(derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash))) + rkyv(derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)) )] #[cfg_attr(feature = "rkyv-validation", archive(check_bytes))] #[cfg_attr(all(feature = "arbitrary", feature = "std"), derive(arbitrary::Arbitrary))] +#[cfg_attr(feature = "defmt", derive(defmt::Format))] pub enum Month { /// January January = 0, @@ -248,6 +249,7 @@ impl num_traits::FromPrimitive for Month { /// A duration in calendar months #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] #[cfg_attr(all(feature = "arbitrary", feature = "std"), derive(arbitrary::Arbitrary))] +#[cfg_attr(feature = "defmt", derive(defmt::Format))] pub struct Months(pub(crate) u32); impl Months { @@ -287,6 +289,13 @@ impl fmt::Debug for ParseMonthError { } } +#[cfg(feature = "defmt")] +impl defmt::Format for ParseMonthError { + fn format(&self, fmt: defmt::Formatter) { + defmt::write!(fmt, "ParseMonthError {{ .. }}") + } +} + #[cfg(feature = "serde")] mod month_serde { use super::Month; diff --git a/patch/chrono-0.4.42/src/naive/date/mod.rs b/patch/chrono-0.4.43/src/naive/date/mod.rs similarity index 98% rename from patch/chrono-0.4.42/src/naive/date/mod.rs rename to patch/chrono-0.4.43/src/naive/date/mod.rs index a0d410d..d4090d4 100644 --- a/patch/chrono-0.4.42/src/naive/date/mod.rs +++ b/patch/chrono-0.4.43/src/naive/date/mod.rs @@ -96,7 +96,7 @@ mod tests; any(feature = "rkyv", feature = "rkyv-16", feature = "rkyv-32", feature = "rkyv-64"), derive(Archive, Deserialize, Serialize), rkyv(compare(PartialEq, PartialOrd)), - rkyv(attr(derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash))) + rkyv(derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)) )] #[cfg_attr(feature = "rkyv-validation", archive(check_bytes))] pub struct NaiveDate { @@ -1148,7 +1148,7 @@ impl NaiveDate { /// ); /// ``` #[must_use] - pub const fn signed_duration_since(self, rhs: NaiveDate) -> TimeDelta { + pub const fn signed_duration_since(self, rhs: Self) -> TimeDelta { let year1 = self.year(); let year2 = rhs.year(); let (year1_div_400, year1_mod_400) = div_mod_floor(year1, 400); @@ -1161,11 +1161,40 @@ impl NaiveDate { expect(TimeDelta::try_days(days), "always in range") } + /// Returns the absolute difference between two `NaiveDate`s measured as the number of days. + /// + /// This is always an integer, non-negative number, similar to `abs_diff` in `std`. + /// + /// # Example + /// + /// ``` + /// # use chrono::{Days, NaiveDate}; + /// # + /// let date1: NaiveDate = "2020-01-01".parse().unwrap(); + /// let date2: NaiveDate = "2020-01-31".parse().unwrap(); + /// assert_eq!(date2.abs_diff(date1), Days::new(30)); + /// assert_eq!(date1.abs_diff(date2), Days::new(30)); + /// ``` + pub const fn abs_diff(self, rhs: Self) -> Days { + Days::new(i32::abs_diff(self.num_days_from_ce(), rhs.num_days_from_ce()) as u64) + } + /// Returns the number of whole years from the given `base` until `self`. /// /// # Errors /// /// Returns `None` if `base > self`. + /// + /// # Example + /// + /// ``` + /// # use chrono::{NaiveDate}; + /// # + /// let base: NaiveDate = "2025-01-01".parse().unwrap(); + /// let date: NaiveDate = "2030-01-01".parse().unwrap(); + /// + /// assert_eq!(date.years_since(base), Some(5)) + /// ``` #[must_use] pub const fn years_since(&self, base: Self) -> Option { let mut years = self.year() - base.year(); @@ -2284,6 +2313,23 @@ impl fmt::Debug for NaiveDate { } } +#[cfg(feature = "defmt")] +impl defmt::Format for NaiveDate { + fn format(&self, fmt: defmt::Formatter) { + let year = self.year(); + let mdf = self.mdf(); + if (0..=9999).contains(&year) { + defmt::write!(fmt, "{:02}{:02}", year / 100, year % 100); + } else { + // ISO 8601 requires the explicit sign for out-of-range years + let sign = ['+', '-'][(year < 0) as usize]; + defmt::write!(fmt, "{}{:05}", sign, year.abs()); + } + + defmt::write!(fmt, "-{:02}-{:02}", mdf.month(), mdf.day()); + } +} + /// The `Display` output of the naive date `d` is the same as /// [`d.format("%Y-%m-%d")`](crate::format::strftime). /// diff --git a/patch/chrono-0.4.42/src/naive/date/tests.rs b/patch/chrono-0.4.43/src/naive/date/tests.rs similarity index 100% rename from patch/chrono-0.4.42/src/naive/date/tests.rs rename to patch/chrono-0.4.43/src/naive/date/tests.rs diff --git a/patch/chrono-0.4.42/src/naive/datetime/mod.rs b/patch/chrono-0.4.43/src/naive/datetime/mod.rs similarity index 99% rename from patch/chrono-0.4.42/src/naive/datetime/mod.rs rename to patch/chrono-0.4.43/src/naive/datetime/mod.rs index 3441a91..76c98f1 100644 --- a/patch/chrono-0.4.42/src/naive/datetime/mod.rs +++ b/patch/chrono-0.4.43/src/naive/datetime/mod.rs @@ -69,7 +69,7 @@ pub const MAX_DATETIME: NaiveDateTime = NaiveDateTime::MAX; any(feature = "rkyv", feature = "rkyv-16", feature = "rkyv-32", feature = "rkyv-64"), derive(Archive, Deserialize, Serialize), rkyv(compare(PartialEq, PartialOrd)), - rkyv(attr(derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash))) + rkyv(derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)) )] #[cfg_attr(feature = "rkyv-validation", archive(check_bytes))] #[cfg_attr(all(feature = "arbitrary", feature = "std"), derive(arbitrary::Arbitrary))] @@ -2057,6 +2057,13 @@ impl fmt::Debug for NaiveDateTime { } } +#[cfg(feature = "defmt")] +impl defmt::Format for NaiveDateTime { + fn format(&self, fmt: defmt::Formatter) { + defmt::write!(fmt, "{}T{}", self.date, self.time); + } +} + /// The `Display` output of the naive date and time `dt` is the same as /// [`dt.format("%Y-%m-%d %H:%M:%S%.f")`](crate::format::strftime). /// diff --git a/patch/chrono-0.4.42/src/naive/datetime/serde.rs b/patch/chrono-0.4.43/src/naive/datetime/serde.rs similarity index 100% rename from patch/chrono-0.4.42/src/naive/datetime/serde.rs rename to patch/chrono-0.4.43/src/naive/datetime/serde.rs diff --git a/patch/chrono-0.4.42/src/naive/datetime/tests.rs b/patch/chrono-0.4.43/src/naive/datetime/tests.rs similarity index 100% rename from patch/chrono-0.4.42/src/naive/datetime/tests.rs rename to patch/chrono-0.4.43/src/naive/datetime/tests.rs diff --git a/patch/chrono-0.4.42/src/naive/internals.rs b/patch/chrono-0.4.43/src/naive/internals.rs similarity index 100% rename from patch/chrono-0.4.42/src/naive/internals.rs rename to patch/chrono-0.4.43/src/naive/internals.rs diff --git a/patch/chrono-0.4.42/src/naive/isoweek.rs b/patch/chrono-0.4.43/src/naive/isoweek.rs similarity index 93% rename from patch/chrono-0.4.42/src/naive/isoweek.rs rename to patch/chrono-0.4.43/src/naive/isoweek.rs index efe0722..644f792 100644 --- a/patch/chrono-0.4.42/src/naive/isoweek.rs +++ b/patch/chrono-0.4.43/src/naive/isoweek.rs @@ -21,7 +21,7 @@ use rkyv::{Archive, Deserialize, Serialize}; any(feature = "rkyv", feature = "rkyv-16", feature = "rkyv-32", feature = "rkyv-64"), derive(Archive, Deserialize, Serialize), rkyv(compare(PartialEq, PartialOrd)), - rkyv(attr(derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash))) + rkyv(derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)) )] #[cfg_attr(feature = "rkyv-validation", archive(check_bytes))] pub struct IsoWeek { @@ -160,6 +160,21 @@ impl fmt::Debug for IsoWeek { } } +#[cfg(feature = "defmt")] +impl defmt::Format for IsoWeek { + fn format(&self, fmt: defmt::Formatter) { + let year = self.year(); + let week = self.week(); + if (0..=9999).contains(&year) { + defmt::write!(fmt, "{:04}-W{:02}", year, week) + } else { + // ISO 8601 requires the explicit sign for out-of-range years + let sign = ['+', '-'][(year < 0) as usize]; + defmt::write!(fmt, "{}{:05}-W{:02}", sign, year.abs(), week) + } + } +} + #[cfg(test)] mod tests { #[cfg(feature = "rkyv-validation")] diff --git a/patch/chrono-0.4.42/src/naive/mod.rs b/patch/chrono-0.4.43/src/naive/mod.rs similarity index 99% rename from patch/chrono-0.4.42/src/naive/mod.rs rename to patch/chrono-0.4.43/src/naive/mod.rs index efb9822..ff94584 100644 --- a/patch/chrono-0.4.42/src/naive/mod.rs +++ b/patch/chrono-0.4.43/src/naive/mod.rs @@ -31,6 +31,7 @@ pub use self::internals::YearFlags as __BenchYearFlags; /// A week represented by a [`NaiveDate`] and a [`Weekday`] which is the first /// day of the week. #[derive(Clone, Copy, Debug, Eq)] +#[cfg_attr(feature = "defmt", derive(defmt::Format))] pub struct NaiveWeek { date: NaiveDate, start: Weekday, @@ -226,6 +227,7 @@ impl Hash for NaiveWeek { /// difference applies only when dealing with `DateTime` data types and in other cases /// `TimeDelta::days(n)` and `Days::new(n)` are equivalent. #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] +#[cfg_attr(feature = "defmt", derive(defmt::Format))] pub struct Days(pub(crate) u64); impl Days { diff --git a/patch/chrono-0.4.42/src/naive/time/mod.rs b/patch/chrono-0.4.43/src/naive/time/mod.rs similarity index 98% rename from patch/chrono-0.4.42/src/naive/time/mod.rs rename to patch/chrono-0.4.43/src/naive/time/mod.rs index dbf1db9..2a54acb 100644 --- a/patch/chrono-0.4.42/src/naive/time/mod.rs +++ b/patch/chrono-0.4.43/src/naive/time/mod.rs @@ -214,7 +214,7 @@ mod tests; any(feature = "rkyv", feature = "rkyv-16", feature = "rkyv-32", feature = "rkyv-64"), derive(Archive, Deserialize, Serialize), rkyv(compare(PartialEq, PartialOrd)), - rkyv(attr(derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash))) + rkyv(derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)) )] #[cfg_attr(feature = "rkyv-validation", archive(check_bytes))] pub struct NaiveTime { @@ -1529,6 +1529,31 @@ impl fmt::Debug for NaiveTime { } } +#[cfg(feature = "defmt")] +impl defmt::Format for NaiveTime { + fn format(&self, fmt: defmt::Formatter) { + let (hour, min, sec) = self.hms(); + let (sec, nano) = if self.frac >= 1_000_000_000 { + (sec + 1, self.frac - 1_000_000_000) + } else { + (sec, self.frac) + }; + + let (hour, min, sec) = (hour as u8, min as u8, sec as u8); + defmt::write!(fmt, "{:02}:{:02}:{:02}", hour, min, sec); + + if nano == 0 { + return; + } else if nano % 1_000_000 == 0 { + defmt::write!(fmt, ".{:03}", nano / 1_000_000); + } else if nano % 1_000 == 0 { + defmt::write!(fmt, ".{:06}", nano / 1_000); + } else { + defmt::write!(fmt, ".{:09}", nano); + } + } +} + /// The `Display` output of the naive time `t` is the same as /// [`t.format("%H:%M:%S%.f")`](crate::format::strftime). /// diff --git a/patch/chrono-0.4.42/src/naive/time/serde.rs b/patch/chrono-0.4.43/src/naive/time/serde.rs similarity index 100% rename from patch/chrono-0.4.42/src/naive/time/serde.rs rename to patch/chrono-0.4.43/src/naive/time/serde.rs diff --git a/patch/chrono-0.4.42/src/naive/time/tests.rs b/patch/chrono-0.4.43/src/naive/time/tests.rs similarity index 100% rename from patch/chrono-0.4.42/src/naive/time/tests.rs rename to patch/chrono-0.4.43/src/naive/time/tests.rs diff --git a/patch/chrono-0.4.42/src/offset/fixed.rs b/patch/chrono-0.4.43/src/offset/fixed.rs similarity index 92% rename from patch/chrono-0.4.42/src/offset/fixed.rs rename to patch/chrono-0.4.43/src/offset/fixed.rs index 2c04537..f2b8fdf 100644 --- a/patch/chrono-0.4.42/src/offset/fixed.rs +++ b/patch/chrono-0.4.43/src/offset/fixed.rs @@ -24,7 +24,7 @@ use crate::naive::{NaiveDate, NaiveDateTime}; any(feature = "rkyv", feature = "rkyv-16", feature = "rkyv-32", feature = "rkyv-64"), derive(Archive, Deserialize, Serialize), rkyv(compare(PartialEq)), - rkyv(attr(derive(Clone, Copy, PartialEq, Eq, Hash, Debug))) + rkyv(derive(Clone, Copy, PartialEq, Eq, Hash, Debug)) )] #[cfg_attr(feature = "rkyv-validation", archive(check_bytes))] pub struct FixedOffset { @@ -174,6 +174,23 @@ impl fmt::Display for FixedOffset { } } +#[cfg(feature = "defmt")] +impl defmt::Format for FixedOffset { + fn format(&self, f: defmt::Formatter) { + let offset = self.local_minus_utc; + let (sign, offset) = if offset < 0 { ('-', -offset) } else { ('+', offset) }; + let sec = offset.rem_euclid(60); + let mins = offset.div_euclid(60); + let min = mins.rem_euclid(60); + let hour = mins.div_euclid(60); + if sec == 0 { + defmt::write!(f, "{}{:02}:{:02}", sign, hour, min) + } else { + defmt::write!(f, "{}{:02}:{:02}:{:02}", sign, hour, min, sec) + } + } +} + #[cfg(all(feature = "arbitrary", feature = "std"))] impl arbitrary::Arbitrary<'_> for FixedOffset { fn arbitrary(u: &mut arbitrary::Unstructured) -> arbitrary::Result { diff --git a/patch/chrono-0.4.42/src/offset/local/mod.rs b/patch/chrono-0.4.43/src/offset/local/mod.rs similarity index 99% rename from patch/chrono-0.4.42/src/offset/local/mod.rs rename to patch/chrono-0.4.43/src/offset/local/mod.rs index ee68571..5741d24 100644 --- a/patch/chrono-0.4.42/src/offset/local/mod.rs +++ b/patch/chrono-0.4.43/src/offset/local/mod.rs @@ -122,6 +122,7 @@ mod tz_info; )] #[cfg_attr(feature = "rkyv-validation", archive(check_bytes))] #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +#[cfg_attr(feature = "defmt", derive(defmt::Format))] pub struct Local; impl Local { diff --git a/patch/chrono-0.4.42/src/offset/local/tz_data.rs b/patch/chrono-0.4.43/src/offset/local/tz_data.rs similarity index 100% rename from patch/chrono-0.4.42/src/offset/local/tz_data.rs rename to patch/chrono-0.4.43/src/offset/local/tz_data.rs diff --git a/patch/chrono-0.4.42/src/offset/local/tz_info/mod.rs b/patch/chrono-0.4.43/src/offset/local/tz_info/mod.rs similarity index 100% rename from patch/chrono-0.4.42/src/offset/local/tz_info/mod.rs rename to patch/chrono-0.4.43/src/offset/local/tz_info/mod.rs diff --git a/patch/chrono-0.4.42/src/offset/local/tz_info/parser.rs b/patch/chrono-0.4.43/src/offset/local/tz_info/parser.rs similarity index 100% rename from patch/chrono-0.4.42/src/offset/local/tz_info/parser.rs rename to patch/chrono-0.4.43/src/offset/local/tz_info/parser.rs diff --git a/patch/chrono-0.4.42/src/offset/local/tz_info/rule.rs b/patch/chrono-0.4.43/src/offset/local/tz_info/rule.rs similarity index 100% rename from patch/chrono-0.4.42/src/offset/local/tz_info/rule.rs rename to patch/chrono-0.4.43/src/offset/local/tz_info/rule.rs diff --git a/patch/chrono-0.4.42/src/offset/local/tz_info/timezone.rs b/patch/chrono-0.4.43/src/offset/local/tz_info/timezone.rs similarity index 100% rename from patch/chrono-0.4.42/src/offset/local/tz_info/timezone.rs rename to patch/chrono-0.4.43/src/offset/local/tz_info/timezone.rs diff --git a/patch/chrono-0.4.42/src/offset/local/unix.rs b/patch/chrono-0.4.43/src/offset/local/unix.rs similarity index 100% rename from patch/chrono-0.4.42/src/offset/local/unix.rs rename to patch/chrono-0.4.43/src/offset/local/unix.rs diff --git a/patch/chrono-0.4.42/src/offset/local/win_bindings.rs b/patch/chrono-0.4.43/src/offset/local/win_bindings.rs similarity index 100% rename from patch/chrono-0.4.42/src/offset/local/win_bindings.rs rename to patch/chrono-0.4.43/src/offset/local/win_bindings.rs diff --git a/patch/chrono-0.4.42/src/offset/local/windows.rs b/patch/chrono-0.4.43/src/offset/local/windows.rs similarity index 100% rename from patch/chrono-0.4.42/src/offset/local/windows.rs rename to patch/chrono-0.4.43/src/offset/local/windows.rs diff --git a/patch/chrono-0.4.42/src/offset/mod.rs b/patch/chrono-0.4.43/src/offset/mod.rs similarity index 99% rename from patch/chrono-0.4.42/src/offset/mod.rs rename to patch/chrono-0.4.43/src/offset/mod.rs index 38380c1..295700b 100644 --- a/patch/chrono-0.4.42/src/offset/mod.rs +++ b/patch/chrono-0.4.43/src/offset/mod.rs @@ -76,7 +76,7 @@ pub use self::utc::Utc; /// The type of `T` is usually a [`DateTime`] but may also be only an offset. pub type MappedLocalTime = LocalResult; #[derive(Clone, PartialEq, Debug, Copy, Eq, Hash)] - +#[cfg_attr(feature = "defmt", derive(defmt::Format))] /// Old name of [`MappedLocalTime`]. See that type for more documentation. pub enum LocalResult { /// The local time maps to a single unique result. diff --git a/patch/chrono-0.4.42/src/offset/utc.rs b/patch/chrono-0.4.43/src/offset/utc.rs similarity index 95% rename from patch/chrono-0.4.42/src/offset/utc.rs rename to patch/chrono-0.4.43/src/offset/utc.rs index be2b520..92f6518 100644 --- a/patch/chrono-0.4.42/src/offset/utc.rs +++ b/patch/chrono-0.4.43/src/offset/utc.rs @@ -45,7 +45,7 @@ use crate::{Date, DateTime}; any(feature = "rkyv", feature = "rkyv-16", feature = "rkyv-32", feature = "rkyv-64"), derive(Archive, Deserialize, Serialize), rkyv(compare(PartialEq)), - rkyv(attr(derive(Clone, Copy, PartialEq, Eq, Debug, Hash))) + rkyv(derive(Clone, Copy, PartialEq, Eq, Debug, Hash)) )] #[cfg_attr(feature = "rkyv-validation", archive(check_bytes))] #[cfg_attr(all(feature = "arbitrary", feature = "std"), derive(arbitrary::Arbitrary))] @@ -150,3 +150,10 @@ impl fmt::Display for Utc { write!(f, "UTC") } } + +#[cfg(feature = "defmt")] +impl defmt::Format for Utc { + fn format(&self, fmt: defmt::Formatter) { + defmt::write!(fmt, "Z"); + } +} diff --git a/patch/chrono-0.4.42/src/round.rs b/patch/chrono-0.4.43/src/round.rs similarity index 99% rename from patch/chrono-0.4.42/src/round.rs rename to patch/chrono-0.4.43/src/round.rs index 9f575a6..34c4fe4 100644 --- a/patch/chrono-0.4.42/src/round.rs +++ b/patch/chrono-0.4.43/src/round.rs @@ -305,6 +305,7 @@ where /// /// See: [`DurationRound`] #[derive(Debug, Clone, PartialEq, Eq, Copy)] +#[cfg_attr(feature = "defmt", derive(defmt::Format))] pub enum RoundingError { /// Error when the TimeDelta exceeds the TimeDelta from or until the Unix epoch. /// diff --git a/patch/chrono-0.4.42/src/time_delta.rs b/patch/chrono-0.4.43/src/time_delta.rs similarity index 99% rename from patch/chrono-0.4.42/src/time_delta.rs rename to patch/chrono-0.4.43/src/time_delta.rs index 0b467cc..066be18 100644 --- a/patch/chrono-0.4.42/src/time_delta.rs +++ b/patch/chrono-0.4.43/src/time_delta.rs @@ -56,9 +56,10 @@ const SECS_PER_WEEK: i64 = 604_800; any(feature = "rkyv", feature = "rkyv-16", feature = "rkyv-32", feature = "rkyv-64"), derive(Archive, Deserialize, Serialize), rkyv(compare(PartialEq, PartialOrd)), - rkyv(attr(derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash))) + rkyv(derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)) )] #[cfg_attr(feature = "rkyv-validation", archive(check_bytes))] +#[cfg_attr(feature = "defmt", derive(defmt::Format))] pub struct TimeDelta { secs: i64, nanos: i32, // Always 0 <= nanos < NANOS_PER_SEC @@ -624,6 +625,7 @@ impl fmt::Display for TimeDelta { /// *seconds*, while this module supports signed range of up to /// `i64::MAX` of *milliseconds*. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr(feature = "defmt", derive(defmt::Format))] pub struct OutOfRangeError(()); impl fmt::Display for OutOfRangeError { diff --git a/patch/chrono-0.4.42/src/traits.rs b/patch/chrono-0.4.43/src/traits.rs similarity index 100% rename from patch/chrono-0.4.42/src/traits.rs rename to patch/chrono-0.4.43/src/traits.rs diff --git a/patch/chrono-0.4.42/src/weekday.rs b/patch/chrono-0.4.43/src/weekday.rs similarity index 97% rename from patch/chrono-0.4.42/src/weekday.rs rename to patch/chrono-0.4.43/src/weekday.rs index fcb4183..f739566 100644 --- a/patch/chrono-0.4.42/src/weekday.rs +++ b/patch/chrono-0.4.43/src/weekday.rs @@ -34,10 +34,11 @@ use crate::OutOfRange; any(feature = "rkyv", feature = "rkyv-16", feature = "rkyv-32", feature = "rkyv-64"), derive(Archive, Deserialize, Serialize), rkyv(compare(PartialEq)), - rkyv(attr(derive(Clone, Copy, PartialEq, Eq, Debug, Hash))) + rkyv(derive(Clone, Copy, PartialEq, Eq, Debug, Hash)) )] #[cfg_attr(feature = "rkyv-validation", archive(check_bytes))] #[cfg_attr(all(feature = "arbitrary", feature = "std"), derive(arbitrary::Arbitrary))] +#[cfg_attr(feature = "defmt", derive(defmt::Format))] pub enum Weekday { /// Monday. Mon = 0, @@ -256,6 +257,13 @@ impl fmt::Debug for ParseWeekdayError { } } +#[cfg(feature = "defmt")] +impl defmt::Format for ParseWeekdayError { + fn format(&self, fmt: defmt::Formatter) { + defmt::write!(fmt, "ParseWeekdayError {{ .. }}") + } +} + // the actual `FromStr` implementation is in the `format` module to leverage the existing code #[cfg(feature = "serde")] diff --git a/patch/chrono-0.4.42/src/weekday_set.rs b/patch/chrono-0.4.43/src/weekday_set.rs similarity index 97% rename from patch/chrono-0.4.42/src/weekday_set.rs rename to patch/chrono-0.4.43/src/weekday_set.rs index ce60bf3..989131c 100644 --- a/patch/chrono-0.4.42/src/weekday_set.rs +++ b/patch/chrono-0.4.43/src/weekday_set.rs @@ -331,6 +331,23 @@ impl Debug for WeekdaySet { } } +#[cfg(feature = "defmt")] +impl defmt::Format for WeekdaySet { + fn format(&self, f: defmt::Formatter<'_>) { + defmt::write!( + f, + "WeekdaySet({}{}{}{}{}{}{})", + 0x1 & (self.0 >> 6), + 0x1 & (self.0 >> 5), + 0x1 & (self.0 >> 4), + 0x1 & (self.0 >> 3), + 0x1 & (self.0 >> 2), + 0x1 & (self.0 >> 1), + 0x1 & (self.0 >> 0), + ) + } +} + /// An iterator over a collection of weekdays, starting from a given day. /// /// See [`WeekdaySet::iter()`]. diff --git a/patch/dashmap/Cargo.toml b/patch/dashmap/Cargo.toml deleted file mode 100644 index 06ca4e9..0000000 --- a/patch/dashmap/Cargo.toml +++ /dev/null @@ -1,35 +0,0 @@ -[package] -name = "dashmap" -version = "7.0.0-rc2" -authors = ["Joel Wejdenstål "] -edition = "2021" -# rust-version = "1.70" -license = "MIT" -repository = "https://github.com/xacrimon/dashmap" -homepage = "https://github.com/xacrimon/dashmap" -description = "Blazing fast concurrent HashMap for Rust." -readme = "README.md" -documentation = "https://docs.rs/dashmap" -keywords = ["atomic", "concurrent", "hashmap"] -categories = ["concurrency", "algorithms", "data-structures"] - -[features] -all = ["raw-api", "typesize", "serde", "rayon", "arbitrary"] -raw-api = [] -typesize = ["dep:typesize"] -inline-more = ["hashbrown/inline-more"] - -[dependencies] -lock_api = "0.4.12" -parking_lot_core = "0.9.10" -equivalent = "1.0.1" -hashbrown = { version = "0.16.1", default-features = false } -serde = { version = "1.0.217", optional = true, features = ["derive"] } -cfg-if = "1.0.0" -rayon = { version = "1.10.0", optional = true } -arbitrary = { version = "1.4.1", optional = true } -crossbeam-utils = "0.8" -typesize = { version = "0.1.13", default-features = false, features = ["hashbrown_15"], optional = true } - -[package.metadata.docs.rs] -features = ["all"] diff --git a/patch/dashmap/LICENSE b/patch/dashmap/LICENSE deleted file mode 100644 index 7217acd..0000000 --- a/patch/dashmap/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2025 Joel Wejdenstål - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/patch/dashmap/README.md b/patch/dashmap/README.md deleted file mode 100644 index 7d20392..0000000 --- a/patch/dashmap/README.md +++ /dev/null @@ -1,72 +0,0 @@ -# DashMap - -Blazingly fast concurrent map in Rust. - -DashMap is an implementation of a concurrent associative array/hashmap in Rust. - -DashMap tries to implement an easy to use API similar to `std::collections::HashMap` -with some slight changes to handle concurrency. - -DashMap tries to be very simple to use and to be a direct replacement for `RwLock>`. -To accomplish these goals, all methods take `&self` instead of modifying methods taking `&mut self`. -This allows you to put a DashMap in an `Arc` and share it between threads while still being able to modify it. - -DashMap puts great effort into performance and aims to be as fast as possible. -If you have any suggestions or tips do not hesitate to open an issue or a PR. - -The current MSRV is 1.70 and is not changed in patch releases. You can pin a minor version if you want -perfect stability. Though `dashmap` always stays at least 1 year behind the current stable release. - -[![version](https://img.shields.io/crates/v/dashmap)](https://crates.io/crates/dashmap) - -[![documentation](https://docs.rs/dashmap/badge.svg)](https://docs.rs/dashmap) - -[![downloads](https://img.shields.io/crates/d/dashmap)](https://crates.io/crates/dashmap) - -[![minimum rustc version](https://img.shields.io/badge/rustc-1.70-orange.svg)](https://crates.io/crates/dashmap) - -## Cargo features - -- `serde` - Enables serde support. - -- `raw-api` - Enables the unstable raw-shard api. - -- `rayon` - Enables rayon support. - -- `inline-more` - Enables `inline-more` feature from the `hashbrown` crate. Comes with the usual tradeoffs of possibly excessive inlining. - -- `arbitrary` - Enables support for the `arbitrary` crate. - -## Contributing - -DashMap gladly accepts contributions! -Do not hesitate to open issues or PR's. - -I will take a look as soon as I have time for it. - -That said I do not get paid (yet) to work on open-source. This means -that my time is limited and my work here comes after my personal life. - -## Performance - -A comprehensive benchmark suite including DashMap can be found [here](https://github.com/xacrimon/conc-map-bench). - -## Special thanks - -- [Conrad Ludgate](https://github.com/conradludgate) - -- [Jon Gjengset](https://github.com/jonhoo) - -- [Yato](https://github.com/RustyYato) - -- [Karl Bergström](https://github.com/kabergstrom) - -- [Dylan DPC](https://github.com/Dylan-DPC) - -- [Lokathor](https://github.com/Lokathor) - -- [namibj](https://github.com/namibj) - -## License - -This project is licensed under MIT. diff --git a/patch/dashmap/src/arbitrary.rs b/patch/dashmap/src/arbitrary.rs deleted file mode 100644 index a760964..0000000 --- a/patch/dashmap/src/arbitrary.rs +++ /dev/null @@ -1,13 +0,0 @@ -use arbitrary::{Arbitrary, Unstructured}; -use core::hash::BuildHasher; - -impl<'a, K, V, S> Arbitrary<'a> for crate::DashMap -where - K: Eq + std::hash::Hash + Arbitrary<'a>, - V: Arbitrary<'a>, - S: Default + BuildHasher + Clone, -{ - fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result { - u.arbitrary_iter()?.collect() - } -} diff --git a/patch/dashmap/src/iter.rs b/patch/dashmap/src/iter.rs deleted file mode 100644 index 887e439..0000000 --- a/patch/dashmap/src/iter.rs +++ /dev/null @@ -1,217 +0,0 @@ -use crossbeam_utils::CachePadded; -use hashbrown::hash_table; - -use super::mapref::multiple::{RefMulti, RefMutMulti}; -use crate::lock::{RwLock, RwLockReadGuardDetached, RwLockWriteGuardDetached}; -use crate::{DashMap, HashMap}; -use core::hash::Hash; -use std::sync::Arc; - -/// Iterator over a DashMap yielding key value pairs. -/// -/// # Examples -/// -/// ``` -/// use dashmap::DashMap; -/// -/// let map = DashMap::new(); -/// map.insert("hello", "world"); -/// map.insert("alex", "steve"); -/// let pairs: Vec<(&'static str, &'static str)> = map.into_iter().collect(); -/// assert_eq!(pairs.len(), 2); -/// ``` -pub struct OwningIter { - shards: std::vec::IntoIter>>>, - current: Option>, -} - -impl OwningIter { - pub(crate) fn new(map: DashMap) -> Self { - Self { - shards: map.shards.into_vec().into_iter(), - current: None, - } - } -} - -type GuardOwningIter = hash_table::IntoIter<(K, V)>; - -impl Iterator for OwningIter { - type Item = (K, V); - - fn next(&mut self) -> Option { - loop { - if let Some(current) = self.current.as_mut() { - if let Some((k, v)) = current.next() { - return Some((k, v)); - } - } - - let iter = self.shards.next()?.into_inner().into_inner().into_iter(); - self.current = Some(iter); - } - } -} - -type GuardIter<'a, K, V> = ( - Arc>, - hash_table::Iter<'a, (K, V)>, -); - -type GuardIterMut<'a, K, V> = ( - Arc>, - hash_table::IterMut<'a, (K, V)>, -); - -/// Iterator over a DashMap yielding immutable references. -/// -/// # Examples -/// -/// ``` -/// use dashmap::DashMap; -/// -/// let map = DashMap::new(); -/// map.insert("hello", "world"); -/// assert_eq!(map.iter().count(), 1); -/// ``` -pub struct Iter<'a, K, V> { - shards: std::slice::Iter<'a, CachePadded>>>, - current: Option>, -} - -impl<'i, K: Clone + Hash + Eq, V: Clone> Clone for Iter<'i, K, V> { - fn clone(&self) -> Self { - Iter { - shards: self.shards.clone(), - current: self.current.clone(), - } - } -} - -impl<'a, K: 'a, V: 'a> Iter<'a, K, V> { - pub(crate) fn new(map: &'a DashMap) -> Self { - Self { - shards: map.shards.iter(), - current: None, - } - } -} - -impl<'a, K: 'a, V: 'a> Iterator for Iter<'a, K, V> { - type Item = RefMulti<'a, K, V>; - - fn next(&mut self) -> Option { - loop { - if let Some(current) = self.current.as_mut() { - if let Some((k, v)) = current.1.next() { - let guard = current.0.clone(); - return Some(RefMulti::new(guard, k, v)); - } - } - - let guard = self.shards.next()?.read(); - // SAFETY: we keep the guard alive with the shard iterator, - // and with any refs produced by the iterator - let (guard, shard) = unsafe { RwLockReadGuardDetached::detach_from(guard) }; - - let iter = shard.iter(); - - self.current = Some((Arc::new(guard), iter)); - } - } -} - -/// Iterator over a DashMap yielding mutable references. -/// -/// # Examples -/// -/// ``` -/// use dashmap::DashMap; -/// -/// let map = DashMap::new(); -/// map.insert("Johnny", 21); -/// map.iter_mut().for_each(|mut r| *r += 1); -/// assert_eq!(*map.get("Johnny").unwrap(), 22); -/// ``` -pub struct IterMut<'a, K, V> { - shards: std::slice::Iter<'a, CachePadded>>>, - current: Option>, -} - -impl<'a, K: 'a, V: 'a> IterMut<'a, K, V> { - pub(crate) fn new(map: &'a DashMap) -> Self { - Self { - shards: map.shards.iter(), - current: None, - } - } -} - -impl<'a, K: 'a, V: 'a> Iterator for IterMut<'a, K, V> { - type Item = RefMutMulti<'a, K, V>; - - fn next(&mut self) -> Option { - loop { - if let Some(current) = self.current.as_mut() { - if let Some((k, v)) = current.1.next() { - let guard = current.0.clone(); - return Some(RefMutMulti::new(guard, k, v)); - } - } - - let guard = self.shards.next()?.write(); - - // SAFETY: we keep the guard alive with the shard iterator, - // and with any refs produced by the iterator - let (guard, shard) = unsafe { RwLockWriteGuardDetached::detach_from(guard) }; - - let iter = shard.iter_mut(); - - self.current = Some((Arc::new(guard), iter)); - } - } -} - -#[cfg(test)] -mod tests { - use crate::DashMap; - - #[test] - fn iter_mut_manual_count() { - let map = DashMap::new(); - - map.insert("Johnny", 21); - - assert_eq!(map.len(), 1); - - let mut c = 0; - - for shard in map.shards() { - c += shard.write().iter().count(); - } - - assert_eq!(c, 1); - } - - #[test] - fn iter_mut_count() { - let map = DashMap::new(); - - map.insert("Johnny", 21); - - assert_eq!(map.len(), 1); - - assert_eq!(map.iter_mut().count(), 1); - } - - #[test] - fn iter_count() { - let map = DashMap::new(); - - map.insert("Johnny", 21); - - assert_eq!(map.len(), 1); - - assert_eq!(map.iter().count(), 1); - } -} diff --git a/patch/dashmap/src/iter_set.rs b/patch/dashmap/src/iter_set.rs deleted file mode 100644 index 0b90ec6..0000000 --- a/patch/dashmap/src/iter_set.rs +++ /dev/null @@ -1,38 +0,0 @@ -use crate::setref::multiple::RefMulti; -use core::hash::Hash; - -pub struct OwningIter { - inner: crate::iter::OwningIter, -} - -impl OwningIter { - pub(crate) fn new(inner: crate::iter::OwningIter) -> Self { - Self { inner } - } -} - -impl Iterator for OwningIter { - type Item = K; - - fn next(&mut self) -> Option { - self.inner.next().map(|(k, _)| k) - } -} - -pub struct Iter<'a, K> { - inner: crate::iter::Iter<'a, K, ()>, -} - -impl<'a, K: Eq + Hash + 'a> Iter<'a, K> { - pub(crate) fn new(inner: crate::iter::Iter<'a, K, ()>) -> Self { - Self { inner } - } -} - -impl<'a, K: Eq + Hash + 'a> Iterator for Iter<'a, K> { - type Item = RefMulti<'a, K>; - - fn next(&mut self) -> Option { - self.inner.next().map(RefMulti::new) - } -} diff --git a/patch/dashmap/src/lib.rs b/patch/dashmap/src/lib.rs deleted file mode 100644 index 31d356f..0000000 --- a/patch/dashmap/src/lib.rs +++ /dev/null @@ -1,1502 +0,0 @@ -#![doc = include_str!("../README.md")] -#![allow(clippy::type_complexity)] - -#[cfg(feature = "arbitrary")] -mod arbitrary; -pub mod iter; -pub mod iter_set; -mod lock; -pub mod mapref; -mod read_only; -#[cfg(feature = "serde")] -mod serde; -mod set; -pub mod setref; -pub mod try_result; -mod util; - -#[cfg(feature = "rayon")] -pub mod rayon { - pub mod map; - pub mod read_only; - pub mod set; -} - -#[cfg(not(feature = "raw-api"))] -use crate::lock::RwLock; -#[cfg(feature = "raw-api")] -pub use crate::lock::{RawRwLock, RwLock}; -use crate::mapref::entry_ref::{EntryRef, OccupiedEntryRef, VacantEntryRef}; -use cfg_if::cfg_if; -use core::{ - fmt, - hash::{BuildHasher, Hash, Hasher}, - iter::FromIterator, - ops::{BitAnd, BitOr, Shl, Shr, Sub}, -}; -use crossbeam_utils::CachePadded; -pub use equivalent::Equivalent; -use hashbrown::hash_table; -use iter::{Iter, IterMut, OwningIter}; -use lock::{RwLockReadGuardDetached, RwLockWriteGuardDetached}; -pub use mapref::entry::{Entry, OccupiedEntry, VacantEntry}; -use mapref::{ - multiple::RefMulti, - one::{Ref, RefMut}, -}; -pub use read_only::ReadOnlyView; -pub use set::DashSet; -use std::{collections::hash_map::RandomState, sync::OnceLock}; -use try_result::TryResult; - -pub(crate) type HashMap = hash_table::HashTable<(K, V)>; - -// Temporary reimplementation of [`std::collections::TryReserveError`] -// util [`std::collections::TryReserveError`] stabilises. -// We cannot easily create `std::collections` error type from `hashbrown` error type -// without access to `TryReserveError::kind` method. -#[non_exhaustive] -#[derive(Clone, PartialEq, Eq, Debug)] -pub struct TryReserveError {} - -fn default_shard_amount() -> usize { - static DEFAULT_SHARD_AMOUNT: OnceLock = OnceLock::new(); - *DEFAULT_SHARD_AMOUNT.get_or_init(|| { - (std::thread::available_parallelism().map_or(1, usize::from) * 4).next_power_of_two() - }) -} - -fn ncb(shard_amount: usize) -> usize { shard_amount.trailing_zeros() as usize } - -/// DashMap is an implementation of a concurrent associative array/hashmap in Rust. -/// -/// DashMap tries to implement an easy to use API similar to `std::collections::HashMap` -/// with some slight changes to handle concurrency. -/// -/// DashMap tries to be very simple to use and to be a direct replacement for `RwLock>`. -/// To accomplish this, all methods take `&self` instead of modifying methods taking `&mut self`. -/// This allows you to put a DashMap in an `Arc` and share it between threads while being able to modify it. -/// -/// Documentation mentioning locking behaviour acts in the reference frame of the calling thread. -/// This means that it is safe to ignore it across multiple threads. -pub struct DashMap { - shift: usize, - shards: Box<[CachePadded>>]>, - hasher: S, -} - -impl Clone for DashMap { - fn clone(&self) -> Self { - fn clone_rwlock(lock: &CachePadded>) -> CachePadded> { - CachePadded::new(RwLock::new(lock.read().clone())) - } - - Self { - shift: self.shift, - shards: self.shards.iter().map(clone_rwlock).collect(), - hasher: self.hasher.clone(), - } - } -} - -impl Default for DashMap -where - K: Eq + Hash, - S: Default + BuildHasher + Clone, -{ - fn default() -> Self { Self::with_hasher(Default::default()) } -} - -impl<'a, K: Eq + Hash, V> DashMap { - /// Creates a new DashMap with a capacity of 0. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// - /// let reviews = DashMap::new(); - /// reviews.insert("Veloren", "What a fantastic game!"); - /// ``` - pub fn new() -> Self { DashMap::with_hasher(RandomState::default()) } - - /// Creates a new DashMap with a specified starting capacity. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// - /// let mappings = DashMap::with_capacity(2); - /// mappings.insert(2, 4); - /// mappings.insert(8, 16); - /// ``` - pub fn with_capacity(capacity: usize) -> Self { - DashMap::with_capacity_and_hasher(capacity, RandomState::default()) - } - - /// Creates a new DashMap with a specified shard amount - /// - /// shard_amount should greater than 0 and be a power of two. - /// If a shard_amount which is not a power of two is provided, the function will panic. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// - /// let mappings = DashMap::with_shard_amount(32); - /// mappings.insert(2, 4); - /// mappings.insert(8, 16); - /// ``` - pub fn with_shard_amount(shard_amount: usize) -> Self { - Self::with_capacity_and_hasher_and_shard_amount(0, RandomState::default(), shard_amount) - } - - /// Creates a new DashMap with a specified capacity and shard amount. - /// - /// shard_amount should greater than 0 and be a power of two. - /// If a shard_amount which is not a power of two is provided, the function will panic. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// - /// let mappings = DashMap::with_capacity_and_shard_amount(32, 32); - /// mappings.insert(2, 4); - /// mappings.insert(8, 16); - /// ``` - pub fn with_capacity_and_shard_amount(capacity: usize, shard_amount: usize) -> Self { - Self::with_capacity_and_hasher_and_shard_amount( - capacity, - RandomState::default(), - shard_amount, - ) - } -} - -impl<'a, K, V, S> DashMap { - /// Wraps this `DashMap` into a read-only view. This view allows to obtain raw references to the stored values. - pub fn into_read_only(self) -> ReadOnlyView { ReadOnlyView::new(self) } - - /// Creates a new DashMap with a capacity of 0 and the provided hasher. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// use std::collections::hash_map::RandomState; - /// - /// let s = RandomState::new(); - /// let reviews = DashMap::with_hasher(s); - /// reviews.insert("Veloren", "What a fantastic game!"); - /// ``` - pub fn with_hasher(hasher: S) -> Self { Self::with_capacity_and_hasher(0, hasher) } - - /// Creates a new DashMap with a specified starting capacity and hasher. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// use std::collections::hash_map::RandomState; - /// - /// let s = RandomState::new(); - /// let mappings = DashMap::with_capacity_and_hasher(2, s); - /// mappings.insert(2, 4); - /// mappings.insert(8, 16); - /// ``` - pub fn with_capacity_and_hasher(capacity: usize, hasher: S) -> Self { - Self::with_capacity_and_hasher_and_shard_amount(capacity, hasher, default_shard_amount()) - } - - /// Creates a new DashMap with a specified hasher and shard amount - /// - /// shard_amount should be greater than 0 and a power of two. - /// If a shard_amount which is not a power of two is provided, the function will panic. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// use std::collections::hash_map::RandomState; - /// - /// let s = RandomState::new(); - /// let mappings = DashMap::with_hasher_and_shard_amount(s, 32); - /// mappings.insert(2, 4); - /// mappings.insert(8, 16); - /// ``` - pub fn with_hasher_and_shard_amount(hasher: S, shard_amount: usize) -> Self { - Self::with_capacity_and_hasher_and_shard_amount(0, hasher, shard_amount) - } - - /// Creates a new DashMap with a specified starting capacity, hasher and shard_amount. - /// - /// shard_amount should greater than 0 and be a power of two. - /// If a shard_amount which is not a power of two is provided, the function will panic. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// use std::collections::hash_map::RandomState; - /// - /// let s = RandomState::new(); - /// let mappings = DashMap::with_capacity_and_hasher_and_shard_amount(2, s, 32); - /// mappings.insert(2, 4); - /// mappings.insert(8, 16); - /// ``` - pub fn with_capacity_and_hasher_and_shard_amount( - mut capacity: usize, - hasher: S, - shard_amount: usize, - ) -> Self { - assert!(shard_amount > 1); - assert!(shard_amount.is_power_of_two()); - - let shift = util::ptr_size_bits() - ncb(shard_amount); - - if capacity != 0 { - capacity = (capacity + (shard_amount - 1)) & !(shard_amount - 1); - } - - let cps = capacity / shard_amount; - - let shards = (0..shard_amount) - .map(|_| CachePadded::new(RwLock::new(HashMap::with_capacity(cps)))) - .collect(); - - Self { shift, shards, hasher } - } - - cfg_if! { - if #[cfg(feature = "raw-api")] { - /// Allows you to peek at the inner shards that store your data. - /// You should probably not use this unless you know what you are doing. - /// - /// Requires the `raw-api` feature to be enabled. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// - /// let map = DashMap::<(), ()>::new(); - /// println!("Amount of shards: {}", map.shards().len()); - /// ``` - pub fn shards(&self) -> &[CachePadded>>] { - &self.shards - } - - /// Provides mutable access to the inner shards that store your data. - /// You should probably not use this unless you know what you are doing. - /// - /// Requires the `raw-api` feature to be enabled. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// use std::hash::{Hash, Hasher, BuildHasher}; - /// - /// let mut map = DashMap::::new(); - /// let shard_ind = map.determine_map(&42); - /// let mut factory = map.hasher().clone(); - /// let hasher = |tuple: &(i32, &'static str)| { - /// let mut hasher = factory.build_hasher(); - /// tuple.0.hash(&mut hasher); - /// hasher.finish() - /// }; - /// let data = (42, "forty two"); - /// let hash = hasher(&data); - /// map.shards_mut()[shard_ind].get_mut().insert_unique(hash, data, hasher); - /// assert_eq!(*map.get(&42).unwrap(), "forty two"); - /// ``` - pub fn shards_mut(&mut self) -> &mut [CachePadded>>] { - &mut self.shards - } - - /// Consumes this `DashMap` and returns the inner shards. - /// You should probably not use this unless you know what you are doing. - /// - /// Requires the `raw-api` feature to be enabled. - /// - /// See [`DashMap::shards()`] and [`DashMap::shards_mut()`] for more information. - pub fn into_shards(self) -> Box<[CachePadded>>]> { - self.shards - } - } else { - #[allow(dead_code)] - pub(crate) fn shards(&self) -> &[CachePadded>>] { - &self.shards - } - - #[allow(dead_code)] - pub(crate) fn shards_mut(&mut self) -> &mut [CachePadded>>] { - &mut self.shards - } - - #[allow(dead_code)] - pub(crate) fn into_shards(self) -> Box<[CachePadded>>]> { - self.shards - } - } - } - - cfg_if! { - if #[cfg(feature = "raw-api")] { - /// Finds which shard a certain key is stored in. - /// You should probably not use this unless you know what you are doing. - /// Note that shard selection is dependent on the default or provided HashBuilder. - /// - /// Requires the `raw-api` feature to be enabled. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// - /// let map = DashMap::new(); - /// map.insert("coca-cola", 1.4); - /// println!("coca-cola is stored in shard: {}", map.determine_map("coca-cola")); - /// ``` - pub fn determine_map(&self, key: &Q) -> usize - where - Q: Hash + Equivalent + ?Sized, - { - let hash = self.hash_usize(&key); - self.determine_shard(hash) - } - } - } - - cfg_if! { - if #[cfg(feature = "raw-api")] { - /// Finds which shard a certain hash is stored in. - /// - /// Requires the `raw-api` feature to be enabled. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// - /// let map: DashMap = DashMap::new(); - /// let key = "key"; - /// let hash = map.hash_usize(&key); - /// println!("hash is stored in shard: {}", map.determine_shard(hash)); - /// ``` - pub fn determine_shard(&self, hash: usize) -> usize { - // Leave the high 7 bits for the HashBrown SIMD tag. - let idx = (hash << 7) >> self.shift; - - // hint to llvm that the panic bounds check can be removed - if idx >= self.shards.len() { - if cfg!(debug_assertions) { - unreachable!("invalid shard index") - } else { - // SAFETY: shards is always a power of two, - // and shift is calculated such that the resulting idx is always - // less than the shards length - unsafe { - std::hint::unreachable_unchecked(); - } - } - } - - idx - } - } else { - - pub(crate) fn determine_shard(&self, hash: usize) -> usize { - // Leave the high 7 bits for the HashBrown SIMD tag. - let idx = (hash << 7) >> self.shift; - - // hint to llvm that the panic bounds check can be removed - if idx >= self.shards.len() { - if cfg!(debug_assertions) { - unreachable!("invalid shard index") - } else { - // SAFETY: shards is always a power of two, - // and shift is calculated such that the resulting idx is always - // less than the shards length - unsafe { - std::hint::unreachable_unchecked(); - } - } - } - - idx - } - } - } - - /// Returns a reference to the map's [`BuildHasher`]. - /// - /// # Examples - /// - /// ```rust - /// use dashmap::DashMap; - /// use std::collections::hash_map::RandomState; - /// - /// let hasher = RandomState::new(); - /// let map: DashMap = DashMap::new(); - /// let hasher: &RandomState = map.hasher(); - /// ``` - /// - /// [`BuildHasher`]: https://doc.rust-lang.org/std/hash/trait.BuildHasher.html - pub fn hasher(&self) -> &S { &self.hasher } -} - -impl<'a, K, V, S: BuildHasher> DashMap { - /// Hash a given item to produce a usize. - /// Uses the provided or default HashBuilder. - pub fn hash_usize(&self, item: &T) -> usize { self.hash_u64(item) as usize } - - fn hash_u64(&self, item: &T) -> u64 { - let mut hasher = self.hasher.build_hasher(); - - item.hash(&mut hasher); - - hasher.finish() - } - - /// Inserts a key and a value into the map. Returns the old value associated with the key if there was one. - /// Does not update the key if it was already present. - /// - /// **Locking behaviour:** May deadlock if called when holding any sort of reference into the map. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// - /// let map = DashMap::new(); - /// map.insert("I am the key!", "And I am the value!"); - /// ``` - pub fn insert(&self, key: K, value: V) -> Option - where K: Eq + Hash { - self._insert(key, value) - } - - /// Removes an entry from the map, returning the key and value if they existed in the map. - /// - /// **Locking behaviour:** May deadlock if called when holding any sort of reference into the map. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// - /// let soccer_team = DashMap::new(); - /// soccer_team.insert("Jack", "Goalie"); - /// assert_eq!(soccer_team.remove("Jack").unwrap().1, "Goalie"); - /// ``` - pub fn remove(&self, key: &Q) -> Option<(K, V)> - where Q: Hash + Equivalent + ?Sized { - self._remove(key) - } - - /// Removes an entry from the map, returning the key and value - /// if the entry existed and the provided conditional function returned true. - /// - /// **Locking behaviour:** May deadlock if called when holding any sort of reference into the map. - /// - /// ``` - /// use dashmap::DashMap; - /// - /// let soccer_team = DashMap::new(); - /// soccer_team.insert("Sam", "Forward"); - /// soccer_team.remove_if("Sam", |_, position| position == &"Goalie"); - /// assert!(soccer_team.contains_key("Sam")); - /// ``` - /// ``` - /// use dashmap::DashMap; - /// - /// let soccer_team = DashMap::new(); - /// soccer_team.insert("Sam", "Forward"); - /// soccer_team.remove_if("Sam", |_, position| position == &"Forward"); - /// assert!(!soccer_team.contains_key("Sam")); - /// ``` - pub fn remove_if(&self, key: &Q, f: impl FnOnce(&K, &V) -> bool) -> Option<(K, V)> - where Q: Hash + Equivalent + ?Sized { - self._remove_if(key, f) - } - - pub fn remove_if_mut(&self, key: &Q, f: impl FnOnce(&K, &mut V) -> bool) -> Option<(K, V)> - where Q: Hash + Equivalent + ?Sized { - self._remove_if_mut(key, f) - } - - /// Creates an iterator over a DashMap yielding immutable references. - /// - /// **Locking behaviour:** May deadlock if called when holding a mutable reference into the map. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// - /// let words = DashMap::new(); - /// words.insert("hello", "world"); - /// assert_eq!(words.iter().count(), 1); - /// ``` - pub fn iter(&'a self) -> Iter<'a, K, V> { self._iter() } - - /// Iterator over a DashMap yielding mutable references. - /// - /// **Locking behaviour:** May deadlock if called when holding any sort of reference into the map. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// - /// let map = DashMap::new(); - /// map.insert("Johnny", 21); - /// map.iter_mut().for_each(|mut r| *r += 1); - /// assert_eq!(*map.get("Johnny").unwrap(), 22); - /// ``` - pub fn iter_mut(&'a self) -> IterMut<'a, K, V> { self._iter_mut() } - - /// Get an immutable reference to an entry in the map - /// - /// **Locking behaviour:** May deadlock if called when holding a mutable reference into the map. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// - /// let youtubers = DashMap::new(); - /// youtubers.insert("Bosnian Bill", 457000); - /// assert_eq!(*youtubers.get("Bosnian Bill").unwrap(), 457000); - /// ``` - pub fn get(&'a self, key: &Q) -> Option> - where Q: Hash + Equivalent + ?Sized { - self._get(key) - } - - /// Get a mutable reference to an entry in the map - /// - /// **Locking behaviour:** May deadlock if called when holding any sort of reference into the map. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// - /// let class = DashMap::new(); - /// class.insert("Albin", 15); - /// *class.get_mut("Albin").unwrap() -= 1; - /// assert_eq!(*class.get("Albin").unwrap(), 14); - /// ``` - pub fn get_mut(&'a self, key: &Q) -> Option> - where Q: Hash + Equivalent + ?Sized { - self._get_mut(key) - } - - /// Get an immutable reference to an entry in the map, if the shard is not locked. - /// If the shard is locked, the function will return [TryResult::Locked]. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// use dashmap::try_result::TryResult; - /// - /// let map = DashMap::new(); - /// map.insert("Johnny", 21); - /// - /// assert_eq!(*map.try_get("Johnny").unwrap(), 21); - /// - /// let _result1_locking = map.get_mut("Johnny"); - /// - /// let result2 = map.try_get("Johnny"); - /// assert!(result2.is_locked()); - /// ``` - pub fn try_get(&'a self, key: &Q) -> TryResult> - where Q: Hash + Equivalent + ?Sized { - self._try_get(key) - } - - /// Get a mutable reference to an entry in the map, if the shard is not locked. - /// If the shard is locked, the function will return [TryResult::Locked]. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// use dashmap::try_result::TryResult; - /// - /// let map = DashMap::new(); - /// map.insert("Johnny", 21); - /// - /// *map.try_get_mut("Johnny").unwrap() += 1; - /// assert_eq!(*map.get("Johnny").unwrap(), 22); - /// - /// let _result1_locking = map.get("Johnny"); - /// - /// let result2 = map.try_get_mut("Johnny"); - /// assert!(result2.is_locked()); - /// ``` - pub fn try_get_mut(&'a self, key: &Q) -> TryResult> - where Q: Hash + Equivalent + ?Sized { - self._try_get_mut(key) - } - - /// Remove excess capacity to reduce memory usage. - /// - /// **Locking behaviour:** May deadlock if called when holding any sort of reference into the map. - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// use dashmap::try_result::TryResult; - /// - /// let map = DashMap::new(); - /// map.insert("Johnny", 21); - /// assert!(map.capacity() > 0); - /// map.remove("Johnny"); - /// map.shrink_to_fit(); - /// assert_eq!(map.capacity(), 0); - /// ``` - pub fn shrink_to_fit(&self) - where K: Hash { - self._shrink_to_fit(); - } - - /// Retain elements that whose predicates return true - /// and discard elements whose predicates return false. - /// - /// **Locking behaviour:** May deadlock if called when holding any sort of reference into the map. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// - /// let people = DashMap::new(); - /// people.insert("Albin", 15); - /// people.insert("Jones", 22); - /// people.insert("Charlie", 27); - /// people.retain(|_, v| *v > 20); - /// assert_eq!(people.len(), 2); - /// ``` - pub fn retain(&self, f: impl FnMut(&K, &mut V) -> bool) { self._retain(f); } - - /// Fetches the total number of key-value pairs stored in the map. - /// - /// **Locking behaviour:** May deadlock if called when holding a mutable reference into the map. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// - /// let people = DashMap::new(); - /// people.insert("Albin", 15); - /// people.insert("Jones", 22); - /// people.insert("Charlie", 27); - /// assert_eq!(people.len(), 3); - /// ``` - pub fn len(&self) -> usize { self._len() } - - /// Checks if the map is empty or not. - /// - /// **Locking behaviour:** May deadlock if called when holding a mutable reference into the map. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// - /// let map = DashMap::<(), ()>::new(); - /// assert!(map.is_empty()); - /// ``` - pub fn is_empty(&self) -> bool { self._is_empty() } - - /// Removes all key-value pairs in the map. - /// - /// **Locking behaviour:** May deadlock if called when holding any sort of reference into the map. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// - /// let stats = DashMap::new(); - /// stats.insert("Goals", 4); - /// assert!(!stats.is_empty()); - /// stats.clear(); - /// assert!(stats.is_empty()); - /// ``` - pub fn clear(&self) { self._clear(); } - - /// Returns how many key-value pairs the map can store without reallocating. - /// - /// **Locking behaviour:** May deadlock if called when holding a mutable reference into the map. - pub fn capacity(&self) -> usize { self._capacity() } - - /// Modify a specific value according to a function. - /// - /// **Locking behaviour:** May deadlock if called when holding any sort of reference into the map. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// - /// let stats = DashMap::new(); - /// stats.insert("Goals", 4); - /// stats.alter("Goals", |_, v| v * 2); - /// assert_eq!(*stats.get("Goals").unwrap(), 8); - /// ``` - /// - /// # Panics - /// - /// If the given closure panics, then `alter` will abort the process - pub fn alter(&self, key: &Q, f: impl FnOnce(&K, V) -> V) - where Q: Hash + Equivalent + ?Sized { - self._alter(key, f); - } - - /// Modify every value in the map according to a function. - /// - /// **Locking behaviour:** May deadlock if called when holding any sort of reference into the map. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// - /// let stats = DashMap::new(); - /// stats.insert("Wins", 4); - /// stats.insert("Losses", 2); - /// stats.alter_all(|_, v| v + 1); - /// assert_eq!(*stats.get("Wins").unwrap(), 5); - /// assert_eq!(*stats.get("Losses").unwrap(), 3); - /// ``` - /// - /// # Panics - /// - /// If the given closure panics, then `alter_all` will abort the process - pub fn alter_all(&self, f: impl FnMut(&K, V) -> V) { self._alter_all(f); } - - /// Scoped access into an item of the map according to a function. - /// - /// **Locking behaviour:** May deadlock if called when holding any sort of reference into the map. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// - /// let warehouse = DashMap::new(); - /// warehouse.insert(4267, ("Banana", 100)); - /// warehouse.insert(2359, ("Pear", 120)); - /// let fruit = warehouse.view(&4267, |_k, v| *v); - /// assert_eq!(fruit, Some(("Banana", 100))); - /// ``` - /// - /// # Panics - /// - /// If the given closure panics, then `view` will abort the process - pub fn view(&self, key: &Q, f: impl FnOnce(&K, &V) -> R) -> Option - where Q: Hash + Equivalent + ?Sized { - self._view(key, f) - } - - /// Checks if the map contains a specific key. - /// - /// **Locking behaviour:** May deadlock if called when holding a mutable reference into the map. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// - /// let team_sizes = DashMap::new(); - /// team_sizes.insert("Dakota Cherries", 23); - /// assert!(team_sizes.contains_key("Dakota Cherries")); - /// ``` - pub fn contains_key(&self, key: &Q) -> bool - where Q: Hash + Equivalent + ?Sized { - self._contains_key(key) - } - - /// Advanced entry API that tries to mimic `std::collections::HashMap`. - /// See the documentation on `dashmap::mapref::entry` for more details. - /// - /// **Locking behaviour:** May deadlock if called when holding any sort of reference into the map. - pub fn entry(&'a self, key: K) -> Entry<'a, K, V> - where K: Eq + Hash { - self._entry(key) - } - - /// Advanced entry API that tries to mimic `std::collections::HashMap`. - /// See the documentation on `dashmap::mapref::entry` for more details. - /// - /// Returns None if the shard is currently locked. - pub fn try_entry(&'a self, key: K) -> Option> - where K: Eq + Hash { - self._try_entry(key) - } - - /// Advanced entry API that tries to mimic `hashbrown::HashMap::entry_ref`. - /// See the documentation on `dashmap::mapref::entry_ref` for more details. - /// - /// **Locking behaviour:** May deadlock if called when holding any sort of reference into the map. - pub fn entry_ref<'q, Q>(&'a self, key: &'q Q) -> EntryRef<'a, 'q, K, Q, V> - where - K: Eq + Hash, - Q: Hash + Equivalent, - { - self._entry_ref(key) - } - - /// Advanced entry API that tries to mimic `std::collections::HashMap::try_reserve`. - /// Tries to reserve capacity for at least `shard * additional` - /// and may reserve more space to avoid frequent reallocations. - /// - /// # Errors - /// - /// If the capacity overflows, or the allocator reports a failure, then an error is returned. - // TODO: return std::collections::TryReserveError once std::collections::TryReserveErrorKind stabilises. - pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> - where K: Hash { - for shard in self.shards.iter() { - shard - .write() - .try_reserve(additional, |(k, _v)| { - let mut hasher = self.hasher.build_hasher(); - k.hash(&mut hasher); - hasher.finish() - }) - .map_err(|_| TryReserveError {})?; - } - Ok(()) - } -} - -impl<'a, K, V, S: BuildHasher> DashMap { - fn _insert(&self, key: K, value: V) -> Option - where K: Eq + Hash { - match self.entry(key) { - Entry::Occupied(mut o) => Some(o.insert(value)), - Entry::Vacant(v) => { - v.insert(value); - None - } - } - } - - fn _remove(&self, key: &Q) -> Option<(K, V)> - where Q: Hash + Equivalent + ?Sized { - let hash = self.hash_u64(&key); - - let idx = self.determine_shard(hash as usize); - - let mut shard = self.shards[idx].write(); - - if let Ok(entry) = shard.find_entry(hash, |(k, _v)| key.equivalent(k)) { - let ((k, v), _) = entry.remove(); - Some((k, v)) - } else { - None - } - } - - fn _remove_if(&self, key: &Q, f: impl FnOnce(&K, &V) -> bool) -> Option<(K, V)> - where Q: Hash + Equivalent + ?Sized { - let hash = self.hash_u64(&key); - - let idx = self.determine_shard(hash as usize); - - let mut shard = self.shards[idx].write(); - - if let Ok(entry) = shard.find_entry(hash, |(k, _v)| key.equivalent(k)) { - let (k, v) = entry.get(); - if f(k, v) { - let ((k, v), _) = entry.remove(); - Some((k, v)) - } else { - None - } - } else { - None - } - } - - fn _remove_if_mut(&self, key: &Q, f: impl FnOnce(&K, &mut V) -> bool) -> Option<(K, V)> - where Q: Hash + Equivalent + ?Sized { - let hash = self.hash_u64(&key); - - let idx = self.determine_shard(hash as usize); - - let mut shard = self.shards[idx].write(); - - if let Ok(mut entry) = shard.find_entry(hash, |(k, _v)| key.equivalent(k)) { - let (k, v) = entry.get_mut(); - if f(k, v) { - let ((k, v), _) = entry.remove(); - Some((k, v)) - } else { - None - } - } else { - None - } - } - - fn _iter(&'a self) -> Iter<'a, K, V> { Iter::new(self) } - - fn _iter_mut(&'a self) -> IterMut<'a, K, V> { IterMut::new(self) } - - fn _get(&'a self, key: &Q) -> Option> - where Q: Hash + Equivalent + ?Sized { - let hash = self.hash_u64(&key); - - let idx = self.determine_shard(hash as usize); - - let shard = self.shards[idx].read(); - // SAFETY: The data will not outlive the guard, since we pass the guard to `Ref`. - let (guard, shard) = unsafe { RwLockReadGuardDetached::detach_from(shard) }; - - if let Some((k, v)) = shard.find(hash, |(k, _v)| key.equivalent(k)) { - Some(Ref::new(guard, k, v)) - } else { - None - } - } - - fn _get_mut(&'a self, key: &Q) -> Option> - where Q: Hash + Equivalent + ?Sized { - let hash = self.hash_u64(&key); - - let idx = self.determine_shard(hash as usize); - - let shard = self.shards[idx].write(); - // SAFETY: The data will not outlive the guard, since we pass the guard to `RefMut`. - let (guard, shard) = unsafe { RwLockWriteGuardDetached::detach_from(shard) }; - - if let Some((k, v)) = shard.find_mut(hash, |(k, _v)| key.equivalent(k)) { - Some(RefMut::new(guard, k, v)) - } else { - None - } - } - - fn _try_get(&'a self, key: &Q) -> TryResult> - where Q: Hash + Equivalent + ?Sized { - let hash = self.hash_u64(&key); - - let idx = self.determine_shard(hash as usize); - - let shard = match self.shards[idx].try_read() { - Some(shard) => shard, - None => return TryResult::Locked, - }; - // SAFETY: The data will not outlive the guard, since we pass the guard to `Ref`. - let (guard, shard) = unsafe { RwLockReadGuardDetached::detach_from(shard) }; - - if let Some((k, v)) = shard.find(hash, |(k, _v)| key.equivalent(k)) { - TryResult::Present(Ref::new(guard, k, v)) - } else { - TryResult::Absent - } - } - - fn _try_get_mut(&'a self, key: &Q) -> TryResult> - where Q: Hash + Equivalent + ?Sized { - let hash = self.hash_u64(&key); - - let idx = self.determine_shard(hash as usize); - - let shard = match self.shards[idx].try_write() { - Some(shard) => shard, - None => return TryResult::Locked, - }; - // SAFETY: The data will not outlive the guard, since we pass the guard to `RefMut`. - let (guard, shard) = unsafe { RwLockWriteGuardDetached::detach_from(shard) }; - - if let Some((k, v)) = shard.find_mut(hash, |(k, _v)| key.equivalent(k)) { - TryResult::Present(RefMut::new(guard, k, v)) - } else { - TryResult::Absent - } - } - - fn _shrink_to_fit(&self) - where K: Hash { - self.shards.iter().for_each(|s| { - let mut shard = s.write(); - let size = shard.len(); - shard.shrink_to(size, |(k, _v)| { - let mut hasher = self.hasher.build_hasher(); - k.hash(&mut hasher); - hasher.finish() - }) - }); - } - - fn _retain(&self, mut f: impl FnMut(&K, &mut V) -> bool) { - self.shards.iter().for_each(|s| { - s.write().retain(|(k, v)| f(k, v)); - }); - } - - fn _len(&self) -> usize { self.shards.iter().map(|s| s.read().len()).sum() } - - fn _capacity(&self) -> usize { self.shards.iter().map(|s| s.read().capacity()).sum() } - - fn _alter(&self, key: &Q, f: impl FnOnce(&K, V) -> V) - where Q: Hash + Equivalent + ?Sized { - if let Some(mut r) = self.get_mut(key) { - util::map_in_place_2(r.pair_mut(), f); - } - } - - fn _alter_all(&self, mut f: impl FnMut(&K, V) -> V) { - self.iter_mut().for_each(|mut m| util::map_in_place_2(m.pair_mut(), &mut f)); - } - - fn _view(&self, key: &Q, f: impl FnOnce(&K, &V) -> R) -> Option - where Q: Hash + Equivalent + ?Sized { - self.get(key).map(|r| { - let (k, v) = r.pair(); - f(k, v) - }) - } - - fn _entry(&'a self, key: K) -> Entry<'a, K, V> - where K: Eq + Hash { - let hash = self.hash_u64(&key); - - let idx = self.determine_shard(hash as usize); - - let shard = self.shards[idx].write(); - // SAFETY: The data will not outlive the guard, since we pass the guard to `Entry`. - let (guard, shard) = unsafe { RwLockWriteGuardDetached::detach_from(shard) }; - - match shard.entry( - hash, - |(k, _v)| k == &key, - |(k, _v)| { - let mut hasher = self.hasher.build_hasher(); - k.hash(&mut hasher); - hasher.finish() - }, - ) { - hash_table::Entry::Occupied(entry) => { - Entry::Occupied(OccupiedEntry::new(guard, key, entry)) - } - hash_table::Entry::Vacant(entry) => Entry::Vacant(VacantEntry::new(guard, key, entry)), - } - } - - fn _try_entry(&'a self, key: K) -> Option> - where K: Eq + Hash { - let hash = self.hash_u64(&key); - - let idx = self.determine_shard(hash as usize); - - let shard = match self.shards[idx].try_write() { - Some(shard) => shard, - None => return None, - }; - // SAFETY: The data will not outlive the guard, since we pass the guard to `Entry`. - let (guard, shard) = unsafe { RwLockWriteGuardDetached::detach_from(shard) }; - - match shard.entry( - hash, - |(k, _v)| k == &key, - |(k, _v)| { - let mut hasher = self.hasher.build_hasher(); - k.hash(&mut hasher); - hasher.finish() - }, - ) { - hash_table::Entry::Occupied(entry) => { - Some(Entry::Occupied(OccupiedEntry::new(guard, key, entry))) - } - hash_table::Entry::Vacant(entry) => { - Some(Entry::Vacant(VacantEntry::new(guard, key, entry))) - } - } - } - - fn _entry_ref<'q, Q>(&'a self, key: &'q Q) -> EntryRef<'a, 'q, K, Q, V> - where - K: Eq + Hash, - Q: Hash + Equivalent, - { - let hash = self.hash_u64(&key); - - let idx = self.determine_shard(hash as usize); - - let shard = self.shards[idx].write(); - // SAFETY: The data will not outlive the guard, since we pass the guard to `Entry`. - let (guard, shard) = unsafe { RwLockWriteGuardDetached::detach_from(shard) }; - - match shard.entry( - hash, - |(k, _v)| key.equivalent(k), - |(k, _v)| { - let mut hasher = self.hasher.build_hasher(); - k.hash(&mut hasher); - hasher.finish() - }, - ) { - hash_table::Entry::Occupied(entry) => { - EntryRef::Occupied(OccupiedEntryRef::new(guard, key, entry)) - } - hash_table::Entry::Vacant(entry) => { - EntryRef::Vacant(VacantEntryRef::new(guard, key, entry)) - } - } - } - - fn _try_entry_ref<'q, Q>(&'a self, key: &'q Q) -> Option> - where - K: Eq + Hash, - Q: Hash + Equivalent, - { - let hash = self.hash_u64(&key); - - let idx = self.determine_shard(hash as usize); - - let shard = match self.shards[idx].try_write() { - Some(shard) => shard, - None => return None, - }; - // SAFETY: The data will not outlive the guard, since we pass the guard to `Entry`. - let (guard, shard) = unsafe { RwLockWriteGuardDetached::detach_from(shard) }; - - match shard.entry( - hash, - |(k, _v)| key.equivalent(k), - |(k, _v)| { - let mut hasher = self.hasher.build_hasher(); - k.hash(&mut hasher); - hasher.finish() - }, - ) { - hash_table::Entry::Occupied(entry) => { - Some(EntryRef::Occupied(OccupiedEntryRef::new(guard, key, entry))) - } - hash_table::Entry::Vacant(entry) => { - Some(EntryRef::Vacant(VacantEntryRef::new(guard, key, entry))) - } - } - } - - fn _clear(&self) { self._retain(|_, _| false) } - - fn _contains_key(&'a self, key: &Q) -> bool - where Q: Hash + Equivalent + ?Sized { - self._get(key).is_some() - } - - fn _is_empty(&self) -> bool { self._len() == 0 } -} - -impl fmt::Debug - for DashMap -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let mut pmap = f.debug_map(); - - for r in self { - let (k, v) = r.pair(); - - pmap.entry(k, v); - } - - pmap.finish() - } -} - -impl<'a, K: Eq + Hash, V, S: BuildHasher + Clone> Shl<(K, V)> for &'a DashMap { - type Output = Option; - - fn shl(self, pair: (K, V)) -> Self::Output { self.insert(pair.0, pair.1) } -} - -impl<'a, K: Eq + Hash, V, S: BuildHasher + Clone, Q> Shr<&Q> for &'a DashMap -where Q: Hash + Equivalent + ?Sized -{ - type Output = Ref<'a, K, V>; - - fn shr(self, key: &Q) -> Self::Output { self.get(key).unwrap() } -} - -impl<'a, K: Eq + Hash, V, S: BuildHasher + Clone, Q> BitOr<&Q> for &'a DashMap -where Q: Hash + Equivalent + ?Sized -{ - type Output = RefMut<'a, K, V>; - - fn bitor(self, key: &Q) -> Self::Output { self.get_mut(key).unwrap() } -} - -impl<'a, K: Eq + Hash, V, S: BuildHasher + Clone, Q> Sub<&Q> for &'a DashMap -where Q: Hash + Equivalent + ?Sized -{ - type Output = Option<(K, V)>; - - fn sub(self, key: &Q) -> Self::Output { self.remove(key) } -} - -impl<'a, K: Eq + Hash, V, S: BuildHasher + Clone, Q> BitAnd<&Q> for &'a DashMap -where Q: Hash + Equivalent + ?Sized -{ - type Output = bool; - - fn bitand(self, key: &Q) -> Self::Output { self.contains_key(key) } -} - -impl<'a, K: Eq + Hash, V: PartialEq, S: BuildHasher + Clone> PartialEq - for DashMap -{ - fn eq(&self, other: &Self) -> bool { - self.len() == other.len() - && self.iter().all(|r| other.get(r.key()).map_or(false, |ro| r.value() == ro.value())) - } -} - -impl<'a, K: Eq + Hash, V: Eq, S: BuildHasher + Clone> Eq for DashMap {} - -impl IntoIterator for DashMap { - type Item = (K, V); - - type IntoIter = OwningIter; - - fn into_iter(self) -> Self::IntoIter { OwningIter::new(self) } -} - -impl<'a, K: Eq + Hash, V, S: BuildHasher + Clone> IntoIterator for &'a DashMap { - type Item = RefMulti<'a, K, V>; - - type IntoIter = Iter<'a, K, V>; - - fn into_iter(self) -> Self::IntoIter { self.iter() } -} - -impl Extend<(K, V)> for DashMap { - fn extend>(&mut self, intoiter: I) { - for pair in intoiter.into_iter() { - self.insert(pair.0, pair.1); - } - } -} - -impl FromIterator<(K, V)> for DashMap { - fn from_iter>(intoiter: I) -> Self { - let mut map = DashMap::default(); - - map.extend(intoiter); - - map - } -} - -#[cfg(feature = "typesize")] -impl typesize::TypeSize for DashMap -where - K: typesize::TypeSize + Eq + Hash, - V: typesize::TypeSize, - S: typesize::TypeSize + Clone + BuildHasher, -{ - fn extra_size(&self) -> usize { - let shards_extra_size: usize = self - .shards - .iter() - .map(|shard_lock| { - core::mem::size_of::>>>() - + shard_lock.read().extra_size() - }) - .sum(); - - self.hasher.extra_size() + shards_extra_size - } - - typesize::if_typesize_details! { - fn get_collection_item_count(&self) -> Option { - Some(self.len()) - } - } -} - -#[cfg(test)] -mod tests { - use crate::DashMap; - use std::collections::hash_map::RandomState; - - #[test] - fn test_basic() { - let dm = DashMap::new(); - - dm.insert(0, 0); - - assert_eq!(dm.get(&0).unwrap().value(), &0); - } - - #[test] - fn test_default() { - let dm: DashMap = DashMap::default(); - - dm.insert(0, 0); - - assert_eq!(dm.get(&0).unwrap().value(), &0); - } - - #[test] - fn test_equal() { - let dm1 = DashMap::new(); - let dm2 = DashMap::new(); - assert_eq!(dm1, dm2); - - dm1.insert(0, "Hello, world!"); - assert_ne!(dm1, dm2); - - dm1.insert(1, "Goodbye, world!"); - assert_ne!(dm1, dm2); - - dm2.insert(0, "Hello, world!"); - assert_ne!(dm1, dm2); - - dm2.insert(1, "Goodbye, world!"); - assert_eq!(dm1, dm2); - } - - #[test] - fn test_multiple_hashes() { - let dm: DashMap = DashMap::default(); - - for i in 0..100 { - dm.insert(0, i); - - dm.insert(i, i); - } - - for i in 1..100 { - let r = dm.get(&i).unwrap(); - - assert_eq!(i, *r.value()); - - assert_eq!(i, *r.key()); - } - - let r = dm.get(&0).unwrap(); - - assert_eq!(99, *r.value()); - } - - #[test] - fn test_more_complex_values() { - #[derive(Hash, PartialEq, Debug, Clone)] - - struct T0 { - s: String, - u: u8, - } - - let dm = DashMap::new(); - - let range = 0..10; - - for i in range { - let t = T0 { s: i.to_string(), u: i as u8 }; - - dm.insert(i, t.clone()); - - assert_eq!(&t, dm.get(&i).unwrap().value()); - } - } - - #[test] - fn test_different_hashers_randomstate() { - let dm_hm_default: DashMap = - DashMap::with_hasher(RandomState::new()); - - for i in 0..10 { - dm_hm_default.insert(i, i); - - assert_eq!(i, *dm_hm_default.get(&i).unwrap().value()); - } - } - - #[test] - fn test_map_view() { - let dm = DashMap::new(); - - let vegetables: [String; 4] = - ["Salad".to_string(), "Beans".to_string(), "Potato".to_string(), "Tomato".to_string()]; - - // Give it some values - dm.insert(0, "Banana".to_string()); - dm.insert(4, "Pear".to_string()); - dm.insert(9, "Potato".to_string()); - dm.insert(12, "Chicken".to_string()); - - let potato_vegetableness = dm.view(&9, |_, v| vegetables.contains(v)); - assert_eq!(potato_vegetableness, Some(true)); - - let chicken_vegetableness = dm.view(&12, |_, v| vegetables.contains(v)); - assert_eq!(chicken_vegetableness, Some(false)); - - let not_in_map = dm.view(&30, |_k, _v| false); - assert_eq!(not_in_map, None); - } - - #[test] - fn test_try_get() { - { - let map = DashMap::new(); - map.insert("Johnny", 21); - - assert_eq!(*map.try_get("Johnny").unwrap(), 21); - - let _result1_locking = map.get_mut("Johnny"); - - let result2 = map.try_get("Johnny"); - assert!(result2.is_locked()); - } - - { - let map = DashMap::new(); - map.insert("Johnny", 21); - - *map.try_get_mut("Johnny").unwrap() += 1; - assert_eq!(*map.get("Johnny").unwrap(), 22); - - let _result1_locking = map.get("Johnny"); - - let result2 = map.try_get_mut("Johnny"); - assert!(result2.is_locked()); - } - } - - #[test] - fn test_try_reserve() { - let mut map: DashMap = DashMap::new(); - // DashMap is empty and doesn't allocate memory - assert_eq!(map.capacity(), 0); - - map.try_reserve(10).unwrap(); - - // And now map can hold at least 10 elements - assert!(map.capacity() >= 10); - } - - #[test] - fn test_try_reserve_errors() { - let mut map: DashMap = DashMap::new(); - - match map.try_reserve(usize::MAX) { - Err(_) => {} - _ => panic!("should have raised CapacityOverflow error"), - } - } -} diff --git a/patch/dashmap/src/lock.rs b/patch/dashmap/src/lock.rs deleted file mode 100644 index c762ac9..0000000 --- a/patch/dashmap/src/lock.rs +++ /dev/null @@ -1,300 +0,0 @@ -use core::sync::atomic::{AtomicUsize, Ordering}; -use parking_lot_core::{ParkToken, SpinWait, UnparkToken}; - -pub type RwLock = lock_api::RwLock; -pub(crate) type RwLockReadGuardDetached<'a> = crate::util::RwLockReadGuardDetached<'a, RawRwLock>; -pub(crate) type RwLockWriteGuardDetached<'a> = crate::util::RwLockWriteGuardDetached<'a, RawRwLock>; - -const READERS_PARKED: usize = 0b0001; -const WRITERS_PARKED: usize = 0b0010; -const ONE_READER: usize = 0b0100; -const ONE_WRITER: usize = !(READERS_PARKED | WRITERS_PARKED); - -pub struct RawRwLock { - state: AtomicUsize, -} - -unsafe impl lock_api::RawRwLock for RawRwLock { - #[allow(clippy::declare_interior_mutable_const)] - const INIT: Self = Self { - state: AtomicUsize::new(0), - }; - - type GuardMarker = lock_api::GuardSend; - - #[inline] - fn try_lock_exclusive(&self) -> bool { - self.state - .compare_exchange(0, ONE_WRITER, Ordering::Acquire, Ordering::Relaxed) - .is_ok() - } - - #[inline] - fn lock_exclusive(&self) { - if self - .state - .compare_exchange_weak(0, ONE_WRITER, Ordering::Acquire, Ordering::Relaxed) - .is_err() - { - self.lock_exclusive_slow(); - } - } - - #[inline] - unsafe fn unlock_exclusive(&self) { - if self - .state - .compare_exchange(ONE_WRITER, 0, Ordering::Release, Ordering::Relaxed) - .is_err() - { - self.unlock_exclusive_slow(); - } - } - - #[inline] - fn try_lock_shared(&self) -> bool { - self.try_lock_shared_fast() || self.try_lock_shared_slow() - } - - #[inline] - fn lock_shared(&self) { - if !self.try_lock_shared_fast() { - self.lock_shared_slow(); - } - } - - #[inline] - unsafe fn unlock_shared(&self) { - let state = self.state.fetch_sub(ONE_READER, Ordering::Release); - - if state == (ONE_READER | WRITERS_PARKED) { - self.unlock_shared_slow(); - } - } -} - -unsafe impl lock_api::RawRwLockDowngrade for RawRwLock { - #[inline] - unsafe fn downgrade(&self) { - let state = self - .state - .fetch_and(ONE_READER | WRITERS_PARKED, Ordering::Release); - if state & READERS_PARKED != 0 { - parking_lot_core::unpark_all((self as *const _ as usize) + 1, UnparkToken(0)); - } - } -} - -impl RawRwLock { - #[cold] - fn lock_exclusive_slow(&self) { - let mut acquire_with = 0; - loop { - let mut spin = SpinWait::new(); - let mut state = self.state.load(Ordering::Relaxed); - - loop { - while state & ONE_WRITER == 0 { - match self.state.compare_exchange_weak( - state, - state | ONE_WRITER | acquire_with, - Ordering::Acquire, - Ordering::Relaxed, - ) { - Ok(_) => return, - Err(e) => state = e, - } - } - - if state & WRITERS_PARKED == 0 { - if spin.spin() { - state = self.state.load(Ordering::Relaxed); - continue; - } - - if let Err(e) = self.state.compare_exchange_weak( - state, - state | WRITERS_PARKED, - Ordering::Relaxed, - Ordering::Relaxed, - ) { - state = e; - continue; - } - } - - let _ = unsafe { - parking_lot_core::park( - self as *const _ as usize, - || { - let state = self.state.load(Ordering::Relaxed); - (state & ONE_WRITER != 0) && (state & WRITERS_PARKED != 0) - }, - || {}, - |_, _| {}, - ParkToken(0), - None, - ) - }; - - acquire_with = WRITERS_PARKED; - break; - } - } - } - - #[cold] - fn unlock_exclusive_slow(&self) { - let state = self.state.load(Ordering::Relaxed); - assert_eq!(state & ONE_WRITER, ONE_WRITER); - - let mut parked = state & (READERS_PARKED | WRITERS_PARKED); - assert_ne!(parked, 0); - - if parked != (READERS_PARKED | WRITERS_PARKED) { - if let Err(new_state) = - self.state - .compare_exchange(state, 0, Ordering::Release, Ordering::Relaxed) - { - assert_eq!(new_state, ONE_WRITER | READERS_PARKED | WRITERS_PARKED); - parked = READERS_PARKED | WRITERS_PARKED; - } - } - - if parked == (READERS_PARKED | WRITERS_PARKED) { - self.state.store(WRITERS_PARKED, Ordering::Release); - parked = READERS_PARKED; - } - - if parked == READERS_PARKED { - return unsafe { - parking_lot_core::unpark_all((self as *const _ as usize) + 1, UnparkToken(0)); - }; - } - - assert_eq!(parked, WRITERS_PARKED); - unsafe { - parking_lot_core::unpark_one(self as *const _ as usize, |_| UnparkToken(0)); - } - } - - #[inline(always)] - fn try_lock_shared_fast(&self) -> bool { - let state = self.state.load(Ordering::Relaxed); - - if let Some(new_state) = state.checked_add(ONE_READER) { - if new_state & ONE_WRITER != ONE_WRITER { - return self - .state - .compare_exchange_weak(state, new_state, Ordering::Acquire, Ordering::Relaxed) - .is_ok(); - } - } - - false - } - - #[cold] - fn try_lock_shared_slow(&self) -> bool { - let mut state = self.state.load(Ordering::Relaxed); - - while let Some(new_state) = state.checked_add(ONE_READER) { - if new_state & ONE_WRITER == ONE_WRITER { - break; - } - - match self.state.compare_exchange_weak( - state, - new_state, - Ordering::Acquire, - Ordering::Relaxed, - ) { - Ok(_) => return true, - Err(e) => state = e, - } - } - - false - } - - #[cold] - fn lock_shared_slow(&self) { - loop { - let mut spin = SpinWait::new(); - let mut state = self.state.load(Ordering::Relaxed); - - loop { - let mut backoff = SpinWait::new(); - while let Some(new_state) = state.checked_add(ONE_READER) { - assert_ne!( - new_state & ONE_WRITER, - ONE_WRITER, - "reader count overflowed", - ); - - if self - .state - .compare_exchange_weak( - state, - new_state, - Ordering::Acquire, - Ordering::Relaxed, - ) - .is_ok() - { - return; - } - - backoff.spin_no_yield(); - state = self.state.load(Ordering::Relaxed); - } - - if state & READERS_PARKED == 0 { - if spin.spin() { - state = self.state.load(Ordering::Relaxed); - continue; - } - - if let Err(e) = self.state.compare_exchange_weak( - state, - state | READERS_PARKED, - Ordering::Relaxed, - Ordering::Relaxed, - ) { - state = e; - continue; - } - } - - let _ = unsafe { - parking_lot_core::park( - (self as *const _ as usize) + 1, - || { - let state = self.state.load(Ordering::Relaxed); - (state & ONE_WRITER == ONE_WRITER) && (state & READERS_PARKED != 0) - }, - || {}, - |_, _| {}, - ParkToken(0), - None, - ) - }; - - break; - } - } - } - - #[cold] - fn unlock_shared_slow(&self) { - if self - .state - .compare_exchange(WRITERS_PARKED, 0, Ordering::Relaxed, Ordering::Relaxed) - .is_ok() - { - unsafe { - parking_lot_core::unpark_one(self as *const _ as usize, |_| UnparkToken(0)); - } - } - } -} diff --git a/patch/dashmap/src/mapref/entry.rs b/patch/dashmap/src/mapref/entry.rs deleted file mode 100644 index 6edf542..0000000 --- a/patch/dashmap/src/mapref/entry.rs +++ /dev/null @@ -1,288 +0,0 @@ -use hashbrown::hash_table; - -use super::one::RefMut; -use crate::lock::RwLockWriteGuardDetached; -use core::hash::Hash; -use core::mem; - -pub enum Entry<'a, K, V> { - Occupied(OccupiedEntry<'a, K, V>), - Vacant(VacantEntry<'a, K, V>), -} - -impl<'a, K: Eq + Hash, V> Entry<'a, K, V> { - /// Apply a function to the stored value if it exists. - pub fn and_modify(self, f: impl FnOnce(&mut V)) -> Self { - match self { - Entry::Occupied(mut entry) => { - f(entry.get_mut()); - - Entry::Occupied(entry) - } - - Entry::Vacant(entry) => Entry::Vacant(entry), - } - } - - /// Get the key of the entry. - pub fn key(&self) -> &K { - match *self { - Entry::Occupied(ref entry) => entry.key(), - Entry::Vacant(ref entry) => entry.key(), - } - } - - /// Into the key of the entry. - pub fn into_key(self) -> K { - match self { - Entry::Occupied(entry) => entry.into_key(), - Entry::Vacant(entry) => entry.into_key(), - } - } - - /// Return a mutable reference to the element if it exists, - /// otherwise insert the default and return a mutable reference to that. - pub fn or_default(self) -> RefMut<'a, K, V> - where - V: Default, - { - match self { - Entry::Occupied(entry) => entry.into_ref(), - Entry::Vacant(entry) => entry.insert(V::default()), - } - } - - /// Return a mutable reference to the element if it exists, - /// otherwise a provided value and return a mutable reference to that. - pub fn or_insert(self, value: V) -> RefMut<'a, K, V> { - match self { - Entry::Occupied(entry) => entry.into_ref(), - Entry::Vacant(entry) => entry.insert(value), - } - } - - /// Return a mutable reference to the element if it exists, - /// otherwise insert the result of a provided function and return a mutable reference to that. - pub fn or_insert_with(self, value: impl FnOnce() -> V) -> RefMut<'a, K, V> { - match self { - Entry::Occupied(entry) => entry.into_ref(), - Entry::Vacant(entry) => entry.insert(value()), - } - } - - pub fn or_try_insert_with( - self, - value: impl FnOnce() -> Result, - ) -> Result, E> { - match self { - Entry::Occupied(entry) => Ok(entry.into_ref()), - Entry::Vacant(entry) => Ok(entry.insert(value()?)), - } - } - - /// Sets the value of the entry, and returns a reference to the inserted value. - pub fn insert(self, value: V) -> RefMut<'a, K, V> { - match self { - Entry::Occupied(mut entry) => { - entry.insert(value); - entry.into_ref() - } - Entry::Vacant(entry) => entry.insert(value), - } - } - - /// Sets the value of the entry, and returns an OccupiedEntry. - /// - /// If you are not interested in the occupied entry, - /// consider [`insert`] as it doesn't need to clone the key. - /// - /// [`insert`]: Entry::insert - pub fn insert_entry(self, value: V) -> OccupiedEntry<'a, K, V> - where - K: Clone, - { - match self { - Entry::Occupied(mut entry) => { - entry.insert(value); - entry - } - Entry::Vacant(entry) => entry.insert_entry(value), - } - } -} - -pub struct VacantEntry<'a, K, V> { - shard: RwLockWriteGuardDetached<'a>, - key: K, - entry: hash_table::VacantEntry<'a, (K, V)>, -} - -impl<'a, K: Eq + Hash, V> VacantEntry<'a, K, V> { - pub(crate) fn new( - shard: RwLockWriteGuardDetached<'a>, - key: K, - entry: hash_table::VacantEntry<'a, (K, V)>, - ) -> Self { - Self { shard, key, entry } - } - - pub fn insert(self, value: V) -> RefMut<'a, K, V> { - let occupied = self.entry.insert((self.key, value)); - - let (k, v) = occupied.into_mut(); - - RefMut::new(self.shard, k, v) - } - - /// Sets the value of the entry with the VacantEntry’s key, and returns an OccupiedEntry. - pub fn insert_entry(self, value: V) -> OccupiedEntry<'a, K, V> - where - K: Clone, - { - let entry = self.entry.insert((self.key.clone(), value)); - OccupiedEntry::new(self.shard, self.key, entry) - } - - pub fn into_key(self) -> K { - self.key - } - - pub fn key(&self) -> &K { - &self.key - } -} - -pub struct OccupiedEntry<'a, K, V> { - shard: RwLockWriteGuardDetached<'a>, - entry: hash_table::OccupiedEntry<'a, (K, V)>, - key: K, -} - -impl<'a, K: Eq + Hash, V> OccupiedEntry<'a, K, V> { - pub(crate) fn new( - shard: RwLockWriteGuardDetached<'a>, - key: K, - entry: hash_table::OccupiedEntry<'a, (K, V)>, - ) -> Self { - Self { shard, entry, key } - } - - pub fn get(&self) -> &V { - &self.entry.get().1 - } - - pub fn get_mut(&mut self) -> &mut V { - &mut self.entry.get_mut().1 - } - - pub fn insert(&mut self, value: V) -> V { - mem::replace(self.get_mut(), value) - } - - pub fn into_ref(self) -> RefMut<'a, K, V> { - let (k, v) = self.entry.into_mut(); - RefMut::new(self.shard, k, v) - } - - pub fn into_key(self) -> K { - self.key - } - - pub fn key(&self) -> &K { - &self.entry.get().0 - } - - pub fn remove(self) -> V { - let ((_k, v), _) = self.entry.remove(); - v - } - - pub fn remove_entry(self) -> (K, V) { - let ((k, v), _) = self.entry.remove(); - (k, v) - } - - pub fn replace_entry(self, value: V) -> (K, V) { - let (k, v) = mem::replace(self.entry.into_mut(), (self.key, value)); - (k, v) - } -} - -#[cfg(test)] -mod tests { - use crate::DashMap; - - use super::*; - - #[test] - fn test_insert_into_vacant() { - let map: DashMap = DashMap::new(); - - let entry = map.entry(1); - - assert!(matches!(entry, Entry::Vacant(_))); - - let val = entry.insert(2); - - assert_eq!(*val, 2); - - drop(val); - - assert_eq!(*map.get(&1).unwrap(), 2); - } - - #[test] - fn test_insert_into_occupied() { - let map: DashMap = DashMap::new(); - - map.insert(1, 1000); - - let entry = map.entry(1); - - assert!(matches!(&entry, Entry::Occupied(entry) if *entry.get() == 1000)); - - let val = entry.insert(2); - - assert_eq!(*val, 2); - - drop(val); - - assert_eq!(*map.get(&1).unwrap(), 2); - } - - #[test] - fn test_insert_entry_into_vacant() { - let map: DashMap = DashMap::new(); - - let entry = map.entry(1); - - assert!(matches!(entry, Entry::Vacant(_))); - - let entry = entry.insert_entry(2); - - assert_eq!(*entry.get(), 2); - - drop(entry); - - assert_eq!(*map.get(&1).unwrap(), 2); - } - - #[test] - fn test_insert_entry_into_occupied() { - let map: DashMap = DashMap::new(); - - map.insert(1, 1000); - - let entry = map.entry(1); - - assert!(matches!(&entry, Entry::Occupied(entry) if *entry.get() == 1000)); - - let entry = entry.insert_entry(2); - - assert_eq!(*entry.get(), 2); - - drop(entry); - - assert_eq!(*map.get(&1).unwrap(), 2); - } -} diff --git a/patch/dashmap/src/mapref/entry_ref.rs b/patch/dashmap/src/mapref/entry_ref.rs deleted file mode 100644 index da8e7c2..0000000 --- a/patch/dashmap/src/mapref/entry_ref.rs +++ /dev/null @@ -1,319 +0,0 @@ -use hashbrown::hash_table; - -use super::one::RefMut; -use crate::lock::RwLockWriteGuardDetached; -use core::hash::Hash; -use std::mem; - -/// Entry with a borrowed key. -pub enum EntryRef<'a, 'q, K, Q, V> { - Occupied(OccupiedEntryRef<'a, 'q, K, Q, V>), - Vacant(VacantEntryRef<'a, 'q, K, Q, V>), -} - -impl<'a, 'q, K: Eq + Hash, Q, V> EntryRef<'a, 'q, K, Q, V> { - /// Apply a function to the stored value if it exists. - pub fn and_modify(self, f: impl FnOnce(&mut V)) -> Self { - match self { - EntryRef::Occupied(mut entry) => { - f(entry.get_mut()); - - EntryRef::Occupied(entry) - } - - EntryRef::Vacant(entry) => EntryRef::Vacant(entry), - } - } -} - -impl<'a, 'q, K: Eq + Hash + From<&'q Q>, Q, V> EntryRef<'a, 'q, K, Q, V> { - /// Get the key of the entry. - pub fn key(&self) -> &Q { - match *self { - EntryRef::Occupied(ref entry) => entry.key(), - EntryRef::Vacant(ref entry) => entry.key(), - } - } - - /// Into the key of the entry. - pub fn into_key(self) -> K { - match self { - EntryRef::Occupied(entry) => entry.into_key(), - EntryRef::Vacant(entry) => entry.into_key(), - } - } - - /// Return a mutable reference to the element if it exists, - /// otherwise insert the default and return a mutable reference to that. - pub fn or_default(self) -> RefMut<'a, K, V> - where - V: Default, - { - match self { - EntryRef::Occupied(entry) => entry.into_ref(), - EntryRef::Vacant(entry) => entry.insert(V::default()), - } - } - - /// Return a mutable reference to the element if it exists, - /// otherwise a provided value and return a mutable reference to that. - pub fn or_insert(self, value: V) -> RefMut<'a, K, V> { - match self { - EntryRef::Occupied(entry) => entry.into_ref(), - EntryRef::Vacant(entry) => entry.insert(value), - } - } - - /// Return a mutable reference to the element if it exists, - /// otherwise insert the result of a provided function and return a mutable reference to that. - pub fn or_insert_with(self, value: impl FnOnce() -> V) -> RefMut<'a, K, V> { - match self { - EntryRef::Occupied(entry) => entry.into_ref(), - EntryRef::Vacant(entry) => entry.insert(value()), - } - } - - pub fn or_try_insert_with( - self, - value: impl FnOnce() -> Result, - ) -> Result, E> { - match self { - EntryRef::Occupied(entry) => Ok(entry.into_ref()), - EntryRef::Vacant(entry) => Ok(entry.insert(value()?)), - } - } - - /// Sets the value of the entry, and returns a reference to the inserted value. - pub fn insert(self, value: V) -> RefMut<'a, K, V> { - match self { - EntryRef::Occupied(mut entry) => { - entry.insert(value); - entry.into_ref() - } - EntryRef::Vacant(entry) => entry.insert(value), - } - } - - /// Sets the value of the entry, and returns an OccupiedEntryRef. - /// - /// If you are not interested in the occupied entry, - /// consider [`insert`] as it doesn't need to clone the key. - /// - /// [`insert`]: EntryRef::insert - pub fn insert_entry(self, value: V) -> OccupiedEntryRef<'a, 'q, K, Q, V> - where - K: Clone, - { - match self { - EntryRef::Occupied(mut entry) => { - entry.insert(value); - entry - } - EntryRef::Vacant(entry) => entry.insert_entry(value), - } - } -} - -pub struct VacantEntryRef<'a, 'q, K, Q, V> { - shard: RwLockWriteGuardDetached<'a>, - entry: hash_table::VacantEntry<'a, (K, V)>, - key: &'q Q, -} - -impl<'a, 'q, K: Eq + Hash, Q, V> VacantEntryRef<'a, 'q, K, Q, V> { - pub(crate) fn new( - shard: RwLockWriteGuardDetached<'a>, - key: &'q Q, - entry: hash_table::VacantEntry<'a, (K, V)>, - ) -> Self { - Self { shard, entry, key } - } - - pub fn insert(self, value: V) -> RefMut<'a, K, V> - where - K: From<&'q Q>, - { - let k = K::from(self.key); - let occupied = self.entry.insert((k, value)); - let (k, v) = occupied.into_mut(); - - RefMut::new(self.shard, k, v) - } - - /// Sets the value of the entry with the VacantEntryRef’s key, and returns an OccupiedEntry. - pub fn insert_entry(self, value: V) -> OccupiedEntryRef<'a, 'q, K, Q, V> - where - K: From<&'q Q>, - { - let k = K::from(self.key); - let entry = self.entry.insert((k, value)); - OccupiedEntryRef::new(self.shard, self.key, entry) - } - - pub fn into_key(self) -> K - where - K: From<&'q Q>, - { - K::from(self.key) - } - - pub fn key(&self) -> &'q Q { - self.key - } -} - -pub struct OccupiedEntryRef<'a, 'q, K, Q, V> { - shard: RwLockWriteGuardDetached<'a>, - entry: hash_table::OccupiedEntry<'a, (K, V)>, - key: &'q Q, -} - -impl<'a, 'q, K: Eq + Hash, Q, V> OccupiedEntryRef<'a, 'q, K, Q, V> { - pub(crate) fn new( - shard: RwLockWriteGuardDetached<'a>, - key: &'q Q, - entry: hash_table::OccupiedEntry<'a, (K, V)>, - ) -> Self { - Self { shard, entry, key } - } - - pub fn get(&self) -> &V { - &self.entry.get().1 - } - - pub fn get_mut(&mut self) -> &mut V { - &mut self.entry.get_mut().1 - } - - pub fn insert(&mut self, value: V) -> V { - mem::replace(self.get_mut(), value) - } - - pub fn into_ref(self) -> RefMut<'a, K, V> { - let (k, v) = self.entry.into_mut(); - RefMut::new(self.shard, k, v) - } - - pub fn into_key(self) -> K - where - K: From<&'q Q>, - { - K::from(self.key) - } - - pub fn key(&self) -> &'q Q { - self.key - } - - pub fn remove(self) -> V { - let ((_k, v), _) = self.entry.remove(); - v - } - - pub fn remove_entry(self) -> (K, V) { - let ((k, v), _) = self.entry.remove(); - (k, v) - } - - pub fn replace_entry(self, value: V) -> (K, V) - where - K: From<&'q Q>, - { - let (k, v) = mem::replace(self.entry.into_mut(), (K::from(self.key), value)); - (k, v) - } -} - -#[cfg(test)] -mod tests { - use equivalent::Equivalent; - - use crate::DashMap; - - use super::*; - - #[derive(Hash, PartialEq, Eq, Debug)] - struct K(u32); - impl From<&K> for u32 { - fn from(value: &K) -> Self { - value.0 - } - } - impl Equivalent for K { - fn equivalent(&self, key: &u32) -> bool { - self.0 == *key - } - } - - #[test] - fn test_insert_into_vacant() { - let map: DashMap = DashMap::new(); - - let entry = map.entry_ref(&K(1)); - - assert!(matches!(entry, EntryRef::Vacant(_))); - - let val = entry.insert(2); - - assert_eq!(*val, 2); - - drop(val); - - assert_eq!(*map.get(&1).unwrap(), 2); - } - - #[test] - fn test_insert_into_occupied() { - let map: DashMap = DashMap::new(); - - map.insert(1, 1000); - - let entry = map.entry_ref(&K(1)); - - assert!(matches!(&entry, EntryRef::Occupied(entry) if *entry.get() == 1000)); - - let val = entry.insert(2); - - assert_eq!(*val, 2); - - drop(val); - - assert_eq!(*map.get(&1).unwrap(), 2); - } - - #[test] - fn test_insert_entry_into_vacant() { - let map: DashMap = DashMap::new(); - - let entry = map.entry_ref(&K(1)); - - assert!(matches!(entry, EntryRef::Vacant(_))); - - let entry = entry.insert_entry(2); - - assert_eq!(*entry.get(), 2); - - drop(entry); - - assert_eq!(*map.get(&1).unwrap(), 2); - } - - #[test] - fn test_insert_entry_into_occupied() { - let map: DashMap = DashMap::new(); - - map.insert(1, 1000); - - let entry = map.entry_ref(&K(1)); - - assert!(matches!(&entry, EntryRef::Occupied(entry) if *entry.get() == 1000)); - - let entry = entry.insert_entry(2); - - assert_eq!(*entry.get(), 2); - - drop(entry); - - assert_eq!(*map.get(&1).unwrap(), 2); - } -} diff --git a/patch/dashmap/src/mapref/mod.rs b/patch/dashmap/src/mapref/mod.rs deleted file mode 100644 index de25262..0000000 --- a/patch/dashmap/src/mapref/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub mod entry; -pub mod entry_ref; -pub mod multiple; -pub mod one; -pub mod raw_entry; diff --git a/patch/dashmap/src/mapref/multiple.rs b/patch/dashmap/src/mapref/multiple.rs deleted file mode 100644 index 36b5708..0000000 --- a/patch/dashmap/src/mapref/multiple.rs +++ /dev/null @@ -1,90 +0,0 @@ -use crate::lock::{RwLockReadGuardDetached, RwLockWriteGuardDetached}; -use core::hash::Hash; -use core::ops::{Deref, DerefMut}; -use std::sync::Arc; - -pub struct RefMulti<'a, K, V> { - _guard: Arc>, - k: &'a K, - v: &'a V, -} - -impl<'a, K, V> RefMulti<'a, K, V> { - pub(crate) fn new(guard: Arc>, k: &'a K, v: &'a V) -> Self { - Self { - _guard: guard, - k, - v, - } - } - - pub fn key(&self) -> &K { - self.pair().0 - } - - pub fn value(&self) -> &V { - self.pair().1 - } - - pub fn pair(&self) -> (&K, &V) { - (self.k, self.v) - } -} - -impl<'a, K: Eq + Hash, V> Deref for RefMulti<'a, K, V> { - type Target = V; - - fn deref(&self) -> &V { - self.value() - } -} - -pub struct RefMutMulti<'a, K, V> { - _guard: Arc>, - k: &'a K, - v: &'a mut V, -} - -impl<'a, K, V> RefMutMulti<'a, K, V> { - pub(crate) fn new(guard: Arc>, k: &'a K, v: &'a mut V) -> Self { - Self { - _guard: guard, - k, - v, - } - } - - pub fn key(&self) -> &K { - self.pair().0 - } - - pub fn value(&self) -> &V { - self.pair().1 - } - - pub fn value_mut(&mut self) -> &mut V { - self.pair_mut().1 - } - - pub fn pair(&self) -> (&K, &V) { - (self.k, self.v) - } - - pub fn pair_mut(&mut self) -> (&K, &mut V) { - (self.k, self.v) - } -} - -impl<'a, K: Eq + Hash, V> Deref for RefMutMulti<'a, K, V> { - type Target = V; - - fn deref(&self) -> &V { - self.value() - } -} - -impl<'a, K: Eq + Hash, V> DerefMut for RefMutMulti<'a, K, V> { - fn deref_mut(&mut self) -> &mut V { - self.value_mut() - } -} diff --git a/patch/dashmap/src/mapref/one.rs b/patch/dashmap/src/mapref/one.rs deleted file mode 100644 index 8979b5f..0000000 --- a/patch/dashmap/src/mapref/one.rs +++ /dev/null @@ -1,386 +0,0 @@ -use crate::lock::{RwLockReadGuardDetached, RwLockWriteGuardDetached}; -use core::ops::{Deref, DerefMut}; -use std::fmt::{Debug, Formatter}; - -pub struct Ref<'a, K: ?Sized, V: ?Sized> { - _guard: RwLockReadGuardDetached<'a>, - k: &'a K, - v: &'a V, -} - -impl<'a, K: ?Sized, V: ?Sized> Ref<'a, K, V> { - pub(crate) fn new(guard: RwLockReadGuardDetached<'a>, k: &'a K, v: &'a V) -> Self { - Self { - _guard: guard, - k, - v, - } - } - - pub fn key(&self) -> &K { - self.pair().0 - } - - pub fn value(&self) -> &V { - self.pair().1 - } - - pub fn pair(&self) -> (&K, &V) { - (self.k, self.v) - } - - pub fn map(self, f: F) -> MappedRef<'a, K, T> - where - F: FnOnce(&V) -> &T, - { - MappedRef { - _guard: self._guard, - k: self.k, - v: f(self.v), - } - } - - pub fn try_map(self, f: F) -> Result, Self> - where - F: FnOnce(&V) -> Option<&T>, - { - if let Some(v) = f(self.v) { - Ok(MappedRef { - _guard: self._guard, - k: self.k, - v, - }) - } else { - Err(self) - } - } -} - -impl<'a, K: Debug + ?Sized, V: Debug + ?Sized> Debug for Ref<'a, K, V> { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Ref") - .field("k", &self.k) - .field("v", &self.v) - .finish() - } -} - -impl<'a, K: ?Sized, V: ?Sized> Deref for Ref<'a, K, V> { - type Target = V; - - fn deref(&self) -> &V { - self.value() - } -} - -pub struct RefMut<'a, K: ?Sized, V: ?Sized> { - guard: RwLockWriteGuardDetached<'a>, - k: &'a K, - v: &'a mut V, -} - -impl<'a, K: ?Sized, V: ?Sized> RefMut<'a, K, V> { - pub(crate) fn new(guard: RwLockWriteGuardDetached<'a>, k: &'a K, v: &'a mut V) -> Self { - Self { guard, k, v } - } - - pub fn key(&self) -> &K { - self.pair().0 - } - - pub fn value(&self) -> &V { - self.pair().1 - } - - pub fn value_mut(&mut self) -> &mut V { - self.pair_mut().1 - } - - pub fn pair(&self) -> (&K, &V) { - (self.k, self.v) - } - - pub fn pair_mut(&mut self) -> (&K, &mut V) { - (self.k, self.v) - } - - pub fn downgrade(self) -> Ref<'a, K, V> { - Ref::new( - unsafe { RwLockWriteGuardDetached::downgrade(self.guard) }, - self.k, - self.v, - ) - } - - pub fn map(self, f: F) -> MappedRefMut<'a, K, T> - where - F: FnOnce(&mut V) -> &mut T, - { - MappedRefMut { - _guard: self.guard, - k: self.k, - v: f(&mut *self.v), - } - } - - pub fn try_map(self, f: F) -> Result, Self> - where - F: FnOnce(&mut V) -> Option<&mut T>, - { - let v = match f(unsafe { &mut *(self.v as *mut _) }) { - Some(v) => v, - None => return Err(self), - }; - let guard = self.guard; - let k = self.k; - Ok(MappedRefMut { - _guard: guard, - k, - v, - }) - } -} - -impl<'a, K: Debug+ ?Sized, V: Debug + ?Sized> Debug for RefMut<'a, K, V> { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("RefMut") - .field("k", &self.k) - .field("v", &self.v) - .finish() - } -} - -impl<'a, K: ?Sized, V: ?Sized> Deref for RefMut<'a, K, V> { - type Target = V; - - fn deref(&self) -> &V { - self.value() - } -} - -impl<'a, K: ?Sized, V: ?Sized> DerefMut for RefMut<'a, K, V> { - fn deref_mut(&mut self) -> &mut V { - self.value_mut() - } -} - -pub struct MappedRef<'a, K: ?Sized, T: ?Sized> { - _guard: RwLockReadGuardDetached<'a>, - k: &'a K, - v: &'a T, -} - -impl<'a, K: ?Sized, T: ?Sized> MappedRef<'a, K, T> { - pub fn key(&self) -> &K { - self.pair().0 - } - - pub fn value(&self) -> &T { - self.pair().1 - } - - pub fn pair(&self) -> (&K, &T) { - (self.k, self.v) - } - - pub fn map(self, f: F) -> MappedRef<'a, K, T2> - where - F: FnOnce(&T) -> &T2, - { - MappedRef { - _guard: self._guard, - k: self.k, - v: f(self.v), - } - } - - pub fn try_map(self, f: F) -> Result, Self> - where - F: FnOnce(&T) -> Option<&T2>, - { - let v = match f(self.v) { - Some(v) => v, - None => return Err(self), - }; - let guard = self._guard; - Ok(MappedRef { - _guard: guard, - k: self.k, - v, - }) - } -} - -impl<'a, K: Debug + ?Sized, T: Debug + ?Sized> Debug for MappedRef<'a, K, T> { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("MappedRef") - .field("k", &self.k) - .field("v", &self.v) - .finish() - } -} - -impl<'a, K: ?Sized, T: ?Sized> Deref for MappedRef<'a, K, T> { - type Target = T; - - fn deref(&self) -> &T { - self.value() - } -} - -impl<'a, K: ?Sized, T: std::fmt::Display + ?Sized> std::fmt::Display for MappedRef<'a, K, T> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - std::fmt::Display::fmt(self.value(), f) - } -} - -impl<'a, K: ?Sized, T: ?Sized + AsRef, TDeref: ?Sized> AsRef - for MappedRef<'a, K, T> -{ - fn as_ref(&self) -> &TDeref { - self.value().as_ref() - } -} - -pub struct MappedRefMut<'a, K: ?Sized, T: ?Sized> { - _guard: RwLockWriteGuardDetached<'a>, - k: &'a K, - v: &'a mut T, -} - -impl<'a, K: ?Sized, T: ?Sized> MappedRefMut<'a, K, T> { - pub fn key(&self) -> &K { - self.pair().0 - } - - pub fn value(&self) -> &T { - self.pair().1 - } - - pub fn value_mut(&mut self) -> &mut T { - self.pair_mut().1 - } - - pub fn pair(&self) -> (&K, &T) { - (self.k, self.v) - } - - pub fn pair_mut(&mut self) -> (&K, &mut T) { - (self.k, self.v) - } - - pub fn map(self, f: F) -> MappedRefMut<'a, K, T2> - where - F: FnOnce(&mut T) -> &mut T2, - { - MappedRefMut { - _guard: self._guard, - k: self.k, - v: f(self.v), - } - } - - pub fn try_map(self, f: F) -> Result, Self> - where - F: FnOnce(&mut T) -> Option<&mut T2>, - { - let v = match f(unsafe { &mut *(self.v as *mut _) }) { - Some(v) => v, - None => return Err(self), - }; - let guard = self._guard; - let k = self.k; - Ok(MappedRefMut { - _guard: guard, - k, - v, - }) - } -} - -impl<'a, K: Debug + ?Sized, T: Debug + ?Sized> Debug for MappedRefMut<'a, K, T> { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("MappedRefMut") - .field("k", &self.k) - .field("v", &self.v) - .finish() - } -} - -impl<'a, K: ?Sized, T: ?Sized> Deref for MappedRefMut<'a, K, T> { - type Target = T; - - fn deref(&self) -> &T { - self.value() - } -} - -impl<'a, K: ?Sized, T: ?Sized> DerefMut for MappedRefMut<'a, K, T> { - fn deref_mut(&mut self) -> &mut T { - self.value_mut() - } -} - -#[cfg(test)] -mod tests { - use crate::DashMap; - - #[test] - fn downgrade() { - let data = DashMap::new(); - data.insert("test", "test"); - if let Some(mut w_ref) = data.get_mut("test") { - *w_ref.value_mut() = "test2"; - let r_ref = w_ref.downgrade(); - assert_eq!(*r_ref.value(), "test2"); - }; - } - - #[test] - fn mapped_mut() { - let data = DashMap::new(); - data.insert("test", *b"test"); - if let Some(b_ref) = data.get_mut("test") { - let mut s_ref = b_ref.try_map(|b| std::str::from_utf8_mut(b).ok()).unwrap(); - s_ref.value_mut().make_ascii_uppercase(); - } - - assert_eq!(data.get("test").unwrap().value(), b"TEST"); - } - - #[test] - fn mapped_mut_again() { - let data = DashMap::new(); - data.insert("test", *b"hello world"); - if let Some(b_ref) = data.get_mut("test") { - let s_ref = b_ref.try_map(|b| std::str::from_utf8_mut(b).ok()).unwrap(); - let mut hello_ref = s_ref.try_map(|s| s.get_mut(..5)).unwrap(); - hello_ref.value_mut().make_ascii_uppercase(); - } - - assert_eq!(data.get("test").unwrap().value(), b"HELLO world"); - } - - #[test] - fn mapped_ref() { - let data = DashMap::new(); - data.insert("test", *b"test"); - if let Some(b_ref) = data.get("test") { - let s_ref = b_ref.try_map(|b| std::str::from_utf8(b).ok()).unwrap(); - - assert_eq!(s_ref.value(), "test"); - }; - } - - #[test] - fn mapped_ref_again() { - let data = DashMap::new(); - data.insert("test", *b"hello world"); - if let Some(b_ref) = data.get("test") { - let s_ref = b_ref.try_map(|b| std::str::from_utf8(b).ok()).unwrap(); - let hello_ref = s_ref.try_map(|s| s.get(..5)).unwrap(); - - assert_eq!(hello_ref.value(), "hello"); - }; - } -} diff --git a/patch/dashmap/src/mapref/raw_entry.rs b/patch/dashmap/src/mapref/raw_entry.rs deleted file mode 100644 index d0132b7..0000000 --- a/patch/dashmap/src/mapref/raw_entry.rs +++ /dev/null @@ -1,373 +0,0 @@ -use crate::{ - DashMap, Equivalent, - lock::{RwLockReadGuardDetached, RwLockWriteGuardDetached}, - mapref::one::Ref, -}; -use core::{ - hash::{BuildHasher, Hash}, - mem, -}; -use hashbrown::hash_table; - -#[cfg_attr(feature = "inline-more", inline)] -pub(crate) fn make_hash(hash_builder: &S, val: &Q) -> u64 -where - Q: Hash + ?Sized, - S: BuildHasher, -{ - hash_builder.hash_one(val) -} - -#[cfg_attr(feature = "inline-more", inline)] -pub(crate) fn make_hasher(hash_builder: &S) -> impl Fn(&(Q, V)) -> u64 + '_ -where - Q: Hash, - S: BuildHasher, -{ - move |val| make_hash::(hash_builder, &val.0) -} - -impl DashMap { - #[cfg_attr(feature = "inline-more", inline)] - pub fn raw_entry_mut(&self) -> RawEntryBuilderMut<'_, K, V, S> { - RawEntryBuilderMut { map: self } - } - - #[cfg_attr(feature = "inline-more", inline)] - pub fn raw_entry(&self) -> RawEntryBuilder<'_, K, V, S> { RawEntryBuilder { map: self } } -} - -pub struct RefMut<'a, K: ?Sized, V: ?Sized> { - guard: RwLockWriteGuardDetached<'a>, - pub key: &'a mut K, - pub value: &'a mut V, -} - -impl<'a, K: ?Sized, V: ?Sized> RefMut<'a, K, V> { - pub(crate) fn new( - guard: RwLockWriteGuardDetached<'a>, - key: &'a mut K, - value: &'a mut V, - ) -> Self { - Self { guard, key, value } - } - - pub fn downgrade(self) -> Ref<'a, K, V> { - Ref::new(unsafe { RwLockWriteGuardDetached::downgrade(self.guard) }, self.key, self.value) - } -} - -impl<'a, K: core::fmt::Debug + ?Sized, V: core::fmt::Debug + ?Sized> core::fmt::Debug - for RefMut<'a, K, V> -{ - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.debug_struct("RefMut").field("key", &self.key).field("value", &self.value).finish() - } -} - -/// A builder for creating a raw entry in a `DashMap`. -pub struct RawEntryBuilderMut<'a, K, V, S> { - map: &'a DashMap, -} - -impl<'a, K, V, S> RawEntryBuilderMut<'a, K, V, S> { - /// Access an entry by key. - #[cfg_attr(feature = "inline-more", inline)] - pub fn from_key(self, k: &Q) -> RawEntryMut<'a, K, V, S> - where - S: BuildHasher, - K: Hash, - Q: Hash + Equivalent + ?Sized, - { - let hash = self.map.hash_u64(k); - self.from_key_hashed_nocheck(hash, k) - } - - /// Access an entry by a pre-computed hash and a key. - #[cfg_attr(feature = "inline-more", inline)] - pub fn from_key_hashed_nocheck(self, hash: u64, k: &Q) -> RawEntryMut<'a, K, V, S> - where - S: BuildHasher, - K: Hash, - Q: Equivalent + ?Sized, - { - self.from_hash(hash, |q| k.equivalent(q)) - } - - /// Access an entry by a pre-computed hash and a matching function. - pub fn from_hash(self, hash: u64, mut is_match: F) -> RawEntryMut<'a, K, V, S> - where - S: BuildHasher, - K: Hash, - F: FnMut(&K) -> bool, - { - let idx = self.map.determine_shard(hash as usize); - let shard_lock = self.map.shards[idx].write(); - - // SAFETY: The guard is stored in the returned RawEntryMut, ensuring the lock - // is held as long as the entry exists. - let (shard_guard, table) = unsafe { RwLockWriteGuardDetached::detach_from(shard_lock) }; - - match table.find_entry( - hash, - |(k, _)| is_match(k), - ) { - Ok(entry) => RawEntryMut::Occupied(RawOccupiedEntryMut { - shard: shard_guard, - entry, - }), - Err(entry) => RawEntryMut::Vacant(RawVacantEntryMut { - shard: shard_guard, - table: entry.into_table(), - hash_builder: &self.map.hasher, - }), - } - } -} - -/// A raw entry in the map. -pub enum RawEntryMut<'a, K, V, S> { - Occupied(RawOccupiedEntryMut<'a, K, V>), - Vacant(RawVacantEntryMut<'a, K, V, S>), -} - -impl<'a, K, V, S> RawEntryMut<'a, K, V, S> { - /// Sets the value of the entry, and returns an OccupiedEntry. - #[cfg_attr(feature = "inline-more", inline)] - pub fn insert(self, key: K, value: V) -> RawOccupiedEntryMut<'a, K, V> - where - K: Hash, - S: BuildHasher, - { - match self { - RawEntryMut::Occupied(mut entry) => { - entry.insert(value); - entry - } - RawEntryMut::Vacant(entry) => entry.insert_entry(key, value), - } - } - - #[cfg_attr(feature = "inline-more", inline)] - pub fn or_insert(self, default_key: K, default_val: V) -> RefMut<'a, K, V> - where - K: Hash, - S: BuildHasher, - { - match self { - RawEntryMut::Occupied(entry) => entry.into_key_value(), - RawEntryMut::Vacant(entry) => entry.insert(default_key, default_val), - } - } - - #[cfg_attr(feature = "inline-more", inline)] - pub fn or_insert_with(self, default: F) -> RefMut<'a, K, V> - where - F: FnOnce() -> (K, V), - K: Hash, - S: BuildHasher, - { - match self { - RawEntryMut::Occupied(entry) => entry.into_key_value(), - RawEntryMut::Vacant(entry) => { - let (k, v) = default(); - entry.insert(k, v) - } - } - } - - #[cfg_attr(feature = "inline-more", inline)] - pub fn and_modify(mut self, f: F) -> Self - where F: FnOnce(&mut K, &mut V) { - if let RawEntryMut::Occupied(entry) = &mut self { - let (k, v) = entry.get_key_value_mut(); - f(k, v); - } - self - } -} - -pub struct RawOccupiedEntryMut<'a, K, V> { - shard: RwLockWriteGuardDetached<'a>, - entry: hash_table::OccupiedEntry<'a, (K, V)>, - // hash_builder: &'a S, -} - -impl<'a, K, V> RawOccupiedEntryMut<'a, K, V> { - #[cfg_attr(feature = "inline-more", inline)] - pub fn key(&self) -> &K { &self.entry.get().0 } - - #[cfg_attr(feature = "inline-more", inline)] - pub fn key_mut(&mut self) -> &mut K { &mut self.entry.get_mut().0 } - - #[cfg_attr(feature = "inline-more", inline)] - pub fn into_key(self) -> &'a mut K { &mut self.entry.into_mut().0 } - - #[cfg_attr(feature = "inline-more", inline)] - pub fn get(&self) -> &V { &self.entry.get().1 } - - #[cfg_attr(feature = "inline-more", inline)] - pub fn into_mut(self) -> &'a mut V { &mut self.entry.into_mut().1 } - - #[cfg_attr(feature = "inline-more", inline)] - pub fn get_mut(&mut self) -> &mut V { &mut self.entry.get_mut().1 } - - #[cfg_attr(feature = "inline-more", inline)] - pub fn get_key_value(&self) -> (&K, &V) { - let (k, v) = self.entry.get(); - (k, v) - } - - #[cfg_attr(feature = "inline-more", inline)] - pub fn get_key_value_mut(&mut self) -> (&mut K, &mut V) { - let (k, v) = self.entry.get_mut(); - (k, v) - } - - #[cfg_attr(feature = "inline-more", inline)] - pub fn into_key_value(self) -> RefMut<'a, K, V> { - let (k, v) = self.entry.into_mut(); - RefMut::new(self.shard, k, v) - } - - #[cfg_attr(feature = "inline-more", inline)] - pub fn insert(&mut self, value: V) -> V { mem::replace(self.get_mut(), value) } - - #[cfg_attr(feature = "inline-more", inline)] - pub fn insert_key(&mut self, key: K) -> K { mem::replace(self.key_mut(), key) } - - #[cfg_attr(feature = "inline-more", inline)] - pub fn remove(self) -> V { self.remove_entry().1 } - - #[cfg_attr(feature = "inline-more", inline)] - pub fn remove_entry(self) -> (K, V) { self.entry.remove().0 } - - // #[cfg_attr(feature = "inline-more", inline)] - // pub fn replace_entry_with(self, f: F) -> RawEntryMut<'a, K, V, S> - // where F: FnOnce(&K, V) -> Option { - // let proxy: hashbrown::hash_map::RawOccupiedEntryMut<'a, K, V, S> = unsafe { - // let (bucket, table): ( - // core::ptr::NonNull<(K, V)>, - // &'a mut hash_table::HashTable<(K, V)>, - // ) = core::mem::transmute(self.entry); - // core::mem::transmute((bucket, table, self.hash_builder)) - // }; - // let result = proxy.replace_entry_with(f); - // match result { - // hashbrown::hash_map::RawEntryMut::Occupied(entry) => { - // let (bucket, table, hash_builder): ( - // core::ptr::NonNull<(K, V)>, - // &'a mut hash_table::HashTable<(K, V)>, - // &'a S, - // ) = unsafe { core::mem::transmute(entry) }; - // RawEntryMut::Occupied(RawOccupiedEntryMut { - // shard: self.shard, - // entry: unsafe { core::mem::transmute((bucket, table)) }, - // hash_builder, - // }) - // } - // hashbrown::hash_map::RawEntryMut::Vacant(entry) => { - // let (table, hash_builder) = unsafe { core::mem::transmute(entry) }; - // RawEntryMut::Vacant(RawVacantEntryMut { shard: self.shard, table, hash_builder }) - // } - // } - // } -} - -pub struct RawVacantEntryMut<'a, K, V, S> { - shard: RwLockWriteGuardDetached<'a>, - table: &'a mut hash_table::HashTable<(K, V)>, - hash_builder: &'a S, -} - -impl<'a, K, V, S> RawVacantEntryMut<'a, K, V, S> { - #[cfg_attr(feature = "inline-more", inline)] - pub fn insert(self, key: K, value: V) -> RefMut<'a, K, V> - where - K: Hash, - S: BuildHasher, - { - let hash = make_hash::(self.hash_builder, &key); - self.insert_hashed_nocheck(hash, key, value) - } - - #[cfg_attr(feature = "inline-more", inline)] - #[allow(clippy::shadow_unrelated)] - pub fn insert_hashed_nocheck(self, hash: u64, key: K, value: V) -> RefMut<'a, K, V> - where - K: Hash, - S: BuildHasher, - { - let &mut (ref mut k, ref mut v) = self - .table - .insert_unique(hash, (key, value), make_hasher::<_, V, S>(self.hash_builder)) - .into_mut(); - RefMut::new(self.shard, k, v) - } - - #[cfg_attr(feature = "inline-more", inline)] - pub fn insert_with_hasher( - self, - hash: u64, - key: K, - value: V, - hasher: H, - ) -> (&'a mut K, &'a mut V) - where - H: Fn(&K) -> u64, - { - let &mut (ref mut k, ref mut v) = - self.table.insert_unique(hash, (key, value), |x| hasher(&x.0)).into_mut(); - (k, v) - } - - /// Helper to match hashbrown behavior for API compatibility. - #[cfg_attr(feature = "inline-more", inline)] - pub fn insert_entry(self, key: K, value: V) -> RawOccupiedEntryMut<'a, K, V> - where - K: Hash, - S: BuildHasher, - { - let hash = make_hash::(self.hash_builder, &key); - let entry = - self.table.insert_unique(hash, (key, value), make_hasher::<_, V, S>(self.hash_builder)); - RawOccupiedEntryMut { shard: self.shard, entry } - } -} - -pub struct RawEntryBuilder<'a, K, V, S> { - map: &'a DashMap, -} - -impl<'a, K, V, S> RawEntryBuilder<'a, K, V, S> { - #[cfg_attr(feature = "inline-more", inline)] - pub fn from_key(self, k: &Q) -> Option> - where - S: BuildHasher, - K: Hash, - Q: Hash + Equivalent + ?Sized, - { - let hash = self.map.hash_u64(k); - self.from_key_hashed_nocheck(hash, k) - } - - #[cfg_attr(feature = "inline-more", inline)] - pub fn from_key_hashed_nocheck(self, hash: u64, k: &Q) -> Option> - where Q: Equivalent + ?Sized { - self.from_hash(hash, |q| k.equivalent(q)) - } - - pub fn from_hash(self, hash: u64, mut is_match: F) -> Option> - where F: FnMut(&K) -> bool { - let idx = self.map.determine_shard(hash as usize); - let shard_lock = self.map.shards[idx].read(); - - // SAFETY: Detach guard to return Ref which holds the lock. - let (shard_guard, table) = unsafe { RwLockReadGuardDetached::detach_from(shard_lock) }; - - match table.find(hash, |(k, _)| is_match(k)) { - Some((k, v)) => Some(Ref::new(shard_guard, k, v)), - None => None, - } - } -} diff --git a/patch/dashmap/src/rayon/map.rs b/patch/dashmap/src/rayon/map.rs deleted file mode 100644 index c90fbb3..0000000 --- a/patch/dashmap/src/rayon/map.rs +++ /dev/null @@ -1,212 +0,0 @@ -use crate::lock::{RwLock, RwLockReadGuardDetached, RwLockWriteGuardDetached}; -use crate::mapref::multiple::{RefMulti, RefMutMulti}; -use crate::{DashMap, HashMap}; -use core::hash::{BuildHasher, Hash}; -use crossbeam_utils::CachePadded; -use rayon::iter::plumbing::UnindexedConsumer; -use rayon::iter::{FromParallelIterator, IntoParallelIterator, ParallelExtend, ParallelIterator}; -use std::sync::Arc; - -impl ParallelExtend<(K, V)> for DashMap -where - K: Send + Sync + Eq + Hash, - V: Send + Sync, - S: Send + Sync + Clone + BuildHasher, -{ - fn par_extend(&mut self, par_iter: I) - where - I: IntoParallelIterator, - { - (&*self).par_extend(par_iter); - } -} - -// Since we don't actually need mutability, we can implement this on a -// reference, similar to `io::Write for &File`. -impl ParallelExtend<(K, V)> for &'_ DashMap -where - K: Send + Sync + Eq + Hash, - V: Send + Sync, - S: Send + Sync + Clone + BuildHasher, -{ - fn par_extend(&mut self, par_iter: I) - where - I: IntoParallelIterator, - { - let &mut map = self; - par_iter.into_par_iter().for_each(move |(key, value)| { - map.insert(key, value); - }); - } -} - -impl FromParallelIterator<(K, V)> for DashMap -where - K: Send + Sync + Eq + Hash, - V: Send + Sync, - S: Send + Sync + Clone + Default + BuildHasher, -{ - fn from_par_iter(par_iter: I) -> Self - where - I: IntoParallelIterator, - { - let map = Self::default(); - (&map).par_extend(par_iter); - map - } -} - -// Implementation note: while the shards will iterate in parallel, we flatten -// sequentially within each shard (`flat_map_iter`), because the standard -// `HashMap` only implements `ParallelIterator` by collecting to a `Vec` first. -// There is real parallel support in the `hashbrown/rayon` feature, but we don't -// always use that map. - -impl IntoParallelIterator for DashMap -where - K: Send + Eq + Hash, - V: Send, - S: Send + Clone + BuildHasher, -{ - type Iter = OwningIter; - type Item = (K, V); - - fn into_par_iter(self) -> Self::Iter { - OwningIter { - shards: self.shards, - } - } -} - -pub struct OwningIter { - pub(super) shards: Box<[CachePadded>>]>, -} - -impl ParallelIterator for OwningIter -where - K: Send + Eq + Hash, - V: Send, -{ - type Item = (K, V); - - fn drive_unindexed(self, consumer: C) -> C::Result - where - C: UnindexedConsumer, - { - Vec::from(self.shards) - .into_par_iter() - .flat_map_iter(|shard| shard.into_inner().into_inner().into_iter()) - .drive_unindexed(consumer) - } -} - -// This impl also enables `IntoParallelRefIterator::par_iter` -impl<'a, K, V, S> IntoParallelIterator for &'a DashMap -where - K: Send + Sync + Eq + Hash, - V: Send + Sync, - S: Send + Sync + Clone + BuildHasher, -{ - type Iter = Iter<'a, K, V>; - type Item = RefMulti<'a, K, V>; - - fn into_par_iter(self) -> Self::Iter { - Iter { - shards: &self.shards, - } - } -} - -pub struct Iter<'a, K, V> { - pub(super) shards: &'a [CachePadded>>], -} - -impl<'a, K, V> ParallelIterator for Iter<'a, K, V> -where - K: Send + Sync + Eq + Hash, - V: Send + Sync, -{ - type Item = RefMulti<'a, K, V>; - - fn drive_unindexed(self, consumer: C) -> C::Result - where - C: UnindexedConsumer, - { - self.shards - .into_par_iter() - .flat_map_iter(|shard| { - // SAFETY: we keep the guard alive with the shard iterator, - // and with any refs produced by the iterator - let (guard, shard) = unsafe { RwLockReadGuardDetached::detach_from(shard.read()) }; - - let guard = Arc::new(guard); - shard.iter().map(move |(k, v)| { - let guard = Arc::clone(&guard); - RefMulti::new(guard, k, v) - }) - }) - .drive_unindexed(consumer) - } -} - -// This impl also enables `IntoParallelRefMutIterator::par_iter_mut` -impl<'a, K, V> IntoParallelIterator for &'a mut DashMap -where - K: Send + Sync + Eq + Hash, - V: Send + Sync, -{ - type Iter = IterMut<'a, K, V>; - type Item = RefMutMulti<'a, K, V>; - - fn into_par_iter(self) -> Self::Iter { - IterMut { - shards: &self.shards, - } - } -} - -impl DashMap -where - K: Send + Sync + Eq + Hash, - V: Send + Sync, -{ - // Unlike `IntoParallelRefMutIterator::par_iter_mut`, we only _need_ `&self`. - pub fn par_iter_mut(&self) -> IterMut<'_, K, V> { - IterMut { - shards: &self.shards, - } - } -} - -pub struct IterMut<'a, K, V> { - shards: &'a [CachePadded>>], -} - -impl<'a, K, V> ParallelIterator for IterMut<'a, K, V> -where - K: Send + Sync + Eq + Hash, - V: Send + Sync, -{ - type Item = RefMutMulti<'a, K, V>; - - fn drive_unindexed(self, consumer: C) -> C::Result - where - C: UnindexedConsumer, - { - self.shards - .into_par_iter() - .flat_map_iter(|shard| { - // SAFETY: we keep the guard alive with the shard iterator, - // and with any refs produced by the iterator - let (guard, shard) = - unsafe { RwLockWriteGuardDetached::detach_from(shard.write()) }; - - let guard = Arc::new(guard); - shard.iter_mut().map(move |(k, v)| { - let guard = Arc::clone(&guard); - RefMutMulti::new(guard, k, v) - }) - }) - .drive_unindexed(consumer) - } -} diff --git a/patch/dashmap/src/rayon/read_only.rs b/patch/dashmap/src/rayon/read_only.rs deleted file mode 100644 index d37bb87..0000000 --- a/patch/dashmap/src/rayon/read_only.rs +++ /dev/null @@ -1,96 +0,0 @@ -use crate::mapref::multiple::RefMulti; -use crate::rayon::map::Iter; -use crate::ReadOnlyView; -use core::hash::{BuildHasher, Hash}; -use rayon::iter::IntoParallelIterator; - -impl IntoParallelIterator for ReadOnlyView -where - K: Send + Eq + Hash, - V: Send, - S: Send + Clone + BuildHasher, -{ - type Iter = super::map::OwningIter; - type Item = (K, V); - - fn into_par_iter(self) -> Self::Iter { - super::map::OwningIter { - shards: self.map.shards, - } - } -} - -// This impl also enables `IntoParallelRefIterator::par_iter` -impl<'a, K, V, S> IntoParallelIterator for &'a ReadOnlyView -where - K: Send + Sync + Eq + Hash, - V: Send + Sync, - S: Send + Sync + Clone + BuildHasher, -{ - type Iter = Iter<'a, K, V>; - type Item = RefMulti<'a, K, V>; - - fn into_par_iter(self) -> Self::Iter { - Iter { - shards: &self.map.shards, - } - } -} - -#[cfg(test)] -mod tests { - use crate::DashMap; - use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}; - - fn construct_sample_map() -> DashMap { - let map = DashMap::new(); - - map.insert(1, "one".to_string()); - - map.insert(10, "ten".to_string()); - - map.insert(27, "twenty seven".to_string()); - - map.insert(45, "forty five".to_string()); - - map - } - - #[test] - fn test_par_iter() { - let map = construct_sample_map(); - - let view = map.clone().into_read_only(); - - view.par_iter().for_each(|entry| { - let key = *entry.key(); - - assert!(view.contains_key(&key)); - - let map_entry = map.get(&key).unwrap(); - - assert_eq!(view.get(&key).unwrap(), map_entry.value()); - - let key_value: (&i32, &String) = view.get_key_value(&key).unwrap(); - - assert_eq!(key_value.0, map_entry.key()); - - assert_eq!(key_value.1, map_entry.value()); - }); - } - - #[test] - fn test_into_par_iter() { - let map = construct_sample_map(); - - let view = map.clone().into_read_only(); - - view.into_par_iter().for_each(|(key, value)| { - let map_entry = map.get(&key).unwrap(); - - assert_eq!(&key, map_entry.key()); - - assert_eq!(&value, map_entry.value()); - }); - } -} diff --git a/patch/dashmap/src/rayon/set.rs b/patch/dashmap/src/rayon/set.rs deleted file mode 100644 index c92e2cd..0000000 --- a/patch/dashmap/src/rayon/set.rs +++ /dev/null @@ -1,118 +0,0 @@ -use crate::setref::multiple::RefMulti; -use crate::DashSet; -use core::hash::{BuildHasher, Hash}; -use rayon::iter::plumbing::UnindexedConsumer; -use rayon::iter::{FromParallelIterator, IntoParallelIterator, ParallelExtend, ParallelIterator}; - -impl ParallelExtend for DashSet -where - K: Send + Sync + Eq + Hash, - S: Send + Sync + Clone + BuildHasher, -{ - fn par_extend(&mut self, par_iter: I) - where - I: IntoParallelIterator, - { - (&*self).par_extend(par_iter); - } -} - -// Since we don't actually need mutability, we can implement this on a -// reference, similar to `io::Write for &File`. -impl ParallelExtend for &'_ DashSet -where - K: Send + Sync + Eq + Hash, - S: Send + Sync + Clone + BuildHasher, -{ - fn par_extend(&mut self, par_iter: I) - where - I: IntoParallelIterator, - { - let &mut set = self; - par_iter.into_par_iter().for_each(move |key| { - set.insert(key); - }); - } -} - -impl FromParallelIterator for DashSet -where - K: Send + Sync + Eq + Hash, - S: Send + Sync + Clone + Default + BuildHasher, -{ - fn from_par_iter(par_iter: I) -> Self - where - I: IntoParallelIterator, - { - let set = Self::default(); - (&set).par_extend(par_iter); - set - } -} - -impl IntoParallelIterator for DashSet -where - K: Send + Eq + Hash, - S: Send + Clone + BuildHasher, -{ - type Iter = OwningIter; - type Item = K; - - fn into_par_iter(self) -> Self::Iter { - OwningIter { - inner: self.inner.into_par_iter(), - } - } -} - -pub struct OwningIter { - inner: super::map::OwningIter, -} - -impl ParallelIterator for OwningIter -where - K: Send + Eq + Hash, -{ - type Item = K; - - fn drive_unindexed(self, consumer: C) -> C::Result - where - C: UnindexedConsumer, - { - self.inner.map(|(k, _)| k).drive_unindexed(consumer) - } -} - -// This impl also enables `IntoParallelRefIterator::par_iter` -impl<'a, K, S> IntoParallelIterator for &'a DashSet -where - K: Send + Sync + Eq + Hash, - S: Send + Sync + Clone + BuildHasher, -{ - type Iter = Iter<'a, K>; - type Item = RefMulti<'a, K>; - - fn into_par_iter(self) -> Self::Iter { - Iter { - inner: (&self.inner).into_par_iter(), - } - } -} - -pub struct Iter<'a, K> { - inner: super::map::Iter<'a, K, ()>, -} - -impl<'a, K> ParallelIterator for Iter<'a, K> -where - K: Send + Sync + Eq + Hash, -{ - type Item = RefMulti<'a, K>; - - fn drive_unindexed(self, consumer: C) -> C::Result - where - C: UnindexedConsumer, - { - self.inner.map(RefMulti::new).drive_unindexed(consumer) - } -} diff --git a/patch/dashmap/src/read_only.rs b/patch/dashmap/src/read_only.rs deleted file mode 100644 index f31af48..0000000 --- a/patch/dashmap/src/read_only.rs +++ /dev/null @@ -1,255 +0,0 @@ -use crate::lock::RwLock; -use crate::{DashMap, HashMap}; -use cfg_if::cfg_if; -use core::fmt; -use core::hash::{BuildHasher, Hash}; -use crossbeam_utils::CachePadded; -use equivalent::Equivalent; -use std::collections::hash_map::RandomState; - -/// A read-only view into a `DashMap`. Allows to obtain raw references to the stored values. -pub struct ReadOnlyView { - pub(crate) map: DashMap, -} - -impl Clone for ReadOnlyView { - fn clone(&self) -> Self { - Self { - map: self.map.clone(), - } - } -} - -impl fmt::Debug - for ReadOnlyView -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.map.fmt(f) - } -} - -impl ReadOnlyView { - pub(crate) fn new(map: DashMap) -> Self { - Self { map } - } - - /// Consumes this `ReadOnlyView`, returning the underlying `DashMap`. - pub fn into_inner(self) -> DashMap { - self.map - } -} - -impl<'a, K: 'a + Eq + Hash, V: 'a, S: BuildHasher + Clone> ReadOnlyView { - /// Returns the number of elements in the map. - pub fn len(&self) -> usize { - self.map.len() - } - - /// Returns `true` if the map contains no elements. - pub fn is_empty(&self) -> bool { - self.map.is_empty() - } - - /// Returns the number of elements the map can hold without reallocating. - pub fn capacity(&self) -> usize { - self.map.capacity() - } - - /// Returns `true` if the map contains a value for the specified key. - pub fn contains_key(&'a self, key: &Q) -> bool - where - Q: Hash + Equivalent + ?Sized, - { - self.get(key).is_some() - } - - /// Returns a reference to the value corresponding to the key. - pub fn get(&'a self, key: &Q) -> Option<&'a V> - where - Q: Hash + Equivalent + ?Sized, - { - self.get_key_value(key).map(|(_k, v)| v) - } - - /// Returns the key-value pair corresponding to the supplied key. - pub fn get_key_value(&'a self, key: &Q) -> Option<(&'a K, &'a V)> - where - Q: Hash + Equivalent + ?Sized, - { - let hash = self.map.hash_u64(&key); - - let idx = self.map.determine_shard(hash as usize); - - let shard = &self.map.shards[idx]; - let shard = unsafe { &*shard.data_ptr() }; - - shard - .find(hash, |(k, _v)| key.equivalent(k)) - .map(|(k, v)| (k, v)) - } - - /// An iterator visiting all key-value pairs in arbitrary order. The iterator element type is `(&'a K, &'a V)`. - pub fn iter(&'a self) -> impl Iterator + 'a { - self.map - .shards - .iter() - .map(|shard| unsafe { &*shard.data_ptr() }) - .flat_map(|shard| shard.iter()) - .map(|(k, v)| (k, v)) - } - - /// An iterator visiting all keys in arbitrary order. The iterator element type is `&'a K`. - pub fn keys(&'a self) -> impl Iterator + 'a { - self.iter().map(|(k, _v)| k) - } - - /// An iterator visiting all values in arbitrary order. The iterator element type is `&'a V`. - pub fn values(&'a self) -> impl Iterator + 'a { - self.iter().map(|(_k, v)| v) - } - - cfg_if! { - if #[cfg(feature = "raw-api")] { - /// Allows you to peek at the inner shards that store your data. - /// You should probably not use this unless you know what you are doing. - /// - /// Requires the `raw-api` feature to be enabled. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashMap; - /// - /// let map = DashMap::<(), ()>::new().into_read_only(); - /// println!("Amount of shards: {}", map.shards().len()); - /// ``` - pub fn shards(&self) -> &[CachePadded>>] { - &self.map.shards - } - } else { - #[allow(dead_code)] - pub(crate) fn shards(&self) -> &[CachePadded>>] { - &self.map.shards - } - } - } -} - -#[cfg(test)] - -mod tests { - - use crate::DashMap; - - fn construct_sample_map() -> DashMap { - let map = DashMap::new(); - - map.insert(1, "one".to_string()); - - map.insert(10, "ten".to_string()); - - map.insert(27, "twenty seven".to_string()); - - map.insert(45, "forty five".to_string()); - - map - } - - #[test] - - fn test_properties() { - let map = construct_sample_map(); - - let view = map.clone().into_read_only(); - - assert_eq!(view.is_empty(), map.is_empty()); - - assert_eq!(view.len(), map.len()); - - assert_eq!(view.capacity(), map.capacity()); - - let new_map = view.into_inner(); - - assert_eq!(new_map.is_empty(), map.is_empty()); - - assert_eq!(new_map.len(), map.len()); - - assert_eq!(new_map.capacity(), map.capacity()); - } - - #[test] - - fn test_get() { - let map = construct_sample_map(); - - let view = map.clone().into_read_only(); - - for key in map.iter().map(|entry| *entry.key()) { - assert!(view.contains_key(&key)); - - let map_entry = map.get(&key).unwrap(); - - assert_eq!(view.get(&key).unwrap(), map_entry.value()); - - let key_value: (&i32, &String) = view.get_key_value(&key).unwrap(); - - assert_eq!(key_value.0, map_entry.key()); - - assert_eq!(key_value.1, map_entry.value()); - } - } - - #[test] - - fn test_iters() { - let map = construct_sample_map(); - - let view = map.clone().into_read_only(); - - let mut visited_items = Vec::new(); - - for (key, value) in view.iter() { - map.contains_key(key); - - let map_entry = map.get(key).unwrap(); - - assert_eq!(key, map_entry.key()); - - assert_eq!(value, map_entry.value()); - - visited_items.push((key, value)); - } - - let mut visited_keys = Vec::new(); - - for key in view.keys() { - map.contains_key(key); - - let map_entry = map.get(key).unwrap(); - - assert_eq!(key, map_entry.key()); - - assert_eq!(view.get(key).unwrap(), map_entry.value()); - - visited_keys.push(key); - } - - let mut visited_values = Vec::new(); - - for value in view.values() { - visited_values.push(value); - } - - for entry in map.iter() { - let key = entry.key(); - - let value = entry.value(); - - assert!(visited_keys.contains(&key)); - - assert!(visited_values.contains(&value)); - - assert!(visited_items.contains(&(key, value))); - } - } -} diff --git a/patch/dashmap/src/serde.rs b/patch/dashmap/src/serde.rs deleted file mode 100644 index 9e7d41c..0000000 --- a/patch/dashmap/src/serde.rs +++ /dev/null @@ -1,203 +0,0 @@ -use crate::{mapref, setref, DashMap, DashSet}; -use core::fmt; -use core::hash::{BuildHasher, Hash}; -use core::marker::PhantomData; -use serde::de::{Deserialize, MapAccess, SeqAccess, Visitor}; -use serde::ser::{Serialize, SerializeMap, SerializeSeq, Serializer}; -use serde::Deserializer; - -pub struct DashMapVisitor { - marker: PhantomData DashMap>, -} - -impl DashMapVisitor -where - K: Eq + Hash, - S: BuildHasher + Clone, -{ - fn new() -> Self { - DashMapVisitor { - marker: PhantomData, - } - } -} - -impl<'de, K, V, S> Visitor<'de> for DashMapVisitor -where - K: Deserialize<'de> + Eq + Hash, - V: Deserialize<'de>, - S: BuildHasher + Clone + Default, -{ - type Value = DashMap; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("a DashMap") - } - - fn visit_map(self, mut access: M) -> Result - where - M: MapAccess<'de>, - { - let map = - DashMap::with_capacity_and_hasher(access.size_hint().unwrap_or(0), Default::default()); - - while let Some((key, value)) = access.next_entry()? { - map.insert(key, value); - } - - Ok(map) - } -} - -impl<'de, K, V, S> Deserialize<'de> for DashMap -where - K: Deserialize<'de> + Eq + Hash, - V: Deserialize<'de>, - S: BuildHasher + Clone + Default, -{ - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - deserializer.deserialize_map(DashMapVisitor::::new()) - } -} - -impl Serialize for DashMap -where - K: Serialize + Eq + Hash, - V: Serialize, - H: BuildHasher + Clone, -{ - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let mut map = serializer.serialize_map(Some(self.len()))?; - - for ref_multi in self.iter() { - map.serialize_entry(ref_multi.key(), ref_multi.value())?; - } - - map.end() - } -} - -pub struct DashSetVisitor { - marker: PhantomData DashSet>, -} - -impl DashSetVisitor -where - K: Eq + Hash, - S: BuildHasher + Clone, -{ - fn new() -> Self { - DashSetVisitor { - marker: PhantomData, - } - } -} - -impl<'de, K, S> Visitor<'de> for DashSetVisitor -where - K: Deserialize<'de> + Eq + Hash, - S: BuildHasher + Clone + Default, -{ - type Value = DashSet; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("a DashSet") - } - - fn visit_seq(self, mut access: M) -> Result - where - M: SeqAccess<'de>, - { - let map = - DashSet::with_capacity_and_hasher(access.size_hint().unwrap_or(0), Default::default()); - - while let Some(key) = access.next_element()? { - map.insert(key); - } - - Ok(map) - } -} - -impl<'de, K, S> Deserialize<'de> for DashSet -where - K: Deserialize<'de> + Eq + Hash, - S: BuildHasher + Clone + Default, -{ - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - deserializer.deserialize_seq(DashSetVisitor::::new()) - } -} - -impl Serialize for DashSet -where - K: Serialize + Eq + Hash, - H: BuildHasher + Clone, -{ - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let mut seq = serializer.serialize_seq(Some(self.len()))?; - - for ref_multi in self.iter() { - seq.serialize_element(ref_multi.key())?; - } - - seq.end() - } -} - -macro_rules! serialize_impl { - () => { - fn serialize(&self, serializer: Ser) -> Result - where - Ser: serde::Serializer, - { - std::ops::Deref::deref(self).serialize(serializer) - } - }; -} - -// Map -impl<'a, K: Eq + Hash, V: Serialize> Serialize for mapref::multiple::RefMulti<'a, K, V> { - serialize_impl! {} -} - -impl<'a, K: Eq + Hash, V: Serialize> Serialize for mapref::multiple::RefMutMulti<'a, K, V> { - serialize_impl! {} -} - -impl<'a, K: Eq + Hash, V: Serialize> Serialize for mapref::one::Ref<'a, K, V> { - serialize_impl! {} -} - -impl<'a, K: Eq + Hash, V: Serialize> Serialize for mapref::one::RefMut<'a, K, V> { - serialize_impl! {} -} - -impl<'a, K: Eq + Hash, T: Serialize> Serialize for mapref::one::MappedRef<'a, K, T> { - serialize_impl! {} -} - -impl<'a, K: Eq + Hash, T: Serialize> Serialize for mapref::one::MappedRefMut<'a, K, T> { - serialize_impl! {} -} - -// Set -impl<'a, V: Hash + Eq + Serialize> Serialize for setref::multiple::RefMulti<'a, V> { - serialize_impl! {} -} - -impl<'a, V: Hash + Eq + Serialize> Serialize for setref::one::Ref<'a, V> { - serialize_impl! {} -} diff --git a/patch/dashmap/src/set.rs b/patch/dashmap/src/set.rs deleted file mode 100644 index 4ab98e1..0000000 --- a/patch/dashmap/src/set.rs +++ /dev/null @@ -1,500 +0,0 @@ -use crate::iter_set::{Iter, OwningIter}; -#[cfg(feature = "raw-api")] -use crate::lock::RwLock; -use crate::setref::one::Ref; -use crate::DashMap; -#[cfg(feature = "raw-api")] -use crate::HashMap; -use cfg_if::cfg_if; -use core::fmt; -use core::hash::{BuildHasher, Hash}; -use core::iter::FromIterator; -#[cfg(feature = "raw-api")] -use crossbeam_utils::CachePadded; -use equivalent::Equivalent; -use std::collections::hash_map::RandomState; - -/// DashSet is a thin wrapper around [`DashMap`] using `()` as the value type. It uses -/// methods and types which are more convenient to work with on a set. -/// -/// [`DashMap`]: struct.DashMap.html -pub struct DashSet { - pub(crate) inner: DashMap, -} - -impl fmt::Debug for DashSet { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Debug::fmt(&self.inner, f) - } -} - -impl Clone for DashSet { - fn clone(&self) -> Self { - Self { - inner: self.inner.clone(), - } - } - - fn clone_from(&mut self, source: &Self) { - self.inner.clone_from(&source.inner) - } -} - -impl Default for DashSet -where - K: Eq + Hash, - S: Default + BuildHasher + Clone, -{ - fn default() -> Self { - Self::with_hasher(Default::default()) - } -} - -impl<'a, K: 'a + Eq + Hash> DashSet { - /// Creates a new DashSet with a capacity of 0. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashSet; - /// - /// let games = DashSet::new(); - /// games.insert("Veloren"); - /// ``` - pub fn new() -> Self { - Self::with_hasher(RandomState::default()) - } - - /// Creates a new DashMap with a specified starting capacity. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashSet; - /// - /// let numbers = DashSet::with_capacity(2); - /// numbers.insert(2); - /// numbers.insert(8); - /// ``` - pub fn with_capacity(capacity: usize) -> Self { - Self::with_capacity_and_hasher(capacity, RandomState::default()) - } -} - -impl<'a, K: 'a + Eq + Hash, S: BuildHasher + Clone> DashSet { - /// Creates a new DashMap with a capacity of 0 and the provided hasher. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashSet; - /// use std::collections::hash_map::RandomState; - /// - /// let s = RandomState::new(); - /// let games = DashSet::with_hasher(s); - /// games.insert("Veloren"); - /// ``` - pub fn with_hasher(hasher: S) -> Self { - Self::with_capacity_and_hasher(0, hasher) - } - - /// Creates a new DashMap with a specified starting capacity and hasher. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashSet; - /// use std::collections::hash_map::RandomState; - /// - /// let s = RandomState::new(); - /// let numbers = DashSet::with_capacity_and_hasher(2, s); - /// numbers.insert(2); - /// numbers.insert(8); - /// ``` - pub fn with_capacity_and_hasher(capacity: usize, hasher: S) -> Self { - Self { - inner: DashMap::with_capacity_and_hasher(capacity, hasher), - } - } - - /// Hash a given item to produce a usize. - /// Uses the provided or default HashBuilder. - pub fn hash_usize(&self, item: &T) -> usize { - self.inner.hash_usize(item) - } - - cfg_if! { - if #[cfg(feature = "raw-api")] { - /// Allows you to peek at the inner shards that store your data. - /// You should probably not use this unless you know what you are doing. - /// - /// Requires the `raw-api` feature to be enabled. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashSet; - /// - /// let set = DashSet::<()>::new(); - /// println!("Amount of shards: {}", set.shards().len()); - /// ``` - pub fn shards(&self) -> &[CachePadded>>] { - self.inner.shards() - } - } - } - - cfg_if! { - if #[cfg(feature = "raw-api")] { - /// Finds which shard a certain key is stored in. - /// You should probably not use this unless you know what you are doing. - /// Note that shard selection is dependent on the default or provided HashBuilder. - /// - /// Requires the `raw-api` feature to be enabled. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashSet; - /// - /// let set = DashSet::new(); - /// set.insert("coca-cola"); - /// println!("coca-cola is stored in shard: {}", set.determine_map("coca-cola")); - /// ``` - pub fn determine_map(&self, key: &Q) -> usize - where - Q: Hash + Equivalent + ?Sized, - { - self.inner.determine_map(key) - } - } - } - - cfg_if! { - if #[cfg(feature = "raw-api")] { - /// Finds which shard a certain hash is stored in. - /// - /// Requires the `raw-api` feature to be enabled. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashSet; - /// - /// let set: DashSet = DashSet::new(); - /// let key = "key"; - /// let hash = set.hash_usize(&key); - /// println!("hash is stored in shard: {}", set.determine_shard(hash)); - /// ``` - pub fn determine_shard(&self, hash: usize) -> usize { - self.inner.determine_shard(hash) - } - } - } - - /// Inserts a key into the set. Returns true if the key was not already in the set. - /// Does not update the key if it was already present. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashSet; - /// - /// let set = DashSet::new(); - /// set.insert("I am the key!"); - /// ``` - pub fn insert(&self, key: K) -> bool { - self.inner.insert(key, ()).is_none() - } - - /// Removes an entry from the map, returning the key if it existed in the map. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashSet; - /// - /// let soccer_team = DashSet::new(); - /// soccer_team.insert("Jack"); - /// assert_eq!(soccer_team.remove("Jack").unwrap(), "Jack"); - /// ``` - pub fn remove(&self, key: &Q) -> Option - where - Q: Hash + Equivalent + ?Sized, - { - self.inner.remove(key).map(|(k, _)| k) - } - - /// Removes an entry from the set, returning the key - /// if the entry existed and the provided conditional function returned true. - /// - /// ``` - /// use dashmap::DashSet; - /// - /// let soccer_team = DashSet::new(); - /// soccer_team.insert("Sam"); - /// soccer_team.remove_if("Sam", |player| player.starts_with("Ja")); - /// assert!(soccer_team.contains("Sam")); - /// ``` - /// ``` - /// use dashmap::DashSet; - /// - /// let soccer_team = DashSet::new(); - /// soccer_team.insert("Sam"); - /// soccer_team.remove_if("Jacob", |player| player.starts_with("Ja")); - /// assert!(!soccer_team.contains("Jacob")); - /// ``` - pub fn remove_if(&self, key: &Q, f: impl FnOnce(&K) -> bool) -> Option - where - Q: Hash + Equivalent + ?Sized, - { - // TODO: Don't create another closure around f - self.inner.remove_if(key, |k, _| f(k)).map(|(k, _)| k) - } - - /// Creates an iterator over a DashMap yielding immutable references. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashSet; - /// - /// let words = DashSet::new(); - /// words.insert("hello"); - /// assert_eq!(words.iter().count(), 1); - /// ``` - pub fn iter(&'a self) -> Iter<'a, K> { - let iter = self.inner.iter(); - - Iter::new(iter) - } - - /// Get a reference to an entry in the set - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashSet; - /// - /// let youtubers = DashSet::new(); - /// youtubers.insert("Bosnian Bill"); - /// assert_eq!(*youtubers.get("Bosnian Bill").unwrap(), "Bosnian Bill"); - /// ``` - pub fn get(&'a self, key: &Q) -> Option> - where - Q: Hash + Equivalent + ?Sized, - { - self.inner.get(key).map(Ref::new) - } - - /// Remove excess capacity to reduce memory usage. - pub fn shrink_to_fit(&self) { - self.inner.shrink_to_fit() - } - - /// Retain elements that whose predicates return true - /// and discard elements whose predicates return false. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashSet; - /// - /// let people = DashSet::new(); - /// people.insert("Albin"); - /// people.insert("Jones"); - /// people.insert("Charlie"); - /// people.retain(|name| name.contains('i')); - /// assert_eq!(people.len(), 2); - /// ``` - pub fn retain(&self, mut f: impl FnMut(&K) -> bool) { - self.inner.retain(|k, _| f(k)) - } - - /// Fetches the total number of keys stored in the set. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashSet; - /// - /// let people = DashSet::new(); - /// people.insert("Albin"); - /// people.insert("Jones"); - /// people.insert("Charlie"); - /// assert_eq!(people.len(), 3); - /// ``` - pub fn len(&self) -> usize { - self.inner.len() - } - - /// Checks if the set is empty or not. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashSet; - /// - /// let map = DashSet::<()>::new(); - /// assert!(map.is_empty()); - /// ``` - pub fn is_empty(&self) -> bool { - self.inner.is_empty() - } - - /// Removes all keys in the set. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashSet; - /// - /// let people = DashSet::new(); - /// people.insert("Albin"); - /// assert!(!people.is_empty()); - /// people.clear(); - /// assert!(people.is_empty()); - /// ``` - pub fn clear(&self) { - self.inner.clear() - } - - /// Returns how many keys the set can store without reallocating. - pub fn capacity(&self) -> usize { - self.inner.capacity() - } - - /// Checks if the set contains a specific key. - /// - /// # Examples - /// - /// ``` - /// use dashmap::DashSet; - /// - /// let people = DashSet::new(); - /// people.insert("Dakota Cherries"); - /// assert!(people.contains("Dakota Cherries")); - /// ``` - pub fn contains(&self, key: &Q) -> bool - where - Q: Hash + Equivalent + ?Sized, - { - self.inner.contains_key(key) - } -} - -impl PartialEq for DashSet { - fn eq(&self, other: &Self) -> bool { - self.len() == other.len() && self.iter().all(|r| other.contains(r.key())) - } -} - -impl Eq for DashSet {} - -impl IntoIterator for DashSet { - type Item = K; - - type IntoIter = OwningIter; - - fn into_iter(self) -> Self::IntoIter { - OwningIter::new(self.inner.into_iter()) - } -} - -impl Extend for DashSet { - fn extend>(&mut self, iter: T) { - let iter = iter.into_iter().map(|k| (k, ())); - - self.inner.extend(iter) - } -} - -impl FromIterator for DashSet { - fn from_iter>(iter: I) -> Self { - let mut set = DashSet::default(); - - set.extend(iter); - - set - } -} - -#[cfg(feature = "typesize")] -impl typesize::TypeSize for DashSet -where - K: typesize::TypeSize + Eq + Hash, - S: typesize::TypeSize + Clone + BuildHasher, -{ - fn extra_size(&self) -> usize { - self.inner.extra_size() - } - - typesize::if_typesize_details! { - fn get_collection_item_count(&self) -> Option { - Some(self.len()) - } - } -} - -#[cfg(test)] -mod tests { - use crate::DashSet; - - #[test] - fn test_basic() { - let set = DashSet::new(); - - set.insert(0); - - assert_eq!(set.get(&0).as_deref(), Some(&0)); - } - - #[test] - fn test_default() { - let set: DashSet = DashSet::default(); - - set.insert(0); - - assert_eq!(set.get(&0).as_deref(), Some(&0)); - } - - #[test] - fn test_equal() { - let set1 = DashSet::new(); - let set2 = DashSet::new(); - assert_eq!(set1, set2); - - set1.insert("Hello, world!"); - assert_ne!(set1, set2); - - set1.insert("Goodbye, world!"); - assert_ne!(set1, set2); - - set2.insert("Hello, world!"); - assert_ne!(set1, set2); - - set2.insert("Goodbye, world!"); - assert_eq!(set1, set2); - } - - #[test] - fn test_multiple_hashes() { - let set = DashSet::::default(); - - for i in 0..100 { - assert!(set.insert(i)); - } - - for i in 0..100 { - assert!(!set.insert(i)); - } - - for i in 0..100 { - assert_eq!(Some(i), set.remove(&i)); - } - - for i in 0..100 { - assert_eq!(None, set.remove(&i)); - } - } -} diff --git a/patch/dashmap/src/setref/mod.rs b/patch/dashmap/src/setref/mod.rs deleted file mode 100644 index 95ae8f4..0000000 --- a/patch/dashmap/src/setref/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod multiple; -pub mod one; diff --git a/patch/dashmap/src/setref/multiple.rs b/patch/dashmap/src/setref/multiple.rs deleted file mode 100644 index 4806c34..0000000 --- a/patch/dashmap/src/setref/multiple.rs +++ /dev/null @@ -1,25 +0,0 @@ -use crate::mapref; -use core::hash::Hash; -use core::ops::Deref; - -pub struct RefMulti<'a, K> { - inner: mapref::multiple::RefMulti<'a, K, ()>, -} - -impl<'a, K: Eq + Hash> RefMulti<'a, K> { - pub(crate) fn new(inner: mapref::multiple::RefMulti<'a, K, ()>) -> Self { - Self { inner } - } - - pub fn key(&self) -> &K { - self.inner.key() - } -} - -impl<'a, K: Eq + Hash> Deref for RefMulti<'a, K> { - type Target = K; - - fn deref(&self) -> &K { - self.key() - } -} diff --git a/patch/dashmap/src/setref/one.rs b/patch/dashmap/src/setref/one.rs deleted file mode 100644 index 5f6d71d..0000000 --- a/patch/dashmap/src/setref/one.rs +++ /dev/null @@ -1,25 +0,0 @@ -use crate::mapref; -use core::hash::Hash; -use core::ops::Deref; - -pub struct Ref<'a, K> { - inner: mapref::one::Ref<'a, K, ()>, -} - -impl<'a, K: Eq + Hash> Ref<'a, K> { - pub(crate) fn new(inner: mapref::one::Ref<'a, K, ()>) -> Self { - Self { inner } - } - - pub fn key(&self) -> &K { - self.inner.key() - } -} - -impl<'a, K: Eq + Hash> Deref for Ref<'a, K> { - type Target = K; - - fn deref(&self) -> &K { - self.key() - } -} diff --git a/patch/dashmap/src/try_result.rs b/patch/dashmap/src/try_result.rs deleted file mode 100644 index aa93d3b..0000000 --- a/patch/dashmap/src/try_result.rs +++ /dev/null @@ -1,46 +0,0 @@ -/// Represents the result of a non-blocking read from a [DashMap](crate::DashMap). -#[derive(Debug)] -pub enum TryResult { - /// The value was present in the map, and the lock for the shard was successfully obtained. - Present(R), - /// The shard wasn't locked, and the value wasn't present in the map. - Absent, - /// The shard was locked. - Locked, -} - -impl TryResult { - /// Returns `true` if the value was present in the map, and the lock for the shard was successfully obtained. - pub fn is_present(&self) -> bool { - matches!(self, TryResult::Present(_)) - } - - /// Returns `true` if the shard wasn't locked, and the value wasn't present in the map. - pub fn is_absent(&self) -> bool { - matches!(self, TryResult::Absent) - } - - /// Returns `true` if the shard was locked. - pub fn is_locked(&self) -> bool { - matches!(self, TryResult::Locked) - } - - /// If `self` is [Present](TryResult::Present), returns the reference to the value in the map. - /// Panics if `self` is not [Present](TryResult::Present). - pub fn unwrap(self) -> R { - match self { - TryResult::Present(r) => r, - TryResult::Locked => panic!("Called unwrap() on TryResult::Locked"), - TryResult::Absent => panic!("Called unwrap() on TryResult::Absent"), - } - } - - /// If `self` is [Present](TryResult::Present), returns the reference to the value in the map. - /// If `self` is not [Present](TryResult::Present), returns `None`. - pub fn try_unwrap(self) -> Option { - match self { - TryResult::Present(r) => Some(r), - _ => None, - } - } -} diff --git a/patch/dashmap/src/util.rs b/patch/dashmap/src/util.rs deleted file mode 100644 index 6464aa6..0000000 --- a/patch/dashmap/src/util.rs +++ /dev/null @@ -1,125 +0,0 @@ -use core::{mem, ptr}; -use std::{marker::PhantomData, mem::ManuallyDrop}; - -use lock_api::{RawRwLock, RawRwLockDowngrade, RwLockReadGuard, RwLockWriteGuard}; - -pub const fn ptr_size_bits() -> usize { - mem::size_of::() * 8 -} - -pub fn map_in_place_2 T>((k, v): (U, &mut T), f: F) { - unsafe { - // # Safety - // - // If the closure panics, we must abort otherwise we could double drop `T` - let promote_panic_to_abort = AbortOnPanic; - - ptr::write(v, f(k, ptr::read(v))); - - // If we made it here, the calling thread could have already have panicked, in which case - // We know that the closure did not panic, so don't bother checking. - std::mem::forget(promote_panic_to_abort); - } -} - -struct AbortOnPanic; - -impl Drop for AbortOnPanic { - fn drop(&mut self) { - if std::thread::panicking() { - std::process::abort() - } - } -} - -/// A [`RwLockReadGuard`], without the data -pub(crate) struct RwLockReadGuardDetached<'a, R: RawRwLock> { - lock: &'a R, - _marker: PhantomData, -} - -impl Drop for RwLockReadGuardDetached<'_, R> { - fn drop(&mut self) { - // Safety: An RwLockReadGuardDetached always holds a shared lock. - unsafe { - self.lock.unlock_shared(); - } - } -} - -/// A [`RwLockWriteGuard`], without the data -pub(crate) struct RwLockWriteGuardDetached<'a, R: RawRwLock> { - lock: &'a R, - _marker: PhantomData, -} - -impl Drop for RwLockWriteGuardDetached<'_, R> { - fn drop(&mut self) { - // Safety: An RwLockWriteGuardDetached always holds an exclusive lock. - unsafe { - self.lock.unlock_exclusive(); - } - } -} - -impl<'a, R: RawRwLock> RwLockReadGuardDetached<'a, R> { - /// Separates the data from the [`RwLockReadGuard`] - /// - /// # Safety - /// - /// The data must not outlive the detached guard - pub(crate) unsafe fn detach_from(guard: RwLockReadGuard<'a, R, T>) -> (Self, &'a T) { - let rwlock = RwLockReadGuard::rwlock(&ManuallyDrop::new(guard)); - - // Safety: There will be no concurrent writes as we are "forgetting" the existing guard, - // with the safety assumption that the caller will not drop the new detached guard early. - let data = unsafe { &*rwlock.data_ptr() }; - let guard = RwLockReadGuardDetached { - // Safety: We are imitating the original RwLockReadGuard. It's the callers - // responsibility to not drop the guard early. - lock: unsafe { rwlock.raw() }, - _marker: PhantomData, - }; - (guard, data) - } -} - -impl<'a, R: RawRwLock> RwLockWriteGuardDetached<'a, R> { - /// Separates the data from the [`RwLockWriteGuard`] - /// - /// # Safety - /// - /// The data must not outlive the detached guard - pub(crate) unsafe fn detach_from(guard: RwLockWriteGuard<'a, R, T>) -> (Self, &'a mut T) { - let rwlock = RwLockWriteGuard::rwlock(&ManuallyDrop::new(guard)); - - // Safety: There will be no concurrent reads/writes as we are "forgetting" the existing guard, - // with the safety assumption that the caller will not drop the new detached guard early. - let data = unsafe { &mut *rwlock.data_ptr() }; - let guard = RwLockWriteGuardDetached { - // Safety: We are imitating the original RwLockWriteGuard. It's the callers - // responsibility to not drop the guard early. - lock: unsafe { rwlock.raw() }, - _marker: PhantomData, - }; - (guard, data) - } -} - -impl<'a, R: RawRwLockDowngrade> RwLockWriteGuardDetached<'a, R> { - /// # Safety - /// - /// The associated data must not mut mutated after downgrading - pub(crate) unsafe fn downgrade(self) -> RwLockReadGuardDetached<'a, R> { - // Do not drop the write guard - otherwise we will trigger a downgrade + unlock_exclusive, - // which is incorrect - let this = ManuallyDrop::new(self); - - // Safety: An RwLockWriteGuardDetached always holds an exclusive lock. - unsafe { this.lock.downgrade() } - RwLockReadGuardDetached { - lock: this.lock, - _marker: this._marker, - } - } -} diff --git a/patch/reqwest-0.12.18/src/lib.rs b/patch/reqwest-0.12.18/src/lib.rs index 2df47e1..0a048d3 100644 --- a/patch/reqwest-0.12.18/src/lib.rs +++ b/patch/reqwest-0.12.18/src/lib.rs @@ -1,7 +1,7 @@ #![deny(missing_docs)] #![deny(missing_debug_implementations)] #![cfg_attr(docsrs, feature(doc_cfg))] -#![cfg_attr(not(test), warn(unused_crate_dependencies))] +// #![cfg_attr(not(test), warn(unused_crate_dependencies))] #![cfg_attr(test, deny(warnings))] //! # reqwest diff --git a/patch/scc-3.4.8/CHANGELOG.md b/patch/scc-3.4.8/CHANGELOG.md index bd32086..d69a53c 100644 --- a/patch/scc-3.4.8/CHANGELOG.md +++ b/patch/scc-3.4.8/CHANGELOG.md @@ -3,6 +3,31 @@ ## Version 3 +3.4.14 + +* Implement `DoubleEndedIterator` for `tree_index::Iter`: [#217](https://codeberg.org/wvwwvwwv/scalable-concurrent-containers/issues/217). + +3.4.13 + +* Remove the `V: Clone` bound from `TreeIndex`. + +3.4.12 + +* Optimize `TreeIndex` entry iteration. + +3.4.11 - 3.4.12 + +* Code cleanup. + +3.4.10 + +* Minor `Hash*` performance improvements. + +3.4.9 + +* Migrate to [`codeberg`](https://codeberg.org/wvwwvwwv/scalable-concurrent-containers). +* Better `Hash*` OOM handling. + 3.4.8 * Minor updates for `codeberg` migration. diff --git a/patch/scc-3.4.8/Cargo.toml b/patch/scc-3.4.8/Cargo.toml index 579fa99..6c5009b 100644 --- a/patch/scc-3.4.8/Cargo.toml +++ b/patch/scc-3.4.8/Cargo.toml @@ -1,13 +1,13 @@ [package] name = "scc" -description = "A collection of high-performance containers providing both asynchronous and synchronous interfaces" +description = "A collection of high-performance asynchronous/concurrent containers providing both asynchronous and synchronous interfaces" documentation = "https://docs.rs/scc" -version = "3.4.8" +version = "3.4.14" authors = ["wvwwvwwv "] edition = "2024" rust-version = "1.85.0" readme = "README.md" -repository = "https://github.com/wvwwvwwv/scalable-concurrent-containers/" +repository = "https://codeberg.org/wvwwvwwv/scalable-concurrent-containers/" license = "Apache-2.0" categories = ["asynchronous", "caching", "concurrency", "data-structures"] keywords = ["async", "cache", "concurrent", "hashmap", "tree"] @@ -32,7 +32,7 @@ futures = "0.3" proptest = "1.9" serde_test = "1.0" static_assertions = "1.1" -tokio = { version = "1.48", features = ["full"] } +tokio = { version = "1.49", features = ["full"] } [[bench]] name = "hash_map" diff --git a/patch/scc-3.4.8/README.md b/patch/scc-3.4.8/README.md index d7e1e55..caf2ca8 100644 --- a/patch/scc-3.4.8/README.md +++ b/patch/scc-3.4.8/README.md @@ -2,9 +2,8 @@ [![Cargo](https://img.shields.io/crates/v/scc)](https://crates.io/crates/scc) ![Crates.io](https://img.shields.io/crates/l/scc) -![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/wvwwvwwv/scalable-concurrent-containers/scc.yml?branch=main) -A collection of high-performance containers providing both asynchronous and synchronous interfaces. +A collection of high-performance asynchronous/concurrent containers providing both asynchronous and synchronous interfaces. #### Features @@ -14,15 +13,15 @@ A collection of high-performance containers providing both asynchronous and sync #### Concurrent Containers -- [`HashMap`](#hashmap) is a concurrent hash map. -- [`HashSet`](#hashset) is a concurrent hash set. -- [`HashIndex`](#hashindex) is a read-optimized concurrent hash map. -- [`HashCache`](#hashcache) is a 32-way associative concurrent cache backed by [`HashMap`](#hashmap). -- [`TreeIndex`](#treeindex) is a read-optimized concurrent B-plus tree. +- [`HashMap`](#hashmap) is an asynchronous/concurrent hash map. +- [`HashSet`](#hashset) is an asynchronous/concurrent hash set. +- [`HashIndex`](#hashindex) is a read-optimized asynchronous/concurrent hash map. +- [`HashCache`](#hashcache) is a 32-way associative asynchronous/concurrent cache backed by [`HashMap`](#hashmap). +- [`TreeIndex`](#treeindex) is a read-optimized asynchronous/concurrent B-plus tree. ## `HashMap` -[`HashMap`](#hashmap) is a concurrent hash map optimized for highly parallel write-heavy workloads. [`HashMap`](#hashmap) is structured as a lock-free stack of entry bucket arrays. The entry bucket array is managed by [`sdd`](https://crates.io/crates/sdd), thus enabling lock-free access to it and non-blocking container resizing. Each bucket is a fixed-size array of entries, protected by a read-write lock that simultaneously provides blocking and asynchronous methods. +[`HashMap`](#hashmap) is an asynchronous/concurrent hash map optimized for highly parallel write-heavy workloads. [`HashMap`](#hashmap) is structured as a lock-free stack of entry bucket arrays. The entry bucket array is managed by [`sdd`](https://crates.io/crates/sdd), thus enabling lock-free access to it and non-blocking container resizing. Each bucket is a fixed-size array of entries, protected by a read-write lock that simultaneously provides blocking and asynchronous methods. ### Locking behavior @@ -213,7 +212,7 @@ assert_eq!(iter.next(), None); ## `HashCache` -[`HashCache`](#hashcache) is a 32-way associative concurrent cache based on the [`HashMap`](#hashmap) implementation. [`HashCache`](#hashcache) does not keep track of the least recently used entry in the entire cache. Instead, each bucket maintains a doubly linked list of occupied entries, which is updated on entry access. +[`HashCache`](#hashcache) is a 32-way associative asynchronous/concurrent cache based on the [`HashMap`](#hashmap) implementation. [`HashCache`](#hashcache) does not keep track of the least recently used entry in the entire cache. Instead, each bucket maintains a doubly linked list of occupied entries, which is updated on entry access. ### Examples @@ -328,4 +327,4 @@ The expected tail latency of a distribution of latencies of 1048576 insertion op - [Results on Intel Xeon (48 cores, avx2)](https://codeberg.org/wvwwvwwv/conc-map-bench). -## [Changelog](https://github.com/wvwwvwwv/scalable-concurrent-containers/blob/main/CHANGELOG.md) +## [Changelog](https://codeberg.org/wvwwvwwv/scalable-concurrent-containers/src/branch/main/CHANGELOG.md) diff --git a/patch/scc-3.4.8/benches/hash_cache.rs b/patch/scc-3.4.8/benches/hash_cache.rs index 4baa40f..352fd00 100644 --- a/patch/scc-3.4.8/benches/hash_cache.rs +++ b/patch/scc-3.4.8/benches/hash_cache.rs @@ -3,18 +3,19 @@ use std::time::Instant; use criterion::{Criterion, criterion_group, criterion_main}; use scc::HashCache; -fn get(c: &mut Criterion) { - c.bench_function("HashCache: get", |b| { +fn get_saturated(c: &mut Criterion) { + let hashcache: HashCache = HashCache::with_capacity(64, 64); + for k in 0..256 { + assert!(hashcache.put_sync(k, k).is_ok()); + } + let mut max_key = 256; + c.bench_function("HashCache: get, saturated", |b| { b.iter_custom(|iters| { - let hashcache: HashCache = - HashCache::with_capacity(iters as usize * 2, iters as usize * 2); - for i in 0..iters { - assert!(hashcache.put_sync(i, i).is_ok()); - } let start = Instant::now(); - for i in 0..iters { + for i in max_key..(max_key + iters) { drop(hashcache.get_sync(&i)); } + max_key += iters; start.elapsed() }) }); @@ -38,5 +39,5 @@ fn put_saturated(c: &mut Criterion) { }); } -criterion_group!(hash_cache, get, put_saturated); +criterion_group!(hash_cache, get_saturated, put_saturated); criterion_main!(hash_cache); diff --git a/patch/scc-3.4.8/benches/hash_map.rs b/patch/scc-3.4.8/benches/hash_map.rs index cefdd9e..6561ed6 100644 --- a/patch/scc-3.4.8/benches/hash_map.rs +++ b/patch/scc-3.4.8/benches/hash_map.rs @@ -4,28 +4,6 @@ use criterion::async_executor::FuturesExecutor; use criterion::{Criterion, criterion_group, criterion_main}; use scc::HashMap; -fn insert_single_async(c: &mut Criterion) { - c.bench_function("HashMap: insert_single_async", |b| { - let hashmap: HashMap = HashMap::default(); - assert!(hashmap.insert_sync(0, 0).is_ok()); - async fn test(hashmap: &HashMap) { - assert!(hashmap.insert_async(0, 0).await.is_err()); - } - b.to_async(FuturesExecutor).iter(|| test(&hashmap)); - }); -} - -fn insert_single_sync(c: &mut Criterion) { - c.bench_function("HashMap: insert_single_sync", |b| { - let hashmap: HashMap = HashMap::default(); - assert!(hashmap.insert_sync(0, 0).is_ok()); - fn test(hashmap: &HashMap) { - assert!(hashmap.insert_sync(0, 0).is_err()); - } - b.iter(|| test(&hashmap)); - }); -} - fn insert_remove_single_async(c: &mut Criterion) { c.bench_function("HashMap: insert_remove_single_async", |b| { let hashmap: HashMap = HashMap::default(); @@ -160,8 +138,6 @@ fn insert_tail_latency(c: &mut Criterion) { criterion_group!( hash_map, - insert_single_async, - insert_single_sync, insert_remove_single_async, insert_remove_single_sync, insert_cold_async, diff --git a/patch/scc-3.4.8/benches/tree_index.rs b/patch/scc-3.4.8/benches/tree_index.rs index e986960..d52f40a 100644 --- a/patch/scc-3.4.8/benches/tree_index.rs +++ b/patch/scc-3.4.8/benches/tree_index.rs @@ -43,8 +43,8 @@ fn insert_rev(c: &mut Criterion) { }); } -fn iter_with(c: &mut Criterion) { - c.bench_function("TreeIndex: iter_with", |b| { +fn iter(c: &mut Criterion) { + c.bench_function("TreeIndex: iter", |b| { b.iter_custom(|iters| { let treeindex: TreeIndex = TreeIndex::default(); for i in 0..iters { @@ -61,6 +61,25 @@ fn iter_with(c: &mut Criterion) { }); } +fn range(c: &mut Criterion) { + c.bench_function("TreeIndex: range", |b| { + b.iter_custom(|iters| { + let treeindex: TreeIndex = TreeIndex::default(); + for i in 0..iters { + assert!(treeindex.insert_sync(i, i).is_ok()); + } + let start = Instant::now(); + let guard = Guard::new(); + for s in 0..iters { + let range = s..iters; + let mut iter = treeindex.range(range.clone(), &guard); + assert_eq!(range.is_empty(), iter.next().is_none()); + } + start.elapsed() + }) + }); +} + fn peek(c: &mut Criterion) { c.bench_function("TreeIndex: peek", |b| { b.iter_custom(|iters| { @@ -83,7 +102,8 @@ criterion_group!( insert_async, insert_sync, insert_rev, - iter_with, + iter, + range, peek ); criterion_main!(tree_index); diff --git a/patch/scc-3.4.8/src/hash_cache.rs b/patch/scc-3.4.8/src/hash_cache.rs index b2a085f..6edc7cf 100644 --- a/patch/scc-3.4.8/src/hash_cache.rs +++ b/patch/scc-3.4.8/src/hash_cache.rs @@ -1,4 +1,5 @@ -//! [`HashCache`] is a concurrent 32-way associative cache backed by [`HashMap`](super::HashMap). +//! [`HashCache`] is an asynchronous/concurrent 32-way associative cache backed by +//! [`HashMap`](super::HashMap). use std::collections::hash_map::RandomState; use std::fmt::{self, Debug}; @@ -19,9 +20,9 @@ use super::hash_table::bucket::{CACHE, DoublyLinkedList, EntryPtr}; use super::hash_table::bucket_array::BucketArray; use super::hash_table::{HashTable, LockedBucket}; -/// Scalable concurrent 32-way associative cache backed by [`HashMap`](super::HashMap). +/// Scalable asynchronous/concurrent 32-way associative cache backed by [`HashMap`](super::HashMap). /// -/// [`HashCache`] is a concurrent 32-way associative cache based on the +/// [`HashCache`] is an asynchronous/concurrent 32-way associative cache based on the /// [`HashMap`](super::HashMap) implementation. [`HashCache`] does not keep track of the least /// recently used entry in the entire cache. Instead, each bucket maintains a doubly linked list of /// occupied entries, updated on access to entries to keep track of the least recently used entries @@ -826,7 +827,7 @@ where let mut result = true; self.for_each_reader_async(|reader, data_block| { let mut entry_ptr = EntryPtr::null(); - while entry_ptr.move_to_next(&reader) { + while entry_ptr.find_next(&reader) { let (k, v) = entry_ptr.get(data_block); if !f(k, v) { result = false; @@ -869,7 +870,7 @@ where let guard = Guard::new(); self.for_each_reader_sync(&guard, |reader, data_block| { let mut entry_ptr = EntryPtr::null(); - while entry_ptr.move_to_next(&reader) { + while entry_ptr.find_next(&reader) { let (k, v) = entry_ptr.get(data_block); if !f(k, v) { result = false; @@ -916,7 +917,7 @@ where let mut result = true; self.for_each_writer_async(0, 0, |mut locked_bucket, removed| { let mut entry_ptr = EntryPtr::null(); - while entry_ptr.move_to_next(&locked_bucket.writer) { + while entry_ptr.find_next(&locked_bucket.writer) { let consumable_entry = ConsumableEntry { locked_bucket: &mut locked_bucket, entry_ptr: &mut entry_ptr, @@ -924,10 +925,10 @@ where }; if !f(consumable_entry) { result = false; - return true; + return false; } } - false + true }) .await; result @@ -966,7 +967,7 @@ where let guard = Guard::new(); self.for_each_writer_sync(0, 0, &guard, |mut locked_bucket, removed| { let mut entry_ptr = EntryPtr::null(); - while entry_ptr.move_to_next(&locked_bucket.writer) { + while entry_ptr.find_next(&locked_bucket.writer) { let consumable_entry = ConsumableEntry { locked_bucket: &mut locked_bucket, entry_ptr: &mut entry_ptr, @@ -974,10 +975,10 @@ where }; if !f(consumable_entry) { result = false; - return true; + return false; } } - false + true }); result } diff --git a/patch/scc-3.4.8/src/hash_index.rs b/patch/scc-3.4.8/src/hash_index.rs index 93bcff7..33f2ca3 100644 --- a/patch/scc-3.4.8/src/hash_index.rs +++ b/patch/scc-3.4.8/src/hash_index.rs @@ -1,4 +1,4 @@ -//! [`HashIndex`] is a read-optimized concurrent hash map. +//! [`HashIndex`] is a read-optimized asynchronous/concurrent hash map. use std::collections::hash_map::RandomState; use std::fmt::{self, Debug}; @@ -21,11 +21,11 @@ use super::hash_table::bucket::{Bucket, EntryPtr, INDEX}; use super::hash_table::bucket_array::BucketArray; use super::hash_table::{HashTable, LockedBucket}; -/// Scalable concurrent hash index. +/// Scalable asynchronous/concurrent hash index. /// -/// [`HashIndex`] is a concurrent hash map data structure optimized for parallel read operations. -/// The key characteristics of [`HashIndex`] are similar to that of [`HashMap`](super::HashMap) -/// except its read operations are lock-free. +/// [`HashIndex`] is an asynchronous/concurrent hash map data structure optimized for parallel read +/// operations. The key characteristics of [`HashIndex`] are similar to that of +/// [`HashMap`](super::HashMap) except its read operations are lock-free. /// /// ## The key differences between [`HashIndex`] and [`HashMap`](crate::HashMap). /// @@ -430,7 +430,7 @@ where let mut entry = None; self.for_each_writer_async(0, 0, |locked_bucket, _| { let mut entry_ptr = EntryPtr::null(); - while entry_ptr.move_to_next(&locked_bucket.writer) { + while entry_ptr.find_next(&locked_bucket.writer) { let (k, v) = locked_bucket.entry(&entry_ptr); if pred(k, v) { entry = Some(OccupiedEntry { @@ -438,10 +438,10 @@ where locked_bucket, entry_ptr, }); - return true; + return false; } } - false + true }) .await; entry @@ -472,7 +472,7 @@ where let guard = Guard::new(); self.for_each_writer_sync(0, 0, &guard, |locked_bucket, _| { let mut entry_ptr = EntryPtr::null(); - while entry_ptr.move_to_next(&locked_bucket.writer) { + while entry_ptr.find_next(&locked_bucket.writer) { let (k, v) = locked_bucket.entry(&entry_ptr); if pred(k, v) { entry = Some(OccupiedEntry { @@ -480,10 +480,10 @@ where locked_bucket, entry_ptr, }); - return true; + return false; } } - false + true }); entry } @@ -871,7 +871,7 @@ where let mut result = true; self.for_each_reader_async(|reader, data_block| { let mut entry_ptr = EntryPtr::null(); - while entry_ptr.move_to_next(&reader) { + while entry_ptr.find_next(&reader) { let (k, v) = entry_ptr.get(data_block); if !f(k, v) { result = false; @@ -915,7 +915,7 @@ where let guard = Guard::new(); self.for_each_reader_sync(&guard, |reader, data_block| { let mut entry_ptr = EntryPtr::null(); - while entry_ptr.move_to_next(&reader) { + while entry_ptr.find_next(&reader) { let (k, v) = entry_ptr.get(data_block); if !f(k, v) { result = false; @@ -948,7 +948,7 @@ where self.reclaim_memory(); self.for_each_writer_async(0, 0, |mut locked_bucket, removed| { let mut entry_ptr = EntryPtr::null(); - while entry_ptr.move_to_next(&locked_bucket.writer) { + while entry_ptr.find_next(&locked_bucket.writer) { let (k, v) = locked_bucket.entry_mut(&mut entry_ptr); if !pred(k, v) { locked_bucket @@ -957,7 +957,7 @@ where *removed = true; } } - false + true }) .await; } @@ -991,14 +991,14 @@ where let guard = Guard::new(); self.for_each_writer_sync(0, 0, &guard, |mut locked_bucket, removed| { let mut entry_ptr = EntryPtr::null(); - while entry_ptr.move_to_next(&locked_bucket.writer) { + while entry_ptr.find_next(&locked_bucket.writer) { let (k, v) = locked_bucket.entry_mut(&mut entry_ptr); if !pred(k, v) { locked_bucket.writer.mark_removed(&mut entry_ptr, &guard); *removed = true; } } - false + true }); } @@ -1135,7 +1135,8 @@ where where Q: Equivalent + Hash + ?Sized, { - self.calculate_bucket_index(key) + let hash = self.hash(key); + self.calculate_bucket_index(hash) } /// Returns an [`Iter`]. @@ -2082,8 +2083,8 @@ where #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Iter") - .field("current_index", &self.index) - .field("current_entry_ptr", &self.entry_ptr) + .field("current_bucket_index", &self.index) + .field("current_index_in_bucket", &self.entry_ptr.index()) .finish() } } @@ -2116,7 +2117,7 @@ where loop { if let Some(bucket) = self.bucket.take() { // Move to the next entry in the bucket. - if self.entry_ptr.move_to_next(bucket) { + if self.entry_ptr.find_next(bucket) { let (k, v) = self.entry_ptr.get(array.data_block(self.index)); self.bucket.replace(bucket); return Some((k, v)); diff --git a/patch/scc-3.4.8/src/hash_map.rs b/patch/scc-3.4.8/src/hash_map.rs index 559a769..3739f58 100644 --- a/patch/scc-3.4.8/src/hash_map.rs +++ b/patch/scc-3.4.8/src/hash_map.rs @@ -1,4 +1,4 @@ -//! [`HashMap`] is a concurrent hash map. +//! [`HashMap`] is an asynchronous/concurrent hash map. mod raw_entry; pub use raw_entry::{RawEntry, RawOccupiedEntry, RawVacantEntry}; @@ -21,12 +21,12 @@ use super::hash_table::bucket::{EntryPtr, MAP}; use super::hash_table::bucket_array::BucketArray; use super::hash_table::{HashTable, LockedBucket}; -/// Scalable concurrent hash map. +/// Scalable asynchronous/concurrent hash map. /// -/// [`HashMap`] is a concurrent hash map data structure optimized for highly concurrent workloads. -/// [`HashMap`] has a dynamically sized array of buckets where a bucket is a fixed-size hash table -/// with linear probing that can be expanded by allocating a linked list of smaller buckets when it -/// is full. +/// [`HashMap`] is an asynchronous/concurrent hash map data structure optimized for highly +/// concurrent workloads. [`HashMap`] has a dynamically sized array of buckets where a bucket is a +/// fixed-size hash table with linear probing that can be expanded by allocating a linked list of +/// smaller buckets when it is full. /// /// ## The key features of [`HashMap`] /// @@ -454,7 +454,7 @@ where let mut entry = None; self.for_each_writer_async(0, 0, |locked_bucket, _| { let mut entry_ptr = EntryPtr::null(); - while entry_ptr.move_to_next(&locked_bucket.writer) { + while entry_ptr.find_next(&locked_bucket.writer) { let (k, v) = locked_bucket.entry(&entry_ptr); if pred(k, v) { entry = Some(OccupiedEntry { @@ -462,10 +462,10 @@ where locked_bucket, entry_ptr, }); - return true; + return false; } } - false + true }) .await; entry @@ -495,7 +495,7 @@ where let guard = Guard::new(); self.for_each_writer_sync(0, 0, &guard, |locked_bucket, _| { let mut entry_ptr = EntryPtr::null(); - while entry_ptr.move_to_next(&locked_bucket.writer) { + while entry_ptr.find_next(&locked_bucket.writer) { let (k, v) = locked_bucket.entry(&entry_ptr); if pred(k, v) { entry = Some(OccupiedEntry { @@ -503,10 +503,10 @@ where locked_bucket, entry_ptr, }); - return true; + return false; } } - false + true }); entry } @@ -1126,7 +1126,7 @@ where let mut result = true; self.for_each_reader_async(|reader, data_block| { let mut entry_ptr = EntryPtr::null(); - while entry_ptr.move_to_next(&reader) { + while entry_ptr.find_next(&reader) { let (k, v) = entry_ptr.get(data_block); if !f(k, v) { result = false; @@ -1169,7 +1169,7 @@ where let guard = Guard::new(); self.for_each_reader_sync(&guard, |reader, data_block| { let mut entry_ptr = EntryPtr::null(); - while entry_ptr.move_to_next(&reader) { + while entry_ptr.find_next(&reader) { let (k, v) = entry_ptr.get(data_block); if !f(k, v) { result = false; @@ -1216,7 +1216,7 @@ where let mut result = true; self.for_each_writer_async(0, 0, |mut locked_bucket, removed| { let mut entry_ptr = EntryPtr::null(); - while entry_ptr.move_to_next(&locked_bucket.writer) { + while entry_ptr.find_next(&locked_bucket.writer) { let consumable_entry = ConsumableEntry { locked_bucket: &mut locked_bucket, entry_ptr: &mut entry_ptr, @@ -1224,10 +1224,10 @@ where }; if !f(consumable_entry) { result = false; - return true; + return false; } } - false + true }) .await; result @@ -1266,7 +1266,7 @@ where let guard = Guard::new(); self.for_each_writer_sync(0, 0, &guard, |mut locked_bucket, removed| { let mut entry_ptr = EntryPtr::null(); - while entry_ptr.move_to_next(&locked_bucket.writer) { + while entry_ptr.find_next(&locked_bucket.writer) { let consumable_entry = ConsumableEntry { locked_bucket: &mut locked_bucket, entry_ptr: &mut entry_ptr, @@ -1274,10 +1274,10 @@ where }; if !f(consumable_entry) { result = false; - return true; + return false; } } - false + true }); result } @@ -1483,7 +1483,8 @@ where where Q: Equivalent + Hash + ?Sized, { - self.calculate_bucket_index(key) + let hash = self.hash(key); + self.calculate_bucket_index(hash) } } diff --git a/patch/scc-3.4.8/src/hash_set.rs b/patch/scc-3.4.8/src/hash_set.rs index 7f6393c..6236981 100644 --- a/patch/scc-3.4.8/src/hash_set.rs +++ b/patch/scc-3.4.8/src/hash_set.rs @@ -1,4 +1,4 @@ -//! [`HashSet`] is a concurrent hash set. +//! [`HashSet`] is an asynchronous/concurrent hash set. #![deny(unsafe_code)] @@ -12,9 +12,9 @@ use super::hash_map; use super::hash_table::HashTable; use super::{Equivalent, HashMap}; -/// Scalable concurrent hash set. +/// Scalable asynchronous/concurrent hash set. /// -/// [`HashSet`] is a concurrent hash set based on [`HashMap`]. +/// [`HashSet`] is an asynchronous/concurrent hash set based on [`HashMap`]. pub struct HashSet where H: BuildHasher, diff --git a/patch/scc-3.4.8/src/hash_table.rs b/patch/scc-3.4.8/src/hash_table.rs index cd2794a..14cccac 100644 --- a/patch/scc-3.4.8/src/hash_table.rs +++ b/patch/scc-3.4.8/src/hash_table.rs @@ -43,6 +43,7 @@ where fn bucket_array_var(&self) -> &AtomicShared>; /// Returns a reference to the current [`BucketArray`]. + #[inline] fn bucket_array<'g>(&self, guard: &'g Guard) -> Option<&'g BucketArray> { unsafe { self.bucket_array_var() @@ -59,15 +60,12 @@ where /// Calculates the bucket index from the supplied key. #[inline] - fn calculate_bucket_index(&self, key: &Q) -> usize - where - Q: Equivalent + Hash + ?Sized, - { + fn calculate_bucket_index(&self, hash: u64) -> usize { unsafe { self.bucket_array_var() .load(Acquire, &Guard::new()) .as_ref_unchecked() - .map_or(0, |a| a.bucket_index(self.hash(key))) + .map_or(0, |a| a.bucket_index(hash)) } } @@ -75,6 +73,7 @@ where fn minimum_capacity_var(&self) -> &AtomicUsize; /// Returns the current minimum allowed capacity. + #[inline] fn minimum_capacity(&self) -> usize { self.minimum_capacity_var().load(Relaxed) & (!RESIZING) } @@ -82,6 +81,7 @@ where /// Returns the maximum capacity. /// /// The maximum capacity must be a power of `2`. + #[inline] fn maximum_capacity(&self) -> usize { MAXIMUM_CAPACITY_LIMIT } @@ -613,7 +613,7 @@ where /// Iterates over all the buckets in the [`HashTable`]. /// - /// This method stops iterating when the closure returns `true`. + /// This method stops iterating when the closure returns `false`. #[inline] async fn for_each_writer_async( &self, @@ -665,8 +665,7 @@ where bucket_index, bucket_array: into_non_null(current_array), }; - let stop = f(locked_bucket, &mut removed); - if stop { + if !f(locked_bucket, &mut removed) { // Stop iterating over buckets. start_index = current_array_len; break; @@ -695,7 +694,7 @@ where /// Iterates over all the buckets in the [`HashTable`]. /// - /// This methods stops iterating when the closure returns `true`. + /// This methods stops iterating when the closure returns `false`. #[inline] fn for_each_writer_sync( &self, @@ -734,8 +733,7 @@ where bucket_index, bucket_array: into_non_null(current_array), }; - let stop = f(locked_bucket, &mut removed); - if stop { + if !f(locked_bucket, &mut removed) { // Stop iterating over buckets. start_index = current_array_len; break; @@ -1020,7 +1018,7 @@ where let mut hash_data = [0_u64; BUCKET_LEN]; // Collect data for relocation. - while entry_ptr.move_to_next(old_writer) { + while entry_ptr.find_next(old_writer) { let (offset, hash) = if old_array.len() >= current_array.len() { (0, u64::from(entry_ptr.partial_hash(&**old_writer))) } else { @@ -1069,7 +1067,7 @@ where // Relocate entries; it is infallible. entry_ptr = EntryPtr::null(); position = 0; - while entry_ptr.move_to_next(old_writer) { + while entry_ptr.find_next(old_writer) { let hash = if old_array.len() >= current_array.len() { u64::from(entry_ptr.partial_hash(&**old_writer)) } else if position == BUCKET_LEN { @@ -1103,18 +1101,20 @@ where let rehashing_metadata = old_array.rehashing_metadata(); let mut current = rehashing_metadata.load(Relaxed); loop { - if current >= old_array.len() || (current & (BUCKET_LEN - 1)) == BUCKET_LEN - 1 { - // Only `BUCKET_LEN` threads are allowed to rehash a `Bucket` at a moment. + if current >= old_array.len() || (current & (BUCKET_LEN * 2 - 1)) == BUCKET_LEN * 2 - 1 + { + // Only `BUCKET_LEN * 2` concurrent threads are allowed to rehash `BUCKET_LEN * 2` + // buckets. return None; } match rehashing_metadata.compare_exchange_weak( current, - current + BUCKET_LEN + 1, + current + BUCKET_LEN * 2 + 1, Acquire, Relaxed, ) { Ok(_) => { - current &= !(BUCKET_LEN - 1); + current &= !(BUCKET_LEN * 2 - 1); return Some(current); } Err(result) => current = result, @@ -1133,7 +1133,7 @@ where if success { // Keep the index as it is. let metadata = rehashing_metadata.fetch_sub(1, Release) - 1; - (metadata & (BUCKET_LEN - 1) == 0) && metadata >= old_array.len() + (metadata & (BUCKET_LEN * 2 - 1) == 0) && metadata >= old_array.len() } else { // On failure, `rehashing` reverts to its previous state. let mut current = rehashing_metadata.load(Relaxed); @@ -1141,7 +1141,7 @@ where let new = if current <= prev { current - 1 } else { - let refs = current & (BUCKET_LEN - 1); + let refs = current & (BUCKET_LEN * 2 - 1); prev | (refs - 1) }; match rehashing_metadata.compare_exchange_weak(current, new, Release, Relaxed) { @@ -1172,7 +1172,7 @@ where }); for bucket_index in - rehashing_guard.1..(rehashing_guard.1 + BUCKET_LEN).min(old_array.len()) + rehashing_guard.1..(rehashing_guard.1 + BUCKET_LEN * 2).min(old_array.len()) { let old_bucket = rehashing_guard.0.bucket(bucket_index); let writer = Writer::lock_async(old_bucket, async_guard).await; @@ -1218,7 +1218,7 @@ where }); for bucket_index in - rehashing_guard.1..(rehashing_guard.1 + BUCKET_LEN).min(old_array.len()) + rehashing_guard.1..(rehashing_guard.1 + BUCKET_LEN * 2).min(old_array.len()) { let old_bucket = rehashing_guard.0.bucket(bucket_index); let writer = if TRY_LOCK { @@ -1321,28 +1321,8 @@ where } else if current_array.has_linked_array() { // Cannot resize with a bucket array linked to the current bucket array. return; - } - - if self - .minimum_capacity_var() - .fetch_update(AcqRel, Acquire, |lock_state| { - if lock_state >= RESIZING { - None - } else { - Some(lock_state + RESIZING) - } - }) - .is_err() - { - // The bucket array is being replaced with a new one. - return; - } - let _lock_guard = ExitGuard::new((), |()| { - self.minimum_capacity_var().fetch_sub(RESIZING, Release); - }); - - if self.bucket_array_var().load(Acquire, guard) != current_array_ptr { - // Resized in the meantime. + } else if self.minimum_capacity_var().load(Relaxed) >= RESIZING { + // The table is being resized. return; } @@ -1385,6 +1365,29 @@ where return; } + if self + .minimum_capacity_var() + .fetch_update(AcqRel, Acquire, |lock_state| { + if lock_state >= RESIZING { + None + } else { + Some(lock_state + RESIZING) + } + }) + .is_err() + { + // The bucket array is being replaced with a new one. + return; + } + let _lock_guard = ExitGuard::new((), |()| { + self.minimum_capacity_var().fetch_sub(RESIZING, Release); + }); + + if self.bucket_array_var().load(Acquire, guard) != current_array_ptr { + // Resized in the meantime. + return; + } + if try_drop_table { // Try to drop the hash table with all the buckets locked. let mut writer_guard = ExitGuard::new((0, false), |(len, success): (usize, bool)| { @@ -1448,7 +1451,6 @@ pub(super) const MAXIMUM_CAPACITY_LIMIT: usize = 1_usize << (usize::BITS - 2); pub(super) const RESIZING: usize = 1_usize << (usize::BITS - 1); /// [`LockedBucket`] has exclusive access to a [`Bucket`]. -#[derive(Debug)] pub(crate) struct LockedBucket { /// Holds an exclusive lock on the [`Bucket`]. pub writer: Writer, @@ -1573,7 +1575,7 @@ impl LockedBucket where H: BuildHasher, { - if entry_ptr.move_to_next(&self.writer) { + if entry_ptr.find_next(&self.writer) { return Some(self); } @@ -1594,11 +1596,11 @@ impl LockedBucket hash_table .for_each_writer_async(next_index, len, |locked_bucket, _| { *entry_ptr = EntryPtr::null(); - if entry_ptr.move_to_next(&locked_bucket.writer) { + if entry_ptr.find_next(&locked_bucket.writer) { next_entry = Some(locked_bucket); - return true; + return false; } - false + true }) .await; @@ -1615,7 +1617,7 @@ impl LockedBucket where H: BuildHasher, { - if entry_ptr.move_to_next(&self.writer) { + if entry_ptr.find_next(&self.writer) { return Some(self); } @@ -1635,11 +1637,11 @@ impl LockedBucket let mut next_entry = None; hash_table.for_each_writer_sync(next_index, len, &Guard::new(), |locked_bucket, _| { *entry_ptr = EntryPtr::null(); - if entry_ptr.move_to_next(&locked_bucket.writer) { + if entry_ptr.find_next(&locked_bucket.writer) { next_entry = Some(locked_bucket); - return true; + return false; } - false + true }); next_entry @@ -1679,6 +1681,7 @@ const fn from_index_to_range(from_len: usize, to_len: usize, from_index: usize) } /// Turns a reference into a [`NonNull`] pointer. +#[inline] const fn into_non_null(t: &T) -> NonNull { unsafe { NonNull::new_unchecked(from_ref(t).cast_mut()) } } diff --git a/patch/scc-3.4.8/src/hash_table/bucket.rs b/patch/scc-3.4.8/src/hash_table/bucket.rs index 696c865..daeb73b 100644 --- a/patch/scc-3.4.8/src/hash_table/bucket.rs +++ b/patch/scc-3.4.8/src/hash_table/bucket.rs @@ -1,7 +1,6 @@ use std::cell::UnsafeCell; -use std::fmt::{self, Debug}; use std::mem::{MaybeUninit, forget, needs_drop}; -use std::ops::{Deref, Index}; +use std::ops::Deref; use std::ptr::{self, NonNull, from_ref}; use std::sync::atomic::Ordering::{Acquire, Relaxed, Release}; use std::sync::atomic::{AtomicPtr, AtomicU32, AtomicUsize}; @@ -20,10 +19,10 @@ use crate::async_helper::AsyncGuard; pub struct Bucket { /// Number of entries in the [`Bucket`]. len: AtomicUsize, - /// [`Bucket`] metadata. - metadata: Metadata, /// Reader-writer lock. rw_lock: Lock, + /// [`Bucket`] metadata. + metadata: Metadata, /// The LRU list of the [`Bucket`]. lru_list: L, } @@ -44,13 +43,11 @@ pub const BUCKET_LEN: usize = u32::BITS as usize; pub struct DataBlock([UnsafeCell>; LEN]); /// [`Writer`] holds an exclusive lock on a [`Bucket`]. -#[derive(Debug)] pub struct Writer { bucket_ptr: NonNull>, } /// [`Reader`] holds a shared lock on a [`Bucket`]. -#[derive(Debug)] pub struct Reader { bucket_ptr: NonNull>, } @@ -60,7 +57,7 @@ pub struct EntryPtr { /// Pointer to a [`LinkedBucket`]. link_ptr: *const LinkedBucket, /// Index of the entry. - index: usize, + index: u32, } /// Doubly-linked list interfaces to efficiently manage least-recently-used entries. @@ -85,21 +82,21 @@ pub trait LruList: 'static + Default { } /// [`DoublyLinkedList`] is an array of `(u8, u8)` implementing [`LruList`]. -#[derive(Debug, Default)] +#[derive(Default)] pub struct DoublyLinkedList([UnsafeCell<(u8, u8)>; BUCKET_LEN]); /// [`Metadata`] is a collection of metadata fields of [`Bucket`] and [`LinkedBucket`]. struct Metadata { - /// Linked list of entries. - link: AtomicShared>, /// Occupied slot bitmap. occupied_bitmap: AtomicU32, - /// Removed slot bitmap, or the 1-based index of the most recently used entry if - /// `TYPE = CACHE` where `0` represents `nil`. + /// Removed slot bitmap, or the 1-based index of the most recently used entry where `0` + /// represents `nil` if `TYPE = CACHE`. removed_bitmap: AtomicU32, /// Partial hash array for fast hash lookup, or the epoch when the corresponding entry was /// removed if `TYPE = INDEX`. - partial_hash_array: [UnsafeCell; LEN], + partial_hash_array: UnsafeCell<[u8; LEN]>, + /// Linked list of entries. + link: AtomicShared>, } /// The size of the linked data block. @@ -110,10 +107,10 @@ const LINKED_BUCKET_LEN: usize = BUCKET_LEN / 4; struct LinkedBucket { /// [`LinkedBucket`] metadata. metadata: Metadata, - /// Own data block. - data_block: DataBlock, /// Previous [`LinkedBucket`]. prev_link: AtomicPtr>, + /// Own data block. + data_block: DataBlock, } impl Bucket { @@ -124,10 +121,10 @@ impl Bucket { len: AtomicUsize::new(0), rw_lock: Lock::default(), metadata: Metadata { - link: AtomicShared::default(), occupied_bitmap: AtomicU32::default(), removed_bitmap: AtomicU32::default(), - partial_hash_array: Default::default(), + partial_hash_array: UnsafeCell::new(Default::default()), + link: AtomicShared::default(), }, lru_list: L::default(), } @@ -147,28 +144,31 @@ impl Bucket { hash: u64, entry: (K, V), ) -> EntryPtr { + let partial_hash = Self::partial_hash(hash); let occupied_bitmap = self.metadata.occupied_bitmap.load(Relaxed); let occupied_bitmap = if TYPE == INDEX + && partial_hash % 8 == 0 && occupied_bitmap == u32::MAX - && self.metadata.removed_bitmap.load(Relaxed) != 0 + && (self.metadata.removed_bitmap.load(Relaxed) != 0 + || !self.metadata.link.is_null(Relaxed)) { self.clear_unreachable_entries(data_block_ref(data_block)) } else { occupied_bitmap }; - let free_index = occupied_bitmap.trailing_ones() as usize; - if free_index == BUCKET_LEN { - self.insert_overflow(hash, entry) + + let free_index = occupied_bitmap.trailing_ones(); + if free_index as usize == BUCKET_LEN { + self.insert_overflow(partial_hash, entry) } else { self.insert_entry( &self.metadata, data_block_ref(data_block), free_index, occupied_bitmap, - hash, + partial_hash, entry, ); - self.len.store(self.len.load(Relaxed) + 1, Relaxed); EntryPtr { link_ptr: ptr::null(), index: free_index, @@ -184,8 +184,8 @@ impl Bucket { entry_ptr: &mut EntryPtr, ) -> (K, V) { debug_assert_ne!(TYPE, INDEX); - debug_assert_ne!(entry_ptr.index, usize::MAX); - debug_assert_ne!(entry_ptr.index, BUCKET_LEN); + debug_assert_ne!(entry_ptr.index, u32::MAX); + debug_assert_ne!(entry_ptr.index as usize, BUCKET_LEN); self.len.store(self.len.load(Relaxed) - 1, Relaxed); @@ -197,9 +197,9 @@ impl Bucket { link.metadata .occupied_bitmap .store(occupied_bitmap, Relaxed); - let removed = Self::read_data_block(&link.data_block, entry_ptr.index); + let removed = link.data_block.read_entry(entry_ptr.index); if occupied_bitmap == 0 && TYPE != INDEX { - entry_ptr.unlink(&self.metadata.link, link); + entry_ptr.unlink(&self.metadata.link); } removed } else { @@ -213,7 +213,7 @@ impl Bucket { self.metadata .occupied_bitmap .store(occupied_bitmap & !(1_u32 << entry_ptr.index), Relaxed); - Self::read_data_block(data_block_ref(data_block), entry_ptr.index) + data_block_ref(data_block).read_entry(entry_ptr.index) } } @@ -221,26 +221,22 @@ impl Bucket { #[inline] pub(crate) fn mark_removed(&self, entry_ptr: &mut EntryPtr, guard: &Guard) { debug_assert_eq!(TYPE, INDEX); - debug_assert_ne!(entry_ptr.index, usize::MAX); - debug_assert_ne!(entry_ptr.index, BUCKET_LEN); + debug_assert_ne!(entry_ptr.index, u32::MAX); + debug_assert_ne!(entry_ptr.index as usize, BUCKET_LEN); self.len.store(self.len.load(Relaxed) - 1, Relaxed); if let Some(link) = link_ref(entry_ptr.link_ptr) { - self.write_cell(&link.metadata.partial_hash_array[entry_ptr.index], |h| { - *h = u8::from(guard.epoch()); - }); - + link.metadata + .update_partial_hash(entry_ptr.index, u8::from(guard.epoch())); let mut removed_bitmap = link.metadata.removed_bitmap.load(Relaxed); debug_assert_eq!(removed_bitmap & (1_u32 << entry_ptr.index), 0); removed_bitmap |= 1_u32 << entry_ptr.index; link.metadata.removed_bitmap.store(removed_bitmap, Release); } else { - self.write_cell(&self.metadata.partial_hash_array[entry_ptr.index], |h| { - *h = u8::from(guard.epoch()); - }); - + self.metadata + .update_partial_hash(entry_ptr.index, u8::from(guard.epoch())); let mut removed_bitmap = self.metadata.removed_bitmap.load(Relaxed); debug_assert_eq!(removed_bitmap & (1_u32 << entry_ptr.index), 0); @@ -258,13 +254,13 @@ impl Bucket { debug_assert_eq!(TYPE, CACHE); let occupied_bitmap = self.metadata.occupied_bitmap.load(Relaxed); - if occupied_bitmap == 0b1111_1111_1111_1111_1111_1111_1111_1111 { + if occupied_bitmap == u32::MAX { self.len.store(self.len.load(Relaxed) - 1, Relaxed); let tail = self.metadata.removed_bitmap.load(Relaxed); let evicted = if let Some((evicted, new_tail)) = self.lru_list.evict(tail) { self.metadata.removed_bitmap.store(new_tail, Relaxed); - evicted as usize + u32::from(evicted) } else { // Evict the first occupied entry. 0 @@ -274,7 +270,7 @@ impl Bucket { self.metadata .occupied_bitmap .store(occupied_bitmap & !(1_u32 << evicted), Relaxed); - return Some(Self::read_data_block(data_block_ref(data_block), evicted)); + return Some(data_block_ref(data_block).read_entry(evicted)); } None @@ -284,8 +280,8 @@ impl Bucket { #[inline] pub(crate) fn update_lru_tail(&self, entry_ptr: &EntryPtr) { debug_assert_eq!(TYPE, CACHE); - debug_assert_ne!(entry_ptr.index, usize::MAX); - debug_assert_ne!(entry_ptr.index, BUCKET_LEN); + debug_assert_ne!(entry_ptr.index, u32::MAX); + debug_assert_ne!(entry_ptr.index as usize, BUCKET_LEN); if entry_ptr.link_ptr.is_null() { #[allow(clippy::cast_possible_truncation)] @@ -302,27 +298,40 @@ impl Bucket { pub(crate) fn reserve_slots(&self, additional: usize) { debug_assert!(self.rw_lock.is_locked(Relaxed)); - let mut capacity = - BUCKET_LEN - self.metadata.occupied_bitmap.load(Relaxed).count_ones() as usize; - if capacity < additional { - let mut link_ptr = self.metadata.load_link(); - while let Some(link) = link_ref(link_ptr) { - capacity += LINKED_BUCKET_LEN - - link.metadata.occupied_bitmap.load(Relaxed).count_ones() as usize; - if capacity >= additional { - return; - } - let mut next_link_ptr = link.metadata.load_link(); - if next_link_ptr.is_null() { - let new_link = unsafe { Shared::new_unchecked(LinkedBucket::new(None)) }; - new_link.prev_link.store(link_ptr.cast_mut(), Relaxed); - next_link_ptr = new_link.as_ptr(); - link.metadata - .link - .swap((Some(new_link), Tag::None), Release); - } - link_ptr = next_link_ptr; + let required = additional + self.len(); + let mut capacity = BUCKET_LEN; + if capacity >= required { + return; + } + + let mut link_ptr = self.metadata.load_link(); + while let Some(link) = link_ref(link_ptr) { + capacity += LINKED_BUCKET_LEN; + if capacity >= required { + return; } + let new_link_ptr = link.metadata.load_link(); + if new_link_ptr.is_null() { + break; + } + link_ptr = new_link_ptr; + } + + // Allocate additional overflow buckets. + for _ in 0..(required - capacity).div_ceil(LINKED_BUCKET_LEN) { + let new_link = LinkedBucket::new(None); + let new_link_ptr = new_link.as_ptr(); + if let Some(link) = link_ref(link_ptr) { + new_link.prev_link.store(link_ptr.cast_mut(), Relaxed); + link.metadata + .link + .swap((Some(new_link), Tag::None), Release); + } else { + self.metadata + .link + .swap((Some(new_link), Tag::None), Release); + } + link_ptr = new_link_ptr; } } @@ -339,9 +348,9 @@ impl Bucket { debug_assert!(self.rw_lock.is_locked(Relaxed)); let entry = if let Some(link) = link_ref(from_entry_ptr.link_ptr) { - Self::read_data_block(&link.data_block, from_entry_ptr.index) + link.data_block.read_entry(from_entry_ptr.index) } else { - Self::read_data_block(data_block_ref(from_data_block), from_entry_ptr.index) + data_block_ref(from_data_block).read_entry(from_entry_ptr.index) }; self.insert(data_block, hash, entry); @@ -383,29 +392,35 @@ impl Bucket { let mut occupied_bitmap = self.metadata.occupied_bitmap.load(Relaxed); while occupied_bitmap != 0 { let index = occupied_bitmap.trailing_zeros(); - Self::drop_entry(data_block_ref(data_block), index as usize); + data_block_ref(data_block).drop_entry(index); occupied_bitmap -= 1_u32 << index; } } } + /// Returns the partial hash value of the given hash. + #[allow(clippy::cast_possible_truncation)] + #[inline] + const fn partial_hash(hash: u64) -> u8 { + hash as u8 + } + /// Inserts an entry into an overflow bucket. - fn insert_overflow(&self, hash: u64, entry: (K, V)) -> EntryPtr { + fn insert_overflow(&self, partial_hash: u8, entry: (K, V)) -> EntryPtr { let mut link_ptr = self.metadata.load_link(); while let Some(link) = link_ref(link_ptr) { let occupied_bitmap = link.metadata.occupied_bitmap.load(Relaxed); - let free_index = occupied_bitmap.trailing_ones() as usize; - if free_index != LINKED_BUCKET_LEN { - debug_assert!(free_index < LINKED_BUCKET_LEN); + let free_index = occupied_bitmap.trailing_ones(); + if free_index as usize != LINKED_BUCKET_LEN { + debug_assert!((free_index as usize) < LINKED_BUCKET_LEN); self.insert_entry( &link.metadata, &link.data_block, free_index, occupied_bitmap, - hash, + partial_hash, entry, ); - self.len.store(self.len.load(Relaxed) + 1, Relaxed); return EntryPtr { link_ptr, index: free_index, @@ -416,20 +431,13 @@ impl Bucket { // Insert a new `LinkedBucket` at the linked list head. let head = self.metadata.link.get_shared(Relaxed, fake_ref(self)); - let link = unsafe { Shared::new_unchecked(LinkedBucket::new(head)) }; - self.write_cell(&link.data_block[0], |block| unsafe { - block.as_mut_ptr().write(entry); - }); - self.write_cell(&link.metadata.partial_hash_array[0], |h| { - *h = Self::partial_hash(hash); - }); - link.metadata.occupied_bitmap.store(1, Relaxed); + let link = LinkedBucket::new(head); + self.insert_entry(&link.metadata, &link.data_block, 0, 1, partial_hash, entry); if let Some(head) = link_ref(link.metadata.load_link()) { head.prev_link.store(link.as_ptr().cast_mut(), Relaxed); } let link_ptr = link.as_ptr(); self.metadata.link.swap((Some(link), Tag::None), Release); - self.len.store(self.len.load(Relaxed) + 1, Relaxed); EntryPtr { link_ptr, index: 0 } } @@ -439,24 +447,23 @@ impl Bucket { &self, metadata: &Metadata, data_block: &DataBlock, - index: usize, + index: u32, occupied_bitmap: u32, - hash: u64, + partial_hash: u8, entry: (K, V), ) { - debug_assert!(index < LEN); + debug_assert!((index as usize) < LEN); debug_assert_eq!(metadata.occupied_bitmap.load(Relaxed) & (1_u32 << index), 0); - self.write_cell(&data_block[index], |block| unsafe { - block.as_mut_ptr().write(entry); - }); - self.write_cell(&metadata.partial_hash_array[index], |h| { - *h = Self::partial_hash(hash); - }); + unsafe { + data_block.entry_ptr(index).cast_mut().write(entry); + } + metadata.update_partial_hash(index, partial_hash); metadata.occupied_bitmap.store( occupied_bitmap | (1_u32 << index), if TYPE == INDEX { Release } else { Relaxed }, ); + self.len.store(self.len.load(Relaxed) + 1, Relaxed); } /// Clears unreachable entries. @@ -475,7 +482,7 @@ impl Bucket { && next_link_ptr.is_null() { debug_assert!(link.metadata.link.is_null(Relaxed)); - let unlinked = if let Some(prev) = unsafe { prev_link_ptr.as_ref() } { + let unlinked = if let Some(prev) = link_ref(prev_link_ptr) { prev.metadata.link.swap((None, Tag::None), Acquire).0 } else { self.metadata.link.swap((None, Tag::None), Acquire).0 @@ -505,15 +512,16 @@ impl Bucket { let mut dropped_bitmap = metadata.removed_bitmap.load(Relaxed); let current_epoch = guard.epoch(); - for i in 0..LEN { - if Epoch::try_from(*Self::read_cell(&metadata.partial_hash_array[i])) + #[allow(clippy::cast_possible_truncation)] + for i in 0..LEN as u32 { + if Epoch::try_from(metadata.read_partial_hash(i)) .is_ok_and(|e| e.in_same_generation(current_epoch)) { dropped_bitmap &= !(1_u32 << i); } } - // Store ordering: `occupied_bitmap` -> `release` -> `removed_bitmap`. + // Store order: `occupied_bitmap` -> `release` -> `removed_bitmap`. let occupied_bitmap = metadata.occupied_bitmap.load(Relaxed) & !dropped_bitmap; metadata.occupied_bitmap.store(occupied_bitmap, Release); let removed_bitmap = metadata.removed_bitmap.load(Relaxed) & !dropped_bitmap; @@ -525,7 +533,7 @@ impl Bucket { if needs_drop::<(K, V)>() { while dropped_bitmap != 0 { let index = dropped_bitmap.trailing_zeros(); - Self::drop_entry(data_block, index as usize); + data_block.drop_entry(index); dropped_bitmap -= 1_u32 << index; } } @@ -533,20 +541,12 @@ impl Bucket { occupied_bitmap } - /// Drops the data in place. - #[inline] - fn drop_entry(data_block: &DataBlock, index: usize) { - unsafe { - (*data_block[index].get()).as_mut_ptr().drop_in_place(); - } - } - /// Removes the entry from the LRU linked list. #[inline] fn remove_from_lru_list(&self, entry_ptr: &EntryPtr) { debug_assert_eq!(TYPE, CACHE); - debug_assert_ne!(entry_ptr.index, usize::MAX); - debug_assert_ne!(entry_ptr.index, BUCKET_LEN); + debug_assert_ne!(entry_ptr.index, u32::MAX); + debug_assert_ne!(entry_ptr.index as usize, BUCKET_LEN); if entry_ptr.link_ptr.is_null() { #[allow(clippy::cast_possible_truncation)] @@ -557,53 +557,6 @@ impl Bucket { } } } - - /// Returns the partial hash value of the given hash. - #[allow(clippy::cast_possible_truncation)] - #[inline] - const fn partial_hash(hash: u64) -> u8 { - hash as u8 - } - - /// Reads the data at the given index. - #[inline] - const fn read_data_block( - data_block: &DataBlock, - index: usize, - ) -> (K, V) { - unsafe { (*data_block.0[index].get()).as_ptr().read() } - } - - /// Returns a pointer to the slot in the [`DataBlock`]. - #[inline] - const fn entry_ptr( - data_block: &DataBlock, - index: usize, - ) -> *const (K, V) { - Self::read_cell(&data_block.0[index]).as_ptr() - } - - /// Returns a mutable pointer to the slot in the [`DataBlock`]. - #[inline] - const fn entry_mut_ptr( - data_block: &DataBlock, - index: usize, - ) -> *mut (K, V) { - unsafe { (*data_block.0[index].get()).as_mut_ptr() } - } - - /// Reads the cell. - #[inline] - const fn read_cell(cell: &UnsafeCell) -> &T { - unsafe { &*cell.get() } - } - - /// Writes the cell. - #[inline] - fn write_cell R>(&self, cell: &UnsafeCell, f: F) -> R { - debug_assert!(self.rw_lock.is_locked(Relaxed)); - unsafe { f(&mut *cell.get()) } - } } impl Bucket { @@ -621,23 +574,26 @@ impl Bucket { Q: Equivalent + ?Sized, { if self.len() != 0 { - if let Some((entry, _)) = - Self::search_data_block(&self.metadata, data_block_ref(data_block), key, hash) - { + let partial_hash = Self::partial_hash(hash); + if let Some((entry, _)) = Self::search_data_block( + &self.metadata, + data_block_ref(data_block), + key, + partial_hash, + ) { return Some(entry); } let mut link_ptr = self.metadata.load_link(); while let Some(link) = link_ref(link_ptr) { if let Some((entry, _)) = - Self::search_data_block(&link.metadata, &link.data_block, key, hash) + Self::search_data_block(&link.metadata, &link.data_block, key, partial_hash) { return Some(entry); } link_ptr = link.metadata.load_link(); } } - None } @@ -655,9 +611,13 @@ impl Bucket { Q: Equivalent + ?Sized, { if self.len() != 0 { - if let Some((_, index)) = - Self::search_data_block(&self.metadata, data_block_ref(data_block), key, hash) - { + let partial_hash = Self::partial_hash(hash); + if let Some((_, index)) = Self::search_data_block( + &self.metadata, + data_block_ref(data_block), + key, + partial_hash, + ) { return EntryPtr { link_ptr: ptr::null(), index, @@ -667,7 +627,7 @@ impl Bucket { let mut current_link_ptr = self.metadata.load_link(); while let Some(link) = link_ref(current_link_ptr) { if let Some((_, index)) = - Self::search_data_block(&link.metadata, &link.data_block, key, hash) + Self::search_data_block(&link.metadata, &link.data_block, key, partial_hash) { return EntryPtr { link_ptr: current_link_ptr, @@ -686,33 +646,29 @@ impl Bucket { metadata: &Metadata, data_block: &'g DataBlock, key: &Q, - hash: u64, - ) -> Option<(&'g (K, V), usize)> + partial_hash: u8, + ) -> Option<(&'g (K, V), u32)> where Q: Equivalent + ?Sized, { - let mut bitmap = if TYPE == INDEX { - // Load ordering: `removed_bitmap` -> `acquire` -> `occupied_bitmap`. - (!metadata.removed_bitmap.load(Acquire)) & metadata.occupied_bitmap.load(Acquire) - } else { - metadata.occupied_bitmap.load(Relaxed) - }; + let mut bitmap = metadata.bitmap::(); - // Expect that the loop is vectorized by the compiler. - for i in 0..LEN { - if *Self::read_cell(&metadata.partial_hash_array[i]) != Self::partial_hash(hash) { + // Expect that the loop is vectorized by the compiler (https://godbolt.org/z/bcWYsaPbY). + let partial_hash_array = unsafe { &*metadata.partial_hash_array.get() }; + (0..LEN).for_each(|i| { + if partial_hash_array[i] != partial_hash { bitmap &= !(1_u32 << i); } - } + }); - let mut offset = bitmap.trailing_zeros(); - while offset != u32::BITS { - let entry = unsafe { &*Self::entry_ptr(data_block, offset as usize) }; + let mut index = bitmap.trailing_zeros(); + while index != u32::BITS { + let entry = unsafe { &*data_block.entry_ptr(index) }; if key.equivalent(&entry.0) { - return Some((entry, offset as usize)); + return Some((entry, index)); } - bitmap &= !(1_u32 << offset); - offset = bitmap.trailing_zeros(); + bitmap -= 1_u32 << index; + index = bitmap.trailing_zeros(); } None @@ -725,12 +681,32 @@ unsafe impl S { } -impl Index for DataBlock { - type Output = UnsafeCell>; - +impl DataBlock { + /// Creates a new [`DataBlock`]. + #[allow(clippy::uninit_assumed_init)] #[inline] - fn index(&self, index: usize) -> &Self::Output { - &self.0[index] + const fn new() -> DataBlock { + DataBlock(unsafe { MaybeUninit::uninit().assume_init() }) + } + + /// Returns a pointer to the entry at the given index. + #[inline] + const fn entry_ptr(&self, index: u32) -> *const (K, V) { + self.0[index as usize].get().cast::<(K, V)>() + } + + /// Reads the entry at the given index. + #[inline] + const fn read_entry(&self, index: u32) -> (K, V) { + unsafe { self.entry_ptr(index).read() } + } + + /// Drops the entry at the given index. + #[inline] + fn drop_entry(&self, index: u32) { + unsafe { + self.entry_ptr(index).cast_mut().drop_in_place(); + } } } @@ -738,6 +714,14 @@ unsafe impl Send for DataBlock {} unsafe impl Sync for DataBlock {} impl Writer { + /// Creates a new [`Writer`] from a [`Bucket`]. + #[inline] + pub(crate) const fn from_bucket(bucket: &Bucket) -> Writer { + Writer { + bucket_ptr: bucket_ptr(bucket), + } + } + /// Locks the [`Bucket`] asynchronously. #[inline] pub(crate) async fn lock_async<'g>( @@ -777,14 +761,6 @@ impl Writer { } } - /// Creates a new [`Writer`] from a [`Bucket`]. - #[inline] - pub(crate) const fn from_bucket(bucket: &Bucket) -> Writer { - Writer { - bucket_ptr: bucket_ptr(bucket), - } - } - /// Marks the [`Bucket`] killed by poisoning the lock. #[inline] pub(super) fn kill(self) { @@ -800,9 +776,9 @@ impl Writer { debug_assert!(poisoned); if (TYPE != INDEX || !needs_drop::<(K, V)>()) && !self.metadata.link.is_null(Relaxed) { - // In case `TYPE == INDEX`, overflow buckets may contain removed entries that may be - // still accessible to readers; those `(K, V)` that need `drop` should be dropped in - // `drop_entries`. + // In case `TYPE == INDEX`, `(K, V)` that need `drop` should be dropped in + // `drop_entries` to make sure that they are dropped before the container is dropped; + // they should never be passed to the garbage collector. let mut link = self.metadata.link.swap((None, Tag::None), Acquire).0; while let Some(current) = link { link = current.metadata.link.swap((None, Tag::None), Acquire).0; @@ -906,14 +882,28 @@ impl EntryPtr { pub(crate) const fn null() -> Self { Self { link_ptr: ptr::null(), - index: BUCKET_LEN, + index: u32::MAX, + } + } + + #[inline] + pub(crate) const fn clone(&self) -> Self { + Self { + link_ptr: self.link_ptr, + index: self.index, } } /// Returns `true` if the [`EntryPtr`] points to, or has pointed to, an occupied entry. #[inline] pub(crate) const fn is_valid(&self) -> bool { - self.index != BUCKET_LEN + self.index != u32::MAX + } + + /// Returns the current index. + #[inline] + pub(crate) const fn index(&self) -> u32 { + self.index } /// Gets the partial hash value of the entry. @@ -922,9 +912,9 @@ impl EntryPtr { #[inline] pub(crate) const fn partial_hash(&self, bucket: &Bucket) -> u8 { if let Some(link) = link_ref(self.link_ptr) { - *Bucket::::read_cell(&link.metadata.partial_hash_array[self.index]) + link.metadata.read_partial_hash(self.index) } else { - *Bucket::::read_cell(&bucket.metadata.partial_hash_array[self.index]) + bucket.metadata.read_partial_hash(self.index) } } @@ -937,9 +927,9 @@ impl EntryPtr { data_block: NonNull>, ) -> &'e (K, V) { let entry_ptr = if let Some(link) = link_ref(self.link_ptr) { - Bucket::::entry_ptr(&link.data_block, self.index) + link.data_block.entry_ptr(self.index) } else { - Bucket::::entry_ptr(data_block_ref(data_block), self.index) + data_block_ref(data_block).entry_ptr(self.index) }; unsafe { &(*entry_ptr) } } @@ -954,55 +944,51 @@ impl EntryPtr { _writer: &Writer, ) -> &mut (K, V) { let entry_ptr = if let Some(link) = link_ref(self.link_ptr) { - Bucket::::entry_mut_ptr(&link.data_block, self.index) + link.data_block.entry_ptr(self.index) } else { - Bucket::::entry_mut_ptr(data_block_ref(data_block), self.index) + data_block_ref(data_block).entry_ptr(self.index) }; - unsafe { &mut (*entry_ptr) } + unsafe { &mut (*entry_ptr.cast_mut()) } } /// Moves the [`EntryPtr`] to point to the next occupied entry. /// /// Returns `true` if it successfully found the next occupied entry. #[inline] - pub(crate) fn move_to_next(&mut self, bucket: &Bucket) -> bool { - if self.index != usize::MAX { - if self.link_ptr.is_null() && self.next_entry::(&bucket.metadata) { + pub(crate) fn find_next(&mut self, bucket: &Bucket) -> bool { + if self.link_ptr.is_null() && self.next_entry::(&bucket.metadata) { + return true; + } + while let Some(link) = link_ref(self.link_ptr) { + if self.next_entry::(&link.metadata) { return true; } - while let Some(link) = link_ref(self.link_ptr) { - if self.next_entry::(&link.metadata) { - return true; - } - } - - // Fuse itself. - self.index = usize::MAX; } - false } /// Unlinks the [`LinkedBucket`] currently pointed to by this [`EntryPtr`] from the linked list. /// /// The associated [`Bucket`] must be locked. - fn unlink(&mut self, link_head: &AtomicShared>, link: &LinkedBucket) { + fn unlink(&mut self, link_head: &AtomicShared>) { debug_assert_ne!(TYPE, INDEX); - let prev_link_ptr = link.prev_link.load(Relaxed); - let next = link.metadata.link.swap((None, Tag::None), Relaxed).0; + let prev_link_ptr = + link_ref(self.link_ptr).map_or(ptr::null_mut(), |link| link.prev_link.load(Relaxed)); + let next = link_ref(self.link_ptr) + .and_then(|link| link.metadata.link.swap((None, Tag::None), Relaxed).0); if let Some(next) = next.as_ref() { - // Go to the next `Link`. + // Move the pointer to the next `Link`. next.prev_link.store(prev_link_ptr, Relaxed); self.link_ptr = next.as_ptr(); - self.index = LINKED_BUCKET_LEN; + self.index = u32::MAX; } else { - // Fuse the `EntryPtr`. - self.link_ptr = ptr::null(); - self.index = usize::MAX; + // Move the pointer to the previous `Link`. + self.link_ptr = prev_link_ptr; + self.index = u32::MAX - 1; } - let unlinked = if let Some(prev) = unsafe { prev_link_ptr.as_ref() } { + let unlinked = if let Some(prev) = link_ref(prev_link_ptr) { prev.metadata.link.swap((next, Tag::None), Acquire).0 } else { link_head.swap((next, Tag::None), Acquire).0 @@ -1019,51 +1005,28 @@ impl EntryPtr { #[inline] fn next_entry(&mut self, metadata: &Metadata) -> bool { // Search for the next occupied entry. - let current_index = if self.index == LEN { 0 } else { self.index + 1 }; + let current_index = if self.index == u32::MAX { + 0 + } else { + self.index + 1 + }; - if current_index < LEN { - let bitmap = if TYPE == INDEX { - // Load order: `removed_bitmap` -> `acquire` -> `occupied_bitmap`. - (!metadata.removed_bitmap.load(Acquire) & metadata.occupied_bitmap.load(Acquire)) - & (!((1_u32 << current_index) - 1)) - } else { - metadata.occupied_bitmap.load(Relaxed) & (!((1_u32 << current_index) - 1)) - }; - - let next_index = bitmap.trailing_zeros() as usize; - if next_index < LEN { + if (current_index as usize) < LEN { + let bitmap = metadata.bitmap::() & (!((1_u32 << current_index) - 1)); + let next_index = bitmap.trailing_zeros(); + if (next_index as usize) < LEN { self.index = next_index; return true; } } self.link_ptr = metadata.load_link(); - self.index = LINKED_BUCKET_LEN; + self.index = u32::MAX; false } } -impl Clone for EntryPtr { - #[inline] - fn clone(&self) -> Self { - Self { - link_ptr: self.link_ptr, - index: self.index, - } - } -} - -impl Debug for EntryPtr { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("EntryPtr") - .field("link_ptr", &self.link_ptr) - .field("index", &self.index) - .finish() - } -} - unsafe impl Send for EntryPtr {} unsafe impl Sync for EntryPtr {} @@ -1072,14 +1035,14 @@ impl LruList for () {} impl DoublyLinkedList { /// Reads the slot. #[inline] - const fn read(&self, index: usize) -> (u8, u8) { - unsafe { *self.0[index].get() } + fn read(&self, index: u32) -> (u8, u8) { + unsafe { *self.0.get_unchecked(index as usize).get() } } /// Writes the slot. #[inline] - fn write R>(&self, index: usize, f: F) -> R { - unsafe { f(&mut *self.0[index].get()) } + fn write R>(&self, index: u32, f: F) -> R { + unsafe { f(&mut *self.0.get_unchecked(index as usize).get()) } } } @@ -1089,24 +1052,24 @@ impl LruList for DoublyLinkedList { if tail == 0 { None } else { - let lru = self.read(tail as usize - 1).0; + let lru = self.read(tail - 1).0; let new_tail = if tail - 1 == u32::from(lru) { // Reset the linked list. 0 } else { - let new_lru = self.read(lru as usize).0; + let new_lru = self.read(u32::from(lru)).0; { #![allow(clippy::cast_possible_truncation)] - self.write(new_lru as usize, |v| { + self.write(u32::from(new_lru), |v| { v.1 = tail as u8 - 1; }); } - self.write(tail as usize - 1, |v| { + self.write(tail - 1, |v| { v.0 = new_lru; }); tail }; - self.write(lru as usize, |v| { + self.write(u32::from(lru), |v| { *v = (0, 0); }); Some((lru, new_tail)) @@ -1116,32 +1079,32 @@ impl LruList for DoublyLinkedList { #[inline] fn remove(&self, tail: u32, entry: u8) -> Option { if tail == 0 - || (self.read(entry as usize) == (0, 0) + || (self.read(u32::from(entry)) == (0, 0) && (self.read(0) != (entry, entry) || (tail != 1 && tail != u32::from(entry) + 1))) { // The linked list is empty, or the entry is not a part of the linked list. return None; } - if self.read(entry as usize).0 == entry { + if self.read(u32::from(entry)).0 == entry { // It is the head and the only entry of the linked list. debug_assert_eq!(tail, u32::from(entry) + 1); - self.write(entry as usize, |v| { + self.write(u32::from(entry), |v| { *v = (0, 0); }); return Some(0); } // Adjust `prev -> current`. - let (prev, next) = self.read(entry as usize); - debug_assert_eq!(self.read(prev as usize).1, entry); - self.write(prev as usize, |v| { + let (prev, next) = self.read(u32::from(entry)); + debug_assert_eq!(self.read(u32::from(prev)).1, entry); + self.write(u32::from(prev), |v| { v.1 = next; }); // Adjust `next -> current`. - debug_assert_eq!(self.read(next as usize).0, entry); - self.write(next as usize, |v| { + debug_assert_eq!(self.read(u32::from(next)).0, entry); + self.write(u32::from(next), |v| { v.0 = prev; }); @@ -1151,7 +1114,7 @@ impl LruList for DoublyLinkedList { } else { None }; - self.write(entry as usize, |v| { + self.write(u32::from(entry), |v| { *v = (0, 0); }); @@ -1165,45 +1128,45 @@ impl LruList for DoublyLinkedList { return None; } else if tail == 0 { // The linked list is empty. - self.write(entry as usize, |v| { + self.write(u32::from(entry), |v| { *v = (entry, entry); }); return Some(u32::from(entry) + 1); } // Remove the entry from the linked list only if it is a part of it. - if self.read(entry as usize) != (0, 0) || (self.read(0) == (entry, entry) && tail == 1) { + if self.read(u32::from(entry)) != (0, 0) || (self.read(0) == (entry, entry) && tail == 1) { // Adjust `prev -> current`. - let (prev, next) = self.read(entry as usize); - debug_assert_eq!(self.read(prev as usize).1, entry); - self.write(prev as usize, |v| { + let (prev, next) = self.read(u32::from(entry)); + debug_assert_eq!(self.read(u32::from(prev)).1, entry); + self.write(u32::from(prev), |v| { v.1 = next; }); // Adjust `next -> current`. - debug_assert_eq!(self.read(next as usize).0, entry); - self.write(next as usize, |v| { + debug_assert_eq!(self.read(u32::from(next)).0, entry); + self.write(u32::from(next), |v| { v.0 = prev; }); } // Adjust `oldest -> head`. - let oldest = self.read(tail as usize - 1).0; - debug_assert_eq!(u32::from(self.read(oldest as usize).1) + 1, tail); - self.write(oldest as usize, |v| { + let oldest = self.read(tail - 1).0; + debug_assert_eq!(u32::from(self.read(u32::from(oldest)).1) + 1, tail); + self.write(u32::from(oldest), |v| { v.1 = entry; }); - self.write(entry as usize, |v| { + self.write(u32::from(entry), |v| { v.0 = oldest; }); // Adjust `head -> new head` - self.write(tail as usize - 1, |v| { + self.write(tail - 1, |v| { v.0 = entry; }); { #![allow(clippy::cast_possible_truncation)] - self.write(entry as usize, |v| { + self.write(u32::from(entry), |v| { v.1 = tail as u8 - 1; }); } @@ -1217,6 +1180,41 @@ unsafe impl Send for DoublyLinkedList {} unsafe impl Sync for DoublyLinkedList {} impl Metadata { + /// Returns the partial hash at the given index. + #[inline] + const fn read_partial_hash(&self, index: u32) -> u8 { + unsafe { + self.partial_hash_array + .get() + .cast::() + .add(index as usize) + .read() + } + } + + /// Updates the partial hash at the given index. + #[inline] + const fn update_partial_hash(&self, index: u32, partial_hash: u8) { + unsafe { + self.partial_hash_array + .get() + .cast::() + .add(index as usize) + .write(partial_hash); + } + } + + /// Returns a bitmap representing valid entries. + #[inline] + fn bitmap(&self) -> u32 { + if TYPE == INDEX { + // Load order: `removed_bitmap` -> `acquire` -> `occupied_bitmap`. + !self.removed_bitmap.load(Acquire) & self.occupied_bitmap.load(Acquire) + } else { + self.occupied_bitmap.load(Relaxed) + } + } + /// Loads the linked bucket pointer. #[inline] fn load_link(&self) -> *const LinkedBucket { @@ -1230,31 +1228,21 @@ unsafe impl Sync for Metadata< impl LinkedBucket { /// Creates an empty [`LinkedBucket`]. #[inline] - fn new(next: Option>>) -> Self { + fn new(next: Option>>) -> Shared { let mut bucket = Self { metadata: Metadata { - link: AtomicShared::default(), occupied_bitmap: AtomicU32::default(), removed_bitmap: AtomicU32::default(), - partial_hash_array: Default::default(), - }, - data_block: unsafe { - #[allow(clippy::uninit_assumed_init)] - MaybeUninit::uninit().assume_init() + partial_hash_array: UnsafeCell::new(Default::default()), + link: AtomicShared::default(), }, prev_link: AtomicPtr::default(), + data_block: DataBlock::new(), }; if let Some(next) = next { bucket.metadata.link = AtomicShared::from(next); } - bucket - } -} - -impl Debug for LinkedBucket { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("LinkedBucket").finish() + unsafe { Shared::new_unchecked(bucket) } } } @@ -1265,7 +1253,7 @@ impl Drop for LinkedBucket { let mut occupied_bitmap = self.metadata.occupied_bitmap.load(Relaxed); while occupied_bitmap != 0 { let index = occupied_bitmap.trailing_zeros(); - Bucket::::drop_entry(&self.data_block, index as usize); + self.data_block.drop_entry(index); occupied_bitmap -= 1_u32 << index; } } @@ -1306,7 +1294,6 @@ const fn fake_ref<'l, T, U>(v: &T) -> &'l U { mod test { use super::*; - use std::mem::MaybeUninit; use std::sync::atomic::AtomicPtr; use std::sync::atomic::Ordering::Relaxed; @@ -1323,8 +1310,7 @@ mod test { #[cfg_attr(miri, ignore)] #[test] fn evict_untracked(xs in 0..BUCKET_LEN * 2) { - let data_block: DataBlock = - unsafe { MaybeUninit::uninit().assume_init() }; + let data_block: DataBlock = DataBlock::new(); let data_block_ptr = unsafe { NonNull::new_unchecked(from_ref(&data_block).cast_mut()) }; let bucket: Bucket = Bucket::new(); @@ -1340,8 +1326,7 @@ mod test { #[cfg_attr(miri, ignore)] #[test] fn evict_overflowed(xs in 1..BUCKET_LEN * 2) { - let data_block: DataBlock = - unsafe { MaybeUninit::uninit().assume_init() }; + let data_block: DataBlock = DataBlock::new(); let data_block_ptr = unsafe { NonNull::new_unchecked(from_ref(&data_block).cast_mut()) }; let bucket: Bucket = Bucket::new(); @@ -1358,7 +1343,7 @@ mod test { } assert_eq!( writer.lru_list.read - (writer.metadata.removed_bitmap.load(Relaxed) as usize - 1) + (writer.metadata.removed_bitmap.load(Relaxed) - 1) .0, 0 ); @@ -1387,8 +1372,7 @@ mod test { #[cfg_attr(miri, ignore)] #[test] fn evict_tracked(xs in 0..BUCKET_LEN * 2) { - let data_block: DataBlock = - unsafe { MaybeUninit::uninit().assume_init() }; + let data_block: DataBlock = DataBlock::new(); let data_block_ptr = unsafe { NonNull::new_unchecked(from_ref(&data_block).cast_mut()) }; let bucket: Bucket = Bucket::new(); @@ -1399,28 +1383,28 @@ mod test { let mut entry_ptr = writer.insert(data_block_ptr, 0, (v, v)); writer.update_lru_tail(&entry_ptr); assert_eq!( - writer.metadata.removed_bitmap.load(Relaxed) as usize, + writer.metadata.removed_bitmap.load(Relaxed), entry_ptr.index + 1 ); if v >= BUCKET_LEN { - entry_ptr.index = xs % BUCKET_LEN; + entry_ptr.index = u32::try_from(xs % BUCKET_LEN).unwrap_or(0); writer.update_lru_tail(&entry_ptr); assert_eq!( - writer.metadata.removed_bitmap.load(Relaxed) as usize, + writer.metadata.removed_bitmap.load(Relaxed), entry_ptr.index + 1 ); let mut iterated = 1; - let mut i = writer.lru_list.read(entry_ptr.index).1 as usize; + let mut i = u32::from(writer.lru_list.read(entry_ptr.index).1); while i != entry_ptr.index { iterated += 1; - i = writer.lru_list.read(i).1 as usize; + i = u32::from(writer.lru_list.read(i).1); } assert_eq!(iterated, BUCKET_LEN); iterated = 1; - i = writer.lru_list.read(entry_ptr.index).0 as usize; + i = u32::from(writer.lru_list.read(entry_ptr.index).0); while i != entry_ptr.index { iterated += 1; - i = writer.lru_list.read(i).0 as usize; + i = u32::from(writer.lru_list.read(i).0); } assert_eq!(iterated, BUCKET_LEN); } @@ -1430,8 +1414,7 @@ mod test { #[cfg_attr(miri, ignore)] #[test] fn removed(xs in 0..BUCKET_LEN) { - let data_block: DataBlock = - unsafe { MaybeUninit::uninit().assume_init() }; + let data_block: DataBlock = DataBlock::new(); let data_block_ptr = unsafe { NonNull::new_unchecked(from_ref(&data_block).cast_mut()) }; let bucket: Bucket = Bucket::new(); @@ -1440,10 +1423,10 @@ mod test { let entry_ptr = writer.insert(data_block_ptr, 0, (v, v)); writer.update_lru_tail(&entry_ptr); let mut iterated = 1; - let mut i = writer.lru_list.read(entry_ptr.index).1 as usize; + let mut i = u32::from(writer.lru_list.read(entry_ptr.index).1); while i != entry_ptr.index { iterated += 1; - i = writer.lru_list.read(i).1 as usize; + i = u32::from(writer.lru_list.read(i).1); } assert_eq!(iterated, v + 1); } @@ -1453,10 +1436,10 @@ mod test { unsafe { NonNull::new_unchecked(from_ref(&data_block).cast_mut()) }; let entry_ptr = writer.get_entry_ptr(data_block_ptr, &v, 0); let mut iterated = 1; - let mut i = writer.lru_list.read(entry_ptr.index).1 as usize; + let mut i = u32::from(writer.lru_list.read(entry_ptr.index).1); while i != entry_ptr.index { iterated += 1; - i = writer.lru_list.read(i).1 as usize; + i = u32::from(writer.lru_list.read(i).1); } assert_eq!(iterated, xs - v); writer.remove_from_lru_list(&entry_ptr); @@ -1470,8 +1453,7 @@ mod test { async fn bucket_lock_sync() { let num_tasks = BUCKET_LEN + 2; let barrier = Shared::new(Barrier::new(num_tasks)); - let data_block: Shared> = - Shared::new(unsafe { MaybeUninit::uninit().assume_init() }); + let data_block: Shared> = Shared::new(DataBlock::new()); let mut bucket: Shared> = Shared::new(Bucket::new()); let mut data: [u64; 128] = [0; 128]; let mut task_handles = Vec::with_capacity(num_tasks); @@ -1543,14 +1525,14 @@ mod test { let mut count = 0; let mut entry_ptr = EntryPtr::null(); - while entry_ptr.move_to_next(&bucket) { + while entry_ptr.find_next(&bucket) { count += 1; } assert_eq!(bucket.len(), count); entry_ptr = EntryPtr::null(); let writer = Writer::lock_sync(&bucket).unwrap(); - while entry_ptr.move_to_next(&writer) { + while entry_ptr.find_next(&writer) { writer.remove( unsafe { NonNull::new_unchecked(data_block.as_ptr().cast_mut()) }, &mut entry_ptr, diff --git a/patch/scc-3.4.8/src/hash_table/bucket_array.rs b/patch/scc-3.4.8/src/hash_table/bucket_array.rs index 94e961e..040c126 100644 --- a/patch/scc-3.4.8/src/hash_table/bucket_array.rs +++ b/patch/scc-3.4.8/src/hash_table/bucket_array.rs @@ -206,11 +206,9 @@ impl Drop for BucketArray { } else { self.num_cleared_buckets.load(Relaxed) }; - if num_cleared_buckets < self.array_len { - for index in num_cleared_buckets..self.array_len { - self.bucket(index).drop_entries(self.data_block(index)); - } - } + (num_cleared_buckets..self.array_len).for_each(|i| { + self.bucket(i).drop_entries(self.data_block(i)); + }); #[cfg(feature = "loom")] for i in 0..self.array_len { diff --git a/patch/scc-3.4.8/src/serde.rs b/patch/scc-3.4.8/src/serde.rs index b0a599b..1c4c762 100644 --- a/patch/scc-3.4.8/src/serde.rs +++ b/patch/scc-3.4.8/src/serde.rs @@ -355,7 +355,7 @@ where } /// Helper type to allow `serde` to access [`TreeIndex`] entries. -pub struct TreeIndexVisitor { +pub struct TreeIndexVisitor { #[allow(clippy::type_complexity)] marker: PhantomData TreeIndex>, } @@ -363,7 +363,6 @@ pub struct TreeIndexVisitor { impl TreeIndexVisitor where K: Clone + Ord, - V: Clone, { fn new() -> Self { TreeIndexVisitor { @@ -375,7 +374,7 @@ where impl<'d, K, V> Visitor<'d> for TreeIndexVisitor where K: 'static + Clone + Deserialize<'d> + Ord, - V: 'static + Clone + Deserialize<'d>, + V: 'static + Deserialize<'d>, { type Value = TreeIndex; @@ -398,7 +397,7 @@ where impl<'d, K, V> Deserialize<'d> for TreeIndex where K: 'static + Clone + Deserialize<'d> + Ord, - V: 'static + Clone + Deserialize<'d>, + V: 'static + Deserialize<'d>, { fn deserialize(deserializer: D) -> Result where @@ -411,7 +410,7 @@ where impl Serialize for TreeIndex where K: 'static + Clone + Ord + Serialize, - V: 'static + Clone + Serialize, + V: 'static + Serialize, { fn serialize(&self, serializer: S) -> Result where diff --git a/patch/scc-3.4.8/src/tests/unit_tests.rs b/patch/scc-3.4.8/src/tests/unit_tests.rs index fd12917..e95c370 100644 --- a/patch/scc-3.4.8/src/tests/unit_tests.rs +++ b/patch/scc-3.4.8/src/tests/unit_tests.rs @@ -983,11 +983,11 @@ mod hashmap { let task = tokio::task::spawn(async move { // test insert barrier_clone.wait().await; - let mut scanned = 0; + let mut iterated = 0; let mut checker = BTreeSet::new(); let mut max = inserted_clone.load(Acquire); hashmap_clone.retain_sync(|k, _| { - scanned += 1; + iterated += 1; checker.insert(*k); true }); @@ -996,12 +996,12 @@ mod hashmap { } barrier_clone.wait().await; - scanned = 0; + iterated = 0; checker = BTreeSet::new(); max = inserted_clone.load(Acquire); hashmap_clone .retain_async(|k, _| { - scanned += 1; + iterated += 1; checker.insert(*k); true }) @@ -1012,20 +1012,20 @@ mod hashmap { // test remove barrier_clone.wait().await; - scanned = 0; + iterated = 0; max = removed_clone.load(Acquire); hashmap_clone.retain_sync(|k, _| { - scanned += 1; + iterated += 1; assert!(*k < max); true }); barrier_clone.wait().await; - scanned = 0; + iterated = 0; max = removed_clone.load(Acquire); hashmap_clone .retain_async(|k, _| { - scanned += 1; + iterated += 1; assert!(*k < max); true }) @@ -1745,11 +1745,11 @@ mod hashindex { let task = tokio::task::spawn(async move { // test insert barrier_clone.wait().await; - let mut scanned = 0; + let mut iterated = 0; let mut checker = BTreeSet::new(); let mut max = inserted_clone.load(Acquire); hashindex_clone.retain_sync(|k, _| { - scanned += 1; + iterated += 1; checker.insert(*k); true }); @@ -1758,12 +1758,12 @@ mod hashindex { } barrier_clone.wait().await; - scanned = 0; + iterated = 0; checker = BTreeSet::new(); max = inserted_clone.load(Acquire); hashindex_clone .retain_async(|k, _| { - scanned += 1; + iterated += 1; checker.insert(*k); true }) @@ -1774,20 +1774,20 @@ mod hashindex { // test remove barrier_clone.wait().await; - scanned = 0; + iterated = 0; max = removed_clone.load(Acquire); hashindex_clone.retain_sync(|k, _| { - scanned += 1; + iterated += 1; assert!(*k < max); true }); barrier_clone.wait().await; - scanned = 0; + iterated = 0; max = removed_clone.load(Acquire); hashindex_clone .retain_async(|k, _| { - scanned += 1; + iterated += 1; assert!(*k < max); true }) @@ -2721,6 +2721,55 @@ mod treeindex { } } + #[test] + fn double_ended_iter() { + let workload_size = 16384; + let tree: TreeIndex = TreeIndex::default(); + for k in 0..workload_size { + assert!(tree.insert_sync(k, ()).is_ok()); + } + + let guard = Guard::new(); + let mut de_iter = tree.iter(&guard); + for k in 0..workload_size / 4 { + assert_eq!(de_iter.next_back(), Some((&(workload_size - k - 1), &()))); + } + for k in 0..workload_size / 4 { + assert_eq!(de_iter.next(), Some((&k, &()))); + } + for k in 0..workload_size / 4 { + assert_eq!( + de_iter.next_back(), + Some((&(workload_size - k - 1 - workload_size / 4), &())) + ); + } + for k in 0..workload_size / 4 { + assert_eq!(de_iter.next(), Some((&(k + workload_size / 4), &()))); + } + assert!(de_iter.next().is_none()); + assert!(de_iter.next_back().is_none()); + + // Only one end of the iterator was exhausted. + for rev in [true, false] { + let mut de_iter = tree.iter(&guard); + if rev { + let mut prev = usize::MAX; + while let Some((key, ())) = de_iter.next_back() { + assert!(prev > *key); + prev = *key; + } + assert!(de_iter.next().is_none()); + } else { + let mut prev = 0; + for (key, ()) in de_iter.by_ref() { + assert!(prev == 0 || prev < *key); + prev = *key; + } + assert!(de_iter.next_back().is_none()); + } + } + } + #[test] fn range() { let tree: TreeIndex = TreeIndex::default(); @@ -2946,9 +2995,9 @@ mod treeindex { } let guard = Guard::new(); - let scanner = tree.iter(&guard); + let iter = tree.iter(&guard); let mut prev = 0; - for entry in scanner { + for entry in iter { assert!(prev == 0 || prev < *entry.0); assert_eq!(*entry.0, *entry.1); prev = *entry.0; @@ -2957,93 +3006,101 @@ mod treeindex { #[test] fn insert_peek_remove_range_sync() { - let range = if cfg!(miri) { 8 } else { 4096 }; - let num_threads = if cfg!(miri) { 2 } else { 8 }; + let (num_threads, range) = if cfg!(miri) { (2, 8) } else { (8, 4096) }; let tree: Arc> = Arc::new(TreeIndex::new()); for t in 0..num_threads { - // insert markers + // Fixed points. assert!(tree.insert_sync(t * range, t * range).is_ok()); } let stopped: Arc = Arc::new(AtomicBool::new(false)); let barrier = Arc::new(Barrier::new(num_threads + 1)); let mut threads = Vec::with_capacity(num_threads); for thread_id in 0..num_threads { - let tree = tree.clone(); - let stopped = stopped.clone(); - let barrier = barrier.clone(); + let (tree, stopped, barrier) = (tree.clone(), stopped.clone(), barrier.clone()); threads.push(thread::spawn(move || { - let first_key = thread_id * range; + let first = thread_id * range; barrier.wait(); while !stopped.load(Relaxed) { - for key in (first_key + 1)..(first_key + range) { - assert!(tree.insert_sync(key, key).is_ok()); + for k in (first + 1)..(first + range) { + assert!(tree.insert_sync(k, k).is_ok()); } - for key in (first_key + 1)..(first_key + range) { - assert!( - tree.peek_with(&key, |key, val| assert_eq!(key, val)) - .is_some() - ); + for k in (first + 1)..(first + range) { + assert!(tree.peek_with(&k, |k, v| assert_eq!(k, v)).is_some()); } { let guard = Guard::new(); - let mut range_scanner = tree.range(first_key.., &guard); - let mut entry = range_scanner.next().unwrap(); - assert_eq!(entry, (&first_key, &first_key)); - entry = range_scanner.next().unwrap(); - assert_eq!(entry, (&(first_key + 1), &(first_key + 1))); - entry = range_scanner.next().unwrap(); - assert_eq!(entry, (&(first_key + 2), &(first_key + 2))); - entry = range_scanner.next().unwrap(); - assert_eq!(entry, (&(first_key + 3), &(first_key + 3))); + let mut range_iter = tree.range(first.., &guard); + let mut entry = range_iter.next().unwrap(); + assert_eq!(entry, (&first, &first)); + entry = range_iter.next().unwrap(); + assert_eq!(entry, (&(first + 1), &(first + 1))); + entry = range_iter.next().unwrap(); + assert_eq!(entry, (&(first + 2), &(first + 2))); + entry = range_iter.next().unwrap(); + assert_eq!(entry, (&(first + 3), &(first + 3))); } - let key_at_halfway = first_key + range / 2; - for key in (first_key + 1)..(first_key + range) { + let key_at_halfway = first + range / 2; + for key in (first + 1)..(first + range) { if key == key_at_halfway { let guard = Guard::new(); - let mut range_scanner = tree.range((first_key + 1).., &guard); - let entry = range_scanner.next().unwrap(); - assert_eq!(entry, (&key_at_halfway, &key_at_halfway)); - let entry = range_scanner.next().unwrap(); + let mut range_iter = tree.range((first + 1).., &guard); + let entry = range_iter.next().unwrap(); + assert_eq!(entry, (&key_at_halfway, &key_at_halfway)); // TODO! left: (26153, 26153) right: (26624, 26624) + let entry = range_iter.next().unwrap(); assert_eq!(entry, (&(key_at_halfway + 1), &(key_at_halfway + 1))); } assert!(tree.remove_sync(&key)); assert!(!tree.remove_sync(&key)); - assert!(tree.peek_with(&(first_key + 1), |_, _| ()).is_none()); + assert!(tree.peek_with(&(first + 1), |_, _| ()).is_none()); assert!(tree.peek_with(&key, |_, _| ()).is_none()); } - for key in (first_key + 1)..(first_key + range) { - assert!( - tree.peek_with(&key, |key, val| assert_eq!(key, val)) - .is_none() - ); + for k in (first + 1)..(first + range) { + assert!(tree.peek_with(&k, |k, v| assert_eq!(k, v)).is_none()); } } })); } barrier.wait(); - let iteration = if cfg!(miri) { 16 } else { 512 }; - for _ in 0..iteration { - let mut found_0 = false; - let mut found_markers = 0; - let mut prev_marker = 0; - let mut prev = 0; + let repeat = if cfg!(miri) { 16 } else { 512 }; + for _ in 0..repeat { + let (mut found_0, mut found_markers) = (false, 0); + let (mut prev_marker, mut prev_marker_rev) = (0, 0); + let (mut prev, mut prev_rev) = (0, usize::MAX); let guard = Guard::new(); - for iter in tree.iter(&guard) { - let current = *iter.0; - if current % range == 0 { + let (mut iter, mut iter_ended, mut iter_rev_ended, mut rev) = + (tree.iter(&guard), false, false, false); + while !iter_ended || !iter_rev_ended { + rev = iter_ended || (!iter_rev_ended && rev); + let current = if rev { iter.next_back() } else { iter.next() }; + let Some((key, _)) = current else { + if rev { + iter_rev_ended = true; + } else { + iter_ended = true; + } + continue; + }; + if key % range == 0 { found_markers += 1; - if current == 0 { + if *key == 0 { found_0 = true; + } else if rev { + assert!(prev_marker_rev == 0 || prev_marker_rev - range == *key); + prev_marker_rev = *key; + } else { + assert_eq!(prev_marker + range, *key); + prev_marker = *key; } - if current > 0 { - assert_eq!(prev_marker + range, current); - } - prev_marker = current; } - assert!(prev == 0 || prev < current); - prev = current; + assert!((rev && prev_rev > *key) || (!rev && (prev == 0 || prev < *key))); + if rev { + prev_rev = *key; + } else { + prev = *key; + } + rev = !rev; } assert!(found_0); assert_eq!(found_markers, num_threads); diff --git a/patch/scc-3.4.8/src/tree_index.rs b/patch/scc-3.4.8/src/tree_index.rs index b0a40fe..b606a6d 100644 --- a/patch/scc-3.4.8/src/tree_index.rs +++ b/patch/scc-3.4.8/src/tree_index.rs @@ -1,4 +1,4 @@ -//! [`TreeIndex`] is a read-optimized concurrent B-plus tree. +//! [`TreeIndex`] is a read-optimized asynchronous/concurrent B-plus tree. mod internal_node; mod leaf; @@ -18,14 +18,16 @@ use sdd::{AtomicShared, Guard, Ptr, Shared, Tag}; use crate::Comparable; use crate::async_helper::AsyncWait; -use leaf::{InsertResult, Leaf, RemoveResult, Scanner}; +use leaf::Iter as LeafIter; +use leaf::RevIter as LeafRevIter; +use leaf::{InsertResult, Leaf, RemoveResult}; use node::Node; -/// Scalable concurrent B-plus tree. +/// Scalable asynchronous/concurrent B-plus tree. /// -/// [`TreeIndex`] is a concurrent B-plus tree variant optimized for read operations. Read -/// operations, such as read iteration over entries, are neither blocked nor interrupted by other -/// threads or tasks. Write operations, such as insert and remove, do not block if structural +/// [`TreeIndex`] is a asynchronous/concurrent B-plus tree variant optimized for read operations. +/// Read operations, such as read iteration over entries, are neither blocked nor interrupted by +/// other threads or tasks. Write operations, such as insert and remove, do not block if structural /// changes are not required. /// /// ## Note @@ -72,16 +74,16 @@ pub struct TreeIndex { /// monotonically increasing order. pub struct Iter<'t, 'g, K, V> { root: &'t AtomicShared>, - leaf_scanner: Option>, + forward: Option>, + backward: Option>, guard: &'g Guard, } /// An iterator over a sub-range of entries in a [`TreeIndex`]. pub struct Range<'t, 'g, K, V, Q: ?Sized, R: RangeBounds> { root: &'t AtomicShared>, - leaf_scanner: Option>, + leaf_iter: Option>, bounds: R, - check_lower_bound: bool, check_upper_bound: bool, guard: &'g Guard, query: PhantomData Q>, @@ -159,7 +161,7 @@ impl TreeIndex { impl TreeIndex where K: 'static + Clone + Ord, - V: 'static + Clone, + V: 'static, { /// Inserts a key-value pair. /// @@ -178,63 +180,43 @@ where #[inline] pub async fn insert_async(&self, mut key: K, mut val: V) -> Result<(), (K, V)> { let mut pinned_async_wait = pin!(AsyncWait::default()); - let mut new_root = None; loop { - let need_await = { + { let guard = Guard::new(); let root_ptr = self.root.load(Acquire, &guard); if let Some(root_ref) = root_ptr.as_ref() { match root_ref.insert(key, val, &mut pinned_async_wait, &guard) { Ok(r) => match r { InsertResult::Success => return Ok(()), - InsertResult::Frozen(k, v) | InsertResult::Retry(k, v) => { - key = k; - val = v; - root_ref.cleanup_link(&key, false, &guard); - true + InsertResult::Duplicate(k, v) | InsertResult::Frozen(k, v) => { + return Err((k, v)); } - InsertResult::Duplicate(k, v) => return Err((k, v)), InsertResult::Full(k, v) => { - let (k, v) = Node::split_root(root_ptr, &self.root, k, v, &guard); key = k; val = v; + Node::split_root(root_ptr, &self.root, &guard); continue; } - InsertResult::Retired(k, v) => { - key = k; - val = v; - !Node::cleanup_root(&self.root, &mut pinned_async_wait, &guard) - } }, Err((k, v)) => { key = k; val = v; - true } } - } else { - false + } else if let Err((Some(new_node), _)) = self.root.compare_exchange( + Ptr::null(), + (Some(Shared::new(Node::new_leaf_node())), Tag::None), + AcqRel, + Acquire, + &Guard::new(), + ) { + unsafe { + let _: bool = new_node.drop_in_place(); + } + continue; } }; - - if need_await { - pinned_async_wait.wait().await; - } - - let node = if let Some(new_root) = new_root.take() { - new_root - } else { - Shared::new(Node::new_leaf_node()) - }; - if let Err((node, _)) = self.root.compare_exchange( - Ptr::null(), - (Some(node), Tag::None), - AcqRel, - Acquire, - &Guard::new(), - ) { - new_root = node; - } + pinned_async_wait.wait().await; } } @@ -257,7 +239,6 @@ where /// ``` #[inline] pub fn insert_sync(&self, mut key: K, mut val: V) -> Result<(), (K, V)> { - let mut new_root = None; loop { let guard = Guard::new(); let root_ptr = self.root.load(Acquire, &guard); @@ -265,22 +246,13 @@ where match root_ref.insert(key, val, &mut (), &guard) { Ok(r) => match r { InsertResult::Success => return Ok(()), - InsertResult::Frozen(k, v) | InsertResult::Retry(k, v) => { - key = k; - val = v; - root_ref.cleanup_link(&key, false, &guard); + InsertResult::Duplicate(k, v) | InsertResult::Frozen(k, v) => { + return Err((k, v)); } - InsertResult::Duplicate(k, v) => return Err((k, v)), InsertResult::Full(k, v) => { - let (k, v) = Node::split_root(root_ptr, &self.root, k, v, &guard); key = k; val = v; - continue; - } - InsertResult::Retired(k, v) => { - key = k; - val = v; - let _result = Node::cleanup_root(&self.root, &mut (), &guard); + Node::split_root(root_ptr, &self.root, &guard); } }, Err((k, v)) => { @@ -288,21 +260,16 @@ where val = v; } } - } - - let node = if let Some(new_root) = new_root.take() { - new_root - } else { - Shared::new(Node::new_leaf_node()) - }; - if let Err((node, _)) = self.root.compare_exchange( + } else if let Err((Some(new_node), _)) = self.root.compare_exchange( Ptr::null(), - (Some(node), Tag::None), + (Some(Shared::new(Node::new_leaf_node())), Tag::None), AcqRel, Acquire, - &guard, + &Guard::new(), ) { - new_root = node; + unsafe { + let _: bool = new_node.drop_in_place(); + } } } } @@ -385,12 +352,9 @@ where &mut pinned_async_wait, &guard, ) { - if matches!(result, RemoveResult::Cleanup) { - root_ref.cleanup_link(key, false, &guard); - } match result { RemoveResult::Success => return true, - RemoveResult::Cleanup | RemoveResult::Retired => { + RemoveResult::Retired => { if Node::cleanup_root(&self.root, &mut pinned_async_wait, &guard) { return true; } @@ -450,12 +414,9 @@ where if let Ok(result) = root_ref.remove_if::<_, _, _>(key, &mut condition, &mut (), &guard) { - if matches!(result, RemoveResult::Cleanup) { - root_ref.cleanup_link(key, false, &guard); - } match result { RemoveResult::Success => return true, - RemoveResult::Cleanup | RemoveResult::Retired => { + RemoveResult::Retired => { if Node::cleanup_root(&self.root, &mut (), &guard) { return true; } @@ -747,10 +708,10 @@ where /// Returns an [`Iter`]. /// - /// The returned [`Iter`] starts scanning from the minimum key-value pair. Key-value pairs - /// are scanned in ascending order, and key-value pairs that have existed since the invocation - /// of the method are guaranteed to be visited if they are not removed. However, it is possible - /// to visit removed key-value pairs momentarily. + /// The returned [`Iter`] is a [`DoubleEndedIterator`] that allows scanning in both ascending + /// and descending order. [`Iter`] may miss newly inserted key-value pairs after the invocation + /// of this method, because [`Self::iter`] is the linearization point whereas [`Iter::next`] and + /// [`Iter::next_back`] are not. /// /// # Examples /// @@ -761,12 +722,18 @@ where /// /// let treeindex: TreeIndex = TreeIndex::new(); /// + /// assert!(treeindex.insert_sync(1, 2).is_ok()); + /// assert!(treeindex.insert_sync(3, 4).is_ok()); + /// /// let guard = Guard::new(); /// let mut iter = treeindex.iter(&guard); - /// assert!(iter.next().is_none()); + /// assert_eq!(iter.next(), Some((&1, &2))); + /// assert_eq!(iter.next_back(), Some((&3, &4))); + /// assert_eq!(iter.next(), None); + /// assert_eq!(iter.next_back(), None); /// ``` #[inline] - pub fn iter<'t, 'g>(&'t self, guard: &'g Guard) -> Iter<'t, 'g, K, V> { + pub const fn iter<'t, 'g>(&'t self, guard: &'g Guard) -> Iter<'t, 'g, K, V> { Iter::new(&self.root, guard) } @@ -789,7 +756,7 @@ where /// assert_eq!(treeindex.range(4..=8, &guard).count(), 0); /// ``` #[inline] - pub fn range<'t, 'g, Q, R: RangeBounds>( + pub const fn range<'t, 'g, Q, R: RangeBounds>( &'t self, range: R, guard: &'g Guard, @@ -819,7 +786,7 @@ where impl Debug for TreeIndex where K: 'static + Clone + Debug + Ord, - V: 'static + Clone + Debug, + V: 'static + Debug, { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -854,7 +821,7 @@ impl Drop for TreeIndex { impl PartialEq for TreeIndex where K: 'static + Clone + Ord, - V: 'static + Clone + PartialEq, + V: 'static + PartialEq, { #[inline] fn eq(&self, other: &Self) -> bool { @@ -868,39 +835,107 @@ impl UnwindSafe for TreeIndex {} impl<'t, 'g, K, V> Iter<'t, 'g, K, V> { #[inline] - fn new(root: &'t AtomicShared>, guard: &'g Guard) -> Iter<'t, 'g, K, V> { + const fn new(root: &'t AtomicShared>, guard: &'g Guard) -> Iter<'t, 'g, K, V> { Iter::<'t, 'g, K, V> { root, - leaf_scanner: None, + forward: None, + backward: None, guard, } } } +impl<'g, K, V> Iter<'_, 'g, K, V> +where + K: Ord, +{ + fn check_collision( + &mut self, + entry: (&'g K, &'g V), + ) -> Option<(&'g K, &'g V)> { + let other_entry = if FORWARD { + self.backward.as_ref().and_then(LeafRevIter::get) + } else { + self.forward.as_ref().and_then(LeafIter::get) + }; + let Some(other_entry) = other_entry else { + // The other iterator was exhausted. + return None; + }; + if (FORWARD && other_entry.0 > entry.0) || (!FORWARD && other_entry.0 < entry.0) { + return Some(entry); + } + None + } +} + impl Debug for Iter<'_, '_, K, V> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Iter") .field("root", &self.root) - .field("leaf_scanner", &self.leaf_scanner) + .field("leaf_iter", &self.forward) .finish() } } +impl DoubleEndedIterator for Iter<'_, '_, K, V> +where + K: 'static + Clone + Ord, + V: 'static, +{ + #[inline] + fn next_back(&mut self) -> Option { + // Start iteration. + if self.backward.is_none() { + let root_ptr = self.root.load(Acquire, self.guard); + if let Some(root_ref) = root_ptr.as_ref() { + if let Some(rev_iter) = root_ref.max(self.guard) { + self.backward.replace(rev_iter); + } + } else { + return None; + } + } + + // Go to the prev entry. + if let Some(rev_iter) = self.backward.as_mut() { + if let Some(entry) = rev_iter.next() { + if self.forward.is_some() { + return self.check_collision::(entry); + } + return Some(entry); + } + // Go to the prev leaf node. + if let Some(new_rev_iter) = rev_iter.jump(self.guard) { + if let Some(entry) = new_rev_iter.get() { + self.backward.replace(new_rev_iter); + if self.forward.is_some() { + return self.check_collision::(entry); + } + return Some(entry); + } + } + } + + None + } +} + impl<'g, K, V> Iterator for Iter<'_, 'g, K, V> where K: 'static + Clone + Ord, - V: 'static + Clone, + V: 'static, { type Item = (&'g K, &'g V); #[inline] fn next(&mut self) -> Option { - // Starts scanning. - if self.leaf_scanner.is_none() { + // Start iteration. + if self.forward.is_none() { let root_ptr = self.root.load(Acquire, self.guard); if let Some(root_ref) = root_ptr.as_ref() { - if let Some(scanner) = root_ref.min(self.guard) { - self.leaf_scanner.replace(scanner); + if let Some(iter) = root_ref.min(self.guard) { + self.forward.replace(iter); } } else { return None; @@ -908,20 +943,25 @@ where } // Go to the next entry. - if let Some(mut scanner) = self.leaf_scanner.take() { - let min_allowed_key = scanner.get().map(|(key, _)| key); - if let Some(result) = scanner.next() { - self.leaf_scanner.replace(scanner); - return Some(result); + if let Some(iter) = self.forward.as_mut() { + if let Some(entry) = iter.next() { + if self.backward.is_some() { + return self.check_collision::(entry); + } + return Some(entry); } // Go to the next leaf node. - if let Some(new_scanner) = scanner.jump(min_allowed_key, self.guard) { - if let Some(entry) = new_scanner.get() { - self.leaf_scanner.replace(new_scanner); + if let Some(new_iter) = iter.jump(self.guard) { + if let Some(entry) = new_iter.get() { + self.forward.replace(new_iter); + if self.backward.is_some() { + return self.check_collision::(entry); + } return Some(entry); } } } + None } } @@ -929,7 +969,7 @@ where impl FusedIterator for Iter<'_, '_, K, V> where K: 'static + Clone + Ord, - V: 'static + Clone, + V: 'static, { } @@ -937,16 +977,15 @@ impl UnwindSafe for Iter<'_, '_, K, V> {} impl<'t, 'g, K, V, Q: ?Sized, R: RangeBounds> Range<'t, 'g, K, V, Q, R> { #[inline] - fn new( + const fn new( root: &'t AtomicShared>, range: R, guard: &'g Guard, ) -> Range<'t, 'g, K, V, Q, R> { Range::<'t, 'g, K, V, Q, R> { root, - leaf_scanner: None, + leaf_iter: None, bounds: range, - check_lower_bound: true, check_upper_bound: false, guard, query: PhantomData, @@ -957,59 +996,63 @@ impl<'t, 'g, K, V, Q: ?Sized, R: RangeBounds> Range<'t, 'g, K, V, Q, R> { impl<'g, K, V, Q, R> Range<'_, 'g, K, V, Q, R> where K: 'static + Clone + Ord, - V: 'static + Clone, + V: 'static, Q: Comparable + ?Sized, R: RangeBounds, { - #[inline] - fn next_unbounded(&mut self) -> Option<(&'g K, &'g V)> { - if self.leaf_scanner.is_none() { - // Start scanning. - let root_ptr = self.root.load(Acquire, self.guard); - if let Some(root_ref) = root_ptr.as_ref() { - let min_allowed_key = match self.bounds.start_bound() { - Excluded(key) | Included(key) => Some(key), - Unbounded => { - self.check_lower_bound = false; - None - } - }; - let mut leaf_scanner = min_allowed_key - .and_then(|min_allowed_key| root_ref.max_le_appr(min_allowed_key, self.guard)); - if leaf_scanner.is_none() { - // No `min_allowed_key` is supplied, or no keys smaller than or equal to - // `min_allowed_key` found. - if let Some(mut scanner) = root_ref.min(self.guard) { - // It's possible that the leaf has just been emptied, so go to the next. - scanner.next(); - while scanner.get().is_none() { - scanner = scanner.jump(None, self.guard)?; - } - leaf_scanner.replace(scanner); - } + fn start(&mut self) -> Option<(&'g K, &'g V)> { + // Start iteration. + let root_ptr = self.root.load(Acquire, self.guard); + if let Some(root) = root_ptr.as_ref() { + let mut leaf_iter = match self.bounds.start_bound() { + Excluded(k) | Included(k) => root.approximate::<_, true>(k, self.guard), + Unbounded => None, + }; + if leaf_iter.is_none() { + if let Some(mut iter) = root.min(self.guard) { + iter.next(); + leaf_iter.replace(iter); } - if let Some(leaf_scanner) = leaf_scanner { - if let Some(result) = leaf_scanner.get() { - self.set_check_upper_bound(&leaf_scanner); - self.leaf_scanner.replace(leaf_scanner); - return Some(result); + } + if let Some(mut leaf_iter) = leaf_iter { + while let Some((k, v)) = leaf_iter.get() { + let check_failed = match self.bounds.start_bound() { + Excluded(key) => key.compare(k).is_ge(), + Included(key) => key.compare(k).is_gt(), + Unbounded => false, + }; + if check_failed { + if leaf_iter.next().is_none() { + leaf_iter = leaf_iter.jump(self.guard)?; + } + continue; } + + self.set_check_upper_bound(&leaf_iter); + self.leaf_iter.replace(leaf_iter); + return Some((k, v)); } } } + None + } + + #[inline] + fn next_unbounded(&mut self) -> Option<(&'g K, &'g V)> { + if self.leaf_iter.is_none() { + return self.start(); + } // Go to the next entry. - if let Some(mut leaf_scanner) = self.leaf_scanner.take() { - let min_allowed_key = leaf_scanner.get().map(|(key, _)| key); - if let Some(result) = leaf_scanner.next() { - self.leaf_scanner.replace(leaf_scanner); + if let Some(leaf_iter) = self.leaf_iter.as_mut() { + if let Some(result) = leaf_iter.next() { return Some(result); } // Go to the next leaf node. - if let Some(new_scanner) = leaf_scanner.jump(min_allowed_key, self.guard) { - if let Some(entry) = new_scanner.get() { - self.set_check_upper_bound(&new_scanner); - self.leaf_scanner.replace(new_scanner); + if let Some(new_iter) = leaf_iter.jump(self.guard) { + if let Some(entry) = new_iter.get() { + self.set_check_upper_bound(&new_iter); + self.leaf_iter.replace(new_iter); return Some(entry); } } @@ -1019,14 +1062,10 @@ where } #[inline] - fn set_check_upper_bound(&mut self, scanner: &Scanner) { + fn set_check_upper_bound(&mut self, leaf_iter: &LeafIter) { self.check_upper_bound = match self.bounds.end_bound() { - Excluded(key) => scanner - .max_key() - .is_some_and(|max_key| key.compare(max_key).is_le()), - Included(key) => scanner - .max_key() - .is_some_and(|max_key| key.compare(max_key).is_lt()), + Excluded(key) => leaf_iter.max_key().is_some_and(|k| key.compare(k).is_le()), + Included(key) => leaf_iter.max_key().is_some_and(|k| key.compare(k).is_lt()), Unbounded => false, }; } @@ -1037,8 +1076,7 @@ impl> Debug for Range<'_, '_, K, V, Q, R> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Range") .field("root", &self.root) - .field("leaf_scanner", &self.leaf_scanner) - .field("check_lower_bound", &self.check_lower_bound) + .field("leaf_iter", &self.leaf_iter) .field("check_upper_bound", &self.check_upper_bound) .finish() } @@ -1047,7 +1085,7 @@ impl> Debug for Range<'_, '_, K, V, Q, R> { impl<'g, K, V, Q, R> Iterator for Range<'_, 'g, K, V, Q, R> where K: 'static + Clone + Ord, - V: 'static + Clone, + V: 'static, Q: Comparable + ?Sized, R: RangeBounds, { @@ -1055,23 +1093,7 @@ where #[inline] fn next(&mut self) -> Option { - while let Some((k, v)) = self.next_unbounded() { - if self.check_lower_bound { - match self.bounds.start_bound() { - Excluded(key) => { - if key.compare(k).is_ge() { - continue; - } - } - Included(key) => { - if key.compare(k).is_gt() { - continue; - } - } - Unbounded => (), - } - } - self.check_lower_bound = false; + if let Some((k, v)) = self.next_unbounded() { if self.check_upper_bound { match self.bounds.end_bound() { Excluded(key) => { @@ -1088,9 +1110,9 @@ where return Some((k, v)); } } - break; + } else { + return Some((k, v)); } - return Some((k, v)); } None } @@ -1099,7 +1121,7 @@ where impl FusedIterator for Range<'_, '_, K, V, Q, R> where K: 'static + Clone + Ord, - V: 'static + Clone, + V: 'static, Q: Comparable + ?Sized, R: RangeBounds, { diff --git a/patch/scc-3.4.8/src/tree_index/internal_node.rs b/patch/scc-3.4.8/src/tree_index/internal_node.rs index 86718f6..e193da1 100644 --- a/patch/scc-3.4.8/src/tree_index/internal_node.rs +++ b/patch/scc-3.4.8/src/tree_index/internal_node.rs @@ -1,20 +1,17 @@ -use std::cmp::Ordering::{Equal, Greater, Less}; use std::mem::forget; use std::ops::RangeBounds; use std::ptr; -use std::sync::atomic::AtomicPtr; use std::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed, Release}; use saa::Lock; use sdd::{AtomicShared, Guard, Ptr, Shared, Tag}; -use super::leaf::{DIMENSION, InsertResult, Leaf, RemoveResult, Scanner}; -use super::leaf_node::RETIRED; -use super::leaf_node::RemoveRangeState; +use super::leaf::{InsertResult, Iter, Leaf, RemoveResult, RevIter}; +use super::leaf_node::Locker as LeafNodeLocker; +use super::leaf_node::{LeafNode, RemoveRangeState}; use super::node::Node; use crate::Comparable; use crate::async_helper::TryWait; -use crate::exit_guard::ExitGuard; /// Internal node. /// @@ -27,8 +24,6 @@ pub struct InternalNode { /// It stores the maximum key in the node, and key-value pairs are first pushed to this [`Node`] /// until it splits. pub(super) unbounded_child: AtomicShared>, - /// Ongoing split operation. - split_op: StructuralChange, /// [`Lock`] to protect the [`InternalNode`]. pub(super) lock: Lock, } @@ -38,18 +33,6 @@ pub(super) struct Locker<'n, K, V> { internal_node: &'n InternalNode, } -/// [`StructuralChange`] stores intermediate results during a split operation. -/// -/// `AtomicPtr` members may point to values under the protection of the [`Guard`] used for the -/// split operation. -struct StructuralChange { - origin_node_key: AtomicPtr, - origin_node: AtomicShared>, - low_key_node: AtomicShared>, - middle_key: AtomicPtr, - high_key_node: AtomicShared>, -} - impl InternalNode { /// Creates a new empty internal node. #[inline] @@ -57,7 +40,6 @@ impl InternalNode { InternalNode { children: Leaf::new(), unbounded_child: AtomicShared::null(), - split_op: StructuralChange::default(), lock: Lock::default(), } } @@ -65,8 +47,8 @@ impl InternalNode { /// Clears the internal node. #[inline] pub(super) fn clear(&self, guard: &Guard) { - let scanner = Scanner::new(&self.children); - for (_, child) in scanner { + let iter = Iter::new(&self.children); + for (_, child) in iter { let child_ptr = child.load(Acquire, guard); if let Some(child) = child_ptr.as_ref() { child.clear(guard); @@ -90,15 +72,15 @@ impl InternalNode { /// Returns `true` if the [`InternalNode`] has retired. #[inline] - pub(super) fn retired(&self) -> bool { - self.unbounded_child.tag(Acquire) == RETIRED + pub(super) fn is_retired(&self) -> bool { + self.lock.is_poisoned(Acquire) } } impl InternalNode where K: 'static + Clone + Ord, - V: 'static + Clone, + V: 'static, { /// Searches for an entry containing the specified key. #[inline] @@ -158,84 +140,109 @@ where } } - /// Returns the minimum key entry. + /// Returns the minimum key entry in the entire tree. #[inline] - pub(super) fn min<'g>(&self, guard: &'g Guard) -> Option> { - loop { - let mut retry = false; - let scanner = Scanner::new(&self.children); - let metadata = scanner.metadata(); - for (_, child) in scanner { + pub(super) fn min<'g>(&self, guard: &'g Guard) -> Option> { + let mut unbounded_ptr = self.unbounded_child.load(Acquire, guard); + while let Some(unbounded) = unbounded_ptr.as_ref() { + let mut iter = Iter::new(&self.children); + for (_, child) in iter.by_ref() { let child_ptr = child.load(Acquire, guard); if let Some(child) = child_ptr.as_ref() { - if self.children.validate(metadata) { - // Data race resolution - see `LeafNode::search_entry`. - if let Some(scanner) = child.min(guard) { - return Some(scanner); - } - continue; + if let Some(iter) = child.min(guard) { + return Some(iter); } } - // It is not a hot loop - see `LeafNode::search_entry`. - retry = true; + } + if let Some(iter) = unbounded.min(guard) { + return Some(iter); + } + // `post_remove` may be replacing the retired unbounded child with an existing child. + let new_ptr = self.unbounded_child.load(Acquire, guard); + if unbounded_ptr == new_ptr && self.children.validate(iter.metadata()) { + // All the children are empty or retired. break; } - if retry { - continue; - } - let unbounded_ptr = self.unbounded_child.load(Acquire, guard); - if let Some(unbounded) = unbounded_ptr.as_ref() { - if self.children.validate(metadata) { - return unbounded.min(guard); - } - continue; - } - return None; + unbounded_ptr = new_ptr; } + + None } - /// Returns a [`Scanner`] pointing to an entry that is close enough to the entry with the - /// maximum key among those keys that are smaller than or equal to the given key. - /// - /// Returns `None` if all keys in the [`InternalNode`] are greater than the given key. + /// Returns the maximum key entry in the entire tree. #[inline] - pub(super) fn max_le_appr<'g, Q>(&self, key: &Q, guard: &'g Guard) -> Option> + pub(super) fn max<'g>(&self, guard: &'g Guard) -> Option> { + let mut unbounded_ptr = self.unbounded_child.load(Acquire, guard); + while let Some(unbounded) = unbounded_ptr.as_ref() { + let mut rev_iter = RevIter::new(&self.children); + if let Some(iter) = unbounded.max(guard) { + return Some(iter); + } + // `post_remove` may be replacing the retired unbounded child with an existing child. + for (_, child) in rev_iter.by_ref() { + let child_ptr = child.load(Acquire, guard); + if let Some(child) = child_ptr.as_ref() { + if let Some(iter) = child.max(guard) { + return Some(iter); + } + } + } + let new_ptr = self.unbounded_child.load(Acquire, guard); + if unbounded_ptr == new_ptr && self.children.validate(rev_iter.metadata()) { + // All the children are empty or retired. + break; + } + unbounded_ptr = new_ptr; + } + + None + } + + /// Returns a [`Iter`] pointing to an entry that is close enough to the specified key. + #[inline] + pub(super) fn approximate<'g, Q, const LE: bool>( + &self, + key: &Q, + guard: &'g Guard, + ) -> Option> where K: 'g, Q: Comparable + ?Sized, { - loop { - if let Some(scanner) = Scanner::max_less(&self.children, key) { - if let Some((_, child)) = scanner.get() { - if let Some(child) = child.load(Acquire, guard).as_ref() { - if self.children.validate(scanner.metadata()) { - // Data race resolution - see `LeafNode::search_entry`. - if let Some(scanner) = child.max_le_appr(key, guard) { - return Some(scanner); - } - // Fallback. - break; - } + let mut unbounded_ptr = self.unbounded_child.load(Acquire, guard); + while let Some(unbounded) = unbounded_ptr.as_ref() { + // Firstly, try to find a key in the optimal child. + if let Some((_, child)) = self.children.min_greater_equal(key).0 { + let child_ptr = child.load(Acquire, guard); + if let Some(child) = child_ptr.as_ref() { + if let Some(iter) = child.approximate::<_, LE>(key, guard) { + return Some(iter); } + } else { // It is not a hot loop - see `LeafNode::search_entry`. continue; } + } else if let Some(iter) = unbounded.approximate::<_, LE>(key, guard) { + return Some(iter); } - // Fallback. - break; - } - // Starts scanning from the minimum key. - let mut min_scanner = self.min(guard)?; - min_scanner.next(); - loop { - if let Some((k, _)) = min_scanner.get() { - if key.compare(k).is_ge() { - return Some(min_scanner); + // Secondly, try to find a key in any child. + let mut iter = Iter::new(&self.children); + for (_, child) in iter.by_ref() { + let child_ptr = child.load(Acquire, guard); + if let Some(child) = child_ptr.as_ref() { + if let Some(iter) = child.approximate::<_, LE>(key, guard) { + return Some(iter); + } } + } + + let new_ptr = self.unbounded_child.load(Acquire, guard); + if unbounded_ptr == new_ptr && self.children.validate(iter.metadata()) { + // All the children are empty or retired. break; } - min_scanner = min_scanner.jump(None, guard)?; + unbounded_ptr = new_ptr; } None @@ -252,7 +259,7 @@ where ) -> Result, (K, V)> { loop { let (child, metadata) = self.children.min_greater_equal(&key); - if let Some((child_key, child)) = child { + if let Some((_, child)) = child { let child_ptr = child.load(Acquire, guard); if let Some(child_ref) = child_ptr.as_ref() { if self.children.validate(metadata) { @@ -263,38 +270,15 @@ where | InsertResult::Duplicate(..) | InsertResult::Frozen(..) => return Ok(insert_result), InsertResult::Full(k, v) => { - let split_result = self.split_node( - (k, v), - Some(child_key), - child_ptr, - child, - false, - async_wait, - guard, - )?; - if let InsertResult::Retry(k, v) = split_result { - key = k; - val = v; - continue; + match self.split_node(child_ptr, child, async_wait, guard) { + Ok(true) => { + key = k; + val = v; + continue; + } + Ok(false) => return Ok(InsertResult::Full(k, v)), + Err(()) => return Err((k, v)), } - return Ok(split_result); - } - InsertResult::Retired(k, v) => { - debug_assert!(child_ref.retired()); - if self.coalesce(guard) == RemoveResult::Retired { - debug_assert!(self.retired()); - return Ok(InsertResult::Retired(k, v)); - } - return Err((k, v)); - } - InsertResult::Retry(k, v) => { - // `child` has been split, therefore it can be retried. - if self.cleanup_link(&k, false, guard) { - key = k; - val = v; - continue; - } - return Ok(InsertResult::Retry(k, v)); } }; } @@ -315,42 +299,24 @@ where | InsertResult::Duplicate(..) | InsertResult::Frozen(..) => return Ok(insert_result), InsertResult::Full(k, v) => { - let split_result = self.split_node( - (k, v), - None, + match self.split_node( unbounded_ptr, &self.unbounded_child, - false, async_wait, guard, - )?; - if let InsertResult::Retry(k, v) = split_result { - key = k; - val = v; - continue; + ) { + Ok(true) => { + key = k; + val = v; + continue; + } + Ok(false) => return Ok(InsertResult::Full(k, v)), + Err(()) => return Err((k, v)), } - return Ok(split_result); - } - InsertResult::Retired(k, v) => { - debug_assert!(unbounded.retired()); - if self.coalesce(guard) == RemoveResult::Retired { - debug_assert!(self.retired()); - return Ok(InsertResult::Retired(k, v)); - } - return Err((k, v)); - } - InsertResult::Retry(k, v) => { - if self.cleanup_link(&k, false, guard) { - key = k; - val = v; - continue; - } - return Ok(InsertResult::Retry(k, v)); } }; } - debug_assert!(unbounded_ptr.tag() == RETIRED); - return Ok(InsertResult::Retired(key, val)); + return Ok(InsertResult::Full(key, val)); } } @@ -380,14 +346,8 @@ where // Data race resolution - see `LeafNode::search_entry`. let result = child.remove_if::<_, _, _>(key, condition, async_wait, guard)?; - if result == RemoveResult::Cleanup { - if self.cleanup_link(key, false, guard) { - return Ok(RemoveResult::Success); - } - return Ok(RemoveResult::Cleanup); - } if result == RemoveResult::Retired { - return Ok(self.coalesce(guard)); + return Ok(self.post_remove(None, guard)); } return Ok(result); } @@ -403,14 +363,8 @@ where continue; } let result = unbounded.remove_if::<_, _, _>(key, condition, async_wait, guard)?; - if result == RemoveResult::Cleanup { - if self.cleanup_link(key, false, guard) { - return Ok(RemoveResult::Success); - } - return Ok(RemoveResult::Cleanup); - } if result == RemoveResult::Retired { - return Ok(self.coalesce(guard)); + return Ok(self.post_remove(None, guard)); } return Ok(result); } @@ -448,7 +402,7 @@ where let mut lower_border = None; let mut upper_border = None; - for (key, node) in Scanner::new(&self.children) { + for (key, node) in Iter::new(&self.children) { current_state = current_state.next(key, range, start_unbounded); match current_state { RemoveRangeState::Below => { @@ -463,13 +417,17 @@ where // There can be another thread inserting keys into the node, and this may // render those concurrent operations completely ineffective. self.children.remove_if(key, &mut |_| true); - node.swap((None, Tag::None), AcqRel); + if let Some(node) = node.swap((None, Tag::None), AcqRel).0 { + node.clear(guard); + } } RemoveRangeState::MaybeAbove => { if valid_upper_min_node.is_some() { // `valid_upper_min_node` is not in this sub-tree. self.children.remove_if(key, &mut |_| true); - node.swap((None, Tag::None), AcqRel); + if let Some(node) = node.swap((None, Tag::None), AcqRel).0 { + node.clear(guard); + } } else { num_children += 1; upper_border.replace(node); @@ -564,479 +522,278 @@ where /// Splits a full node. /// + /// Returns `false` if the parent node needs to be split. + /// /// # Errors /// - /// Returns an error if a retry is required. - #[allow(clippy::too_many_arguments, clippy::too_many_lines)] + /// Returns an error if locking failed or the full internal node was changed. + #[allow(clippy::too_many_lines)] pub(super) fn split_node( &self, - entry: (K, V), - full_node_key: Option<&K>, full_node_ptr: Ptr>, full_node: &AtomicShared>, - root_split: bool, async_wait: &mut W, guard: &Guard, - ) -> Result, (K, V)> { - let target = full_node_ptr.as_ref().unwrap(); - if !self.lock.try_lock() { - target.rollback(guard); - async_wait.try_wait(&self.lock); - return Err(entry); + ) -> Result { + if self.is_retired() { + // Let the parent node clean up this node. + return Ok(false); } - debug_assert!(!self.retired()); + + let Some(locker) = Locker::try_lock(self) else { + async_wait.try_wait(&self.lock); + return Err(()); + }; if full_node_ptr != full_node.load(Relaxed, guard) { - self.lock.release_lock(); - target.rollback(guard); - return Err(entry); + return Err(()); } - let prev = self - .split_op - .origin_node - .swap((full_node.get_shared(Relaxed, guard), Tag::None), Relaxed) - .0; - debug_assert!(prev.is_none()); - - if let Some(full_node_key) = full_node_key { - self.split_op - .origin_node_key - .store(ptr::from_ref(full_node_key).cast_mut(), Relaxed); + let target = full_node_ptr.as_ref().unwrap(); + if target.is_retired() { + // It is not possible to split a retired node. + self.post_remove(Some(locker), guard); + return Err(()); } - let exit_guard = ExitGuard::new((), |()| { - self.rollback(guard); - }); + let is_full = self.children.is_full(); match target { - Node::Internal(full_internal_node) => { - // Copies nodes except for the known full node to the newly allocated internal node entries. - let internal_nodes = ( - Shared::new(Node::new_internal_node()), - Shared::new(Node::new_internal_node()), - ); - let Node::Internal(low_key_nodes) = internal_nodes.0.as_ref() else { - unreachable!() + Node::Internal(target) => { + let Some(locker) = Locker::try_lock(target) else { + async_wait.try_wait(&target.lock); + return Err(()); }; - let Node::Internal(high_key_nodes) = internal_nodes.1.as_ref() else { - unreachable!() - }; - - // Builds a list of valid nodes. - #[allow(clippy::type_complexity)] - let mut entry_array: [Option<( - Option<&K>, - AtomicShared>, - )>; - DIMENSION.num_entries + 2] = Default::default(); - let mut num_entries = 0; - let scanner = Scanner::new(&full_internal_node.children); - let recommended_boundary = Leaf::::optimal_boundary(scanner.metadata()); - for entry in scanner { - if unsafe { - full_internal_node - .split_op - .origin_node_key - .load(Relaxed) - .as_ref() - .map_or_else(|| false, |key| entry.0 == key) - } { - let low_key_node_ptr = full_internal_node - .split_op - .low_key_node - .load(Relaxed, guard); - if !low_key_node_ptr.is_null() { - entry_array[num_entries].replace(( - Some(unsafe { - full_internal_node - .split_op - .middle_key - .load(Relaxed) - .as_ref() - .unwrap() - }), - full_internal_node - .split_op - .low_key_node - .clone(Relaxed, guard), - )); - num_entries += 1; - } - let high_key_node_ptr = full_internal_node - .split_op - .high_key_node - .load(Relaxed, guard); - if !high_key_node_ptr.is_null() { - entry_array[num_entries].replace(( - Some(entry.0), - full_internal_node - .split_op - .high_key_node - .clone(Relaxed, guard), - )); - num_entries += 1; + let low_key_node = InternalNode::new(); + let high_key_node = InternalNode::new(); + let mut low_i = 0; + let mut boundary_key = None; + let mut high_i = 0; + if !target.children.distribute(|k, v, _, boundary, _| { + let Some(child) = v.get_shared(Acquire, guard) else { + return true; + }; + if child.is_retired() { + return true; + } + if low_i < boundary { + if low_i == boundary - 1 { + low_key_node + .unbounded_child + .swap((Some(child), Tag::None), Relaxed); + boundary_key.replace(k.clone()); + } else { + low_key_node.children.insert_unchecked( + k.clone(), + AtomicShared::from(child), + low_i, + ); } + low_i += 1; + } else if is_full { + return false; } else { - entry_array[num_entries] - .replace((Some(entry.0), entry.1.clone(Acquire, guard))); - num_entries += 1; - } - } - if full_internal_node - .split_op - .origin_node_key - .load(Relaxed) - .is_null() - { - // If the origin is an unbounded node, assign the high key node to the high key - // node's unbounded. - let low_key_node_ptr = full_internal_node - .split_op - .low_key_node - .load(Relaxed, guard); - if !low_key_node_ptr.is_null() { - entry_array[num_entries].replace(( - Some(unsafe { - full_internal_node - .split_op - .middle_key - .load(Relaxed) - .as_ref() - .unwrap() - }), - full_internal_node - .split_op - .low_key_node - .clone(Relaxed, guard), - )); - num_entries += 1; - } - let high_key_node_ptr = full_internal_node - .split_op - .high_key_node - .load(Relaxed, guard); - if !high_key_node_ptr.is_null() { - entry_array[num_entries].replace(( - None, - full_internal_node - .split_op - .high_key_node - .clone(Relaxed, guard), - )); - num_entries += 1; - } - } else { - // If the origin is a bounded node, assign the unbounded node to the high key - // node's unbounded. - entry_array[num_entries].replace(( - None, - full_internal_node.unbounded_child.clone(Relaxed, guard), - )); - num_entries += 1; - } - debug_assert!(num_entries >= 2); - - let low_key_node_array_size = recommended_boundary.min(num_entries - 1); - for (i, entry) in entry_array.iter().enumerate() { - if let Some((k, v)) = entry { - match (i + 1).cmp(&low_key_node_array_size) { - Less => { - low_key_nodes.children.insert_unchecked( - k.unwrap().clone(), - v.clone(Relaxed, guard), - i, - ); - } - Equal => { - if let Some(&k) = k.as_ref() { - self.split_op - .middle_key - .store(ptr::from_ref(k).cast_mut(), Relaxed); - } - low_key_nodes - .unbounded_child - .swap((v.get_shared(Relaxed, guard), Tag::None), Relaxed); - } - Greater => { - if let Some(k) = k.cloned() { - high_key_nodes.children.insert_unchecked( - k, - v.clone(Relaxed, guard), - i - low_key_node_array_size, - ); - } else { - high_key_nodes - .unbounded_child - .swap((v.get_shared(Relaxed, guard), Tag::None), Relaxed); - } - } - } - } else { - break; + high_key_node.children.insert_unchecked( + k.clone(), + AtomicShared::from(child), + high_i, + ); + high_i += 1; } + true + }) { + return Ok(false); } - // Turns the new nodes into internal nodes. - self.split_op - .low_key_node - .swap((Some(internal_nodes.0), Tag::None), Relaxed); - self.split_op - .high_key_node - .swap((Some(internal_nodes.1), Tag::None), Relaxed); - } - Node::Leaf(full_leaf_node) => { - // Copies leaves except for the known full leaf to the newly allocated leaf node entries. - let leaf_nodes = ( - Shared::new(Node::new_leaf_node()), - Shared::new(Node::new_leaf_node()), - ); - let low_key_leaf_node = if let Node::Leaf(low_key_leaf_node) = leaf_nodes.0.as_ref() - { - Some(low_key_leaf_node) - } else { - None - }; - let high_key_leaf_node = - if let Node::Leaf(high_key_leaf_node) = &leaf_nodes.1.as_ref() { - Some(high_key_leaf_node) + let high_key_node_empty = + if high_i == 0 && low_key_node.unbounded_child.is_null(Relaxed) { + low_key_node.unbounded_child.swap( + (target.unbounded_child.get_shared(Acquire, guard), Tag::None), + Relaxed, + ); + true + } else if is_full { + return Ok(false); } else { - None + high_key_node.unbounded_child.swap( + (target.unbounded_child.get_shared(Acquire, guard), Tag::None), + Relaxed, + ); + false }; - self.split_op.middle_key.store( - ptr::from_ref(full_leaf_node.split_leaf_node( - low_key_leaf_node.unwrap(), - high_key_leaf_node.unwrap(), - guard, - )) - .cast_mut(), - Relaxed, - ); - - // Turns the new leaves into leaf nodes. - self.split_op - .low_key_node - .swap((Some(leaf_nodes.0), Tag::None), Relaxed); - self.split_op - .high_key_node - .swap((Some(leaf_nodes.1), Tag::None), Relaxed); - } - } - - // Inserts the newly allocated internal nodes into the main array. - match self.children.insert( - unsafe { - self.split_op - .middle_key - .load(Relaxed) - .as_ref() - .unwrap() - .clone() - }, - self.split_op.low_key_node.clone(Relaxed, guard), - ) { - InsertResult::Success => (), - InsertResult::Duplicate(..) | InsertResult::Frozen(..) | InsertResult::Retry(..) => { - unreachable!() - } - InsertResult::Full(..) | InsertResult::Retired(..) => { - // Insertion failed: expects that the parent splits this node. - exit_guard.forget(); - return Ok(InsertResult::Full(entry.0, entry.1)); - } - } - exit_guard.forget(); - - // Replace the full node with the high-key node. - let unused_node = full_node - .swap( - ( - self.split_op.high_key_node.get_shared(Relaxed, guard), - Tag::None, - ), - Release, - ) - .0; - - if root_split { - // Return without unlocking it. - return Ok(InsertResult::Retry(entry.0, entry.1)); - } - - // Unlock the node. - self.finish_split(); - - // Drop the deprecated nodes. - if let Some(unused_node) = unused_node { - // Clean up the split operation by committing it. - unused_node.commit(guard); - let _: bool = unused_node.release(); - } - - // Since a new node has been inserted, the caller can retry. - Ok(InsertResult::Retry(entry.0, entry.1)) - } - - /// Finishes splitting the [`InternalNode`]. - #[inline] - pub(super) fn finish_split(&self) { - let origin = self.split_op.reset(); - self.lock.release_lock(); - origin.map(Shared::release); - } - - /// Commits an ongoing structural change recursively. - #[inline] - pub(super) fn commit(&self, guard: &Guard) { - let origin = self.split_op.reset(); - - // Prevent further exclusive access to the internal node. - self.lock.poison_lock(); - if let Some(origin) = origin { - origin.commit(guard); - let _: bool = origin.release(); - } - } - - /// Rolls back the ongoing split operation recursively. - #[inline] - pub(super) fn rollback(&self, guard: &Guard) { - let origin = self.split_op.reset(); - self.lock.release_lock(); - if let Some(origin) = origin { - origin.rollback(guard); - let _: bool = origin.release(); - } - } - - /// Cleans up logically deleted leaves in the linked list. - /// - /// Returns `false` if the target leaf node does not exist in the subtree. - #[inline] - pub(super) fn cleanup_link<'g, Q>(&self, key: &Q, traverse_max: bool, guard: &'g Guard) -> bool - where - K: 'g, - Q: Comparable + ?Sized, - { - if traverse_max { - // It just has to search for the maximum leaf node in the tree. - if let Some(unbounded) = self.unbounded_child.load(Acquire, guard).as_ref() { - return unbounded.cleanup_link(key, true, guard); - } - } else if let Some(child_scanner) = Scanner::max_less(&self.children, key) { - if let Some((_, child)) = child_scanner.get() { - if let Some(child) = child.load(Acquire, guard).as_ref() { - return child.cleanup_link(key, true, guard); - } - } - } - false - } - - /// Tries to coalesce nodes. - fn coalesce(&self, guard: &Guard) -> RemoveResult { - let mut node_deleted = false; - while let Some(lock) = Locker::try_lock(self) { - let mut max_key_entry = None; - for (key, node) in Scanner::new(&self.children) { - let node_ptr = node.load(Acquire, guard); - let node_ref = node_ptr.as_ref().unwrap(); - if node_ref.retired() { - let result = self.children.remove_if(key, &mut |_| true); - debug_assert_ne!(result, RemoveResult::Fail); - - // Once the key is removed, it is safe to deallocate the node as the validation - // loop ensures the absence of readers. - if let Some(node) = node.swap((None, Tag::None), Release).0 { - let _: bool = node.release(); - node_deleted = true; - } + debug_assert!(!low_key_node.unbounded_child.is_null(Relaxed)); + if high_key_node_empty { + full_node.swap( + (Some(Shared::new(Node::Internal(low_key_node))), Tag::None), + AcqRel, + ); + } else if let Some(key) = boundary_key { + let high_key_node = Shared::new(Node::Internal(high_key_node)); + let result = self + .children + .insert(key, AtomicShared::new(Node::Internal(low_key_node))); + debug_assert!(matches!(result, InsertResult::Success)); + full_node.swap((Some(high_key_node), Tag::None), Release); } else { - max_key_entry.replace((key, node)); + return Ok(false); } - } - // The unbounded node is replaced with the maximum key node if retired. - let unbounded_ptr = self.unbounded_child.load(Acquire, guard); - let fully_empty = if let Some(unbounded) = unbounded_ptr.as_ref() { - if unbounded.retired() { - if let Some((key, max_key_child)) = max_key_entry { - if let Some(obsolete_node) = self - .unbounded_child - .swap( - (max_key_child.get_shared(Relaxed, guard), Tag::None), - Release, - ) - .0 - { - debug_assert!(obsolete_node.retired()); - let _: bool = obsolete_node.release(); - node_deleted = true; + locker.unlock_retire(); + } + Node::Leaf(target) => { + let Some(locker) = LeafNodeLocker::try_lock(target) else { + async_wait.try_wait(&target.lock); + return Err(()); + }; + let low_key_node = LeafNode::new(); + let high_key_node = LeafNode::new(); + let mut low_i = 0; + let mut boundary_key = None; + let mut high_i = 0; + if !target.children.distribute(|k, v, _, boundary, _| { + let Some(child) = v.get_shared(Acquire, guard) else { + return true; + }; + if low_i < boundary { + if low_i == boundary - 1 { + low_key_node + .unbounded_child + .swap((Some(child), Tag::None), Relaxed); + boundary_key.replace(k.clone()); + } else { + low_key_node.children.insert_unchecked( + k.clone(), + AtomicShared::from(child), + low_i, + ); } - let result = self.children.remove_if(key, &mut |_| true); - debug_assert_ne!(result, RemoveResult::Fail); - if let Some(node) = max_key_child.swap((None, Tag::None), Release).0 { - let _: bool = node.release(); - node_deleted = true; - } - false + low_i += 1; + } else if is_full { + return false; } else { - if let Some(obsolete_node) = - self.unbounded_child.swap((None, RETIRED), Release).0 - { - debug_assert!(obsolete_node.retired()); - let _: bool = obsolete_node.release(); - node_deleted = true; - } - true + high_key_node.children.insert_unchecked( + k.clone(), + AtomicShared::from(child), + high_i, + ); + high_i += 1; } - } else { - false + true + }) { + return Ok(false); } - } else { - debug_assert!(unbounded_ptr.tag() == RETIRED); - true - }; - if fully_empty { + let high_key_node_empty = + if high_i == 0 && low_key_node.unbounded_child.is_null(Relaxed) { + low_key_node.unbounded_child.swap( + (target.unbounded_child.get_shared(Acquire, guard), Tag::None), + Relaxed, + ); + true + } else if is_full { + return Ok(false); + } else { + high_key_node.unbounded_child.swap( + (target.unbounded_child.get_shared(Acquire, guard), Tag::None), + Relaxed, + ); + false + }; + + debug_assert!(!low_key_node.unbounded_child.is_null(Relaxed)); + if high_key_node_empty { + full_node.swap( + (Some(Shared::new(Node::Leaf(low_key_node))), Tag::None), + AcqRel, + ); + } else if let Some(key) = boundary_key { + let high_key_node = Shared::new(Node::Leaf(high_key_node)); + let result = self + .children + .insert(key, AtomicShared::new(Node::Leaf(low_key_node))); + debug_assert!(matches!(result, InsertResult::Success)); + full_node.swap((Some(high_key_node), Tag::None), Release); + } else { + return Ok(false); + } + + locker.unlock_retire(); + } + } + + Ok(true) + } + + /// Tries to delete retired nodes after a successful removal of an entry. + fn post_remove(&self, locker: Option>, guard: &Guard) -> RemoveResult { + let Some(lock) = locker.or_else(|| Locker::try_lock(self)) else { + if self.is_retired() { return RemoveResult::Retired; } + return RemoveResult::Success; + }; - drop(lock); - if !self.has_retired_node(guard) { - break; + let mut max_key_entry = None; + let mut iter = Iter::new(&self.children); + while let Some((key, node)) = iter.next() { + let node_ptr = node.load(Acquire, guard); + let node_ref = node_ptr.as_ref().unwrap(); + if node_ref.is_retired() { + let result = iter.remove_unchecked(); + debug_assert_ne!(result, RemoveResult::Fail); + + // Once the key is removed, it is safe to deallocate the node as the validation + // loop ensures the absence of readers. + node.swap((None, Tag::None), Release); + } else { + max_key_entry.replace((key, node)); } } - if node_deleted { - RemoveResult::Cleanup + // The unbounded node is replaced with the maximum key node if retired. + let unbounded_ptr = self.unbounded_child.load(Acquire, guard); + let fully_empty = if let Some(unbounded) = unbounded_ptr.as_ref() { + if unbounded.is_retired() { + if let Some((key, max_key_child)) = max_key_entry { + if let Some(obsolete_node) = self + .unbounded_child + .swap( + (max_key_child.get_shared(Relaxed, guard), Tag::None), + Release, + ) + .0 + { + debug_assert!(obsolete_node.is_retired()); + let _: bool = obsolete_node.release(); + } + self.children.remove_if(key, &mut |_| true); + max_key_child.swap((None, Tag::None), Release); + false + } else { + // `Tag::First` prevents `insert` from allocating a new node. + if let Some(obsolete_node) = + self.unbounded_child.swap((None, Tag::First), Release).0 + { + debug_assert!(obsolete_node.is_retired()); + let _: bool = obsolete_node.release(); + } + true + } + } else { + false + } + } else { + debug_assert!(unbounded_ptr.tag() != Tag::None); + true + }; + + if fully_empty { + lock.unlock_retire(); + RemoveResult::Retired } else { RemoveResult::Success } } - - /// Checks if the [`InternalNode`] has a retired [`Node`]. - fn has_retired_node(&self, guard: &Guard) -> bool { - let mut has_valid_node = false; - for entry in Scanner::new(&self.children) { - let leaf_ptr = entry.1.load(Relaxed, guard); - if let Some(leaf) = leaf_ptr.as_ref() { - if leaf.retired() { - return true; - } - has_valid_node = true; - } - } - if !has_valid_node { - let unbounded_ptr = self.unbounded_child.load(Relaxed, guard); - if let Some(unbounded) = unbounded_ptr.as_ref() { - if unbounded.retired() { - return true; - } - } - } - false - } } impl<'n, K, V> Locker<'n, K, V> { @@ -1050,7 +807,8 @@ impl<'n, K, V> Locker<'n, K, V> { } } - /// Retires the node with the lock released. + /// Retires the leaf node by poisoning the lock. + #[inline] pub(super) fn unlock_retire(self) { self.internal_node.lock.poison_lock(); forget(self); @@ -1064,29 +822,6 @@ impl Drop for Locker<'_, K, V> { } } -impl StructuralChange { - fn reset(&self) -> Option>> { - self.origin_node_key.store(ptr::null_mut(), Relaxed); - self.low_key_node.swap((None, Tag::None), Relaxed); - self.middle_key.store(ptr::null_mut(), Relaxed); - self.high_key_node.swap((None, Tag::None), Relaxed); - self.origin_node.swap((None, Tag::None), Relaxed).0 - } -} - -impl Default for StructuralChange { - #[inline] - fn default() -> Self { - Self { - origin_node_key: AtomicPtr::default(), - origin_node: AtomicShared::null(), - low_key_node: AtomicShared::null(), - middle_key: AtomicPtr::default(), - high_key_node: AtomicShared::null(), - } - } -} - #[cfg(not(feature = "loom"))] #[cfg(test)] mod test { @@ -1100,10 +835,8 @@ mod test { unbounded_child: AtomicShared::new(Node::Internal(InternalNode { children: Leaf::new(), unbounded_child: AtomicShared::new(Node::new_leaf_node()), - split_op: StructuralChange::default(), lock: Lock::default(), })), - split_op: StructuralChange::default(), lock: Lock::default(), } } @@ -1121,11 +854,8 @@ mod test { InsertResult::Success => { assert_eq!(internal_node.search_entry(&k, &guard), Some((&k, &k))); } - InsertResult::Duplicate(..) - | InsertResult::Frozen(..) - | InsertResult::Retired(..) => unreachable!(), + InsertResult::Duplicate(..) | InsertResult::Frozen(..) => unreachable!(), InsertResult::Full(_, _) => { - internal_node.rollback(&guard); for j in 0..k { assert_eq!(internal_node.search_entry(&j, &guard), Some((&j, &j))); if j == k - 1 { @@ -1149,11 +879,6 @@ mod test { } break; } - InsertResult::Retry(k, v) => { - let result = internal_node.insert(k, v, &mut (), &guard); - assert!(result.is_ok()); - assert_eq!(internal_node.search_entry(&k, &guard), Some((&k, &k))); - } }, Err((k, v)) => { let result = internal_node.insert(k, v, &mut (), &guard); @@ -1198,12 +923,11 @@ mod test { break; } InsertResult::Full(..) => { - internal_node_clone.rollback(&guard); max_key.replace(id); break; } - InsertResult::Frozen(..) | InsertResult::Retry(..) => (), - _ => unreachable!(), + InsertResult::Frozen(..) => (), + InsertResult::Duplicate(..) => unreachable!(), } } } @@ -1232,9 +956,7 @@ mod test { &guard, ) { match r { - RemoveResult::Success - | RemoveResult::Cleanup - | RemoveResult::Fail => break, + RemoveResult::Success | RemoveResult::Fail => break, RemoveResult::Frozen | RemoveResult::Retired => unreachable!(), } } @@ -1284,19 +1006,13 @@ mod test { { barrier_clone.wait().await; let guard = Guard::new(); - match internal_node_clone.insert( + if let Ok(InsertResult::Success) = internal_node_clone.insert( fixed_point, fixed_point, &mut (), &guard, ) { - Ok(InsertResult::Success) => { - assert!(!inserted_clone.swap(true, Relaxed)); - } - Ok(InsertResult::Full(_, _) | InsertResult::Retired(_, _)) => { - internal_node_clone.rollback(&guard); - } - _ => (), + assert!(!inserted_clone.swap(true, Relaxed)); } assert_eq!( internal_node_clone @@ -1310,12 +1026,8 @@ mod test { let guard = Guard::new(); for i in 0..workload_size { if i != fixed_point { - if let Ok( - InsertResult::Full(_, _) | InsertResult::Retired(_, _), - ) = internal_node_clone.insert(i, i, &mut (), &guard) - { - internal_node_clone.rollback(&guard); - } + let result = internal_node_clone.insert(i, i, &mut (), &guard); + drop(result); } assert_eq!( internal_node_clone @@ -1325,17 +1037,16 @@ mod test { ); } for i in 0..workload_size { - let max_scanner = internal_node_clone - .max_le_appr(&fixed_point, &guard) + let max_iter = internal_node_clone + .approximate::<_, true>(&fixed_point, &guard) .unwrap(); - assert!(*max_scanner.get().unwrap().0 <= fixed_point); - let mut min_scanner = internal_node_clone.min(&guard).unwrap(); - if let Some((f, v)) = min_scanner.next() { + assert!(*max_iter.get().unwrap().0 <= fixed_point); + let mut min_iter = internal_node_clone.min(&guard).unwrap(); + if let Some((f, v)) = min_iter.next() { assert_eq!(*f, *v); assert!(*f <= fixed_point); } else { - let (f, v) = - min_scanner.jump(None, &guard).unwrap().get().unwrap(); + let (f, v) = min_iter.jump(&guard).unwrap().get().unwrap(); assert_eq!(*f, *v); assert!(*f <= fixed_point); } diff --git a/patch/scc-3.4.8/src/tree_index/leaf.rs b/patch/scc-3.4.8/src/tree_index/leaf.rs index e5845ff..5ff5e81 100644 --- a/patch/scc-3.4.8/src/tree_index/leaf.rs +++ b/patch/scc-3.4.8/src/tree_index/leaf.rs @@ -4,16 +4,17 @@ use std::fmt::{self, Debug}; use std::mem::{MaybeUninit, needs_drop}; use std::ops::Bound::{Excluded, Included, Unbounded}; use std::ops::RangeBounds; -#[cfg(not(feature = "loom"))] -use std::sync::atomic::AtomicUsize; +use std::ptr; use std::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed, Release}; +#[cfg(not(feature = "loom"))] +use std::sync::atomic::{AtomicPtr, AtomicUsize}; -use sdd::{AtomicShared, Guard, Shared}; +use saa::Lock; +use sdd::Guard; use crate::Comparable; -use crate::LinkedList; #[cfg(feature = "loom")] -use loom::sync::atomic::AtomicUsize; +use loom::sync::atomic::{AtomicPtr, AtomicUsize}; /// [`Leaf`] is an ordered array of key-value pairs. /// @@ -30,10 +31,14 @@ pub struct Leaf { /// The entry state transitions as follows. /// * `uninit -> removed -> rank -> removed`. metadata: AtomicUsize, - /// The array of key-value pairs. + /// Entry array. entry_array: UnsafeCell>, - /// A pointer that points to the next adjacent [`Leaf`]. - link: AtomicShared>, + /// Lock to protect the linked list. + lock: Lock, + /// Pointer to the previous [`Leaf`]. + pub(super) prev: AtomicPtr>, + /// Pointer to the next [`Leaf`]. + pub(super) next: AtomicPtr>, } /// The number of entries and number of state bits per entry. @@ -43,7 +48,7 @@ pub struct Dimension { pub num_bits_per_entry: usize, } -/// The result of insertion. +/// Insertion result. pub enum InsertResult { /// Insertion succeeded. Success, @@ -55,22 +60,14 @@ pub enum InsertResult { /// /// This is not a terminal state as a frozen [`Leaf`] can be unfrozen. Frozen(K, V), - /// Insertion failed as the [`Leaf`] has retired. - /// - /// It is a terminal state. - Retired(K, V), - /// The operation can be retried. - Retry(K, V), } -/// The result of removal. +/// Remove result. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum RemoveResult { /// Remove succeeded. Success, /// Remove succeeded and cleanup required. - Cleanup, - /// Remove succeeded and the [`Leaf`] has retired without usable entries left. Retired, /// Remove failed. Fail, @@ -78,7 +75,30 @@ pub enum RemoveResult { Frozen, } +/// Each constructed entry in an `EntryArray` is never dropped until the [`Leaf`] is dropped. +pub type EntryArray = ( + [MaybeUninit; DIMENSION.num_entries], + [MaybeUninit; DIMENSION.num_entries], +); + +/// Leaf entry iterator. +pub struct Iter<'l, K, V> { + leaf: &'l Leaf, + metadata: usize, + index: [u8; DIMENSION.num_entries], + rank: u8, +} + +/// Leaf entry iterator, reversed. +pub struct RevIter<'l, K, V> { + leaf: &'l Leaf, + metadata: usize, + index: [u8; DIMENSION.num_entries], + rev_rank: u8, +} + /// Emulates `RangeBounds::contains`. +#[inline] pub(crate) fn range_contains>(range: &R, key: &K) -> bool where Q: Comparable + ?Sized, @@ -103,7 +123,9 @@ impl Leaf { Leaf { metadata: AtomicUsize::new(0), entry_array: UnsafeCell::new(unsafe { MaybeUninit::uninit().assume_init() }), - link: AtomicShared::null(), + lock: Lock::new(), + prev: AtomicPtr::new(ptr::null_mut()), + next: AtomicPtr::new(ptr::null_mut()), } } @@ -114,22 +136,24 @@ impl Leaf { Leaf { metadata: AtomicUsize::new(0), entry_array: UnsafeCell::new(unsafe { MaybeUninit::uninit().assume_init() }), - link: AtomicShared::null(), + lock: Lock::new(), + prev: AtomicPtr::new(ptr::null_mut()), + next: AtomicPtr::new(ptr::null_mut()), } } - /// Thaws the [`Leaf`]. + /// Returns `true` if the [`Leaf`] has no reachable entry. #[inline] - pub(super) fn thaw(&self) -> bool { - self.metadata - .fetch_update(Release, Relaxed, |p| { - if Dimension::frozen(p) { - Some(Dimension::thaw(p)) - } else { - None - } - }) - .is_ok() + pub(super) fn is_empty(&self) -> bool { + Self::len(self.metadata.load(Relaxed)) == 0 + } + + /// Checks if the leaf is full or retired. + #[inline] + pub(super) fn is_full(&self) -> bool { + let metadata = self.metadata.load(Relaxed); + let rank = DIMENSION.rank(metadata, DIMENSION.num_entries - 1); + rank != Dimension::uninit_rank() || Dimension::retired(metadata) } /// Returns `true` if the [`Leaf`] has retired. @@ -138,32 +162,89 @@ impl Leaf { Dimension::retired(self.metadata.load(Acquire)) } - /// Returns `true` if the [`Leaf`] has no reachable entry. + /// Replaces itself in the linked list with others as defined in the specified closure. #[inline] - pub(super) fn is_empty(&self) -> bool { - let mut mutable_metadata = self.metadata.load(Acquire); - for _ in 0..DIMENSION.num_entries { + pub(super) fn replace_link, Option<&Self>, &Guard)>( + &self, + f: F, + guard: &Guard, + ) { + let mut prev = self.prev.load(Acquire); + loop { + if let Some(prev_link) = unsafe { prev.as_ref() } { + prev_link.lock.lock_sync(); + } + self.lock.lock_sync(); + let prev_check = self.prev.load(Acquire); + if prev_check == prev { + break; + } + if let Some(prev_link) = unsafe { prev.as_ref() } { + prev_link.lock.release_lock(); + } + self.lock.release_lock(); + prev = prev_check; + } + let prev = unsafe { prev.as_ref() }; + let next = unsafe { self.next.load(Acquire).as_ref() }; + if let Some(next_link) = next { + next_link.lock.lock_sync(); + } + + f(prev, next, guard); + + self.metadata.fetch_or(Dimension::UNLINKED, Release); + if let Some(prev_link) = prev { + let released = prev_link.lock.release_lock(); + debug_assert!(released); + } + let released = self.lock.release_lock(); + debug_assert!(released); + if let Some(next_link) = next { + let released = next_link.lock.release_lock(); + debug_assert!(released); + } + } + + /// Deletes itself from the linked list. + #[inline] + pub(super) fn unlink(&self, guard: &Guard) { + self.replace_link( + |prev, next, _| { + if let Some(prev_link) = prev { + prev_link.next.store(self.next.load(Acquire), Release); + } + if let Some(next_link) = next { + next_link.prev.store(self.prev.load(Acquire), Release); + } + }, + guard, + ); + } + + /// Returns the number of reachable entries. + #[inline] + pub(super) const fn len(metadata: usize) -> usize { + let mut mutable_metadata = metadata; + let mut count = 0; + let mut i = 0; + while i != DIMENSION.num_entries { if mutable_metadata == 0 { break; } let rank = mutable_metadata % (1_usize << DIMENSION.num_bits_per_entry); if rank != Dimension::uninit_rank() && rank != DIMENSION.removed_rank() { - return false; + count += 1; } mutable_metadata >>= DIMENSION.num_bits_per_entry; + i += 1; } - true + count } /// Returns a reference to the max key. #[inline] pub(super) fn max_key(&self) -> Option<&K> { - self.max_entry().map(|(k, _)| k) - } - - /// Returns a reference to the max entry. - #[inline] - pub(super) fn max_entry(&self) -> Option<(&K, &V)> { let mut mutable_metadata = self.metadata.load(Acquire); let mut max_rank = 0; let mut max_index = DIMENSION.num_entries; @@ -179,7 +260,7 @@ impl Leaf { mutable_metadata >>= DIMENSION.num_bits_per_entry; } if max_index != DIMENSION.num_entries { - return Some((self.key_at(max_index), self.value_at(max_index))); + return Some(self.key_at(max_index)); } None } @@ -196,6 +277,53 @@ impl Leaf { self.metadata.store(new_metadata, Release); } + /// Removes the entry at the specified position without checking the metadata. + #[inline] + pub(super) fn remove_unchecked(&self, mut metadata: usize, index: usize) -> RemoveResult { + loop { + let mut empty = true; + let mut mutable_metadata = metadata; + for j in 0..DIMENSION.num_entries { + if mutable_metadata == 0 { + break; + } + if index != j { + let rank = mutable_metadata % (1_usize << DIMENSION.num_bits_per_entry); + if rank != Dimension::uninit_rank() && rank != DIMENSION.removed_rank() { + empty = false; + break; + } + } + mutable_metadata >>= DIMENSION.num_bits_per_entry; + } + + let mut new_metadata = metadata | DIMENSION.rank_mask(index); + if empty { + new_metadata = Dimension::retire(new_metadata); + } + match self + .metadata + .compare_exchange(metadata, new_metadata, AcqRel, Acquire) + { + Ok(_) => { + if empty { + return RemoveResult::Retired; + } + return RemoveResult::Success; + } + Err(actual) => { + if DIMENSION.rank(actual, index) == DIMENSION.removed_rank() { + return RemoveResult::Fail; + } + if Dimension::frozen(actual) { + return RemoveResult::Frozen; + } + metadata = actual; + } + } + } + } + /// Compares the given metadata value with the current one. #[inline] pub(super) fn validate(&self, metadata: usize) -> bool { @@ -219,6 +347,20 @@ impl Leaf { .is_ok() } + /// Unfreezes the [`Leaf`]. + #[inline] + pub(super) fn unfreeze(&self) -> bool { + self.metadata + .fetch_update(Release, Relaxed, |p| { + if Dimension::frozen(p) { + Some(Dimension::unfreeze(p)) + } else { + None + } + }) + .is_ok() + } + /// Returns the recommended number of entries that the left-side node should store when a /// [`Leaf`] is split. /// @@ -256,16 +398,19 @@ impl Leaf { } /// Returns a reference to the key at the given index. + #[inline] const fn key_at(&self, index: usize) -> &K { unsafe { &*(*self.entry_array.get()).0[index].as_ptr() } } /// Returns a reference to the key at the given index. + #[inline] const fn value_at(&self, index: usize) -> &V { unsafe { &*(*self.entry_array.get()).1[index].as_ptr() } } /// Writes the key and value at the given index. + #[inline] const fn write(&self, index: usize, key: K, val: V) { unsafe { (*self.entry_array.get()).0[index].as_mut_ptr().write(key); @@ -273,69 +418,40 @@ impl Leaf { } } - /// Takes the key and value at the given index. - const fn take(&self, index: usize) -> (K, V) { - unsafe { + /// Rolls back the insertion at the given index. + fn rollback(&self, index: usize) -> (K, V) { + let (k, v) = unsafe { ( (*self.entry_array.get()).0[index].as_ptr().read(), (*self.entry_array.get()).1[index].as_ptr().read(), ) - } - } - - /// Rolls back the insertion at the given index. - fn rollback(&self, index: usize) -> InsertResult { - let (key, val) = self.take(index); - let result = self - .metadata - .fetch_and(!DIMENSION.rank_mask(index), Release) - & (!DIMENSION.rank_mask(index)); - if Dimension::retired(result) { - InsertResult::Retired(key, val) - } else if Dimension::frozen(result) { - InsertResult::Frozen(key, val) - } else { - InsertResult::Duplicate(key, val) - } - } - - /// Returns the index of the corresponding entry of the next higher ranked entry. - fn next(index: usize, mut mutable_metadata: usize) -> usize { - debug_assert_ne!(index, usize::MAX); - let current_entry_rank = if index == DIMENSION.num_entries { - 0 - } else { - DIMENSION.rank(mutable_metadata, index) }; - let mut next_index = DIMENSION.num_entries; - if current_entry_rank < DIMENSION.num_entries { - let mut next_rank = DIMENSION.removed_rank(); - for i in 0..DIMENSION.num_entries { - if mutable_metadata == 0 { - break; - } - if i != index { - let rank = mutable_metadata % (1_usize << DIMENSION.num_bits_per_entry); - if rank != Dimension::uninit_rank() && rank < next_rank { - if rank == current_entry_rank + 1 { - return i; - } else if rank > current_entry_rank { - next_rank = rank; - next_index = i; - } - } - } - mutable_metadata >>= DIMENSION.num_bits_per_entry; + self.metadata + .fetch_and(!DIMENSION.rank_mask(index), Release); + (k, v) + } + + /// Builds a rank to index map from metadata. + #[allow(clippy::cast_possible_truncation)] + #[inline] + const fn build_index(metadata: usize) -> [u8; DIMENSION.num_entries] { + let mut index = [0; DIMENSION.num_entries]; + let mut i = 0; + while i != DIMENSION.num_entries { + let rank = DIMENSION.rank(metadata, i); + i += 1; + if rank != Dimension::uninit_rank() && rank != DIMENSION.removed_rank() { + index[rank - 1] = i as u8; } } - next_index + index } } impl Leaf where - K: 'static + Clone + Ord, - V: 'static + Clone, + K: 'static + Ord, + V: 'static, { /// Inserts a key value pair. #[inline] @@ -343,7 +459,7 @@ where let mut metadata = self.metadata.load(Acquire); 'after_read_metadata: loop { if Dimension::retired(metadata) { - return InsertResult::Retired(key, val); + return InsertResult::Full(key, val); } else if Dimension::frozen(metadata) { return InsertResult::Frozen(key, val); } @@ -388,7 +504,7 @@ where where Q: Comparable + ?Sized, { - let mut metadata = self.metadata.load(Acquire); + let metadata = self.metadata.load(Acquire); if Dimension::frozen(metadata) { return RemoveResult::Frozen; } @@ -414,57 +530,11 @@ where } Ordering::Equal => { // Found the key. - loop { - if !condition(self.value_at(i)) { - // The given condition is not met. - return RemoveResult::Fail; - } - let mut empty = true; - mutable_metadata = metadata; - for j in 0..DIMENSION.num_entries { - if mutable_metadata == 0 { - break; - } - if i != j { - let rank = mutable_metadata - % (1_usize << DIMENSION.num_bits_per_entry); - if rank != Dimension::uninit_rank() - && rank != DIMENSION.removed_rank() - { - empty = false; - break; - } - } - mutable_metadata >>= DIMENSION.num_bits_per_entry; - } - - let mut new_metadata = metadata | DIMENSION.rank_mask(i); - if empty { - new_metadata = Dimension::retire(new_metadata); - } - match self.metadata.compare_exchange( - metadata, - new_metadata, - AcqRel, - Acquire, - ) { - Ok(_) => { - if empty { - return RemoveResult::Retired; - } - return RemoveResult::Success; - } - Err(actual) => { - if DIMENSION.rank(actual, i) == DIMENSION.removed_rank() { - return RemoveResult::Fail; - } - if Dimension::frozen(actual) { - return RemoveResult::Frozen; - } - metadata = actual; - } - } + if !condition(self.value_at(i)) { + // The given condition is not met. + return RemoveResult::Fail; } + return self.remove_unchecked(metadata, i); } } } @@ -519,43 +589,6 @@ where self.search_slot(key, metadata).map(|i| self.value_at(i)) } - /// Returns the index of the key-value pair that has a key smaller than the given key. - #[inline] - pub(super) fn max_less(&self, mut mutable_metadata: usize, key: &Q) -> usize - where - Q: Comparable + ?Sized, - { - let mut min_max_rank = DIMENSION.removed_rank(); - let mut max_min_rank = 0; - let mut max_min_index = DIMENSION.num_entries; - for i in 0..DIMENSION.num_entries { - if mutable_metadata == 0 { - break; - } - let rank = mutable_metadata % (1_usize << DIMENSION.num_bits_per_entry); - if rank < min_max_rank && rank > max_min_rank { - match self.compare(i, key) { - Ordering::Less => { - if max_min_rank < rank { - max_min_rank = rank; - max_min_index = i; - } - } - Ordering::Greater => { - if min_max_rank > rank { - min_max_rank = rank; - } - } - Ordering::Equal => { - min_max_rank = rank; - } - } - } - mutable_metadata >>= DIMENSION.num_bits_per_entry; - } - max_min_index - } - /// Returns the minimum entry among those that are not `Ordering::Less` than the given key. /// /// It additionally returns the current version of its metadata so the caller can validate the @@ -605,40 +638,26 @@ where (None, metadata) } - /// Freezes the [`Leaf`] and distributes entries to two new leaves. + /// Distributes entries to given leaves. + /// + /// `dist` is a function to distribute entries to other containers where the first argument is + /// the key, the second argument is the value, the third argument is the index, the fourth + /// argument is the boundary, and the fifth argument is the length. Stops distribution if the + /// function returns `false`, and this method returns `false`. #[inline] - pub(super) fn freeze_and_distribute( + pub(super) fn distribute bool>( &self, - low_key_leaf: &mut Shared>, - high_key_leaf: &mut Option>>, - ) { - let metadata = unsafe { - self.metadata - .fetch_update(AcqRel, Acquire, |p| { - if Dimension::frozen(p) { - None - } else { - Some(Dimension::freeze(p)) - } - }) - .unwrap_unchecked() - }; - - let boundary = Self::optimal_boundary(metadata); - let scanner = Scanner { - leaf: self, - metadata, - entry_index: DIMENSION.num_entries, - }; - for (i, (k, v)) in scanner.enumerate() { - if i < boundary { - low_key_leaf.insert_unchecked(k.clone(), v.clone(), i); - } else { - high_key_leaf - .get_or_insert_with(|| Shared::new(Leaf::new())) - .insert_unchecked(k.clone(), v.clone(), i - boundary); + mut dist: F, + ) -> bool { + let iter = Iter::new(self); + let len = Self::len(iter.metadata); + let boundary = Self::optimal_boundary(iter.metadata); + for (i, (k, v)) in iter.enumerate() { + if !dist(k, v, i, boundary, len) { + return false; } } + true } /// Post-processing after reserving a free slot. @@ -669,7 +688,8 @@ where } Ordering::Equal => { // Duplicate key. - return self.rollback(free_slot_index); + let (k, v) = self.rollback(free_slot_index); + return InsertResult::Duplicate(k, v); } } } else if rank != DIMENSION.removed_rank() && rank > min_max_rank { @@ -684,8 +704,14 @@ where self.metadata .compare_exchange(prev_metadata, final_metadata, AcqRel, Acquire) { - if Dimension::frozen(actual) || Dimension::retired(actual) { - return self.rollback(free_slot_index); + let frozen = Dimension::frozen(actual); + let retired = Dimension::retired(actual); + if frozen || retired { + let (k, v) = self.rollback(free_slot_index); + if frozen { + return InsertResult::Frozen(k, v); + } + return InsertResult::Full(k, v); } prev_metadata = actual; continue; @@ -696,6 +722,7 @@ where } /// Searches for a slot in which the key is stored. + #[inline] fn search_slot(&self, key: &Q, mut mutable_metadata: usize) -> Option where Q: Comparable + ?Sized, @@ -729,6 +756,7 @@ where None } + #[inline] fn compare(&self, index: usize, key: &Q) -> Ordering where Q: Comparable + ?Sized, @@ -740,16 +768,27 @@ where impl Drop for Leaf { #[inline] fn drop(&mut self) { - if needs_drop::<(K, V)>() { + if needs_drop::() || needs_drop::() { let mut mutable_metadata = self.metadata.load(Acquire); + let is_frozen = Dimension::frozen(mutable_metadata); for i in 0..DIMENSION.num_entries { if mutable_metadata == 0 { break; } - if mutable_metadata % (1_usize << DIMENSION.num_bits_per_entry) - != Dimension::uninit_rank() - { - self.take(i); + let rank = mutable_metadata % (1_usize << DIMENSION.num_bits_per_entry); + if rank != Dimension::uninit_rank() { + if needs_drop::() { + unsafe { + (*self.entry_array.get()).0[i].as_mut_ptr().drop_in_place(); + } + } + if needs_drop::() && (!is_frozen || rank == DIMENSION.removed_rank()) { + // `self` being frozen means that reachable values have copied to another + // leaf, and they should not be dropped here. + unsafe { + (*self.entry_array.get()).1[i].as_mut_ptr().drop_in_place(); + } + } } mutable_metadata >>= DIMENSION.num_bits_per_entry; } @@ -757,64 +796,81 @@ impl Drop for Leaf { } } -/// [`LinkedList`] implementation for [`Leaf`]. -impl LinkedList for Leaf { - #[inline] - fn link_ref(&self) -> &AtomicShared> { - &self.link - } -} - unsafe impl Send for Leaf {} unsafe impl Sync for Leaf {} impl Dimension { + /// Flags indicating that the [`Leaf`] is unlinked. + const UNLINKED: usize = 1_usize << (usize::BITS - 3); + + /// Flags indicating that the [`Leaf`] is frozen. + const FROZEN: usize = 1_usize << (usize::BITS - 2); + + /// Flags indicating that the [`Leaf`] is retired. + const RETIRED: usize = 1_usize << (usize::BITS - 1); + + /// Checks if the [`Leaf`] is unlinked. + #[inline] + const fn unlinked(metadata: usize) -> bool { + metadata & Self::UNLINKED != 0 + } + /// Checks if the [`Leaf`] is frozen. + #[inline] const fn frozen(metadata: usize) -> bool { - metadata & (1_usize << (usize::BITS - 2)) != 0 + metadata & Self::FROZEN != 0 } /// Makes the metadata represent a frozen state. + #[inline] const fn freeze(metadata: usize) -> usize { - metadata | (1_usize << (usize::BITS - 2)) + metadata | Self::FROZEN } /// Updates the metadata to represent a non-frozen state. - const fn thaw(metadata: usize) -> usize { - metadata & (!(1_usize << (usize::BITS - 2))) + #[inline] + const fn unfreeze(metadata: usize) -> usize { + metadata & (!Self::FROZEN) } /// Checks if the [`Leaf`] is retired. + #[inline] const fn retired(metadata: usize) -> bool { - metadata & (1_usize << (usize::BITS - 1)) != 0 + metadata & Self::RETIRED != 0 } /// Makes the metadata represent a retired state. + #[inline] const fn retire(metadata: usize) -> usize { - metadata | (1_usize << (usize::BITS - 1)) + metadata | Self::RETIRED } /// Returns a bit mask for an entry. + #[inline] const fn rank_mask(&self, index: usize) -> usize { ((1_usize << self.num_bits_per_entry) - 1) << (index * self.num_bits_per_entry) } /// Returns the rank of an entry. + #[inline] const fn rank(&self, metadata: usize, index: usize) -> usize { (metadata >> (index * self.num_bits_per_entry)) % (1_usize << self.num_bits_per_entry) } /// Returns the uninitialized rank value which is smaller than all the valid rank values. + #[inline] const fn uninit_rank() -> usize { 0 } /// Returns the removed rank value which is greater than all the valid rank values. + #[inline] const fn removed_rank(&self) -> usize { (1_usize << self.num_bits_per_entry) - 1 } /// Augments the rank to the given metadata. + #[inline] const fn augment(&self, metadata: usize, index: usize, rank: usize) -> usize { (metadata & (!self.rank_mask(index))) | (rank << (index * self.num_bits_per_entry)) } @@ -825,11 +881,11 @@ impl Dimension { /// * `M`: The maximum number of entries. /// * `B`: The minimum number of bits to express the state of an entry. /// * `2`: The number of special states of an entry: uninitialized, removed. -/// * `2`: The number of special states of a [`Leaf`]: frozen, retired. +/// * `3`: The number of special states of a [`Leaf`]: frozen, retired, and unlinked. /// * `U`: `usize::BITS`. /// * `Eq1 = M + 2 <= 2^B`: `B` bits represent at least `M + 2` states. -/// * `Eq2 = B * M + 2 <= U`: `M entries + 2` special state. -/// * `Eq3 = Ceil(Log2(M + 2)) * M + 2 <= U`: derived from `Eq1` and `Eq2`. +/// * `Eq2 = B * M + 3 <= U`: `M entries + 3` special state. +/// * `Eq3 = Ceil(Log2(M + 2)) * M + 3 <= U`: derived from `Eq1` and `Eq2`. /// /// Therefore, when `U = 64 => M = 14 / B = 4`, and `U = 32 => M = 7 / B = 4`. pub const DIMENSION: Dimension = match usize::BITS / 8 { @@ -855,147 +911,296 @@ pub const DIMENSION: Dimension = match usize::BITS / 8 { }, }; -/// Each constructed entry in an `EntryArray` is never dropped until the [`Leaf`] is dropped. -pub type EntryArray = ( - [MaybeUninit; DIMENSION.num_entries], - [MaybeUninit; DIMENSION.num_entries], -); - -/// Leaf scanner. -pub struct Scanner<'l, K, V> { - leaf: &'l Leaf, - metadata: usize, - entry_index: usize, -} - -impl<'l, K, V> Scanner<'l, K, V> { - /// Creates a new [`Scanner`]. +impl<'l, K, V> Iter<'l, K, V> { + /// Creates a new [`Iter`]. #[inline] - pub(super) fn new(leaf: &'l Leaf) -> Scanner<'l, K, V> { - Scanner { - leaf, - metadata: leaf.metadata.load(Acquire), - entry_index: DIMENSION.num_entries, + pub(super) fn new(leaf: &'l Leaf) -> Iter<'l, K, V> { + let metadata = leaf.metadata.load(Acquire); + Self::with_metadata(leaf, metadata) + } + + /// Clones the iterator. + #[inline] + pub(super) const fn clone(&self) -> Iter<'l, K, V> { + Iter { ..*self } + } + + /// Rewinds the iterator to the beginning. + #[inline] + pub(super) const fn rewind(&mut self) { + self.rank = 0; + } + + /// Converts itself into a [`RevIter`]. + #[inline] + pub(super) const fn rev(self) -> RevIter<'l, K, V> { + // `DIMENSION.num_entries - (self.rev_rank as usize) == (self.rank as usize) - 1`. + #[allow(clippy::cast_possible_truncation)] + let rev_rank = if self.rank == 0 { + 0 + } else { + DIMENSION.num_entries as u8 + 1 - self.rank + }; + RevIter { + leaf: self.leaf, + index: self.index, + metadata: self.metadata, + rev_rank, } } - /// Returns the metadata that the [`Scanner`] is currently using. + /// Returns the snapshot of leaf metadata that the [`Iter`] took. #[inline] pub(super) const fn metadata(&self) -> usize { self.metadata } - /// Returns a reference to the entry that the scanner is currently pointing to. + /// Returns a reference to the entry that the iterator is currently pointing to. #[inline] pub(super) const fn get(&self) -> Option<(&'l K, &'l V)> { - if self.entry_index >= DIMENSION.num_entries { + if self.rank == 0 { return None; } - Some(( - self.leaf.key_at(self.entry_index), - self.leaf.value_at(self.entry_index), - )) + let index = self.index[(self.rank as usize) - 1] as usize - 1; + Some((self.leaf.key_at(index), self.leaf.value_at(index))) + } + + /// Removes the entry that the iterator is currently pointing to. + #[inline] + pub(super) fn remove_unchecked(&self) -> RemoveResult { + // `self.metadata` cannot be passed to the method as it may be outdated. + let index = self.index[(self.rank as usize) - 1] as usize - 1; + self.leaf + .remove_unchecked(self.leaf.metadata.load(Acquire), index) } /// Returns a reference to the max key. #[inline] pub(super) fn max_key(&self) -> Option<&'l K> { - self.leaf.max_key() - } - - /// Traverses the linked list. - #[inline] - pub(super) fn jump<'g>( - &self, - min_allowed_key: Option<&K>, - guard: &'g Guard, - ) -> Option> - where - K: Ord, - { - let mut next_leaf_ptr = self.leaf.next_ptr(Acquire, guard); - while let Some(next_leaf_ref) = next_leaf_ptr.as_ref() { - let mut leaf_scanner = Scanner::new(next_leaf_ref); - if let Some(key) = min_allowed_key { - if !self.leaf.is_clear(Relaxed) { - // Data race resolution: compare keys if the current leaf has been deleted. - // - // There is a chance that the current leaf has been deleted, and smaller - // keys have been inserted into the next leaf. - while let Some((k, _)) = leaf_scanner.next() { - if key.cmp(k) == Ordering::Less { - return Some(leaf_scanner); - } - } - next_leaf_ptr = next_leaf_ref.next_ptr(Acquire, guard); - continue; - } + for i in self.index.iter().rev() { + if *i != 0 { + return Some(self.leaf.key_at(*i as usize - 1)); } - if leaf_scanner.next().is_some() { - return Some(leaf_scanner); - } - next_leaf_ptr = next_leaf_ref.next_ptr(Acquire, guard); } None } - /// Proceeds to the next entry. - fn proceed(&mut self) { - if self.entry_index == usize::MAX { - return; - } - let index = Leaf::::next(self.entry_index, self.metadata); - if index == DIMENSION.num_entries { - // Fuse the iterator. - self.entry_index = usize::MAX; - } else { - self.entry_index = index; - } - } -} - -impl<'l, K, V> Scanner<'l, K, V> -where - K: 'static + Clone + Ord, - V: 'static + Clone, -{ - /// Returns a [`Scanner`] pointing to the max-less entry if there is one. + /// Jumps to the next non-empty leaf. #[inline] - pub(super) fn max_less(leaf: &'l Leaf, key: &Q) -> Option> + pub(super) fn jump(&self, _guard: &'l Guard) -> Option> where - Q: Comparable + ?Sized, + K: Ord, { - let metadata = leaf.metadata.load(Acquire); - let index = leaf.max_less(metadata, key); - if index == DIMENSION.num_entries { - None - } else { - Some(Scanner { - leaf, - metadata, - entry_index: index, - }) + let max_key = self.max_key(); + let mut found_unlinked = false; + let mut next_leaf = Some(self.leaf); + while let Some(current_leaf) = next_leaf.take() { + let next_leaf_ptr = current_leaf.next.load(Acquire); + let Some(leaf) = (unsafe { next_leaf_ptr.as_ref() }) else { + break; + }; + let metadata = leaf.metadata.load(Acquire); + let mut iter = Iter::with_metadata(leaf, metadata); + + found_unlinked |= Dimension::unlinked(current_leaf.metadata.load(Acquire)); + if found_unlinked { + // Data race resolution: + // - T1: remove(L1) -> range(L0) -> traverse(L1) + // - T2: unlink(L0) -> delete(L0) + // - T3: insertSmall(L1) + // + // T1 must not see T3's insertion while it still needs to observe its own deletion. + // Therefore, keys that are smaller than the max key in the current leaf should be + // filtered out here. + while let Some((k, _)) = iter.next() { + if max_key.is_none_or(|max| max < k) { + return Some(iter); + } + } + } + if iter.next().is_some() { + return Some(iter); + } + // Empty leaf: continue. + next_leaf = Some(leaf); + } + None + } + + /// Creates a new [`Iter`] with the supplied metadata. + #[inline] + const fn with_metadata(leaf: &'l Leaf, metadata: usize) -> Iter<'l, K, V> { + let index = Leaf::::build_index(metadata); + Iter { + leaf, + metadata, + index, + rank: 0, } } } -impl Debug for Scanner<'_, K, V> { +impl Debug for Iter<'_, K, V> { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Scanner") + f.debug_struct("Iter") .field("metadata", &self.metadata) - .field("entry_index", &self.entry_index) + .field("rank", &self.rank) .finish() } } -impl<'l, K, V> Iterator for Scanner<'l, K, V> { +impl<'l, K, V> Iterator for Iter<'l, K, V> { type Item = (&'l K, &'l V); #[inline] fn next(&mut self) -> Option { - self.proceed(); - self.get() + loop { + self.rank += 1; + if (self.rank as usize) > DIMENSION.num_entries { + self.rank = 0; + return None; + } + let index = self.index[(self.rank as usize) - 1] as usize; + if index != 0 { + return Some((self.leaf.key_at(index - 1), self.leaf.value_at(index - 1))); + } + } + } +} + +impl<'l, K, V> RevIter<'l, K, V> { + /// Creates a new [`RevIter`]. + #[inline] + pub(super) fn new(leaf: &'l Leaf) -> RevIter<'l, K, V> { + let metadata = leaf.metadata.load(Acquire); + Self::with_metadata(leaf, metadata) + } + + /// Rewinds the iterator to the beginning. + #[inline] + pub(super) const fn rewind(&mut self) { + self.rev_rank = 0; + } + + /// Converts itself into an [`Iter`]. + #[inline] + pub(super) const fn rev(self) -> Iter<'l, K, V> { + // `DIMENSION.num_entries - (self.rev_rank as usize) == (self.rank as usize) - 1`. + #[allow(clippy::cast_possible_truncation)] + let rank = if self.rev_rank == 0 { + 0 + } else { + DIMENSION.num_entries as u8 + 1 - self.rev_rank + }; + + Iter { + leaf: self.leaf, + metadata: self.metadata, + index: self.index, + rank, + } + } + + /// Returns the snapshot of leaf metadata that the [`RevIter`] took. + #[inline] + pub(super) const fn metadata(&self) -> usize { + self.metadata + } + + /// Returns a reference to the entry that the iterator is currently pointing to. + #[inline] + pub(super) const fn get(&self) -> Option<(&'l K, &'l V)> { + if self.rev_rank == 0 { + return None; + } + let index = self.index[DIMENSION.num_entries - (self.rev_rank as usize)] as usize - 1; + Some((self.leaf.key_at(index), self.leaf.value_at(index))) + } + + /// Returns a reference to the min key entry. + #[inline] + pub(super) fn min_key(&self) -> Option<&'l K> { + for i in self.index { + if i != 0 { + return Some(self.leaf.key_at(i as usize - 1)); + } + } + None + } + + /// Jumps to the prev non-empty leaf. + #[inline] + pub(super) fn jump(&self, _guard: &'l Guard) -> Option> + where + K: Ord, + { + let min_key = self.min_key(); + let mut prev_leaf = Some(self.leaf); + let mut found_unlinked = false; + while let Some(current_leaf) = prev_leaf.take() { + let prev_leaf_ptr = current_leaf.prev.load(Acquire); + let Some(leaf) = (unsafe { prev_leaf_ptr.as_ref() }) else { + break; + }; + let metadata = leaf.metadata.load(Acquire); + let mut iter = RevIter::with_metadata(leaf, metadata); + + found_unlinked |= Dimension::unlinked(current_leaf.metadata.load(Acquire)); + if found_unlinked { + // See `Iter::jump`. + while let Some((k, _)) = iter.next() { + if min_key.is_none_or(|min| min > k) { + return Some(iter); + } + } + } + if iter.next().is_some() { + return Some(iter); + } + // Empty leaf: continue. + prev_leaf = Some(leaf); + } + None + } + + #[inline] + const fn with_metadata(leaf: &'l Leaf, metadata: usize) -> RevIter<'l, K, V> { + let index = Leaf::::build_index(metadata); + RevIter { + leaf, + metadata, + index, + rev_rank: 0, + } + } +} + +impl Debug for RevIter<'_, K, V> { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RevIter") + .field("rev_rank", &self.rev_rank) + .finish() + } +} + +impl<'l, K, V> Iterator for RevIter<'l, K, V> { + type Item = (&'l K, &'l V); + + #[inline] + fn next(&mut self) -> Option { + loop { + self.rev_rank += 1; + if (self.rev_rank as usize) > DIMENSION.num_entries { + self.rev_rank = 0; + return None; + } + let index = self.index[DIMENSION.num_entries - (self.rev_rank as usize)] as usize; + if index != 0 { + return Some((self.leaf.key_at(index - 1), self.leaf.value_at(index - 1))); + } + } } } @@ -1004,6 +1209,7 @@ impl<'l, K, V> Iterator for Scanner<'l, K, V> { mod test { use super::*; use proptest::prelude::*; + use sdd::Shared; use std::sync::atomic::AtomicBool; use tokio::sync::Barrier; @@ -1067,9 +1273,9 @@ mod test { InsertResult::Full(..) )); - let mut scanner = Scanner::new(&leaf); + let mut iter = Iter::new(&leaf); for i in 0..DIMENSION.num_entries { - if let Some(e) = scanner.next() { + if let Some(e) = iter.next() { assert_eq!(e.0, &i.to_string()); assert_eq!(e.1, &i.to_string()); assert_ne!( @@ -1083,10 +1289,54 @@ mod test { assert!(matches!( leaf.insert("200".to_owned(), "200".to_owned()), - InsertResult::Retired(..) + InsertResult::Full(..) )); } + #[test] + fn iter_rev_iter() { + let leaf: Leaf = Leaf::new(); + for i in 0..DIMENSION.num_entries { + if i % 2 == 0 { + assert!(matches!( + leaf.insert(i * 1024 + 1, i), + InsertResult::Success + )); + } else { + assert!(matches!(leaf.insert(i * 2, i), InsertResult::Success)); + } + } + assert!(matches!( + leaf.remove_if(&6, &mut |_| true), + RemoveResult::Success + )); + + let mut iter = Iter::new(&leaf); + assert_eq!(iter.next(), Some((&1, &0))); + let rev_iter = iter.rev(); + assert_eq!(rev_iter.get(), Some((&1, &0))); + iter = rev_iter.rev(); + assert_eq!(iter.get(), Some((&1, &0))); + + let mut prev_key = 0; + let mut sum = 0; + for (key, _) in Iter::new(&leaf) { + assert_ne!(*key, 6); + assert!(prev_key < *key); + prev_key = *key; + sum += *key; + } + prev_key = usize::MAX; + + for (key, _) in RevIter::new(&leaf) { + assert_ne!(*key, 6); + assert!(prev_key > *key); + prev_key = *key; + sum -= *key; + } + assert_eq!(sum, 0); + } + #[test] fn calculate_boundary() { let leaf: Leaf = Leaf::new(); @@ -1137,24 +1387,32 @@ mod test { assert!(matches!(leaf.insert(11, 17), InsertResult::Success)); assert!(matches!(leaf.insert(17, 11), InsertResult::Success)); - let mut leaf1 = Shared::new(Leaf::new()); - let mut leaf2 = None; - leaf.freeze_and_distribute(&mut leaf1, &mut leaf2); + let leaf1 = Leaf::new(); + let leaf2 = Leaf::new(); + assert!(leaf.freeze()); + leaf.distribute(|k, v, i, b, _| { + if i < b { + leaf1.insert_unchecked(*k, *v, i); + } else { + leaf2.insert_unchecked(*k, *v, i - b); + } + true + }); assert_eq!(leaf1.search_entry(&11), Some((&11, &17))); assert_eq!(leaf1.search_entry(&17), Some((&17, &11))); - assert!(leaf2.is_none()); + assert!(leaf2.is_empty()); assert!(matches!(leaf.insert(1, 7), InsertResult::Frozen(..))); assert_eq!(leaf.remove_if(&17, &mut |_| true), RemoveResult::Frozen); assert!(matches!(leaf.insert(3, 5), InsertResult::Frozen(..))); - assert!(leaf.thaw()); + assert!(leaf.unfreeze()); assert!(matches!(leaf.insert(1, 7), InsertResult::Success)); assert_eq!(leaf.remove_if(&1, &mut |_| true), RemoveResult::Success); assert_eq!(leaf.remove_if(&17, &mut |_| true), RemoveResult::Success); assert_eq!(leaf.remove_if(&11, &mut |_| true), RemoveResult::Retired); - assert!(matches!(leaf.insert(5, 3), InsertResult::Retired(..))); + assert!(matches!(leaf.insert(5, 3), InsertResult::Full(..))); } proptest! { @@ -1165,11 +1423,6 @@ mod test { assert!(leaf.is_empty()); for i in 0..insert { assert!(matches!(leaf.insert(i, i), InsertResult::Success)); - if i != 0 { - let result = leaf.max_less(leaf.metadata.load(Relaxed), &i); - assert_eq!(*leaf.key_at(result), i - 1); - assert_eq!(*leaf.value_at(result), i - 1); - } } if insert == 0 { assert_eq!(leaf.max_key(), None); @@ -1195,7 +1448,7 @@ mod test { if i == insert - 1 { assert!(matches!(leaf.remove_if(&i, &mut |_| true), RemoveResult::Retired)); for i in 0..insert { - assert!(matches!(leaf.insert(i, i), InsertResult::Retired(..))); + assert!(matches!(leaf.insert(i, i), InsertResult::Full(..))); } } else { assert!(matches!(leaf.remove_if(&i, &mut |_| true), RemoveResult::Success)); @@ -1246,10 +1499,7 @@ mod test { assert_eq!(leaf_clone.search_entry(&t).unwrap(), (&t, &t)); true } - InsertResult::Duplicate(_, _) - | InsertResult::Frozen(_, _) - | InsertResult::Retired(_, _) - | InsertResult::Retry(_, _) => { + InsertResult::Duplicate(_, _) | InsertResult::Frozen(_, _) => { unreachable!(); } InsertResult::Full(k, v) => { @@ -1261,8 +1511,8 @@ mod test { }; { let mut prev = 0; - let mut scanner = Scanner::new(&leaf_clone); - for e in scanner.by_ref() { + let mut iter = Iter::new(&leaf_clone); + for e in iter.by_ref() { assert_eq!(e.0, e.1); assert!(*e.0 > prev); prev = *e.0; @@ -1275,15 +1525,15 @@ mod test { assert_eq!(leaf_clone.search_entry(&t).unwrap(), (&t, &t)); } { - let scanner = Scanner::new(&leaf_clone); - assert_eq!(scanner.count(), DIMENSION.num_entries); + let iter = Iter::new(&leaf_clone); + assert_eq!(iter.count(), DIMENSION.num_entries); } barrier_clone.wait().await; match leaf_clone.remove_if(&t, &mut |_| true) { RemoveResult::Success => assert!(inserted), RemoveResult::Fail => assert!(!inserted), - RemoveResult::Frozen | RemoveResult::Cleanup => unreachable!(), + RemoveResult::Frozen => unreachable!(), RemoveResult::Retired => { assert!(inserted); assert_eq!(retire_clone.swap(1, Relaxed), 0); diff --git a/patch/scc-3.4.8/src/tree_index/leaf_node.rs b/patch/scc-3.4.8/src/tree_index/leaf_node.rs index 18cdda2..d6fa6a4 100644 --- a/patch/scc-3.4.8/src/tree_index/leaf_node.rs +++ b/patch/scc-3.4.8/src/tree_index/leaf_node.rs @@ -1,23 +1,19 @@ use std::cmp::Ordering::{Equal, Greater, Less}; +use std::mem::forget; use std::ops::{Bound, RangeBounds}; use std::ptr; -use std::sync::atomic::AtomicPtr; use std::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed, Release}; use saa::Lock; use sdd::{AtomicShared, Guard, Ptr, Shared, Tag}; use super::Leaf; -use super::leaf::{DIMENSION, InsertResult, RemoveResult, Scanner, range_contains}; +use super::leaf::{InsertResult, Iter, RemoveResult, RevIter, range_contains}; use super::node::Node; use crate::Comparable; -use crate::LinkedList; use crate::async_helper::TryWait; use crate::exit_guard::ExitGuard; -/// [`Tag::First`] indicates that the corresponding node has retired. -pub const RETIRED: Tag = Tag::First; - /// [`LeafNode`] contains a list of instances of `K, V` [`Leaf`]. /// /// The layout of a leaf node: `|ptr(entry array)/max(child keys)|...|ptr(entry array)|` @@ -28,10 +24,8 @@ pub struct LeafNode { /// /// It stores the maximum key in the node, and key-value pairs are first pushed to this /// [`Leaf`] until it splits. - unbounded_child: AtomicShared>, - /// Ongoing split operation. - split_op: StructuralChange, - /// [`Lock`] to protecct the [`LeafNode`]. + pub(super) unbounded_child: AtomicShared>, + /// [`Lock`] to protect the [`LeafNode`]. pub(super) lock: Lock, } @@ -55,19 +49,6 @@ pub(super) enum RemoveRangeState { MaybeAbove, } -/// [`StructuralChange`] stores intermediate results during a split operation. -/// -/// `AtomicPtr` members may point to values under the protection of the [`Guard`] used for the -/// split operation. -pub(super) struct StructuralChange { - origin_leaf_key: AtomicPtr, - origin_leaf: AtomicShared>, - low_key_leaf: AtomicShared>, - high_key_leaf: AtomicShared>, - low_key_leaf_node: AtomicPtr>, - high_key_leaf_node: AtomicPtr>, -} - impl LeafNode { /// Creates a new empty [`LeafNode`]. #[inline] @@ -75,7 +56,6 @@ impl LeafNode { LeafNode { children: Leaf::new(), unbounded_child: AtomicShared::null(), - split_op: StructuralChange::default(), lock: Lock::default(), } } @@ -83,30 +63,35 @@ impl LeafNode { /// Clears the leaf node by unlinking all the leaves. #[inline] pub(super) fn clear(&self, guard: &Guard) { - let scanner = Scanner::new(&self.children); - for (_, child) in scanner { + // Mark the unbounded to prevent any on-going split operation to cleanup itself. + self.unbounded_child + .update_tag_if(Tag::First, |_| true, Release, Relaxed); + + // Unlink all the children + let iter = Iter::new(&self.children); + for (_, child) in iter { let child_ptr = child.load(Acquire, guard); if let Some(child) = child_ptr.as_ref() { - child.link_ref().swap((None, Tag::None), Acquire); + child.unlink(guard); } } let unbounded_ptr = self.unbounded_child.load(Acquire, guard); if let Some(unbounded) = unbounded_ptr.as_ref() { - unbounded.link_ref().swap((None, Tag::None), Acquire); + unbounded.unlink(guard); } } /// Returns `true` if the [`LeafNode`] has retired. #[inline] - pub(super) fn retired(&self) -> bool { - self.unbounded_child.tag(Acquire) == RETIRED + pub(super) fn is_retired(&self) -> bool { + self.lock.is_poisoned(Acquire) } } impl LeafNode where K: 'static + Clone + Ord, - V: 'static + Clone, + V: 'static, { /// Searches for an entry containing the specified key. #[inline] @@ -180,78 +165,117 @@ where } } - /// Returns the minimum key entry. + /// Returns the minimum key entry in the entire tree. #[inline] - pub(super) fn min<'g>(&self, guard: &'g Guard) -> Option> { - loop { - let mut scanner = Scanner::new(&self.children); - let metadata = scanner.metadata(); - if let Some((_, child)) = scanner.next() { - let child_ptr = child.load(Acquire, guard); - if let Some(child) = child_ptr.as_ref() { - if self.children.validate(metadata) { - // Data race resolution - see `LeafNode::search_entry`. - return Some(Scanner::new(child)); - } - } - // It is not a hot loop - see `LeafNode::search_entry`. - continue; + pub(super) fn min<'g>(&self, guard: &'g Guard) -> Option> { + let mut min_leaf = None; + for (_, child) in Iter::new(&self.children) { + let child_ptr = child.load(Acquire, guard); + if let Some(child) = child_ptr.as_ref() { + min_leaf.replace(child); + break; } + } + if min_leaf.is_none() { let unbounded_ptr = self.unbounded_child.load(Acquire, guard); if let Some(unbounded) = unbounded_ptr.as_ref() { - if self.children.validate(metadata) { - return Some(Scanner::new(unbounded)); - } - continue; + min_leaf.replace(unbounded); } - return None; } + + let Some(min_leaf) = min_leaf else { + // `unbounded_child` being null means that the leaf was retired of empty. + return None; + }; + + let mut rev_iter = RevIter::new(min_leaf); + while let Some(next_rev_iter) = rev_iter.jump(guard) { + rev_iter = next_rev_iter; + } + rev_iter.rewind(); + Some(rev_iter.rev()) } - /// Returns a [`Scanner`] pointing to an entry that is close enough to the entry with the - /// maximum key among those keys that are smaller than or equal to the given key. - /// - /// Returns `None` if all keys in the [`LeafNode`] are greater than the given key. + /// Returns the maximum key entry in the entire tree. #[inline] - pub(super) fn max_le_appr<'g, Q>(&self, key: &Q, guard: &'g Guard) -> Option> + pub(super) fn max<'g>(&self, guard: &'g Guard) -> Option> { + let unbounded_ptr = self.unbounded_child.load(Acquire, guard); + if let Some(unbounded) = unbounded_ptr.as_ref() { + let mut iter = Iter::new(unbounded); + while let Some(next_iter) = iter.jump(guard) { + iter = next_iter; + iter.rewind(); + } + return Some(iter.rev()); + } + // `unbounded_child` being null means that the leaf was retired of empty. + None + } + + /// Returns a [`Iter`] pointing to an entry that is close enough to the specified key. + #[inline] + pub(super) fn approximate<'g, Q, const LE: bool>( + &self, + key: &Q, + guard: &'g Guard, + ) -> Option> where K: 'g, Q: Comparable + ?Sized, { - loop { - if let Some(scanner) = Scanner::max_less(&self.children, key) { - if let Some((_, child)) = scanner.get() { - if let Some(child) = child.load(Acquire, guard).as_ref() { - if self.children.validate(scanner.metadata()) { - // Data race resolution - see `LeafNode::search_entry`. - if let Some(scanner) = Scanner::max_less(child, key) { - return Some(scanner); - } - // Fallback. - break; - } - } - // It is not a hot loop - see `LeafNode::search_entry`. - continue; + let leaf = loop { + if let (Some((_, child)), _) = self.children.min_greater_equal(key) { + if let Some(child) = child.load(Acquire, guard).as_ref() { + break child; } + // It is not a hot loop - see `LeafNode::search_entry`. + continue; + } + if let Some(unbounded) = self.unbounded_child.load(Acquire, guard).as_ref() { + break unbounded; + } + // `unbounded_child` being null means that the leaf was retired of empty. + return None; + }; + + // Tries to find "any" leaf that contains a reachable entry. + let origin = Iter::new(leaf); + let mut iter = origin.clone(); + if iter.next().is_none() { + if let Some(next) = iter.jump(guard) { + iter = next; + } else if let Some(prev) = origin.rev().jump(guard) { + iter = prev.rev(); + } else { + return None; + } + } + iter.rewind(); + + if LE { + while let Some((k, _)) = iter.next() { + if let Equal | Greater = key.compare(k) { + return Some(iter); + } + // Go to the prev leaf node that shall contain smaller keys. + iter = iter.rev().jump(guard)?.rev(); + // Rewind the iterator to point to the smallest key in the leaf. + iter.rewind(); + } + } else { + let mut rev_iter = iter.rev(); + while let Some((k, _)) = rev_iter.next() { + if let Less | Equal = key.compare(k) { + return Some(rev_iter.rev()); + } + // Go to the next leaf node that shall contain larger keys. + rev_iter = rev_iter.rev().jump(guard)?.rev(); + // Rewind the iterator to point to the largest key in the leaf. + rev_iter.rewind(); } - // Fallback. - break; - } - - // Starts scanning from the minimum key. - let mut min_scanner = self.min(guard)?; - min_scanner.next(); - loop { - if let Some((k, _)) = min_scanner.get() { - if key.compare(k).is_ge() { - return Some(min_scanner); - } - break; - } - min_scanner = min_scanner.jump(None, guard)?; } + // Reached the end of the linked list. None } @@ -270,31 +294,26 @@ where ) -> Result, (K, V)> { loop { let (child, metadata) = self.children.min_greater_equal(&key); - if let Some((child_key, child)) = child { + if let Some((_, child)) = child { let child_ptr = child.load(Acquire, guard); if let Some(child_ref) = child_ptr.as_ref() { if self.children.validate(metadata) { // Data race resolution - see `LeafNode::search_entry`. let insert_result = child_ref.insert(key, val); match insert_result { - InsertResult::Success - | InsertResult::Duplicate(..) - | InsertResult::Retry(..) => return Ok(insert_result), - InsertResult::Full(k, v) | InsertResult::Retired(k, v) => { - let split_result = self.split_leaf( - (k, v), - Some(child_key), - child_ptr, - child, - async_wait, - guard, - )?; - if let InsertResult::Retry(k, v) = split_result { - key = k; - val = v; - continue; + InsertResult::Success | InsertResult::Duplicate(..) => { + return Ok(insert_result); + } + InsertResult::Full(k, v) => { + match self.split_leaf(child_ptr, child, async_wait, guard) { + Ok(true) => { + key = k; + val = v; + continue; + } + Ok(false) => return Ok(InsertResult::Full(k, v)), + Err(()) => return Err((k, v)), } - return Ok(split_result); } InsertResult::Frozen(k, v) => { // The `Leaf` is being split: retry. @@ -326,32 +345,29 @@ where } } if let Some(unbounded) = unbounded_ptr.as_ref() { - debug_assert!(unbounded_ptr.tag() == Tag::None); if !self.children.validate(metadata) { continue; } let insert_result = unbounded.insert(key, val); match insert_result { - InsertResult::Success - | InsertResult::Duplicate(..) - | InsertResult::Retry(..) => { + InsertResult::Success | InsertResult::Duplicate(..) => { return Ok(insert_result); } - InsertResult::Full(k, v) | InsertResult::Retired(k, v) => { - let split_result = self.split_leaf( - (k, v), - None, + InsertResult::Full(k, v) => { + match self.split_leaf( unbounded_ptr, &self.unbounded_child, async_wait, guard, - )?; - if let InsertResult::Retry(k, v) = split_result { - key = k; - val = v; - continue; + ) { + Ok(true) => { + key = k; + val = v; + continue; + } + Ok(false) => return Ok(InsertResult::Full(k, v)), + Err(()) => return Err((k, v)), } - return Ok(split_result); } InsertResult::Frozen(k, v) => { async_wait.try_wait(&self.lock); @@ -359,7 +375,7 @@ where } }; } - return Ok(InsertResult::Retired(key, val)); + return Ok(InsertResult::Full(key, val)); } } @@ -388,12 +404,11 @@ where // Data race resolution - see `LeafNode::search_entry`. let result = child.remove_if(key, condition); if result == RemoveResult::Frozen { - // When a `Leaf` is frozen, its entries may be being copied to new - // `Leaves`. + // Its entries may be being relocated. async_wait.try_wait(&self.lock); return Err(()); } else if result == RemoveResult::Retired { - return Ok(self.coalesce(guard)); + return Ok(self.post_remove(guard)); } return Ok(result); } @@ -403,7 +418,6 @@ where } let unbounded_ptr = self.unbounded_child.load(Acquire, guard); if let Some(unbounded) = unbounded_ptr.as_ref() { - debug_assert!(unbounded_ptr.tag() == Tag::None); if !self.children.validate(metadata) { // Data race resolution - see `LeafNode::search_entry`. continue; @@ -413,7 +427,7 @@ where async_wait.try_wait(&self.lock); return Err(()); } else if result == RemoveResult::Retired { - return Ok(self.coalesce(guard)); + return Ok(self.post_remove(guard)); } return Ok(result); } @@ -449,7 +463,8 @@ where let mut num_leaves = 1; let mut first_valid_leaf = None; - for (key, leaf) in Scanner::new(&self.children) { + let mut iter = Iter::new(&self.children); + while let Some((key, leaf)) = iter.next() { current_state = current_state.next(key, range, start_unbounded); match current_state { RemoveRangeState::Below | RemoveRangeState::MaybeBelow => { @@ -462,12 +477,12 @@ where } } RemoveRangeState::FullyContained => { - // There can be another thread inserting keys into the leaf, and this may - // render those operations completely ineffective. - self.children.remove_if(key, &mut |_| true); if let Some(leaf) = leaf.swap((None, Tag::None), AcqRel).0 { - leaf.delete_self(Release); + leaf.unlink(guard); } + // There can be another thread inserting keys into the leaf, and this may render + // those operations completely ineffective. + iter.remove_unchecked(); } RemoveRangeState::MaybeAbove => { if let Some(leaf) = leaf.load(Acquire, guard).as_ref() { @@ -491,13 +506,16 @@ where if first_valid_leaf.is_none() { first_valid_leaf.replace(&self.unbounded_child); } - valid_lower_max_leaf.link_ref().swap( - ( - first_valid_leaf.and_then(|l| l.get_shared(Acquire, guard)), - Tag::None, - ), - AcqRel, - ); + let first_valid_leaf_ptr = + first_valid_leaf.map_or(Ptr::null(), |l| l.load(Acquire, guard)); + valid_lower_max_leaf + .next + .store(first_valid_leaf_ptr.as_ptr().cast_mut(), Release); + if let Some(first_valid_leaf) = first_valid_leaf_ptr.as_ref() { + first_valid_leaf + .prev + .store(ptr::from_ref(valid_lower_max_leaf).cast_mut(), Release); + } } else if let Some(valid_upper_min_node) = valid_upper_min_node { // Connect the unbounded child with the minimum valid leaf in the node. valid_upper_min_node.remove_range( @@ -513,504 +531,203 @@ where Ok(num_leaves) } - /// Splits itself into the given leaf nodes, and returns the middle key value. - #[allow(clippy::too_many_lines)] - pub(super) fn split_leaf_node<'g>( - &'g self, - low_key_leaf_node: &LeafNode, - high_key_leaf_node: &LeafNode, - guard: &'g Guard, - ) -> &'g K { - let mut middle_key = None; - - low_key_leaf_node.lock.lock_sync(); - high_key_leaf_node.lock.lock_sync(); - - // It is safe to keep the pointers to the new leaf nodes in this full leaf node since the - // whole split operation is protected under a single `ebr::Guard`, and the pointers are - // only dereferenced during the operation. - self.split_op - .low_key_leaf_node - .swap(ptr::from_ref(low_key_leaf_node).cast_mut(), Relaxed); - self.split_op - .high_key_leaf_node - .swap(ptr::from_ref(high_key_leaf_node).cast_mut(), Relaxed); - - // Builds a list of valid leaves - #[allow(clippy::type_complexity)] - let mut entry_array: [Option<(Option<&K>, AtomicShared>)>; - DIMENSION.num_entries + 2] = Default::default(); - let mut num_entries = 0; - let low_key_leaf_ref = self - .split_op - .low_key_leaf - .load(Relaxed, guard) - .as_ref() - .unwrap(); - let middle_key_ref = low_key_leaf_ref.max_key().unwrap(); - let scanner = Scanner::new(&self.children); - let recommended_boundary = Leaf::::optimal_boundary(scanner.metadata()); - for entry in scanner { - if unsafe { - self.split_op - .origin_leaf_key - .load(Relaxed) - .as_ref() - .map_or_else(|| false, |key| entry.0 == key) - } { - entry_array[num_entries].replace(( - Some(middle_key_ref), - self.split_op.low_key_leaf.clone(Relaxed, guard), - )); - num_entries += 1; - if !self.split_op.high_key_leaf.is_null(Relaxed) { - entry_array[num_entries].replace(( - Some(entry.0), - self.split_op.high_key_leaf.clone(Relaxed, guard), - )); - num_entries += 1; - } - } else { - entry_array[num_entries].replace((Some(entry.0), entry.1.clone(Acquire, guard))); - num_entries += 1; - } - } - - if self.split_op.origin_leaf_key.load(Relaxed).is_null() { - // If the origin is an unbounded node, assign the high key node to the high key node's - // unbounded. - entry_array[num_entries].replace(( - Some(middle_key_ref), - self.split_op.low_key_leaf.clone(Relaxed, guard), - )); - num_entries += 1; - if !self.split_op.high_key_leaf.is_null(Relaxed) { - entry_array[num_entries] - .replace((None, self.split_op.high_key_leaf.clone(Relaxed, guard))); - num_entries += 1; - } - } else { - // If the origin is a bounded node, assign the unbounded leaf as the high key node's - // unbounded. - entry_array[num_entries].replace((None, self.unbounded_child.clone(Relaxed, guard))); - num_entries += 1; - } - debug_assert!(num_entries >= 2); - - let low_key_leaf_array_size = recommended_boundary.min(num_entries - 1); - for (i, entry) in entry_array.iter().enumerate() { - if let Some((k, v)) = entry { - match (i + 1).cmp(&low_key_leaf_array_size) { - Less => { - low_key_leaf_node.children.insert_unchecked( - k.unwrap().clone(), - v.clone(Relaxed, guard), - i, - ); - } - Equal => { - middle_key.replace(k.unwrap()); - low_key_leaf_node - .unbounded_child - .swap((v.get_shared(Relaxed, guard), Tag::None), Relaxed); - } - Greater => { - if let Some(k) = k.cloned() { - high_key_leaf_node.children.insert_unchecked( - k, - v.clone(Relaxed, guard), - i - low_key_leaf_array_size, - ); - } else { - high_key_leaf_node - .unbounded_child - .swap((v.get_shared(Relaxed, guard), Tag::None), Relaxed); - } - } - } - } else { - break; - } - } - - middle_key.unwrap() - } - - /// Commits an ongoing structural change. - #[inline] - pub(super) fn commit(&self, guard: &Guard) { - // Unfreeze both leaves. - if let Some(origin_leaf) = self.split_op.origin_leaf.swap((None, Tag::None), Relaxed).0 { - // Make the origin leaf unreachable before making the new leaves updatable. - origin_leaf.delete_self(Release); - let _: bool = origin_leaf.release(); - } - let low_key_leaf = self - .split_op - .low_key_leaf - .load(Relaxed, guard) - .as_ref() - .unwrap(); - let high_key_leaf = self - .split_op - .high_key_leaf - .load(Relaxed, guard) - .as_ref() - .unwrap(); - let unfrozen = low_key_leaf.thaw(); - debug_assert!(unfrozen); - let unfrozen = high_key_leaf.thaw(); - debug_assert!(unfrozen); - - // It is safe to dereference those pointers since the whole split operation is under a - // single `ebr::Guard`. - if let Some(low_key_leaf_node) = - unsafe { self.split_op.low_key_leaf_node.load(Relaxed).as_ref() } - { - let origin = low_key_leaf_node.split_op.reset(); - low_key_leaf_node.lock.release_lock(); - origin.map(Shared::release); - } - - if let Some(high_key_leaf_node) = - unsafe { self.split_op.high_key_leaf_node.load(Relaxed).as_ref() } - { - let origin = high_key_leaf_node.split_op.reset(); - high_key_leaf_node.lock.release_lock(); - origin.map(Shared::release); - } - - let origin = self.split_op.reset(); - - // The leaf node can no longer be locked. - self.lock.poison_lock(); - origin.map(Shared::release); - } - - /// Rolls back the ongoing split operation. - #[inline] - pub(super) fn rollback(&self, guard: &Guard) { - let low_key_leaf = self - .split_op - .low_key_leaf - .load(Relaxed, guard) - .as_ref() - .unwrap(); - let high_key_leaf = self - .split_op - .high_key_leaf - .load(Relaxed, guard) - .as_ref() - .unwrap(); - - // Roll back the linked list state. - // - // `high_key_leaf` must be deleted first in order for `Scanners` not to omit entries. - let deleted = high_key_leaf.delete_self(Release); - debug_assert!(deleted); - let deleted = low_key_leaf.delete_self(Release); - debug_assert!(deleted); - - if let Some(origin_leaf) = self.split_op.origin_leaf.swap((None, Tag::None), Relaxed).0 { - // Unfreeze the origin. - let unfrozen = origin_leaf.thaw(); - debug_assert!(unfrozen); - - // Remove the mark from the full leaf node. - // - // `clear` clears the tag, so there is no guarantee that the tag has been kept. - origin_leaf.unmark(Release); - } - - let origin = self.split_op.reset(); - self.lock.release_lock(); - origin.map(Shared::release); - } - - /// Cleans up logically deleted leaf instances in the linked list. - /// - /// Returns `false` if the target leaf does not exist in the [`LeafNode`]. - #[inline] - pub(super) fn cleanup_link<'g, Q>(&self, key: &Q, traverse_max: bool, guard: &'g Guard) -> bool - where - K: 'g, - Q: Comparable + ?Sized, - { - let scanner = if traverse_max { - if let Some(unbounded) = self.unbounded_child.load(Acquire, guard).as_ref() { - Scanner::new(unbounded) - } else { - return false; - } - } else if let Some(leaf_scanner) = Scanner::max_less(&self.children, key) { - if let Some((_, child)) = leaf_scanner.get() { - if let Some(child) = child.load(Acquire, guard).as_ref() { - Scanner::new(child) - } else { - return false; - } - } else { - return false; - } - } else { - return false; - }; - - // It *would* be the maximum leaf node among those containing keys smaller than the - // target key. Hopefully, two jumps will be sufficient. - scanner.jump(None, guard).map(|s| s.jump(None, guard)); - true - } - /// Splits a full leaf. /// + /// Returns `false` if the parent node needs to be split. + /// /// # Errors /// - /// Returns an error if a retry is required. - #[allow(clippy::too_many_lines)] + /// Returns an error if locking failed or the full leaf node was changed. fn split_leaf( &self, - entry: (K, V), - full_leaf_key: Option<&K>, full_leaf_ptr: Ptr>, full_leaf: &AtomicShared>, async_wait: &mut W, guard: &Guard, - ) -> Result, (K, V)> { - if !self.lock.try_lock() { - async_wait.try_wait(&self.lock); - return Err(entry); - } else if self.retired() { - self.lock.release_lock(); - return Ok(InsertResult::Retired(entry.0, entry.1)); - } else if full_leaf_ptr != full_leaf.load(Relaxed, guard) { - self.lock.release_lock(); - return Err(entry); + ) -> Result { + if self.is_retired() { + // Let the parent node clean up this node. + return Ok(false); } - let prev = self - .split_op - .origin_leaf - .swap((full_leaf.get_shared(Relaxed, guard), Tag::None), Relaxed) - .0; - debug_assert!(prev.is_none()); + let Some(_locker) = Locker::try_lock(self) else { + async_wait.try_wait(&self.lock); + return Err(()); + }; - if let Some(full_leaf_key) = full_leaf_key { - self.split_op - .origin_leaf_key - .store(ptr::from_ref(full_leaf_key).cast_mut(), Relaxed); + if self.unbounded_child.tag(Relaxed) != Tag::None + || full_leaf_ptr != full_leaf.load(Relaxed, guard) + { + // The leaf node is being cleared, or the leaf node was already split. + return Err(()); } let target = full_leaf_ptr.as_ref().unwrap(); + let frozen = target.freeze(); + debug_assert!(frozen); - // Distribute entries to two leaves after make the target retired. let exit_guard = ExitGuard::new((), |()| { - target.thaw(); - self.split_op.reset(); - self.lock.release_lock(); + target.unfreeze(); }); - let mut low_key_leaf_shared = Shared::new(Leaf::new()); - let mut high_key_leaf_shared = None; - target.freeze_and_distribute(&mut low_key_leaf_shared, &mut high_key_leaf_shared); + let low_key_leaf = Shared::new(Leaf::new()); + let high_key_leaf = Leaf::new(); + let is_full = self.children.is_full(); - self.split_op - .low_key_leaf - .swap((Some(low_key_leaf_shared), Tag::None), Relaxed); - self.split_op - .high_key_leaf - .swap((high_key_leaf_shared.take(), Tag::None), Relaxed); + // Need to freeze new leaves before making them reachable. + let frozen_low = low_key_leaf.freeze(); + let frozen_high = high_key_leaf.freeze(); + debug_assert!(frozen_low && frozen_high); - // When a new leaf is added to the linked list, the leaf is marked to let `Scanners` - // acknowledge that the newly added leaf may contain keys that are smaller than those - // having been `scanned`. - let low_key_leaf_ptr = self.split_op.low_key_leaf.load(Relaxed, guard); - let high_key_leaf_ptr = self.split_op.high_key_leaf.load(Relaxed, guard); - let unused_leaf = if let Some(high_key_leaf) = high_key_leaf_ptr.as_ref() { - // From here, `Scanners` can reach the new leaves. - let result = target.push_back( - high_key_leaf_ptr.get_shared().unwrap(), - true, - Release, + let mut is_high_key_leaf_empty = true; + if !target.distribute(|k, v, i, b, l| { + if b < l && is_full { + // E.g., `b == 2, l == 2`, then `i` can be as large as `1`: `high_key_leaf` is not + // needed. + return false; + } + // `v` is moved, not cloned; those new leaves do not own them until unfrozen. + let v = unsafe { ptr::from_ref(v).read() }; + if i < b { + low_key_leaf.insert_unchecked(k.clone(), v, i); + } else { + high_key_leaf.insert_unchecked(k.clone(), v, i - b); + is_high_key_leaf_empty = false; + } + true + }) { + return Ok(false); + } + + if is_high_key_leaf_empty { + target.replace_link( + |prev, next, _| { + low_key_leaf.prev.store(target.prev.load(Acquire), Relaxed); + low_key_leaf.next.store(target.next.load(Acquire), Relaxed); + if let Some(prev) = prev { + prev.next.store(low_key_leaf.as_ptr().cast_mut(), Release); + } + if let Some(next) = next { + next.prev.store(low_key_leaf.as_ptr().cast_mut(), Release); + } + // From here, `Iter` can reach the new leaf. + }, + guard, + ); + + // Unfreeze the leaves; the leaf now takes ownership of the copied values. + let unfrozen_low = low_key_leaf.unfreeze(); + debug_assert!(unfrozen_low); + full_leaf.swap((Some(low_key_leaf), Tag::None), Release); + } else { + let low_key_max = low_key_leaf.max_key().unwrap().clone(); + let high_key_leaf = Shared::new(high_key_leaf); + low_key_leaf + .next + .store(high_key_leaf.as_ptr().cast_mut(), Relaxed); + high_key_leaf + .prev + .store(low_key_leaf.as_ptr().cast_mut(), Relaxed); + + target.replace_link( + |prev, next, _| { + low_key_leaf.prev.store(target.prev.load(Acquire), Relaxed); + high_key_leaf.next.store(target.next.load(Acquire), Relaxed); + if let Some(prev) = prev { + prev.next.store(low_key_leaf.as_ptr().cast_mut(), Release); + } + if let Some(next) = next { + next.prev.store(high_key_leaf.as_ptr().cast_mut(), Release); + } + // From here, `Iter` can reach the new leaf. + }, guard, ); - debug_assert!(result.is_ok()); - let result = - target.push_back(low_key_leaf_ptr.get_shared().unwrap(), true, Release, guard); - debug_assert!(result.is_ok()); // Take the max key value stored in the low key leaf as the leaf key. - let low_key_leaf = low_key_leaf_ptr.as_ref().unwrap(); + let result = self + .children + .insert(low_key_max, AtomicShared::from(low_key_leaf.clone())); + debug_assert!(matches!(result, InsertResult::Success)); - // Need to freeze the leaf before trying to make it reachable. - let frozen = low_key_leaf.freeze(); - debug_assert!(frozen); + // Unfreeze the leaves; those leaves now take ownership of the copied values. + let unfrozen_low = low_key_leaf.unfreeze(); + let unfrozen_high = high_key_leaf.unfreeze(); + debug_assert!(unfrozen_low && unfrozen_high); + full_leaf.swap((Some(high_key_leaf), Tag::None), Release); + } - match self.children.insert( - low_key_leaf.max_key().unwrap().clone(), - self.split_op.low_key_leaf.clone(Relaxed, guard), - ) { - InsertResult::Success => (), - InsertResult::Duplicate(..) - | InsertResult::Frozen(..) - | InsertResult::Retry(..) => unreachable!(), - InsertResult::Full(_, _) | InsertResult::Retired(_, _) => { - // Need to freeze the other leaf. - let frozen = high_key_leaf.freeze(); - debug_assert!(frozen); - exit_guard.forget(); - return Ok(InsertResult::Full(entry.0, entry.1)); - } - } - - // Mark the full leaf deleted before making the new one reachable and updatable. - // - // If the order is reversed, there emerges a possibility that entries were removed from - // the replaced leaf node whereas those entries still remain in `unused_leaf`; if that - // happens, iterators may see the removed entries momentarily. - let deleted = target.delete_self(Release); - debug_assert!(deleted); - - // Unfreeze the leaf. - let unfrozen = low_key_leaf.thaw(); - debug_assert!(unfrozen); - - // Replace the full leaf with the high-key leaf. - let high_key_leaf = self - .split_op - .high_key_leaf - .swap((None, Tag::None), Relaxed) - .0; - full_leaf.swap((high_key_leaf, Tag::None), Release).0 - } else { - // From here, `Scanner` can reach the new leaf. - // - // The full leaf is marked so that readers know that the next leaves may contain - // smaller keys. - let result = - target.push_back(low_key_leaf_ptr.get_shared().unwrap(), true, Release, guard); - debug_assert!(result.is_ok()); - - // Mark the full leaf deleted before making the new one reachable and updatable. - let deleted = target.delete_self(Release); - debug_assert!(deleted); - - full_leaf - .swap( - self.split_op.low_key_leaf.swap((None, Tag::None), Release), - Release, - ) - .0 - }; + // The removed leaf stays frozen: ownership of the copied values is transferred. exit_guard.forget(); - let origin = self.split_op.reset(); - self.lock.release_lock(); - origin.map(Shared::release); - unused_leaf.map(Shared::release); + // If there was a clear operation in the meantime, new leaves will need to be cleaned up. + if self.unbounded_child.tag(Acquire) != Tag::None { + self.clear(guard); + } - // Since a new leaf has been inserted, the caller can retry. - Ok(InsertResult::Retry(entry.0, entry.1)) + Ok(true) } - /// Tries to coalesce empty or obsolete leaves after a successful removal of an entry. - fn coalesce(&self, guard: &Guard) -> RemoveResult { - let mut uncleaned_leaf = false; - let mut prev_valid_leaf = None; - while let Some(lock) = Locker::try_lock(self) { - prev_valid_leaf.take(); - for entry in Scanner::new(&self.children) { - let leaf_ptr = entry.1.load(Acquire, guard); - let leaf = leaf_ptr.as_ref().unwrap(); - if leaf.is_retired() { - let deleted = leaf.delete_self(Release); - debug_assert!(deleted); - let result = self.children.remove_if(entry.0, &mut |_| true); - debug_assert_ne!(result, RemoveResult::Fail); - - // The pointer is nullified after the metadata of `self.children` is updated so - // that readers are able to retry when they find it being `null`. - if let Some(leaf) = entry.1.swap((None, Tag::None), Release).0 { - let _: bool = leaf.release(); - if let Some(prev_leaf) = prev_valid_leaf.as_ref() { - // One jump is sufficient. - Scanner::new(*prev_leaf).jump(None, guard); - } else { - uncleaned_leaf = true; - } - } - } else { - prev_valid_leaf.replace(leaf); - } - } - - // The unbounded leaf becomes unreachable after all the other leaves are gone. - let fully_empty = if prev_valid_leaf.is_some() { - false - } else { - let unbounded_ptr = self.unbounded_child.load(Acquire, guard); - if let Some(unbounded) = unbounded_ptr.as_ref() { - if unbounded.is_retired() { - let deleted = unbounded.delete_self(Release); - debug_assert!(deleted); - - // It has to mark the pointer in order to prevent `LeafNode::insert` from - // replacing it with a new `Leaf`. - if let Some(obsolete_leaf) = - self.unbounded_child.swap((None, RETIRED), Release).0 - { - let _: bool = obsolete_leaf.release(); - uncleaned_leaf = true; - } - true - } else { - false - } - } else { - debug_assert!(unbounded_ptr.tag() == RETIRED); - true - } - }; - - if fully_empty { + /// Tries to delete retired leaves after a successful removal of an entry. + fn post_remove(&self, guard: &Guard) -> RemoveResult { + let Some(lock) = Locker::try_lock(self) else { + if self.is_retired() { return RemoveResult::Retired; } + return RemoveResult::Success; + }; - drop(lock); - if !self.has_retired_leaf(guard) { - break; + let mut prev_valid_leaf = None; + let mut iter = Iter::new(&self.children); + while let Some(entry) = iter.next() { + let leaf_ptr = entry.1.load(Acquire, guard); + let leaf = leaf_ptr.as_ref().unwrap(); + if leaf.is_retired() { + leaf.unlink(guard); + + // As soon as the leaf is removed from the leaf node, the next leaf can store keys + // that are smaller than those that were previously stored in the removed leaf node. + // + // Therefore, when unlinking a leaf, the current snapshot of metadata of neighboring + // leaves is stored inside the leaf which will be used by iterators. + let result = iter.remove_unchecked(); + debug_assert_ne!(result, RemoveResult::Fail); + + // The pointer is set to null after the metadata of `self.children` is updated + // to enable readers to retry when they find it being null. + entry.1.swap((None, Tag::None), Release); + } else { + prev_valid_leaf.replace(leaf); } } - if uncleaned_leaf { - RemoveResult::Cleanup + // The unbounded leaf becomes unreachable after all the other leaves are gone. + let fully_empty = if prev_valid_leaf.is_some() { + false + } else { + let unbounded_ptr = self.unbounded_child.load(Acquire, guard); + if let Some(unbounded) = unbounded_ptr.as_ref() { + if unbounded.is_retired() { + unbounded.unlink(guard); + + // `Tag::First` prevents `insert` from allocating a new leaf. + self.unbounded_child.swap((None, Tag::First), Release); + true + } else { + false + } + } else { + true + } + }; + + if fully_empty { + lock.unlock_retire(); + RemoveResult::Retired } else { RemoveResult::Success } } - - /// Checks if the [`LeafNode`] has a retired [`Leaf`]. - fn has_retired_leaf(&self, guard: &Guard) -> bool { - let mut has_valid_leaf = false; - for entry in Scanner::new(&self.children) { - let leaf_ptr = entry.1.load(Relaxed, guard); - if let Some(leaf) = leaf_ptr.as_ref() { - if leaf.is_retired() { - return true; - } - has_valid_leaf = true; - } - } - if !has_valid_leaf { - let unbounded_ptr = self.unbounded_child.load(Relaxed, guard); - if let Some(unbounded) = unbounded_ptr.as_ref() { - if unbounded.is_retired() { - return true; - } - } - } - false - } } impl<'n, K, V> Locker<'n, K, V> { @@ -1023,6 +740,13 @@ impl<'n, K, V> Locker<'n, K, V> { None } } + + /// Retires the leaf node by poisoning the lock. + #[inline] + pub(super) fn unlock_retire(self) { + self.leaf_node.lock.poison_lock(); + forget(self); + } } impl Drop for Locker<'_, K, V> { @@ -1079,31 +803,6 @@ impl RemoveRangeState { } } -impl StructuralChange { - fn reset(&self) -> Option>> { - self.origin_leaf_key.store(ptr::null_mut(), Relaxed); - self.low_key_leaf.swap((None, Tag::None), Relaxed); - self.high_key_leaf.swap((None, Tag::None), Relaxed); - self.low_key_leaf_node.store(ptr::null_mut(), Relaxed); - self.high_key_leaf_node.store(ptr::null_mut(), Relaxed); - self.origin_leaf.swap((None, Tag::None), Relaxed).0 - } -} - -impl Default for StructuralChange { - #[inline] - fn default() -> Self { - Self { - origin_leaf_key: AtomicPtr::default(), - origin_leaf: AtomicShared::null(), - low_key_leaf: AtomicShared::null(), - high_key_leaf: AtomicShared::null(), - low_key_leaf_node: AtomicPtr::default(), - high_key_leaf_node: AtomicPtr::default(), - } - } -} - #[cfg(not(feature = "loom"))] #[cfg(test)] mod test { @@ -1164,7 +863,7 @@ mod test { )); assert!(matches!( leaf_node.insert("HI".to_owned(), "HO".to_owned(), &mut (), &guard), - Ok(InsertResult::Retired(..)) + Ok(InsertResult::Full(..)) )); } @@ -1181,11 +880,8 @@ mod test { InsertResult::Success => { assert_eq!(leaf_node.search_entry(&k, &guard), Some((&k, &k))); } - InsertResult::Duplicate(..) - | InsertResult::Frozen(..) - | InsertResult::Retired(..) => unreachable!(), + InsertResult::Duplicate(..) | InsertResult::Frozen(..) => unreachable!(), InsertResult::Full(_, _) => { - leaf_node.rollback(&guard); for r in 0..(k - 1) { assert_eq!(leaf_node.search_entry(&r, &guard), Some((&r, &r))); assert!( @@ -1206,9 +902,6 @@ mod test { assert_eq!(leaf_node.search_entry(&(k - 1), &guard), None); break; } - InsertResult::Retry(..) => { - assert!(leaf_node.insert(k, k, &mut (), &guard).is_ok()); - } } } } @@ -1247,12 +940,12 @@ mod test { break; } InsertResult::Full(..) => { - leaf_node_clone.rollback(&guard); max_key.replace(id); break; } - InsertResult::Frozen(..) | InsertResult::Retry(..) => (), - _ => unreachable!(), + InsertResult::Duplicate(..) | InsertResult::Frozen(..) => { + unreachable!() + } } } } @@ -1278,9 +971,7 @@ mod test { &guard, ) { match r { - RemoveResult::Success - | RemoveResult::Cleanup - | RemoveResult::Fail => break, + RemoveResult::Success | RemoveResult::Fail => break, RemoveResult::Frozen | RemoveResult::Retired => unreachable!(), } } @@ -1332,14 +1023,10 @@ mod test { { barrier_clone.wait().await; let guard = Guard::new(); - match leaf_node_clone.insert(k, k, &mut (), &guard) { - Ok(InsertResult::Success) => { - assert!(!inserted_clone.swap(true, Relaxed)); - } - Ok(InsertResult::Full(_, _) | InsertResult::Retired(_, _)) => { - leaf_node_clone.rollback(&guard); - } - _ => (), + if let Ok(InsertResult::Success) = + leaf_node_clone.insert(k, k, &mut (), &guard) + { + assert!(!inserted_clone.swap(true, Relaxed)); } } { @@ -1347,12 +1034,8 @@ mod test { let guard = Guard::new(); for i in 0..workload_size { if i != k { - if let Ok( - InsertResult::Full(_, _) | InsertResult::Retired(_, _), - ) = leaf_node_clone.insert(i, i, &mut (), &guard) - { - leaf_node_clone.rollback(&guard); - } + let result = leaf_node_clone.insert(i, i, &mut (), &guard); + drop(result); } assert_eq!( leaf_node_clone.search_entry(&k, &guard).unwrap(), @@ -1360,15 +1043,16 @@ mod test { ); } for i in 0..workload_size { - let max_scanner = leaf_node_clone.max_le_appr(&k, &guard).unwrap(); - assert!(*max_scanner.get().unwrap().0 <= k); - let mut min_scanner = leaf_node_clone.min(&guard).unwrap(); - if let Some((k_ref, v_ref)) = min_scanner.next() { + let max_iter = + leaf_node_clone.approximate::<_, true>(&k, &guard).unwrap(); + assert!(*max_iter.get().unwrap().0 <= k); + let mut min_iter = leaf_node_clone.min(&guard).unwrap(); + if let Some((k_ref, v_ref)) = min_iter.next() { assert_eq!(*k_ref, *v_ref); assert!(*k_ref <= k); } else { let (k_ref, v_ref) = - min_scanner.jump(None, &guard).unwrap().get().unwrap(); + min_iter.jump(&guard).unwrap().get().unwrap(); assert_eq!(*k_ref, *v_ref); assert!(*k_ref <= k); } diff --git a/patch/scc-3.4.8/src/tree_index/node.rs b/patch/scc-3.4.8/src/tree_index/node.rs index d746302..4f24e6c 100644 --- a/patch/scc-3.4.8/src/tree_index/node.rs +++ b/patch/scc-3.4.8/src/tree_index/node.rs @@ -4,12 +4,12 @@ use std::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed}; use sdd::{AtomicShared, Guard, Ptr, Shared, Tag}; -use super::internal_node::{self, InternalNode}; -use super::leaf::{InsertResult, Leaf, RemoveResult, Scanner}; -use super::leaf_node::{self, LeafNode}; +use super::internal_node::InternalNode; +use super::internal_node::Locker as InternalNodeLocker; +use super::leaf::{InsertResult, Iter, Leaf, RemoveResult, RevIter}; +use super::leaf_node::LeafNode; use crate::Comparable; use crate::async_helper::TryWait; -use crate::exit_guard::ExitGuard; /// [`Node`] is either [`Self::Internal`] or [`Self::Leaf`]. pub enum Node { @@ -52,10 +52,10 @@ impl Node { /// Checks if the node has retired. #[inline] - pub(super) fn retired(&self) -> bool { + pub(super) fn is_retired(&self) -> bool { match &self { - Self::Internal(internal_node) => internal_node.retired(), - Self::Leaf(leaf_node) => leaf_node.retired(), + Self::Internal(internal_node) => internal_node.is_retired(), + Self::Leaf(leaf_node) => leaf_node.is_retired(), } } } @@ -63,7 +63,7 @@ impl Node { impl Node where K: 'static + Clone + Ord, - V: 'static + Clone, + V: 'static, { /// Searches for an entry containing the specified key. #[inline] @@ -91,30 +91,41 @@ where } } - /// Returns the minimum key-value pair. - /// - /// This method is not linearizable. + /// Returns the minimum key entry in the entire tree. #[inline] - pub(super) fn min<'g>(&self, guard: &'g Guard) -> Option> { + pub(super) fn min<'g>(&self, guard: &'g Guard) -> Option> { match &self { Self::Internal(internal_node) => internal_node.min(guard), Self::Leaf(leaf_node) => leaf_node.min(guard), } } - /// Returns a [`Scanner`] pointing to an entry that is close enough to the entry with the - /// maximum key among those keys that are smaller than or equal to the given key. - /// - /// This method is not linearizable. + /// Returns the maximum key entry in the entire tree. #[inline] - pub(super) fn max_le_appr<'g, Q>(&self, key: &Q, guard: &'g Guard) -> Option> + pub(super) fn max<'g>(&self, guard: &'g Guard) -> Option> { + match &self { + Self::Internal(internal_node) => internal_node.max(guard), + Self::Leaf(leaf_node) => leaf_node.max(guard), + } + } + + /// Returns a [`Iter`] pointing to an entry that is close enough to the specified key. + /// + /// If `LE == true`, the returned [`Iter`] does not contain any keys larger than the specified + /// key. If not, the returned [`Iter`] does not contain any keys smaller than the specified key. + #[inline] + pub(super) fn approximate<'g, Q, const LE: bool>( + &self, + key: &Q, + guard: &'g Guard, + ) -> Option> where K: 'g, Q: Comparable + ?Sized, { match &self { - Self::Internal(internal_node) => internal_node.max_le_appr(key, guard), - Self::Leaf(leaf_node) => leaf_node.max_le_appr(key, guard), + Self::Internal(internal_node) => internal_node.approximate::<_, LE>(key, guard), + Self::Leaf(leaf_node) => leaf_node.approximate::<_, LE>(key, guard), } } @@ -197,76 +208,28 @@ where pub(super) fn split_root( root_ptr: Ptr>, root: &AtomicShared>, - mut key: K, - mut val: V, guard: &Guard, - ) -> (K, V) { - let mut exit_guard = ExitGuard::new(true, |rollback| { - if rollback { - if let Some(old_root) = root_ptr.as_ref() { - old_root.rollback(guard); - } - } - }); - - // The fact that the `TreeIndex` calls this function means the root is full and locked. - let mut new_root = Shared::new(Node::new_internal_node()); - if let (Some(Self::Internal(internal_node)), Some(old_root)) = - (unsafe { new_root.get_mut() }, root_ptr.get_shared()) - { - if root.load(Relaxed, guard) != root_ptr { - // The root has changed. - return (key, val); - } - - internal_node.unbounded_child = AtomicShared::from(old_root); - - // Now, the old root is connected to the new root, so the split operation will be rolled - // back in the `split_node` method if memory allocation fails. - *exit_guard = false; - - let result = internal_node.split_node( - (key, val), - None, - root_ptr, - &internal_node.unbounded_child, - true, - &mut (), - guard, - ); - let Ok(InsertResult::Retry(k, v)) = result else { - unreachable!() + ) { + if let Some(old_root) = root_ptr.get_shared() { + let new_root = if old_root.is_retired() { + Some(Shared::new(Node::new_leaf_node())) + } else { + let mut internal_node = Node::new_internal_node(); + let Node::Internal(node) = &mut internal_node else { + return; + }; + node.unbounded_child + .swap((Some(old_root), Tag::None), Relaxed); + Some(Shared::new(internal_node)) }; - key = k; - val = v; - // Updates the pointer before unlocking the root. - match root.compare_exchange( - root_ptr, - (Some(new_root), Tag::None), - AcqRel, - Acquire, - guard, - ) { - Ok((old_root, new_root_ptr)) => { - if let Some(Self::Internal(internal_node)) = new_root_ptr.as_ref() { - internal_node.finish_split(); - } - if let Some(old_root) = old_root { - old_root.commit(guard); - } - } - Err((new_root, _)) => { - // The root has been changed. - if let Some(Self::Internal(internal_node)) = new_root.as_deref() { - internal_node.finish_split(); - } - // The old root needs to rollback the split operation. - *exit_guard = true; - } + if let Err((Some(new_root), _)) = + root.compare_exchange(root_ptr, (new_root, Tag::None), AcqRel, Acquire, guard) + { + let dropped = unsafe { new_root.drop_in_place() }; + debug_assert!(dropped); } } - (key, val) } /// Cleans up or removes the current root node. @@ -283,109 +246,48 @@ where ) -> bool { let mut root_ptr = root.load(Acquire, guard); while let Some(root_ref) = root_ptr.as_ref() { - let mut internal_node_locker = None; - let mut leaf_node_locker = None; - match root_ref { - Self::Internal(internal_node) => { - if !internal_node.retired() && !internal_node.children.is_empty() { - // The internal node is still usable. - break; - } else if let Some(locker) = internal_node::Locker::try_lock(internal_node) { - internal_node_locker.replace(locker); - } else { - async_wait.try_wait(&internal_node.lock); - } - } - Self::Leaf(leaf_node) => { - if !leaf_node.retired() { - // The leaf node is still usable. - break; - } else if let Some(locker) = leaf_node::Locker::try_lock(leaf_node) { - leaf_node_locker.replace(locker); - } else { - async_wait.try_wait(&leaf_node.lock); - } + if root_ref.is_retired() { + if let Err((_, new_root_ptr)) = + root.compare_exchange(root_ptr, (None, Tag::None), AcqRel, Acquire, guard) + { + root_ptr = new_root_ptr; + continue; } + // The entire tree was truncated. + break; } - if internal_node_locker.is_none() && leaf_node_locker.is_none() { - // The root node is locked by another thread. - return false; - } - - let new_root = match root_ref { - Node::Internal(internal_node) => { - if internal_node.retired() { - // The internal node is empty, therefore the entire tree can be emptied. - None - } else if internal_node.children.is_empty() { - // Replace the root with the unbounded child. - internal_node.unbounded_child.get_shared(Acquire, guard) - } else { - // The internal node is not empty. - break; - } - } - Node::Leaf(leaf_node) => { - if leaf_node.retired() { - // The leaf node is empty, therefore the entire tree can be emptied. - None - } else { - // The leaf node is not empty. - break; - } - } + // Try to lower the tree. + let Self::Internal(internal_node) = root_ref else { + break; }; - - match root.compare_exchange(root_ptr, (new_root, Tag::None), AcqRel, Acquire, guard) { - Ok((_, new_root_ptr)) => { - root_ptr = new_root_ptr; - if let Some(internal_node_locker) = internal_node_locker { - internal_node_locker.unlock_retire(); + if let Some(locker) = InternalNodeLocker::try_lock(internal_node) { + let new_root = if internal_node.children.is_empty() { + // Replace the root with the unbounded child. + internal_node.unbounded_child.get_shared(Acquire, guard) + } else { + // The internal node is not empty. + break; + }; + match root.compare_exchange(root_ptr, (new_root, Tag::None), AcqRel, Acquire, guard) + { + Ok((_, new_root_ptr)) => { + locker.unlock_retire(); + root_ptr = new_root_ptr; + } + Err((_, new_root_ptr)) => { + // The root node has been changed. + root_ptr = new_root_ptr; } } - Err((_, new_root_ptr)) => { - // The root node has been changed. - root_ptr = new_root_ptr; - } + } else { + async_wait.try_wait(&internal_node.lock); + return false; } } true } - - /// Commits an ongoing structural change. - #[inline] - pub(super) fn commit(&self, guard: &Guard) { - match &self { - Self::Internal(internal_node) => internal_node.commit(guard), - Self::Leaf(leaf_node) => leaf_node.commit(guard), - } - } - - /// Rolls back an ongoing structural change. - #[inline] - pub(super) fn rollback(&self, guard: &Guard) { - match &self { - Self::Internal(internal_node) => internal_node.rollback(guard), - Self::Leaf(leaf_node) => leaf_node.rollback(guard), - } - } - - /// Cleans up logically deleted [`LeafNode`] instances in the linked list. - /// - /// Returns `false` if the target leaf node does not exist in the subtree. - #[inline] - pub(super) fn cleanup_link<'g, Q>(&self, key: &Q, traverse_max: bool, guard: &'g Guard) -> bool - where - K: 'g, - Q: Comparable + ?Sized, - { - match &self { - Self::Internal(internal_node) => internal_node.cleanup_link(key, traverse_max, guard), - Self::Leaf(leaf_node) => leaf_node.cleanup_link(key, traverse_max, guard), - } - } } impl Debug for Node { diff --git a/src/app/constant/header.rs b/src/app/constant/header.rs index a862c3c..b0b070c 100644 --- a/src/app/constant/header.rs +++ b/src/app/constant/header.rs @@ -1,10 +1,5 @@ use crate::common::model::HeaderValue; -mod version; -pub use version::{ - cursor_client_version, cursor_version, header_value_ua_cursor_latest, initialize_cursor_version, -}; - /// 定义 HeaderName 常量 /// /// # Example @@ -40,21 +35,6 @@ macro_rules! def_header_value { }; } -pub const UA: http::header::HeaderValue = http::header::HeaderValue::from_static( - cfg_select! { - target_os = "windows" => {"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36"} - target_os = "macos" => {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36"} - target_os = "linux" => {"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36"} - } -); - -pub const PLATFORM: http::header::HeaderValue = - http::header::HeaderValue::from_static(cfg_select! { - target_os = "windows" => {"\"Windows\""} - target_os = "macos" => {"\"macOS\""} - target_os = "linux" => {"\"Linux\""} - }); - def_header_value! { (NONE, ""), (ZERO, "0"), diff --git a/src/app/lazy.rs b/src/app/lazy.rs index 3d9a233..f23c2d8 100644 --- a/src/app/lazy.rs +++ b/src/app/lazy.rs @@ -8,7 +8,13 @@ use super::{ }, model::{DateTime, GcppHost}, }; -use crate::common::{model::HeaderValue, utils::parse_from_env}; +use crate::common::{ + model::{ + HeaderValue, + niche_types::{NonNegativeI8, NonNegativeI16}, + }, + utils::{ParseFromEnv, parse_from_env}, +}; use alloc::borrow::Cow; use manually_init::ManuallyInit; pub use path::{ @@ -26,6 +32,7 @@ pub static START_TIME: ManuallyInit = ManuallyInit::new(); pub fn init_start_time() { START_TIME.init(DateTime::now()) } #[inline] +#[rustfmt::skip] pub fn init() { PRI_REVERSE_PROXY_HOST.init(parse_from_env("PRI_REVERSE_PROXY_HOST", EMPTY_STRING)); PUB_REVERSE_PROXY_HOST.init(parse_from_env("PUB_REVERSE_PROXY_HOST", EMPTY_STRING)); @@ -39,7 +46,11 @@ pub fn init() { USE_PUB_REVERSE_PROXY = !PUB_REVERSE_PROXY_HOST.is_empty(); } TCP_KEEPALIVE - .init(parse_from_env("TCP_KEEPALIVE", DEFAULT_TCP_KEEPALIVE).min(MAX_TCP_KEEPALIVE)); + .init(parse_from_env("TCP_KEEPALIVE", ToDuration(Some(DEFAULT_TCP_KEEPALIVE)))); + TCP_KEEPALIVE_INTERVAL + .init(parse_from_env("TCP_KEEPALIVE_INTERVAL", ToDuration(Some(DEFAULT_TCP_KEEPALIVE_INTERVAL)))); + TCP_KEEPALIVE_RETRIES + .init(parse_from_env("TCP_KEEPALIVE_RETRIES", ToCount(Some(DEFAULT_TCP_KEEPALIVE_RETRIES)))); SERVICE_TIMEOUT .init(parse_from_env("SERVICE_TIMEOUT", DEFAULT_SERVICE_TIMEOUT).min(MAX_SERVICE_TIMEOUT)); REAL_USAGE.init(parse_from_env("REAL_USAGE", true)); @@ -254,13 +265,61 @@ def_cursor_api_url! { } // TCP 和超时相关常量 -const DEFAULT_TCP_KEEPALIVE: u64 = 90; -const MAX_TCP_KEEPALIVE: u64 = 600; -pub static TCP_KEEPALIVE: ManuallyInit = ManuallyInit::new(); +const DEFAULT_TCP_KEEPALIVE: NonNegativeI16 = NonNegativeI16::new(15).unwrap(); +const MAX_TCP_KEEPALIVE: NonNegativeI16 = NonNegativeI16::new(600).unwrap(); +pub static TCP_KEEPALIVE: ManuallyInit> = ManuallyInit::new(); -const DEFAULT_SERVICE_TIMEOUT: u64 = 30; -const MAX_SERVICE_TIMEOUT: u64 = 600; -pub static SERVICE_TIMEOUT: ManuallyInit = ManuallyInit::new(); +const DEFAULT_TCP_KEEPALIVE_INTERVAL: NonNegativeI16 = NonNegativeI16::new(15).unwrap(); +const MAX_TCP_KEEPALIVE_INTERVAL: NonNegativeI16 = NonNegativeI16::new(600).unwrap(); +pub static TCP_KEEPALIVE_INTERVAL: ManuallyInit> = + ManuallyInit::new(); + +const DEFAULT_TCP_KEEPALIVE_RETRIES: NonNegativeI8 = NonNegativeI8::new(3).unwrap(); +const MAX_TCP_KEEPALIVE_RETRIES: NonNegativeI8 = NonNegativeI8::new(20).unwrap(); +pub static TCP_KEEPALIVE_RETRIES: ManuallyInit> = + ManuallyInit::new(); + +const DEFAULT_SERVICE_TIMEOUT: u16 = 30; +const MAX_SERVICE_TIMEOUT: u16 = 600; +pub static SERVICE_TIMEOUT: ManuallyInit = ManuallyInit::new(); + +#[derive(Debug, Clone, Copy)] +pub struct ToDuration(Option); + +impl ParseFromEnv for ToDuration { + fn parse_from_env(key: &str) -> Option { + let v = i32::parse_from_env(key)?; + Some(if v < 0 { + Self(None) + } else { + Self(NonNegativeI16::new(v.min(MAX.as_inner() as i32) as i16)) + }) + } +} + +impl ToDuration { + pub fn to_duration(self) -> Option { + self.0.map(|v| core::time::Duration::from_secs(v.as_inner() as _)) + } +} + +#[derive(Debug, Clone, Copy)] +pub struct ToCount(Option); + +impl ParseFromEnv for ToCount { + fn parse_from_env(key: &str) -> Option { + let v = i16::parse_from_env(key)?; + Some(if v < 0 { + Self(None) + } else { + Self(NonNegativeI8::new(v.min(MAX.as_inner() as i16) as i8)) + }) + } +} + +impl ToCount { + pub fn to_count(self) -> Option { self.0.map(|v| v.as_inner() as _) } +} pub static REAL_USAGE: ManuallyInit = ManuallyInit::new(); diff --git a/src/app/model.rs b/src/app/model.rs index 39b86fd..756e3e1 100644 --- a/src/app/model.rs +++ b/src/app/model.rs @@ -4,6 +4,7 @@ mod checksum; mod config; mod context_fill_mode; mod cpp; +pub mod cursor_version; mod default_instructions; pub mod dynamic_key; mod exchange_map; @@ -11,6 +12,7 @@ mod fetch_model; mod hash; mod id_source; mod log; +pub mod platform; mod proxy; pub mod proxy_pool; mod state; diff --git a/src/app/model/config.rs b/src/app/model/config.rs index 676e28a..02aea1a 100644 --- a/src/app/model/config.rs +++ b/src/app/model/config.rs @@ -1,5 +1,8 @@ use super::{FetchMode, UsageCheck, VisionAbility}; -use crate::app::{lazy::CONFIG_FILE_PATH, model::Hash}; +use crate::app::{ + lazy::CONFIG_FILE_PATH, + model::{Hash, cursor_version::Version, platform::PlatformType}, +}; use arc_swap::ArcSwap; use byte_str::ByteStr; use manually_init::ManuallyInit; @@ -16,6 +19,8 @@ pub struct AppConfig { pub share_token: String, pub web_references_included: bool, pub raw_model_fetch_mode: FetchMode, + pub emulated_platform: PlatformType, + pub cursor_client_version: Version, } pub struct AppConfigManager { @@ -70,7 +75,7 @@ impl AppConfig { super::super::lazy::init_all_cursor_urls(); super::super::lazy::log::init(); super::hash::init(); - super::super::constant::header::initialize_cursor_version(); + // super::super::constant::header::initialize_cursor_version(); // super::super::constant::init_thinking_tags(); crate::core::model::init_resolver(); super::token::parse_providers(); @@ -78,16 +83,24 @@ impl AppConfig { } crate::core::constant::create_models(); - let (content, config) = if let Ok(s) = std::fs::read_to_string(&*CONFIG_FILE_PATH) - && let Ok(config) = toml::from_str(&s) - { - (s.into(), config) + let (content, config) = if let Ok(s) = std::fs::read_to_string(&*CONFIG_FILE_PATH) { + match toml::from_str(&s) { + Ok(config) => (s.into(), config), + Err(e) => { + eprintln!("Warning: configuration parse failed: {e}"); + (ByteStr::new(), AppConfig::default()) + } + } } else { (ByteStr::new(), AppConfig::default()) }; + { + use super::cursor_version::init; + init(config.emulated_platform, config.cursor_client_version) + } { use super::dynamic_key::{Secret, init}; - init(Secret::parse_str(&config.dynamic_key_secret).0.unwrap_or_else(|| [0; 64])); + init(Secret::parse_str(&config.dynamic_key_secret).0.unwrap_or([0; 64])); } APP_CONFIG.init(ArcSwap::from_pointee(AppConfigManager { hash: hash(&config), @@ -98,10 +111,19 @@ impl AppConfig { pub fn update(config: Self, content: ByteStr) -> bool { let hash = hash(&config); - if APP_CONFIG.load().hash != hash { - if APP_CONFIG.load().dynamic_key_secret != config.dynamic_key_secret { + let guard = APP_CONFIG.load(); + if guard.hash != hash { + { + use super::cursor_version::{update, update_platform_only}; + if guard.cursor_client_version != config.cursor_client_version { + update(config.emulated_platform, config.cursor_client_version) + } else if guard.emulated_platform != config.emulated_platform { + update_platform_only(config.emulated_platform) + } + } + if guard.dynamic_key_secret != config.dynamic_key_secret { use super::dynamic_key::{Secret, update}; - update(Secret::parse_str(&config.dynamic_key_secret).0.unwrap_or_else(|| [0; 64])); + update(Secret::parse_str(&config.dynamic_key_secret).0.unwrap_or([0; 64])); } APP_CONFIG.store(alloc::sync::Arc::new(AppConfigManager { hash, @@ -130,6 +152,7 @@ impl AppConfig { // share_token: Str; web_references_included: bool as is_web_references_included; raw_model_fetch_mode: FetchMode; + emulated_platform: PlatformType; ); #[inline] @@ -148,19 +171,23 @@ fn hash(config: &AppConfig) -> Hash { hasher.update(b"vision_ability"); hasher.update(config.vision_ability.as_str().as_bytes()); hasher.update(b"slow_pool_enabled"); - hasher.update(&[config.slow_pool_enabled as u8]); + hasher.update([config.slow_pool_enabled as u8]); hasher.update(b"long_context_enabled"); - hasher.update(&[config.long_context_enabled as u8]); + hasher.update([config.long_context_enabled as u8]); hasher.update(b"model_usage_checks"); config.model_usage_checks.hash(&mut hasher); hasher.update(b"dynamic_key_secret"); - hasher.update(config.dynamic_key_secret.as_str().as_bytes()); + hasher.update(config.dynamic_key_secret.as_bytes()); hasher.update(b"share_token"); - hasher.update(config.share_token.as_str().as_bytes()); + hasher.update(config.share_token.as_bytes()); hasher.update(b"web_references_included"); - hasher.update(&[config.web_references_included as u8]); + hasher.update([config.web_references_included as u8]); hasher.update(b"raw_model_fetch_mode"); hasher.update(config.raw_model_fetch_mode.as_str().as_bytes()); + hasher.update(b"emulated_platform"); + hasher.update(config.emulated_platform.as_str().as_bytes()); + hasher.update(b"cursor_client_version"); + hasher.update(config.cursor_client_version.to_bytes()); Hash(hasher.finalize().0) } diff --git a/src/app/model/cursor_version.rs b/src/app/model/cursor_version.rs new file mode 100644 index 0000000..02f2e92 --- /dev/null +++ b/src/app/model/cursor_version.rs @@ -0,0 +1,109 @@ +use alloc::sync::Arc; +use arc_swap::ArcSwap; +use core::{ + fmt::{self, Display, Formatter}, + num::ParseIntError, + str::FromStr, +}; +use manually_init::ManuallyInit; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Version { + pub major: u16, + pub minor: u16, + pub patch: u16, +} + +impl Version { + pub fn to_bytes(self) -> [u8; 6] { + unsafe { core::mem::transmute([self.major, self.minor, self.patch]) } + } +} + +impl Default for Version { + fn default() -> Self { Self { major: 2, minor: 0, patch: 0 } } +} + +impl<'de> serde::Deserialize<'de> for Version { + fn deserialize(deserializer: D) -> Result + where D: serde::Deserializer<'de> { + let s = String::deserialize(deserializer)?; + Self::from_str(&s).map_err(serde::de::Error::custom) + } +} + +impl FromStr for Version { + type Err = ParseIntError; + fn from_str(s: &str) -> Result { + let mut parts = [0; 3]; + for (s, i) in s.split('.').zip(parts.iter_mut()) { + *i = s.parse()?; + } + Ok(Version { major: parts[0], minor: parts[1], patch: parts[2] }) + } +} + +impl Display for Version { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let mut buffer = itoa::Buffer::new(); + f.write_str(buffer.format(self.major))?; + f.write_str(".")?; + f.write_str(buffer.format(self.minor))?; + f.write_str(".")?; + f.write_str(buffer.format(self.patch))?; + Ok(()) + } +} + +struct VersionValues { + /// 客户端版本的 HeaderValue + client_version: http::header::HeaderValue, + /// Cursor User-Agent 的 HeaderValue + ua_cursor: http::header::HeaderValue, +} + +static INSTANCE: ManuallyInit> = ManuallyInit::new(); + +pub fn get() -> bytes::Bytes { + use crate::common::model::HeaderValue; + let v: &HeaderValue = (&INSTANCE.load().client_version).into(); + v.inner.clone() +} + +pub fn client_version() -> http::header::HeaderValue { INSTANCE.load().client_version.clone() } + +pub fn ua_cursor() -> http::header::HeaderValue { INSTANCE.load().ua_cursor.clone() } + +pub(super) fn init(platform: super::platform::PlatformType, version: Version) { + use crate::common::model::HeaderValue; + let version = version.to_string(); + INSTANCE.init(ArcSwap::from_pointee(unsafe { + VersionValues { + ua_cursor: HeaderValue::from(platform.as_platform().client_ua(&version)).into(), + client_version: HeaderValue::from(version).into(), + } + })) +} + +pub(super) fn update(platform: super::platform::PlatformType, version: Version) { + use crate::common::model::HeaderValue; + let version = version.to_string(); + INSTANCE.store(Arc::new(unsafe { + VersionValues { + ua_cursor: HeaderValue::from(platform.as_platform().client_ua(&version)).into(), + client_version: HeaderValue::from(version).into(), + } + })) +} + +pub(super) fn update_platform_only(platform: super::platform::PlatformType) { + use crate::common::model::HeaderValue; + let guard = INSTANCE.load(); + let version = unsafe { core::str::from_utf8_unchecked(guard.client_version.as_bytes()) }; + INSTANCE.store(Arc::new(unsafe { + VersionValues { + ua_cursor: HeaderValue::from(platform.as_platform().client_ua(version)).into(), + client_version: guard.client_version.clone(), + } + })) +} diff --git a/src/app/model/dynamic_key.rs b/src/app/model/dynamic_key.rs index b384d5f..10a5b39 100644 --- a/src/app/model/dynamic_key.rs +++ b/src/app/model/dynamic_key.rs @@ -40,7 +40,7 @@ impl Secret { if ok && hex_pairs.len() < result.len() - && let Some(&hi) = hex_rests.get(0) + && let Some(&hi) = hex_rests.first() { if let Some(byte) = hex_to_byte(hi, b'0') { result[hex_pairs.len()] = byte; diff --git a/src/app/model/log/command.rs b/src/app/model/log/command.rs index 268f4d2..16a048d 100644 --- a/src/app/model/log/command.rs +++ b/src/app/model/log/command.rs @@ -41,7 +41,7 @@ pub enum LogCommand { }, // 添加一条日志 AddLog { - log: RequestLog, + log: Box, token: ExtToken, }, // 获取下一个日志ID diff --git a/src/app/model/log/manager.rs b/src/app/model/log/manager.rs index 7758f2c..42bd718 100644 --- a/src/app/model/log/manager.rs +++ b/src/app/model/log/manager.rs @@ -312,7 +312,7 @@ fn handle_command(mgr: &mut LogManager, cmd: LogCommand) -> bool { }; } } - mgr.logs.push_back(log); + mgr.logs.push_back(*log); match mgr.tokens.entry(key) { Entry::Occupied(e) => { let a = e.into_mut(); @@ -383,7 +383,7 @@ trait Expect: Sized { } impl Expect for Result<(), tokio::sync::mpsc::error::SendError> { type T = (); - fn expect(self) -> () { + fn expect(self) { core::result::Result::expect(self, "Log actor is dead - this should never happen") } } @@ -403,7 +403,7 @@ pub async fn get_logs(params: super::GetLogsParams) -> (u64, Vec) { } pub async fn add_log(log: RequestLog, token: ExtToken) { - expect(LOG_COMMAND_SENDER.send(LogCommand::AddLog { log, token }).await) + expect(LOG_COMMAND_SENDER.send(LogCommand::AddLog { log: Box::new(log), token }).await) } pub async fn get_next_log_id() -> u64 { diff --git a/src/app/model/platform.rs b/src/app/model/platform.rs new file mode 100644 index 0000000..31a1f31 --- /dev/null +++ b/src/app/model/platform.rs @@ -0,0 +1,128 @@ +use crate::common::{model::HeaderValue, platform::DEFAULT}; + +#[allow(non_camel_case_types)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PlatformType { + Windows, + macOS, + Linux, +} + +impl Default for PlatformType { + fn default() -> Self { DEFAULT } +} + +impl PlatformType { + const WINDOWS: &str = "Windows"; + const MAC_OS: &str = "macOS"; + const LINUX: &str = "Linux"; + pub const fn as_str(self) -> &'static str { + match self { + PlatformType::Windows => Self::WINDOWS, + PlatformType::macOS => Self::MAC_OS, + PlatformType::Linux => Self::LINUX, + } + } + pub fn from_str(s: &str) -> Option { + match s { + Self::WINDOWS => Some(PlatformType::Windows), + Self::MAC_OS => Some(PlatformType::macOS), + Self::LINUX => Some(PlatformType::Linux), + _ => None, + } + } +} + +impl serde::Serialize for PlatformType { + fn serialize(&self, serializer: S) -> Result + where S: serde::Serializer { + self.as_str().serialize(serializer) + } +} + +impl<'de> serde::Deserialize<'de> for PlatformType { + fn deserialize(deserializer: D) -> Result + where D: serde::Deserializer<'de> { + let s = String::deserialize(deserializer)?; + Self::from_str(&s) + .ok_or_else(|| serde::de::Error::custom(format_args!("{s:?} is unsupported"))) + } +} + +pub struct Platforms { + pub windows: Platform, + pub macos: Platform, + pub linux: Platform, +} + +pub struct Platform { + pub web_ua: &'static str, + pub string: &'static str, + pub ua_prefix: &'static str, + // pub default_ua: &'static str, +} + +impl PlatformType { + pub const fn as_platform(self) -> &'static Platform { + match self { + PlatformType::Windows => &PLATFORMS.windows, + PlatformType::macOS => &PLATFORMS.macos, + PlatformType::Linux => &PLATFORMS.linux, + } + } +} + +/// User-Agent 后缀 +pub const UA_SUFFIX: &'static str = " Chrome/138.0.7204.251 Electron/37.7.0 Safari/537.36"; + +// const UA_SUFFIX_LEN: usize = " Chrome/138.0.7204.251 Electron/37.7.0 Safari/537.36".len(); + +static PLATFORMS: Platforms = Platforms { + windows: Platform { + web_ua: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36", + string: "\"Windows\"", + ua_prefix: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Cursor/", + // default_ua: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Cursor/2.0.0 Chrome/138.0.7204.251 Electron/37.7.0 Safari/537.36", + }, + macos: Platform { + web_ua: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36", + string: "\"macOS\"", + ua_prefix: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Cursor/", + // default_ua: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Cursor/2.0.0 Chrome/138.0.7204.251 Electron/37.7.0 Safari/537.36", + }, + linux: Platform { + web_ua: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36", + string: "\"Linux\"", + ua_prefix: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Cursor/", + // default_ua: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Cursor/2.0.0 Chrome/138.0.7204.251 Electron/37.7.0 Safari/537.36", + }, +}; + +impl Platform { + pub const fn web_ua(&self) -> http::header::HeaderValue { + unsafe { HeaderValue::from_static(self.web_ua).into() } + } + // pub const fn as_str(&self) -> &'static str { + // let ptr = self.string.as_ptr(); + // let len = core::ptr::metadata(self.string); + // unsafe { &*core::ptr::from_raw_parts(ptr.add(1), len.unchecked_sub(2)) } + // } + pub const fn as_header_value(&self) -> http::header::HeaderValue { + unsafe { HeaderValue::from_static(self.string).into() } + } + /// User-Agent 前缀 + pub const fn ua_prefix(&self) -> &'static str { self.ua_prefix } + // const fn ua_suffix(&self) -> &'static str { + // use core::slice::SliceIndex as _; + // unsafe { + // &*(self.default_ua.len().unchecked_sub(UA_SUFFIX_LEN)..).get_unchecked(self.default_ua) + // } + // } + // /// 默认的 User-Agent + // pub const fn default_ua(&self) -> &'static str { self.default_ua } + pub fn client_ua(&self, version: &str) -> String { + [self.ua_prefix(), version, UA_SUFFIX].concat() + } +} + +pub fn current() -> &'static Platform { super::AppConfig::emulated_platform().as_platform() } diff --git a/src/app/model/proxy_pool.rs b/src/app/model/proxy_pool.rs index e48f8fc..387bbc8 100644 --- a/src/app/model/proxy_pool.rs +++ b/src/app/model/proxy_pool.rs @@ -1,4 +1,4 @@ -use crate::app::lazy::{PROXIES_FILE_PATH, SERVICE_TIMEOUT, TCP_KEEPALIVE}; +use crate::app::lazy::{PROXIES_FILE_PATH, SERVICE_TIMEOUT, TCP_KEEPALIVE, TCP_KEEPALIVE_INTERVAL, TCP_KEEPALIVE_RETRIES}; use alloc::sync::Arc; use arc_swap::{ArcSwap, ArcSwapAny}; use core::{str::FromStr, time::Duration}; @@ -236,8 +236,10 @@ impl SingleProxy { fn insert_to(&self, clients: &mut HashMap) { let builder = Client::builder() .https_only(true) - .tcp_keepalive(Duration::from_secs(*TCP_KEEPALIVE)) - .connect_timeout(Duration::from_secs(*SERVICE_TIMEOUT)) + .tcp_keepalive(TCP_KEEPALIVE.to_duration()) + .tcp_keepalive_interval(TCP_KEEPALIVE_INTERVAL.to_duration()) + .tcp_keepalive_retries(TCP_KEEPALIVE_RETRIES.to_count()) + .connect_timeout(Duration::from_secs(*SERVICE_TIMEOUT as _)) .webpki_roots_only(); let client = match self { SingleProxy::Non => builder.no_proxy().build().expect("创建无代理客户端失败"), diff --git a/src/app/model/state/token.rs b/src/app/model/state/token.rs index 1109d47..c38dec0 100644 --- a/src/app/model/state/token.rs +++ b/src/app/model/state/token.rs @@ -90,7 +90,7 @@ impl TokenManager { alias = Cow::Owned(generate_unnamed_alias(id)); } - if self.alias_map.contains_key(alias.as_ref()) { + if self.alias_map.contains_key(&*alias) { return Err(TokenError::AliasExists); } @@ -172,7 +172,7 @@ impl TokenManager { if alias == UNNAMED || alias.starts_with(UNNAMED_PATTERN) { alias = Cow::Owned(generate_unnamed_alias(id)); } - if self.alias_map.contains_key(alias.as_ref()) { + if self.alias_map.contains_key(&*alias) { return Err(TokenError::AliasExists); } diff --git a/src/app/model/state/token/queue.rs b/src/app/model/state/token/queue.rs index 8d7e46b..ab64b94 100644 --- a/src/app/model/state/token/queue.rs +++ b/src/app/model/state/token/queue.rs @@ -183,7 +183,7 @@ impl TokenQueue { pub fn remove( &mut self, token_key: &TokenKey, - tokens: &Vec>, + tokens: &[Option], ) -> Option { let vec_index = self.map.remove(token_key)?; diff --git a/src/app/model/token.rs b/src/app/model/token.rs index 2e5528e..ece9625 100644 --- a/src/app/model/token.rs +++ b/src/app/model/token.rs @@ -156,7 +156,7 @@ impl<'de> ::serde::Deserialize<'de> for Randomness { const _: [u8; 8] = [0; core::mem::size_of::()]; const _: () = assert!(core::mem::align_of::() == 8); -#[derive(Clone, Copy, PartialEq, Hash)] +#[derive(Clone, Copy, PartialEq, Eq, Hash)] pub struct Subject { pub provider: Provider, pub id: UserId, @@ -378,7 +378,7 @@ impl<'de> ::serde::Deserialize<'de> for UserId { const _: [u8; 16] = [0; core::mem::size_of::()]; const _: () = assert!(core::mem::align_of::() == 1); -#[derive(Clone, Copy, PartialEq, Hash, ::rkyv::Archive, ::rkyv::Deserialize, ::rkyv::Serialize)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::rkyv::Archive, ::rkyv::Deserialize, ::rkyv::Serialize)] pub struct Duration { pub start: i64, pub end: i64, @@ -428,7 +428,7 @@ impl fmt::Display for TokenError { } } -#[derive(Clone, Copy, Hash)] +#[derive(Clone, Copy)] pub struct RawToken { /// 用户标识符 pub subject: Subject, @@ -444,7 +444,30 @@ pub struct RawToken { impl PartialEq for RawToken { #[inline] - fn eq(&self, other: &Self) -> bool { self.signature == other.signature } + fn eq(&self, other: &Self) -> bool { + if self.signature != other.signature { + return false; + }; + core::intrinsics::likely( + self.subject == other.subject + && self.duration == other.duration + && self.randomness == other.randomness + && self.is_session == other.is_session, + ) + } +} + +impl Eq for RawToken {} + +impl ::core::hash::Hash for RawToken { + #[inline] + fn hash(&self, state: &mut H) { + self.signature.hash(state); + self.subject.hash(state); + self.duration.hash(state); + self.randomness.hash(state); + self.is_session.hash(state); + } } impl RawToken { diff --git a/src/app/model/token/provider.rs b/src/app/model/token/provider.rs index ffa4bca..6bae3c6 100644 --- a/src/app/model/token/provider.rs +++ b/src/app/model/token/provider.rs @@ -23,7 +23,7 @@ static mut PROVIDERS: &'static [&'static str] = DEFAULT_PROVIDERS; /// /// 这是一个对静态字符串标识符的包装, /// 该标识符会与支持的提供者列表进行验证 -#[derive(Clone, Copy, Hash)] +#[derive(Clone, Copy)] #[repr(transparent)] pub struct Provider(usize); @@ -32,6 +32,13 @@ impl PartialEq for Provider { fn eq(&self, other: &Self) -> bool { self.0 == other.0 } } +impl Eq for Provider {} + +impl ::core::hash::Hash for Provider { + #[inline] + fn hash(&self, state: &mut H) { self.0.hash(state) } +} + impl fmt::Display for Provider { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(self.as_str()) } @@ -40,7 +47,7 @@ impl fmt::Display for Provider { impl Provider { #[inline] #[allow(static_mut_refs)] - pub fn as_str(self) -> &'static str { unsafe { *PROVIDERS.get_unchecked(self.0) } } + pub fn as_str(self) -> &'static str { unsafe { PROVIDERS.get_unchecked(self.0) } } #[inline] pub(super) fn from_str(s: &str) -> Result { diff --git a/src/app/model/tz.rs b/src/app/model/tz.rs index 8839c35..2788ce2 100644 --- a/src/app/model/tz.rs +++ b/src/app/model/tz.rs @@ -1,6 +1,5 @@ -use manually_init::ManuallyInit; - use crate::common::utils::parse_from_env; +use manually_init::ManuallyInit; pub static TZ: ManuallyInit = ManuallyInit::new(); @@ -48,19 +47,13 @@ impl DateTime { /// 获取当前时刻的 UTC 时间。 #[inline(always)] - pub fn utc_now() -> chrono::DateTime { - now_naive().and_utc() - } + pub fn utc_now() -> chrono::DateTime { now_naive().and_utc() } #[inline(always)] - pub fn naive_now() -> chrono::NaiveDateTime { - now_naive() - } + pub fn naive_now() -> chrono::NaiveDateTime { now_naive() } #[inline(always)] - pub fn naive(&self) -> chrono::NaiveDateTime { - self.0.naive_utc() - } + pub fn naive(&self) -> chrono::NaiveDateTime { self.0.naive_utc() } #[inline(always)] pub fn from_naive(naive: &chrono::NaiveDateTime) -> Self { @@ -73,23 +66,17 @@ impl ::core::ops::Deref for DateTime { type Target = chrono::DateTime; #[inline(always)] - fn deref(&self) -> &Self::Target { - &self.0 - } + fn deref(&self) -> &Self::Target { &self.0 } } impl From for DateTime { #[inline] - fn from(naive: chrono::NaiveDateTime) -> Self { - Self::from_naive(&naive) - } + fn from(naive: chrono::NaiveDateTime) -> Self { Self::from_naive(&naive) } } impl From for chrono::NaiveDateTime { #[inline] - fn from(date_time: DateTime) -> Self { - date_time.naive() - } + fn from(date_time: DateTime) -> Self { date_time.naive() } } mod serde_impls { @@ -98,9 +85,7 @@ mod serde_impls { impl ::serde::Serialize for DateTime { #[inline] fn serialize(&self, serializer: S) -> Result - where - S: ::serde::Serializer, - { + where S: ::serde::Serializer { self.0.serialize(serializer) } } @@ -108,11 +93,8 @@ mod serde_impls { impl<'de> ::serde::Deserialize<'de> for DateTime { #[inline] fn deserialize(deserializer: D) -> Result - where - D: ::serde::Deserializer<'de>, - { - deserializer - .deserialize_str(chrono::serde::DateTimeVisitor) + where D: ::serde::Deserializer<'de> { + chrono::DateTime::::deserialize(deserializer) .map(|dt| Self(dt.with_timezone(&TZ))) } } @@ -120,9 +102,7 @@ mod serde_impls { impl ::core::cmp::PartialEq for DateTime { #[inline] - fn eq(&self, other: &DateTime) -> bool { - self.0 == other.0 - } + fn eq(&self, other: &DateTime) -> bool { self.0 == other.0 } } impl ::core::cmp::Eq for DateTime {} @@ -136,7 +116,5 @@ impl ::core::cmp::PartialOrd for DateTime { impl ::core::cmp::Ord for DateTime { #[inline] - fn cmp(&self, other: &DateTime) -> ::core::cmp::Ordering { - self.0.cmp(&other.0) - } + fn cmp(&self, other: &DateTime) -> ::core::cmp::Ordering { self.0.cmp(&other.0) } } diff --git a/src/common.rs b/src/common.rs index 2cb40bb..ea7714d 100644 --- a/src/common.rs +++ b/src/common.rs @@ -7,3 +7,7 @@ pub mod utils; pub mod build { include!(concat!(env!("OUT_DIR"), "/build_info.rs")); } + +pub mod platform { + include!(concat!(env!("OUT_DIR"), "/platform_info.rs")); +} diff --git a/src/common/client.rs b/src/common/client.rs index 8009217..92ede19 100644 --- a/src/common/client.rs +++ b/src/common/client.rs @@ -9,10 +9,9 @@ use crate::app::{ CURSOR_REFERER_URL, CURSOR_STREAMING, CURSOR_TIMEZONE, EMPTY, ENCODING, ENCODINGS, FALSE, FS_CLIENT_KEY, GHOST_MODE, HEADER_VALUE_ACCEPT, HOST, JSON, KEEP_ALIVE, LANGUAGE, MOBILE_NO, NEW_ONBOARDING_COMPLETED, NO_CACHE, NONE, NOT_A_BRAND, ONE, - PLATFORM, PRIORITY, PROTO, PROXY_HOST, REQUEST_ID, SAME_ORIGIN, SEC_CH_UA, - SEC_CH_UA_MOBILE, SEC_CH_UA_PLATFORM, SEC_FETCH_DEST, SEC_FETCH_MODE, SEC_FETCH_SITE, - SEC_GPC, SESSION_ID, TRAILERS, TRUE, U_EQ_0, U_EQ_1_I, UA, VSCODE_ORIGIN, ZERO, - cursor_client_version, header_value_ua_cursor_latest, + PRIORITY, PROTO, PROXY_HOST, REQUEST_ID, SAME_ORIGIN, SEC_CH_UA, SEC_CH_UA_MOBILE, + SEC_CH_UA_PLATFORM, SEC_FETCH_DEST, SEC_FETCH_MODE, SEC_FETCH_SITE, SEC_GPC, + SESSION_ID, TRAILERS, TRUE, U_EQ_0, U_EQ_1_I, VSCODE_ORIGIN, ZERO, }, }, lazy::{ @@ -20,7 +19,7 @@ use crate::app::{ pub_reverse_proxy_host, sessions_url, stripe_url, token_refresh_url, token_upgrade_url, usage_api_url, }, - model::ExtToken, + model::{ExtToken, cursor_version, platform}, }; use http::{ header::{ @@ -132,11 +131,7 @@ fn get_client( use_pri: bool, real_host: &'static str, ) -> RequestBuilder { - if use_pri && unsafe { USE_PRI_REVERSE_PROXY } { - client - .request(method, url) - .header(PROXY_HOST, unsafe { super::model::HeaderValue::from_static(real_host).into() }) - } else if !use_pri && unsafe { USE_PUB_REVERSE_PROXY } { + if (use_pri && unsafe { USE_PRI_REVERSE_PROXY }) || (!use_pri && unsafe { USE_PUB_REVERSE_PROXY }) { client .request(method, url) .header(PROXY_HOST, unsafe { super::model::HeaderValue::from_static(real_host).into() }) @@ -223,7 +218,7 @@ pub fn build_client_request(req: AiServiceRequest) -> RequestBuilder { req.ext_token.checksum.to_str(&mut buf); HeaderValue::from_bytes(&buf).into() }) - .header(CURSOR_CLIENT_VERSION, cursor_client_version()) + .header(CURSOR_CLIENT_VERSION, cursor_version::client_version()) .opt_header_map(CURSOR_CONFIG_VERSION, req.ext_token.config_version, |v| { v.hyphenated().encode_lower(unsafe { &mut *(buf.as_mut_ptr() as *mut [u8; 36]) }); unsafe { HeaderValue::from_bytes(buf.get_unchecked(..36)).into() } @@ -260,15 +255,15 @@ pub fn build_stripe_request( builder .version(http::Version::HTTP_2) - .header(SEC_CH_UA_PLATFORM, PLATFORM) + .header(SEC_CH_UA_PLATFORM, platform::current().as_header_value()) .header(AUTHORIZATION, bearer_token) - .header(CURSOR_CLIENT_VERSION, cursor_client_version()) + .header(CURSOR_CLIENT_VERSION, cursor_version::client_version()) .header(NEW_ONBOARDING_COMPLETED, FALSE) .header(SEC_CH_UA, NOT_A_BRAND) .header(SEC_CH_UA_MOBILE, MOBILE_NO) // skip traceparent .header(GHOST_MODE, TRUE) - .header(USER_AGENT, header_value_ua_cursor_latest()) + .header(USER_AGENT, cursor_version::ua_cursor()) .header(ACCEPT, HEADER_VALUE_ACCEPT) .header(ORIGIN, VSCODE_ORIGIN) .header(SEC_FETCH_SITE, CROSS_SITE) @@ -295,7 +290,7 @@ pub fn build_usage_request( builder .version(http::Version::HTTP_2) // .header(HOST, host) - .header(USER_AGENT, UA) + .header(USER_AGENT, platform::current().web_ua()) .header(ACCEPT, HEADER_VALUE_ACCEPT) .header(ACCEPT_LANGUAGE, LANGUAGE) .header(ACCEPT_ENCODING, ENCODINGS) @@ -393,7 +388,7 @@ pub fn build_token_upgrade_request( builder .version(http::Version::HTTP_2) // .header(HOST, host) - .header(USER_AGENT, UA) + .header(USER_AGENT, platform::current().web_ua()) .header(ACCEPT, HEADER_VALUE_ACCEPT) .header(ACCEPT_LANGUAGE, LANGUAGE) .header(ACCEPT_ENCODING, ENCODINGS) @@ -422,7 +417,7 @@ pub fn build_token_poll_request(client: &Client, url: Url, use_pri: bool) -> Req .header(ACCEPT_ENCODING, ENCODINGS) .header(ACCEPT_LANGUAGE, LANGUAGE) .header(CONTENT_LENGTH, ZERO) - .header(USER_AGENT, header_value_ua_cursor_latest()) + .header(USER_AGENT, cursor_version::ua_cursor()) .header(ORIGIN, VSCODE_ORIGIN) .header(GHOST_MODE, TRUE) .header(ACCEPT, HEADER_VALUE_ACCEPT) @@ -450,7 +445,7 @@ pub fn build_token_refresh_request( .header(ACCEPT_LANGUAGE, LANGUAGE) .header(CONTENT_TYPE, JSON) .header(CONTENT_LENGTH, HeaderValue::from_integer(body.len())) - .header(USER_AGENT, header_value_ua_cursor_latest()) + .header(USER_AGENT, cursor_version::ua_cursor()) .header(ORIGIN, VSCODE_ORIGIN) .header(GHOST_MODE, TRUE) .header(ACCEPT, HEADER_VALUE_ACCEPT) @@ -469,7 +464,7 @@ pub fn build_proto_web_request( builder .version(http::Version::HTTP_2) // .header(HOST, host) - .header(USER_AGENT, UA) + .header(USER_AGENT, platform::current().web_ua()) .header(ACCEPT, HEADER_VALUE_ACCEPT) .header(ACCEPT_LANGUAGE, LANGUAGE) .header(ACCEPT_ENCODING, ENCODINGS) @@ -502,7 +497,7 @@ pub fn build_sessions_request( builder .version(http::Version::HTTP_2) // .header(HOST, host) - .header(USER_AGENT, UA) + .header(USER_AGENT, platform::current().web_ua()) .header(ACCEPT, HEADER_VALUE_ACCEPT) .header(ACCEPT_LANGUAGE, LANGUAGE) .header(ACCEPT_ENCODING, ENCODINGS) diff --git a/src/common/model.rs b/src/common/model.rs index e4692e4..5b2e155 100644 --- a/src/common/model.rs +++ b/src/common/model.rs @@ -1,5 +1,7 @@ +pub mod cached; pub mod error; pub mod health; +pub mod niche_types; pub mod ntp; pub mod raw_json; pub mod token; diff --git a/src/common/model/cached.rs b/src/common/model/cached.rs new file mode 100644 index 0000000..c9efa51 --- /dev/null +++ b/src/common/model/cached.rs @@ -0,0 +1,66 @@ +use super::raw_json::{RawJson, to_raw_json}; + +pub struct Cached { + value: T, + cache: U, +} + +impl Cached { + #[inline] + fn new(value: T, f: F) -> Result, E> + where F: FnOnce(&T) -> Result { + Ok(Cached { cache: f(&value)?, value }) + } +} + +impl core::ops::Deref for Cached { + type Target = T; + #[inline(always)] + fn deref(&self) -> &Self::Target { &self.value } +} + +impl core::ops::DerefMut for Cached { + #[inline(always)] + fn deref_mut(&mut self) -> &mut Self::Target { &mut self.value } +} + +pub struct JsonCached { + inner: Cached, +} + +impl JsonCached { + #[inline] + pub fn new(value: T) -> serde_json::Result { + Ok(JsonCached { inner: Cached::new(value, to_raw_json)? }) + } + #[inline] + pub fn cache(&self) -> RawJson { self.inner.cache.clone() } +} + +impl core::ops::Deref for JsonCached { + type Target = Cached; + #[inline(always)] + fn deref(&self) -> &Self::Target { &self.inner } +} + +impl core::ops::DerefMut for JsonCached { + #[inline(always)] + fn deref_mut(&mut self) -> &mut Self::Target { &mut self.inner } +} + +// pub struct StringCached { +// inner: Cached>, +// } + +// impl StringCached { +// #[inline] +// pub fn new(value: T) -> Self { +// StringCached { +// inner: match Cached::new(value, |v| Ok::<_, !>(v.to_string().into_boxed_str())) { +// Ok(val) => val, +// }, +// } +// } +// #[inline] +// pub fn cache(&self) -> &str { &*self.inner.cache } +// } diff --git a/src/common/model/niche_types.rs b/src/common/model/niche_types.rs new file mode 100644 index 0000000..ae05f4f --- /dev/null +++ b/src/common/model/niche_types.rs @@ -0,0 +1,8 @@ +use ::core::marker::ConstParamTy; + +define_valid_range_type! { + #[derive(ConstParamTy)] + pub struct NonNegativeI8(i8 as u8 in 0..=0x7f); + #[derive(ConstParamTy)] + pub struct NonNegativeI16(i16 as u16 in 0..=0x7f_ff); +} diff --git a/src/common/utils.rs b/src/common/utils.rs index db25d44..05de524 100644 --- a/src/common/utils.rs +++ b/src/common/utils.rs @@ -30,7 +30,6 @@ pub mod ulid; use super::model::userinfo::{Session, StripeProfile, UsageProfile, UserProfile}; use crate::{ app::{ - constant::EMPTY_STRING, lazy::{ chat_models_url, filtered_usage_events_url, get_privacy_mode_url, is_on_new_pricing_url, server_config_url, token_poll_url, user_api_url, @@ -61,47 +60,46 @@ use reqwest::Client; use std::time::{SystemTime, UNIX_EPOCH}; pub trait ParseFromEnv: Sized + 'static { - type Result: Sized + 'static = Self; - fn parse_from_env(key: &str, default: Self) -> Self::Result; + type Result: Sized + From + 'static = Self; + fn parse_from_env(key: &str) -> Option; + #[inline] + fn parse_from_env_or(key: &str, default: Self) -> Self::Result { + Self::parse_from_env(key).unwrap_or(default.into()) + } } impl ParseFromEnv for bool { #[inline] - fn parse_from_env(key: &str, default: bool) -> bool { - ::std::env::var(key) - .ok() - .map(|mut val| { - let trimmed = val.trim(); - let start = trimmed.as_ptr() as usize - val.as_ptr() as usize; - let len = trimmed.len(); + fn parse_from_env(key: &str) -> Option { + ::std::env::var(key).ok().and_then(|mut val| { + let trimmed = val.trim(); + let start = trimmed.as_ptr() as usize - val.as_ptr() as usize; + let len = trimmed.len(); - // 只对 trim 后的部分转小写 - unsafe { - val.as_bytes_mut().get_unchecked_mut(start..start + len).make_ascii_lowercase(); - } + // 只对 trim 后的部分转小写 + unsafe { + val.as_bytes_mut().get_unchecked_mut(start..start + len).make_ascii_lowercase(); + } - // SAFETY: trimmed 是从有效 UTF-8 字符串 trim 得到的, - // make_ascii_lowercase 保持 UTF-8 有效性 - let result = unsafe { - ::core::str::from_utf8_unchecked( - val.as_bytes().get_unchecked(start..start + len), - ) - }; + // SAFETY: trimmed 是从有效 UTF-8 字符串 trim 得到的, + // make_ascii_lowercase 保持 UTF-8 有效性 + let result = unsafe { + ::core::str::from_utf8_unchecked(val.as_bytes().get_unchecked(start..start + len)) + }; - match result { - "true" | "1" => true, - "false" | "0" => false, - _ => default, - } - }) - .unwrap_or(default) + match result { + "true" | "1" => Some(true), + "false" | "0" => Some(false), + _ => None, + } + }) } } impl ParseFromEnv for &'static str { type Result = Cow<'static, str>; #[inline] - fn parse_from_env(key: &str, default: &'static str) -> Cow<'static, str> { + fn parse_from_env(key: &str) -> Option> { match ::std::env::var(key) { Ok(mut value) => { let trimmed = value.trim(); @@ -109,10 +107,10 @@ impl ParseFromEnv for &'static str { if trimmed_len == 0 { // 如果 trim 后为空,使用默认值(不分配) - Cow::Borrowed(default) + None } else if trimmed_len == value.len() { // 不需要 trim,直接使用 - Cow::Owned(value) + Some(Cow::Owned(value)) } else { // 需要 trim - 就地修改 let start_offset = trimmed.as_ptr() as usize - value.as_ptr() as usize; @@ -135,10 +133,10 @@ impl ParseFromEnv for &'static str { vec.set_len(trimmed_len); } - Cow::Owned(value) + Some(Cow::Owned(value)) } } - Err(_) => Cow::Borrowed(default), + Err(_) => None, } } } @@ -148,8 +146,8 @@ macro_rules! impl_parse_num_from_env { $( impl ParseFromEnv for $ty { #[inline] - fn parse_from_env(key: &str, default: $ty) -> $ty { - ::std::env::var(key).ok().and_then(|v| v.trim().parse().ok()).unwrap_or(default) + fn parse_from_env(key: &str) -> Option<$ty> { + ::std::env::var(key).ok().and_then(|v| v.trim().parse().ok()) } } )* @@ -159,9 +157,9 @@ macro_rules! impl_parse_num_from_env { impl_parse_num_from_env!(i8 u8 i16 u16 i32 u32 i64 u64 i128 u128 isize usize); impl ParseFromEnv for duration_fmt::DurationFormat { - fn parse_from_env(key: &str, default: Self) -> Self::Result { - let s = <&'static str as ParseFromEnv>::parse_from_env(key, EMPTY_STRING); - match &*s { + fn parse_from_env(key: &str) -> Option { + let s = <&'static str as ParseFromEnv>::parse_from_env(key)?; + Some(match &*s { "auto" => Self::Auto, "compact" => Self::Compact, "standard" => Self::Standard, @@ -171,28 +169,28 @@ impl ParseFromEnv for duration_fmt::DurationFormat { "numeric" => Self::Numeric, "verbose" => Self::Verbose, "random" => Self::Random, - _ => default, - } + _ => return None, + }) } } impl ParseFromEnv for duration_fmt::Language { - fn parse_from_env(key: &str, default: Self) -> Self::Result { - let s = <&'static str as ParseFromEnv>::parse_from_env(key, EMPTY_STRING); - match &*s { + fn parse_from_env(key: &str) -> Option { + let s = <&'static str as ParseFromEnv>::parse_from_env(key)?; + Some(match &*s { "english" => Self::English, "chinese" => Self::Chinese, "japanese" => Self::Japanese, "spanish" => Self::Spanish, "german" => Self::German, "random" => Self::Random, - _ => default, - } + _ => return None, + }) } } #[inline] pub fn parse_from_env(key: &str, default: T) -> T::Result { - T::parse_from_env(key, default) + T::parse_from_env_or(key, default) } pub fn now() -> Duration { now_with_epoch(UNIX_EPOCH, "system time before Unix epoch") } diff --git a/src/common/utils/duration_fmt.rs b/src/common/utils/duration_fmt.rs index 57aa8c1..4b0395b 100644 --- a/src/common/utils/duration_fmt.rs +++ b/src/common/utils/duration_fmt.rs @@ -323,7 +323,6 @@ impl HumanDuration { /// let duration = Duration::from_secs(65); /// let human_duration = HumanDuration::new(duration); /// ``` - #[must_use] #[inline] pub const fn new(duration: Duration) -> Self { Self { duration, format: DurationFormat::Auto, language: Language::Chinese } @@ -949,7 +948,6 @@ impl fmt::Display for HumanDuration { /// .format(DurationFormat::Compact) /// .language(Language::English)); /// ``` -#[must_use] #[inline(always)] pub const fn human(duration: Duration) -> HumanDuration { HumanDuration::new(duration) } diff --git a/src/core/adapter/anthropic.rs b/src/core/adapter/anthropic.rs index b989124..71a2cc8 100644 --- a/src/core/adapter/anthropic.rs +++ b/src/core/adapter/anthropic.rs @@ -47,13 +47,10 @@ impl ToolParam for Tool { impl ToolResult for (Option, bool) { fn is_error(&self) -> bool { self.1 } fn size_hint(&self) -> Option { - match self.0 { - Some(ref c) => Some(match c { + self.0.as_ref().map(|c| match c { ToolResultContent::String(..) => 1, ToolResultContent::Array(cs) => cs.len(), - }), - None => None, - } + }) } async fn add_to(self, builder: &mut ToolResultBuilder) -> Result<(), AdapterError> { if let Some(c) = self.0 { diff --git a/src/core/adapter/traits.rs b/src/core/adapter/traits.rs index dbc665f..d272cc6 100644 --- a/src/core/adapter/traits.rs +++ b/src/core/adapter/traits.rs @@ -98,7 +98,7 @@ pub(super) trait Adapter: Sized + 'static { match res { Ok((image_data, dimension)) => { images.push(ImageProto { - data: image_data.into(), + data: image_data, dimension, uuid: base_uuid.add_and_to_string(), // task_specific_description: None, diff --git a/src/core/auth/utils.rs b/src/core/auth/utils.rs index be7a548..ad4bc3e 100644 --- a/src/core/auth/utils.rs +++ b/src/core/auth/utils.rs @@ -62,7 +62,7 @@ pub(super) fn get_environment_info( }), exthost_arch: get(headers, STAINLESS_ARCH), local_timestamp: request_time.to_utc().to_rfc3339_opts(chrono::SecondsFormat::Millis, true), - cursor_version: crate::app::constant::header::cursor_version(), + cursor_version: crate::app::model::cursor_version::get(), } } diff --git a/src/core/constant.rs b/src/core/constant.rs index c7a2701..c727b33 100644 --- a/src/core/constant.rs +++ b/src/core/constant.rs @@ -6,7 +6,10 @@ use crate::{ constant::UNKNOWN, model::{AppConfig, FetchMode, ModelIdSource}, }, - common::model::raw_json::{RawJson, to_raw_json}, + common::model::{ + cached::JsonCached, + raw_json::RawJson, + }, core::model::ModelsResponse, }; use alloc::sync::Arc; @@ -231,9 +234,9 @@ macro_rules! create_models { let find_ids = HashMap::from_iter(models.iter().enumerate().map(|(i, m)| (m.id(), i))); INSTANCE.init(ArcSwap::from_pointee(Models { - models: __unwrap!(Cached::new(ModelsResponse(models))), + models: __unwrap!(JsonCached::new(ModelsResponse(models))), raw_models: None, - ids: __unwrap!(Cached::new(ids)), + ids: __unwrap!(JsonCached::new(ids)), find_ids, last_update: Instant::now(), })) @@ -241,35 +244,10 @@ macro_rules! create_models { }; } -pub struct Cached { - value: T, - json: RawJson, -} - -impl Cached { - #[inline] - pub fn new(value: T) -> serde_json::Result { - Ok(Self { json: to_raw_json(&value)?, value }) - } - #[inline] - pub fn cache(&self) -> RawJson { self.json.clone() } -} - -impl core::ops::Deref for Cached { - type Target = T; - #[inline(always)] - fn deref(&self) -> &Self::Target { &self.value } -} - -impl core::ops::DerefMut for Cached { - #[inline(always)] - fn deref_mut(&mut self) -> &mut Self::Target { &mut self.value } -} - pub struct Models { - models: Cached, - raw_models: Option>, - ids: Cached>, + models: JsonCached, + raw_models: Option>, + ids: JsonCached>, find_ids: HashMap<&'static str, usize>, last_update: Instant, @@ -284,7 +262,7 @@ impl Models { #[inline] pub fn get_raw_models_cache() -> Option { - Self::get().raw_models.as_ref().map(Cached::cache) + Self::get().raw_models.as_ref().map(JsonCached::cache) } #[inline] @@ -357,6 +335,7 @@ impl Models { return None; } + #[allow(clippy::get_first)] match *bytes.get(0)? { b'g' => match *bytes.get(1)? { b'p' => Some(OPENAI), // g + p → "gp" (gpt) @@ -395,7 +374,8 @@ impl Models { .unwrap_or(UNKNOWN) }; let is_thinking = model.supports_thinking.unwrap_or_default(); - let is_image = if server_id == DEFAULT { true } else { model.supports_images.unwrap_or_default() }; + let is_image = + if server_id == DEFAULT { true } else { model.supports_images.unwrap_or_default() }; let is_max = model.supports_max_mode.unwrap_or_default(); let is_non_max = model.supports_non_max_mode.unwrap_or_default(); @@ -514,9 +494,9 @@ impl Models { let find_ids = HashMap::from_iter(new_models.iter().enumerate().map(|(i, m)| (m.id(), i))); INSTANCE.store(Arc::new(Models { - models: __unwrap!(Cached::new(ModelsResponse(new_models))), - raw_models: Some(__unwrap!(Cached::new(available_models))), - ids: __unwrap!(Cached::new(ids)), + models: __unwrap!(JsonCached::new(ModelsResponse(new_models))), + raw_models: Some(__unwrap!(JsonCached::new(available_models))), + ids: __unwrap!(JsonCached::new(ids)), find_ids, last_update: Instant::now(), })); diff --git a/src/core/constant/leak_ids.rs b/src/core/constant/leak_ids.rs index e9cf6ef..9ed8e6b 100644 --- a/src/core/constant/leak_ids.rs +++ b/src/core/constant/leak_ids.rs @@ -69,7 +69,7 @@ fn __intern(pool: &HashSet<&'static str>, s: &str) -> (Id, bool) { }; let id = match pool.raw_entry().from_key_sync(key) { - RawEntry::Occupied(entry) => Id::from_ref(*entry.key()), + RawEntry::Occupied(entry) => Id::from_ref(entry.key()), RawEntry::Vacant(entry) => { let leaked = unsafe { alloc_ids(s) }; entry.insert(leaked.non_suffix(), ()); diff --git a/src/core/error/cursor.rs b/src/core/error/cursor.rs index 1d746f1..4198046 100644 --- a/src/core/error/cursor.rs +++ b/src/core/error/cursor.rs @@ -63,7 +63,7 @@ impl CursorError { pub(super) fn code(&self) -> &str { &self.error.code } pub fn error(&self) -> Option { - self.error.details.get(0)?.value.error.get().try_into().ok() + self.error.details.first()?.value.error.get().try_into().ok() } #[inline] diff --git a/src/core/route/page.rs b/src/core/route/page.rs index bdfcaa0..4538136 100644 --- a/src/core/route/page.rs +++ b/src/core/route/page.rs @@ -1,4 +1,7 @@ -use crate::app::constant::header::{HEADER_VALUE_TEXT_HTML_UTF8, HEADER_VALUE_TEXT_PLAIN_UTF8}; +use crate::{ + app::constant::header::{HEADER_VALUE_TEXT_HTML_UTF8, HEADER_VALUE_TEXT_PLAIN_UTF8}, + common::platform::CONFIG_EXAMPLE, +}; use axum::{body::Body, http::header::CONTENT_TYPE, response::Response}; // const MAX_FILE_SIZE_BYTES: u64 = 4_000_000_000; @@ -15,7 +18,7 @@ pub async fn handle_config_example() -> Response { __unwrap!( Response::builder() .header(CONTENT_TYPE, HEADER_VALUE_TEXT_PLAIN_UTF8) - .body(Body::from(include_str!("../../../config.example.toml"))) + .body(Body::from(CONFIG_EXAMPLE)) ) } diff --git a/src/core/service.rs b/src/core/service.rs index 5ad9097..b8dc3d4 100644 --- a/src/core/service.rs +++ b/src/core/service.rs @@ -97,7 +97,7 @@ pub async fn handle_models( }; // 获取token信息 - let (ext_token, use_pri) = (async || { + let (ext_token, use_pri) = async { // 管理员 Token if let Some(part) = auth_token.strip_prefix(&**AUTH_TOKEN) { let token_manager = state.token_manager.read().await; @@ -135,15 +135,14 @@ pub async fn handle_models( } else // 动态密钥 if AppConfig::is_dynamic_key_enabled() { - if let Some(parsed_config) = parse_dynamic_token(auth_token) { - if let Some(ext_token) = parsed_config.into_tuple().and_then(tokeninfo_to_token) { + if let Some(parsed_config) = parse_dynamic_token(auth_token) + && let Some(ext_token) = parsed_config.into_tuple().and_then(tokeninfo_to_token) { return Ok((ext_token, false)); } - } } Err(AuthError::Unauthorized) - })() + } .await .map_err(AuthError::into_generic_tuple)?; @@ -799,7 +798,7 @@ pub async fn handle_chat_completions( let response = openai::ChatCompletionChunk { id: &response_id, object: openai::ObjectChatCompletionChunk, - created: created, + created, model: model.id, choices: Some(openai::chat_completion_chunk::Choice { index: (), diff --git a/src/core/stream/decoder.rs b/src/core/stream/decoder.rs index 4840f72..566304f 100644 --- a/src/core/stream/decoder.rs +++ b/src/core/stream/decoder.rs @@ -243,7 +243,7 @@ impl StreamDecoder { let mut messages = Vec::with_capacity(count); - while let Some(raw_msg) = iter.next() { + for raw_msg in iter.by_ref() { if raw_msg.data.is_empty() { #[cfg(test)] messages.push(StreamMessage::ContentStart); @@ -343,7 +343,7 @@ impl StreamDecoder { fn handle_text_message(msg_data: &[u8], ctx: &mut Context) -> Option { // let count = self.counter.fetch_add(1, Ordering::SeqCst); - if let Ok(wrapper) = StreamUnifiedChatResponseWithTools::decode(&*msg_data) { + if let Ok(wrapper) = StreamUnifiedChatResponseWithTools::decode(msg_data) { // crate::debug!("StreamUnifiedChatResponseWithTools [hex: {}]: {:#?}", hex::encode(msg_data), response); // crate::debug!("{count}: {response:?}"); if let Some(response) = wrapper.response { diff --git a/src/lib.rs b/src/lib.rs index ad3b83b..282815c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,24 +1,50 @@ +#![allow(internal_features)] #![feature(const_trait_impl)] +#![feature(allow_internal_unsafe)] +#![feature(allow_internal_unstable)] -pub const trait UnwrapUnchecked: Sized { fn unwrap_unchecked(self) -> T; } +pub const trait UnwrapUnchecked: Sized { + fn unwrap_unchecked(self) -> T; +} -impl const UnwrapUnchecked for Option { #[inline(always)] fn unwrap_unchecked(self) -> T { unsafe { self.unwrap_unchecked() } } } +impl const UnwrapUnchecked for Option { + #[inline(always)] + fn unwrap_unchecked(self) -> T { unsafe { self.unwrap_unchecked() } } +} -impl UnwrapUnchecked for Result { #[inline(always)] fn unwrap_unchecked(self) -> T { unsafe { self.unwrap_unchecked() } } } +impl UnwrapUnchecked for Result { + #[inline(always)] + fn unwrap_unchecked(self) -> T { unsafe { self.unwrap_unchecked() } } +} #[cfg(debug_assertions)] #[macro_export] #[doc(hidden)] -macro_rules! __unwrap { ($expr:expr) => { $expr.unwrap() } } +macro_rules! __unwrap { + ($expr:expr) => { + $expr.unwrap() + }; +} #[cfg(not(debug_assertions))] #[macro_export] #[doc(hidden)] -macro_rules! __unwrap { ($expr:expr) => { $crate::UnwrapUnchecked::unwrap_unchecked($expr) } } +macro_rules! __unwrap { + ($expr:expr) => { + $crate::UnwrapUnchecked::unwrap_unchecked($expr) + }; +} #[macro_export] #[doc(hidden)] -macro_rules! __unwrap_panic { ($result:expr) => { match $result { ::core::result::Result::Ok(t) => t, ::core::result::Result::Err(e) => $crate::__panic(&e) } } } +macro_rules! __unwrap_panic { + ($result:expr) => { + match $result { + ::core::result::Result::Ok(t) => t, + ::core::result::Result::Err(e) => $crate::__panic(&e), + } + }; +} #[inline(never)] #[cold] @@ -27,11 +53,27 @@ pub fn __panic(error: &dyn ::core::fmt::Display) -> ! { ::core::panic!("{error}" #[macro_export] #[doc(hidden)] -macro_rules! __unreachable { () => {{ #[cfg(debug_assertions)] { unreachable!() } #[cfg(not(debug_assertions))] unsafe { ::core::hint::unreachable_unchecked() } }} } +macro_rules! __unreachable { + () => {{ + #[cfg(debug_assertions)] + { + unreachable!() + } + #[cfg(not(debug_assertions))] + unsafe { + ::core::hint::unreachable_unchecked() + } + }}; +} +#[allow_internal_unstable(cold_path)] #[macro_export] #[doc(hidden)] -macro_rules! __cold_path { () => { ::core::hint::cold_path() } } +macro_rules! __cold_path { + () => { + ::core::hint::cold_path() + }; +} #[cfg(unix)] pub const LF: &[u8] = b"\n"; @@ -41,21 +83,48 @@ pub const LF: &[u8] = b"\r\n"; #[macro_export] #[doc(hidden)] -macro_rules! __print { ($expr:expr) => { $crate::_print($expr.as_bytes()) } } +macro_rules! __print { + ($expr:expr) => { + $crate::_print($expr.as_bytes()) + }; +} #[macro_export] #[doc(hidden)] -macro_rules! __println { () => { $crate::_print($crate::LF) }; ($expr:expr) => { $crate::_print(concat!($expr, "\n").as_bytes()) } } +macro_rules! __println { + () => { + $crate::_print($crate::LF) + }; + ($expr:expr) => { + $crate::_print(concat!($expr, "\n").as_bytes()) + }; +} #[macro_export] #[doc(hidden)] -macro_rules! __eprint { ($expr:expr) => { $crate::_eprint($expr.as_bytes()) } } +macro_rules! __eprint { + ($expr:expr) => { + $crate::_eprint($expr.as_bytes()) + }; +} #[macro_export] #[doc(hidden)] -macro_rules! __eprintln { () => { $crate::_eprint($crate::LF) }; ($expr:expr) => { $crate::_eprint(concat!($expr, "\n").as_bytes()) } } +macro_rules! __eprintln { + () => { + $crate::_eprint($crate::LF) + }; + ($expr:expr) => { + $crate::_eprint(concat!($expr, "\n").as_bytes()) + }; +} -fn print_to(bytes: &'_ [u8], global_s: fn() -> T, label: &str) where T: std::io::Write { if let Err(e) = global_s().write_all(bytes) { panic!("failed printing to {label}: {e}"); } } +fn print_to(bytes: &'_ [u8], global_s: fn() -> T, label: &str) +where T: std::io::Write { + if let Err(e) = global_s().write_all(bytes) { + panic!("failed printing to {label}: {e}"); + } +} #[doc(hidden)] #[inline] @@ -64,3 +133,105 @@ pub fn _print(bytes: &'_ [u8]) { print_to(bytes, std::io::stdout, "stdout"); } #[doc(hidden)] #[inline] pub fn _eprint(bytes: &'_ [u8]) { print_to(bytes, std::io::stderr, "stderr"); } + +/// See: https://github.com/rust-lang/rust/blob/main/library/core/src/num/niche_types.rs +/// [`core::num::niche_types`] +#[allow_internal_unsafe] +#[allow_internal_unstable(rustc_attrs, structural_match)] +#[macro_export] +macro_rules! define_valid_range_type { + ($( + $(#[$m:meta])* + $vis:vis struct $name:ident($int:ident as $uint:ident in $low:literal..=$high:literal); + )+) => {$( + #[derive(Clone, Copy, Eq)] + #[repr(transparent)] + #[rustc_layout_scalar_valid_range_start($low)] + #[rustc_layout_scalar_valid_range_end($high)] + $(#[$m])* + $vis struct $name($int); + + const _: () = { + // With the `valid_range` attributes, it's always specified as unsigned + ::core::assert!(<$uint>::MIN == 0); + let ulow: $uint = $low; + let uhigh: $uint = $high; + ::core::assert!(ulow <= uhigh); + + ::core::assert!(::core::mem::size_of::<$int>() == ::core::mem::size_of::<$uint>()); + }; + + impl $name { + pub const MIN: $name = unsafe { $name($low as $int) }; + pub const MAX: $name = unsafe { $name($high as $int) }; + + #[inline] + pub const fn new(val: $int) -> Option { + if (val as $uint) >= ($low as $uint) && (val as $uint) <= ($high as $uint) { + // SAFETY: just checked the inclusive range + Some(unsafe { $name(val) }) + } else { + None + } + } + + /// Constructs an instance of this type from the underlying integer + /// primitive without checking whether its zero. + /// + /// # Safety + /// Immediate language UB if `val` is not within the valid range for this + /// type, as it violates the validity invariant. + #[inline] + pub const unsafe fn new_unchecked(val: $int) -> Self { + // SAFETY: Caller promised that `val` is within the valid range. + unsafe { $name(val) } + } + + #[inline] + pub const fn as_inner(self) -> $int { + // SAFETY: This is a transparent wrapper, so unwrapping it is sound + // (Not using `.0` due to MCP#807.) + unsafe { ::core::mem::transmute(self) } + } + } + + // This is required to allow matching a constant. We don't get it from a derive + // because the derived `PartialEq` would do a field projection, which is banned + // by . + impl ::core::marker::StructuralPartialEq for $name {} + + impl ::core::cmp::PartialEq for $name { + #[inline] + fn eq(&self, other: &Self) -> bool { + self.as_inner() == other.as_inner() + } + } + + impl ::core::cmp::Ord for $name { + #[inline] + fn cmp(&self, other: &Self) -> ::core::cmp::Ordering { + ::core::cmp::Ord::cmp(&self.as_inner(), &other.as_inner()) + } + } + + impl ::core::cmp::PartialOrd for $name { + #[inline] + fn partial_cmp(&self, other: &Self) -> ::core::option::Option<::core::cmp::Ordering> { + ::core::option::Option::Some(::core::cmp::Ord::cmp(self, other)) + } + } + + impl ::core::hash::Hash for $name { + // Required method + fn hash(&self, state: &mut H) { + ::core::hash::Hash::hash(&self.as_inner(), state); + } + } + + impl ::core::fmt::Debug for $name { + fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result { + <$int as ::core::fmt::Debug>::fmt(&self.as_inner(), f) + } + } + )+}; +} diff --git a/src/main.rs b/src/main.rs index d6a4130..65c69cd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,6 @@ #![allow(internal_features)] #![feature( addr_parse_ascii, - cold_path, hasher_prefixfree_extras, const_trait_impl, const_default, @@ -15,7 +14,8 @@ fn_traits, ptr_metadata, maybe_uninit_as_bytes, - cfg_select + never_type, + adt_const_params )] #![allow(clippy::redundant_static_lifetimes, clippy::enum_variant_names, clippy::let_and_return)] @@ -108,11 +108,10 @@ fn main() { } } else { println!("数据兼容版本标识不存在,当前需要: v{MIN_COMPAT_VERSION}"); - if let Ok(mut f) = std::fs::File::create(&path) { - if let Err(e) = MIN_COMPAT_VERSION.write_to(&mut f) { + if let Ok(mut f) = std::fs::File::create(&path) + && let Err(e) = MIN_COMPAT_VERSION.write_to(&mut f) { eprintln!("{e}"); } - } } }