From c413d0121d4f029032c6b349319e7d04121f818b Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 2 Sep 2026 07:55:26 +0900 Subject: [PATCH 1/3] lzma: export FILTERS_MAX and raise ValueError Rename the private LZMA_FILTERS_MAX constant to pub FILTERS_MAX and use it from parse_filter_chain. An over-long filter chain now raises ValueError instead of LZMAError. Assisted-by: Grok:4.6 --- crates/common/src/compression/lzma.rs | 6 +++--- crates/stdlib/src/lzma.rs | 11 +++++------ extra_tests/snippets/stdlib_lzma.py | 7 +++++++ 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/crates/common/src/compression/lzma.rs b/crates/common/src/compression/lzma.rs index 1b7943d07fb..a0d80ec7915 100644 --- a/crates/common/src/compression/lzma.rs +++ b/crates/common/src/compression/lzma.rs @@ -16,7 +16,6 @@ use super::{CHUNKSIZE, Chunker}; pub const BUFSIZ: usize = 8192; const DEF_BUF_SIZE: usize = 16 * 1024; const USE_AFTER_FINISH_ERR: &str = "Error -2: inconsistent stream state"; -const LZMA_FILTERS_MAX: usize = 4; pub const CHECK_NONE: i32 = xz_sys::LZMA_CHECK_NONE as _; pub const CHECK_CRC32: i32 = xz_sys::LZMA_CHECK_CRC32 as _; @@ -48,6 +47,7 @@ pub const FILTER_IA64: u64 = xz_sys::LZMA_FILTER_IA64; pub const FILTER_ARM: u64 = xz_sys::LZMA_FILTER_ARM; pub const FILTER_ARMTHUMB: u64 = xz_sys::LZMA_FILTER_ARMTHUMB; pub const FILTER_SPARC: u64 = xz_sys::LZMA_FILTER_SPARC; +pub const FILTERS_MAX: usize = 4; pub const PRESET_DEFAULT: u32 = xz_sys::LZMA_PRESET_DEFAULT; pub const PRESET_EXTREME: u32 = xz_sys::LZMA_PRESET_EXTREME; @@ -189,9 +189,9 @@ fn add_bcj_filter(filters: &mut Filters, id: u64, start_offset: u32) -> Result<( } fn build_filters(specs: &[FilterSpec]) -> Result { - if specs.len() > LZMA_FILTERS_MAX { + if specs.len() > FILTERS_MAX { return Err(Error::Lzma(format!( - "Too many filters - liblzma supports a maximum of {LZMA_FILTERS_MAX}" + "Too many filters - liblzma supports a maximum of {FILTERS_MAX}" ))); } let mut filters = Filters::new(); diff --git a/crates/stdlib/src/lzma.rs b/crates/stdlib/src/lzma.rs index 23f3a4393a7..a9510d8fce0 100644 --- a/crates/stdlib/src/lzma.rs +++ b/crates/stdlib/src/lzma.rs @@ -195,13 +195,12 @@ mod _lzma { filter_specs: PyObjectRef, vm: &VirtualMachine, ) -> PyResult> { - const LZMA_FILTERS_MAX: usize = 4; let length = filter_specs.length(vm)?; - if length > LZMA_FILTERS_MAX { - return Err(new_lzma_error( - format!("Too many filters - liblzma supports a maximum of {LZMA_FILTERS_MAX}"), - vm, - )); + if length > backend::FILTERS_MAX { + return Err(vm.new_value_error(format!( + "Too many filters - liblzma supports a maximum of {}", + backend::FILTERS_MAX + ))); } let sequence = filter_specs.try_sequence(vm)?; (0..length) diff --git a/extra_tests/snippets/stdlib_lzma.py b/extra_tests/snippets/stdlib_lzma.py index 5ebce3c7fb1..e7e040f56b4 100644 --- a/extra_tests/snippets/stdlib_lzma.py +++ b/extra_tests/snippets/stdlib_lzma.py @@ -12,6 +12,13 @@ filters=({"id": lzma.FILTER_LZMA2} for _ in itertools.count()), ) +# Length is checked before any specifier is parsed: five invalid ids report the +# chain length, not the id. +with assert_raises(ValueError) as raised: + lzma.LZMACompressor(format=lzma.FORMAT_RAW, filters=[{"id": 999}] * 5) +assert type(raised.exception) is ValueError +assert str(raised.exception) == "Too many filters - liblzma supports a maximum of 4" + compressor = lzma.LZMACompressor( format=lzma.FORMAT_RAW, filters=[{"id": lzma.FILTER_LZMA2}] ) From 0474123269721313f6aee2b415d71228921b5940 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 2 Sep 2026 08:36:14 +0900 Subject: [PATCH 2/3] common: port the lzma engine to xz-core Drive `common/compression/lzma.rs` through the pure-Rust `xz-core` port of liblzma rather than the `xz` / `xz-sys` bindings to the C library, and drop the android / wasm32 `cfg` the C dependency carried. `LZMA_FILTERS_MAX` becomes `pub const FILTERS_MAX`, so a caller that reports the filter-chain limit does not have to restate the value. `xz-core` 0.1.0-rc.0 registers a `lzma12_optmap` static initializer that MSVC's `_initterm_e` calls through the wrong signature, and its `alone_decoder` dictionary-size check overflows in a debug build for `dict_size == 0`. simnalamburt/xz-rs#21 fixes both and is merged but not yet released, so the workspace pins the upstream commit carrying it; the pin drops once a release reaches crates.io. Assisted-by: Grok --- Cargo.lock | 25 +- Cargo.toml | 11 +- crates/common/Cargo.toml | 9 +- crates/common/src/compression/lzma.rs | 808 +++++++++++++------------- crates/common/src/compression/mod.rs | 5 +- 5 files changed, 420 insertions(+), 438 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f327b3ba419..638326ca1f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3345,8 +3345,7 @@ dependencies = [ "rustpython-unicode", "rustpython-wtf8", "siphasher", - "xz", - "xz-sys", + "xz-core", "zlib-rs", ] @@ -4942,36 +4941,16 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "636f85e5ca6488e96401b61eb7de54f4e44755c988af0f52cf90230c312a1a89" -[[package]] -name = "xz" -version = "0.4.6-rc.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7af8851c229f574c5e4f4e9b6d90ca06c43974a563f7f42de4b88d651bf8f19c" -dependencies = [ - "xz-core", -] - [[package]] name = "xz-core" version = "0.1.0-rc.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef4d12cf9c122b2523cce22d6ab3083f4eb56d1221e5d30ddcc70fbaac553589" +source = "git+https://github.com/simnalamburt/xz-rs?rev=5bf95411d0307e052ca2aeb06f0d75565c9a7362#5bf95411d0307e052ca2aeb06f0d75565c9a7362" dependencies = [ "libc", "memchr", "windows-sys 0.61.2", ] -[[package]] -name = "xz-sys" -version = "0.4.6-rc.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "114e3a14e5bb2d46513305741650595041b7e7e1677aa5c9715a09b7a8da08fd" -dependencies = [ - "libc", - "xz-core", -] - [[package]] name = "yoke" version = "0.8.3" diff --git a/Cargo.toml b/Cargo.toml index f85f68b1d22..fa9565e904b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -126,6 +126,14 @@ panic = "abort" [patch.crates-io] parking_lot_core = { git = "https://github.com/youknowone/parking_lot", branch = "rustpython" } +# xz-core 0.1.0-rc.0 as published registers a `lzma12_optmap` static +# initializer. On MSVC it lands in `.CRT$XIB`, which `_initterm_e` walks as +# `int (__cdecl *)(void)` while the initializer's Rust signature returns `()`, +# so a Windows build exits 255 with empty stdout and stderr. That release also +# overflows the `alone_decoder` dictionary-size check in a debug build for +# `dict_size == 0`. simnalamburt/xz-rs#21 fixes both and is merged, so the pin +# drops once a release carrying it reaches crates.io. +xz-core = { git = "https://github.com/simnalamburt/xz-rs", rev = "5bf95411d0307e052ca2aeb06f0d75565c9a7362" } # REDOX START, Uncomment when you want to compile/check with redoxer # REDOX END @@ -236,8 +244,7 @@ lexical-parse-float = "1.0.6" libc = "0.2.186" libffi = "5" libloading = "0.9" -xz = "0.4.6-rc.0" -xz-sys = "0.4.6-rc.0" +xz-core = "0.1.0-rc.0" libsqlite3-sys = "0.38" libz-rs-sys = "0.6" zlib-rs = { version = "=0.6.7", default-features = false, features = ["rust-allocator", "__internal-api"] } diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 72d7ce4a6f7..67cb846342b 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -15,7 +15,7 @@ bz2 = ["std", "dep:bzip2"] cjk-codecs = [] inet = ["std"] json = ["dep:memchr", "std"] -lzma = ["std", "dep:xz", "dep:xz-sys"] +lzma = ["std", "dep:xz-core"] std = [] threading = ["parking_lot", "std"] wasm_js = ["getrandom/wasm_js"] @@ -46,12 +46,7 @@ zlib-rs = { workspace = true, optional = true } lock_api = { workspace = true } siphasher = { workspace = true } num-complex = { workspace = true } - -[target.'cfg(not(any(target_os = "android", target_arch = "wasm32")))'.dependencies] -# RustPython does not expose `_lzma` on these targets; keep the native engine's -# dependency boundary with its common-module owner rather than in stdlib. -xz = { workspace = true, optional = true } -xz-sys = { workspace = true, optional = true } +xz-core = { workspace = true, optional = true } [lints] workspace = true diff --git a/crates/common/src/compression/lzma.rs b/crates/common/src/compression/lzma.rs index a0d80ec7915..4b68f034ccd 100644 --- a/crates/common/src/compression/lzma.rs +++ b/crates/common/src/compression/lzma.rs @@ -1,61 +1,67 @@ -// spell-checker:ignore ARMTHUMB chunker memlimit +// spell-checker:ignore ARMTHUMB chunker lclp memlimit //! VM-independent liblzma stream engine. //! -//! `_lzma` is not built on Android or WebAssembly in RustPython, so the xz -//! dependency and this module have the same target boundary. Python object -//! conversion and exception construction remain in `rustpython-stdlib`. - -use xz::stream::{ - Action, Check, Error as XzError, Filters, LzmaOptions, MatchFinder, Mode, Status, Stream, - TELL_ANY_CHECK, TELL_NO_CHECK, +//! The engine is the `xz-core` crate, a pure-Rust port of liblzma with the +//! same entry points. Python object conversion and exception construction +//! remain in `rustpython-stdlib`. + +use core::ffi::c_void; +use core::mem::MaybeUninit; + +use super::Chunker; +use xz_core::common::alone_decoder::lzma_alone_decoder; +use xz_core::common::alone_encoder::lzma_alone_encoder; +use xz_core::common::auto_decoder::lzma_auto_decoder; +use xz_core::common::common::{lzma_code, lzma_end, lzma_get_check}; +use xz_core::common::easy_encoder::lzma_easy_encoder; +use xz_core::common::filter_common::lzma_filters_free; +use xz_core::common::filter_decoder::{lzma_properties_decode, lzma_raw_decoder}; +use xz_core::common::filter_encoder::{ + lzma_properties_encode, lzma_properties_size, lzma_raw_encoder, }; +use xz_core::common::stream_decoder::lzma_stream_decoder; +use xz_core::common::stream_encoder::lzma_stream_encoder; +use xz_core::lzma::lzma_encoder_presets::lzma_lzma_preset; +use xz_core::types::*; -use super::{CHUNKSIZE, Chunker}; - -pub const BUFSIZ: usize = 8192; -const DEF_BUF_SIZE: usize = 16 * 1024; +const INITIAL_BUFFER_SIZE: usize = 8192; const USE_AFTER_FINISH_ERR: &str = "Error -2: inconsistent stream state"; -pub const CHECK_NONE: i32 = xz_sys::LZMA_CHECK_NONE as _; -pub const CHECK_CRC32: i32 = xz_sys::LZMA_CHECK_CRC32 as _; -pub const CHECK_CRC64: i32 = xz_sys::LZMA_CHECK_CRC64 as _; -pub const CHECK_SHA256: i32 = xz_sys::LZMA_CHECK_SHA256 as _; -pub const CHECK_ID_MAX: i32 = 15; +pub const CHECK_NONE: i32 = LZMA_CHECK_NONE as _; +pub const CHECK_CRC32: i32 = LZMA_CHECK_CRC32 as _; +pub const CHECK_CRC64: i32 = LZMA_CHECK_CRC64 as _; +pub const CHECK_SHA256: i32 = LZMA_CHECK_SHA256 as _; +pub const CHECK_ID_MAX: i32 = LZMA_CHECK_ID_MAX as _; pub const CHECK_UNKNOWN: i32 = CHECK_ID_MAX + 1; -pub const MF_HC3: i32 = xz_sys::LZMA_MF_HC3 as _; -pub const MF_HC4: i32 = xz_sys::LZMA_MF_HC4 as _; -pub const MF_BT2: i32 = xz_sys::LZMA_MF_BT2 as _; -pub const MF_BT3: i32 = xz_sys::LZMA_MF_BT3 as _; -pub const MF_BT4: i32 = xz_sys::LZMA_MF_BT4 as _; +pub const MF_HC3: i32 = LZMA_MF_HC3 as _; +pub const MF_HC4: i32 = LZMA_MF_HC4 as _; +pub const MF_BT2: i32 = LZMA_MF_BT2 as _; +pub const MF_BT3: i32 = LZMA_MF_BT3 as _; +pub const MF_BT4: i32 = LZMA_MF_BT4 as _; -pub const MODE_FAST: i32 = xz_sys::LZMA_MODE_FAST as _; -pub const MODE_NORMAL: i32 = xz_sys::LZMA_MODE_NORMAL as _; +pub const MODE_FAST: i32 = LZMA_MODE_FAST as _; +pub const MODE_NORMAL: i32 = LZMA_MODE_NORMAL as _; pub const FORMAT_AUTO: i32 = 0; pub const FORMAT_XZ: i32 = 1; pub const FORMAT_ALONE: i32 = 2; pub const FORMAT_RAW: i32 = 3; -pub const FILTER_LZMA1: u64 = xz_sys::LZMA_FILTER_LZMA1; -pub const FILTER_LZMA2: u64 = xz_sys::LZMA_FILTER_LZMA2; -pub const FILTER_DELTA: u64 = xz_sys::LZMA_FILTER_DELTA; -pub const FILTER_X86: u64 = xz_sys::LZMA_FILTER_X86; -pub const FILTER_POWERPC: u64 = xz_sys::LZMA_FILTER_POWERPC; -pub const FILTER_IA64: u64 = xz_sys::LZMA_FILTER_IA64; -pub const FILTER_ARM: u64 = xz_sys::LZMA_FILTER_ARM; -pub const FILTER_ARMTHUMB: u64 = xz_sys::LZMA_FILTER_ARMTHUMB; -pub const FILTER_SPARC: u64 = xz_sys::LZMA_FILTER_SPARC; +pub const FILTER_LZMA1: u64 = LZMA_FILTER_LZMA1; +pub const FILTER_LZMA2: u64 = LZMA_FILTER_LZMA2; +pub const FILTER_DELTA: u64 = LZMA_FILTER_DELTA; +pub const FILTER_X86: u64 = LZMA_FILTER_X86; +pub const FILTER_POWERPC: u64 = LZMA_FILTER_POWERPC; +pub const FILTER_IA64: u64 = LZMA_FILTER_IA64; +pub const FILTER_ARM: u64 = LZMA_FILTER_ARM; +pub const FILTER_ARMTHUMB: u64 = LZMA_FILTER_ARMTHUMB; +pub const FILTER_SPARC: u64 = LZMA_FILTER_SPARC; pub const FILTERS_MAX: usize = 4; -pub const PRESET_DEFAULT: u32 = xz_sys::LZMA_PRESET_DEFAULT; -pub const PRESET_EXTREME: u32 = xz_sys::LZMA_PRESET_EXTREME; - -const DEFAULT_LC: u32 = xz_sys::LZMA_LC_DEFAULT; -const DEFAULT_LP: u32 = xz_sys::LZMA_LP_DEFAULT; -const DEFAULT_PB: u32 = xz_sys::LZMA_PB_DEFAULT; -const DICT_POW2: [u8; 10] = [18, 20, 21, 22, 22, 23, 23, 24, 25, 26]; +pub const PRESET_DEFAULT: u32 = 6; +pub const PRESET_EXTREME: u32 = LZMA_PRESET_EXTREME; #[derive(Debug)] pub enum Error { @@ -65,17 +71,22 @@ pub enum Error { Eof, } -impl From for Error { - fn from(err: XzError) -> Self { - match err { - XzError::UnsupportedCheck => Self::Lzma("Unsupported integrity check".to_owned()), - XzError::Mem => Self::Memory, - XzError::MemLimit => Self::Lzma("Memory usage limit exceeded".to_owned()), - XzError::Format => Self::Lzma("Input format not supported by decoder".to_owned()), - XzError::Options => Self::Lzma("Invalid or unsupported options".to_owned()), - XzError::Data | XzError::NoCheck => Self::Lzma("Corrupt input data".to_owned()), - XzError::Program => Self::Lzma("Internal error".to_owned()), - } +fn check_lzma(ret: lzma_ret) -> Result { + match ret { + LZMA_OK | LZMA_GET_CHECK | LZMA_NO_CHECK | LZMA_STREAM_END => Ok(ret), + LZMA_UNSUPPORTED_CHECK => Err(Error::Lzma("Unsupported integrity check".to_owned())), + LZMA_MEM_ERROR => Err(Error::Memory), + LZMA_MEMLIMIT_ERROR => Err(Error::Lzma("Memory usage limit exceeded".to_owned())), + LZMA_FORMAT_ERROR => Err(Error::Lzma( + "Input format not supported by decoder".to_owned(), + )), + LZMA_OPTIONS_ERROR => Err(Error::Lzma("Invalid or unsupported options".to_owned())), + LZMA_DATA_ERROR => Err(Error::Lzma("Corrupt input data".to_owned())), + LZMA_BUF_ERROR => Err(Error::Lzma("Insufficient buffer space".to_owned())), + LZMA_PROG_ERROR => Err(Error::Lzma("Internal error".to_owned())), + other => Err(Error::Lzma(format!( + "Unrecognized error from liblzma: {other}" + ))), } } @@ -95,326 +106,313 @@ pub struct FilterSpec { pub start_offset: Option, } -fn int_to_check(check: i32) -> Option { - if check == -1 { - return Some(Check::Crc64); - } - match check { - CHECK_NONE => Some(Check::None), - CHECK_CRC32 => Some(Check::Crc32), - CHECK_CRC64 => Some(Check::Crc64), - CHECK_SHA256 => Some(Check::Sha256), - _ => None, - } -} - -fn u32_to_mode(value: u32) -> Option { - match value as i32 { - MODE_FAST => Some(Mode::Fast), - MODE_NORMAL => Some(Mode::Normal), - _ => None, - } +enum FilterOptions { + Lzma(Box), + Delta(Box), + Bcj(Box), } -fn u32_to_mf(value: u32) -> Option { - match value as i32 { - MF_HC3 => Some(MatchFinder::HashChain3), - MF_HC4 => Some(MatchFinder::HashChain4), - MF_BT2 => Some(MatchFinder::BinaryTree2), - MF_BT3 => Some(MatchFinder::BinaryTree3), - MF_BT4 => Some(MatchFinder::BinaryTree4), - _ => None, - } +struct FilterChain { + filters: Vec, + owned: Vec, } -fn lzma_options(spec: &FilterSpec) -> Result { +fn lzma_options(spec: &FilterSpec) -> Result, Error> { let preset = spec.preset.unwrap_or(PRESET_DEFAULT); - let mut options = LzmaOptions::new_preset(preset) - .map_err(|_| Error::Lzma(format!("Invalid compression preset: {preset}")))?; + let mut options = Box::new(unsafe { MaybeUninit::::zeroed().assume_init() }); + if unsafe { lzma_lzma_preset(&mut *options, preset) } != 0 { + return Err(Error::Lzma(format!("Invalid compression preset: {preset}"))); + } if let Some(value) = spec.dict_size { - options.dict_size(value); + options.dict_size = value; } if let Some(value) = spec.lc { - options.literal_context_bits(value); + options.lc = value; } if let Some(value) = spec.lp { - options.literal_position_bits(value); + options.lp = value; } if let Some(value) = spec.pb { - options.position_bits(value); + options.pb = value; } if let Some(value) = spec.mode { - let mode = u32_to_mode(value) - .ok_or_else(|| Error::Value("Invalid filter specifier for LZMA filter".to_owned()))?; - options.mode(mode); + options.mode = value as lzma_mode; } if let Some(value) = spec.nice_len { - options.nice_len(value); + options.nice_len = value; } if let Some(value) = spec.mf { - let mf = u32_to_mf(value) - .ok_or_else(|| Error::Value("Invalid filter specifier for LZMA filter".to_owned()))?; - options.match_finder(mf); + options.mf = value as lzma_match_finder; } if let Some(value) = spec.depth { - options.depth(value); + options.depth = value; } Ok(options) } -fn add_bcj_filter(filters: &mut Filters, id: u64, start_offset: u32) -> Result<(), Error> { - if start_offset == 0 { - match id { - FILTER_X86 => filters.x86(), - FILTER_POWERPC => filters.powerpc(), - FILTER_IA64 => filters.ia64(), - FILTER_ARM => filters.arm(), - FILTER_ARMTHUMB => filters.arm_thumb(), - FILTER_SPARC => filters.sparc(), - _ => unreachable!(), - }; - } else { - let properties = start_offset.to_le_bytes(); - match id { - FILTER_X86 => filters.x86_properties(&properties)?, - FILTER_POWERPC => filters.powerpc_properties(&properties)?, - FILTER_IA64 => filters.ia64_properties(&properties)?, - FILTER_ARM => filters.arm_properties(&properties)?, - FILTER_ARMTHUMB => filters.arm_thumb_properties(&properties)?, - FILTER_SPARC => filters.sparc_properties(&properties)?, - _ => unreachable!(), - }; +fn filter_options(spec: &FilterSpec) -> Result { + match spec.id { + FILTER_LZMA1 | FILTER_LZMA2 => Ok(FilterOptions::Lzma(lzma_options(spec)?)), + FILTER_DELTA => { + let mut options = + Box::new(unsafe { MaybeUninit::::zeroed().assume_init() }); + options.type_ = LZMA_DELTA_TYPE_BYTE; + options.dist = spec.dist.unwrap_or(1); + Ok(FilterOptions::Delta(options)) + } + FILTER_X86 | FILTER_POWERPC | FILTER_IA64 | FILTER_ARM | FILTER_ARMTHUMB | FILTER_SPARC => { + let mut options = + Box::new(unsafe { MaybeUninit::::zeroed().assume_init() }); + options.start_offset = spec.start_offset.unwrap_or(0); + Ok(FilterOptions::Bcj(options)) + } + id => Err(Error::Value(format!("Invalid filter ID: {id}"))), } - Ok(()) } -fn build_filters(specs: &[FilterSpec]) -> Result { - if specs.len() > FILTERS_MAX { - return Err(Error::Lzma(format!( - "Too many filters - liblzma supports a maximum of {FILTERS_MAX}" - ))); - } - let mut filters = Filters::new(); - for spec in specs { - match spec.id { - FILTER_LZMA1 => { - filters.lzma1(&lzma_options(spec)?); - } - FILTER_LZMA2 => { - filters.lzma2(&lzma_options(spec)?); - } - FILTER_DELTA => { - let dist = spec.dist.unwrap_or(1); - if !(1..=256).contains(&dist) { - return Err(Error::Value( - "Invalid filter specifier for delta filter".to_owned(), - )); - } - filters.delta_properties(&[(dist - 1) as u8])?; - } - FILTER_X86 | FILTER_POWERPC | FILTER_IA64 | FILTER_ARM | FILTER_ARMTHUMB - | FILTER_SPARC => { - add_bcj_filter(&mut filters, spec.id, spec.start_offset.unwrap_or(0))?; +impl FilterChain { + fn new(specs: &[FilterSpec]) -> Result { + if specs.len() > FILTERS_MAX { + return Err(Error::Lzma(format!( + "Too many filters - liblzma supports a maximum of {FILTERS_MAX}" + ))); + } + let mut chain = Self { + filters: Vec::with_capacity(specs.len() + 1), + owned: Vec::with_capacity(specs.len()), + }; + for spec in specs { + let options = filter_options(spec)?; + let pointer = match &options { + FilterOptions::Lzma(o) => &raw const **o as *mut c_void, + FilterOptions::Delta(o) => &raw const **o as *mut c_void, + FilterOptions::Bcj(o) => &raw const **o as *mut c_void, + }; + chain.owned.push(options); + chain.filters.push(lzma_filter { + id: spec.id, + options: pointer, + }); + } + chain.filters.push(lzma_filter { + id: LZMA_VLI_UNKNOWN, + options: core::ptr::null_mut(), + }); + Ok(chain) + } + + fn as_ptr(&self) -> *const lzma_filter { + self.filters.as_ptr() + } + + fn lone_lzma1_options(&self) -> Option<*const lzma_options_lzma> { + match (self.filters.len(), self.owned.first()) { + (2, Some(FilterOptions::Lzma(options))) if self.filters[0].id == FILTER_LZMA1 => { + Some(&raw const **options) } - id => return Err(Error::Value(format!("Invalid filter ID: {id}"))), + _ => None, } } - Ok(filters) } -fn preset_dict_size(preset: u32) -> u32 { - let level = (preset & xz_sys::LZMA_PRESET_LEVEL_MASK) as usize; - DICT_POW2.get(level).map_or(0, |power| 1u32 << power) +struct Stream { + raw: lzma_stream, } -fn lzma2_dict_size_from_prop(prop: u8) -> u32 { - if prop >= 40 { - return u32::MAX; +// The raw pointers belong to this stream and are never accessed without the +// mutable owner. Moving the owner between threads does not share the stream. +unsafe impl Send for Stream {} + +impl Stream { + fn new() -> Self { + Self { + raw: unsafe { MaybeUninit::::zeroed().assume_init() }, + } + } + + fn code( + &mut self, + input: &[u8], + consumed: &mut usize, + block: &mut [u8], + action: lzma_action, + ) -> (lzma_ret, usize) { + let tail = &input[*consumed..]; + self.raw.next_in = tail.as_ptr(); + self.raw.avail_in = tail.len(); + self.raw.next_out = block.as_mut_ptr(); + self.raw.avail_out = block.len(); + let ret = unsafe { lzma_code(&mut self.raw, action) }; + *consumed = input.len() - self.raw.avail_in; + let produced = block.len() - self.raw.avail_out; + self.raw.next_in = core::ptr::null(); + self.raw.avail_in = 0; + self.raw.next_out = core::ptr::null_mut(); + self.raw.avail_out = 0; + (ret, produced) } - let prop = u32::from(prop); - (2 | (prop & 1)) << (prop / 2 + 11) } -fn lzma2_prop_from_dict_size(dict_size: u32) -> u8 { - if dict_size == u32::MAX { - return 40; +impl Drop for Stream { + fn drop(&mut self) { + unsafe { lzma_end(&mut self.raw) }; } - (0u8..40) - .find(|&property| lzma2_dict_size_from_prop(property) >= dict_size) - .unwrap_or(40) } -pub fn encode_filter_properties(spec: &FilterSpec) -> Result, Error> { - match spec.id { - FILTER_LZMA1 => { - let preset = spec.preset.unwrap_or(PRESET_DEFAULT); - let lc = spec.lc.unwrap_or(DEFAULT_LC); - let lp = spec.lp.unwrap_or(DEFAULT_LP); - let pb = spec.pb.unwrap_or(DEFAULT_PB); - if lc > 4 || lp > 4 || lc + lp > 4 || pb > 4 { - return Err(Error::Lzma("Invalid or unsupported options".to_owned())); - } - let dict_size = spec.dict_size.unwrap_or_else(|| preset_dict_size(preset)); - let mut result = vec![0u8; 5]; - result[0] = ((pb * 5 + lp) * 9 + lc) as u8; - result[1..].copy_from_slice(&dict_size.to_le_bytes()); - Ok(result) +fn next_block(produced_so_far: usize, max_length: usize) -> Vec { + vec![0u8; INITIAL_BUFFER_SIZE.min(max_length - produced_so_far)] +} + +fn decompress_buf( + stream: &mut Stream, + check: &mut i32, + eof: &mut bool, + chunks: &mut Chunker<'_>, + max_length: Option, +) -> Result<(Vec, bool), Error> { + let max_length = max_length.unwrap_or(usize::MAX); + let mut out = Vec::new(); + let mut block = next_block(0, max_length); + let mut capped = false; + loop { + let chunk = chunks.chunk(); + let mut consumed = 0usize; + let (ret, produced) = stream.code(chunk, &mut consumed, &mut block, LZMA_RUN); + chunks.advance(consumed); + out.extend_from_slice(&block[..produced]); + let filled = produced == block.len(); + // BUF_ERROR after the current slice is exhausted is "need more + // input", including when the next chained slice still has bytes. + let ret = if ret == LZMA_BUF_ERROR && consumed == chunk.len() && !filled { + LZMA_OK + } else { + ret + }; + check_lzma(ret)?; + if ret == LZMA_GET_CHECK || ret == LZMA_NO_CHECK { + *check = lzma_get_check(&stream.raw) as i32; } - FILTER_LZMA2 => { - let preset = spec.preset.unwrap_or(PRESET_DEFAULT); - let dict_size = spec.dict_size.unwrap_or_else(|| preset_dict_size(preset)); - Ok(vec![lzma2_prop_from_dict_size(dict_size)]) + if ret == LZMA_STREAM_END { + *eof = true; + break; } - FILTER_DELTA => { - let dist = spec.dist.unwrap_or(1); - if !(1..=256).contains(&dist) { - return Err(Error::Value( - "Invalid filter specifier for delta filter".to_owned(), - )); + if filled { + if out.len() == max_length { + capped = true; + break; } - Ok(vec![(dist - 1) as u8]) + block = next_block(out.len(), max_length); + } else if chunks.is_empty() { + break; } - FILTER_X86 | FILTER_POWERPC | FILTER_IA64 | FILTER_ARM | FILTER_ARMTHUMB | FILTER_SPARC => { - let start_offset = spec.start_offset.unwrap_or(0); - Ok(if start_offset == 0 { - vec![] - } else { - start_offset.to_le_bytes().to_vec() - }) + } + out.shrink_to_fit(); + Ok((out, capped)) +} + +fn int_to_check(check: i32) -> Option { + if check == -1 { + return Some(LZMA_CHECK_CRC64); + } + match check { + CHECK_NONE => Some(LZMA_CHECK_NONE), + CHECK_CRC32 => Some(LZMA_CHECK_CRC32), + CHECK_CRC64 => Some(LZMA_CHECK_CRC64), + CHECK_SHA256 => Some(LZMA_CHECK_SHA256), + _ => None, + } +} + +pub fn encode_filter_properties(spec: &FilterSpec) -> Result, Error> { + if spec.id == FILTER_LZMA1 { + let lc = spec.lc.unwrap_or(3); + let lp = spec.lp.unwrap_or(0); + let pb = spec.pb.unwrap_or(2); + if lc > LZMA_LCLP_MAX || lp > LZMA_LCLP_MAX || lc + lp > LZMA_LCLP_MAX || pb > LZMA_PB_MAX { + return Err(Error::Lzma("Invalid or unsupported options".to_owned())); } - id => Err(Error::Value(format!("Invalid filter ID: {id}"))), } + let chain = FilterChain::new(core::slice::from_ref(spec))?; + let filter = chain.filters[0]; + let mut encoded_size: u32 = 0; + check_lzma(unsafe { lzma_properties_size(&mut encoded_size, &filter) })?; + let mut properties = vec![0u8; encoded_size as usize]; + check_lzma(unsafe { lzma_properties_encode(&filter, properties.as_mut_ptr()) })?; + Ok(properties) } pub fn decode_filter_properties(id: u64, properties: &[u8]) -> Result { - let mut spec = FilterSpec { + let mut filter = lzma_filter { id, + options: core::ptr::null_mut(), + }; + check_lzma(unsafe { + lzma_properties_decode( + &mut filter, + core::ptr::null(), + properties.as_ptr(), + properties.len(), + ) + })?; + let spec = unsafe { filter_to_spec(&filter) }; + let mut chain = [ + filter, + lzma_filter { + id: LZMA_VLI_UNKNOWN, + options: core::ptr::null_mut(), + }, + ]; + unsafe { lzma_filters_free(chain.as_mut_ptr(), core::ptr::null()) }; + spec +} + +/// # Safety +/// `filter.options` must be the option struct the filter's id implies, or +/// null for a filter whose properties carry nothing. +unsafe fn filter_to_spec(filter: &lzma_filter) -> Result { + let mut spec = FilterSpec { + id: filter.id, ..FilterSpec::default() }; - match id { + match filter.id { FILTER_LZMA1 => { - let [property, a, b, c, d, ..] = properties else { - return Err(Error::Lzma("Invalid or unsupported options".to_owned())); - }; - let mut value = u32::from(*property); - spec.lc = Some(value % 9); - value /= 9; - spec.lp = Some(value % 5); - spec.pb = Some(value / 5); - spec.dict_size = Some(u32::from_le_bytes([*a, *b, *c, *d])); + let options = unsafe { &*(filter.options as *const lzma_options_lzma) }; + spec.lc = Some(options.lc); + spec.lp = Some(options.lp); + spec.pb = Some(options.pb); + spec.dict_size = Some(options.dict_size); } FILTER_LZMA2 => { - let [property] = properties else { - return Err(Error::Lzma("Invalid or unsupported options".to_owned())); - }; - spec.dict_size = Some(lzma2_dict_size_from_prop(*property)); + let options = unsafe { &*(filter.options as *const lzma_options_lzma) }; + spec.dict_size = Some(options.dict_size); } FILTER_DELTA => { - let [property] = properties else { - return Err(Error::Lzma("Invalid or unsupported options".to_owned())); - }; - spec.dist = Some(u32::from(*property) + 1); + let options = unsafe { &*(filter.options as *const lzma_options_delta) }; + spec.dist = Some(options.dist); } FILTER_X86 | FILTER_POWERPC | FILTER_IA64 | FILTER_ARM | FILTER_ARMTHUMB | FILTER_SPARC => { - match properties { - [] => {} - [a, b, c, d] => spec.start_offset = Some(u32::from_le_bytes([*a, *b, *c, *d])), - _ => return Err(Error::Lzma("Invalid or unsupported options".to_owned())), + if !filter.options.is_null() { + let options = unsafe { &*(filter.options as *const lzma_options_bcj) }; + spec.start_offset = Some(options.start_offset); } } - _ => return Err(Error::Value(format!("Invalid filter ID: {id}"))), + id => return Err(Error::Value(format!("Invalid filter ID: {id}"))), } Ok(spec) } #[must_use] pub fn is_check_supported(check_id: i32) -> bool { - unsafe { xz_sys::lzma_check_is_supported(check_id as _) != 0 } + xz_core::check::check::lzma_check_is_supported(check_id as lzma_check) != 0 } -struct LzmaStream { +pub struct Decompressor { stream: Stream, check: i32, - header_buf: [u8; 8], - header_collected: u8, - track_header: bool, -} - -impl LzmaStream { - fn new(stream: Stream, check: i32, track_header: bool) -> Self { - Self { - stream, - check, - header_buf: [0; 8], - header_collected: 0, - track_header, - } - } - - fn process(&mut self, input: &[u8], output: &mut Vec) -> Result { - if self.track_header && self.header_collected < 8 { - let count = (8 - usize::from(self.header_collected)).min(input.len()); - let start = usize::from(self.header_collected); - self.header_buf[start..start + count].copy_from_slice(&input[..count]); - self.header_collected += count as u8; - } - match self.stream.process_vec(input, output, Action::Run) { - Ok(Status::GetCheck) => { - if self.header_collected >= 8 { - self.check = i32::from(self.header_buf[7] & 0x0f); - } - Ok(Status::Ok) - } - Err(XzError::NoCheck) => { - self.check = CHECK_NONE; - Ok(Status::Ok) - } - other => other, - } - } -} - -fn decompress_chunks( - chunks: &mut Chunker<'_>, - stream: &mut LzmaStream, - max_length: Option, -) -> Result<(Vec, bool), XzError> { - if chunks.is_empty() { - return Ok((Vec::new(), true)); - } - let max_length = max_length.unwrap_or(usize::MAX); - let mut output = Vec::new(); - 'outer: loop { - let chunk = chunks.chunk(); - loop { - let additional = BUFSIZ.min(max_length - output.capacity()); - if additional == 0 { - return Ok((output, false)); - } - output.reserve_exact(additional); - let previous_in = stream.stream.total_in(); - let result = stream.process(chunk, &mut output); - let consumed = (stream.stream.total_in() - previous_in) as usize; - chunks.advance(consumed); - let status = result?; - if status == Status::StreamEnd || chunks.is_empty() { - output.shrink_to_fit(); - return Ok((output, status == Status::StreamEnd)); - } - if !chunk.is_empty() && consumed == 0 { - continue; - } - continue 'outer; - } - } -} - -pub struct Decompressor { - stream: LzmaStream, - unused_data: Vec, - input_buffer: Vec, eof: bool, needs_input: bool, + unused_data: Vec, + input_buffer: Vec, } impl Decompressor { @@ -438,74 +436,74 @@ impl Decompressor { "Cannot specify filters except with FORMAT_RAW".to_owned(), )); } + const DECODER_FLAGS: u32 = LZMA_TELL_ANY_CHECK | LZMA_TELL_NO_CHECK; let memlimit = memlimit.unwrap_or(u64::MAX); - let flags = TELL_ANY_CHECK | TELL_NO_CHECK; - let stream = match format { - FORMAT_AUTO => LzmaStream::new( - Stream::new_auto_decoder(memlimit, flags)?, - CHECK_UNKNOWN, - true, - ), - FORMAT_XZ => LzmaStream::new( - Stream::new_stream_decoder(memlimit, flags)?, - CHECK_UNKNOWN, - true, - ), - FORMAT_ALONE => LzmaStream::new(Stream::new_lzma_decoder(memlimit)?, CHECK_NONE, false), + let mut decompressor = Self { + stream: Stream::new(), + check: CHECK_UNKNOWN, + eof: false, + needs_input: true, + unused_data: Vec::new(), + input_buffer: Vec::new(), + }; + let raw = &mut decompressor.stream.raw; + let ret = match format { + FORMAT_AUTO => unsafe { lzma_auto_decoder(raw, memlimit, DECODER_FLAGS) }, + FORMAT_XZ => unsafe { lzma_stream_decoder(raw, memlimit, DECODER_FLAGS) }, + FORMAT_ALONE => { + decompressor.check = CHECK_NONE; + unsafe { lzma_alone_decoder(raw, memlimit) } + } FORMAT_RAW => { - let filters = build_filters(filters.as_deref().expect("validated raw filters"))?; - LzmaStream::new(Stream::new_raw_decoder(&filters)?, CHECK_NONE, false) + decompressor.check = CHECK_NONE; + let chain = FilterChain::new(filters.as_deref().expect("validated raw filters"))?; + unsafe { lzma_raw_decoder(raw, chain.as_ptr()) } } _ => return Err(Error::Value(format!("Invalid container format: {format}"))), }; - Ok(Self { - stream, - unused_data: Vec::new(), - input_buffer: Vec::new(), - eof: false, - needs_input: true, - }) + check_lzma(ret)?; + Ok(decompressor) } pub fn decompress(&mut self, data: &[u8], max_length: Option) -> Result, Error> { if self.eof { return Err(Error::Eof); } - let input_buffer = &mut self.input_buffer; - let stream = &mut self.stream; - let mut chunks = Chunker::chain(input_buffer, data); - let previous_len = chunks.len(); - let result = decompress_chunks(&mut chunks, stream, max_length); - let stream_end = match &result { - Ok((_, stream_end)) => *stream_end, - Err(_) => false, + let (out, leftover, capped) = { + let mut chunks = Chunker::chain(&self.input_buffer, data); + let (out, capped) = decompress_buf( + &mut self.stream, + &mut self.check, + &mut self.eof, + &mut chunks, + max_length, + )?; + let leftover = if chunks.is_empty() { + None + } else { + Some(chunks.to_vec()) + }; + (out, leftover, capped) }; - let consumed = previous_len - chunks.len(); - self.eof |= stream_end; if self.eof { self.needs_input = false; - if !chunks.is_empty() { - self.unused_data = chunks.to_vec(); + self.input_buffer.clear(); + if let Some(unused) = leftover { + self.unused_data = unused; } - } else if chunks.is_empty() { - input_buffer.clear(); - self.needs_input = true; - } else { + } else if let Some(remaining) = leftover { self.needs_input = false; - if let Some(consumed_from_data) = consumed.checked_sub(input_buffer.len()) { - input_buffer.clear(); - input_buffer.extend_from_slice(&data[consumed_from_data..]); - } else { - input_buffer.drain(..consumed); - input_buffer.extend_from_slice(data); - } + self.input_buffer = remaining; + } else { + self.needs_input = !capped; + self.input_buffer.clear(); } - result.map(|(output, _)| output).map_err(Error::from) + Ok(out) } #[must_use] pub fn check(&self) -> i32 { - self.stream.check + self.check } #[must_use] pub fn eof(&self) -> bool { @@ -522,7 +520,8 @@ impl Decompressor { } pub struct Compressor { - stream: Option, + stream: Stream, + flushed: bool, } impl Compressor { @@ -537,85 +536,90 @@ impl Compressor { "Integrity checks are only supported by FORMAT_XZ".to_owned(), )); } - let stream = match format { + let mut compressor = Self { + stream: Stream::new(), + flushed: false, + }; + let raw = &mut compressor.stream.raw; + let ret = match format { FORMAT_XZ => { let check = int_to_check(check) .ok_or_else(|| Error::Value("Invalid check value".to_owned()))?; if let Some(specs) = filters { - Stream::new_stream_encoder(&build_filters(&specs)?, check)? + let chain = FilterChain::new(&specs)?; + unsafe { lzma_stream_encoder(raw, chain.as_ptr(), check) } } else { - Stream::new_easy_encoder(preset, check)? + unsafe { lzma_easy_encoder(raw, preset, check) } } } - FORMAT_ALONE => { - let options = match filters { - None => LzmaOptions::new_preset(preset).map_err(|_| { - Error::Lzma(format!("Invalid compression preset: {preset}")) - })?, - Some(specs) => match specs.as_slice() { - [spec] if spec.id == FILTER_LZMA1 => lzma_options(spec)?, - _ => { - return Err(Error::Value( - "Invalid filter chain for FORMAT_ALONE - must be a single LZMA1 filter" - .to_owned(), - )); - } - }, - }; - Stream::new_lzma_encoder(&options)? - } + FORMAT_ALONE => match filters { + None => { + let options = lzma_options(&FilterSpec { + preset: Some(preset), + ..FilterSpec::default() + })?; + unsafe { lzma_alone_encoder(raw, &*options) } + } + Some(specs) => { + let chain = FilterChain::new(&specs)?; + let Some(options) = chain.lone_lzma1_options() else { + return Err(Error::Value( + "Invalid filter chain for FORMAT_ALONE - must be a single LZMA1 filter" + .to_owned(), + )); + }; + unsafe { lzma_alone_encoder(raw, options) } + } + }, FORMAT_RAW => { let specs = filters.ok_or_else(|| { Error::Value("Must specify filters for FORMAT_RAW".to_owned()) })?; - Stream::new_raw_encoder(&build_filters(&specs)?)? + let chain = FilterChain::new(&specs)?; + unsafe { lzma_raw_encoder(raw, chain.as_ptr()) } } _ => return Err(Error::Value(format!("Invalid container format: {format}"))), }; - Ok(Self { - stream: Some(stream), - }) + check_lzma(ret)?; + Ok(compressor) } pub fn compress(&mut self, data: &[u8]) -> Result, Error> { - let stream = self - .stream - .as_mut() - .ok_or_else(|| Error::Lzma(USE_AFTER_FINISH_ERR.to_owned()))?; - let mut output = Vec::new(); - for mut chunk in data.chunks(CHUNKSIZE) { - while !chunk.is_empty() { - output.reserve(DEF_BUF_SIZE); - let previous_in = stream.total_in(); - stream.process_vec(chunk, &mut output, Action::Run)?; - let consumed = (stream.total_in() - previous_in) as usize; - chunk = &chunk[consumed..]; - } + if self.flushed { + return Err(Error::Lzma(USE_AFTER_FINISH_ERR.to_owned())); } - output.shrink_to_fit(); - Ok(output) + self.code(data, LZMA_RUN) } pub fn flush(&mut self) -> Result, Error> { - let stream = self - .stream - .as_mut() - .ok_or_else(|| Error::Lzma(USE_AFTER_FINISH_ERR.to_owned()))?; - let mut output = Vec::new(); - let status = loop { - if output.len() == output.capacity() { - output.reserve(DEF_BUF_SIZE); - } - let status = stream.process_vec(&[], &mut output, Action::Finish)?; - if output.len() != output.capacity() { - break status; + if self.flushed { + return Err(Error::Lzma(USE_AFTER_FINISH_ERR.to_owned())); + } + self.flushed = true; + self.code(&[], LZMA_FINISH) + } + + fn code(&mut self, data: &[u8], action: lzma_action) -> Result, Error> { + let mut out = Vec::new(); + let mut block = vec![0u8; INITIAL_BUFFER_SIZE]; + let mut consumed = 0usize; + loop { + let (ret, produced) = self.stream.code(data, &mut consumed, &mut block, action); + out.extend_from_slice(&block[..produced]); + let ret = if ret == LZMA_BUF_ERROR && data.is_empty() && produced < block.len() { + LZMA_OK + } else { + ret + }; + check_lzma(ret)?; + if (action == LZMA_RUN && consumed == data.len()) + || (action == LZMA_FINISH && ret == LZMA_STREAM_END) + { + break; } - }; - if status == Status::StreamEnd { - self.stream = None; } - output.shrink_to_fit(); - Ok(output) + out.shrink_to_fit(); + Ok(out) } } diff --git a/crates/common/src/compression/mod.rs b/crates/common/src/compression/mod.rs index 9f84d2590dd..bcb3f921944 100644 --- a/crates/common/src/compression/mod.rs +++ b/crates/common/src/compression/mod.rs @@ -71,10 +71,7 @@ impl<'a> Chunker<'a> { #[cfg(feature = "bz2")] pub mod bz2; -#[cfg(all( - feature = "lzma", - not(any(target_os = "android", target_arch = "wasm32")) -))] +#[cfg(feature = "lzma")] pub mod lzma; #[cfg(feature = "zlib")] pub mod zlib; From 4854c8666af8b1ecce1e3bef6c010e4ba22d652c Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 2 Sep 2026 08:17:36 +0900 Subject: [PATCH 3/3] test_lzma: drop three passing expectedFailure markers Remove the markers on test_decompressor_chunks_empty, test_decompressor_chunks_maxsize, and test_issue21872. Assisted-by: Grok:4.6 --- Lib/test/test_lzma.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py index fff261d890e..8f269c49ec4 100644 --- a/Lib/test/test_lzma.py +++ b/Lib/test/test_lzma.py @@ -143,7 +143,6 @@ def test_decompressor_chunks(self): self.assertTrue(lzd.eof) self.assertEqual(lzd.unused_data, b"") - @unittest.expectedFailure # TODO: RUSTPYTHON; EOFError: End of stream already reached def test_decompressor_chunks_empty(self): lzd = LZMADecompressor() out = [] @@ -159,7 +158,6 @@ def test_decompressor_chunks_empty(self): self.assertTrue(lzd.eof) self.assertEqual(lzd.unused_data, b"") - @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check' def test_decompressor_chunks_maxsize(self): lzd = LZMADecompressor() max_length = 100 @@ -1352,7 +1350,6 @@ def test_tell_bad_args(self): f.close() self.assertRaises(ValueError, f.tell) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: True is not false def test_issue21872(self): # sometimes decompress data incompletely