diff --git a/crates/rustc_codegen_spirv/src/builder/intrinsics.rs b/crates/rustc_codegen_spirv/src/builder/intrinsics.rs index 622347b8a08..acb18aa087e 100644 --- a/crates/rustc_codegen_spirv/src/builder/intrinsics.rs +++ b/crates/rustc_codegen_spirv/src/builder/intrinsics.rs @@ -542,39 +542,22 @@ impl Builder<'_, '_> { let u32 = SpirvType::Integer(32, false).def(self.span(), self); let glsl = self.ext_inst.borrow_mut().import_glsl(self); - let find_xsb = |this: &mut Self, arg, offset: i32| { + let opgl = |this: &mut Self, op: GLOp, arg| { + this.emit() + .ext_inst(u32, None, glsl, op as u32, [Operand::IdRef(arg)]) + .unwrap() + }; + let find_xsb = |this: &mut Self, arg| { if trailing { - let lsb = this - .emit() - .ext_inst( - u32, - None, - glsl, - GLOp::FindILsb as u32, - [Operand::IdRef(arg)], - ) - .unwrap(); - if offset == 0 { - lsb - } else { - let const_offset = this.constant_i32(this.span(), offset).def(this); - this.emit().i_add(u32, None, const_offset, lsb).unwrap() - } + // trailing zeros = lsb bit + opgl(this, GLOp::FindILsb, arg) } else { // rust is always unsigned, so FindUMsb - let msb_bit = this - .emit() - .ext_inst( - u32, - None, - glsl, - GLOp::FindUMsb as u32, - [Operand::IdRef(arg)], - ) - .unwrap(); + let msb_bit = opgl(this, GLOp::FindUMsb, arg); // the glsl op returns the Msb bit, not the amount of leading zeros of this u32 - // leading zeros = 31 - Msb bit - let const_offset = this.constant_i32(this.span(), 31 - offset).def(this); + // leading zeros = bits - 1 - Msb bit + let const_offset = + this.constant_i32(this.span(), bits as i32 - 1).def(this); this.emit().i_sub(u32, None, const_offset, msb_bit).unwrap() } }; @@ -582,13 +565,9 @@ impl Builder<'_, '_> { let converted = match bits { 8 | 16 => { let arg = self.emit().u_convert(u32, None, arg.def(self)).unwrap(); - if trailing { - find_xsb(self, arg, 0) - } else { - find_xsb(self, arg, bits as i32 - 32) - } + find_xsb(self, arg) } - 32 => find_xsb(self, arg.def(self), 0), + 32 => find_xsb(self, arg.def(self)), 64 => { let u32_0 = self.constant_int(u32, 0).def(self); let u32_32 = self.constant_u32(self.span(), 32).def(self); @@ -602,19 +581,26 @@ impl Builder<'_, '_> { let higher = self.emit().u_convert(u32, None, higher).unwrap(); if trailing { - let use_lower = self.emit().i_equal(bool, None, lower, u32_0).unwrap(); - let lower_bits = find_xsb(self, lower, 0); - let higher_bits = find_xsb(self, higher, 32); + let lower_bits = opgl(self, GLOp::FindILsb, lower); + let higher_bits = opgl(self, GLOp::FindILsb, higher); + let higher_bits = + self.emit().i_add(u32, None, higher_bits, u32_32).unwrap(); + let use_higher = self.emit().i_equal(bool, None, lower, u32_0).unwrap(); self.emit() - .select(u32, None, use_lower, higher_bits, lower_bits) + .select(u32, None, use_higher, higher_bits, lower_bits) .unwrap() } else { - let use_higher = - self.emit().i_equal(bool, None, higher, u32_0).unwrap(); - let lower_bits = find_xsb(self, lower, 0); - let higher_bits = find_xsb(self, higher, 32); + let lower_bits = opgl(self, GLOp::FindUMsb, lower); + let u32_63 = self.constant_u32(self.span(), 63).def(self); + let lower_bits = + self.emit().i_sub(u32, None, u32_63, lower_bits).unwrap(); + let higher_bits = opgl(self, GLOp::FindUMsb, higher); + let u32_31 = self.constant_u32(self.span(), 31).def(self); + let higher_bits = + self.emit().i_sub(u32, None, u32_31, higher_bits).unwrap(); + let use_lower = self.emit().i_equal(bool, None, higher, u32_0).unwrap(); self.emit() - .select(u32, None, use_higher, lower_bits, higher_bits) + .select(u32, None, use_lower, lower_bits, higher_bits) .unwrap() } } diff --git a/tests/difftests/tests/Cargo.lock b/tests/difftests/tests/Cargo.lock index e4e75a72328..9515db26749 100644 --- a/tests/difftests/tests/Cargo.lock +++ b/tests/difftests/tests/Cargo.lock @@ -168,6 +168,23 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "bit-ops-cpu" +version = "0.0.0" +dependencies = [ + "bit-ops-rust", + "difftest", +] + +[[package]] +name = "bit-ops-rust" +version = "0.0.0" +dependencies = [ + "bytemuck", + "difftest", + "spirv-std", +] + [[package]] name = "bit-set" version = "0.10.0" @@ -2187,23 +2204,6 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "trailing_zeros_64-cpu" -version = "0.0.0" -dependencies = [ - "difftest", - "trailing_zeros_64-rust", -] - -[[package]] -name = "trailing_zeros_64-rust" -version = "0.0.0" -dependencies = [ - "bytemuck", - "difftest", - "spirv-std", -] - [[package]] name = "trig_ops-rust" version = "0.0.0" diff --git a/tests/difftests/tests/Cargo.toml b/tests/difftests/tests/Cargo.toml index 2fc956e3e18..e0dde62b905 100644 --- a/tests/difftests/tests/Cargo.toml +++ b/tests/difftests/tests/Cargo.toml @@ -47,8 +47,8 @@ members = [ "lang/core/ops/vector_swizzle/vector_swizzle-wgsl", "lang/core/intrinsics/black_box_noop/with-black-box", "lang/core/intrinsics/black_box_noop/without-black-box", - "lang/core/intrinsics/trailing_zeros_64/trailing_zeros_64-rust", - "lang/core/intrinsics/trailing_zeros_64/trailing_zeros_64-cpu", + "lang/core/intrinsics/bit_ops/bit-ops-rust", + "lang/core/intrinsics/bit_ops/bit-ops-cpu", ] [workspace.package] diff --git a/tests/difftests/tests/lang/core/intrinsics/trailing_zeros_64/trailing_zeros_64-cpu/Cargo.toml b/tests/difftests/tests/lang/core/intrinsics/bit_ops/bit-ops-cpu/Cargo.toml similarity index 52% rename from tests/difftests/tests/lang/core/intrinsics/trailing_zeros_64/trailing_zeros_64-cpu/Cargo.toml rename to tests/difftests/tests/lang/core/intrinsics/bit_ops/bit-ops-cpu/Cargo.toml index d857363be82..e6da1dfa69d 100644 --- a/tests/difftests/tests/lang/core/intrinsics/trailing_zeros_64/trailing_zeros_64-cpu/Cargo.toml +++ b/tests/difftests/tests/lang/core/intrinsics/bit_ops/bit-ops-cpu/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "trailing_zeros_64-cpu" +name = "bit-ops-cpu" edition.workspace = true [lints] @@ -7,4 +7,4 @@ workspace = true [dependencies] difftest.workspace = true -trailing_zeros_64-rust = { path = "../trailing_zeros_64-rust" } +bit-ops-rust = { path = "../bit-ops-rust" } diff --git a/tests/difftests/tests/lang/core/intrinsics/trailing_zeros_64/trailing_zeros_64-cpu/src/main.rs b/tests/difftests/tests/lang/core/intrinsics/bit_ops/bit-ops-cpu/src/main.rs similarity index 61% rename from tests/difftests/tests/lang/core/intrinsics/trailing_zeros_64/trailing_zeros_64-cpu/src/main.rs rename to tests/difftests/tests/lang/core/intrinsics/bit_ops/bit-ops-cpu/src/main.rs index 40f41a94ff4..f67f56f97b2 100644 --- a/tests/difftests/tests/lang/core/intrinsics/trailing_zeros_64/trailing_zeros_64-cpu/src/main.rs +++ b/tests/difftests/tests/lang/core/intrinsics/bit_ops/bit-ops-cpu/src/main.rs @@ -1,10 +1,13 @@ +use bit_ops_rust::{TEST_DATA, leading_zeros_case}; use difftest::config::{Config, TestMetadata}; -use trailing_zeros_64_rust::TEST_DATA; fn main() { let config = Config::from_path(std::env::args().nth(1).unwrap()).unwrap(); - let output: Vec = TEST_DATA.iter().map(|v| v.trailing_zeros()).collect(); + let output: Vec = TEST_DATA + .iter() + .flat_map(|&value| leading_zeros_case(value)) + .collect(); config.write_result(&output).unwrap(); config diff --git a/tests/difftests/tests/lang/core/intrinsics/trailing_zeros_64/trailing_zeros_64-rust/Cargo.toml b/tests/difftests/tests/lang/core/intrinsics/bit_ops/bit-ops-rust/Cargo.toml similarity index 88% rename from tests/difftests/tests/lang/core/intrinsics/trailing_zeros_64/trailing_zeros_64-rust/Cargo.toml rename to tests/difftests/tests/lang/core/intrinsics/bit_ops/bit-ops-rust/Cargo.toml index 2ec1f92b41c..b8592c9f71f 100644 --- a/tests/difftests/tests/lang/core/intrinsics/trailing_zeros_64/trailing_zeros_64-rust/Cargo.toml +++ b/tests/difftests/tests/lang/core/intrinsics/bit_ops/bit-ops-rust/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "trailing_zeros_64-rust" +name = "bit-ops-rust" edition.workspace = true [lints] diff --git a/tests/difftests/tests/lang/core/intrinsics/bit_ops/bit-ops-rust/src/lib.rs b/tests/difftests/tests/lang/core/intrinsics/bit_ops/bit-ops-rust/src/lib.rs new file mode 100644 index 00000000000..ff3cb621221 --- /dev/null +++ b/tests/difftests/tests/lang/core/intrinsics/bit_ops/bit-ops-rust/src/lib.rs @@ -0,0 +1,76 @@ +#![no_std] + +use spirv_std::spirv; + +pub const WORDS_PER_CASE: usize = 26; + +pub const TEST_DATA: [u64; 16] = [ + 0x0000000000000000, + 0x0000000000000001, + 0x0000000000000010, + 0x0000000000000080, + 0x00000000000000FF, + 0x0000000000000100, + 0x0000000000008000, + 0x000000000000FFFF, + 0x0000000080000000, + 0x00000000FFFFFFFF, + 0x0000000100000000, + 0x0010000000000000, + 0x8000000000000000, + 0x8000000000000001, + 0x9ABCDEF012345678, + 0xFFFFFFFFFFFFFFFF, +]; + +pub fn leading_zeros_case(value: u64) -> [u32; WORDS_PER_CASE] { + let (v64, v32, v16, v8) = (value, value as u32, value as u16, value as u8); + let v64_next_power_of_two = v64.checked_next_power_of_two().unwrap_or(0); + let v64_swap_bytes = v64.swap_bytes(); + [ + v64.count_ones(), + v32.count_ones(), + v16.count_ones(), + v8.count_ones(), + v64_swap_bytes as u32, + (v64_swap_bytes >> 32) as u32, + v32.swap_bytes(), + v16.swap_bytes() as u32, + v8.swap_bytes() as u32, + v64.leading_zeros(), + v32.leading_zeros(), + v16.leading_zeros(), + v8.leading_zeros(), + v64.trailing_zeros(), + v32.trailing_zeros(), + v16.trailing_zeros(), + v8.trailing_zeros(), + // `checked_ilog2` counts the leading zeros of a value that is known to be non-zero + v64.checked_ilog2().unwrap_or(u32::MAX), + v32.checked_ilog2().unwrap_or(u32::MAX), + v16.checked_ilog2().unwrap_or(u32::MAX), + v8.checked_ilog2().unwrap_or(u32::MAX), + // The 64-bit result takes two words, low word first + v64_next_power_of_two as u32, + (v64_next_power_of_two >> 32) as u32, + v32.checked_next_power_of_two().unwrap_or(0), + v16.checked_next_power_of_two().unwrap_or(0) as u32, + v8.checked_next_power_of_two().unwrap_or(0) as u32, + ] +} + +#[spirv(compute(threads(1)))] +pub fn main_cs( + #[spirv(storage_buffer, descriptor_set = 0, binding = 0)] input: &[u64], + #[spirv(storage_buffer, descriptor_set = 0, binding = 1)] output: &mut [u32], + #[spirv(global_invocation_id)] global_id: spirv_std::glam::UVec3, +) { + let tid = global_id.x as usize; + + if tid < input.len() { + let words = leading_zeros_case(input[tid]); + for i in 0..WORDS_PER_CASE { + output[tid * WORDS_PER_CASE + i] = words[i]; + } + } +} diff --git a/tests/difftests/tests/lang/core/intrinsics/trailing_zeros_64/trailing_zeros_64-rust/src/main.rs b/tests/difftests/tests/lang/core/intrinsics/bit_ops/bit-ops-rust/src/main.rs similarity index 78% rename from tests/difftests/tests/lang/core/intrinsics/trailing_zeros_64/trailing_zeros_64-rust/src/main.rs rename to tests/difftests/tests/lang/core/intrinsics/bit_ops/bit-ops-rust/src/main.rs index a5ef47c606c..83ff788e5e2 100644 --- a/tests/difftests/tests/lang/core/intrinsics/trailing_zeros_64/trailing_zeros_64-rust/src/main.rs +++ b/tests/difftests/tests/lang/core/intrinsics/bit_ops/bit-ops-rust/src/main.rs @@ -1,11 +1,11 @@ #[cfg(not(target_arch = "spirv"))] fn main() { + use bit_ops_rust::{TEST_DATA, WORDS_PER_CASE}; use difftest::config::{Config, TestMetadata}; use difftest::scaffold::compute::{ AshBackend, BufferConfig, BufferUsage, ComputeShaderTest, RustComputeShader, }; use difftest::spirv_builder::Capability; - use trailing_zeros_64_rust::TEST_DATA; let config = Config::from_path(std::env::args().nth(1).unwrap()).unwrap(); @@ -22,7 +22,7 @@ fn main() { #[cfg(not(target_os = "macos"))] { let input_bytes: Vec = bytemuck::cast_slice(&TEST_DATA).to_vec(); - let output_size = (TEST_DATA.len() * std::mem::size_of::()) as u64; + let output_size = (TEST_DATA.len() * WORDS_PER_CASE * std::mem::size_of::()) as u64; let buffers = vec![ BufferConfig { @@ -37,8 +37,11 @@ fn main() { }, ]; - // Use Ash backend since wgpu/naga doesn't support Int64 - let shader = RustComputeShader::default().with_capability(Capability::Int64); + // Use Ash backend since wgpu/naga doesn't support Int8, Int16 and Int64 + let shader = RustComputeShader::default() + .with_capability(Capability::Int8) + .with_capability(Capability::Int16) + .with_capability(Capability::Int64); let num_workgroups = TEST_DATA.len() as u32; let test = ComputeShaderTest::::new(shader, [num_workgroups, 1, 1], buffers) .unwrap(); diff --git a/tests/difftests/tests/lang/core/intrinsics/trailing_zeros_64/trailing_zeros_64-rust/src/lib.rs b/tests/difftests/tests/lang/core/intrinsics/trailing_zeros_64/trailing_zeros_64-rust/src/lib.rs deleted file mode 100644 index 3e4c3488283..00000000000 --- a/tests/difftests/tests/lang/core/intrinsics/trailing_zeros_64/trailing_zeros_64-rust/src/lib.rs +++ /dev/null @@ -1,38 +0,0 @@ -#![no_std] -#![cfg_attr(target_arch = "spirv", feature(asm_experimental_arch))] - -use spirv_std::spirv; - -/// Test cases for 64-bit trailing_zeros - shared between GPU and CPU tests -#[cfg(not(target_arch = "spirv"))] -pub const TEST_DATA: [u64; 16] = [ - 0x0000000000000000, - 0x0000000000000001, - 0x8000000000000000, - 0xFFFFFFFFFFFFFFFE, - 0x1234000000000000, - 0x0000000100000000, - 0x0000000000001000, - 0x0000000080000000, - 0x0000000000000010, - 0x0000000000000100, - 0x0000000000010000, - 0x0001000000000000, - 0x0100000000000000, - 0xFFFFFFFFFFFFFFFF, - 0x8000000000000001, - 0x4000000000000000, -]; - -#[spirv(compute(threads(1)))] -pub fn main_cs( - #[spirv(storage_buffer, descriptor_set = 0, binding = 0)] input: &[u64], - #[spirv(storage_buffer, descriptor_set = 0, binding = 1)] output: &mut [u32], - #[spirv(global_invocation_id)] global_id: spirv_std::glam::UVec3, -) { - let tid = global_id.x as usize; - - if tid < input.len() && tid < output.len() { - output[tid] = input[tid].trailing_zeros(); - } -}