portable_atomic/imp/
x86.rs

Help
1// SPDX-License-Identifier: Apache-2.0 OR MIT
2
3/*
4Atomic operations implementation on x86/x86_64.
5
6This module provides atomic operations not supported by LLVM or optimizes
7cases where LLVM code generation is not optimal.
8
9Note: On Miri and ThreadSanitizer which do not support inline assembly, we don't use
10this module and use CAS loop instead.
11
12Refs:
13- x86 and amd64 instruction reference https://www.felixcloutier.com/x86
14
15Generated asm:
16- x86_64 https://godbolt.org/z/ETa1MGTP3
17*/
18
19#[cfg(not(portable_atomic_no_asm))]
20use core::arch::asm;
21use core::sync::atomic::Ordering;
22
23use super::core_atomic::{
24    AtomicI8, AtomicI16, AtomicI32, AtomicI64, AtomicIsize, AtomicU8, AtomicU16, AtomicU32,
25    AtomicU64, AtomicUsize,
26};
27
28#[cfg(target_pointer_width = "32")]
29macro_rules! ptr_modifier {
30    () => {
31        ":e"
32    };
33}
34#[cfg(target_pointer_width = "64")]
35macro_rules! ptr_modifier {
36    () => {
37        ""
38    };
39}
40
41macro_rules! atomic_int {
42    ($atomic_type:ident, $ptr_size:tt) => {
43        impl $atomic_type {
44            #[inline]
45            pub(crate) fn not(&self, _order: Ordering) {
46                let dst = self.as_ptr();
47                // SAFETY: any data races are prevented by atomic intrinsics and the raw
48                // pointer passed in is valid because we got it from a reference.
49                //
50                // https://www.felixcloutier.com/x86/not
51                unsafe {
52                    // atomic RMW is always SeqCst.
53                    asm!(
54                        concat!("lock not ", $ptr_size, " ptr [{dst", ptr_modifier!(), "}]"),
55                        dst = in(reg) dst,
56                        options(nostack, preserves_flags),
57                    );
58                }
59            }
60            #[inline]
61            pub(crate) fn neg(&self, _order: Ordering) {
62                let dst = self.as_ptr();
63                // SAFETY: any data races are prevented by atomic intrinsics and the raw
64                // pointer passed in is valid because we got it from a reference.
65                //
66                // https://www.felixcloutier.com/x86/neg
67                unsafe {
68                    // atomic RMW is always SeqCst.
69                    asm!(
70                        concat!("lock neg ", $ptr_size, " ptr [{dst", ptr_modifier!(), "}]"),
71                        dst = in(reg) dst,
72                        // Do not use `preserves_flags` because NEG modifies the CF, OF, SF, ZF, AF, and PF flag.
73                        options(nostack),
74                    );
75                }
76            }
77        }
78    };
79}
80
81atomic_int!(AtomicI8, "byte");
82atomic_int!(AtomicU8, "byte");
83atomic_int!(AtomicI16, "word");
84atomic_int!(AtomicU16, "word");
85atomic_int!(AtomicI32, "dword");
86atomic_int!(AtomicU32, "dword");
87#[cfg(target_arch = "x86_64")]
88atomic_int!(AtomicI64, "qword");
89#[cfg(target_arch = "x86_64")]
90atomic_int!(AtomicU64, "qword");
91#[cfg(target_pointer_width = "32")]
92atomic_int!(AtomicIsize, "dword");
93#[cfg(target_pointer_width = "32")]
94atomic_int!(AtomicUsize, "dword");
95#[cfg(target_pointer_width = "64")]
96atomic_int!(AtomicIsize, "qword");
97#[cfg(target_pointer_width = "64")]
98atomic_int!(AtomicUsize, "qword");
99
100#[cfg(target_arch = "x86")]
101impl AtomicI64 {
102    #[inline]
103    pub(crate) fn not(&self, order: Ordering) {
104        self.fetch_not(order);
105    }
106    #[inline]
107    pub(crate) fn neg(&self, order: Ordering) {
108        self.fetch_neg(order);
109    }
110}
111#[cfg(target_arch = "x86")]
112impl AtomicU64 {
113    #[inline]
114    pub(crate) fn not(&self, order: Ordering) {
115        self.fetch_not(order);
116    }
117    #[inline]
118    pub(crate) fn neg(&self, order: Ordering) {
119        self.fetch_neg(order);
120    }
121}
122
123macro_rules! atomic_bit_opts {
124    ($atomic_type:ident, $int_type:ident, $val_modifier:tt, $ptr_size:tt) => {
125        // LLVM 14 and older don't support generating `lock bt{s,r,c}`.
126        // LLVM 15 only supports generating `lock bt{s,r,c}` for immediate bit offsets.
127        // LLVM 16+ can generate `lock bt{s,r,c}` for both immediate and register bit offsets.
128        // https://godbolt.org/z/TGhr5z4ds
129        // So, use fetch_* based implementations on LLVM 16+, otherwise use asm based implementations.
130        #[cfg(not(portable_atomic_pre_llvm_16))]
131        impl_default_bit_opts!($atomic_type, $int_type);
132        #[cfg(portable_atomic_pre_llvm_16)]
133        impl $atomic_type {
134            // `<integer>::BITS` requires Rust 1.53
135            const BITS: u32 = (core::mem::size_of::<$int_type>() * 8) as u32;
136            #[inline]
137            pub(crate) fn bit_set(&self, bit: u32, _order: Ordering) -> bool {
138                let dst = self.as_ptr();
139                // SAFETY: any data races are prevented by atomic intrinsics and the raw
140                // pointer passed in is valid because we got it from a reference.
141                // the masking by the bit size of the type ensures that we do not shift
142                // out of bounds.
143                //
144                // https://www.felixcloutier.com/x86/bts
145                unsafe {
146                    let r: u8;
147                    // atomic RMW is always SeqCst.
148                    asm!(
149                        concat!("lock bts ", $ptr_size, " ptr [{dst", ptr_modifier!(), "}], {bit", $val_modifier, "}"),
150                        "setb {r}",
151                        dst = in(reg) dst,
152                        bit = in(reg) (bit & (Self::BITS - 1)) as $int_type,
153                        r = out(reg_byte) r,
154                        // Do not use `preserves_flags` because BTS modifies the CF flag.
155                        options(nostack),
156                    );
157                    crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
158                    r != 0
159                }
160            }
161            #[inline]
162            pub(crate) fn bit_clear(&self, bit: u32, _order: Ordering) -> bool {
163                let dst = self.as_ptr();
164                // SAFETY: any data races are prevented by atomic intrinsics and the raw
165                // pointer passed in is valid because we got it from a reference.
166                // the masking by the bit size of the type ensures that we do not shift
167                // out of bounds.
168                //
169                // https://www.felixcloutier.com/x86/btr
170                unsafe {
171                    let r: u8;
172                    // atomic RMW is always SeqCst.
173                    asm!(
174                        concat!("lock btr ", $ptr_size, " ptr [{dst", ptr_modifier!(), "}], {bit", $val_modifier, "}"),
175                        "setb {r}",
176                        dst = in(reg) dst,
177                        bit = in(reg) (bit & (Self::BITS - 1)) as $int_type,
178                        r = out(reg_byte) r,
179                        // Do not use `preserves_flags` because BTR modifies the CF flag.
180                        options(nostack),
181                    );
182                    crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
183                    r != 0
184                }
185            }
186            #[inline]
187            pub(crate) fn bit_toggle(&self, bit: u32, _order: Ordering) -> bool {
188                let dst = self.as_ptr();
189                // SAFETY: any data races are prevented by atomic intrinsics and the raw
190                // pointer passed in is valid because we got it from a reference.
191                // the masking by the bit size of the type ensures that we do not shift
192                // out of bounds.
193                //
194                // https://www.felixcloutier.com/x86/btc
195                unsafe {
196                    let r: u8;
197                    // atomic RMW is always SeqCst.
198                    asm!(
199                        concat!("lock btc ", $ptr_size, " ptr [{dst", ptr_modifier!(), "}], {bit", $val_modifier, "}"),
200                        "setb {r}",
201                        dst = in(reg) dst,
202                        bit = in(reg) (bit & (Self::BITS - 1)) as $int_type,
203                        r = out(reg_byte) r,
204                        // Do not use `preserves_flags` because BTC modifies the CF flag.
205                        options(nostack),
206                    );
207                    crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
208                    r != 0
209                }
210            }
211        }
212    };
213}
214
215impl_default_bit_opts!(AtomicI8, i8);
216impl_default_bit_opts!(AtomicU8, u8);
217atomic_bit_opts!(AtomicI16, i16, ":x", "word");
218atomic_bit_opts!(AtomicU16, u16, ":x", "word");
219atomic_bit_opts!(AtomicI32, i32, ":e", "dword");
220atomic_bit_opts!(AtomicU32, u32, ":e", "dword");
221#[cfg(target_arch = "x86_64")]
222atomic_bit_opts!(AtomicI64, i64, "", "qword");
223#[cfg(target_arch = "x86_64")]
224atomic_bit_opts!(AtomicU64, u64, "", "qword");
225#[cfg(target_arch = "x86")]
226impl_default_bit_opts!(AtomicI64, i64);
227#[cfg(target_arch = "x86")]
228impl_default_bit_opts!(AtomicU64, u64);
229#[cfg(target_pointer_width = "32")]
230atomic_bit_opts!(AtomicIsize, isize, ":e", "dword");
231#[cfg(target_pointer_width = "32")]
232atomic_bit_opts!(AtomicUsize, usize, ":e", "dword");
233#[cfg(target_pointer_width = "64")]
234atomic_bit_opts!(AtomicIsize, isize, "", "qword");
235#[cfg(target_pointer_width = "64")]
236atomic_bit_opts!(AtomicUsize, usize, "", "qword");
portable_atomic/imp/x86.rs

portable_atomic/imp/
x86.rs
