portable_atomic/imp/
mod.rs

Help
1// SPDX-License-Identifier: Apache-2.0 OR MIT
2
3// -----------------------------------------------------------------------------
4// Lock-free implementations
5
6#[cfg(not(any(
7    all(
8        portable_atomic_no_atomic_load_store,
9        not(all(target_arch = "bpf", not(feature = "critical-section"))),
10    ),
11    target_arch = "avr",
12    target_arch = "msp430",
13)))]
14#[cfg_attr(
15    portable_atomic_no_cfg_target_has_atomic,
16    cfg(not(all(
17        any(
18            target_arch = "riscv32",
19            target_arch = "riscv64",
20            feature = "critical-section",
21            portable_atomic_unsafe_assume_single_core,
22        ),
23        portable_atomic_no_atomic_cas,
24    )))
25)]
26#[cfg_attr(
27    not(portable_atomic_no_cfg_target_has_atomic),
28    cfg(not(all(
29        any(
30            target_arch = "riscv32",
31            target_arch = "riscv64",
32            feature = "critical-section",
33            portable_atomic_unsafe_assume_single_core,
34        ),
35        not(target_has_atomic = "ptr"),
36    )))
37)]
38mod core_atomic;
39
40// AVR
41#[cfg(target_arch = "avr")]
42#[cfg(not(portable_atomic_no_asm))]
43#[cfg(not(feature = "critical-section"))]
44mod avr;
45
46// MSP430
47#[cfg(target_arch = "msp430")]
48pub(crate) mod msp430;
49
50// RISC-V without A-extension
51#[cfg(any(test, not(feature = "critical-section")))]
52#[cfg_attr(
53    portable_atomic_no_cfg_target_has_atomic,
54    cfg(any(
55        all(test, not(any(miri, portable_atomic_sanitize_thread))),
56        portable_atomic_no_atomic_cas,
57    ))
58)]
59#[cfg_attr(
60    not(portable_atomic_no_cfg_target_has_atomic),
61    cfg(any(
62        all(test, not(any(miri, portable_atomic_sanitize_thread))),
63        not(target_has_atomic = "ptr"),
64    ))
65)]
66#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
67mod riscv;
68
69// x86-specific optimizations
70// Miri and Sanitizer do not support inline assembly.
71#[cfg(all(
72    any(target_arch = "x86", target_arch = "x86_64"),
73    not(any(miri, portable_atomic_sanitize_thread)),
74    any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
75))]
76mod x86;
77
78// 64-bit atomic implementations on 32-bit architectures
79#[cfg(any(target_arch = "arm", target_arch = "riscv32"))]
80mod atomic64;
81
82// 128-bit atomic implementations on 64-bit architectures
83#[cfg(any(
84    target_arch = "aarch64",
85    target_arch = "arm64ec",
86    target_arch = "powerpc64",
87    target_arch = "riscv64",
88    target_arch = "s390x",
89    target_arch = "x86_64",
90))]
91mod atomic128;
92
93// -----------------------------------------------------------------------------
94// Lock-based fallback implementations
95
96#[cfg(feature = "fallback")]
97#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
98#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
99#[cfg(any(
100    test,
101    not(any(
102        all(
103            target_arch = "aarch64",
104            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
105        ),
106        all(target_arch = "arm64ec", not(portable_atomic_no_asm)),
107        all(
108            target_arch = "x86_64",
109            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
110            any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
111        ),
112        all(
113            target_arch = "riscv64",
114            not(any(miri, portable_atomic_sanitize_thread)),
115            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
116            any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
117        ),
118        all(
119            target_arch = "powerpc64",
120            portable_atomic_unstable_asm_experimental_arch,
121            any(
122                target_feature = "quadword-atomics",
123                portable_atomic_target_feature = "quadword-atomics",
124            ),
125        ),
126        all(target_arch = "s390x", not(portable_atomic_no_asm)),
127    ))
128))]
129mod fallback;
130
131// -----------------------------------------------------------------------------
132// Critical section based fallback implementations
133
134// On AVR, we always use critical section based fallback implementation.
135// AVR can be safely assumed to be single-core, so this is sound.
136// MSP430 as well.
137// See the module-level comments of interrupt module for more.
138#[cfg(any(
139    all(test, target_os = "none"),
140    portable_atomic_unsafe_assume_single_core,
141    feature = "critical-section",
142    target_arch = "avr",
143    target_arch = "msp430",
144))]
145#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(any(test, portable_atomic_no_atomic_cas)))]
146#[cfg_attr(
147    not(portable_atomic_no_cfg_target_has_atomic),
148    cfg(any(test, not(target_has_atomic = "ptr")))
149)]
150#[cfg(any(
151    target_arch = "arm",
152    target_arch = "avr",
153    target_arch = "msp430",
154    target_arch = "riscv32",
155    target_arch = "riscv64",
156    target_arch = "xtensa",
157    feature = "critical-section",
158))]
159mod interrupt;
160
161// -----------------------------------------------------------------------------
162// Atomic float implementations
163
164#[cfg(feature = "float")]
165pub(crate) mod float;
166
167// -----------------------------------------------------------------------------
168
169// has CAS | (has core atomic & !(avr | msp430 | critical section)) => core atomic
170#[cfg(not(any(
171    portable_atomic_no_atomic_load_store,
172    target_arch = "avr",
173    target_arch = "msp430",
174)))]
175#[cfg_attr(
176    portable_atomic_no_cfg_target_has_atomic,
177    cfg(not(all(
178        any(
179            target_arch = "riscv32",
180            target_arch = "riscv64",
181            feature = "critical-section",
182            portable_atomic_unsafe_assume_single_core,
183        ),
184        portable_atomic_no_atomic_cas,
185    )))
186)]
187#[cfg_attr(
188    not(portable_atomic_no_cfg_target_has_atomic),
189    cfg(not(all(
190        any(
191            target_arch = "riscv32",
192            target_arch = "riscv64",
193            feature = "critical-section",
194            portable_atomic_unsafe_assume_single_core,
195        ),
196        not(target_has_atomic = "ptr"),
197    )))
198)]
199items! {
200    pub(crate) use self::core_atomic::{
201        AtomicI16, AtomicI32, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU32, AtomicU8,
202        AtomicUsize,
203    };
204    #[cfg_attr(
205        portable_atomic_no_cfg_target_has_atomic,
206        cfg(any(
207            not(portable_atomic_no_atomic_64),
208            not(any(target_pointer_width = "16", target_pointer_width = "32")),
209        ))
210    )]
211    #[cfg_attr(
212        not(portable_atomic_no_cfg_target_has_atomic),
213        cfg(any(
214            target_has_atomic = "64",
215            not(any(target_pointer_width = "16", target_pointer_width = "32")),
216        ))
217    )]
218    pub(crate) use self::core_atomic::{AtomicI64, AtomicU64};
219}
220// bpf & !(critical section) => core atomic
221#[cfg(all(
222    target_arch = "bpf",
223    portable_atomic_no_atomic_load_store,
224    not(feature = "critical-section"),
225))]
226pub(crate) use self::core_atomic::{AtomicI64, AtomicIsize, AtomicPtr, AtomicU64, AtomicUsize};
227
228// RISC-V without A-extension & !(assume single core | critical section)
229#[cfg(not(any(portable_atomic_unsafe_assume_single_core, feature = "critical-section")))]
230#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_cas))]
231#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "ptr")))]
232#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
233items! {
234    pub(crate) use self::riscv::{
235        AtomicI16, AtomicI32, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU32, AtomicU8,
236        AtomicUsize,
237    };
238    #[cfg(target_arch = "riscv64")]
239    pub(crate) use self::riscv::{AtomicI64, AtomicU64};
240}
241
242// no core atomic CAS & (assume single core | critical section) => critical section based fallback
243#[cfg(any(
244    portable_atomic_unsafe_assume_single_core,
245    feature = "critical-section",
246    target_arch = "avr",
247    target_arch = "msp430",
248))]
249#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_cas))]
250#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "ptr")))]
251items! {
252    pub(crate) use self::interrupt::{
253        AtomicI16, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU8, AtomicUsize,
254    };
255    #[cfg(any(not(target_pointer_width = "16"), feature = "fallback"))]
256    pub(crate) use self::interrupt::{AtomicI32, AtomicU32};
257    #[cfg(any(
258        not(any(target_pointer_width = "16", target_pointer_width = "32")),
259        feature = "fallback",
260    ))]
261    pub(crate) use self::interrupt::{AtomicI64, AtomicU64};
262    #[cfg(feature = "fallback")]
263    pub(crate) use self::interrupt::{AtomicI128, AtomicU128};
264}
265
266// no core (64-bit | 128-bit) atomic & has CAS => use lock-base fallback
267#[cfg(feature = "fallback")]
268#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
269#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
270items! {
271    #[cfg(not(any(
272        all(
273            target_arch = "arm",
274            not(any(miri, portable_atomic_sanitize_thread)),
275            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
276            any(target_os = "linux", target_os = "android"),
277            not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
278            not(portable_atomic_no_outline_atomics),
279        ),
280        all(
281            target_arch = "riscv32",
282            not(any(miri, portable_atomic_sanitize_thread)),
283            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
284            any(
285                target_feature = "zacas",
286                portable_atomic_target_feature = "zacas",
287                all(
288                    feature = "fallback",
289                    not(portable_atomic_no_outline_atomics),
290                    any(target_os = "linux", target_os = "android"),
291                ),
292            ),
293        ),
294    )))]
295    #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_64))]
296    #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "64")))]
297    pub(crate) use self::fallback::{AtomicI64, AtomicU64};
298    #[cfg(not(any(
299        all(
300            target_arch = "aarch64",
301            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
302        ),
303        all(target_arch = "arm64ec", not(portable_atomic_no_asm)),
304        all(
305            target_arch = "x86_64",
306            not(all(
307                any(miri, portable_atomic_sanitize_thread),
308                portable_atomic_no_cmpxchg16b_intrinsic,
309            )),
310            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
311            any(
312                target_feature = "cmpxchg16b",
313                portable_atomic_target_feature = "cmpxchg16b",
314                all(
315                    feature = "fallback",
316                    not(portable_atomic_no_outline_atomics),
317                    not(any(target_env = "sgx", miri)),
318                ),
319            ),
320        ),
321        all(
322            target_arch = "riscv64",
323            not(any(miri, portable_atomic_sanitize_thread)),
324            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
325            any(
326                target_feature = "zacas",
327                portable_atomic_target_feature = "zacas",
328                all(
329                    feature = "fallback",
330                    not(portable_atomic_no_outline_atomics),
331                    any(target_os = "linux", target_os = "android"),
332                ),
333            ),
334        ),
335        all(
336            target_arch = "powerpc64",
337            portable_atomic_unstable_asm_experimental_arch,
338            any(
339                target_feature = "quadword-atomics",
340                portable_atomic_target_feature = "quadword-atomics",
341                all(
342                    feature = "fallback",
343                    not(portable_atomic_no_outline_atomics),
344                    any(
345                        all(
346                            target_os = "linux",
347                            any(
348                                all(
349                                    target_env = "gnu",
350                                    any(target_endian = "little", not(target_feature = "crt-static")),
351                                ),
352                                all(
353                                    any(target_env = "musl", target_env = "ohos", target_env = "uclibc"),
354                                    not(target_feature = "crt-static"),
355                                ),
356                                portable_atomic_outline_atomics,
357                            ),
358                        ),
359                        target_os = "android",
360                        target_os = "freebsd",
361                        target_os = "openbsd",
362                        all(
363                            target_os = "aix",
364                            not(portable_atomic_pre_llvm_20),
365                            portable_atomic_outline_atomics, // TODO(aix): currently disabled by default
366                        ),
367                    ),
368                    not(any(miri, portable_atomic_sanitize_thread)),
369                ),
370            ),
371        ),
372        all(target_arch = "s390x", not(portable_atomic_no_asm)),
373    )))]
374    pub(crate) use self::fallback::{AtomicI128, AtomicU128};
375}
376
377// 64-bit atomics (platform-specific)
378// pre-v6 Arm Linux
379#[cfg(feature = "fallback")]
380#[cfg(all(
381    target_arch = "arm",
382    not(any(miri, portable_atomic_sanitize_thread)),
383    any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
384    any(target_os = "linux", target_os = "android"),
385    not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
386    not(portable_atomic_no_outline_atomics),
387))]
388#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_64))]
389#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "64")))]
390pub(crate) use self::atomic64::arm_linux::{AtomicI64, AtomicU64};
391// riscv32 & (zacas | outline-atomics)
392#[cfg(all(
393    target_arch = "riscv32",
394    not(any(miri, portable_atomic_sanitize_thread)),
395    any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
396    any(
397        target_feature = "zacas",
398        portable_atomic_target_feature = "zacas",
399        all(
400            feature = "fallback",
401            not(portable_atomic_no_outline_atomics),
402            any(target_os = "linux", target_os = "android"),
403        ),
404    ),
405))]
406pub(crate) use self::atomic64::riscv32::{AtomicI64, AtomicU64};
407
408// 128-bit atomics (platform-specific)
409// AArch64
410#[cfg(any(
411    all(target_arch = "aarch64", any(not(portable_atomic_no_asm), portable_atomic_unstable_asm)),
412    all(target_arch = "arm64ec", not(portable_atomic_no_asm))
413))]
414pub(crate) use self::atomic128::aarch64::{AtomicI128, AtomicU128};
415// x86_64 & (cmpxchg16b | outline-atomics)
416#[cfg(all(
417    target_arch = "x86_64",
418    not(all(any(miri, portable_atomic_sanitize_thread), portable_atomic_no_cmpxchg16b_intrinsic)),
419    any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
420    any(
421        target_feature = "cmpxchg16b",
422        portable_atomic_target_feature = "cmpxchg16b",
423        all(
424            feature = "fallback",
425            not(portable_atomic_no_outline_atomics),
426            not(any(target_env = "sgx", miri)),
427        ),
428    ),
429))]
430pub(crate) use self::atomic128::x86_64::{AtomicI128, AtomicU128};
431// riscv64 & (zacas | outline-atomics)
432#[cfg(all(
433    target_arch = "riscv64",
434    not(any(miri, portable_atomic_sanitize_thread)),
435    any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
436    any(
437        target_feature = "zacas",
438        portable_atomic_target_feature = "zacas",
439        all(
440            feature = "fallback",
441            not(portable_atomic_no_outline_atomics),
442            any(target_os = "linux", target_os = "android"),
443        ),
444    ),
445))]
446pub(crate) use self::atomic128::riscv64::{AtomicI128, AtomicU128};
447// powerpc64 & (pwr8 | outline-atomics)
448#[cfg(all(
449    target_arch = "powerpc64",
450    portable_atomic_unstable_asm_experimental_arch,
451    any(
452        target_feature = "quadword-atomics",
453        portable_atomic_target_feature = "quadword-atomics",
454        all(
455            feature = "fallback",
456            not(portable_atomic_no_outline_atomics),
457            any(
458                all(
459                    target_os = "linux",
460                    any(
461                        all(
462                            target_env = "gnu",
463                            any(target_endian = "little", not(target_feature = "crt-static")),
464                        ),
465                        all(
466                            any(target_env = "musl", target_env = "ohos", target_env = "uclibc"),
467                            not(target_feature = "crt-static"),
468                        ),
469                        portable_atomic_outline_atomics,
470                    ),
471                ),
472                target_os = "android",
473                target_os = "freebsd",
474                target_os = "openbsd",
475                all(
476                    target_os = "aix",
477                    not(portable_atomic_pre_llvm_20),
478                    portable_atomic_outline_atomics, // TODO(aix): currently disabled by default
479                ),
480            ),
481            not(any(miri, portable_atomic_sanitize_thread)),
482        ),
483    ),
484))]
485pub(crate) use self::atomic128::powerpc64::{AtomicI128, AtomicU128};
486// s390x
487#[cfg(all(target_arch = "s390x", not(portable_atomic_no_asm)))]
488pub(crate) use self::atomic128::s390x::{AtomicI128, AtomicU128};
portable_atomic/imp/mod.rs

portable_atomic/imp/
mod.rs
