kernel/sync/
arc.rs

1// SPDX-License-Identifier: GPL-2.0
2
3//! A reference-counted pointer.
4//!
5//! This module implements a way for users to create reference-counted objects and pointers to
6//! them. Such a pointer automatically increments and decrements the count, and drops the
7//! underlying object when it reaches zero. It is also safe to use concurrently from multiple
8//! threads.
9//!
10//! It is different from the standard library's [`Arc`] in a few ways:
11//! 1. It is backed by the kernel's `refcount_t` type.
12//! 2. It does not support weak references, which allows it to be half the size.
13//! 3. It saturates the reference count instead of aborting when it goes over a threshold.
14//! 4. It does not provide a `get_mut` method, so the ref counted object is pinned.
15//! 5. The object in [`Arc`] is pinned implicitly.
16//!
17//! [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html
18
19use crate::{
20    alloc::{AllocError, Flags, KBox},
21    bindings,
22    init::{self, InPlaceInit, Init, PinInit},
23    try_init,
24    types::{ForeignOwnable, Opaque},
25};
26use core::{
27    alloc::Layout,
28    fmt,
29    marker::PhantomData,
30    mem::{ManuallyDrop, MaybeUninit},
31    ops::{Deref, DerefMut},
32    pin::Pin,
33    ptr::NonNull,
34};
35use macros::pin_data;
36
37mod std_vendor;
38
39/// A reference-counted pointer to an instance of `T`.
40///
41/// The reference count is incremented when new instances of [`Arc`] are created, and decremented
42/// when they are dropped. When the count reaches zero, the underlying `T` is also dropped.
43///
44/// # Invariants
45///
46/// The reference count on an instance of [`Arc`] is always non-zero.
47/// The object pointed to by [`Arc`] is always pinned.
48///
49/// # Examples
50///
51/// ```
52/// use kernel::sync::Arc;
53///
54/// struct Example {
55///     a: u32,
56///     b: u32,
57/// }
58///
59/// // Create a refcounted instance of `Example`.
60/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
61///
62/// // Get a new pointer to `obj` and increment the refcount.
63/// let cloned = obj.clone();
64///
65/// // Assert that both `obj` and `cloned` point to the same underlying object.
66/// assert!(core::ptr::eq(&*obj, &*cloned));
67///
68/// // Destroy `obj` and decrement its refcount.
69/// drop(obj);
70///
71/// // Check that the values are still accessible through `cloned`.
72/// assert_eq!(cloned.a, 10);
73/// assert_eq!(cloned.b, 20);
74///
75/// // The refcount drops to zero when `cloned` goes out of scope, and the memory is freed.
76/// # Ok::<(), Error>(())
77/// ```
78///
79/// Using `Arc<T>` as the type of `self`:
80///
81/// ```
82/// use kernel::sync::Arc;
83///
84/// struct Example {
85///     a: u32,
86///     b: u32,
87/// }
88///
89/// impl Example {
90///     fn take_over(self: Arc<Self>) {
91///         // ...
92///     }
93///
94///     fn use_reference(self: &Arc<Self>) {
95///         // ...
96///     }
97/// }
98///
99/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
100/// obj.use_reference();
101/// obj.take_over();
102/// # Ok::<(), Error>(())
103/// ```
104///
105/// Coercion from `Arc<Example>` to `Arc<dyn MyTrait>`:
106///
107/// ```
108/// use kernel::sync::{Arc, ArcBorrow};
109///
110/// trait MyTrait {
111///     // Trait has a function whose `self` type is `Arc<Self>`.
112///     fn example1(self: Arc<Self>) {}
113///
114///     // Trait has a function whose `self` type is `ArcBorrow<'_, Self>`.
115///     fn example2(self: ArcBorrow<'_, Self>) {}
116/// }
117///
118/// struct Example;
119/// impl MyTrait for Example {}
120///
121/// // `obj` has type `Arc<Example>`.
122/// let obj: Arc<Example> = Arc::new(Example, GFP_KERNEL)?;
123///
124/// // `coerced` has type `Arc<dyn MyTrait>`.
125/// let coerced: Arc<dyn MyTrait> = obj;
126/// # Ok::<(), Error>(())
127/// ```
128#[repr(transparent)]
129#[cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, derive(core::marker::CoercePointee))]
130pub struct Arc<T: ?Sized> {
131    ptr: NonNull<ArcInner<T>>,
132    // NB: this informs dropck that objects of type `ArcInner<T>` may be used in `<Arc<T> as
133    // Drop>::drop`. Note that dropck already assumes that objects of type `T` may be used in
134    // `<Arc<T> as Drop>::drop` and the distinction between `T` and `ArcInner<T>` is not presently
135    // meaningful with respect to dropck - but this may change in the future so this is left here
136    // out of an abundance of caution.
137    //
138    // See https://doc.rust-lang.org/nomicon/phantom-data.html#generic-parameters-and-drop-checking
139    // for more detail on the semantics of dropck in the presence of `PhantomData`.
140    _p: PhantomData<ArcInner<T>>,
141}
142
143#[pin_data]
144#[repr(C)]
145struct ArcInner<T: ?Sized> {
146    refcount: Opaque<bindings::refcount_t>,
147    data: T,
148}
149
150impl<T: ?Sized> ArcInner<T> {
151    /// Converts a pointer to the contents of an [`Arc`] into a pointer to the [`ArcInner`].
152    ///
153    /// # Safety
154    ///
155    /// `ptr` must have been returned by a previous call to [`Arc::into_raw`], and the `Arc` must
156    /// not yet have been destroyed.
157    unsafe fn container_of(ptr: *const T) -> NonNull<ArcInner<T>> {
158        let refcount_layout = Layout::new::<bindings::refcount_t>();
159        // SAFETY: The caller guarantees that the pointer is valid.
160        let val_layout = Layout::for_value(unsafe { &*ptr });
161        // SAFETY: We're computing the layout of a real struct that existed when compiling this
162        // binary, so its layout is not so large that it can trigger arithmetic overflow.
163        let val_offset = unsafe { refcount_layout.extend(val_layout).unwrap_unchecked().1 };
164
165        // Pointer casts leave the metadata unchanged. This is okay because the metadata of `T` and
166        // `ArcInner<T>` is the same since `ArcInner` is a struct with `T` as its last field.
167        //
168        // This is documented at:
169        // <https://doc.rust-lang.org/std/ptr/trait.Pointee.html>.
170        let ptr = ptr as *const ArcInner<T>;
171
172        // SAFETY: The pointer is in-bounds of an allocation both before and after offsetting the
173        // pointer, since it originates from a previous call to `Arc::into_raw` on an `Arc` that is
174        // still valid.
175        let ptr = unsafe { ptr.byte_sub(val_offset) };
176
177        // SAFETY: The pointer can't be null since you can't have an `ArcInner<T>` value at the null
178        // address.
179        unsafe { NonNull::new_unchecked(ptr.cast_mut()) }
180    }
181}
182
183// This is to allow coercion from `Arc<T>` to `Arc<U>` if `T` can be converted to the
184// dynamically-sized type (DST) `U`.
185#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))]
186impl<T: ?Sized + core::marker::Unsize<U>, U: ?Sized> core::ops::CoerceUnsized<Arc<U>> for Arc<T> {}
187
188// This is to allow `Arc<U>` to be dispatched on when `Arc<T>` can be coerced into `Arc<U>`.
189#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))]
190impl<T: ?Sized + core::marker::Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<Arc<U>> for Arc<T> {}
191
192// SAFETY: It is safe to send `Arc<T>` to another thread when the underlying `T` is `Sync` because
193// it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs
194// `T` to be `Send` because any thread that has an `Arc<T>` may ultimately access `T` using a
195// mutable reference when the reference count reaches zero and `T` is dropped.
196unsafe impl<T: ?Sized + Sync + Send> Send for Arc<T> {}
197
198// SAFETY: It is safe to send `&Arc<T>` to another thread when the underlying `T` is `Sync`
199// because it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally,
200// it needs `T` to be `Send` because any thread that has a `&Arc<T>` may clone it and get an
201// `Arc<T>` on that thread, so the thread may ultimately access `T` using a mutable reference when
202// the reference count reaches zero and `T` is dropped.
203unsafe impl<T: ?Sized + Sync + Send> Sync for Arc<T> {}
204
205impl<T> Arc<T> {
206    /// Constructs a new reference counted instance of `T`.
207    pub fn new(contents: T, flags: Flags) -> Result<Self, AllocError> {
208        // INVARIANT: The refcount is initialised to a non-zero value.
209        let value = ArcInner {
210            // SAFETY: There are no safety requirements for this FFI call.
211            refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }),
212            data: contents,
213        };
214
215        let inner = KBox::new(value, flags)?;
216        let inner = KBox::leak(inner).into();
217
218        // SAFETY: We just created `inner` with a reference count of 1, which is owned by the new
219        // `Arc` object.
220        Ok(unsafe { Self::from_inner(inner) })
221    }
222}
223
224impl<T: ?Sized> Arc<T> {
225    /// Constructs a new [`Arc`] from an existing [`ArcInner`].
226    ///
227    /// # Safety
228    ///
229    /// The caller must ensure that `inner` points to a valid location and has a non-zero reference
230    /// count, one of which will be owned by the new [`Arc`] instance.
231    unsafe fn from_inner(inner: NonNull<ArcInner<T>>) -> Self {
232        // INVARIANT: By the safety requirements, the invariants hold.
233        Arc {
234            ptr: inner,
235            _p: PhantomData,
236        }
237    }
238
239    /// Convert the [`Arc`] into a raw pointer.
240    ///
241    /// The raw pointer has ownership of the refcount that this Arc object owned.
242    pub fn into_raw(self) -> *const T {
243        let ptr = self.ptr.as_ptr();
244        core::mem::forget(self);
245        // SAFETY: The pointer is valid.
246        unsafe { core::ptr::addr_of!((*ptr).data) }
247    }
248
249    /// Recreates an [`Arc`] instance previously deconstructed via [`Arc::into_raw`].
250    ///
251    /// # Safety
252    ///
253    /// `ptr` must have been returned by a previous call to [`Arc::into_raw`]. Additionally, it
254    /// must not be called more than once for each previous call to [`Arc::into_raw`].
255    pub unsafe fn from_raw(ptr: *const T) -> Self {
256        // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an
257        // `Arc` that is still valid.
258        let ptr = unsafe { ArcInner::container_of(ptr) };
259
260        // SAFETY: By the safety requirements we know that `ptr` came from `Arc::into_raw`, so the
261        // reference count held then will be owned by the new `Arc` object.
262        unsafe { Self::from_inner(ptr) }
263    }
264
265    /// Returns an [`ArcBorrow`] from the given [`Arc`].
266    ///
267    /// This is useful when the argument of a function call is an [`ArcBorrow`] (e.g., in a method
268    /// receiver), but we have an [`Arc`] instead. Getting an [`ArcBorrow`] is free when optimised.
269    #[inline]
270    pub fn as_arc_borrow(&self) -> ArcBorrow<'_, T> {
271        // SAFETY: The constraint that the lifetime of the shared reference must outlive that of
272        // the returned `ArcBorrow` ensures that the object remains alive and that no mutable
273        // reference can be created.
274        unsafe { ArcBorrow::new(self.ptr) }
275    }
276
277    /// Compare whether two [`Arc`] pointers reference the same underlying object.
278    pub fn ptr_eq(this: &Self, other: &Self) -> bool {
279        core::ptr::eq(this.ptr.as_ptr(), other.ptr.as_ptr())
280    }
281
282    /// Converts this [`Arc`] into a [`UniqueArc`], or destroys it if it is not unique.
283    ///
284    /// When this destroys the `Arc`, it does so while properly avoiding races. This means that
285    /// this method will never call the destructor of the value.
286    ///
287    /// # Examples
288    ///
289    /// ```
290    /// use kernel::sync::{Arc, UniqueArc};
291    ///
292    /// let arc = Arc::new(42, GFP_KERNEL)?;
293    /// let unique_arc = arc.into_unique_or_drop();
294    ///
295    /// // The above conversion should succeed since refcount of `arc` is 1.
296    /// assert!(unique_arc.is_some());
297    ///
298    /// assert_eq!(*(unique_arc.unwrap()), 42);
299    ///
300    /// # Ok::<(), Error>(())
301    /// ```
302    ///
303    /// ```
304    /// use kernel::sync::{Arc, UniqueArc};
305    ///
306    /// let arc = Arc::new(42, GFP_KERNEL)?;
307    /// let another = arc.clone();
308    ///
309    /// let unique_arc = arc.into_unique_or_drop();
310    ///
311    /// // The above conversion should fail since refcount of `arc` is >1.
312    /// assert!(unique_arc.is_none());
313    ///
314    /// # Ok::<(), Error>(())
315    /// ```
316    pub fn into_unique_or_drop(self) -> Option<Pin<UniqueArc<T>>> {
317        // We will manually manage the refcount in this method, so we disable the destructor.
318        let me = ManuallyDrop::new(self);
319        // SAFETY: We own a refcount, so the pointer is still valid.
320        let refcount = unsafe { me.ptr.as_ref() }.refcount.get();
321
322        // If the refcount reaches a non-zero value, then we have destroyed this `Arc` and will
323        // return without further touching the `Arc`. If the refcount reaches zero, then there are
324        // no other arcs, and we can create a `UniqueArc`.
325        //
326        // SAFETY: We own a refcount, so the pointer is not dangling.
327        let is_zero = unsafe { bindings::refcount_dec_and_test(refcount) };
328        if is_zero {
329            // SAFETY: We have exclusive access to the arc, so we can perform unsynchronized
330            // accesses to the refcount.
331            unsafe { core::ptr::write(refcount, bindings::REFCOUNT_INIT(1)) };
332
333            // INVARIANT: We own the only refcount to this arc, so we may create a `UniqueArc`. We
334            // must pin the `UniqueArc` because the values was previously in an `Arc`, and they pin
335            // their values.
336            Some(Pin::from(UniqueArc {
337                inner: ManuallyDrop::into_inner(me),
338            }))
339        } else {
340            None
341        }
342    }
343}
344
345impl<T: 'static> ForeignOwnable for Arc<T> {
346    type Borrowed<'a> = ArcBorrow<'a, T>;
347    type BorrowedMut<'a> = Self::Borrowed<'a>;
348
349    fn into_foreign(self) -> *mut crate::ffi::c_void {
350        ManuallyDrop::new(self).ptr.as_ptr().cast()
351    }
352
353    unsafe fn from_foreign(ptr: *mut crate::ffi::c_void) -> Self {
354        // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous
355        // call to `Self::into_foreign`.
356        let inner = unsafe { NonNull::new_unchecked(ptr.cast::<ArcInner<T>>()) };
357
358        // SAFETY: By the safety requirement of this function, we know that `ptr` came from
359        // a previous call to `Arc::into_foreign`, which guarantees that `ptr` is valid and
360        // holds a reference count increment that is transferrable to us.
361        unsafe { Self::from_inner(inner) }
362    }
363
364    unsafe fn borrow<'a>(ptr: *mut crate::ffi::c_void) -> ArcBorrow<'a, T> {
365        // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous
366        // call to `Self::into_foreign`.
367        let inner = unsafe { NonNull::new_unchecked(ptr.cast::<ArcInner<T>>()) };
368
369        // SAFETY: The safety requirements of `from_foreign` ensure that the object remains alive
370        // for the lifetime of the returned value.
371        unsafe { ArcBorrow::new(inner) }
372    }
373
374    unsafe fn borrow_mut<'a>(ptr: *mut crate::ffi::c_void) -> ArcBorrow<'a, T> {
375        // SAFETY: The safety requirements for `borrow_mut` are a superset of the safety
376        // requirements for `borrow`.
377        unsafe { Self::borrow(ptr) }
378    }
379}
380
381impl<T: ?Sized> Deref for Arc<T> {
382    type Target = T;
383
384    fn deref(&self) -> &Self::Target {
385        // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is
386        // safe to dereference it.
387        unsafe { &self.ptr.as_ref().data }
388    }
389}
390
391impl<T: ?Sized> AsRef<T> for Arc<T> {
392    fn as_ref(&self) -> &T {
393        self.deref()
394    }
395}
396
397impl<T: ?Sized> Clone for Arc<T> {
398    fn clone(&self) -> Self {
399        // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is
400        // safe to dereference it.
401        let refcount = unsafe { self.ptr.as_ref() }.refcount.get();
402
403        // INVARIANT: C `refcount_inc` saturates the refcount, so it cannot overflow to zero.
404        // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is
405        // safe to increment the refcount.
406        unsafe { bindings::refcount_inc(refcount) };
407
408        // SAFETY: We just incremented the refcount. This increment is now owned by the new `Arc`.
409        unsafe { Self::from_inner(self.ptr) }
410    }
411}
412
413impl<T: ?Sized> Drop for Arc<T> {
414    fn drop(&mut self) {
415        // SAFETY: By the type invariant, there is necessarily a reference to the object. We cannot
416        // touch `refcount` after it's decremented to a non-zero value because another thread/CPU
417        // may concurrently decrement it to zero and free it. It is ok to have a raw pointer to
418        // freed/invalid memory as long as it is never dereferenced.
419        let refcount = unsafe { self.ptr.as_ref() }.refcount.get();
420
421        // INVARIANT: If the refcount reaches zero, there are no other instances of `Arc`, and
422        // this instance is being dropped, so the broken invariant is not observable.
423        // SAFETY: Also by the type invariant, we are allowed to decrement the refcount.
424        let is_zero = unsafe { bindings::refcount_dec_and_test(refcount) };
425        if is_zero {
426            // The count reached zero, we must free the memory.
427            //
428            // SAFETY: The pointer was initialised from the result of `KBox::leak`.
429            unsafe { drop(KBox::from_raw(self.ptr.as_ptr())) };
430        }
431    }
432}
433
434impl<T: ?Sized> From<UniqueArc<T>> for Arc<T> {
435    fn from(item: UniqueArc<T>) -> Self {
436        item.inner
437    }
438}
439
440impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> {
441    fn from(item: Pin<UniqueArc<T>>) -> Self {
442        // SAFETY: The type invariants of `Arc` guarantee that the data is pinned.
443        unsafe { Pin::into_inner_unchecked(item).inner }
444    }
445}
446
447/// A borrowed reference to an [`Arc`] instance.
448///
449/// For cases when one doesn't ever need to increment the refcount on the allocation, it is simpler
450/// to use just `&T`, which we can trivially get from an [`Arc<T>`] instance.
451///
452/// However, when one may need to increment the refcount, it is preferable to use an `ArcBorrow<T>`
453/// over `&Arc<T>` because the latter results in a double-indirection: a pointer (shared reference)
454/// to a pointer ([`Arc<T>`]) to the object (`T`). An [`ArcBorrow`] eliminates this double
455/// indirection while still allowing one to increment the refcount and getting an [`Arc<T>`] when/if
456/// needed.
457///
458/// # Invariants
459///
460/// There are no mutable references to the underlying [`Arc`], and it remains valid for the
461/// lifetime of the [`ArcBorrow`] instance.
462///
463/// # Example
464///
465/// ```
466/// use kernel::sync::{Arc, ArcBorrow};
467///
468/// struct Example;
469///
470/// fn do_something(e: ArcBorrow<'_, Example>) -> Arc<Example> {
471///     e.into()
472/// }
473///
474/// let obj = Arc::new(Example, GFP_KERNEL)?;
475/// let cloned = do_something(obj.as_arc_borrow());
476///
477/// // Assert that both `obj` and `cloned` point to the same underlying object.
478/// assert!(core::ptr::eq(&*obj, &*cloned));
479/// # Ok::<(), Error>(())
480/// ```
481///
482/// Using `ArcBorrow<T>` as the type of `self`:
483///
484/// ```
485/// use kernel::sync::{Arc, ArcBorrow};
486///
487/// struct Example {
488///     a: u32,
489///     b: u32,
490/// }
491///
492/// impl Example {
493///     fn use_reference(self: ArcBorrow<'_, Self>) {
494///         // ...
495///     }
496/// }
497///
498/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
499/// obj.as_arc_borrow().use_reference();
500/// # Ok::<(), Error>(())
501/// ```
502#[repr(transparent)]
503#[cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, derive(core::marker::CoercePointee))]
504pub struct ArcBorrow<'a, T: ?Sized + 'a> {
505    inner: NonNull<ArcInner<T>>,
506    _p: PhantomData<&'a ()>,
507}
508
509// This is to allow `ArcBorrow<U>` to be dispatched on when `ArcBorrow<T>` can be coerced into
510// `ArcBorrow<U>`.
511#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))]
512impl<T: ?Sized + core::marker::Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<ArcBorrow<'_, U>>
513    for ArcBorrow<'_, T>
514{
515}
516
517impl<T: ?Sized> Clone for ArcBorrow<'_, T> {
518    fn clone(&self) -> Self {
519        *self
520    }
521}
522
523impl<T: ?Sized> Copy for ArcBorrow<'_, T> {}
524
525impl<T: ?Sized> ArcBorrow<'_, T> {
526    /// Creates a new [`ArcBorrow`] instance.
527    ///
528    /// # Safety
529    ///
530    /// Callers must ensure the following for the lifetime of the returned [`ArcBorrow`] instance:
531    /// 1. That `inner` remains valid;
532    /// 2. That no mutable references to `inner` are created.
533    unsafe fn new(inner: NonNull<ArcInner<T>>) -> Self {
534        // INVARIANT: The safety requirements guarantee the invariants.
535        Self {
536            inner,
537            _p: PhantomData,
538        }
539    }
540
541    /// Creates an [`ArcBorrow`] to an [`Arc`] that has previously been deconstructed with
542    /// [`Arc::into_raw`].
543    ///
544    /// # Safety
545    ///
546    /// * The provided pointer must originate from a call to [`Arc::into_raw`].
547    /// * For the duration of the lifetime annotated on this `ArcBorrow`, the reference count must
548    ///   not hit zero.
549    /// * For the duration of the lifetime annotated on this `ArcBorrow`, there must not be a
550    ///   [`UniqueArc`] reference to this value.
551    pub unsafe fn from_raw(ptr: *const T) -> Self {
552        // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an
553        // `Arc` that is still valid.
554        let ptr = unsafe { ArcInner::container_of(ptr) };
555
556        // SAFETY: The caller promises that the value remains valid since the reference count must
557        // not hit zero, and no mutable reference will be created since that would involve a
558        // `UniqueArc`.
559        unsafe { Self::new(ptr) }
560    }
561}
562
563impl<T: ?Sized> From<ArcBorrow<'_, T>> for Arc<T> {
564    fn from(b: ArcBorrow<'_, T>) -> Self {
565        // SAFETY: The existence of `b` guarantees that the refcount is non-zero. `ManuallyDrop`
566        // guarantees that `drop` isn't called, so it's ok that the temporary `Arc` doesn't own the
567        // increment.
568        ManuallyDrop::new(unsafe { Arc::from_inner(b.inner) })
569            .deref()
570            .clone()
571    }
572}
573
574impl<T: ?Sized> Deref for ArcBorrow<'_, T> {
575    type Target = T;
576
577    fn deref(&self) -> &Self::Target {
578        // SAFETY: By the type invariant, the underlying object is still alive with no mutable
579        // references to it, so it is safe to create a shared reference.
580        unsafe { &self.inner.as_ref().data }
581    }
582}
583
584/// A refcounted object that is known to have a refcount of 1.
585///
586/// It is mutable and can be converted to an [`Arc`] so that it can be shared.
587///
588/// # Invariants
589///
590/// `inner` always has a reference count of 1.
591///
592/// # Examples
593///
594/// In the following example, we make changes to the inner object before turning it into an
595/// `Arc<Test>` object (after which point, it cannot be mutated directly). Note that `x.into()`
596/// cannot fail.
597///
598/// ```
599/// use kernel::sync::{Arc, UniqueArc};
600///
601/// struct Example {
602///     a: u32,
603///     b: u32,
604/// }
605///
606/// fn test() -> Result<Arc<Example>> {
607///     let mut x = UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
608///     x.a += 1;
609///     x.b += 1;
610///     Ok(x.into())
611/// }
612///
613/// # test().unwrap();
614/// ```
615///
616/// In the following example we first allocate memory for a refcounted `Example` but we don't
617/// initialise it on allocation. We do initialise it later with a call to [`UniqueArc::write`],
618/// followed by a conversion to `Arc<Example>`. This is particularly useful when allocation happens
619/// in one context (e.g., sleepable) and initialisation in another (e.g., atomic):
620///
621/// ```
622/// use kernel::sync::{Arc, UniqueArc};
623///
624/// struct Example {
625///     a: u32,
626///     b: u32,
627/// }
628///
629/// fn test() -> Result<Arc<Example>> {
630///     let x = UniqueArc::new_uninit(GFP_KERNEL)?;
631///     Ok(x.write(Example { a: 10, b: 20 }).into())
632/// }
633///
634/// # test().unwrap();
635/// ```
636///
637/// In the last example below, the caller gets a pinned instance of `Example` while converting to
638/// `Arc<Example>`; this is useful in scenarios where one needs a pinned reference during
639/// initialisation, for example, when initialising fields that are wrapped in locks.
640///
641/// ```
642/// use kernel::sync::{Arc, UniqueArc};
643///
644/// struct Example {
645///     a: u32,
646///     b: u32,
647/// }
648///
649/// fn test() -> Result<Arc<Example>> {
650///     let mut pinned = Pin::from(UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?);
651///     // We can modify `pinned` because it is `Unpin`.
652///     pinned.as_mut().a += 1;
653///     Ok(pinned.into())
654/// }
655///
656/// # test().unwrap();
657/// ```
658pub struct UniqueArc<T: ?Sized> {
659    inner: Arc<T>,
660}
661
662impl<T> UniqueArc<T> {
663    /// Tries to allocate a new [`UniqueArc`] instance.
664    pub fn new(value: T, flags: Flags) -> Result<Self, AllocError> {
665        Ok(Self {
666            // INVARIANT: The newly-created object has a refcount of 1.
667            inner: Arc::new(value, flags)?,
668        })
669    }
670
671    /// Tries to allocate a new [`UniqueArc`] instance whose contents are not initialised yet.
672    pub fn new_uninit(flags: Flags) -> Result<UniqueArc<MaybeUninit<T>>, AllocError> {
673        // INVARIANT: The refcount is initialised to a non-zero value.
674        let inner = KBox::try_init::<AllocError>(
675            try_init!(ArcInner {
676                // SAFETY: There are no safety requirements for this FFI call.
677                refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }),
678                data <- init::uninit::<T, AllocError>(),
679            }? AllocError),
680            flags,
681        )?;
682        Ok(UniqueArc {
683            // INVARIANT: The newly-created object has a refcount of 1.
684            // SAFETY: The pointer from the `KBox` is valid.
685            inner: unsafe { Arc::from_inner(KBox::leak(inner).into()) },
686        })
687    }
688}
689
690impl<T> UniqueArc<MaybeUninit<T>> {
691    /// Converts a `UniqueArc<MaybeUninit<T>>` into a `UniqueArc<T>` by writing a value into it.
692    pub fn write(mut self, value: T) -> UniqueArc<T> {
693        self.deref_mut().write(value);
694        // SAFETY: We just wrote the value to be initialized.
695        unsafe { self.assume_init() }
696    }
697
698    /// Unsafely assume that `self` is initialized.
699    ///
700    /// # Safety
701    ///
702    /// The caller guarantees that the value behind this pointer has been initialized. It is
703    /// *immediate* UB to call this when the value is not initialized.
704    pub unsafe fn assume_init(self) -> UniqueArc<T> {
705        let inner = ManuallyDrop::new(self).inner.ptr;
706        UniqueArc {
707            // SAFETY: The new `Arc` is taking over `ptr` from `self.inner` (which won't be
708            // dropped). The types are compatible because `MaybeUninit<T>` is compatible with `T`.
709            inner: unsafe { Arc::from_inner(inner.cast()) },
710        }
711    }
712
713    /// Initialize `self` using the given initializer.
714    pub fn init_with<E>(mut self, init: impl Init<T, E>) -> core::result::Result<UniqueArc<T>, E> {
715        // SAFETY: The supplied pointer is valid for initialization.
716        match unsafe { init.__init(self.as_mut_ptr()) } {
717            // SAFETY: Initialization completed successfully.
718            Ok(()) => Ok(unsafe { self.assume_init() }),
719            Err(err) => Err(err),
720        }
721    }
722
723    /// Pin-initialize `self` using the given pin-initializer.
724    pub fn pin_init_with<E>(
725        mut self,
726        init: impl PinInit<T, E>,
727    ) -> core::result::Result<Pin<UniqueArc<T>>, E> {
728        // SAFETY: The supplied pointer is valid for initialization and we will later pin the value
729        // to ensure it does not move.
730        match unsafe { init.__pinned_init(self.as_mut_ptr()) } {
731            // SAFETY: Initialization completed successfully.
732            Ok(()) => Ok(unsafe { self.assume_init() }.into()),
733            Err(err) => Err(err),
734        }
735    }
736}
737
738impl<T: ?Sized> From<UniqueArc<T>> for Pin<UniqueArc<T>> {
739    fn from(obj: UniqueArc<T>) -> Self {
740        // SAFETY: It is not possible to move/replace `T` inside a `Pin<UniqueArc<T>>` (unless `T`
741        // is `Unpin`), so it is ok to convert it to `Pin<UniqueArc<T>>`.
742        unsafe { Pin::new_unchecked(obj) }
743    }
744}
745
746impl<T: ?Sized> Deref for UniqueArc<T> {
747    type Target = T;
748
749    fn deref(&self) -> &Self::Target {
750        self.inner.deref()
751    }
752}
753
754impl<T: ?Sized> DerefMut for UniqueArc<T> {
755    fn deref_mut(&mut self) -> &mut Self::Target {
756        // SAFETY: By the `Arc` type invariant, there is necessarily a reference to the object, so
757        // it is safe to dereference it. Additionally, we know there is only one reference when
758        // it's inside a `UniqueArc`, so it is safe to get a mutable reference.
759        unsafe { &mut self.inner.ptr.as_mut().data }
760    }
761}
762
763impl<T: fmt::Display + ?Sized> fmt::Display for UniqueArc<T> {
764    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
765        fmt::Display::fmt(self.deref(), f)
766    }
767}
768
769impl<T: fmt::Display + ?Sized> fmt::Display for Arc<T> {
770    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
771        fmt::Display::fmt(self.deref(), f)
772    }
773}
774
775impl<T: fmt::Debug + ?Sized> fmt::Debug for UniqueArc<T> {
776    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
777        fmt::Debug::fmt(self.deref(), f)
778    }
779}
780
781impl<T: fmt::Debug + ?Sized> fmt::Debug for Arc<T> {
782    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
783        fmt::Debug::fmt(self.deref(), f)
784    }
785}