kernel/init/
macros.rs

1// SPDX-License-Identifier: Apache-2.0 OR MIT
2
3//! This module provides the macros that actually implement the proc-macros `pin_data` and
4//! `pinned_drop`. It also contains `__init_internal` the implementation of the `{try_}{pin_}init!`
5//! macros.
6//!
7//! These macros should never be called directly, since they expect their input to be
8//! in a certain format which is internal. If used incorrectly, these macros can lead to UB even in
9//! safe code! Use the public facing macros instead.
10//!
11//! This architecture has been chosen because the kernel does not yet have access to `syn` which
12//! would make matters a lot easier for implementing these as proc-macros.
13//!
14//! # Macro expansion example
15//!
16//! This section is intended for readers trying to understand the macros in this module and the
17//! `pin_init!` macros from `init.rs`.
18//!
19//! We will look at the following example:
20//!
21//! ```rust,ignore
22//! # use kernel::init::*;
23//! # use core::pin::Pin;
24//! #[pin_data]
25//! #[repr(C)]
26//! struct Bar<T> {
27//!     #[pin]
28//!     t: T,
29//!     pub x: usize,
30//! }
31//!
32//! impl<T> Bar<T> {
33//!     fn new(t: T) -> impl PinInit<Self> {
34//!         pin_init!(Self { t, x: 0 })
35//!     }
36//! }
37//!
38//! #[pin_data(PinnedDrop)]
39//! struct Foo {
40//!     a: usize,
41//!     #[pin]
42//!     b: Bar<u32>,
43//! }
44//!
45//! #[pinned_drop]
46//! impl PinnedDrop for Foo {
47//!     fn drop(self: Pin<&mut Self>) {
48//!         pr_info!("{self:p} is getting dropped.\n");
49//!     }
50//! }
51//!
52//! let a = 42;
53//! let initializer = pin_init!(Foo {
54//!     a,
55//!     b <- Bar::new(36),
56//! });
57//! ```
58//!
59//! This example includes the most common and important features of the pin-init API.
60//!
61//! Below you can find individual section about the different macro invocations. Here are some
62//! general things we need to take into account when designing macros:
63//! - use global paths, similarly to file paths, these start with the separator: `::core::panic!()`
64//!   this ensures that the correct item is used, since users could define their own `mod core {}`
65//!   and then their own `panic!` inside to execute arbitrary code inside of our macro.
66//! - macro `unsafe` hygiene: we need to ensure that we do not expand arbitrary, user-supplied
67//!   expressions inside of an `unsafe` block in the macro, because this would allow users to do
68//!   `unsafe` operations without an associated `unsafe` block.
69//!
70//! ## `#[pin_data]` on `Bar`
71//!
72//! This macro is used to specify which fields are structurally pinned and which fields are not. It
73//! is placed on the struct definition and allows `#[pin]` to be placed on the fields.
74//!
75//! Here is the definition of `Bar` from our example:
76//!
77//! ```rust,ignore
78//! # use kernel::init::*;
79//! #[pin_data]
80//! #[repr(C)]
81//! struct Bar<T> {
82//!     #[pin]
83//!     t: T,
84//!     pub x: usize,
85//! }
86//! ```
87//!
88//! This expands to the following code:
89//!
90//! ```rust,ignore
91//! // Firstly the normal definition of the struct, attributes are preserved:
92//! #[repr(C)]
93//! struct Bar<T> {
94//!     t: T,
95//!     pub x: usize,
96//! }
97//! // Then an anonymous constant is defined, this is because we do not want any code to access the
98//! // types that we define inside:
99//! const _: () = {
100//!     // We define the pin-data carrying struct, it is a ZST and needs to have the same generics,
101//!     // since we need to implement access functions for each field and thus need to know its
102//!     // type.
103//!     struct __ThePinData<T> {
104//!         __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,
105//!     }
106//!     // We implement `Copy` for the pin-data struct, since all functions it defines will take
107//!     // `self` by value.
108//!     impl<T> ::core::clone::Clone for __ThePinData<T> {
109//!         fn clone(&self) -> Self {
110//!             *self
111//!         }
112//!     }
113//!     impl<T> ::core::marker::Copy for __ThePinData<T> {}
114//!     // For every field of `Bar`, the pin-data struct will define a function with the same name
115//!     // and accessor (`pub` or `pub(crate)` etc.). This function will take a pointer to the
116//!     // field (`slot`) and a `PinInit` or `Init` depending on the projection kind of the field
117//!     // (if pinning is structural for the field, then `PinInit` otherwise `Init`).
118//!     #[allow(dead_code)]
119//!     impl<T> __ThePinData<T> {
120//!         unsafe fn t<E>(
121//!             self,
122//!             slot: *mut T,
123//!             // Since `t` is `#[pin]`, this is `PinInit`.
124//!             init: impl ::kernel::init::PinInit<T, E>,
125//!         ) -> ::core::result::Result<(), E> {
126//!             unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) }
127//!         }
128//!         pub unsafe fn x<E>(
129//!             self,
130//!             slot: *mut usize,
131//!             // Since `x` is not `#[pin]`, this is `Init`.
132//!             init: impl ::kernel::init::Init<usize, E>,
133//!         ) -> ::core::result::Result<(), E> {
134//!             unsafe { ::kernel::init::Init::__init(init, slot) }
135//!         }
136//!     }
137//!     // Implement the internal `HasPinData` trait that associates `Bar` with the pin-data struct
138//!     // that we constructed above.
139//!     unsafe impl<T> ::kernel::init::__internal::HasPinData for Bar<T> {
140//!         type PinData = __ThePinData<T>;
141//!         unsafe fn __pin_data() -> Self::PinData {
142//!             __ThePinData {
143//!                 __phantom: ::core::marker::PhantomData,
144//!             }
145//!         }
146//!     }
147//!     // Implement the internal `PinData` trait that marks the pin-data struct as a pin-data
148//!     // struct. This is important to ensure that no user can implement a rogue `__pin_data`
149//!     // function without using `unsafe`.
150//!     unsafe impl<T> ::kernel::init::__internal::PinData for __ThePinData<T> {
151//!         type Datee = Bar<T>;
152//!     }
153//!     // Now we only want to implement `Unpin` for `Bar` when every structurally pinned field is
154//!     // `Unpin`. In other words, whether `Bar` is `Unpin` only depends on structurally pinned
155//!     // fields (those marked with `#[pin]`). These fields will be listed in this struct, in our
156//!     // case no such fields exist, hence this is almost empty. The two phantomdata fields exist
157//!     // for two reasons:
158//!     // - `__phantom`: every generic must be used, since we cannot really know which generics
159//!     //   are used, we declare all and then use everything here once.
160//!     // - `__phantom_pin`: uses the `'__pin` lifetime and ensures that this struct is invariant
161//!     //   over it. The lifetime is needed to work around the limitation that trait bounds must
162//!     //   not be trivial, e.g. the user has a `#[pin] PhantomPinned` field -- this is
163//!     //   unconditionally `!Unpin` and results in an error. The lifetime tricks the compiler
164//!     //   into accepting these bounds regardless.
165//!     #[allow(dead_code)]
166//!     struct __Unpin<'__pin, T> {
167//!         __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
168//!         __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,
169//!         // Our only `#[pin]` field is `t`.
170//!         t: T,
171//!     }
172//!     #[doc(hidden)]
173//!     impl<'__pin, T> ::core::marker::Unpin for Bar<T>
174//!     where
175//!         __Unpin<'__pin, T>: ::core::marker::Unpin,
176//!     {}
177//!     // Now we need to ensure that `Bar` does not implement `Drop`, since that would give users
178//!     // access to `&mut self` inside of `drop` even if the struct was pinned. This could lead to
179//!     // UB with only safe code, so we disallow this by giving a trait implementation error using
180//!     // a direct impl and a blanket implementation.
181//!     trait MustNotImplDrop {}
182//!     // Normally `Drop` bounds do not have the correct semantics, but for this purpose they do
183//!     // (normally people want to know if a type has any kind of drop glue at all, here we want
184//!     // to know if it has any kind of custom drop glue, which is exactly what this bound does).
185//!     #[expect(drop_bounds)]
186//!     impl<T: ::core::ops::Drop> MustNotImplDrop for T {}
187//!     impl<T> MustNotImplDrop for Bar<T> {}
188//!     // Here comes a convenience check, if one implemented `PinnedDrop`, but forgot to add it to
189//!     // `#[pin_data]`, then this will error with the same mechanic as above, this is not needed
190//!     // for safety, but a good sanity check, since no normal code calls `PinnedDrop::drop`.
191//!     #[expect(non_camel_case_types)]
192//!     trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
193//!     impl<
194//!         T: ::kernel::init::PinnedDrop,
195//!     > UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
196//!     impl<T> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for Bar<T> {}
197//! };
198//! ```
199//!
200//! ## `pin_init!` in `impl Bar`
201//!
202//! This macro creates an pin-initializer for the given struct. It requires that the struct is
203//! annotated by `#[pin_data]`.
204//!
205//! Here is the impl on `Bar` defining the new function:
206//!
207//! ```rust,ignore
208//! impl<T> Bar<T> {
209//!     fn new(t: T) -> impl PinInit<Self> {
210//!         pin_init!(Self { t, x: 0 })
211//!     }
212//! }
213//! ```
214//!
215//! This expands to the following code:
216//!
217//! ```rust,ignore
218//! impl<T> Bar<T> {
219//!     fn new(t: T) -> impl PinInit<Self> {
220//!         {
221//!             // We do not want to allow arbitrary returns, so we declare this type as the `Ok`
222//!             // return type and shadow it later when we insert the arbitrary user code. That way
223//!             // there will be no possibility of returning without `unsafe`.
224//!             struct __InitOk;
225//!             // Get the data about fields from the supplied type.
226//!             // - the function is unsafe, hence the unsafe block
227//!             // - we `use` the `HasPinData` trait in the block, it is only available in that
228//!             //   scope.
229//!             let data = unsafe {
230//!                 use ::kernel::init::__internal::HasPinData;
231//!                 Self::__pin_data()
232//!             };
233//!             // Ensure that `data` really is of type `PinData` and help with type inference:
234//!             let init = ::kernel::init::__internal::PinData::make_closure::<
235//!                 _,
236//!                 __InitOk,
237//!                 ::core::convert::Infallible,
238//!             >(data, move |slot| {
239//!                 {
240//!                     // Shadow the structure so it cannot be used to return early. If a user
241//!                     // tries to write `return Ok(__InitOk)`, then they get a type error,
242//!                     // since that will refer to this struct instead of the one defined
243//!                     // above.
244//!                     struct __InitOk;
245//!                     // This is the expansion of `t,`, which is syntactic sugar for `t: t,`.
246//!                     {
247//!                         unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).t), t) };
248//!                     }
249//!                     // Since initialization could fail later (not in this case, since the
250//!                     // error type is `Infallible`) we will need to drop this field if there
251//!                     // is an error later. This `DropGuard` will drop the field when it gets
252//!                     // dropped and has not yet been forgotten.
253//!                     let __t_guard = unsafe {
254//!                         ::pinned_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).t))
255//!                     };
256//!                     // Expansion of `x: 0,`:
257//!                     // Since this can be an arbitrary expression we cannot place it inside
258//!                     // of the `unsafe` block, so we bind it here.
259//!                     {
260//!                         let x = 0;
261//!                         unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).x), x) };
262//!                     }
263//!                     // We again create a `DropGuard`.
264//!                     let __x_guard = unsafe {
265//!                         ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).x))
266//!                     };
267//!                     // Since initialization has successfully completed, we can now forget
268//!                     // the guards. This is not `mem::forget`, since we only have
269//!                     // `&DropGuard`.
270//!                     ::core::mem::forget(__x_guard);
271//!                     ::core::mem::forget(__t_guard);
272//!                     // Here we use the type checker to ensure that every field has been
273//!                     // initialized exactly once, since this is `if false` it will never get
274//!                     // executed, but still type-checked.
275//!                     // Additionally we abuse `slot` to automatically infer the correct type
276//!                     // for the struct. This is also another check that every field is
277//!                     // accessible from this scope.
278//!                     #[allow(unreachable_code, clippy::diverging_sub_expression)]
279//!                     let _ = || {
280//!                         unsafe {
281//!                             ::core::ptr::write(
282//!                                 slot,
283//!                                 Self {
284//!                                     // We only care about typecheck finding every field
285//!                                     // here, the expression does not matter, just conjure
286//!                                     // one using `panic!()`:
287//!                                     t: ::core::panic!(),
288//!                                     x: ::core::panic!(),
289//!                                 },
290//!                             );
291//!                         };
292//!                     };
293//!                 }
294//!                 // We leave the scope above and gain access to the previously shadowed
295//!                 // `__InitOk` that we need to return.
296//!                 Ok(__InitOk)
297//!             });
298//!             // Change the return type from `__InitOk` to `()`.
299//!             let init = move |
300//!                 slot,
301//!             | -> ::core::result::Result<(), ::core::convert::Infallible> {
302//!                 init(slot).map(|__InitOk| ())
303//!             };
304//!             // Construct the initializer.
305//!             let init = unsafe {
306//!                 ::kernel::init::pin_init_from_closure::<
307//!                     _,
308//!                     ::core::convert::Infallible,
309//!                 >(init)
310//!             };
311//!             init
312//!         }
313//!     }
314//! }
315//! ```
316//!
317//! ## `#[pin_data]` on `Foo`
318//!
319//! Since we already took a look at `#[pin_data]` on `Bar`, this section will only explain the
320//! differences/new things in the expansion of the `Foo` definition:
321//!
322//! ```rust,ignore
323//! #[pin_data(PinnedDrop)]
324//! struct Foo {
325//!     a: usize,
326//!     #[pin]
327//!     b: Bar<u32>,
328//! }
329//! ```
330//!
331//! This expands to the following code:
332//!
333//! ```rust,ignore
334//! struct Foo {
335//!     a: usize,
336//!     b: Bar<u32>,
337//! }
338//! const _: () = {
339//!     struct __ThePinData {
340//!         __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,
341//!     }
342//!     impl ::core::clone::Clone for __ThePinData {
343//!         fn clone(&self) -> Self {
344//!             *self
345//!         }
346//!     }
347//!     impl ::core::marker::Copy for __ThePinData {}
348//!     #[allow(dead_code)]
349//!     impl __ThePinData {
350//!         unsafe fn b<E>(
351//!             self,
352//!             slot: *mut Bar<u32>,
353//!             init: impl ::kernel::init::PinInit<Bar<u32>, E>,
354//!         ) -> ::core::result::Result<(), E> {
355//!             unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) }
356//!         }
357//!         unsafe fn a<E>(
358//!             self,
359//!             slot: *mut usize,
360//!             init: impl ::kernel::init::Init<usize, E>,
361//!         ) -> ::core::result::Result<(), E> {
362//!             unsafe { ::kernel::init::Init::__init(init, slot) }
363//!         }
364//!     }
365//!     unsafe impl ::kernel::init::__internal::HasPinData for Foo {
366//!         type PinData = __ThePinData;
367//!         unsafe fn __pin_data() -> Self::PinData {
368//!             __ThePinData {
369//!                 __phantom: ::core::marker::PhantomData,
370//!             }
371//!         }
372//!     }
373//!     unsafe impl ::kernel::init::__internal::PinData for __ThePinData {
374//!         type Datee = Foo;
375//!     }
376//!     #[allow(dead_code)]
377//!     struct __Unpin<'__pin> {
378//!         __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
379//!         __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,
380//!         b: Bar<u32>,
381//!     }
382//!     #[doc(hidden)]
383//!     impl<'__pin> ::core::marker::Unpin for Foo
384//!     where
385//!         __Unpin<'__pin>: ::core::marker::Unpin,
386//!     {}
387//!     // Since we specified `PinnedDrop` as the argument to `#[pin_data]`, we expect `Foo` to
388//!     // implement `PinnedDrop`. Thus we do not need to prevent `Drop` implementations like
389//!     // before, instead we implement `Drop` here and delegate to `PinnedDrop`.
390//!     impl ::core::ops::Drop for Foo {
391//!         fn drop(&mut self) {
392//!             // Since we are getting dropped, no one else has a reference to `self` and thus we
393//!             // can assume that we never move.
394//!             let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };
395//!             // Create the unsafe token that proves that we are inside of a destructor, this
396//!             // type is only allowed to be created in a destructor.
397//!             let token = unsafe { ::kernel::init::__internal::OnlyCallFromDrop::new() };
398//!             ::kernel::init::PinnedDrop::drop(pinned, token);
399//!         }
400//!     }
401//! };
402//! ```
403//!
404//! ## `#[pinned_drop]` on `impl PinnedDrop for Foo`
405//!
406//! This macro is used to implement the `PinnedDrop` trait, since that trait is `unsafe` and has an
407//! extra parameter that should not be used at all. The macro hides that parameter.
408//!
409//! Here is the `PinnedDrop` impl for `Foo`:
410//!
411//! ```rust,ignore
412//! #[pinned_drop]
413//! impl PinnedDrop for Foo {
414//!     fn drop(self: Pin<&mut Self>) {
415//!         pr_info!("{self:p} is getting dropped.\n");
416//!     }
417//! }
418//! ```
419//!
420//! This expands to the following code:
421//!
422//! ```rust,ignore
423//! // `unsafe`, full path and the token parameter are added, everything else stays the same.
424//! unsafe impl ::kernel::init::PinnedDrop for Foo {
425//!     fn drop(self: Pin<&mut Self>, _: ::kernel::init::__internal::OnlyCallFromDrop) {
426//!         pr_info!("{self:p} is getting dropped.\n");
427//!     }
428//! }
429//! ```
430//!
431//! ## `pin_init!` on `Foo`
432//!
433//! Since we already took a look at `pin_init!` on `Bar`, this section will only show the expansion
434//! of `pin_init!` on `Foo`:
435//!
436//! ```rust,ignore
437//! let a = 42;
438//! let initializer = pin_init!(Foo {
439//!     a,
440//!     b <- Bar::new(36),
441//! });
442//! ```
443//!
444//! This expands to the following code:
445//!
446//! ```rust,ignore
447//! let a = 42;
448//! let initializer = {
449//!     struct __InitOk;
450//!     let data = unsafe {
451//!         use ::kernel::init::__internal::HasPinData;
452//!         Foo::__pin_data()
453//!     };
454//!     let init = ::kernel::init::__internal::PinData::make_closure::<
455//!         _,
456//!         __InitOk,
457//!         ::core::convert::Infallible,
458//!     >(data, move |slot| {
459//!         {
460//!             struct __InitOk;
461//!             {
462//!                 unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).a), a) };
463//!             }
464//!             let __a_guard = unsafe {
465//!                 ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).a))
466//!             };
467//!             let init = Bar::new(36);
468//!             unsafe { data.b(::core::addr_of_mut!((*slot).b), b)? };
469//!             let __b_guard = unsafe {
470//!                 ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).b))
471//!             };
472//!             ::core::mem::forget(__b_guard);
473//!             ::core::mem::forget(__a_guard);
474//!             #[allow(unreachable_code, clippy::diverging_sub_expression)]
475//!             let _ = || {
476//!                 unsafe {
477//!                     ::core::ptr::write(
478//!                         slot,
479//!                         Foo {
480//!                             a: ::core::panic!(),
481//!                             b: ::core::panic!(),
482//!                         },
483//!                     );
484//!                 };
485//!             };
486//!         }
487//!         Ok(__InitOk)
488//!     });
489//!     let init = move |
490//!         slot,
491//!     | -> ::core::result::Result<(), ::core::convert::Infallible> {
492//!         init(slot).map(|__InitOk| ())
493//!     };
494//!     let init = unsafe {
495//!         ::kernel::init::pin_init_from_closure::<_, ::core::convert::Infallible>(init)
496//!     };
497//!     init
498//! };
499//! ```
500
501/// Creates a `unsafe impl<...> PinnedDrop for $type` block.
502///
503/// See [`PinnedDrop`] for more information.
504#[doc(hidden)]
505#[macro_export]
506macro_rules! __pinned_drop {
507    (
508        @impl_sig($($impl_sig:tt)*),
509        @impl_body(
510            $(#[$($attr:tt)*])*
511            fn drop($($sig:tt)*) {
512                $($inner:tt)*
513            }
514        ),
515    ) => {
516        // SAFETY: TODO.
517        unsafe $($impl_sig)* {
518            // Inherit all attributes and the type/ident tokens for the signature.
519            $(#[$($attr)*])*
520            fn drop($($sig)*, _: $crate::init::__internal::OnlyCallFromDrop) {
521                $($inner)*
522            }
523        }
524    }
525}
526
527/// This macro first parses the struct definition such that it separates pinned and not pinned
528/// fields. Afterwards it declares the struct and implement the `PinData` trait safely.
529#[doc(hidden)]
530#[macro_export]
531macro_rules! __pin_data {
532    // Proc-macro entry point, this is supplied by the proc-macro pre-parsing.
533    (parse_input:
534        @args($($pinned_drop:ident)?),
535        @sig(
536            $(#[$($struct_attr:tt)*])*
537            $vis:vis struct $name:ident
538            $(where $($whr:tt)*)?
539        ),
540        @impl_generics($($impl_generics:tt)*),
541        @ty_generics($($ty_generics:tt)*),
542        @decl_generics($($decl_generics:tt)*),
543        @body({ $($fields:tt)* }),
544    ) => {
545        // We now use token munching to iterate through all of the fields. While doing this we
546        // identify fields marked with `#[pin]`, these fields are the 'pinned fields'. The user
547        // wants these to be structurally pinned. The rest of the fields are the
548        // 'not pinned fields'. Additionally we collect all fields, since we need them in the right
549        // order to declare the struct.
550        //
551        // In this call we also put some explaining comments for the parameters.
552        $crate::__pin_data!(find_pinned_fields:
553            // Attributes on the struct itself, these will just be propagated to be put onto the
554            // struct definition.
555            @struct_attrs($(#[$($struct_attr)*])*),
556            // The visibility of the struct.
557            @vis($vis),
558            // The name of the struct.
559            @name($name),
560            // The 'impl generics', the generics that will need to be specified on the struct inside
561            // of an `impl<$ty_generics>` block.
562            @impl_generics($($impl_generics)*),
563            // The 'ty generics', the generics that will need to be specified on the impl blocks.
564            @ty_generics($($ty_generics)*),
565            // The 'decl generics', the generics that need to be specified on the struct
566            // definition.
567            @decl_generics($($decl_generics)*),
568            // The where clause of any impl block and the declaration.
569            @where($($($whr)*)?),
570            // The remaining fields tokens that need to be processed.
571            // We add a `,` at the end to ensure correct parsing.
572            @fields_munch($($fields)* ,),
573            // The pinned fields.
574            @pinned(),
575            // The not pinned fields.
576            @not_pinned(),
577            // All fields.
578            @fields(),
579            // The accumulator containing all attributes already parsed.
580            @accum(),
581            // Contains `yes` or `` to indicate if `#[pin]` was found on the current field.
582            @is_pinned(),
583            // The proc-macro argument, this should be `PinnedDrop` or ``.
584            @pinned_drop($($pinned_drop)?),
585        );
586    };
587    (find_pinned_fields:
588        @struct_attrs($($struct_attrs:tt)*),
589        @vis($vis:vis),
590        @name($name:ident),
591        @impl_generics($($impl_generics:tt)*),
592        @ty_generics($($ty_generics:tt)*),
593        @decl_generics($($decl_generics:tt)*),
594        @where($($whr:tt)*),
595        // We found a PhantomPinned field, this should generally be pinned!
596        @fields_munch($field:ident : $($($(::)?core::)?marker::)?PhantomPinned, $($rest:tt)*),
597        @pinned($($pinned:tt)*),
598        @not_pinned($($not_pinned:tt)*),
599        @fields($($fields:tt)*),
600        @accum($($accum:tt)*),
601        // This field is not pinned.
602        @is_pinned(),
603        @pinned_drop($($pinned_drop:ident)?),
604    ) => {
605        ::core::compile_error!(concat!(
606            "The field `",
607            stringify!($field),
608            "` of type `PhantomPinned` only has an effect, if it has the `#[pin]` attribute.",
609        ));
610        $crate::__pin_data!(find_pinned_fields:
611            @struct_attrs($($struct_attrs)*),
612            @vis($vis),
613            @name($name),
614            @impl_generics($($impl_generics)*),
615            @ty_generics($($ty_generics)*),
616            @decl_generics($($decl_generics)*),
617            @where($($whr)*),
618            @fields_munch($($rest)*),
619            @pinned($($pinned)* $($accum)* $field: ::core::marker::PhantomPinned,),
620            @not_pinned($($not_pinned)*),
621            @fields($($fields)* $($accum)* $field: ::core::marker::PhantomPinned,),
622            @accum(),
623            @is_pinned(),
624            @pinned_drop($($pinned_drop)?),
625        );
626    };
627    (find_pinned_fields:
628        @struct_attrs($($struct_attrs:tt)*),
629        @vis($vis:vis),
630        @name($name:ident),
631        @impl_generics($($impl_generics:tt)*),
632        @ty_generics($($ty_generics:tt)*),
633        @decl_generics($($decl_generics:tt)*),
634        @where($($whr:tt)*),
635        // We reached the field declaration.
636        @fields_munch($field:ident : $type:ty, $($rest:tt)*),
637        @pinned($($pinned:tt)*),
638        @not_pinned($($not_pinned:tt)*),
639        @fields($($fields:tt)*),
640        @accum($($accum:tt)*),
641        // This field is pinned.
642        @is_pinned(yes),
643        @pinned_drop($($pinned_drop:ident)?),
644    ) => {
645        $crate::__pin_data!(find_pinned_fields:
646            @struct_attrs($($struct_attrs)*),
647            @vis($vis),
648            @name($name),
649            @impl_generics($($impl_generics)*),
650            @ty_generics($($ty_generics)*),
651            @decl_generics($($decl_generics)*),
652            @where($($whr)*),
653            @fields_munch($($rest)*),
654            @pinned($($pinned)* $($accum)* $field: $type,),
655            @not_pinned($($not_pinned)*),
656            @fields($($fields)* $($accum)* $field: $type,),
657            @accum(),
658            @is_pinned(),
659            @pinned_drop($($pinned_drop)?),
660        );
661    };
662    (find_pinned_fields:
663        @struct_attrs($($struct_attrs:tt)*),
664        @vis($vis:vis),
665        @name($name:ident),
666        @impl_generics($($impl_generics:tt)*),
667        @ty_generics($($ty_generics:tt)*),
668        @decl_generics($($decl_generics:tt)*),
669        @where($($whr:tt)*),
670        // We reached the field declaration.
671        @fields_munch($field:ident : $type:ty, $($rest:tt)*),
672        @pinned($($pinned:tt)*),
673        @not_pinned($($not_pinned:tt)*),
674        @fields($($fields:tt)*),
675        @accum($($accum:tt)*),
676        // This field is not pinned.
677        @is_pinned(),
678        @pinned_drop($($pinned_drop:ident)?),
679    ) => {
680        $crate::__pin_data!(find_pinned_fields:
681            @struct_attrs($($struct_attrs)*),
682            @vis($vis),
683            @name($name),
684            @impl_generics($($impl_generics)*),
685            @ty_generics($($ty_generics)*),
686            @decl_generics($($decl_generics)*),
687            @where($($whr)*),
688            @fields_munch($($rest)*),
689            @pinned($($pinned)*),
690            @not_pinned($($not_pinned)* $($accum)* $field: $type,),
691            @fields($($fields)* $($accum)* $field: $type,),
692            @accum(),
693            @is_pinned(),
694            @pinned_drop($($pinned_drop)?),
695        );
696    };
697    (find_pinned_fields:
698        @struct_attrs($($struct_attrs:tt)*),
699        @vis($vis:vis),
700        @name($name:ident),
701        @impl_generics($($impl_generics:tt)*),
702        @ty_generics($($ty_generics:tt)*),
703        @decl_generics($($decl_generics:tt)*),
704        @where($($whr:tt)*),
705        // We found the `#[pin]` attr.
706        @fields_munch(#[pin] $($rest:tt)*),
707        @pinned($($pinned:tt)*),
708        @not_pinned($($not_pinned:tt)*),
709        @fields($($fields:tt)*),
710        @accum($($accum:tt)*),
711        @is_pinned($($is_pinned:ident)?),
712        @pinned_drop($($pinned_drop:ident)?),
713    ) => {
714        $crate::__pin_data!(find_pinned_fields:
715            @struct_attrs($($struct_attrs)*),
716            @vis($vis),
717            @name($name),
718            @impl_generics($($impl_generics)*),
719            @ty_generics($($ty_generics)*),
720            @decl_generics($($decl_generics)*),
721            @where($($whr)*),
722            @fields_munch($($rest)*),
723            // We do not include `#[pin]` in the list of attributes, since it is not actually an
724            // attribute that is defined somewhere.
725            @pinned($($pinned)*),
726            @not_pinned($($not_pinned)*),
727            @fields($($fields)*),
728            @accum($($accum)*),
729            // Set this to `yes`.
730            @is_pinned(yes),
731            @pinned_drop($($pinned_drop)?),
732        );
733    };
734    (find_pinned_fields:
735        @struct_attrs($($struct_attrs:tt)*),
736        @vis($vis:vis),
737        @name($name:ident),
738        @impl_generics($($impl_generics:tt)*),
739        @ty_generics($($ty_generics:tt)*),
740        @decl_generics($($decl_generics:tt)*),
741        @where($($whr:tt)*),
742        // We reached the field declaration with visibility, for simplicity we only munch the
743        // visibility and put it into `$accum`.
744        @fields_munch($fvis:vis $field:ident $($rest:tt)*),
745        @pinned($($pinned:tt)*),
746        @not_pinned($($not_pinned:tt)*),
747        @fields($($fields:tt)*),
748        @accum($($accum:tt)*),
749        @is_pinned($($is_pinned:ident)?),
750        @pinned_drop($($pinned_drop:ident)?),
751    ) => {
752        $crate::__pin_data!(find_pinned_fields:
753            @struct_attrs($($struct_attrs)*),
754            @vis($vis),
755            @name($name),
756            @impl_generics($($impl_generics)*),
757            @ty_generics($($ty_generics)*),
758            @decl_generics($($decl_generics)*),
759            @where($($whr)*),
760            @fields_munch($field $($rest)*),
761            @pinned($($pinned)*),
762            @not_pinned($($not_pinned)*),
763            @fields($($fields)*),
764            @accum($($accum)* $fvis),
765            @is_pinned($($is_pinned)?),
766            @pinned_drop($($pinned_drop)?),
767        );
768    };
769    (find_pinned_fields:
770        @struct_attrs($($struct_attrs:tt)*),
771        @vis($vis:vis),
772        @name($name:ident),
773        @impl_generics($($impl_generics:tt)*),
774        @ty_generics($($ty_generics:tt)*),
775        @decl_generics($($decl_generics:tt)*),
776        @where($($whr:tt)*),
777        // Some other attribute, just put it into `$accum`.
778        @fields_munch(#[$($attr:tt)*] $($rest:tt)*),
779        @pinned($($pinned:tt)*),
780        @not_pinned($($not_pinned:tt)*),
781        @fields($($fields:tt)*),
782        @accum($($accum:tt)*),
783        @is_pinned($($is_pinned:ident)?),
784        @pinned_drop($($pinned_drop:ident)?),
785    ) => {
786        $crate::__pin_data!(find_pinned_fields:
787            @struct_attrs($($struct_attrs)*),
788            @vis($vis),
789            @name($name),
790            @impl_generics($($impl_generics)*),
791            @ty_generics($($ty_generics)*),
792            @decl_generics($($decl_generics)*),
793            @where($($whr)*),
794            @fields_munch($($rest)*),
795            @pinned($($pinned)*),
796            @not_pinned($($not_pinned)*),
797            @fields($($fields)*),
798            @accum($($accum)* #[$($attr)*]),
799            @is_pinned($($is_pinned)?),
800            @pinned_drop($($pinned_drop)?),
801        );
802    };
803    (find_pinned_fields:
804        @struct_attrs($($struct_attrs:tt)*),
805        @vis($vis:vis),
806        @name($name:ident),
807        @impl_generics($($impl_generics:tt)*),
808        @ty_generics($($ty_generics:tt)*),
809        @decl_generics($($decl_generics:tt)*),
810        @where($($whr:tt)*),
811        // We reached the end of the fields, plus an optional additional comma, since we added one
812        // before and the user is also allowed to put a trailing comma.
813        @fields_munch($(,)?),
814        @pinned($($pinned:tt)*),
815        @not_pinned($($not_pinned:tt)*),
816        @fields($($fields:tt)*),
817        @accum(),
818        @is_pinned(),
819        @pinned_drop($($pinned_drop:ident)?),
820    ) => {
821        // Declare the struct with all fields in the correct order.
822        $($struct_attrs)*
823        $vis struct $name <$($decl_generics)*>
824        where $($whr)*
825        {
826            $($fields)*
827        }
828
829        // We put the rest into this const item, because it then will not be accessible to anything
830        // outside.
831        const _: () = {
832            // We declare this struct which will host all of the projection function for our type.
833            // it will be invariant over all generic parameters which are inherited from the
834            // struct.
835            $vis struct __ThePinData<$($impl_generics)*>
836            where $($whr)*
837            {
838                __phantom: ::core::marker::PhantomData<
839                    fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>
840                >,
841            }
842
843            impl<$($impl_generics)*> ::core::clone::Clone for __ThePinData<$($ty_generics)*>
844            where $($whr)*
845            {
846                fn clone(&self) -> Self { *self }
847            }
848
849            impl<$($impl_generics)*> ::core::marker::Copy for __ThePinData<$($ty_generics)*>
850            where $($whr)*
851            {}
852
853            // Make all projection functions.
854            $crate::__pin_data!(make_pin_data:
855                @pin_data(__ThePinData),
856                @impl_generics($($impl_generics)*),
857                @ty_generics($($ty_generics)*),
858                @where($($whr)*),
859                @pinned($($pinned)*),
860                @not_pinned($($not_pinned)*),
861            );
862
863            // SAFETY: We have added the correct projection functions above to `__ThePinData` and
864            // we also use the least restrictive generics possible.
865            unsafe impl<$($impl_generics)*>
866                $crate::init::__internal::HasPinData for $name<$($ty_generics)*>
867            where $($whr)*
868            {
869                type PinData = __ThePinData<$($ty_generics)*>;
870
871                unsafe fn __pin_data() -> Self::PinData {
872                    __ThePinData { __phantom: ::core::marker::PhantomData }
873                }
874            }
875
876            // SAFETY: TODO.
877            unsafe impl<$($impl_generics)*>
878                $crate::init::__internal::PinData for __ThePinData<$($ty_generics)*>
879            where $($whr)*
880            {
881                type Datee = $name<$($ty_generics)*>;
882            }
883
884            // This struct will be used for the unpin analysis. Since only structurally pinned
885            // fields are relevant whether the struct should implement `Unpin`.
886            #[allow(dead_code)]
887            struct __Unpin <'__pin, $($impl_generics)*>
888            where $($whr)*
889            {
890                __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
891                __phantom: ::core::marker::PhantomData<
892                    fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>
893                >,
894                // Only the pinned fields.
895                $($pinned)*
896            }
897
898            #[doc(hidden)]
899            impl<'__pin, $($impl_generics)*> ::core::marker::Unpin for $name<$($ty_generics)*>
900            where
901                __Unpin<'__pin, $($ty_generics)*>: ::core::marker::Unpin,
902                $($whr)*
903            {}
904
905            // We need to disallow normal `Drop` implementation, the exact behavior depends on
906            // whether `PinnedDrop` was specified as the parameter.
907            $crate::__pin_data!(drop_prevention:
908                @name($name),
909                @impl_generics($($impl_generics)*),
910                @ty_generics($($ty_generics)*),
911                @where($($whr)*),
912                @pinned_drop($($pinned_drop)?),
913            );
914        };
915    };
916    // When no `PinnedDrop` was specified, then we have to prevent implementing drop.
917    (drop_prevention:
918        @name($name:ident),
919        @impl_generics($($impl_generics:tt)*),
920        @ty_generics($($ty_generics:tt)*),
921        @where($($whr:tt)*),
922        @pinned_drop(),
923    ) => {
924        // We prevent this by creating a trait that will be implemented for all types implementing
925        // `Drop`. Additionally we will implement this trait for the struct leading to a conflict,
926        // if it also implements `Drop`
927        trait MustNotImplDrop {}
928        #[expect(drop_bounds)]
929        impl<T: ::core::ops::Drop> MustNotImplDrop for T {}
930        impl<$($impl_generics)*> MustNotImplDrop for $name<$($ty_generics)*>
931        where $($whr)* {}
932        // We also take care to prevent users from writing a useless `PinnedDrop` implementation.
933        // They might implement `PinnedDrop` correctly for the struct, but forget to give
934        // `PinnedDrop` as the parameter to `#[pin_data]`.
935        #[expect(non_camel_case_types)]
936        trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
937        impl<T: $crate::init::PinnedDrop>
938            UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
939        impl<$($impl_generics)*>
940            UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for $name<$($ty_generics)*>
941        where $($whr)* {}
942    };
943    // When `PinnedDrop` was specified we just implement `Drop` and delegate.
944    (drop_prevention:
945        @name($name:ident),
946        @impl_generics($($impl_generics:tt)*),
947        @ty_generics($($ty_generics:tt)*),
948        @where($($whr:tt)*),
949        @pinned_drop(PinnedDrop),
950    ) => {
951        impl<$($impl_generics)*> ::core::ops::Drop for $name<$($ty_generics)*>
952        where $($whr)*
953        {
954            fn drop(&mut self) {
955                // SAFETY: Since this is a destructor, `self` will not move after this function
956                // terminates, since it is inaccessible.
957                let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };
958                // SAFETY: Since this is a drop function, we can create this token to call the
959                // pinned destructor of this type.
960                let token = unsafe { $crate::init::__internal::OnlyCallFromDrop::new() };
961                $crate::init::PinnedDrop::drop(pinned, token);
962            }
963        }
964    };
965    // If some other parameter was specified, we emit a readable error.
966    (drop_prevention:
967        @name($name:ident),
968        @impl_generics($($impl_generics:tt)*),
969        @ty_generics($($ty_generics:tt)*),
970        @where($($whr:tt)*),
971        @pinned_drop($($rest:tt)*),
972    ) => {
973        compile_error!(
974            "Wrong parameters to `#[pin_data]`, expected nothing or `PinnedDrop`, got '{}'.",
975            stringify!($($rest)*),
976        );
977    };
978    (make_pin_data:
979        @pin_data($pin_data:ident),
980        @impl_generics($($impl_generics:tt)*),
981        @ty_generics($($ty_generics:tt)*),
982        @where($($whr:tt)*),
983        @pinned($($(#[$($p_attr:tt)*])* $pvis:vis $p_field:ident : $p_type:ty),* $(,)?),
984        @not_pinned($($(#[$($attr:tt)*])* $fvis:vis $field:ident : $type:ty),* $(,)?),
985    ) => {
986        // For every field, we create a projection function according to its projection type. If a
987        // field is structurally pinned, then it must be initialized via `PinInit`, if it is not
988        // structurally pinned, then it can be initialized via `Init`.
989        //
990        // The functions are `unsafe` to prevent accidentally calling them.
991        #[allow(dead_code)]
992        #[expect(clippy::missing_safety_doc)]
993        impl<$($impl_generics)*> $pin_data<$($ty_generics)*>
994        where $($whr)*
995        {
996            $(
997                $(#[$($p_attr)*])*
998                $pvis unsafe fn $p_field<E>(
999                    self,
1000                    slot: *mut $p_type,
1001                    init: impl $crate::init::PinInit<$p_type, E>,
1002                ) -> ::core::result::Result<(), E> {
1003                    // SAFETY: TODO.
1004                    unsafe { $crate::init::PinInit::__pinned_init(init, slot) }
1005                }
1006            )*
1007            $(
1008                $(#[$($attr)*])*
1009                $fvis unsafe fn $field<E>(
1010                    self,
1011                    slot: *mut $type,
1012                    init: impl $crate::init::Init<$type, E>,
1013                ) -> ::core::result::Result<(), E> {
1014                    // SAFETY: TODO.
1015                    unsafe { $crate::init::Init::__init(init, slot) }
1016                }
1017            )*
1018        }
1019    };
1020}
1021
1022/// The internal init macro. Do not call manually!
1023///
1024/// This is called by the `{try_}{pin_}init!` macros with various inputs.
1025///
1026/// This macro has multiple internal call configurations, these are always the very first ident:
1027/// - nothing: this is the base case and called by the `{try_}{pin_}init!` macros.
1028/// - `with_update_parsed`: when the `..Zeroable::zeroed()` syntax has been handled.
1029/// - `init_slot`: recursively creates the code that initializes all fields in `slot`.
1030/// - `make_initializer`: recursively create the struct initializer that guarantees that every
1031///   field has been initialized exactly once.
1032#[doc(hidden)]
1033#[macro_export]
1034macro_rules! __init_internal {
1035    (
1036        @this($($this:ident)?),
1037        @typ($t:path),
1038        @fields($($fields:tt)*),
1039        @error($err:ty),
1040        // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1041        // case.
1042        @data($data:ident, $($use_data:ident)?),
1043        // `HasPinData` or `HasInitData`.
1044        @has_data($has_data:ident, $get_data:ident),
1045        // `pin_init_from_closure` or `init_from_closure`.
1046        @construct_closure($construct_closure:ident),
1047        @munch_fields(),
1048    ) => {
1049        $crate::__init_internal!(with_update_parsed:
1050            @this($($this)?),
1051            @typ($t),
1052            @fields($($fields)*),
1053            @error($err),
1054            @data($data, $($use_data)?),
1055            @has_data($has_data, $get_data),
1056            @construct_closure($construct_closure),
1057            @zeroed(), // Nothing means default behavior.
1058        )
1059    };
1060    (
1061        @this($($this:ident)?),
1062        @typ($t:path),
1063        @fields($($fields:tt)*),
1064        @error($err:ty),
1065        // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1066        // case.
1067        @data($data:ident, $($use_data:ident)?),
1068        // `HasPinData` or `HasInitData`.
1069        @has_data($has_data:ident, $get_data:ident),
1070        // `pin_init_from_closure` or `init_from_closure`.
1071        @construct_closure($construct_closure:ident),
1072        @munch_fields(..Zeroable::zeroed()),
1073    ) => {
1074        $crate::__init_internal!(with_update_parsed:
1075            @this($($this)?),
1076            @typ($t),
1077            @fields($($fields)*),
1078            @error($err),
1079            @data($data, $($use_data)?),
1080            @has_data($has_data, $get_data),
1081            @construct_closure($construct_closure),
1082            @zeroed(()), // `()` means zero all fields not mentioned.
1083        )
1084    };
1085    (
1086        @this($($this:ident)?),
1087        @typ($t:path),
1088        @fields($($fields:tt)*),
1089        @error($err:ty),
1090        // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1091        // case.
1092        @data($data:ident, $($use_data:ident)?),
1093        // `HasPinData` or `HasInitData`.
1094        @has_data($has_data:ident, $get_data:ident),
1095        // `pin_init_from_closure` or `init_from_closure`.
1096        @construct_closure($construct_closure:ident),
1097        @munch_fields($ignore:tt $($rest:tt)*),
1098    ) => {
1099        $crate::__init_internal!(
1100            @this($($this)?),
1101            @typ($t),
1102            @fields($($fields)*),
1103            @error($err),
1104            @data($data, $($use_data)?),
1105            @has_data($has_data, $get_data),
1106            @construct_closure($construct_closure),
1107            @munch_fields($($rest)*),
1108        )
1109    };
1110    (with_update_parsed:
1111        @this($($this:ident)?),
1112        @typ($t:path),
1113        @fields($($fields:tt)*),
1114        @error($err:ty),
1115        // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1116        // case.
1117        @data($data:ident, $($use_data:ident)?),
1118        // `HasPinData` or `HasInitData`.
1119        @has_data($has_data:ident, $get_data:ident),
1120        // `pin_init_from_closure` or `init_from_closure`.
1121        @construct_closure($construct_closure:ident),
1122        @zeroed($($init_zeroed:expr)?),
1123    ) => {{
1124        // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return
1125        // type and shadow it later when we insert the arbitrary user code. That way there will be
1126        // no possibility of returning without `unsafe`.
1127        struct __InitOk;
1128        // Get the data about fields from the supplied type.
1129        //
1130        // SAFETY: TODO.
1131        let data = unsafe {
1132            use $crate::init::__internal::$has_data;
1133            // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
1134            // information that is associated to already parsed fragments, so a path fragment
1135            // cannot be used in this position. Doing the retokenization results in valid rust
1136            // code.
1137            ::kernel::macros::paste!($t::$get_data())
1138        };
1139        // Ensure that `data` really is of type `$data` and help with type inference:
1140        let init = $crate::init::__internal::$data::make_closure::<_, __InitOk, $err>(
1141            data,
1142            move |slot| {
1143                {
1144                    // Shadow the structure so it cannot be used to return early.
1145                    struct __InitOk;
1146                    // If `$init_zeroed` is present we should zero the slot now and not emit an
1147                    // error when fields are missing (since they will be zeroed). We also have to
1148                    // check that the type actually implements `Zeroable`.
1149                    $({
1150                        fn assert_zeroable<T: $crate::init::Zeroable>(_: *mut T) {}
1151                        // Ensure that the struct is indeed `Zeroable`.
1152                        assert_zeroable(slot);
1153                        // SAFETY: The type implements `Zeroable` by the check above.
1154                        unsafe { ::core::ptr::write_bytes(slot, 0, 1) };
1155                        $init_zeroed // This will be `()` if set.
1156                    })?
1157                    // Create the `this` so it can be referenced by the user inside of the
1158                    // expressions creating the individual fields.
1159                    $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)?
1160                    // Initialize every field.
1161                    $crate::__init_internal!(init_slot($($use_data)?):
1162                        @data(data),
1163                        @slot(slot),
1164                        @guards(),
1165                        @munch_fields($($fields)*,),
1166                    );
1167                    // We use unreachable code to ensure that all fields have been mentioned exactly
1168                    // once, this struct initializer will still be type-checked and complain with a
1169                    // very natural error message if a field is forgotten/mentioned more than once.
1170                    #[allow(unreachable_code, clippy::diverging_sub_expression)]
1171                    let _ = || {
1172                        $crate::__init_internal!(make_initializer:
1173                            @slot(slot),
1174                            @type_name($t),
1175                            @munch_fields($($fields)*,),
1176                            @acc(),
1177                        );
1178                    };
1179                }
1180                Ok(__InitOk)
1181            }
1182        );
1183        let init = move |slot| -> ::core::result::Result<(), $err> {
1184            init(slot).map(|__InitOk| ())
1185        };
1186        // SAFETY: TODO.
1187        let init = unsafe { $crate::init::$construct_closure::<_, $err>(init) };
1188        init
1189    }};
1190    (init_slot($($use_data:ident)?):
1191        @data($data:ident),
1192        @slot($slot:ident),
1193        @guards($($guards:ident,)*),
1194        @munch_fields($(..Zeroable::zeroed())? $(,)?),
1195    ) => {
1196        // Endpoint of munching, no fields are left. If execution reaches this point, all fields
1197        // have been initialized. Therefore we can now dismiss the guards by forgetting them.
1198        $(::core::mem::forget($guards);)*
1199    };
1200    (init_slot($use_data:ident): // `use_data` is present, so we use the `data` to init fields.
1201        @data($data:ident),
1202        @slot($slot:ident),
1203        @guards($($guards:ident,)*),
1204        // In-place initialization syntax.
1205        @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1206    ) => {
1207        let init = $val;
1208        // Call the initializer.
1209        //
1210        // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
1211        // return when an error/panic occurs.
1212        // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`.
1213        unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), init)? };
1214        // Create the drop guard:
1215        //
1216        // We rely on macro hygiene to make it impossible for users to access this local variable.
1217        // We use `paste!` to create new hygiene for `$field`.
1218        ::kernel::macros::paste! {
1219            // SAFETY: We forget the guard later when initialization has succeeded.
1220            let [< __ $field _guard >] = unsafe {
1221                $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
1222            };
1223
1224            $crate::__init_internal!(init_slot($use_data):
1225                @data($data),
1226                @slot($slot),
1227                @guards([< __ $field _guard >], $($guards,)*),
1228                @munch_fields($($rest)*),
1229            );
1230        }
1231    };
1232    (init_slot(): // No `use_data`, so we use `Init::__init` directly.
1233        @data($data:ident),
1234        @slot($slot:ident),
1235        @guards($($guards:ident,)*),
1236        // In-place initialization syntax.
1237        @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1238    ) => {
1239        let init = $val;
1240        // Call the initializer.
1241        //
1242        // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
1243        // return when an error/panic occurs.
1244        unsafe { $crate::init::Init::__init(init, ::core::ptr::addr_of_mut!((*$slot).$field))? };
1245        // Create the drop guard:
1246        //
1247        // We rely on macro hygiene to make it impossible for users to access this local variable.
1248        // We use `paste!` to create new hygiene for `$field`.
1249        ::kernel::macros::paste! {
1250            // SAFETY: We forget the guard later when initialization has succeeded.
1251            let [< __ $field _guard >] = unsafe {
1252                $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
1253            };
1254
1255            $crate::__init_internal!(init_slot():
1256                @data($data),
1257                @slot($slot),
1258                @guards([< __ $field _guard >], $($guards,)*),
1259                @munch_fields($($rest)*),
1260            );
1261        }
1262    };
1263    (init_slot($($use_data:ident)?):
1264        @data($data:ident),
1265        @slot($slot:ident),
1266        @guards($($guards:ident,)*),
1267        // Init by-value.
1268        @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
1269    ) => {
1270        {
1271            $(let $field = $val;)?
1272            // Initialize the field.
1273            //
1274            // SAFETY: The memory at `slot` is uninitialized.
1275            unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) };
1276        }
1277        // Create the drop guard:
1278        //
1279        // We rely on macro hygiene to make it impossible for users to access this local variable.
1280        // We use `paste!` to create new hygiene for `$field`.
1281        ::kernel::macros::paste! {
1282            // SAFETY: We forget the guard later when initialization has succeeded.
1283            let [< __ $field _guard >] = unsafe {
1284                $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
1285            };
1286
1287            $crate::__init_internal!(init_slot($($use_data)?):
1288                @data($data),
1289                @slot($slot),
1290                @guards([< __ $field _guard >], $($guards,)*),
1291                @munch_fields($($rest)*),
1292            );
1293        }
1294    };
1295    (make_initializer:
1296        @slot($slot:ident),
1297        @type_name($t:path),
1298        @munch_fields(..Zeroable::zeroed() $(,)?),
1299        @acc($($acc:tt)*),
1300    ) => {
1301        // Endpoint, nothing more to munch, create the initializer. Since the users specified
1302        // `..Zeroable::zeroed()`, the slot will already have been zeroed and all field that have
1303        // not been overwritten are thus zero and initialized. We still check that all fields are
1304        // actually accessible by using the struct update syntax ourselves.
1305        // We are inside of a closure that is never executed and thus we can abuse `slot` to
1306        // get the correct type inference here:
1307        #[allow(unused_assignments)]
1308        unsafe {
1309            let mut zeroed = ::core::mem::zeroed();
1310            // We have to use type inference here to make zeroed have the correct type. This does
1311            // not get executed, so it has no effect.
1312            ::core::ptr::write($slot, zeroed);
1313            zeroed = ::core::mem::zeroed();
1314            // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
1315            // information that is associated to already parsed fragments, so a path fragment
1316            // cannot be used in this position. Doing the retokenization results in valid rust
1317            // code.
1318            ::kernel::macros::paste!(
1319                ::core::ptr::write($slot, $t {
1320                    $($acc)*
1321                    ..zeroed
1322                });
1323            );
1324        }
1325    };
1326    (make_initializer:
1327        @slot($slot:ident),
1328        @type_name($t:path),
1329        @munch_fields($(,)?),
1330        @acc($($acc:tt)*),
1331    ) => {
1332        // Endpoint, nothing more to munch, create the initializer.
1333        // Since we are in the closure that is never called, this will never get executed.
1334        // We abuse `slot` to get the correct type inference here:
1335        //
1336        // SAFETY: TODO.
1337        unsafe {
1338            // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
1339            // information that is associated to already parsed fragments, so a path fragment
1340            // cannot be used in this position. Doing the retokenization results in valid rust
1341            // code.
1342            ::kernel::macros::paste!(
1343                ::core::ptr::write($slot, $t {
1344                    $($acc)*
1345                });
1346            );
1347        }
1348    };
1349    (make_initializer:
1350        @slot($slot:ident),
1351        @type_name($t:path),
1352        @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1353        @acc($($acc:tt)*),
1354    ) => {
1355        $crate::__init_internal!(make_initializer:
1356            @slot($slot),
1357            @type_name($t),
1358            @munch_fields($($rest)*),
1359            @acc($($acc)* $field: ::core::panic!(),),
1360        );
1361    };
1362    (make_initializer:
1363        @slot($slot:ident),
1364        @type_name($t:path),
1365        @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
1366        @acc($($acc:tt)*),
1367    ) => {
1368        $crate::__init_internal!(make_initializer:
1369            @slot($slot),
1370            @type_name($t),
1371            @munch_fields($($rest)*),
1372            @acc($($acc)* $field: ::core::panic!(),),
1373        );
1374    };
1375}
1376
1377#[doc(hidden)]
1378#[macro_export]
1379macro_rules! __derive_zeroable {
1380    (parse_input:
1381        @sig(
1382            $(#[$($struct_attr:tt)*])*
1383            $vis:vis struct $name:ident
1384            $(where $($whr:tt)*)?
1385        ),
1386        @impl_generics($($impl_generics:tt)*),
1387        @ty_generics($($ty_generics:tt)*),
1388        @body({
1389            $(
1390                $(#[$($field_attr:tt)*])*
1391                $field:ident : $field_ty:ty
1392            ),* $(,)?
1393        }),
1394    ) => {
1395        // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero.
1396        #[automatically_derived]
1397        unsafe impl<$($impl_generics)*> $crate::init::Zeroable for $name<$($ty_generics)*>
1398        where
1399            $($($whr)*)?
1400        {}
1401        const _: () = {
1402            fn assert_zeroable<T: ?::core::marker::Sized + $crate::init::Zeroable>() {}
1403            fn ensure_zeroable<$($impl_generics)*>()
1404                where $($($whr)*)?
1405            {
1406                $(assert_zeroable::<$field_ty>();)*
1407            }
1408        };
1409    };
1410}