1use core::mem::{ManuallyDrop, MaybeUninit};
6use core::ptr::addr_of_mut;
7
8use munge::munge;
9
10use crate::decoder::InternalHandleDecoder;
11use crate::encoder::InternalHandleEncoder;
12use crate::{
13 Decode, DecodeError, Decoder, DecoderExt as _, Encode, EncodeError, Encoder, EncoderExt as _,
14 Slot, Wire, WireU16, WireU32, CHUNK_SIZE,
15};
16
17#[derive(Clone, Copy)]
18#[repr(C)]
19struct Encoded {
20 maybe_num_bytes: WireU32,
21 num_handles: WireU16,
22 flags: WireU16,
23}
24
25const INLINE_SIZE: usize = 4;
26
27#[repr(C, align(8))]
29pub union WireEnvelope {
30 zero: [u8; 8],
31 encoded: Encoded,
32 decoded_inline: [MaybeUninit<u8>; INLINE_SIZE],
33 decoded_out_of_line: *mut (),
34}
35
36unsafe impl Send for WireEnvelope {}
37unsafe impl Sync for WireEnvelope {}
38
39unsafe impl Wire for WireEnvelope {
40 type Decoded<'de> = WireEnvelope;
41
42 fn zero_padding(_: &mut MaybeUninit<Self>) {}
43}
44
45impl WireEnvelope {
46 const IS_INLINE_BIT: u16 = 1;
47
48 #[inline]
50 pub fn encode_zero(out: &mut MaybeUninit<Self>) {
51 out.write(WireEnvelope { zero: [0; 8] });
52 }
53
54 #[inline]
56 pub fn encode_value_static<E: InternalHandleEncoder + ?Sized, T: Encode<E>>(
57 value: T,
58 encoder: &mut E,
59 out: &mut MaybeUninit<Self>,
60 ) -> Result<(), EncodeError> {
61 munge! {
62 let Self {
63 encoded: Encoded {
64 maybe_num_bytes,
65 num_handles,
66 flags,
67 },
68 } = out;
69 }
70
71 let handles_before = encoder.__internal_handle_count();
72
73 let encoded_size = size_of::<T::Encoded>();
74 if encoded_size <= INLINE_SIZE {
75 unsafe {
78 maybe_num_bytes
79 .as_mut_ptr()
80 .cast::<u8>()
81 .add(encoded_size)
82 .write_bytes(0, INLINE_SIZE - encoded_size);
83 }
84 } else {
85 return Err(EncodeError::ExpectedInline(encoded_size));
86 }
87
88 let value_out = unsafe { &mut *maybe_num_bytes.as_mut_ptr().cast() };
89 T::Encoded::zero_padding(value_out);
90 value.encode(encoder, value_out)?;
91
92 flags.write(WireU16(Self::IS_INLINE_BIT));
93
94 let handle_count = (encoder.__internal_handle_count() - handles_before).try_into().unwrap();
95 num_handles.write(WireU16(handle_count));
96
97 Ok(())
98 }
99
100 #[inline]
102 pub fn encode_value<E: Encoder + ?Sized, T: Encode<E>>(
103 value: T,
104 encoder: &mut E,
105 out: &mut MaybeUninit<Self>,
106 ) -> Result<(), EncodeError> {
107 munge! {
108 let Self {
109 encoded: Encoded {
110 maybe_num_bytes,
111 num_handles,
112 flags,
113 },
114 } = out;
115 }
116
117 let handles_before = encoder.__internal_handle_count();
118
119 let encoded_size = size_of::<T::Encoded>();
120 if encoded_size <= INLINE_SIZE {
121 unsafe {
124 maybe_num_bytes
125 .as_mut_ptr()
126 .cast::<u8>()
127 .add(encoded_size)
128 .write_bytes(0, INLINE_SIZE - encoded_size);
129 }
130 let value_out = unsafe { &mut *maybe_num_bytes.as_mut_ptr().cast() };
131 T::Encoded::zero_padding(value_out);
132 value.encode(encoder, value_out)?;
133 flags.write(WireU16(Self::IS_INLINE_BIT));
134 } else {
135 let bytes_before = encoder.bytes_written();
136
137 encoder.encode_next(value)?;
138
139 let bytes_count = (encoder.bytes_written() - bytes_before).try_into().unwrap();
140 maybe_num_bytes.write(WireU32(bytes_count));
141 flags.write(WireU16(0));
142 }
143
144 let handle_count = (encoder.__internal_handle_count() - handles_before).try_into().unwrap();
145 num_handles.write(WireU16(handle_count));
146
147 Ok(())
148 }
149
150 #[inline]
152 pub fn zero() -> Self {
153 Self { zero: [0; 8] }
154 }
155
156 #[inline]
158 pub fn is_encoded_zero(slot: Slot<'_, Self>) -> bool {
159 munge!(let Self { zero } = slot);
160 *zero == [0; 8]
161 }
162
163 #[inline]
165 pub fn is_zero(&self) -> bool {
166 unsafe { self.zero == [0; 8] }
167 }
168
169 #[inline]
170 fn out_of_line_chunks(
171 maybe_num_bytes: Slot<'_, WireU32>,
172 flags: Slot<'_, WireU16>,
173 ) -> Result<Option<usize>, DecodeError> {
174 if **flags & Self::IS_INLINE_BIT == 0 {
175 let num_bytes = **maybe_num_bytes;
176 if num_bytes as usize % CHUNK_SIZE != 0 {
177 return Err(DecodeError::InvalidEnvelopeSize(num_bytes));
178 }
179 if num_bytes <= INLINE_SIZE as u32 {
180 return Err(DecodeError::OutOfLineValueTooSmall(num_bytes));
181 }
182 Ok(Some(num_bytes as usize / CHUNK_SIZE))
183 } else {
184 Ok(None)
185 }
186 }
187
188 #[inline]
190 pub fn decode_unknown_static<D: InternalHandleDecoder + ?Sized>(
191 slot: Slot<'_, Self>,
192 decoder: &mut D,
193 ) -> Result<(), DecodeError> {
194 munge! {
195 let Self {
196 encoded: Encoded {
197 maybe_num_bytes,
198 num_handles,
199 flags,
200 },
201 } = slot;
202 }
203
204 if let Some(count) = Self::out_of_line_chunks(maybe_num_bytes, flags)? {
205 return Err(DecodeError::ExpectedInline(count * CHUNK_SIZE));
206 }
207
208 decoder.__internal_take_handles(**num_handles as usize)?;
209
210 Ok(())
211 }
212
213 #[inline]
215 pub fn decode_unknown<D: Decoder + ?Sized>(
216 slot: Slot<'_, Self>,
217 mut decoder: &mut D,
218 ) -> Result<(), DecodeError> {
219 munge! {
220 let Self {
221 encoded: Encoded {
222 maybe_num_bytes,
223 num_handles,
224 flags,
225 },
226 } = slot;
227 }
228
229 if let Some(count) = Self::out_of_line_chunks(maybe_num_bytes, flags)? {
230 decoder.take_chunks(count)?;
231 }
232
233 decoder.__internal_take_handles(**num_handles as usize)?;
234
235 Ok(())
236 }
237
238 #[inline]
240 pub fn decode_as_static<D: InternalHandleDecoder + ?Sized, T: Decode<D>>(
241 mut slot: Slot<'_, Self>,
242 decoder: &mut D,
243 ) -> Result<(), DecodeError> {
244 munge! {
245 let Self {
246 encoded: Encoded {
247 maybe_num_bytes,
248 num_handles,
249 flags,
250 },
251 } = slot.as_mut();
252 }
253
254 let handles_before = decoder.__internal_handles_remaining();
255 let num_handles = **num_handles as usize;
256
257 if let Some(count) = Self::out_of_line_chunks(maybe_num_bytes, flags)? {
258 return Err(DecodeError::ExpectedInline(count * CHUNK_SIZE));
259 }
260
261 if size_of::<T>() > INLINE_SIZE {
263 return Err(DecodeError::InlineValueTooBig(size_of::<T>()));
264 }
265 munge!(let Self { mut decoded_inline } = slot);
266 let mut slot = unsafe { Slot::<T>::new_unchecked(decoded_inline.as_mut_ptr().cast()) };
267 T::decode(slot.as_mut(), decoder)?;
268
269 let handles_consumed = handles_before - decoder.__internal_handles_remaining();
270 if handles_consumed != num_handles {
271 return Err(DecodeError::IncorrectNumberOfHandlesConsumed {
272 expected: num_handles,
273 actual: handles_consumed,
274 });
275 }
276
277 Ok(())
278 }
279
280 #[inline]
282 pub fn decode_as<D: Decoder + ?Sized, T: Decode<D>>(
283 mut slot: Slot<'_, Self>,
284 mut decoder: &mut D,
285 ) -> Result<(), DecodeError> {
286 munge! {
287 let Self {
288 encoded: Encoded {
289 mut maybe_num_bytes,
290 num_handles,
291 flags,
292 },
293 } = slot.as_mut();
294 }
295
296 let handles_before = decoder.__internal_handles_remaining();
297 let num_handles = **num_handles as usize;
298
299 let out_of_line_chunks = Self::out_of_line_chunks(maybe_num_bytes.as_mut(), flags)?;
300 if let Some(_count) = out_of_line_chunks {
301 let mut value_slot = decoder.take_slot::<T>()?;
305 let value_ptr = value_slot.as_mut_ptr();
306 T::decode(value_slot, decoder)?;
307
308 munge!(let Self { mut decoded_out_of_line } = slot);
309 unsafe { decoded_out_of_line.as_mut_ptr().write(value_ptr.cast()) };
312 } else {
313 if size_of::<T>() > INLINE_SIZE {
315 return Err(DecodeError::InlineValueTooBig(size_of::<T>()));
316 }
317 munge!(let Self { mut decoded_inline } = slot);
318 let mut slot = unsafe { Slot::<T>::new_unchecked(decoded_inline.as_mut_ptr().cast()) };
319 T::decode(slot.as_mut(), decoder)?;
320 }
321
322 let handles_consumed = handles_before - decoder.__internal_handles_remaining();
323 if handles_consumed != num_handles {
324 return Err(DecodeError::IncorrectNumberOfHandlesConsumed {
325 expected: num_handles,
326 actual: handles_consumed,
327 });
328 }
329
330 Ok(())
331 }
332
333 #[inline]
334 unsafe fn as_ptr<T>(this: *mut Self) -> *mut T {
335 if size_of::<T>() <= INLINE_SIZE {
336 let inline = unsafe { addr_of_mut!((*this).decoded_inline) };
337 inline.cast()
338 } else {
339 unsafe { (*this).decoded_out_of_line.cast() }
340 }
341 }
342
343 #[inline]
349 pub unsafe fn deref_unchecked<T>(&self) -> &T {
350 let ptr = unsafe { Self::as_ptr::<T>((self as *const Self).cast_mut()).cast_const() };
351 unsafe { &*ptr }
352 }
353
354 #[inline]
360 pub unsafe fn read_unchecked<T>(&self) -> T {
361 unsafe { Self::as_ptr::<T>((self as *const Self).cast_mut()).read() }
364 }
365
366 #[inline]
372 pub unsafe fn clone_inline_unchecked<T: Clone>(&self) -> Self {
373 debug_assert!(size_of::<T>() <= INLINE_SIZE);
374
375 union ClonedToDecodedInline<T> {
376 cloned: ManuallyDrop<T>,
377 decoded_inline: [MaybeUninit<u8>; INLINE_SIZE],
378 }
379
380 let cloned = unsafe { self.deref_unchecked::<T>().clone() };
381 unsafe {
382 Self {
383 decoded_inline: ClonedToDecodedInline { cloned: ManuallyDrop::new(cloned) }
384 .decoded_inline,
385 }
386 }
387 }
388}