1use core::mem::{ManuallyDrop, MaybeUninit};
6use core::ptr::addr_of_mut;
7
8use munge::munge;
9
10use crate::decoder::InternalHandleDecoder;
11use crate::encoder::InternalHandleEncoder;
12use crate::{
13 CHUNK_SIZE, Constrained, Decode, DecodeError, Decoder, DecoderExt as _, Encode, EncodeError,
14 Encoder, EncoderExt as _, Slot, ValidationError, Wire, wire,
15};
16
17#[derive(Clone, Copy)]
18#[repr(C)]
19struct Encoded {
20 maybe_num_bytes: wire::Uint32,
21 num_handles: wire::Uint16,
22 flags: wire::Uint16,
23}
24
25const INLINE_SIZE: usize = 4;
26
27#[repr(C, align(8))]
29pub union Envelope {
30 zero: [u8; 8],
31 encoded: Encoded,
32 decoded_inline: [MaybeUninit<u8>; INLINE_SIZE],
33 decoded_out_of_line: *mut (),
34}
35
36unsafe impl Send for Envelope {}
37unsafe impl Sync for Envelope {}
38
39impl Constrained for Envelope {
40 type Constraint = ();
41
42 fn validate(_: Slot<'_, Self>, _: Self::Constraint) -> Result<(), ValidationError> {
43 Ok(())
44 }
45}
46
47unsafe impl Wire for Envelope {
48 type Narrowed<'de> = Self;
49
50 fn zero_padding(_: &mut MaybeUninit<Self>) {}
51}
52
53impl Envelope {
54 const IS_INLINE_BIT: u16 = 1;
55
56 #[inline]
58 pub fn encode_zero(out: &mut MaybeUninit<Self>) {
59 out.write(Envelope { zero: [0; 8] });
60 }
61
62 #[inline]
64 pub fn encode_value_static<W: Wire, E: InternalHandleEncoder + ?Sized>(
65 value: impl Encode<W, E>,
66 encoder: &mut E,
67 out: &mut MaybeUninit<Self>,
68 constraint: W::Constraint,
69 ) -> Result<(), EncodeError> {
70 #[allow(unused_unsafe)]
72 let encoded = unsafe {
73 munge!(let Self { encoded } = out);
74 encoded
75 };
76 munge! {
77 let Encoded {
78 maybe_num_bytes,
79 num_handles,
80 flags,
81 } = encoded;
82 }
83
84 let handles_before = encoder.__internal_handle_count();
85
86 let encoded_size = size_of::<W>();
87 if encoded_size <= INLINE_SIZE {
88 unsafe {
91 maybe_num_bytes
92 .as_mut_ptr()
93 .cast::<u8>()
94 .add(encoded_size)
95 .write_bytes(0, INLINE_SIZE - encoded_size);
96 }
97 } else {
98 return Err(EncodeError::ExpectedInline(encoded_size));
99 }
100
101 let value_out = unsafe { &mut *maybe_num_bytes.as_mut_ptr().cast() };
102 W::zero_padding(value_out);
103 value.encode(encoder, value_out, constraint)?;
104
105 flags.write(wire::Uint16(Self::IS_INLINE_BIT));
106
107 let handle_count = (encoder.__internal_handle_count() - handles_before).try_into().unwrap();
108 num_handles.write(wire::Uint16(handle_count));
109
110 Ok(())
111 }
112
113 #[inline]
115 pub fn encode_value<W: Wire, E: Encoder + ?Sized>(
116 value: impl Encode<W, E>,
117 encoder: &mut E,
118 out: &mut MaybeUninit<Self>,
119 constraint: W::Constraint,
120 ) -> Result<(), EncodeError> {
121 #[allow(unused_unsafe)]
123 let encoded = unsafe {
124 munge!(let Self { encoded } = out);
125 encoded
126 };
127 munge! {
128 let Encoded {
129 maybe_num_bytes,
130 num_handles,
131 flags,
132 } = encoded;
133 }
134
135 let handles_before = encoder.__internal_handle_count();
136
137 let encoded_size = size_of::<W>();
138 if encoded_size <= INLINE_SIZE {
139 unsafe {
142 maybe_num_bytes
143 .as_mut_ptr()
144 .cast::<u8>()
145 .add(encoded_size)
146 .write_bytes(0, INLINE_SIZE - encoded_size);
147 }
148 let value_out = unsafe { &mut *maybe_num_bytes.as_mut_ptr().cast() };
149 W::zero_padding(value_out);
150 value.encode(encoder, value_out, constraint)?;
151 flags.write(wire::Uint16(Self::IS_INLINE_BIT));
152 } else {
153 let bytes_before = encoder.bytes_written();
154
155 encoder.encode_next_with_constraint(value, constraint)?;
156
157 let bytes_count = (encoder.bytes_written() - bytes_before).try_into().unwrap();
158 maybe_num_bytes.write(wire::Uint32(bytes_count));
159 flags.write(wire::Uint16(0));
160 }
161
162 let handle_count = (encoder.__internal_handle_count() - handles_before).try_into().unwrap();
163 num_handles.write(wire::Uint16(handle_count));
164
165 Ok(())
166 }
167
168 #[inline]
170 pub fn zero() -> Self {
171 Self { zero: [0; 8] }
172 }
173
174 #[inline]
176 pub fn is_encoded_zero(slot: Slot<'_, Self>) -> bool {
177 #[allow(unused_unsafe)]
179 let zero = unsafe {
180 munge!(let Self { zero } = slot);
181 zero
182 };
183 *zero == [0; 8]
184 }
185
186 #[inline]
188 pub fn is_zero(&self) -> bool {
189 unsafe { self.zero == [0; 8] }
190 }
191
192 #[inline]
193 fn out_of_line_chunks(
194 maybe_num_bytes: Slot<'_, wire::Uint32>,
195 flags: Slot<'_, wire::Uint16>,
196 ) -> Result<Option<usize>, DecodeError> {
197 match **flags {
198 Self::IS_INLINE_BIT => Ok(None),
199 0 => {
200 let num_bytes = **maybe_num_bytes;
201 if !(num_bytes as usize).is_multiple_of(CHUNK_SIZE) {
202 Err(DecodeError::InvalidEnvelopeSize(num_bytes))
203 } else if num_bytes <= INLINE_SIZE as u32 {
204 Err(DecodeError::OutOfLineValueTooSmall(num_bytes))
205 } else {
206 Ok(Some(num_bytes as usize / CHUNK_SIZE))
207 }
208 }
209 _ => Err(DecodeError::InvalidEnvelopeFlags(**flags)),
210 }
211 }
212
213 #[inline]
215 pub fn decode_unknown_static<D: InternalHandleDecoder + ?Sized>(
216 slot: Slot<'_, Self>,
217 decoder: &mut D,
218 ) -> Result<(), DecodeError> {
219 #[allow(unused_unsafe)]
221 let encoded = unsafe {
222 munge!(let Self { encoded } = slot);
223 encoded
224 };
225 munge! {
226 let Encoded {
227 maybe_num_bytes,
228 num_handles,
229 flags,
230 } = encoded;
231 }
232
233 if let Some(count) = Self::out_of_line_chunks(maybe_num_bytes, flags)? {
234 return Err(DecodeError::ExpectedInline(count * CHUNK_SIZE));
235 }
236
237 decoder.__internal_take_handles(**num_handles as usize)?;
238
239 Ok(())
240 }
241
242 #[inline]
244 pub fn decode_unknown<'de, D: Decoder<'de> + ?Sized>(
245 slot: Slot<'_, Self>,
246 decoder: &mut D,
247 ) -> Result<(), DecodeError> {
248 #[allow(unused_unsafe)]
250 let encoded = unsafe {
251 munge!(let Self { encoded } = slot);
252 encoded
253 };
254 munge! {
255 let Encoded {
256 maybe_num_bytes,
257 num_handles,
258 flags,
259 } = encoded;
260 }
261
262 if let Some(count) = Self::out_of_line_chunks(maybe_num_bytes, flags)? {
263 decoder.take_chunks(count)?;
264 }
265
266 decoder.__internal_take_handles(**num_handles as usize)?;
267
268 Ok(())
269 }
270
271 #[inline]
273 pub fn decode_as_static<D: InternalHandleDecoder + ?Sized, T: Decode<D>>(
274 mut slot: Slot<'_, Self>,
275 decoder: &mut D,
276 constraint: T::Constraint,
277 ) -> Result<(), DecodeError> {
278 #[allow(unused_unsafe)]
280 let encoded = unsafe {
281 munge!(let Self { encoded } = slot.as_mut());
282 encoded
283 };
284 munge! {
285 let Encoded {
286 maybe_num_bytes,
287 num_handles,
288 flags,
289 } = encoded;
290 }
291
292 let handles_before = decoder.__internal_handles_remaining();
293 let num_handles = **num_handles as usize;
294
295 if let Some(count) = Self::out_of_line_chunks(maybe_num_bytes, flags)? {
296 return Err(DecodeError::ExpectedInline(count * CHUNK_SIZE));
297 }
298
299 if size_of::<T>() > INLINE_SIZE {
301 return Err(DecodeError::InlineValueTooBig(size_of::<T>()));
302 }
303 #[allow(unused_unsafe)]
305 let mut decoded_inline = unsafe {
306 munge!(let Self { decoded_inline } = slot);
307 decoded_inline
308 };
309 let mut slot = unsafe { Slot::<T>::new_unchecked(decoded_inline.as_mut_ptr().cast()) };
310 T::decode(slot.as_mut(), decoder, constraint)?;
311
312 let handles_consumed = handles_before - decoder.__internal_handles_remaining();
313 if handles_consumed != num_handles {
314 return Err(DecodeError::IncorrectNumberOfHandlesConsumed {
315 expected: num_handles,
316 actual: handles_consumed,
317 });
318 }
319
320 Ok(())
321 }
322
323 #[inline]
325 pub fn decode_as<'de, D: Decoder<'de> + ?Sized, T: Decode<D>>(
326 mut slot: Slot<'_, Self>,
327 decoder: &mut D,
328 constraint: T::Constraint,
329 ) -> Result<(), DecodeError> {
330 #[allow(unused_unsafe)]
332 let encoded = unsafe {
333 munge!(let Self { encoded } = slot.as_mut());
334 encoded
335 };
336 munge! {
337 let Encoded {
338 mut maybe_num_bytes,
339 num_handles,
340 flags,
341 } = encoded;
342 }
343
344 let handles_before = decoder.__internal_handles_remaining();
345 let num_handles = **num_handles as usize;
346
347 let out_of_line_chunks = Self::out_of_line_chunks(maybe_num_bytes.as_mut(), flags)?;
348 if let Some(_count) = out_of_line_chunks {
349 let mut value_slot = decoder.take_slot::<T>()?;
353 let value_ptr = value_slot.as_mut_ptr();
354 T::decode(value_slot, decoder, constraint)?;
355
356 #[allow(unused_unsafe)]
358 let mut decoded_out_of_line = unsafe {
359 munge!(let Self { decoded_out_of_line } = slot);
360 decoded_out_of_line
361 };
362 unsafe { decoded_out_of_line.as_mut_ptr().write(value_ptr.cast()) };
365 } else {
366 if size_of::<T>() > INLINE_SIZE {
368 return Err(DecodeError::InlineValueTooBig(size_of::<T>()));
369 }
370 #[allow(unused_unsafe)]
372 let mut decoded_inline = unsafe {
373 munge!(let Self { decoded_inline } = slot);
374 decoded_inline
375 };
376 let mut slot = unsafe { Slot::<T>::new_unchecked(decoded_inline.as_mut_ptr().cast()) };
377 T::decode(slot.as_mut(), decoder, constraint)?;
378 }
379
380 let handles_consumed = handles_before - decoder.__internal_handles_remaining();
381 if handles_consumed != num_handles {
382 return Err(DecodeError::IncorrectNumberOfHandlesConsumed {
383 expected: num_handles,
384 actual: handles_consumed,
385 });
386 }
387
388 Ok(())
389 }
390
391 #[inline]
392 unsafe fn as_ptr<T>(this: *mut Self) -> *mut T {
393 if size_of::<T>() <= INLINE_SIZE {
394 let inline = unsafe { addr_of_mut!((*this).decoded_inline) };
395 inline.cast()
396 } else {
397 unsafe { (*this).decoded_out_of_line.cast() }
398 }
399 }
400
401 #[inline]
407 pub unsafe fn deref_unchecked<T>(&self) -> &T {
408 let ptr = unsafe { Self::as_ptr::<T>((self as *const Self).cast_mut()).cast_const() };
409 unsafe { &*ptr }
410 }
411
412 #[inline]
418 pub unsafe fn read_unchecked<T>(&self) -> T {
419 unsafe { Self::as_ptr::<T>((self as *const Self).cast_mut()).read() }
422 }
423
424 #[inline]
430 pub unsafe fn clone_inline_unchecked<T: Clone>(&self) -> Self {
431 debug_assert!(size_of::<T>() <= INLINE_SIZE);
432
433 union ClonedToDecodedInline<T> {
434 cloned: ManuallyDrop<T>,
435 decoded_inline: [MaybeUninit<u8>; INLINE_SIZE],
436 }
437
438 let cloned = unsafe { self.deref_unchecked::<T>().clone() };
439 unsafe {
440 Self {
441 decoded_inline: ClonedToDecodedInline { cloned: ManuallyDrop::new(cloned) }
442 .decoded_inline,
443 }
444 }
445 }
446}