]>
Commit | Line | Data |
---|---|---|
60c5eb7d | 1 | use crate::rmeta::*; |
e74abb32 | 2 | |
04454e1e FG |
3 | use rustc_data_structures::fingerprint::Fingerprint; |
4 | use rustc_hir::def::{CtorKind, CtorOf}; | |
60c5eb7d | 5 | use rustc_index::vec::Idx; |
923072b8 | 6 | use rustc_middle::ty::ParameterizedOverTcx; |
064997fb FG |
7 | use rustc_serialize::opaque::FileEncoder; |
8 | use rustc_serialize::Encoder as _; | |
04454e1e | 9 | use rustc_span::hygiene::MacroKind; |
e74abb32 XL |
10 | use std::convert::TryInto; |
11 | use std::marker::PhantomData; | |
12 | use std::num::NonZeroUsize; | |
3dfed10e | 13 | use tracing::debug; |
e74abb32 XL |
14 | |
15 | /// Helper trait, for encoding to, and decoding from, a fixed number of bytes. | |
16 | /// Used mainly for Lazy positions and lengths. | |
17 | /// Unchecked invariant: `Self::default()` should encode as `[0; BYTE_LEN]`, | |
18 | /// but this has no impact on safety. | |
60c5eb7d | 19 | pub(super) trait FixedSizeEncoding: Default { |
04454e1e FG |
20 | /// This should be `[u8; BYTE_LEN]`; |
21 | type ByteArray; | |
22 | ||
23 | fn from_bytes(b: &Self::ByteArray) -> Self; | |
24 | fn write_to_bytes(self, b: &mut Self::ByteArray); | |
25 | } | |
26 | ||
27 | impl FixedSizeEncoding for u32 { | |
28 | type ByteArray = [u8; 4]; | |
29 | ||
30 | #[inline] | |
31 | fn from_bytes(b: &[u8; 4]) -> Self { | |
32 | Self::from_le_bytes(*b) | |
33 | } | |
34 | ||
35 | #[inline] | |
36 | fn write_to_bytes(self, b: &mut [u8; 4]) { | |
37 | *b = self.to_le_bytes(); | |
38 | } | |
39 | } | |
40 | ||
41 | macro_rules! fixed_size_enum { | |
42 | ($ty:ty { $(($($pat:tt)*))* }) => { | |
43 | impl FixedSizeEncoding for Option<$ty> { | |
44 | type ByteArray = [u8;1]; | |
45 | ||
46 | #[inline] | |
47 | fn from_bytes(b: &[u8;1]) -> Self { | |
48 | use $ty::*; | |
49 | if b[0] == 0 { | |
50 | return None; | |
51 | } | |
52 | match b[0] - 1 { | |
53 | $(${index()} => Some($($pat)*),)* | |
54 | _ => panic!("Unexpected ImplPolarity code: {:?}", b[0]), | |
55 | } | |
56 | } | |
57 | ||
58 | #[inline] | |
59 | fn write_to_bytes(self, b: &mut [u8;1]) { | |
60 | use $ty::*; | |
61 | b[0] = match self { | |
62 | None => 0, | |
63 | $(Some($($pat)*) => 1 + ${index()},)* | |
64 | } | |
65 | } | |
e74abb32 | 66 | } |
04454e1e FG |
67 | } |
68 | } | |
69 | ||
70 | fixed_size_enum! { | |
71 | DefKind { | |
72 | ( Mod ) | |
73 | ( Struct ) | |
74 | ( Union ) | |
75 | ( Enum ) | |
76 | ( Variant ) | |
77 | ( Trait ) | |
78 | ( TyAlias ) | |
79 | ( ForeignTy ) | |
80 | ( TraitAlias ) | |
81 | ( AssocTy ) | |
82 | ( TyParam ) | |
83 | ( Fn ) | |
84 | ( Const ) | |
85 | ( ConstParam ) | |
86 | ( AssocFn ) | |
87 | ( AssocConst ) | |
88 | ( ExternCrate ) | |
89 | ( Use ) | |
90 | ( ForeignMod ) | |
91 | ( AnonConst ) | |
92 | ( InlineConst ) | |
93 | ( OpaqueTy ) | |
94 | ( Field ) | |
95 | ( LifetimeParam ) | |
96 | ( GlobalAsm ) | |
97 | ( Impl ) | |
98 | ( Closure ) | |
99 | ( Generator ) | |
100 | ( Static(ast::Mutability::Not) ) | |
101 | ( Static(ast::Mutability::Mut) ) | |
102 | ( Ctor(CtorOf::Struct, CtorKind::Fn) ) | |
103 | ( Ctor(CtorOf::Struct, CtorKind::Const) ) | |
104 | ( Ctor(CtorOf::Struct, CtorKind::Fictive) ) | |
105 | ( Ctor(CtorOf::Variant, CtorKind::Fn) ) | |
106 | ( Ctor(CtorOf::Variant, CtorKind::Const) ) | |
107 | ( Ctor(CtorOf::Variant, CtorKind::Fictive) ) | |
108 | ( Macro(MacroKind::Bang) ) | |
109 | ( Macro(MacroKind::Attr) ) | |
110 | ( Macro(MacroKind::Derive) ) | |
111 | } | |
112 | } | |
113 | ||
114 | fixed_size_enum! { | |
115 | ty::ImplPolarity { | |
116 | ( Positive ) | |
117 | ( Negative ) | |
118 | ( Reservation ) | |
119 | } | |
120 | } | |
121 | ||
122 | fixed_size_enum! { | |
123 | hir::Constness { | |
124 | ( NotConst ) | |
125 | ( Const ) | |
126 | } | |
127 | } | |
128 | ||
129 | fixed_size_enum! { | |
130 | hir::Defaultness { | |
131 | ( Final ) | |
132 | ( Default { has_value: false } ) | |
133 | ( Default { has_value: true } ) | |
134 | } | |
135 | } | |
136 | ||
137 | fixed_size_enum! { | |
138 | hir::IsAsync { | |
139 | ( NotAsync ) | |
140 | ( Async ) | |
141 | } | |
142 | } | |
143 | ||
064997fb | 144 | // We directly encode `DefPathHash` because a `LazyValue` would incur a 25% cost. |
04454e1e FG |
145 | impl FixedSizeEncoding for Option<DefPathHash> { |
146 | type ByteArray = [u8; 16]; | |
147 | ||
148 | #[inline] | |
149 | fn from_bytes(b: &[u8; 16]) -> Self { | |
150 | Some(DefPathHash(Fingerprint::from_le_bytes(*b))) | |
151 | } | |
152 | ||
153 | #[inline] | |
154 | fn write_to_bytes(self, b: &mut [u8; 16]) { | |
155 | let Some(DefPathHash(fingerprint)) = self else { | |
156 | panic!("Trying to encode absent DefPathHash.") | |
157 | }; | |
158 | *b = fingerprint.to_le_bytes(); | |
159 | } | |
160 | } | |
161 | ||
064997fb | 162 | // We directly encode RawDefId because using a `LazyValue` would incur a 50% overhead in the worst case. |
04454e1e FG |
163 | impl FixedSizeEncoding for Option<RawDefId> { |
164 | type ByteArray = [u8; 8]; | |
165 | ||
166 | #[inline] | |
167 | fn from_bytes(b: &[u8; 8]) -> Self { | |
168 | let krate = u32::from_le_bytes(b[0..4].try_into().unwrap()); | |
169 | let index = u32::from_le_bytes(b[4..8].try_into().unwrap()); | |
170 | if krate == 0 { | |
171 | return None; | |
e74abb32 | 172 | } |
04454e1e FG |
173 | Some(RawDefId { krate: krate - 1, index }) |
174 | } | |
175 | ||
176 | #[inline] | |
177 | fn write_to_bytes(self, b: &mut [u8; 8]) { | |
178 | match self { | |
179 | None => *b = [0; 8], | |
180 | Some(RawDefId { krate, index }) => { | |
181 | // CrateNum is less than `CrateNum::MAX_AS_U32`. | |
182 | debug_assert!(krate < u32::MAX); | |
183 | b[0..4].copy_from_slice(&(1 + krate).to_le_bytes()); | |
184 | b[4..8].copy_from_slice(&index.to_le_bytes()); | |
185 | } | |
186 | } | |
187 | } | |
e74abb32 XL |
188 | } |
189 | ||
04454e1e FG |
190 | impl FixedSizeEncoding for Option<()> { |
191 | type ByteArray = [u8; 1]; | |
e74abb32 | 192 | |
04454e1e FG |
193 | #[inline] |
194 | fn from_bytes(b: &[u8; 1]) -> Self { | |
195 | (b[0] != 0).then(|| ()) | |
e74abb32 XL |
196 | } |
197 | ||
04454e1e FG |
198 | #[inline] |
199 | fn write_to_bytes(self, b: &mut [u8; 1]) { | |
200 | b[0] = self.is_some() as u8 | |
e74abb32 XL |
201 | } |
202 | } | |
203 | ||
204 | // NOTE(eddyb) there could be an impl for `usize`, which would enable a more | |
923072b8 FG |
205 | // generic `LazyValue<T>` impl, but in the general case we might not need / want |
206 | // to fit every `usize` in `u32`. | |
207 | impl<T> FixedSizeEncoding for Option<LazyValue<T>> { | |
04454e1e | 208 | type ByteArray = [u8; 4]; |
e74abb32 | 209 | |
04454e1e FG |
210 | #[inline] |
211 | fn from_bytes(b: &[u8; 4]) -> Self { | |
212 | let position = NonZeroUsize::new(u32::from_bytes(b) as usize)?; | |
923072b8 | 213 | Some(LazyValue::from_position(position)) |
e74abb32 XL |
214 | } |
215 | ||
04454e1e FG |
216 | #[inline] |
217 | fn write_to_bytes(self, b: &mut [u8; 4]) { | |
e74abb32 XL |
218 | let position = self.map_or(0, |lazy| lazy.position.get()); |
219 | let position: u32 = position.try_into().unwrap(); | |
e74abb32 XL |
220 | position.write_to_bytes(b) |
221 | } | |
222 | } | |
223 | ||
923072b8 | 224 | impl<T> FixedSizeEncoding for Option<LazyArray<T>> { |
04454e1e | 225 | type ByteArray = [u8; 8]; |
e74abb32 | 226 | |
04454e1e FG |
227 | #[inline] |
228 | fn from_bytes(b: &[u8; 8]) -> Self { | |
229 | let ([ref position_bytes, ref meta_bytes],[])= b.as_chunks::<4>() else { panic!() }; | |
230 | let position = NonZeroUsize::new(u32::from_bytes(position_bytes) as usize)?; | |
231 | let len = u32::from_bytes(meta_bytes) as usize; | |
923072b8 | 232 | Some(LazyArray::from_position_and_num_elems(position, len)) |
e74abb32 XL |
233 | } |
234 | ||
04454e1e FG |
235 | #[inline] |
236 | fn write_to_bytes(self, b: &mut [u8; 8]) { | |
237 | let ([ref mut position_bytes, ref mut meta_bytes],[])= b.as_chunks_mut::<4>() else { panic!() }; | |
238 | ||
239 | let position = self.map_or(0, |lazy| lazy.position.get()); | |
240 | let position: u32 = position.try_into().unwrap(); | |
241 | position.write_to_bytes(position_bytes); | |
e74abb32 | 242 | |
923072b8 | 243 | let len = self.map_or(0, |lazy| lazy.num_elems); |
e74abb32 | 244 | let len: u32 = len.try_into().unwrap(); |
04454e1e | 245 | len.write_to_bytes(meta_bytes); |
e74abb32 XL |
246 | } |
247 | } | |
248 | ||
60c5eb7d | 249 | /// Helper for constructing a table's serialization (also see `Table`). |
dfeec247 XL |
250 | pub(super) struct TableBuilder<I: Idx, T> |
251 | where | |
252 | Option<T>: FixedSizeEncoding, | |
253 | { | |
04454e1e FG |
254 | blocks: IndexVec<I, <Option<T> as FixedSizeEncoding>::ByteArray>, |
255 | _marker: PhantomData<T>, | |
e74abb32 XL |
256 | } |
257 | ||
dfeec247 XL |
258 | impl<I: Idx, T> Default for TableBuilder<I, T> |
259 | where | |
260 | Option<T>: FixedSizeEncoding, | |
261 | { | |
e74abb32 | 262 | fn default() -> Self { |
04454e1e | 263 | TableBuilder { blocks: Default::default(), _marker: PhantomData } |
e74abb32 XL |
264 | } |
265 | } | |
266 | ||
dfeec247 XL |
267 | impl<I: Idx, T> TableBuilder<I, T> |
268 | where | |
269 | Option<T>: FixedSizeEncoding, | |
270 | { | |
04454e1e FG |
271 | pub(crate) fn set<const N: usize>(&mut self, i: I, value: T) |
272 | where | |
273 | Option<T>: FixedSizeEncoding<ByteArray = [u8; N]>, | |
274 | { | |
e74abb32 XL |
275 | // FIXME(eddyb) investigate more compact encodings for sparse tables. |
276 | // On the PR @michaelwoerister mentioned: | |
277 | // > Space requirements could perhaps be optimized by using the HAMT `popcnt` | |
278 | // > trick (i.e. divide things into buckets of 32 or 64 items and then | |
279 | // > store bit-masks of which item in each bucket is actually serialized). | |
04454e1e FG |
280 | self.blocks.ensure_contains_elem(i, || [0; N]); |
281 | Some(value).write_to_bytes(&mut self.blocks[i]); | |
e74abb32 XL |
282 | } |
283 | ||
064997fb | 284 | pub(crate) fn encode<const N: usize>(&self, buf: &mut FileEncoder) -> LazyTable<I, T> |
04454e1e FG |
285 | where |
286 | Option<T>: FixedSizeEncoding<ByteArray = [u8; N]>, | |
287 | { | |
e74abb32 | 288 | let pos = buf.position(); |
04454e1e | 289 | for block in &self.blocks { |
923072b8 | 290 | buf.emit_raw_bytes(block); |
04454e1e FG |
291 | } |
292 | let num_bytes = self.blocks.len() * N; | |
923072b8 FG |
293 | LazyTable::from_position_and_encoded_size( |
294 | NonZeroUsize::new(pos as usize).unwrap(), | |
295 | num_bytes, | |
296 | ) | |
e74abb32 XL |
297 | } |
298 | } | |
299 | ||
923072b8 | 300 | impl<I: Idx, T: ParameterizedOverTcx> LazyTable<I, T> |
dfeec247 XL |
301 | where |
302 | Option<T>: FixedSizeEncoding, | |
303 | { | |
e74abb32 XL |
304 | /// Given the metadata, extract out the value at a particular index (if any). |
305 | #[inline(never)] | |
04454e1e FG |
306 | pub(super) fn get<'a, 'tcx, M: Metadata<'a, 'tcx>, const N: usize>( |
307 | &self, | |
308 | metadata: M, | |
309 | i: I, | |
923072b8 | 310 | ) -> Option<T::Value<'tcx>> |
04454e1e | 311 | where |
923072b8 | 312 | Option<T::Value<'tcx>>: FixedSizeEncoding<ByteArray = [u8; N]>, |
04454e1e | 313 | { |
923072b8 | 314 | debug!("LazyTable::lookup: index={:?} len={:?}", i, self.encoded_size); |
e74abb32 XL |
315 | |
316 | let start = self.position.get(); | |
923072b8 | 317 | let bytes = &metadata.blob()[start..start + self.encoded_size]; |
04454e1e FG |
318 | let (bytes, []) = bytes.as_chunks::<N>() else { panic!() }; |
319 | let bytes = bytes.get(i.index())?; | |
320 | FixedSizeEncoding::from_bytes(bytes) | |
e74abb32 | 321 | } |
3dfed10e XL |
322 | |
323 | /// Size of the table in entries, including possible gaps. | |
04454e1e FG |
324 | pub(super) fn size<const N: usize>(&self) -> usize |
325 | where | |
923072b8 | 326 | for<'tcx> Option<T::Value<'tcx>>: FixedSizeEncoding<ByteArray = [u8; N]>, |
04454e1e | 327 | { |
923072b8 | 328 | self.encoded_size / N |
3dfed10e | 329 | } |
e74abb32 | 330 | } |