]>
Commit | Line | Data |
---|---|---|
ea8adc8c XL |
1 | // Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT |
2 | // file at the top-level directory of this distribution and at | |
3 | // http://rust-lang.org/COPYRIGHT. | |
4 | // | |
5 | // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or | |
6 | // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license | |
7 | // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your | |
8 | // option. This file may not be copied, modified, or distributed | |
9 | // except according to those terms. | |
10 | ||
11 | //! The implementation of the query system itself. Defines the macros | |
12 | //! that generate the actual methods on tcx which find and execute the | |
13 | //! provider, manage the caches, and so forth. | |
14 | ||
abe05a73 XL |
15 | use dep_graph::{DepNodeIndex, DepNode, DepKind, DepNodeColor}; |
16 | use errors::DiagnosticBuilder; | |
83c7162d XL |
17 | use errors::Level; |
18 | use errors::Diagnostic; | |
19 | use errors::FatalError; | |
20 | use ty::tls; | |
ea8adc8c | 21 | use ty::{TyCtxt}; |
94b46f34 XL |
22 | use ty::query::Query; |
23 | use ty::query::config::{QueryConfig, QueryDescription}; | |
24 | use ty::query::job::{QueryJob, QueryResult, QueryInfo}; | |
ea8adc8c XL |
25 | use ty::item_path; |
26 | ||
83c7162d XL |
27 | use util::common::{profq_msg, ProfileQueriesMsg, QueryMsg}; |
28 | ||
ea8adc8c | 29 | use rustc_data_structures::fx::{FxHashMap}; |
83c7162d | 30 | use rustc_data_structures::sync::{Lrc, Lock}; |
ea8adc8c | 31 | use std::mem; |
83c7162d XL |
32 | use std::ptr; |
33 | use std::collections::hash_map::Entry; | |
ea8adc8c | 34 | use syntax_pos::Span; |
b7449926 | 35 | use syntax::source_map::DUMMY_SP; |
ea8adc8c | 36 | |
94b46f34 | 37 | pub struct QueryCache<'tcx, D: QueryConfig<'tcx> + ?Sized> { |
83c7162d XL |
38 | pub(super) results: FxHashMap<D::Key, QueryValue<D::Value>>, |
39 | pub(super) active: FxHashMap<D::Key, QueryResult<'tcx>>, | |
ea8adc8c XL |
40 | } |
41 | ||
42 | pub(super) struct QueryValue<T> { | |
43 | pub(super) value: T, | |
44 | pub(super) index: DepNodeIndex, | |
ea8adc8c XL |
45 | } |
46 | ||
47 | impl<T> QueryValue<T> { | |
48 | pub(super) fn new(value: T, | |
abe05a73 | 49 | dep_node_index: DepNodeIndex) |
ea8adc8c XL |
50 | -> QueryValue<T> { |
51 | QueryValue { | |
52 | value, | |
53 | index: dep_node_index, | |
ea8adc8c XL |
54 | } |
55 | } | |
56 | } | |
57 | ||
0bf4aa26 XL |
58 | impl<'tcx, M: QueryConfig<'tcx>> Default for QueryCache<'tcx, M> { |
59 | fn default() -> QueryCache<'tcx, M> { | |
94b46f34 | 60 | QueryCache { |
0bf4aa26 XL |
61 | results: FxHashMap::default(), |
62 | active: FxHashMap::default(), | |
ea8adc8c XL |
63 | } |
64 | } | |
65 | } | |
66 | ||
83c7162d XL |
67 | // If enabled, send a message to the profile-queries thread |
68 | macro_rules! profq_msg { | |
69 | ($tcx:expr, $msg:expr) => { | |
70 | if cfg!(debug_assertions) { | |
71 | if $tcx.sess.profile_queries() { | |
72 | profq_msg($tcx.sess, $msg) | |
73 | } | |
74 | } | |
75 | } | |
abe05a73 XL |
76 | } |
77 | ||
83c7162d XL |
78 | // If enabled, format a key using its debug string, which can be |
79 | // expensive to compute (in terms of time). | |
80 | macro_rules! profq_query_msg { | |
81 | ($query:expr, $tcx:expr, $key:expr) => {{ | |
82 | let msg = if cfg!(debug_assertions) { | |
83 | if $tcx.sess.profile_queries_and_keys() { | |
84 | Some(format!("{:?}", $key)) | |
85 | } else { None } | |
86 | } else { None }; | |
87 | QueryMsg { | |
88 | query: $query, | |
89 | msg, | |
90 | } | |
91 | }} | |
92 | } | |
93 | ||
94 | /// A type representing the responsibility to execute the job in the `job` field. | |
95 | /// This will poison the relevant query if dropped. | |
96 | pub(super) struct JobOwner<'a, 'tcx: 'a, Q: QueryDescription<'tcx> + 'a> { | |
94b46f34 | 97 | cache: &'a Lock<QueryCache<'tcx, Q>>, |
83c7162d XL |
98 | key: Q::Key, |
99 | job: Lrc<QueryJob<'tcx>>, | |
100 | } | |
101 | ||
102 | impl<'a, 'tcx, Q: QueryDescription<'tcx>> JobOwner<'a, 'tcx, Q> { | |
a1dfa0c6 | 103 | /// Either gets a JobOwner corresponding the query, allowing us to |
83c7162d XL |
104 | /// start executing the query, or it returns with the result of the query. |
105 | /// If the query is executing elsewhere, this will wait for it. | |
106 | /// If the query panicked, this will silently panic. | |
94b46f34 XL |
107 | /// |
108 | /// This function is inlined because that results in a noticeable speedup | |
109 | /// for some compile-time benchmarks. | |
110 | #[inline(always)] | |
83c7162d XL |
111 | pub(super) fn try_get( |
112 | tcx: TyCtxt<'a, 'tcx, '_>, | |
113 | span: Span, | |
114 | key: &Q::Key, | |
115 | ) -> TryGetJob<'a, 'tcx, Q> { | |
94b46f34 | 116 | let cache = Q::query_cache(tcx); |
83c7162d | 117 | loop { |
94b46f34 | 118 | let mut lock = cache.borrow_mut(); |
83c7162d XL |
119 | if let Some(value) = lock.results.get(key) { |
120 | profq_msg!(tcx, ProfileQueriesMsg::CacheHit); | |
b7449926 XL |
121 | tcx.sess.profiler(|p| { |
122 | p.record_query(Q::CATEGORY); | |
123 | p.record_query_hit(Q::CATEGORY); | |
124 | }); | |
125 | ||
83c7162d XL |
126 | let result = Ok((value.value.clone(), value.index)); |
127 | return TryGetJob::JobCompleted(result); | |
128 | } | |
129 | let job = match lock.active.entry((*key).clone()) { | |
130 | Entry::Occupied(entry) => { | |
131 | match *entry.get() { | |
132 | QueryResult::Started(ref job) => job.clone(), | |
133 | QueryResult::Poisoned => FatalError.raise(), | |
134 | } | |
135 | } | |
136 | Entry::Vacant(entry) => { | |
137 | // No job entry for this query. Return a new one to be started later | |
138 | return tls::with_related_context(tcx, |icx| { | |
139 | let info = QueryInfo { | |
140 | span, | |
141 | query: Q::query(key.clone()), | |
142 | }; | |
143 | let job = Lrc::new(QueryJob::new(info, icx.query.clone())); | |
144 | let owner = JobOwner { | |
94b46f34 | 145 | cache, |
83c7162d XL |
146 | job: job.clone(), |
147 | key: (*key).clone(), | |
148 | }; | |
149 | entry.insert(QueryResult::Started(job)); | |
150 | TryGetJob::NotYetStarted(owner) | |
151 | }) | |
152 | } | |
153 | }; | |
154 | mem::drop(lock); | |
155 | ||
156 | if let Err(cycle) = job.await(tcx, span) { | |
157 | return TryGetJob::JobCompleted(Err(cycle)); | |
158 | } | |
159 | } | |
160 | } | |
161 | ||
94b46f34 | 162 | /// Completes the query by updating the query cache with the `result`, |
83c7162d XL |
163 | /// signals the waiter and forgets the JobOwner, so it won't poison the query |
164 | pub(super) fn complete(self, result: &Q::Value, dep_node_index: DepNodeIndex) { | |
165 | // We can move out of `self` here because we `mem::forget` it below | |
166 | let key = unsafe { ptr::read(&self.key) }; | |
167 | let job = unsafe { ptr::read(&self.job) }; | |
94b46f34 | 168 | let cache = self.cache; |
83c7162d XL |
169 | |
170 | // Forget ourself so our destructor won't poison the query | |
171 | mem::forget(self); | |
172 | ||
173 | let value = QueryValue::new(result.clone(), dep_node_index); | |
174 | { | |
94b46f34 | 175 | let mut lock = cache.borrow_mut(); |
83c7162d XL |
176 | lock.active.remove(&key); |
177 | lock.results.insert(key, value); | |
178 | } | |
179 | ||
180 | job.signal_complete(); | |
181 | } | |
182 | ||
183 | /// Executes a job by changing the ImplicitCtxt to point to the | |
184 | /// new query job while it executes. It returns the diagnostics | |
185 | /// captured during execution and the actual result. | |
186 | pub(super) fn start<'lcx, F, R>( | |
187 | &self, | |
188 | tcx: TyCtxt<'_, 'tcx, 'lcx>, | |
189 | compute: F) | |
190 | -> (R, Vec<Diagnostic>) | |
191 | where | |
192 | F: for<'b> FnOnce(TyCtxt<'b, 'tcx, 'lcx>) -> R | |
193 | { | |
194 | // The TyCtxt stored in TLS has the same global interner lifetime | |
195 | // as `tcx`, so we use `with_related_context` to relate the 'gcx lifetimes | |
196 | // when accessing the ImplicitCtxt | |
197 | let r = tls::with_related_context(tcx, move |current_icx| { | |
198 | // Update the ImplicitCtxt to point to our new query job | |
199 | let new_icx = tls::ImplicitCtxt { | |
200 | tcx, | |
201 | query: Some(self.job.clone()), | |
202 | layout_depth: current_icx.layout_depth, | |
203 | task: current_icx.task, | |
204 | }; | |
205 | ||
206 | // Use the ImplicitCtxt while we execute the query | |
207 | tls::enter_context(&new_icx, |_| { | |
208 | compute(tcx) | |
209 | }) | |
210 | }); | |
211 | ||
212 | // Extract the diagnostic from the job | |
213 | let diagnostics = mem::replace(&mut *self.job.diagnostics.lock(), Vec::new()); | |
214 | ||
215 | (r, diagnostics) | |
216 | } | |
217 | } | |
218 | ||
219 | impl<'a, 'tcx, Q: QueryDescription<'tcx>> Drop for JobOwner<'a, 'tcx, Q> { | |
220 | fn drop(&mut self) { | |
221 | // Poison the query so jobs waiting on it panic | |
94b46f34 | 222 | self.cache.borrow_mut().active.insert(self.key.clone(), QueryResult::Poisoned); |
83c7162d XL |
223 | // Also signal the completion of the job, so waiters |
224 | // will continue execution | |
225 | self.job.signal_complete(); | |
226 | } | |
227 | } | |
228 | ||
229 | #[derive(Clone)] | |
94b46f34 | 230 | pub struct CycleError<'tcx> { |
83c7162d XL |
231 | /// The query and related span which uses the cycle |
232 | pub(super) usage: Option<(Span, Query<'tcx>)>, | |
233 | pub(super) cycle: Vec<QueryInfo<'tcx>>, | |
234 | } | |
235 | ||
236 | /// The result of `try_get_lock` | |
237 | pub(super) enum TryGetJob<'a, 'tcx: 'a, D: QueryDescription<'tcx> + 'a> { | |
94b46f34 | 238 | /// The query is not yet started. Contains a guard to the cache eventually used to start it. |
83c7162d XL |
239 | NotYetStarted(JobOwner<'a, 'tcx, D>), |
240 | ||
241 | /// The query was already completed. | |
242 | /// Returns the result of the query and its dep node index | |
243 | /// if it succeeded or a cycle error if it failed | |
244 | JobCompleted(Result<(D::Value, DepNodeIndex), CycleError<'tcx>>), | |
ea8adc8c XL |
245 | } |
246 | ||
247 | impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { | |
83c7162d | 248 | pub(super) fn report_cycle(self, CycleError { usage, cycle: stack }: CycleError<'gcx>) |
ea8adc8c XL |
249 | -> DiagnosticBuilder<'a> |
250 | { | |
ea8adc8c XL |
251 | assert!(!stack.is_empty()); |
252 | ||
83c7162d | 253 | let fix_span = |span: Span, query: &Query<'gcx>| { |
b7449926 | 254 | self.sess.source_map().def_span(query.default_span(self, span)) |
83c7162d XL |
255 | }; |
256 | ||
ea8adc8c XL |
257 | // Disable naming impls with types in this path, since that |
258 | // sometimes cycles itself, leading to extra cycle errors. | |
259 | // (And cycle errors around impls tend to occur during the | |
260 | // collect/coherence phases anyhow.) | |
261 | item_path::with_forced_impl_filename_line(|| { | |
83c7162d XL |
262 | let span = fix_span(stack[1 % stack.len()].span, &stack[0].query); |
263 | let mut err = struct_span_err!(self.sess, | |
264 | span, | |
265 | E0391, | |
266 | "cycle detected when {}", | |
267 | stack[0].query.describe(self)); | |
268 | ||
269 | for i in 1..stack.len() { | |
270 | let query = &stack[i].query; | |
271 | let span = fix_span(stack[(i + 1) % stack.len()].span, query); | |
272 | err.span_note(span, &format!("...which requires {}...", query.describe(self))); | |
ea8adc8c XL |
273 | } |
274 | ||
83c7162d XL |
275 | err.note(&format!("...which again requires {}, completing the cycle", |
276 | stack[0].query.describe(self))); | |
277 | ||
278 | if let Some((span, query)) = usage { | |
279 | err.span_note(fix_span(span, &query), | |
280 | &format!("cycle used when {}", query.describe(self))); | |
281 | } | |
ea8adc8c XL |
282 | |
283 | return err | |
284 | }) | |
285 | } | |
286 | ||
83c7162d XL |
287 | pub fn try_print_query_stack() { |
288 | eprintln!("query stack during panic:"); | |
289 | ||
290 | tls::with_context_opt(|icx| { | |
291 | if let Some(icx) = icx { | |
292 | let mut current_query = icx.query.clone(); | |
293 | let mut i = 0; | |
294 | ||
295 | while let Some(query) = current_query { | |
296 | let mut db = DiagnosticBuilder::new(icx.tcx.sess.diagnostic(), | |
297 | Level::FailureNote, | |
298 | &format!("#{} [{}] {}", | |
299 | i, | |
300 | query.info.query.name(), | |
301 | query.info.query.describe(icx.tcx))); | |
b7449926 | 302 | db.set_span(icx.tcx.sess.source_map().def_span(query.info.span)); |
83c7162d XL |
303 | icx.tcx.sess.diagnostic().force_print_db(db); |
304 | ||
305 | current_query = query.parent.clone(); | |
306 | i += 1; | |
307 | } | |
ea8adc8c | 308 | } |
83c7162d | 309 | }); |
ea8adc8c | 310 | |
83c7162d | 311 | eprintln!("end of query stack"); |
ea8adc8c | 312 | } |
abe05a73 XL |
313 | |
314 | /// Try to read a node index for the node dep_node. | |
315 | /// A node will have an index, when it's already been marked green, or when we can mark it | |
316 | /// green. This function will mark the current task as a reader of the specified node, when | |
a1dfa0c6 | 317 | /// a node index can be found for that node. |
abe05a73 XL |
318 | pub(super) fn try_mark_green_and_read(self, dep_node: &DepNode) -> Option<DepNodeIndex> { |
319 | match self.dep_graph.node_color(dep_node) { | |
320 | Some(DepNodeColor::Green(dep_node_index)) => { | |
321 | self.dep_graph.read_index(dep_node_index); | |
322 | Some(dep_node_index) | |
323 | } | |
324 | Some(DepNodeColor::Red) => { | |
325 | None | |
326 | } | |
327 | None => { | |
328 | // try_mark_green (called below) will panic when full incremental | |
329 | // compilation is disabled. If that's the case, we can't try to mark nodes | |
330 | // as green anyway, so we can safely return None here. | |
331 | if !self.dep_graph.is_fully_enabled() { | |
332 | return None; | |
333 | } | |
ff7c6d11 | 334 | match self.dep_graph.try_mark_green(self.global_tcx(), &dep_node) { |
abe05a73 | 335 | Some(dep_node_index) => { |
0531ce1d | 336 | debug_assert!(self.dep_graph.is_green(&dep_node)); |
abe05a73 XL |
337 | self.dep_graph.read_index(dep_node_index); |
338 | Some(dep_node_index) | |
339 | } | |
340 | None => { | |
341 | None | |
342 | } | |
343 | } | |
344 | } | |
345 | } | |
346 | } | |
ea8adc8c | 347 | |
83c7162d XL |
348 | fn try_get_with<Q: QueryDescription<'gcx>>( |
349 | self, | |
350 | span: Span, | |
351 | key: Q::Key) | |
352 | -> Result<Q::Value, CycleError<'gcx>> | |
353 | { | |
354 | debug!("ty::queries::{}::try_get_with(key={:?}, span={:?})", | |
355 | Q::NAME, | |
356 | key, | |
357 | span); | |
358 | ||
359 | profq_msg!(self, | |
360 | ProfileQueriesMsg::QueryBegin( | |
361 | span.data(), | |
362 | profq_query_msg!(Q::NAME, self, key), | |
363 | ) | |
364 | ); | |
365 | ||
b7449926 XL |
366 | self.sess.profiler(|p| p.record_query(Q::CATEGORY)); |
367 | ||
83c7162d XL |
368 | let job = match JobOwner::try_get(self, span, &key) { |
369 | TryGetJob::NotYetStarted(job) => job, | |
370 | TryGetJob::JobCompleted(result) => { | |
371 | return result.map(|(v, index)| { | |
b7449926 | 372 | self.sess.profiler(|p| p.record_query_hit(Q::CATEGORY)); |
83c7162d XL |
373 | self.dep_graph.read_index(index); |
374 | v | |
375 | }) | |
376 | } | |
377 | }; | |
378 | ||
379 | // Fast path for when incr. comp. is off. `to_dep_node` is | |
380 | // expensive for some DepKinds. | |
381 | if !self.dep_graph.is_fully_enabled() { | |
382 | let null_dep_node = DepNode::new_no_params(::dep_graph::DepKind::Null); | |
383 | return self.force_query_with_job::<Q>(key, job, null_dep_node).map(|(v, _)| v); | |
384 | } | |
385 | ||
386 | let dep_node = Q::to_dep_node(self, &key); | |
387 | ||
388 | if dep_node.kind.is_anon() { | |
389 | profq_msg!(self, ProfileQueriesMsg::ProviderBegin); | |
b7449926 | 390 | self.sess.profiler(|p| p.start_activity(Q::CATEGORY)); |
83c7162d XL |
391 | |
392 | let res = job.start(self, |tcx| { | |
393 | tcx.dep_graph.with_anon_task(dep_node.kind, || { | |
394 | Q::compute(tcx.global_tcx(), key) | |
395 | }) | |
396 | }); | |
397 | ||
b7449926 | 398 | self.sess.profiler(|p| p.end_activity(Q::CATEGORY)); |
83c7162d XL |
399 | profq_msg!(self, ProfileQueriesMsg::ProviderEnd); |
400 | let ((result, dep_node_index), diagnostics) = res; | |
401 | ||
402 | self.dep_graph.read_index(dep_node_index); | |
403 | ||
94b46f34 | 404 | self.queries.on_disk_cache |
83c7162d XL |
405 | .store_diagnostics_for_anon_node(dep_node_index, diagnostics); |
406 | ||
407 | job.complete(&result, dep_node_index); | |
408 | ||
409 | return Ok(result); | |
410 | } | |
411 | ||
412 | if !dep_node.kind.is_input() { | |
413 | if let Some(dep_node_index) = self.try_mark_green_and_read(&dep_node) { | |
414 | profq_msg!(self, ProfileQueriesMsg::CacheHit); | |
b7449926 XL |
415 | self.sess.profiler(|p| p.record_query_hit(Q::CATEGORY)); |
416 | ||
83c7162d XL |
417 | return self.load_from_disk_and_cache_in_memory::<Q>(key, |
418 | job, | |
419 | dep_node_index, | |
420 | &dep_node) | |
421 | } | |
422 | } | |
423 | ||
424 | match self.force_query_with_job::<Q>(key, job, dep_node) { | |
425 | Ok((result, dep_node_index)) => { | |
426 | self.dep_graph.read_index(dep_node_index); | |
427 | Ok(result) | |
ea8adc8c | 428 | } |
83c7162d | 429 | Err(e) => Err(e) |
ea8adc8c XL |
430 | } |
431 | } | |
ea8adc8c | 432 | |
83c7162d XL |
433 | fn load_from_disk_and_cache_in_memory<Q: QueryDescription<'gcx>>( |
434 | self, | |
435 | key: Q::Key, | |
436 | job: JobOwner<'a, 'gcx, Q>, | |
437 | dep_node_index: DepNodeIndex, | |
438 | dep_node: &DepNode | |
439 | ) -> Result<Q::Value, CycleError<'gcx>> | |
440 | { | |
441 | // Note this function can be called concurrently from the same query | |
442 | // We must ensure that this is handled correctly | |
443 | ||
444 | debug_assert!(self.dep_graph.is_green(dep_node)); | |
445 | ||
446 | // First we try to load the result from the on-disk cache | |
447 | let result = if Q::cache_on_disk(key.clone()) && | |
448 | self.sess.opts.debugging_opts.incremental_queries { | |
449 | let prev_dep_node_index = | |
450 | self.dep_graph.prev_dep_node_index_of(dep_node); | |
451 | let result = Q::try_load_from_disk(self.global_tcx(), | |
0bf4aa26 | 452 | prev_dep_node_index); |
83c7162d XL |
453 | |
454 | // We always expect to find a cached result for things that | |
455 | // can be forced from DepNode. | |
456 | debug_assert!(!dep_node.kind.can_reconstruct_query_key() || | |
0bf4aa26 XL |
457 | result.is_some(), |
458 | "Missing on-disk cache entry for {:?}", | |
459 | dep_node); | |
83c7162d XL |
460 | result |
461 | } else { | |
462 | // Some things are never cached on disk. | |
463 | None | |
464 | }; | |
465 | ||
466 | let result = if let Some(result) = result { | |
467 | result | |
468 | } else { | |
469 | // We could not load a result from the on-disk cache, so | |
470 | // recompute. | |
471 | ||
472 | // The diagnostics for this query have already been | |
473 | // promoted to the current session during | |
474 | // try_mark_green(), so we can ignore them here. | |
475 | let (result, _) = job.start(self, |tcx| { | |
476 | // The dep-graph for this computation is already in | |
477 | // place | |
478 | tcx.dep_graph.with_ignore(|| { | |
479 | Q::compute(tcx, key) | |
480 | }) | |
481 | }); | |
482 | result | |
483 | }; | |
484 | ||
485 | // If -Zincremental-verify-ich is specified, re-hash results from | |
486 | // the cache and make sure that they have the expected fingerprint. | |
487 | if self.sess.opts.debugging_opts.incremental_verify_ich { | |
488 | use rustc_data_structures::stable_hasher::{StableHasher, HashStable}; | |
489 | use ich::Fingerprint; | |
490 | ||
491 | assert!(Some(self.dep_graph.fingerprint_of(dep_node_index)) == | |
492 | self.dep_graph.prev_fingerprint_of(dep_node), | |
493 | "Fingerprint for green query instance not loaded \ | |
0bf4aa26 | 494 | from cache: {:?}", dep_node); |
83c7162d XL |
495 | |
496 | debug!("BEGIN verify_ich({:?})", dep_node); | |
497 | let mut hcx = self.create_stable_hashing_context(); | |
498 | let mut hasher = StableHasher::new(); | |
499 | ||
500 | result.hash_stable(&mut hcx, &mut hasher); | |
501 | ||
502 | let new_hash: Fingerprint = hasher.finish(); | |
503 | debug!("END verify_ich({:?})", dep_node); | |
504 | ||
505 | let old_hash = self.dep_graph.fingerprint_of(dep_node_index); | |
506 | ||
507 | assert!(new_hash == old_hash, "Found unstable fingerprints \ | |
508 | for {:?}", dep_node); | |
509 | } | |
510 | ||
511 | if self.sess.opts.debugging_opts.query_dep_graph { | |
512 | self.dep_graph.mark_loaded_from_cache(dep_node_index, true); | |
513 | } | |
514 | ||
515 | job.complete(&result, dep_node_index); | |
516 | ||
517 | Ok(result) | |
518 | } | |
519 | ||
520 | fn force_query_with_job<Q: QueryDescription<'gcx>>( | |
521 | self, | |
522 | key: Q::Key, | |
523 | job: JobOwner<'_, 'gcx, Q>, | |
524 | dep_node: DepNode) | |
525 | -> Result<(Q::Value, DepNodeIndex), CycleError<'gcx>> { | |
526 | // If the following assertion triggers, it can have two reasons: | |
527 | // 1. Something is wrong with DepNode creation, either here or | |
528 | // in DepGraph::try_mark_green() | |
529 | // 2. Two distinct query keys get mapped to the same DepNode | |
530 | // (see for example #48923) | |
531 | assert!(!self.dep_graph.dep_node_exists(&dep_node), | |
532 | "Forcing query with already existing DepNode.\n\ | |
0bf4aa26 XL |
533 | - query-key: {:?}\n\ |
534 | - dep-node: {:?}", | |
83c7162d XL |
535 | key, dep_node); |
536 | ||
537 | profq_msg!(self, ProfileQueriesMsg::ProviderBegin); | |
b7449926 XL |
538 | self.sess.profiler(|p| { |
539 | p.start_activity(Q::CATEGORY); | |
540 | p.record_query(Q::CATEGORY); | |
541 | }); | |
542 | ||
83c7162d XL |
543 | let res = job.start(self, |tcx| { |
544 | if dep_node.kind.is_eval_always() { | |
545 | tcx.dep_graph.with_eval_always_task(dep_node, | |
546 | tcx, | |
547 | key, | |
548 | Q::compute) | |
549 | } else { | |
550 | tcx.dep_graph.with_task(dep_node, | |
551 | tcx, | |
552 | key, | |
553 | Q::compute) | |
554 | } | |
555 | }); | |
b7449926 XL |
556 | |
557 | self.sess.profiler(|p| p.end_activity(Q::CATEGORY)); | |
83c7162d XL |
558 | profq_msg!(self, ProfileQueriesMsg::ProviderEnd); |
559 | ||
560 | let ((result, dep_node_index), diagnostics) = res; | |
561 | ||
562 | if self.sess.opts.debugging_opts.query_dep_graph { | |
563 | self.dep_graph.mark_loaded_from_cache(dep_node_index, false); | |
564 | } | |
565 | ||
566 | if dep_node.kind != ::dep_graph::DepKind::Null { | |
94b46f34 | 567 | self.queries.on_disk_cache |
83c7162d XL |
568 | .store_diagnostics(dep_node_index, diagnostics); |
569 | } | |
570 | ||
571 | job.complete(&result, dep_node_index); | |
572 | ||
573 | Ok((result, dep_node_index)) | |
574 | } | |
575 | ||
576 | /// Ensure that either this query has all green inputs or been executed. | |
577 | /// Executing query::ensure(D) is considered a read of the dep-node D. | |
578 | /// | |
579 | /// This function is particularly useful when executing passes for their | |
580 | /// side-effects -- e.g., in order to report errors for erroneous programs. | |
581 | /// | |
582 | /// Note: The optimization is only available during incr. comp. | |
94b46f34 | 583 | pub(super) fn ensure_query<Q: QueryDescription<'gcx>>(self, key: Q::Key) -> () { |
83c7162d XL |
584 | let dep_node = Q::to_dep_node(self, &key); |
585 | ||
586 | // Ensuring an "input" or anonymous query makes no sense | |
587 | assert!(!dep_node.kind.is_anon()); | |
588 | assert!(!dep_node.kind.is_input()); | |
589 | if self.try_mark_green_and_read(&dep_node).is_none() { | |
590 | // A None return from `try_mark_green_and_read` means that this is either | |
591 | // a new dep node or that the dep node has already been marked red. | |
592 | // Either way, we can't call `dep_graph.read()` as we don't have the | |
593 | // DepNodeIndex. We must invoke the query itself. The performance cost | |
594 | // this introduces should be negligible as we'll immediately hit the | |
595 | // in-memory cache, or another query down the line will. | |
b7449926 XL |
596 | |
597 | self.sess.profiler(|p| { | |
598 | p.start_activity(Q::CATEGORY); | |
599 | p.record_query(Q::CATEGORY); | |
600 | }); | |
601 | ||
83c7162d | 602 | let _ = self.get_query::<Q>(DUMMY_SP, key); |
b7449926 XL |
603 | |
604 | self.sess.profiler(|p| p.end_activity(Q::CATEGORY)); | |
83c7162d XL |
605 | } |
606 | } | |
607 | ||
608 | #[allow(dead_code)] | |
609 | fn force_query<Q: QueryDescription<'gcx>>( | |
610 | self, | |
611 | key: Q::Key, | |
612 | span: Span, | |
613 | dep_node: DepNode | |
614 | ) -> Result<(Q::Value, DepNodeIndex), CycleError<'gcx>> { | |
615 | // We may be concurrently trying both execute and force a query | |
616 | // Ensure that only one of them runs the query | |
617 | let job = match JobOwner::try_get(self, span, &key) { | |
618 | TryGetJob::NotYetStarted(job) => job, | |
619 | TryGetJob::JobCompleted(result) => return result, | |
620 | }; | |
621 | self.force_query_with_job::<Q>(key, job, dep_node) | |
622 | } | |
623 | ||
94b46f34 | 624 | pub(super) fn try_get_query<Q: QueryDescription<'gcx>>( |
83c7162d XL |
625 | self, |
626 | span: Span, | |
94b46f34 | 627 | key: Q::Key, |
83c7162d XL |
628 | ) -> Result<Q::Value, DiagnosticBuilder<'a>> { |
629 | match self.try_get_with::<Q>(span, key) { | |
630 | Ok(e) => Ok(e), | |
631 | Err(e) => Err(self.report_cycle(e)), | |
632 | } | |
633 | } | |
634 | ||
94b46f34 XL |
635 | pub(super) fn get_query<Q: QueryDescription<'gcx>>( |
636 | self, | |
637 | span: Span, | |
638 | key: Q::Key, | |
639 | ) -> Q::Value { | |
83c7162d XL |
640 | self.try_get_query::<Q>(span, key).unwrap_or_else(|mut e| { |
641 | e.emit(); | |
642 | Q::handle_cycle_error(self) | |
643 | }) | |
ea8adc8c XL |
644 | } |
645 | } | |
646 | ||
0531ce1d XL |
647 | macro_rules! handle_cycle_error { |
648 | ([][$this: expr]) => {{ | |
649 | Value::from_cycle_error($this.global_tcx()) | |
650 | }}; | |
651 | ([fatal_cycle$(, $modifiers:ident)*][$this:expr]) => {{ | |
83c7162d | 652 | $this.sess.abort_if_errors(); |
0531ce1d XL |
653 | unreachable!(); |
654 | }}; | |
655 | ([$other:ident$(, $modifiers:ident)*][$($args:tt)*]) => { | |
656 | handle_cycle_error!([$($modifiers),*][$($args)*]) | |
657 | }; | |
658 | } | |
659 | ||
94b46f34 | 660 | macro_rules! define_queries { |
8faf50e0 XL |
661 | (<$tcx:tt> $($category:tt { |
662 | $($(#[$attr:meta])* [$($modifiers:tt)*] fn $name:ident: $node:ident($K:ty) -> $V:ty,)* | |
663 | },)*) => { | |
664 | define_queries_inner! { <$tcx> | |
665 | $($( $(#[$attr])* category<$category> [$($modifiers)*] fn $name: $node($K) -> $V,)*)* | |
666 | } | |
667 | } | |
668 | } | |
669 | ||
670 | macro_rules! define_queries_inner { | |
ea8adc8c | 671 | (<$tcx:tt> |
8faf50e0 XL |
672 | $($(#[$attr:meta])* category<$category:tt> |
673 | [$($modifiers:tt)*] fn $name:ident: $node:ident($K:ty) -> $V:ty,)*) => { | |
ea8adc8c | 674 | |
94b46f34 XL |
675 | use std::mem; |
676 | #[cfg(parallel_queries)] | |
677 | use ty::query::job::QueryResult; | |
83c7162d | 678 | use rustc_data_structures::sync::Lock; |
94b46f34 XL |
679 | use { |
680 | rustc_data_structures::stable_hasher::HashStable, | |
681 | rustc_data_structures::stable_hasher::StableHasherResult, | |
682 | rustc_data_structures::stable_hasher::StableHasher, | |
683 | ich::StableHashingContext | |
684 | }; | |
b7449926 | 685 | use util::profiling::ProfileCategory; |
ea8adc8c | 686 | |
94b46f34 | 687 | define_queries_struct! { |
ea8adc8c XL |
688 | tcx: $tcx, |
689 | input: ($(([$($modifiers)*] [$($attr)*] [$name]))*) | |
690 | } | |
691 | ||
94b46f34 XL |
692 | impl<$tcx> Queries<$tcx> { |
693 | pub fn new( | |
694 | providers: IndexVec<CrateNum, Providers<$tcx>>, | |
b7449926 | 695 | fallback_extern_providers: Providers<$tcx>, |
94b46f34 XL |
696 | on_disk_cache: OnDiskCache<'tcx>, |
697 | ) -> Self { | |
698 | Queries { | |
ea8adc8c | 699 | providers, |
b7449926 | 700 | fallback_extern_providers: Box::new(fallback_extern_providers), |
94b46f34 | 701 | on_disk_cache, |
0bf4aa26 | 702 | $($name: Default::default()),* |
ea8adc8c XL |
703 | } |
704 | } | |
94b46f34 XL |
705 | |
706 | #[cfg(parallel_queries)] | |
707 | pub fn collect_active_jobs(&self) -> Vec<Lrc<QueryJob<$tcx>>> { | |
708 | let mut jobs = Vec::new(); | |
709 | ||
710 | // We use try_lock here since we are only called from the | |
711 | // deadlock handler, and this shouldn't be locked | |
0bf4aa26 XL |
712 | $( |
713 | jobs.extend( | |
714 | self.$name.try_lock().unwrap().active.values().filter_map(|v| | |
715 | if let QueryResult::Started(ref job) = *v { | |
716 | Some(job.clone()) | |
717 | } else { | |
718 | None | |
719 | } | |
720 | ) | |
721 | ); | |
722 | )* | |
723 | ||
724 | jobs | |
94b46f34 | 725 | } |
ea8adc8c XL |
726 | } |
727 | ||
b7449926 | 728 | #[allow(nonstandard_style)] |
94b46f34 | 729 | #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] |
ea8adc8c XL |
730 | pub enum Query<$tcx> { |
731 | $($(#[$attr])* $name($K)),* | |
732 | } | |
733 | ||
ea8adc8c | 734 | impl<$tcx> Query<$tcx> { |
83c7162d XL |
735 | pub fn name(&self) -> &'static str { |
736 | match *self { | |
737 | $(Query::$name(_) => stringify!($name),)* | |
738 | } | |
739 | } | |
740 | ||
0bf4aa26 | 741 | pub fn describe(&self, tcx: TyCtxt<'_, '_, '_>) -> Cow<'static, str> { |
ea8adc8c XL |
742 | let (r, name) = match *self { |
743 | $(Query::$name(key) => { | |
744 | (queries::$name::describe(tcx, key), stringify!($name)) | |
745 | })* | |
746 | }; | |
747 | if tcx.sess.verbose() { | |
0bf4aa26 | 748 | format!("{} [{}]", r, name).into() |
ea8adc8c XL |
749 | } else { |
750 | r | |
751 | } | |
752 | } | |
83c7162d XL |
753 | |
754 | // FIXME(eddyb) Get more valid Span's on queries. | |
755 | pub fn default_span(&self, tcx: TyCtxt<'_, $tcx, '_>, span: Span) -> Span { | |
8faf50e0 | 756 | if !span.is_dummy() { |
83c7162d XL |
757 | return span; |
758 | } | |
759 | // The def_span query is used to calculate default_span, | |
760 | // so exit to avoid infinite recursion | |
0bf4aa26 XL |
761 | if let Query::def_span(..) = *self { |
762 | return span | |
83c7162d XL |
763 | } |
764 | match *self { | |
765 | $(Query::$name(key) => key.default_span(tcx),)* | |
766 | } | |
767 | } | |
ea8adc8c XL |
768 | } |
769 | ||
94b46f34 XL |
770 | impl<'a, $tcx> HashStable<StableHashingContext<'a>> for Query<$tcx> { |
771 | fn hash_stable<W: StableHasherResult>(&self, | |
772 | hcx: &mut StableHashingContext<'a>, | |
773 | hasher: &mut StableHasher<W>) { | |
774 | mem::discriminant(self).hash_stable(hcx, hasher); | |
775 | match *self { | |
776 | $(Query::$name(key) => key.hash_stable(hcx, hasher),)* | |
777 | } | |
778 | } | |
779 | } | |
780 | ||
ea8adc8c XL |
781 | pub mod queries { |
782 | use std::marker::PhantomData; | |
783 | ||
b7449926 | 784 | $(#[allow(nonstandard_style)] |
ea8adc8c XL |
785 | pub struct $name<$tcx> { |
786 | data: PhantomData<&$tcx ()> | |
787 | })* | |
788 | } | |
789 | ||
94b46f34 XL |
790 | // This module and the functions in it exist only to provide a |
791 | // predictable symbol name prefix for query providers. This is helpful | |
792 | // for analyzing queries in profilers. | |
793 | pub(super) mod __query_compute { | |
794 | $(#[inline(never)] | |
795 | pub fn $name<F: FnOnce() -> R, R>(f: F) -> R { | |
796 | f() | |
797 | })* | |
798 | } | |
799 | ||
83c7162d | 800 | $(impl<$tcx> QueryConfig<$tcx> for queries::$name<$tcx> { |
ea8adc8c XL |
801 | type Key = $K; |
802 | type Value = $V; | |
ea8adc8c | 803 | |
83c7162d | 804 | const NAME: &'static str = stringify!($name); |
b7449926 | 805 | const CATEGORY: ProfileCategory = $category; |
94b46f34 | 806 | } |
83c7162d | 807 | |
94b46f34 | 808 | impl<$tcx> QueryAccessors<$tcx> for queries::$name<$tcx> { |
83c7162d XL |
809 | fn query(key: Self::Key) -> Query<'tcx> { |
810 | Query::$name(key) | |
abe05a73 | 811 | } |
abe05a73 | 812 | |
94b46f34 XL |
813 | fn query_cache<'a>(tcx: TyCtxt<'a, $tcx, '_>) -> &'a Lock<QueryCache<$tcx, Self>> { |
814 | &tcx.queries.$name | |
83c7162d | 815 | } |
ea8adc8c XL |
816 | |
817 | #[allow(unused)] | |
83c7162d | 818 | fn to_dep_node(tcx: TyCtxt<'_, $tcx, '_>, key: &Self::Key) -> DepNode { |
ea8adc8c XL |
819 | use dep_graph::DepConstructor::*; |
820 | ||
821 | DepNode::new(tcx, $node(*key)) | |
822 | } | |
823 | ||
94b46f34 | 824 | #[inline] |
83c7162d | 825 | fn compute(tcx: TyCtxt<'_, 'tcx, '_>, key: Self::Key) -> Self::Value { |
94b46f34 | 826 | __query_compute::$name(move || { |
b7449926 XL |
827 | let provider = tcx.queries.providers.get(key.query_crate()) |
828 | // HACK(eddyb) it's possible crates may be loaded after | |
829 | // the query engine is created, and because crate loading | |
830 | // is not yet integrated with the query engine, such crates | |
831 | // would be be missing appropriate entries in `providers`. | |
832 | .unwrap_or(&tcx.queries.fallback_extern_providers) | |
833 | .$name; | |
94b46f34 XL |
834 | provider(tcx.global_tcx(), key) |
835 | }) | |
83c7162d | 836 | } |
ea8adc8c | 837 | |
83c7162d XL |
838 | fn handle_cycle_error(tcx: TyCtxt<'_, 'tcx, '_>) -> Self::Value { |
839 | handle_cycle_error!([$($modifiers)*][tcx]) | |
ea8adc8c | 840 | } |
83c7162d | 841 | } |
ea8adc8c | 842 | |
83c7162d | 843 | impl<'a, $tcx, 'lcx> queries::$name<$tcx> { |
abe05a73 XL |
844 | /// Ensure that either this query has all green inputs or been executed. |
845 | /// Executing query::ensure(D) is considered a read of the dep-node D. | |
846 | /// | |
847 | /// This function is particularly useful when executing passes for their | |
848 | /// side-effects -- e.g., in order to report errors for erroneous programs. | |
849 | /// | |
850 | /// Note: The optimization is only available during incr. comp. | |
851 | pub fn ensure(tcx: TyCtxt<'a, $tcx, 'lcx>, key: $K) -> () { | |
0bf4aa26 | 852 | tcx.ensure_query::<queries::$name<'_>>(key); |
ea8adc8c XL |
853 | } |
854 | })* | |
855 | ||
856 | #[derive(Copy, Clone)] | |
857 | pub struct TyCtxtAt<'a, 'gcx: 'a+'tcx, 'tcx: 'a> { | |
858 | pub tcx: TyCtxt<'a, 'gcx, 'tcx>, | |
859 | pub span: Span, | |
860 | } | |
861 | ||
862 | impl<'a, 'gcx, 'tcx> Deref for TyCtxtAt<'a, 'gcx, 'tcx> { | |
863 | type Target = TyCtxt<'a, 'gcx, 'tcx>; | |
864 | fn deref(&self) -> &Self::Target { | |
865 | &self.tcx | |
866 | } | |
867 | } | |
868 | ||
869 | impl<'a, $tcx, 'lcx> TyCtxt<'a, $tcx, 'lcx> { | |
870 | /// Return a transparent wrapper for `TyCtxt` which uses | |
871 | /// `span` as the location of queries performed through it. | |
872 | pub fn at(self, span: Span) -> TyCtxtAt<'a, $tcx, 'lcx> { | |
873 | TyCtxtAt { | |
874 | tcx: self, | |
875 | span | |
876 | } | |
877 | } | |
878 | ||
879 | $($(#[$attr])* | |
880 | pub fn $name(self, key: $K) -> $V { | |
881 | self.at(DUMMY_SP).$name(key) | |
882 | })* | |
883 | } | |
884 | ||
885 | impl<'a, $tcx, 'lcx> TyCtxtAt<'a, $tcx, 'lcx> { | |
886 | $($(#[$attr])* | |
887 | pub fn $name(self, key: $K) -> $V { | |
0bf4aa26 | 888 | self.tcx.get_query::<queries::$name<'_>>(self.span, key) |
ea8adc8c XL |
889 | })* |
890 | } | |
891 | ||
892 | define_provider_struct! { | |
893 | tcx: $tcx, | |
abe05a73 | 894 | input: ($(([$($modifiers)*] [$name] [$K] [$V]))*) |
ea8adc8c XL |
895 | } |
896 | ||
897 | impl<$tcx> Copy for Providers<$tcx> {} | |
898 | impl<$tcx> Clone for Providers<$tcx> { | |
899 | fn clone(&self) -> Self { *self } | |
900 | } | |
901 | } | |
902 | } | |
903 | ||
94b46f34 | 904 | macro_rules! define_queries_struct { |
ea8adc8c | 905 | (tcx: $tcx:tt, |
0531ce1d | 906 | input: ($(([$($modifiers:tt)*] [$($attr:tt)*] [$name:ident]))*)) => { |
94b46f34 XL |
907 | pub(crate) struct Queries<$tcx> { |
908 | /// This provides access to the incr. comp. on-disk cache for query results. | |
909 | /// Do not access this directly. It is only meant to be used by | |
910 | /// `DepGraph::try_mark_green()` and the query infrastructure. | |
911 | pub(crate) on_disk_cache: OnDiskCache<'tcx>, | |
912 | ||
ea8adc8c | 913 | providers: IndexVec<CrateNum, Providers<$tcx>>, |
b7449926 | 914 | fallback_extern_providers: Box<Providers<$tcx>>, |
94b46f34 XL |
915 | |
916 | $($(#[$attr])* $name: Lock<QueryCache<$tcx, queries::$name<$tcx>>>,)* | |
ea8adc8c XL |
917 | } |
918 | }; | |
919 | } | |
920 | ||
921 | macro_rules! define_provider_struct { | |
ea8adc8c | 922 | (tcx: $tcx:tt, |
abe05a73 | 923 | input: ($(([$($modifiers:tt)*] [$name:ident] [$K:ty] [$R:ty]))*)) => { |
ea8adc8c XL |
924 | pub struct Providers<$tcx> { |
925 | $(pub $name: for<'a> fn(TyCtxt<'a, $tcx, $tcx>, $K) -> $R,)* | |
926 | } | |
927 | ||
928 | impl<$tcx> Default for Providers<$tcx> { | |
929 | fn default() -> Self { | |
930 | $(fn $name<'a, $tcx>(_: TyCtxt<'a, $tcx, $tcx>, key: $K) -> $R { | |
94b46f34 | 931 | bug!("tcx.{}({:?}) unsupported by its crate", |
ea8adc8c XL |
932 | stringify!($name), key); |
933 | })* | |
934 | Providers { $($name),* } | |
935 | } | |
936 | } | |
937 | }; | |
ea8adc8c XL |
938 | } |
939 | ||
abe05a73 XL |
940 | |
941 | /// The red/green evaluation system will try to mark a specific DepNode in the | |
942 | /// dependency graph as green by recursively trying to mark the dependencies of | |
943 | /// that DepNode as green. While doing so, it will sometimes encounter a DepNode | |
944 | /// where we don't know if it is red or green and we therefore actually have | |
945 | /// to recompute its value in order to find out. Since the only piece of | |
946 | /// information that we have at that point is the DepNode we are trying to | |
947 | /// re-evaluate, we need some way to re-run a query from just that. This is what | |
948 | /// `force_from_dep_node()` implements. | |
949 | /// | |
950 | /// In the general case, a DepNode consists of a DepKind and an opaque | |
951 | /// GUID/fingerprint that will uniquely identify the node. This GUID/fingerprint | |
952 | /// is usually constructed by computing a stable hash of the query-key that the | |
953 | /// DepNode corresponds to. Consequently, it is not in general possible to go | |
954 | /// back from hash to query-key (since hash functions are not reversible). For | |
955 | /// this reason `force_from_dep_node()` is expected to fail from time to time | |
956 | /// because we just cannot find out, from the DepNode alone, what the | |
957 | /// corresponding query-key is and therefore cannot re-run the query. | |
958 | /// | |
959 | /// The system deals with this case letting `try_mark_green` fail which forces | |
960 | /// the root query to be re-evaluated. | |
961 | /// | |
962 | /// Now, if force_from_dep_node() would always fail, it would be pretty useless. | |
963 | /// Fortunately, we can use some contextual information that will allow us to | |
964 | /// reconstruct query-keys for certain kinds of DepNodes. In particular, we | |
965 | /// enforce by construction that the GUID/fingerprint of certain DepNodes is a | |
966 | /// valid DefPathHash. Since we also always build a huge table that maps every | |
967 | /// DefPathHash in the current codebase to the corresponding DefId, we have | |
968 | /// everything we need to re-run the query. | |
969 | /// | |
970 | /// Take the `mir_validated` query as an example. Like many other queries, it | |
971 | /// just has a single parameter: the DefId of the item it will compute the | |
972 | /// validated MIR for. Now, when we call `force_from_dep_node()` on a dep-node | |
973 | /// with kind `MirValidated`, we know that the GUID/fingerprint of the dep-node | |
974 | /// is actually a DefPathHash, and can therefore just look up the corresponding | |
975 | /// DefId in `tcx.def_path_hash_to_def_id`. | |
976 | /// | |
977 | /// When you implement a new query, it will likely have a corresponding new | |
978 | /// DepKind, and you'll have to support it here in `force_from_dep_node()`. As | |
979 | /// a rule of thumb, if your query takes a DefId or DefIndex as sole parameter, | |
980 | /// then `force_from_dep_node()` should not fail for it. Otherwise, you can just | |
981 | /// add it to the "We don't have enough information to reconstruct..." group in | |
982 | /// the match below. | |
ea8adc8c XL |
983 | pub fn force_from_dep_node<'a, 'gcx, 'lcx>(tcx: TyCtxt<'a, 'gcx, 'lcx>, |
984 | dep_node: &DepNode) | |
985 | -> bool { | |
ea8adc8c XL |
986 | use hir::def_id::LOCAL_CRATE; |
987 | ||
988 | // We must avoid ever having to call force_from_dep_node() for a | |
989 | // DepNode::CodegenUnit: | |
990 | // Since we cannot reconstruct the query key of a DepNode::CodegenUnit, we | |
991 | // would always end up having to evaluate the first caller of the | |
992 | // `codegen_unit` query that *is* reconstructible. This might very well be | |
94b46f34 | 993 | // the `compile_codegen_unit` query, thus re-codegenning the whole CGU just |
ea8adc8c XL |
994 | // to re-trigger calling the `codegen_unit` query with the right key. At |
995 | // that point we would already have re-done all the work we are trying to | |
996 | // avoid doing in the first place. | |
997 | // The solution is simple: Just explicitly call the `codegen_unit` query for | |
998 | // each CGU, right after partitioning. This way `try_mark_green` will always | |
999 | // hit the cache instead of having to go through `force_from_dep_node`. | |
1000 | // This assertion makes sure, we actually keep applying the solution above. | |
1001 | debug_assert!(dep_node.kind != DepKind::CodegenUnit, | |
1002 | "calling force_from_dep_node() on DepKind::CodegenUnit"); | |
1003 | ||
1004 | if !dep_node.kind.can_reconstruct_query_key() { | |
1005 | return false | |
1006 | } | |
1007 | ||
1008 | macro_rules! def_id { | |
1009 | () => { | |
1010 | if let Some(def_id) = dep_node.extract_def_id(tcx) { | |
1011 | def_id | |
1012 | } else { | |
1013 | // return from the whole function | |
1014 | return false | |
1015 | } | |
1016 | } | |
1017 | }; | |
1018 | ||
1019 | macro_rules! krate { | |
1020 | () => { (def_id!()).krate } | |
1021 | }; | |
1022 | ||
1023 | macro_rules! force { | |
1024 | ($query:ident, $key:expr) => { | |
1025 | { | |
1026 | use $crate::util::common::{ProfileQueriesMsg, profq_msg}; | |
1027 | ||
ea8adc8c XL |
1028 | profq_msg!(tcx, |
1029 | ProfileQueriesMsg::QueryBegin( | |
83c7162d | 1030 | DUMMY_SP.data(), |
94b46f34 | 1031 | profq_query_msg!(::ty::query::queries::$query::NAME, tcx, $key), |
ea8adc8c XL |
1032 | ) |
1033 | ); | |
1034 | ||
0bf4aa26 XL |
1035 | if let Err(e) = tcx.force_query::<::ty::query::queries::$query<'_>>( |
1036 | $key, DUMMY_SP, *dep_node | |
1037 | ) { | |
1038 | tcx.report_cycle(e).emit(); | |
ea8adc8c XL |
1039 | } |
1040 | } | |
1041 | } | |
1042 | }; | |
1043 | ||
1044 | // FIXME(#45015): We should try move this boilerplate code into a macro | |
1045 | // somehow. | |
1046 | match dep_node.kind { | |
1047 | // These are inputs that are expected to be pre-allocated and that | |
1048 | // should therefore always be red or green already | |
1049 | DepKind::AllLocalTraitImpls | | |
1050 | DepKind::Krate | | |
1051 | DepKind::CrateMetadata | | |
1052 | DepKind::HirBody | | |
1053 | DepKind::Hir | | |
1054 | ||
1055 | // This are anonymous nodes | |
abe05a73 XL |
1056 | DepKind::TraitSelect | |
1057 | ||
1058 | // We don't have enough information to reconstruct the query key of | |
1059 | // these | |
ea8adc8c XL |
1060 | DepKind::IsCopy | |
1061 | DepKind::IsSized | | |
1062 | DepKind::IsFreeze | | |
1063 | DepKind::NeedsDrop | | |
1064 | DepKind::Layout | | |
ea8adc8c | 1065 | DepKind::ConstEval | |
a1dfa0c6 | 1066 | DepKind::ConstEvalRaw | |
ea8adc8c XL |
1067 | DepKind::InstanceSymbolName | |
1068 | DepKind::MirShim | | |
1069 | DepKind::BorrowCheckKrate | | |
1070 | DepKind::Specializes | | |
1071 | DepKind::ImplementationsOfTrait | | |
1072 | DepKind::TypeParamPredicates | | |
1073 | DepKind::CodegenUnit | | |
1074 | DepKind::CompileCodegenUnit | | |
abe05a73 XL |
1075 | DepKind::FulfillObligation | |
1076 | DepKind::VtableMethods | | |
1077 | DepKind::EraseRegionsTy | | |
0531ce1d XL |
1078 | DepKind::NormalizeProjectionTy | |
1079 | DepKind::NormalizeTyAfterErasingRegions | | |
8faf50e0 | 1080 | DepKind::ImpliedOutlivesBounds | |
0531ce1d | 1081 | DepKind::DropckOutlives | |
83c7162d | 1082 | DepKind::EvaluateObligation | |
0bf4aa26 | 1083 | DepKind::TypeOpAscribeUserType | |
8faf50e0 XL |
1084 | DepKind::TypeOpEq | |
1085 | DepKind::TypeOpSubtype | | |
1086 | DepKind::TypeOpProvePredicate | | |
1087 | DepKind::TypeOpNormalizeTy | | |
1088 | DepKind::TypeOpNormalizePredicate | | |
1089 | DepKind::TypeOpNormalizePolyFnSig | | |
1090 | DepKind::TypeOpNormalizeFnSig | | |
2c00a5a8 XL |
1091 | DepKind::SubstituteNormalizeAndTestPredicates | |
1092 | DepKind::InstanceDefSizeEstimate | | |
83c7162d | 1093 | DepKind::ProgramClausesForEnv | |
ea8adc8c | 1094 | |
abe05a73 XL |
1095 | // This one should never occur in this context |
1096 | DepKind::Null => { | |
1097 | bug!("force_from_dep_node() - Encountered {:?}", dep_node) | |
ea8adc8c XL |
1098 | } |
1099 | ||
1100 | // These are not queries | |
1101 | DepKind::CoherenceCheckTrait | | |
1102 | DepKind::ItemVarianceConstraints => { | |
1103 | return false | |
1104 | } | |
1105 | ||
1106 | DepKind::RegionScopeTree => { force!(region_scope_tree, def_id!()); } | |
1107 | ||
1108 | DepKind::Coherence => { force!(crate_inherent_impls, LOCAL_CRATE); } | |
1109 | DepKind::CoherenceInherentImplOverlapCheck => { | |
1110 | force!(crate_inherent_impls_overlap_check, LOCAL_CRATE) | |
1111 | }, | |
1112 | DepKind::PrivacyAccessLevels => { force!(privacy_access_levels, LOCAL_CRATE); } | |
1113 | DepKind::MirBuilt => { force!(mir_built, def_id!()); } | |
1114 | DepKind::MirConstQualif => { force!(mir_const_qualif, def_id!()); } | |
1115 | DepKind::MirConst => { force!(mir_const, def_id!()); } | |
1116 | DepKind::MirValidated => { force!(mir_validated, def_id!()); } | |
1117 | DepKind::MirOptimized => { force!(optimized_mir, def_id!()); } | |
1118 | ||
1119 | DepKind::BorrowCheck => { force!(borrowck, def_id!()); } | |
1120 | DepKind::MirBorrowCheck => { force!(mir_borrowck, def_id!()); } | |
1121 | DepKind::UnsafetyCheckResult => { force!(unsafety_check_result, def_id!()); } | |
ff7c6d11 | 1122 | DepKind::UnsafeDeriveOnReprPacked => { force!(unsafe_derive_on_repr_packed, def_id!()); } |
ea8adc8c XL |
1123 | DepKind::Reachability => { force!(reachable_set, LOCAL_CRATE); } |
1124 | DepKind::MirKeys => { force!(mir_keys, LOCAL_CRATE); } | |
1125 | DepKind::CrateVariances => { force!(crate_variances, LOCAL_CRATE); } | |
1126 | DepKind::AssociatedItems => { force!(associated_item, def_id!()); } | |
1127 | DepKind::TypeOfItem => { force!(type_of, def_id!()); } | |
1128 | DepKind::GenericsOfItem => { force!(generics_of, def_id!()); } | |
1129 | DepKind::PredicatesOfItem => { force!(predicates_of, def_id!()); } | |
8faf50e0 | 1130 | DepKind::PredicatesDefinedOnItem => { force!(predicates_defined_on, def_id!()); } |
94b46f34 | 1131 | DepKind::ExplicitPredicatesOfItem => { force!(explicit_predicates_of, def_id!()); } |
abe05a73 | 1132 | DepKind::InferredOutlivesOf => { force!(inferred_outlives_of, def_id!()); } |
83c7162d | 1133 | DepKind::InferredOutlivesCrate => { force!(inferred_outlives_crate, LOCAL_CRATE); } |
ea8adc8c XL |
1134 | DepKind::SuperPredicatesOfItem => { force!(super_predicates_of, def_id!()); } |
1135 | DepKind::TraitDefOfItem => { force!(trait_def, def_id!()); } | |
1136 | DepKind::AdtDefOfItem => { force!(adt_def, def_id!()); } | |
ea8adc8c XL |
1137 | DepKind::ImplTraitRef => { force!(impl_trait_ref, def_id!()); } |
1138 | DepKind::ImplPolarity => { force!(impl_polarity, def_id!()); } | |
ea8adc8c | 1139 | DepKind::FnSignature => { force!(fn_sig, def_id!()); } |
ea8adc8c XL |
1140 | DepKind::CoerceUnsizedInfo => { force!(coerce_unsized_info, def_id!()); } |
1141 | DepKind::ItemVariances => { force!(variances_of, def_id!()); } | |
0bf4aa26 XL |
1142 | DepKind::IsConstFn => { force!(is_const_fn_raw, def_id!()); } |
1143 | DepKind::IsPromotableConstFn => { force!(is_promotable_const_fn, def_id!()); } | |
ea8adc8c XL |
1144 | DepKind::IsForeignItem => { force!(is_foreign_item, def_id!()); } |
1145 | DepKind::SizedConstraint => { force!(adt_sized_constraint, def_id!()); } | |
1146 | DepKind::DtorckConstraint => { force!(adt_dtorck_constraint, def_id!()); } | |
1147 | DepKind::AdtDestructor => { force!(adt_destructor, def_id!()); } | |
1148 | DepKind::AssociatedItemDefIds => { force!(associated_item_def_ids, def_id!()); } | |
1149 | DepKind::InherentImpls => { force!(inherent_impls, def_id!()); } | |
1150 | DepKind::TypeckBodiesKrate => { force!(typeck_item_bodies, LOCAL_CRATE); } | |
1151 | DepKind::TypeckTables => { force!(typeck_tables_of, def_id!()); } | |
abe05a73 | 1152 | DepKind::UsedTraitImports => { force!(used_trait_imports, def_id!()); } |
ea8adc8c XL |
1153 | DepKind::HasTypeckTables => { force!(has_typeck_tables, def_id!()); } |
1154 | DepKind::SymbolName => { force!(def_symbol_name, def_id!()); } | |
1155 | DepKind::SpecializationGraph => { force!(specialization_graph_of, def_id!()); } | |
1156 | DepKind::ObjectSafety => { force!(is_object_safe, def_id!()); } | |
1157 | DepKind::TraitImpls => { force!(trait_impls_of, def_id!()); } | |
ff7c6d11 | 1158 | DepKind::CheckMatch => { force!(check_match, def_id!()); } |
ea8adc8c XL |
1159 | |
1160 | DepKind::ParamEnv => { force!(param_env, def_id!()); } | |
0bf4aa26 | 1161 | DepKind::Environment => { force!(environment, def_id!()); } |
ea8adc8c XL |
1162 | DepKind::DescribeDef => { force!(describe_def, def_id!()); } |
1163 | DepKind::DefSpan => { force!(def_span, def_id!()); } | |
1164 | DepKind::LookupStability => { force!(lookup_stability, def_id!()); } | |
1165 | DepKind::LookupDeprecationEntry => { | |
1166 | force!(lookup_deprecation_entry, def_id!()); | |
1167 | } | |
ea8adc8c XL |
1168 | DepKind::ConstIsRvaluePromotableToStatic => { |
1169 | force!(const_is_rvalue_promotable_to_static, def_id!()); | |
1170 | } | |
abe05a73 | 1171 | DepKind::RvaluePromotableMap => { force!(rvalue_promotable_map, def_id!()); } |
ea8adc8c XL |
1172 | DepKind::ImplParent => { force!(impl_parent, def_id!()); } |
1173 | DepKind::TraitOfItem => { force!(trait_of_item, def_id!()); } | |
0531ce1d | 1174 | DepKind::IsReachableNonGeneric => { force!(is_reachable_non_generic, def_id!()); } |
83c7162d XL |
1175 | DepKind::IsUnreachableLocalDefinition => { |
1176 | force!(is_unreachable_local_definition, def_id!()); | |
1177 | } | |
ea8adc8c XL |
1178 | DepKind::IsMirAvailable => { force!(is_mir_available, def_id!()); } |
1179 | DepKind::ItemAttrs => { force!(item_attrs, def_id!()); } | |
94b46f34 | 1180 | DepKind::CodegenFnAttrs => { force!(codegen_fn_attrs, def_id!()); } |
ea8adc8c | 1181 | DepKind::FnArgNames => { force!(fn_arg_names, def_id!()); } |
83c7162d | 1182 | DepKind::RenderedConst => { force!(rendered_const, def_id!()); } |
ea8adc8c XL |
1183 | DepKind::DylibDepFormats => { force!(dylib_dependency_formats, krate!()); } |
1184 | DepKind::IsPanicRuntime => { force!(is_panic_runtime, krate!()); } | |
1185 | DepKind::IsCompilerBuiltins => { force!(is_compiler_builtins, krate!()); } | |
1186 | DepKind::HasGlobalAllocator => { force!(has_global_allocator, krate!()); } | |
b7449926 | 1187 | DepKind::HasPanicHandler => { force!(has_panic_handler, krate!()); } |
ea8adc8c XL |
1188 | DepKind::ExternCrate => { force!(extern_crate, def_id!()); } |
1189 | DepKind::LintLevels => { force!(lint_levels, LOCAL_CRATE); } | |
1190 | DepKind::InScopeTraits => { force!(in_scope_traits_map, def_id!().index); } | |
1191 | DepKind::ModuleExports => { force!(module_exports, def_id!()); } | |
1192 | DepKind::IsSanitizerRuntime => { force!(is_sanitizer_runtime, krate!()); } | |
1193 | DepKind::IsProfilerRuntime => { force!(is_profiler_runtime, krate!()); } | |
1194 | DepKind::GetPanicStrategy => { force!(panic_strategy, krate!()); } | |
1195 | DepKind::IsNoBuiltins => { force!(is_no_builtins, krate!()); } | |
1196 | DepKind::ImplDefaultness => { force!(impl_defaultness, def_id!()); } | |
0531ce1d XL |
1197 | DepKind::CheckItemWellFormed => { force!(check_item_well_formed, def_id!()); } |
1198 | DepKind::CheckTraitItemWellFormed => { force!(check_trait_item_well_formed, def_id!()); } | |
1199 | DepKind::CheckImplItemWellFormed => { force!(check_impl_item_well_formed, def_id!()); } | |
1200 | DepKind::ReachableNonGenerics => { force!(reachable_non_generics, krate!()); } | |
ea8adc8c XL |
1201 | DepKind::NativeLibraries => { force!(native_libraries, krate!()); } |
1202 | DepKind::PluginRegistrarFn => { force!(plugin_registrar_fn, krate!()); } | |
a1dfa0c6 | 1203 | DepKind::ProcMacroDeclsStatic => { force!(proc_macro_decls_static, krate!()); } |
ea8adc8c XL |
1204 | DepKind::CrateDisambiguator => { force!(crate_disambiguator, krate!()); } |
1205 | DepKind::CrateHash => { force!(crate_hash, krate!()); } | |
1206 | DepKind::OriginalCrateName => { force!(original_crate_name, krate!()); } | |
83c7162d | 1207 | DepKind::ExtraFileName => { force!(extra_filename, krate!()); } |
ea8adc8c XL |
1208 | |
1209 | DepKind::AllTraitImplementations => { | |
1210 | force!(all_trait_implementations, krate!()); | |
1211 | } | |
1212 | ||
0531ce1d XL |
1213 | DepKind::DllimportForeignItems => { |
1214 | force!(dllimport_foreign_items, krate!()); | |
1215 | } | |
ea8adc8c XL |
1216 | DepKind::IsDllimportForeignItem => { |
1217 | force!(is_dllimport_foreign_item, def_id!()); | |
1218 | } | |
1219 | DepKind::IsStaticallyIncludedForeignItem => { | |
1220 | force!(is_statically_included_foreign_item, def_id!()); | |
1221 | } | |
1222 | DepKind::NativeLibraryKind => { force!(native_library_kind, def_id!()); } | |
1223 | DepKind::LinkArgs => { force!(link_args, LOCAL_CRATE); } | |
1224 | ||
ff7c6d11 | 1225 | DepKind::ResolveLifetimes => { force!(resolve_lifetimes, krate!()); } |
ea8adc8c XL |
1226 | DepKind::NamedRegion => { force!(named_region_map, def_id!().index); } |
1227 | DepKind::IsLateBound => { force!(is_late_bound_map, def_id!().index); } | |
1228 | DepKind::ObjectLifetimeDefaults => { | |
1229 | force!(object_lifetime_defaults_map, def_id!().index); | |
1230 | } | |
1231 | ||
1232 | DepKind::Visibility => { force!(visibility, def_id!()); } | |
1233 | DepKind::DepKind => { force!(dep_kind, krate!()); } | |
1234 | DepKind::CrateName => { force!(crate_name, krate!()); } | |
1235 | DepKind::ItemChildren => { force!(item_children, def_id!()); } | |
1236 | DepKind::ExternModStmtCnum => { force!(extern_mod_stmt_cnum, def_id!()); } | |
b7449926 XL |
1237 | DepKind::GetLibFeatures => { force!(get_lib_features, LOCAL_CRATE); } |
1238 | DepKind::DefinedLibFeatures => { force!(defined_lib_features, krate!()); } | |
ea8adc8c XL |
1239 | DepKind::GetLangItems => { force!(get_lang_items, LOCAL_CRATE); } |
1240 | DepKind::DefinedLangItems => { force!(defined_lang_items, krate!()); } | |
1241 | DepKind::MissingLangItems => { force!(missing_lang_items, krate!()); } | |
ea8adc8c XL |
1242 | DepKind::VisibleParentMap => { force!(visible_parent_map, LOCAL_CRATE); } |
1243 | DepKind::MissingExternCrateItem => { | |
1244 | force!(missing_extern_crate_item, krate!()); | |
1245 | } | |
1246 | DepKind::UsedCrateSource => { force!(used_crate_source, krate!()); } | |
1247 | DepKind::PostorderCnums => { force!(postorder_cnums, LOCAL_CRATE); } | |
ea8adc8c XL |
1248 | |
1249 | DepKind::Freevars => { force!(freevars, def_id!()); } | |
1250 | DepKind::MaybeUnusedTraitImport => { | |
1251 | force!(maybe_unused_trait_import, def_id!()); | |
1252 | } | |
1253 | DepKind::MaybeUnusedExternCrates => { force!(maybe_unused_extern_crates, LOCAL_CRATE); } | |
1254 | DepKind::StabilityIndex => { force!(stability_index, LOCAL_CRATE); } | |
83c7162d | 1255 | DepKind::AllTraits => { force!(all_traits, LOCAL_CRATE); } |
ea8adc8c XL |
1256 | DepKind::AllCrateNums => { force!(all_crate_nums, LOCAL_CRATE); } |
1257 | DepKind::ExportedSymbols => { force!(exported_symbols, krate!()); } | |
94b46f34 XL |
1258 | DepKind::CollectAndPartitionMonoItems => { |
1259 | force!(collect_and_partition_mono_items, LOCAL_CRATE); | |
ea8adc8c | 1260 | } |
94b46f34 | 1261 | DepKind::IsCodegenedItem => { force!(is_codegened_item, def_id!()); } |
ea8adc8c | 1262 | DepKind::OutputFilenames => { force!(output_filenames, LOCAL_CRATE); } |
2c00a5a8 XL |
1263 | |
1264 | DepKind::TargetFeaturesWhitelist => { force!(target_features_whitelist, LOCAL_CRATE); } | |
2c00a5a8 | 1265 | |
0531ce1d XL |
1266 | DepKind::Features => { force!(features_query, LOCAL_CRATE); } |
1267 | ||
1268 | DepKind::ProgramClausesFor => { force!(program_clauses_for, def_id!()); } | |
0531ce1d XL |
1269 | DepKind::WasmImportModuleMap => { force!(wasm_import_module_map, krate!()); } |
1270 | DepKind::ForeignModules => { force!(foreign_modules, krate!()); } | |
83c7162d XL |
1271 | |
1272 | DepKind::UpstreamMonomorphizations => { | |
1273 | force!(upstream_monomorphizations, krate!()); | |
1274 | } | |
1275 | DepKind::UpstreamMonomorphizationsFor => { | |
1276 | force!(upstream_monomorphizations_for, def_id!()); | |
1277 | } | |
ea8adc8c XL |
1278 | } |
1279 | ||
1280 | true | |
1281 | } | |
ff7c6d11 XL |
1282 | |
1283 | ||
1284 | // FIXME(#45015): Another piece of boilerplate code that could be generated in | |
94b46f34 | 1285 | // a combined define_dep_nodes!()/define_queries!() macro. |
ff7c6d11 XL |
1286 | macro_rules! impl_load_from_cache { |
1287 | ($($dep_kind:ident => $query_name:ident,)*) => { | |
1288 | impl DepNode { | |
1289 | // Check whether the query invocation corresponding to the given | |
1290 | // DepNode is eligible for on-disk-caching. | |
0bf4aa26 | 1291 | pub fn cache_on_disk(&self, tcx: TyCtxt<'_, '_, '_>) -> bool { |
94b46f34 XL |
1292 | use ty::query::queries; |
1293 | use ty::query::QueryDescription; | |
ff7c6d11 XL |
1294 | |
1295 | match self.kind { | |
1296 | $(DepKind::$dep_kind => { | |
1297 | let def_id = self.extract_def_id(tcx).unwrap(); | |
1298 | queries::$query_name::cache_on_disk(def_id) | |
1299 | })* | |
1300 | _ => false | |
1301 | } | |
1302 | } | |
1303 | ||
1304 | // This is method will execute the query corresponding to the given | |
1305 | // DepNode. It is only expected to work for DepNodes where the | |
1306 | // above `cache_on_disk` methods returns true. | |
1307 | // Also, as a sanity check, it expects that the corresponding query | |
1308 | // invocation has been marked as green already. | |
0bf4aa26 | 1309 | pub fn load_from_on_disk_cache(&self, tcx: TyCtxt<'_, '_, '_>) { |
ff7c6d11 XL |
1310 | match self.kind { |
1311 | $(DepKind::$dep_kind => { | |
1312 | debug_assert!(tcx.dep_graph | |
1313 | .node_color(self) | |
1314 | .map(|c| c.is_green()) | |
1315 | .unwrap_or(false)); | |
1316 | ||
1317 | let def_id = self.extract_def_id(tcx).unwrap(); | |
1318 | let _ = tcx.$query_name(def_id); | |
1319 | })* | |
1320 | _ => { | |
1321 | bug!() | |
1322 | } | |
1323 | } | |
1324 | } | |
1325 | } | |
1326 | } | |
1327 | } | |
1328 | ||
1329 | impl_load_from_cache!( | |
1330 | TypeckTables => typeck_tables_of, | |
1331 | MirOptimized => optimized_mir, | |
1332 | UnsafetyCheckResult => unsafety_check_result, | |
1333 | BorrowCheck => borrowck, | |
1334 | MirBorrowCheck => mir_borrowck, | |
1335 | MirConstQualif => mir_const_qualif, | |
1336 | SymbolName => def_symbol_name, | |
1337 | ConstIsRvaluePromotableToStatic => const_is_rvalue_promotable_to_static, | |
ff7c6d11 | 1338 | CheckMatch => check_match, |
2c00a5a8 XL |
1339 | TypeOfItem => type_of, |
1340 | GenericsOfItem => generics_of, | |
1341 | PredicatesOfItem => predicates_of, | |
1342 | UsedTraitImports => used_trait_imports, | |
94b46f34 | 1343 | CodegenFnAttrs => codegen_fn_attrs, |
0531ce1d | 1344 | SpecializationGraph => specialization_graph_of, |
ff7c6d11 | 1345 | ); |