compiler/rustc_query_impl/src/plumbing.rs

   1 //! The implementation of the query system itself. This defines the macros that
   2 //! generate the actual methods on tcx which find and execute the provider,
   3 //! manage the caches, and so forth.
   4
   5 use crate::rustc_middle::dep_graph::DepContext;
   6 use crate::rustc_middle::ty::TyEncoder;
   7 use crate::QueryConfigRestored;
   8 use rustc_data_structures::stable_hasher::{Hash64, HashStable, StableHasher};
   9 use rustc_data_structures::sync::Lock;
  10 use rustc_errors::Diagnostic;
  11
  12 use rustc_index::Idx;
  13 use rustc_middle::dep_graph::dep_kinds;
  14 use rustc_middle::dep_graph::{
  15     self, DepKind, DepKindStruct, DepNode, DepNodeIndex, SerializedDepNodeIndex,
  16 };
  17 use rustc_middle::query::on_disk_cache::AbsoluteBytePos;
  18 use rustc_middle::query::on_disk_cache::{CacheDecoder, CacheEncoder, EncodedDepNodeIndex};
  19 use rustc_middle::query::Key;
  20 use rustc_middle::ty::tls::{self, ImplicitCtxt};
  21 use rustc_middle::ty::{self, print::with_no_queries, TyCtxt};
  22 use rustc_query_system::dep_graph::{DepNodeParams, HasDepContext};
  23 use rustc_query_system::ich::StableHashingContext;
  24 use rustc_query_system::query::{
  25     force_query, QueryCache, QueryConfig, QueryContext, QueryJobId, QueryMap, QuerySideEffects,
  26     QueryStackFrame,
  27 };
  28 use rustc_query_system::{LayoutOfDepth, QueryOverflow};
  29 use rustc_serialize::Decodable;
  30 use rustc_serialize::Encodable;
  31 use rustc_session::Limit;
  32 use rustc_span::def_id::LOCAL_CRATE;
  33 use std::num::NonZeroU64;
  34 use thin_vec::ThinVec;
  35
  36 #[derive(Copy, Clone)]
  37 pub struct QueryCtxt<'tcx> {
  38     pub tcx: TyCtxt<'tcx>,
  39 }
  40
  41 impl<'tcx> QueryCtxt<'tcx> {
  42     #[inline]
  43     pub fn new(tcx: TyCtxt<'tcx>) -> Self {
  44         QueryCtxt { tcx }
  45     }
  46 }
  47
  48 impl<'tcx> std::ops::Deref for QueryCtxt<'tcx> {
  49     type Target = TyCtxt<'tcx>;
  50
  51     #[inline]
  52     fn deref(&self) -> &Self::Target {
  53         &self.tcx
  54     }
  55 }
  56
  57 impl<'tcx> HasDepContext for QueryCtxt<'tcx> {
  58     type Deps = rustc_middle::dep_graph::DepsType;
  59     type DepContext = TyCtxt<'tcx>;
  60
  61     #[inline]
  62     fn dep_context(&self) -> &Self::DepContext {
  63         &self.tcx
  64     }
  65 }
  66
  67 impl QueryContext for QueryCtxt<'_> {
  68     #[inline]
  69     fn next_job_id(self) -> QueryJobId {
  70         QueryJobId(
  71             NonZeroU64::new(
  72                 self.query_system.jobs.fetch_add(1, rustc_data_structures::sync::Ordering::Relaxed),
  73             )
  74             .unwrap(),
  75         )
  76     }
  77
  78     #[inline]
  79     fn current_query_job(self) -> Option<QueryJobId> {
  80         tls::with_related_context(self.tcx, |icx| icx.query)
  81     }
  82
  83     fn try_collect_active_jobs(self) -> Option<QueryMap> {
  84         let mut jobs = QueryMap::default();
  85
  86         for collect in super::TRY_COLLECT_ACTIVE_JOBS.iter() {
  87             collect(self.tcx, &mut jobs);
  88         }
  89
  90         Some(jobs)
  91     }
  92
  93     // Interactions with on_disk_cache
  94     fn load_side_effects(self, prev_dep_node_index: SerializedDepNodeIndex) -> QuerySideEffects {
  95         self.query_system
  96             .on_disk_cache
  97             .as_ref()
  98             .map(|c| c.load_side_effects(self.tcx, prev_dep_node_index))
  99             .unwrap_or_default()
 100     }
 101
 102     #[inline(never)]
 103     #[cold]
 104     fn store_side_effects(self, dep_node_index: DepNodeIndex, side_effects: QuerySideEffects) {
 105         if let Some(c) = self.query_system.on_disk_cache.as_ref() {
 106             c.store_side_effects(dep_node_index, side_effects)
 107         }
 108     }
 109
 110     #[inline(never)]
 111     #[cold]
 112     fn store_side_effects_for_anon_node(
 113         self,
 114         dep_node_index: DepNodeIndex,
 115         side_effects: QuerySideEffects,
 116     ) {
 117         if let Some(c) = self.query_system.on_disk_cache.as_ref() {
 118             c.store_side_effects_for_anon_node(dep_node_index, side_effects)
 119         }
 120     }
 121
 122     /// Executes a job by changing the `ImplicitCtxt` to point to the
 123     /// new query job while it executes. It returns the diagnostics
 124     /// captured during execution and the actual result.
 125     #[inline(always)]
 126     fn start_query<R>(
 127         self,
 128         token: QueryJobId,
 129         depth_limit: bool,
 130         diagnostics: Option<&Lock<ThinVec<Diagnostic>>>,
 131         compute: impl FnOnce() -> R,
 132     ) -> R {
 133         // The `TyCtxt` stored in TLS has the same global interner lifetime
 134         // as `self`, so we use `with_related_context` to relate the 'tcx lifetimes
 135         // when accessing the `ImplicitCtxt`.
 136         tls::with_related_context(self.tcx, move |current_icx| {
 137             if depth_limit && !self.recursion_limit().value_within_limit(current_icx.query_depth) {
 138                 self.depth_limit_error(token);
 139             }
 140
 141             // Update the `ImplicitCtxt` to point to our new query job.
 142             let new_icx = ImplicitCtxt {
 143                 tcx: self.tcx,
 144                 query: Some(token),
 145                 diagnostics,
 146                 query_depth: current_icx.query_depth + depth_limit as usize,
 147                 task_deps: current_icx.task_deps,
 148             };
 149
 150             // Use the `ImplicitCtxt` while we execute the query.
 151             tls::enter_context(&new_icx, compute)
 152         })
 153     }
 154
 155     fn depth_limit_error(self, job: QueryJobId) {
 156         let mut span = None;
 157         let mut layout_of_depth = None;
 158         if let Some(map) = self.try_collect_active_jobs() {
 159             if let Some((info, depth)) = job.try_find_layout_root(map, dep_kinds::layout_of) {
 160                 span = Some(info.job.span);
 161                 layout_of_depth = Some(LayoutOfDepth { desc: info.query.description, depth });
 162             }
 163         }
 164
 165         let suggested_limit = match self.recursion_limit() {
 166             Limit(0) => Limit(2),
 167             limit => limit * 2,
 168         };
 169
 170         self.sess.emit_fatal(QueryOverflow {
 171             span,
 172             layout_of_depth,
 173             suggested_limit,
 174             crate_name: self.crate_name(LOCAL_CRATE),
 175         });
 176     }
 177 }
 178
 179 pub(super) fn try_mark_green<'tcx>(tcx: TyCtxt<'tcx>, dep_node: &dep_graph::DepNode) -> bool {
 180     tcx.dep_graph.try_mark_green(QueryCtxt::new(tcx), dep_node).is_some()
 181 }
 182
 183 pub(super) fn encode_all_query_results<'tcx>(
 184     tcx: TyCtxt<'tcx>,
 185     encoder: &mut CacheEncoder<'_, 'tcx>,
 186     query_result_index: &mut EncodedDepNodeIndex,
 187 ) {
 188     for encode in super::ENCODE_QUERY_RESULTS.iter().copied().flatten() {
 189         encode(tcx, encoder, query_result_index);
 190     }
 191 }
 192
 193 macro_rules! handle_cycle_error {
 194     ([]) => {{
 195         rustc_query_system::HandleCycleError::Error
 196     }};
 197     ([(fatal_cycle) $($rest:tt)*]) => {{
 198         rustc_query_system::HandleCycleError::Fatal
 199     }};
 200     ([(cycle_delay_bug) $($rest:tt)*]) => {{
 201         rustc_query_system::HandleCycleError::DelayBug
 202     }};
 203     ([$other:tt $($modifiers:tt)*]) => {
 204         handle_cycle_error!([$($modifiers)*])
 205     };
 206 }
 207
 208 macro_rules! is_anon {
 209     ([]) => {{
 210         false
 211     }};
 212     ([(anon) $($rest:tt)*]) => {{
 213         true
 214     }};
 215     ([$other:tt $($modifiers:tt)*]) => {
 216         is_anon!([$($modifiers)*])
 217     };
 218 }
 219
 220 macro_rules! is_eval_always {
 221     ([]) => {{
 222         false
 223     }};
 224     ([(eval_always) $($rest:tt)*]) => {{
 225         true
 226     }};
 227     ([$other:tt $($modifiers:tt)*]) => {
 228         is_eval_always!([$($modifiers)*])
 229     };
 230 }
 231
 232 macro_rules! depth_limit {
 233     ([]) => {{
 234         false
 235     }};
 236     ([(depth_limit) $($rest:tt)*]) => {{
 237         true
 238     }};
 239     ([$other:tt $($modifiers:tt)*]) => {
 240         depth_limit!([$($modifiers)*])
 241     };
 242 }
 243
 244 macro_rules! feedable {
 245     ([]) => {{
 246         false
 247     }};
 248     ([(feedable) $($rest:tt)*]) => {{
 249         true
 250     }};
 251     ([$other:tt $($modifiers:tt)*]) => {
 252         feedable!([$($modifiers)*])
 253     };
 254 }
 255
 256 macro_rules! hash_result {
 257     ([][$V:ty]) => {{
 258         Some(|hcx, result| dep_graph::hash_result(hcx, &restore::<$V>(*result)))
 259     }};
 260     ([(no_hash) $($rest:tt)*][$V:ty]) => {{
 261         None
 262     }};
 263     ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
 264         hash_result!([$($modifiers)*][$($args)*])
 265     };
 266 }
 267
 268 macro_rules! call_provider {
 269     ([][$tcx:expr, $name:ident, $key:expr]) => {{
 270         ($tcx.query_system.fns.local_providers.$name)($tcx, $key)
 271     }};
 272     ([(separate_provide_extern) $($rest:tt)*][$tcx:expr, $name:ident, $key:expr]) => {{
 273         if let Some(key) = $key.as_local_key() {
 274             ($tcx.query_system.fns.local_providers.$name)($tcx, key)
 275         } else {
 276             ($tcx.query_system.fns.extern_providers.$name)($tcx, $key)
 277         }
 278     }};
 279     ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
 280         call_provider!([$($modifiers)*][$($args)*])
 281     };
 282 }
 283
 284 macro_rules! should_ever_cache_on_disk {
 285     ([]$yes:tt $no:tt) => {{
 286         $no
 287     }};
 288     ([(cache) $($rest:tt)*]$yes:tt $no:tt) => {{
 289         $yes
 290     }};
 291     ([$other:tt $($modifiers:tt)*]$yes:tt $no:tt) => {
 292         should_ever_cache_on_disk!([$($modifiers)*]$yes $no)
 293     };
 294 }
 295
 296 pub(crate) fn create_query_frame<
 297     'tcx,
 298     K: Copy + Key + for<'a> HashStable<StableHashingContext<'a>>,
 299 >(
 300     tcx: TyCtxt<'tcx>,
 301     do_describe: fn(TyCtxt<'tcx>, K) -> String,
 302     key: K,
 303     kind: DepKind,
 304     name: &'static str,
 305 ) -> QueryStackFrame {
 306     // Avoid calling queries while formatting the description
 307     let description = ty::print::with_no_queries!(
 308         // Disable visible paths printing for performance reasons.
 309         // Showing visible path instead of any path is not that important in production.
 310         ty::print::with_no_visible_paths!(
 311             // Force filename-line mode to avoid invoking `type_of` query.
 312             ty::print::with_forced_impl_filename_line!(do_describe(tcx, key))
 313         )
 314     );
 315     let description =
 316         if tcx.sess.verbose() { format!("{description} [{name:?}]") } else { description };
 317     let span = if kind == dep_graph::dep_kinds::def_span || with_no_queries() {
 318         // The `def_span` query is used to calculate `default_span`,
 319         // so exit to avoid infinite recursion.
 320         None
 321     } else {
 322         Some(key.default_span(tcx))
 323     };
 324     let def_id = key.key_as_def_id();
 325     let def_kind = if kind == dep_graph::dep_kinds::opt_def_kind || with_no_queries() {
 326         // Try to avoid infinite recursion.
 327         None
 328     } else {
 329         def_id.and_then(|def_id| def_id.as_local()).and_then(|def_id| tcx.opt_def_kind(def_id))
 330     };
 331     let hash = || {
 332         tcx.with_stable_hashing_context(|mut hcx| {
 333             let mut hasher = StableHasher::new();
 334             kind.as_usize().hash_stable(&mut hcx, &mut hasher);
 335             key.hash_stable(&mut hcx, &mut hasher);
 336             hasher.finish::<Hash64>()
 337         })
 338     };
 339     let ty_adt_id = key.ty_adt_id();
 340
 341     QueryStackFrame::new(description, span, def_id, def_kind, kind, ty_adt_id, hash)
 342 }
 343
 344 pub(crate) fn encode_query_results<'a, 'tcx, Q>(
 345     query: Q::Config,
 346     qcx: QueryCtxt<'tcx>,
 347     encoder: &mut CacheEncoder<'a, 'tcx>,
 348     query_result_index: &mut EncodedDepNodeIndex,
 349 ) where
 350     Q: super::QueryConfigRestored<'tcx>,
 351     Q::RestoredValue: Encodable<CacheEncoder<'a, 'tcx>>,
 352 {
 353     let _timer = qcx.profiler().generic_activity_with_arg("encode_query_results_for", query.name());
 354
 355     assert!(query.query_state(qcx).all_inactive());
 356     let cache = query.query_cache(qcx);
 357     cache.iter(&mut |key, value, dep_node| {
 358         if query.cache_on_disk(qcx.tcx, &key) {
 359             let dep_node = SerializedDepNodeIndex::new(dep_node.index());
 360
 361             // Record position of the cache entry.
 362             query_result_index.push((dep_node, AbsoluteBytePos::new(encoder.position())));
 363
 364             // Encode the type check tables with the `SerializedDepNodeIndex`
 365             // as tag.
 366             encoder.encode_tagged(dep_node, &Q::restore(*value));
 367         }
 368     });
 369 }
 370
 371 fn try_load_from_on_disk_cache<'tcx, Q>(query: Q, tcx: TyCtxt<'tcx>, dep_node: DepNode)
 372 where
 373     Q: QueryConfig<QueryCtxt<'tcx>>,
 374 {
 375     debug_assert!(tcx.dep_graph.is_green(&dep_node));
 376
 377     let key = Q::Key::recover(tcx, &dep_node).unwrap_or_else(|| {
 378         panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash)
 379     });
 380     if query.cache_on_disk(tcx, &key) {
 381         let _ = query.execute_query(tcx, key);
 382     }
 383 }
 384
 385 pub(crate) fn loadable_from_disk<'tcx>(tcx: TyCtxt<'tcx>, id: SerializedDepNodeIndex) -> bool {
 386     if let Some(cache) = tcx.query_system.on_disk_cache.as_ref() {
 387         cache.loadable_from_disk(id)
 388     } else {
 389         false
 390     }
 391 }
 392
 393 pub(crate) fn try_load_from_disk<'tcx, V>(
 394     tcx: TyCtxt<'tcx>,
 395     prev_index: SerializedDepNodeIndex,
 396     index: DepNodeIndex,
 397 ) -> Option<V>
 398 where
 399     V: for<'a> Decodable<CacheDecoder<'a, 'tcx>>,
 400 {
 401     let on_disk_cache = tcx.query_system.on_disk_cache.as_ref()?;
 402
 403     let prof_timer = tcx.prof.incr_cache_loading();
 404
 405     // The call to `with_query_deserialization` enforces that no new `DepNodes`
 406     // are created during deserialization. See the docs of that method for more
 407     // details.
 408     let value = tcx
 409         .dep_graph
 410         .with_query_deserialization(|| on_disk_cache.try_load_query_result(tcx, prev_index));
 411
 412     prof_timer.finish_with_query_invocation_id(index.into());
 413
 414     value
 415 }
 416
 417 fn force_from_dep_node<'tcx, Q>(query: Q, tcx: TyCtxt<'tcx>, dep_node: DepNode) -> bool
 418 where
 419     Q: QueryConfig<QueryCtxt<'tcx>>,
 420 {
 421     // We must avoid ever having to call `force_from_dep_node()` for a
 422     // `DepNode::codegen_unit`:
 423     // Since we cannot reconstruct the query key of a `DepNode::codegen_unit`, we
 424     // would always end up having to evaluate the first caller of the
 425     // `codegen_unit` query that *is* reconstructible. This might very well be
 426     // the `compile_codegen_unit` query, thus re-codegenning the whole CGU just
 427     // to re-trigger calling the `codegen_unit` query with the right key. At
 428     // that point we would already have re-done all the work we are trying to
 429     // avoid doing in the first place.
 430     // The solution is simple: Just explicitly call the `codegen_unit` query for
 431     // each CGU, right after partitioning. This way `try_mark_green` will always
 432     // hit the cache instead of having to go through `force_from_dep_node`.
 433     // This assertion makes sure, we actually keep applying the solution above.
 434     debug_assert!(
 435         dep_node.kind != dep_kinds::codegen_unit,
 436         "calling force_from_dep_node() on dep_kinds::codegen_unit"
 437     );
 438
 439     if let Some(key) = Q::Key::recover(tcx, &dep_node) {
 440         #[cfg(debug_assertions)]
 441         let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered();
 442         force_query(query, QueryCtxt::new(tcx), key, dep_node);
 443         true
 444     } else {
 445         false
 446     }
 447 }
 448
 449 pub(crate) fn query_callback<'tcx, Q>(is_anon: bool, is_eval_always: bool) -> DepKindStruct<'tcx>
 450 where
 451     Q: QueryConfigRestored<'tcx>,
 452 {
 453     let fingerprint_style = <Q::Config as QueryConfig<QueryCtxt<'tcx>>>::Key::fingerprint_style();
 454
 455     if is_anon || !fingerprint_style.reconstructible() {
 456         return DepKindStruct {
 457             is_anon,
 458             is_eval_always,
 459             fingerprint_style,
 460             force_from_dep_node: None,
 461             try_load_from_on_disk_cache: None,
 462             name: Q::NAME,
 463         };
 464     }
 465
 466     DepKindStruct {
 467         is_anon,
 468         is_eval_always,
 469         fingerprint_style,
 470         force_from_dep_node: Some(|tcx, dep_node| {
 471             force_from_dep_node(Q::config(tcx), tcx, dep_node)
 472         }),
 473         try_load_from_on_disk_cache: Some(|tcx, dep_node| {
 474             try_load_from_on_disk_cache(Q::config(tcx), tcx, dep_node)
 475         }),
 476         name: Q::NAME,
 477     }
 478 }
 479
 480 macro_rules! item_if_cached {
 481     ([] $tokens:tt) => {};
 482     ([(cache) $($rest:tt)*] { $($tokens:tt)* }) => {
 483         $($tokens)*
 484     };
 485     ([$other:tt $($modifiers:tt)*] $tokens:tt) => {
 486         item_if_cached! { [$($modifiers)*] $tokens }
 487     };
 488 }
 489
 490 macro_rules! expand_if_cached {
 491     ([], $tokens:expr) => {{
 492         None
 493     }};
 494     ([(cache) $($rest:tt)*], $tokens:expr) => {{
 495         Some($tokens)
 496     }};
 497     ([$other:tt $($modifiers:tt)*], $tokens:expr) => {
 498         expand_if_cached!([$($modifiers)*], $tokens)
 499     };
 500 }
 501
 502 /// Don't show the backtrace for query system by default
 503 /// use `RUST_BACKTRACE=full` to show all the backtraces
 504 #[inline(never)]
 505 pub fn __rust_begin_short_backtrace<F, T>(f: F) -> T
 506 where
 507     F: FnOnce() -> T,
 508 {
 509     let result = f();
 510     std::hint::black_box(());
 511     result
 512 }
 513
 514 // NOTE: `$V` isn't used here, but we still need to match on it so it can be passed to other macros
 515 // invoked by `rustc_query_append`.
 516 macro_rules! define_queries {
 517     (
 518      $($(#[$attr:meta])*
 519         [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,)*) => {
 520
 521         pub(crate) mod query_impl { $(pub mod $name {
 522             use super::super::*;
 523             use std::marker::PhantomData;
 524
 525             pub mod get_query_incr {
 526                 use super::*;
 527
 528                 // Adding `__rust_end_short_backtrace` marker to backtraces so that we emit the frames
 529                 // when `RUST_BACKTRACE=1`, add a new mod with `$name` here is to allow duplicate naming
 530                 #[inline(never)]
 531                 pub fn __rust_end_short_backtrace<'tcx>(
 532                     tcx: TyCtxt<'tcx>,
 533                     span: Span,
 534                     key: queries::$name::Key<'tcx>,
 535                     mode: QueryMode,
 536                 ) -> Option<Erase<queries::$name::Value<'tcx>>> {
 537                     #[cfg(debug_assertions)]
 538                     let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered();
 539                     get_query_incr(
 540                         QueryType::config(tcx),
 541                         QueryCtxt::new(tcx),
 542                         span,
 543                         key,
 544                         mode
 545                     )
 546                 }
 547             }
 548
 549             pub mod get_query_non_incr {
 550                 use super::*;
 551
 552                 #[inline(never)]
 553                 pub fn __rust_end_short_backtrace<'tcx>(
 554                     tcx: TyCtxt<'tcx>,
 555                     span: Span,
 556                     key: queries::$name::Key<'tcx>,
 557                     __mode: QueryMode,
 558                 ) -> Option<Erase<queries::$name::Value<'tcx>>> {
 559                     Some(get_query_non_incr(
 560                         QueryType::config(tcx),
 561                         QueryCtxt::new(tcx),
 562                         span,
 563                         key,
 564                     ))
 565                 }
 566             }
 567
 568             pub fn dynamic_query<'tcx>() -> DynamicQuery<'tcx, queries::$name::Storage<'tcx>> {
 569                 DynamicQuery {
 570                     name: stringify!($name),
 571                     eval_always: is_eval_always!([$($modifiers)*]),
 572                     dep_kind: dep_graph::dep_kinds::$name,
 573                     handle_cycle_error: handle_cycle_error!([$($modifiers)*]),
 574                     query_state: offset_of!(QueryStates<'tcx> => $name),
 575                     query_cache: offset_of!(QueryCaches<'tcx> => $name),
 576                     cache_on_disk: |tcx, key| ::rustc_middle::query::cached::$name(tcx, key),
 577                     execute_query: |tcx, key| erase(tcx.$name(key)),
 578                     compute: |tcx, key| {
 579                         #[cfg(debug_assertions)]
 580                         let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered();
 581                         __rust_begin_short_backtrace(||
 582                             queries::$name::provided_to_erased(
 583                                 tcx,
 584                                 {
 585                                     let ret = call_provider!([$($modifiers)*][tcx, $name, key]);
 586                                     tracing::trace!(?ret);
 587                                     ret
 588                                 }
 589                             )
 590                         )
 591                     },
 592                     can_load_from_disk: should_ever_cache_on_disk!([$($modifiers)*] true false),
 593                     try_load_from_disk: should_ever_cache_on_disk!([$($modifiers)*] {
 594                         |tcx, key, prev_index, index| {
 595                             if ::rustc_middle::query::cached::$name(tcx, key) {
 596                                 let value = $crate::plumbing::try_load_from_disk::<
 597                                     queries::$name::ProvidedValue<'tcx>
 598                                 >(
 599                                     tcx,
 600                                     prev_index,
 601                                     index,
 602                                 );
 603                                 value.map(|value| queries::$name::provided_to_erased(tcx, value))
 604                             } else {
 605                                 None
 606                             }
 607                         }
 608                     } {
 609                         |_tcx, _key, _prev_index, _index| None
 610                     }),
 611                     value_from_cycle_error: |tcx, cycle, guar| {
 612                         let result: queries::$name::Value<'tcx> = Value::from_cycle_error(tcx, cycle, guar);
 613                         erase(result)
 614                     },
 615                     loadable_from_disk: |_tcx, _key, _index| {
 616                         should_ever_cache_on_disk!([$($modifiers)*] {
 617                             ::rustc_middle::query::cached::$name(_tcx, _key) &&
 618                                 $crate::plumbing::loadable_from_disk(_tcx, _index)
 619                         } {
 620                             false
 621                         })
 622                     },
 623                     hash_result: hash_result!([$($modifiers)*][queries::$name::Value<'tcx>]),
 624                     format_value: |value| format!("{:?}", restore::<queries::$name::Value<'tcx>>(*value)),
 625                 }
 626             }
 627
 628             #[derive(Copy, Clone, Default)]
 629             pub struct QueryType<'tcx> {
 630                 data: PhantomData<&'tcx ()>
 631             }
 632
 633             impl<'tcx> QueryConfigRestored<'tcx> for QueryType<'tcx> {
 634                 type RestoredValue = queries::$name::Value<'tcx>;
 635                 type Config = DynamicConfig<
 636                     'tcx,
 637                     queries::$name::Storage<'tcx>,
 638                     { is_anon!([$($modifiers)*]) },
 639                     { depth_limit!([$($modifiers)*]) },
 640                     { feedable!([$($modifiers)*]) },
 641                 >;
 642
 643                 const NAME: &'static &'static str = &stringify!($name);
 644
 645                 #[inline(always)]
 646                 fn config(tcx: TyCtxt<'tcx>) -> Self::Config {
 647                     DynamicConfig {
 648                         dynamic: &tcx.query_system.dynamic_queries.$name,
 649                     }
 650                 }
 651
 652                 #[inline(always)]
 653                 fn restore(value: <Self::Config as QueryConfig<QueryCtxt<'tcx>>>::Value) -> Self::RestoredValue {
 654                     restore::<queries::$name::Value<'tcx>>(value)
 655                 }
 656             }
 657
 658             pub fn try_collect_active_jobs<'tcx>(tcx: TyCtxt<'tcx>, qmap: &mut QueryMap) {
 659                 let make_query = |tcx, key| {
 660                     let kind = rustc_middle::dep_graph::dep_kinds::$name;
 661                     let name = stringify!($name);
 662                     $crate::plumbing::create_query_frame(tcx, rustc_middle::query::descs::$name, key, kind, name)
 663                 };
 664                 tcx.query_system.states.$name.try_collect_active_jobs(
 665                     tcx,
 666                     make_query,
 667                     qmap,
 668                 ).unwrap();
 669             }
 670
 671             pub fn alloc_self_profile_query_strings<'tcx>(tcx: TyCtxt<'tcx>, string_cache: &mut QueryKeyStringCache) {
 672                 $crate::profiling_support::alloc_self_profile_query_strings_for_query_cache(
 673                     tcx,
 674                     stringify!($name),
 675                     &tcx.query_system.caches.$name,
 676                     string_cache,
 677                 )
 678             }
 679
 680             item_if_cached! { [$($modifiers)*] {
 681                 pub fn encode_query_results<'tcx>(
 682                     tcx: TyCtxt<'tcx>,
 683                     encoder: &mut CacheEncoder<'_, 'tcx>,
 684                     query_result_index: &mut EncodedDepNodeIndex
 685                 ) {
 686                     $crate::plumbing::encode_query_results::<query_impl::$name::QueryType<'tcx>>(
 687                         query_impl::$name::QueryType::config(tcx),
 688                         QueryCtxt::new(tcx),
 689                         encoder,
 690                         query_result_index,
 691                     )
 692                 }
 693             }}
 694         })*}
 695
 696         pub(crate) fn engine(incremental: bool) -> QueryEngine {
 697             if incremental {
 698                 QueryEngine {
 699                     $($name: query_impl::$name::get_query_incr::__rust_end_short_backtrace,)*
 700                 }
 701             } else {
 702                 QueryEngine {
 703                     $($name: query_impl::$name::get_query_non_incr::__rust_end_short_backtrace,)*
 704                 }
 705             }
 706         }
 707
 708         pub fn dynamic_queries<'tcx>() -> DynamicQueries<'tcx> {
 709             DynamicQueries {
 710                 $(
 711                     $name: query_impl::$name::dynamic_query(),
 712                 )*
 713             }
 714         }
 715
 716         // These arrays are used for iteration and can't be indexed by `DepKind`.
 717
 718         const TRY_COLLECT_ACTIVE_JOBS: &[for<'tcx> fn(TyCtxt<'tcx>, &mut QueryMap)] =
 719             &[$(query_impl::$name::try_collect_active_jobs),*];
 720
 721         const ALLOC_SELF_PROFILE_QUERY_STRINGS: &[
 722             for<'tcx> fn(TyCtxt<'tcx>, &mut QueryKeyStringCache)
 723         ] = &[$(query_impl::$name::alloc_self_profile_query_strings),*];
 724
 725         const ENCODE_QUERY_RESULTS: &[
 726             Option<for<'tcx> fn(
 727                 TyCtxt<'tcx>,
 728                 &mut CacheEncoder<'_, 'tcx>,
 729                 &mut EncodedDepNodeIndex)
 730             >
 731         ] = &[$(expand_if_cached!([$($modifiers)*], query_impl::$name::encode_query_results)),*];
 732
 733         #[allow(nonstandard_style)]
 734         mod query_callbacks {
 735             use super::*;
 736             use rustc_query_system::dep_graph::FingerprintStyle;
 737
 738             // We use this for most things when incr. comp. is turned off.
 739             pub fn Null<'tcx>() -> DepKindStruct<'tcx> {
 740                 DepKindStruct {
 741                     is_anon: false,
 742                     is_eval_always: false,
 743                     fingerprint_style: FingerprintStyle::Unit,
 744                     force_from_dep_node: Some(|_, dep_node| bug!("force_from_dep_node: encountered {:?}", dep_node)),
 745                     try_load_from_on_disk_cache: None,
 746                     name: &"Null",
 747                 }
 748             }
 749
 750             // We use this for the forever-red node.
 751             pub fn Red<'tcx>() -> DepKindStruct<'tcx> {
 752                 DepKindStruct {
 753                     is_anon: false,
 754                     is_eval_always: false,
 755                     fingerprint_style: FingerprintStyle::Unit,
 756                     force_from_dep_node: Some(|_, dep_node| bug!("force_from_dep_node: encountered {:?}", dep_node)),
 757                     try_load_from_on_disk_cache: None,
 758                     name: &"Red",
 759                 }
 760             }
 761
 762             pub fn TraitSelect<'tcx>() -> DepKindStruct<'tcx> {
 763                 DepKindStruct {
 764                     is_anon: true,
 765                     is_eval_always: false,
 766                     fingerprint_style: FingerprintStyle::Unit,
 767                     force_from_dep_node: None,
 768                     try_load_from_on_disk_cache: None,
 769                     name: &"TraitSelect",
 770                 }
 771             }
 772
 773             pub fn CompileCodegenUnit<'tcx>() -> DepKindStruct<'tcx> {
 774                 DepKindStruct {
 775                     is_anon: false,
 776                     is_eval_always: false,
 777                     fingerprint_style: FingerprintStyle::Opaque,
 778                     force_from_dep_node: None,
 779                     try_load_from_on_disk_cache: None,
 780                     name: &"CompileCodegenUnit",
 781                 }
 782             }
 783
 784             pub fn CompileMonoItem<'tcx>() -> DepKindStruct<'tcx> {
 785                 DepKindStruct {
 786                     is_anon: false,
 787                     is_eval_always: false,
 788                     fingerprint_style: FingerprintStyle::Opaque,
 789                     force_from_dep_node: None,
 790                     try_load_from_on_disk_cache: None,
 791                     name: &"CompileMonoItem",
 792                 }
 793             }
 794
 795             $(pub(crate) fn $name<'tcx>()-> DepKindStruct<'tcx> {
 796                 $crate::plumbing::query_callback::<query_impl::$name::QueryType<'tcx>>(
 797                     is_anon!([$($modifiers)*]),
 798                     is_eval_always!([$($modifiers)*]),
 799                 )
 800             })*
 801         }
 802
 803         pub fn query_callbacks<'tcx>(arena: &'tcx Arena<'tcx>) -> &'tcx [DepKindStruct<'tcx>] {
 804             arena.alloc_from_iter(make_dep_kind_array!(query_callbacks))
 805         }
 806     }
 807 }