src/librustc_incremental/persist/save.rs

   1 // Copyright 2014 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 use rustc::dep_graph::DepNode;
  12 use rustc::hir::def_id::DefId;
  13 use rustc::hir::svh::Svh;
  14 use rustc::session::Session;
  15 use rustc::ty::TyCtxt;
  16 use rustc_data_structures::fx::FxHashMap;
  17 use rustc_serialize::Encodable as RustcEncodable;
  18 use rustc_serialize::opaque::Encoder;
  19 use std::hash::Hash;
  20 use std::io::{self, Cursor, Write};
  21 use std::fs::{self, File};
  22 use std::path::PathBuf;
  23
  24 use IncrementalHashesMap;
  25 use ich::Fingerprint;
  26 use super::data::*;
  27 use super::directory::*;
  28 use super::hash::*;
  29 use super::preds::*;
  30 use super::fs::*;
  31 use super::dirty_clean;
  32 use super::file_format;
  33 use super::work_product;
  34 use calculate_svh::IchHasher;
  35
  36 pub fn save_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
  37                                 incremental_hashes_map: &IncrementalHashesMap,
  38                                 svh: Svh) {
  39     debug!("save_dep_graph()");
  40     let _ignore = tcx.dep_graph.in_ignore();
  41     let sess = tcx.sess;
  42     if sess.opts.incremental.is_none() {
  43         return;
  44     }
  45
  46     let mut builder = DefIdDirectoryBuilder::new(tcx);
  47     let query = tcx.dep_graph.query();
  48
  49     if tcx.sess.opts.debugging_opts.incremental_info {
  50         println!("incremental: {} nodes in dep-graph", query.graph.len_nodes());
  51         println!("incremental: {} edges in dep-graph", query.graph.len_edges());
  52     }
  53
  54     let mut hcx = HashContext::new(tcx, incremental_hashes_map);
  55     let preds = Predecessors::new(&query, &mut hcx);
  56     let mut current_metadata_hashes = FxHashMap();
  57
  58     if sess.opts.debugging_opts.incremental_cc ||
  59        sess.opts.debugging_opts.query_dep_graph {
  60         // IMPORTANT: We are saving the metadata hashes *before* the dep-graph,
  61         //            since metadata-encoding might add new entries to the
  62         //            DefIdDirectory (which is saved in the dep-graph file).
  63         save_in(sess,
  64                 metadata_hash_export_path(sess),
  65                 |e| encode_metadata_hashes(tcx,
  66                                            svh,
  67                                            &preds,
  68                                            &mut builder,
  69                                            &mut current_metadata_hashes,
  70                                            e));
  71     }
  72
  73     save_in(sess,
  74             dep_graph_path(sess),
  75             |e| encode_dep_graph(&preds, &mut builder, e));
  76
  77     let prev_metadata_hashes = incremental_hashes_map.prev_metadata_hashes.borrow();
  78     dirty_clean::check_dirty_clean_metadata(tcx,
  79                                             &*prev_metadata_hashes,
  80                                             &current_metadata_hashes);
  81 }
  82
  83 pub fn save_work_products(sess: &Session) {
  84     if sess.opts.incremental.is_none() {
  85         return;
  86     }
  87
  88     debug!("save_work_products()");
  89     let _ignore = sess.dep_graph.in_ignore();
  90     let path = work_products_path(sess);
  91     save_in(sess, path, |e| encode_work_products(sess, e));
  92
  93     // We also need to clean out old work-products, as not all of them are
  94     // deleted during invalidation. Some object files don't change their
  95     // content, they are just not needed anymore.
  96     let new_work_products = sess.dep_graph.work_products();
  97     let previous_work_products = sess.dep_graph.previous_work_products();
  98
  99     for (id, wp) in previous_work_products.iter() {
 100         if !new_work_products.contains_key(id) {
 101             work_product::delete_workproduct_files(sess, wp);
 102             debug_assert!(wp.saved_files.iter().all(|&(_, ref file_name)| {
 103                 !in_incr_comp_dir_sess(sess, file_name).exists()
 104             }));
 105         }
 106     }
 107
 108     // Check that we did not delete one of the current work-products:
 109     debug_assert!({
 110         new_work_products.iter()
 111                          .flat_map(|(_, wp)| wp.saved_files
 112                                                .iter()
 113                                                .map(|&(_, ref name)| name))
 114                          .map(|name| in_incr_comp_dir_sess(sess, name))
 115                          .all(|path| path.exists())
 116     });
 117 }
 118
 119 fn save_in<F>(sess: &Session, path_buf: PathBuf, encode: F)
 120     where F: FnOnce(&mut Encoder) -> io::Result<()>
 121 {
 122     debug!("save: storing data in {}", path_buf.display());
 123
 124     // delete the old dep-graph, if any
 125     // Note: It's important that we actually delete the old file and not just
 126     // truncate and overwrite it, since it might be a shared hard-link, the
 127     // underlying data of which we don't want to modify
 128     if path_buf.exists() {
 129         match fs::remove_file(&path_buf) {
 130             Ok(()) => {
 131                 debug!("save: remove old file");
 132             }
 133             Err(err) => {
 134                 sess.err(&format!("unable to delete old dep-graph at `{}`: {}",
 135                                   path_buf.display(),
 136                                   err));
 137                 return;
 138             }
 139         }
 140     }
 141
 142     // generate the data in a memory buffer
 143     let mut wr = Cursor::new(Vec::new());
 144     file_format::write_file_header(&mut wr).unwrap();
 145     match encode(&mut Encoder::new(&mut wr)) {
 146         Ok(()) => {}
 147         Err(err) => {
 148             sess.err(&format!("could not encode dep-graph to `{}`: {}",
 149                               path_buf.display(),
 150                               err));
 151             return;
 152         }
 153     }
 154
 155     // write the data out
 156     let data = wr.into_inner();
 157     match File::create(&path_buf).and_then(|mut file| file.write_all(&data)) {
 158         Ok(_) => {
 159             debug!("save: data written to disk successfully");
 160         }
 161         Err(err) => {
 162             sess.err(&format!("failed to write dep-graph to `{}`: {}",
 163                               path_buf.display(),
 164                               err));
 165             return;
 166         }
 167     }
 168 }
 169
 170 pub fn encode_dep_graph(preds: &Predecessors,
 171                         builder: &mut DefIdDirectoryBuilder,
 172                         encoder: &mut Encoder)
 173                         -> io::Result<()> {
 174     // First encode the commandline arguments hash
 175     let tcx = builder.tcx();
 176     tcx.sess.opts.dep_tracking_hash().encode(encoder)?;
 177
 178     // Create a flat list of (Input, WorkProduct) edges for
 179     // serialization.
 180     let mut edges = vec![];
 181     for (&target, sources) in &preds.inputs {
 182         match *target {
 183             DepNode::MetaData(ref def_id) => {
 184                 // Metadata *targets* are always local metadata nodes. We have
 185                 // already handled those in `encode_metadata_hashes`.
 186                 assert!(def_id.is_local());
 187                 continue;
 188             }
 189             _ => (),
 190         }
 191         let target = builder.map(target);
 192         for &source in sources {
 193             let source = builder.map(source);
 194             edges.push((source, target.clone()));
 195         }
 196     }
 197
 198     if tcx.sess.opts.debugging_opts.incremental_dump_hash {
 199         for (dep_node, hash) in &preds.hashes {
 200             println!("HIR hash for {:?} is {}", dep_node, hash);
 201         }
 202     }
 203
 204     // Create the serialized dep-graph.
 205     let graph = SerializedDepGraph {
 206         edges: edges,
 207         hashes: preds.hashes
 208             .iter()
 209             .map(|(&dep_node, &hash)| {
 210                 SerializedHash {
 211                     dep_node: builder.map(dep_node),
 212                     hash: hash,
 213                 }
 214             })
 215             .collect(),
 216     };
 217
 218     if tcx.sess.opts.debugging_opts.incremental_info {
 219         println!("incremental: {} edges in serialized dep-graph", graph.edges.len());
 220         println!("incremental: {} hashes in serialized dep-graph", graph.hashes.len());
 221     }
 222
 223     debug!("graph = {:#?}", graph);
 224
 225     // Encode the directory and then the graph data.
 226     builder.directory().encode(encoder)?;
 227     graph.encode(encoder)?;
 228
 229     Ok(())
 230 }
 231
 232 pub fn encode_metadata_hashes(tcx: TyCtxt,
 233                               svh: Svh,
 234                               preds: &Predecessors,
 235                               builder: &mut DefIdDirectoryBuilder,
 236                               current_metadata_hashes: &mut FxHashMap<DefId, Fingerprint>,
 237                               encoder: &mut Encoder)
 238                               -> io::Result<()> {
 239     // For each `MetaData(X)` node where `X` is local, accumulate a
 240     // hash.  These are the metadata items we export. Downstream
 241     // crates will want to see a hash that tells them whether we might
 242     // have changed the metadata for a given item since they last
 243     // compiled.
 244     //
 245     // (I initially wrote this with an iterator, but it seemed harder to read.)
 246     let mut serialized_hashes = SerializedMetadataHashes {
 247         hashes: vec![],
 248         index_map: FxHashMap()
 249     };
 250
 251     let mut def_id_hashes = FxHashMap();
 252
 253     for (&target, sources) in &preds.inputs {
 254         let def_id = match *target {
 255             DepNode::MetaData(def_id) => {
 256                 assert!(def_id.is_local());
 257                 def_id
 258             }
 259             _ => continue,
 260         };
 261
 262         let mut def_id_hash = |def_id: DefId| -> u64 {
 263             *def_id_hashes.entry(def_id)
 264                 .or_insert_with(|| {
 265                     let index = builder.add(def_id);
 266                     let path = builder.lookup_def_path(index);
 267                     path.deterministic_hash(tcx)
 268                 })
 269         };
 270
 271         // To create the hash for each item `X`, we don't hash the raw
 272         // bytes of the metadata (though in principle we
 273         // could). Instead, we walk the predecessors of `MetaData(X)`
 274         // from the dep-graph. This corresponds to all the inputs that
 275         // were read to construct the metadata. To create the hash for
 276         // the metadata, we hash (the hash of) all of those inputs.
 277         debug!("save: computing metadata hash for {:?}", def_id);
 278
 279         // Create a vector containing a pair of (source-id, hash).
 280         // The source-id is stored as a `DepNode<u64>`, where the u64
 281         // is the det. hash of the def-path. This is convenient
 282         // because we can sort this to get a stable ordering across
 283         // compilations, even if the def-ids themselves have changed.
 284         let mut hashes: Vec<(DepNode<u64>, Fingerprint)> = sources.iter()
 285             .map(|dep_node| {
 286                 let hash_dep_node = dep_node.map_def(|&def_id| Some(def_id_hash(def_id))).unwrap();
 287                 let hash = preds.hashes[dep_node];
 288                 (hash_dep_node, hash)
 289             })
 290             .collect();
 291
 292         hashes.sort();
 293         let mut state = IchHasher::new();
 294         hashes.hash(&mut state);
 295         let hash = state.finish();
 296
 297         debug!("save: metadata hash for {:?} is {}", def_id, hash);
 298
 299         if tcx.sess.opts.debugging_opts.incremental_dump_hash {
 300             println!("metadata hash for {:?} is {}", def_id, hash);
 301             for dep_node in sources {
 302                 println!("metadata hash for {:?} depends on {:?} with hash {}",
 303                          def_id, dep_node, preds.hashes[dep_node]);
 304             }
 305         }
 306
 307         serialized_hashes.hashes.push(SerializedMetadataHash {
 308             def_index: def_id.index,
 309             hash: hash,
 310         });
 311     }
 312
 313     if tcx.sess.opts.debugging_opts.query_dep_graph {
 314         for serialized_hash in &serialized_hashes.hashes {
 315             let def_id = DefId::local(serialized_hash.def_index);
 316
 317             // Store entry in the index_map
 318             let def_path_index = builder.add(def_id);
 319             serialized_hashes.index_map.insert(def_id.index, def_path_index);
 320
 321             // Record hash in current_metadata_hashes
 322             current_metadata_hashes.insert(def_id, serialized_hash.hash);
 323         }
 324
 325         debug!("save: stored index_map (len={}) for serialized hashes",
 326                serialized_hashes.index_map.len());
 327     }
 328
 329     // Encode everything.
 330     svh.encode(encoder)?;
 331     serialized_hashes.encode(encoder)?;
 332
 333     Ok(())
 334 }
 335
 336 pub fn encode_work_products(sess: &Session, encoder: &mut Encoder) -> io::Result<()> {
 337     let work_products: Vec<_> = sess.dep_graph
 338         .work_products()
 339         .iter()
 340         .map(|(id, work_product)| {
 341             SerializedWorkProduct {
 342                 id: id.clone(),
 343                 work_product: work_product.clone(),
 344             }
 345         })
 346         .collect();
 347
 348     work_products.encode(encoder)
 349 }