Encode ExpnId using ExpnHash for incr. comp.

compiler/rustc_metadata/src/rmeta/decoder.rs

@@ -79,6 +79,8 @@
     /// `DefIndex`. See `raw_def_id_to_def_id` for more details about how
     /// this is used.
     def_path_hash_map: OnceCell<UnhashMap<DefPathHash, DefIndex>>,
+    /// Likewise for ExpnHash.
+    expn_hash_map: OnceCell<UnhashMap<ExpnHash, ExpnIndex>>,
     /// Used for decoding interpret::AllocIds in a cached & thread-safe manner.
     alloc_decoding_state: AllocDecodingState,
     /// Caches decoded `DefKey`s.
@@ -1619,6 +1621,41 @@ fn def_path_hash(&self, index: DefIndex) -> DefPathHash {
         self.def_path_hash_unlocked(index, &mut def_path_hashes)
     }
 
+    fn expn_hash_to_expn_id(&self, index_guess: u32, hash: ExpnHash) -> ExpnId {
+        debug_assert_eq!(ExpnId::from_hash(hash), None);
+        let index_guess = ExpnIndex::from_u32(index_guess);
+        let old_hash = self.root.expn_hashes.get(self, index_guess).map(|lazy| lazy.decode(self));
+
+        let index = if old_hash == Some(hash) {
+            // Fast path: the expn and its index is unchanged from the
+            // previous compilation session. There is no need to decode anything
+            // else.
+            index_guess
+        } else {
+            // Slow path: We need to find out the new `DefIndex` of the provided
+            // `DefPathHash`, if its still exists. This requires decoding every `DefPathHash`
+            // stored in this crate.
+            let map = self.cdata.expn_hash_map.get_or_init(|| {
+                let end_id = self.root.expn_hashes.size() as u32;
+                let mut map =
+                    UnhashMap::with_capacity_and_hasher(end_id as usize, Default::default());
+                for i in 0..end_id {
+                    let i = ExpnIndex::from_u32(i);
+                    if let Some(hash) = self.root.expn_hashes.get(self, i) {
+                        map.insert(hash.decode(self), i);
+                    } else {
+                        panic!("Missing expn_hash entry for {:?}", i);
+                    }
+                }
+                map
+            });
+            map[&hash]
+        };
+
+        let data = self.root.expn_data.get(self, index).unwrap().decode(self);
+        rustc_span::hygiene::register_expn_id(data, hash)
+    }
+
     /// Imports the source_map from an external crate into the source_map of the crate
     /// currently being compiled (the "local crate").
     ///
@@ -1857,6 +1894,7 @@ impl CrateMetadata {
             raw_proc_macros,
             source_map_import_info: OnceCell::new(),
             def_path_hash_map: Default::default(),
+            expn_hash_map: Default::default(),
             alloc_decoding_state,
             cnum,
             cnum_map,

compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs

@@ -18,7 +18,7 @@
 use rustc_middle::ty::{self, TyCtxt, Visibility};
 use rustc_session::utils::NativeLibKind;
 use rustc_session::{Session, StableCrateId};
-use rustc_span::hygiene::{ExpnData, ExpnHash, ExpnId};
+use rustc_span::hygiene::{ExpnHash, ExpnId};
 use rustc_span::source_map::{Span, Spanned};
 use rustc_span::symbol::Symbol;
 
@@ -494,23 +494,6 @@ impl CrateStore for CStore {
     fn as_any(&self) -> &dyn Any {
         self
     }
-    fn decode_expn_data(&self, sess: &Session, expn_id: ExpnId) -> (ExpnData, ExpnHash) {
-        let crate_data = self.get_crate_data(expn_id.krate);
-        (
-            crate_data
-                .root
-                .expn_data
-                .get(&crate_data, expn_id.local_id)
-                .unwrap()
-                .decode((&crate_data, sess)),
-            crate_data
-                .root
-                .expn_hashes
-                .get(&crate_data, expn_id.local_id)
-                .unwrap()
-                .decode((&crate_data, sess)),
-        )
-    }
 
     fn crate_name(&self, cnum: CrateNum) -> Symbol {
         self.get_crate_data(cnum).root.name
@@ -545,6 +528,10 @@ fn def_path_hash_to_def_id(
         self.get_crate_data(cnum).def_path_hash_to_def_id(cnum, index_guess, hash)
     }
 
+    fn expn_hash_to_expn_id(&self, cnum: CrateNum, index_guess: u32, hash: ExpnHash) -> ExpnId {
+        self.get_crate_data(cnum).expn_hash_to_expn_id(index_guess, hash)
+    }
+
     fn encode_metadata(&self, tcx: TyCtxt<'_>) -> EncodedMetadata {
         encoder::encode_metadata(tcx)
     }

compiler/rustc_middle/src/middle/cstore.rs

@@ -11,8 +11,7 @@
 use rustc_macros::HashStable;
 use rustc_session::search_paths::PathKind;
 use rustc_session::utils::NativeLibKind;
-use rustc_session::Session;
-use rustc_span::hygiene::{ExpnData, ExpnHash, ExpnId};
+use rustc_span::hygiene::{ExpnHash, ExpnId};
 use rustc_span::symbol::Symbol;
 use rustc_span::Span;
 use rustc_target::spec::Target;
@@ -188,7 +187,6 @@ pub trait MetadataLoader {
 /// during resolve)
 pub trait CrateStore: std::fmt::Debug {
     fn as_any(&self) -> &dyn Any;
-    fn decode_expn_data(&self, sess: &Session, expn_id: ExpnId) -> (ExpnData, ExpnHash);
 
     // Foreign definitions.
     // This information is safe to access, since it's hashed as part of the DefPathHash, which incr.
@@ -209,6 +207,7 @@ fn def_path_hash_to_def_id(
         index_guess: u32,
         hash: DefPathHash,
     ) -> Option<DefId>;
+    fn expn_hash_to_expn_id(&self, cnum: CrateNum, index_guess: u32, hash: ExpnHash) -> ExpnId;
 
     // utility functions
     fn encode_metadata(&self, tcx: TyCtxt<'_>) -> EncodedMetadata;

compiler/rustc_middle/src/ty/query/on_disk_cache.rs

@@ -83,7 +83,7 @@ pub struct OnDiskCache<'sess> {
     // `ExpnData` (e.g `ExpnData.krate` may not be `LOCAL_CRATE`). Alternatively,
     // we could look up the `ExpnData` from the metadata of foreign crates,
     // but it seemed easier to have `OnDiskCache` be independent of the `CStore`.
-    expn_data: FxHashMap<u32, AbsoluteBytePos>,
+    expn_data: UnhashMap<ExpnHash, AbsoluteBytePos>,
     // Additional information used when decoding hygiene data.
     hygiene_context: HygieneDecodeContext,
     // Maps `DefPathHash`es to their `RawDefId`s from the *previous*
@@ -91,6 +91,8 @@ pub struct OnDiskCache<'sess> {
     // we try to map a `DefPathHash` to its `DefId` in the current compilation
     // session.
     foreign_def_path_hashes: UnhashMap<DefPathHash, RawDefId>,
+    // Likewise for ExpnId.
+    foreign_expn_data: UnhashMap<ExpnHash, u32>,
 
     // The *next* compilation sessison's `foreign_def_path_hashes` - at
     // the end of our current compilation session, this will get written
@@ -118,8 +120,9 @@ struct Footer {
     // See `OnDiskCache.syntax_contexts`
     syntax_contexts: FxHashMap<u32, AbsoluteBytePos>,
     // See `OnDiskCache.expn_data`
-    expn_data: FxHashMap<u32, AbsoluteBytePos>,
+    expn_data: UnhashMap<ExpnHash, AbsoluteBytePos>,
     foreign_def_path_hashes: UnhashMap<DefPathHash, RawDefId>,
+    foreign_expn_data: UnhashMap<ExpnHash, u32>,
 }
 
 pub type EncodedQueryResultIndex = Vec<(SerializedDepNodeIndex, AbsoluteBytePos)>;
@@ -217,6 +220,7 @@ pub fn new(sess: &'sess Session, data: Vec<u8>, start_pos: usize) -> Self {
             alloc_decoding_state: AllocDecodingState::new(footer.interpret_alloc_index),
             syntax_contexts: footer.syntax_contexts,
             expn_data: footer.expn_data,
+            foreign_expn_data: footer.foreign_expn_data,
             hygiene_context: Default::default(),
             foreign_def_path_hashes: footer.foreign_def_path_hashes,
             latest_foreign_def_path_hashes: Default::default(),
@@ -236,7 +240,8 @@ pub fn new_empty(source_map: &'sess SourceMap) -> Self {
             prev_diagnostics_index: Default::default(),
             alloc_decoding_state: AllocDecodingState::new(Vec::new()),
             syntax_contexts: FxHashMap::default(),
-            expn_data: FxHashMap::default(),
+            expn_data: UnhashMap::default(),
+            foreign_expn_data: UnhashMap::default(),
             hygiene_context: Default::default(),
             foreign_def_path_hashes: Default::default(),
             latest_foreign_def_path_hashes: Default::default(),
@@ -350,7 +355,8 @@ pub fn serialize<'tcx>(
             };
 
             let mut syntax_contexts = FxHashMap::default();
-            let mut expn_ids = FxHashMap::default();
+            let mut expn_data = UnhashMap::default();
+            let mut foreign_expn_data = UnhashMap::default();
 
             // Encode all hygiene data (`SyntaxContextData` and `ExpnData`) from the current
             // session.
@@ -363,13 +369,14 @@ pub fn serialize<'tcx>(
                     syntax_contexts.insert(index, pos);
                     Ok(())
                 },
-                |encoder, index, expn_data, hash| -> FileEncodeResult {
-                    if index.krate == LOCAL_CRATE {
+                |encoder, expn_id, data, hash| -> FileEncodeResult {
+                    if expn_id.krate == LOCAL_CRATE {
                         let pos = AbsoluteBytePos::new(encoder.position());
-                        encoder.encode_tagged(TAG_EXPN_DATA, &(expn_data, hash))?;
-                        expn_ids.insert(index.local_id.as_u32(), pos);
+                        encoder.encode_tagged(TAG_EXPN_DATA, &data)?;
+                        expn_data.insert(hash, pos);
+                    } else {
+                        foreign_expn_data.insert(hash, expn_id.local_id.as_u32());
                     }
-                    // TODO Handle foreign expansions.
                     Ok(())
                 },
             )?;
@@ -387,7 +394,8 @@ pub fn serialize<'tcx>(
                     diagnostics_index,
                     interpret_alloc_index,
                     syntax_contexts,
-                    expn_data: expn_ids,
+                    expn_data,
+                    foreign_expn_data,
                     foreign_def_path_hashes,
                 },
             )?;
@@ -549,6 +557,7 @@ fn with_decoder<'a, 'tcx, T, F: FnOnce(&mut CacheDecoder<'sess, 'tcx>) -> T>(
             alloc_decoding_session: self.alloc_decoding_state.new_decoding_session(),
             syntax_contexts: &self.syntax_contexts,
             expn_data: &self.expn_data,
+            foreign_expn_data: &self.foreign_expn_data,
             hygiene_context: &self.hygiene_context,
         };
         f(&mut decoder)
@@ -643,7 +652,8 @@ pub struct CacheDecoder<'a, 'tcx> {
     file_index_to_stable_id: &'a FxHashMap<SourceFileIndex, EncodedSourceFileId>,
     alloc_decoding_session: AllocDecodingSession<'a>,
     syntax_contexts: &'a FxHashMap<u32, AbsoluteBytePos>,
-    expn_data: &'a FxHashMap<u32, AbsoluteBytePos>,
+    expn_data: &'a UnhashMap<ExpnHash, AbsoluteBytePos>,
+    foreign_expn_data: &'a UnhashMap<ExpnHash, u32>,
     hygiene_context: &'a HygieneDecodeContext,
 }
 
@@ -794,27 +804,43 @@ fn decode(decoder: &mut CacheDecoder<'a, 'tcx>) -> Result<Self, String> {
 
 impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for ExpnId {
     fn decode(decoder: &mut CacheDecoder<'a, 'tcx>) -> Result<Self, String> {
-        let krate = CrateNum::decode(decoder)?;
-        let index = u32::decode(decoder)?;
+        let hash = ExpnHash::decode(decoder)?;
+        if hash.is_root() {
+            return Ok(ExpnId::root());
+        }
 
-        let expn_data = decoder.expn_data;
-        let tcx = decoder.tcx;
-        rustc_span::hygiene::decode_expn_id_incrcomp(
-            krate,
-            index,
-            decoder.hygiene_context,
-            |index| -> Result<(ExpnData, ExpnHash), _> {
-                // This closure is invoked if we haven't already decoded the data for the `ExpnId` we are deserializing.
-                // We look up the position of the associated `ExpnData` and decode it.
-                let pos = expn_data
-                    .get(&index)
-                    .unwrap_or_else(|| panic!("Bad index {:?} (map {:?})", index, expn_data));
+        if let Some(expn_id) = ExpnId::from_hash(hash) {
+            return Ok(expn_id);
+        }
 
-                decoder
-                    .with_position(pos.to_usize(), |decoder| decode_tagged(decoder, TAG_EXPN_DATA))
-            },
-            |expn_id| tcx.untracked_resolutions.cstore.decode_expn_data(tcx.sess, expn_id),
-        )
+        let krate = decoder.cnum_map[&hash.stable_crate_id()];
+
+        let expn_id = if krate == LOCAL_CRATE {
+            // We look up the position of the associated `ExpnData` and decode it.
+            let pos = decoder
+                .expn_data
+                .get(&hash)
+                .unwrap_or_else(|| panic!("Bad hash {:?} (map {:?})", hash, decoder.expn_data));
+
+            let data: ExpnData = decoder
+                .with_position(pos.to_usize(), |decoder| decode_tagged(decoder, TAG_EXPN_DATA))?;
+            rustc_span::hygiene::register_local_expn_id(data, hash)
+        } else {
+            let index_guess = decoder.foreign_expn_data[&hash];
+            decoder.tcx.untracked_resolutions.cstore.expn_hash_to_expn_id(krate, index_guess, hash)
+        };
+
+        #[cfg(debug_assertions)]
+        {
+            use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
+            let mut hcx = decoder.tcx.create_stable_hashing_context();
+            let mut hasher = StableHasher::new();
+            expn_id.expn_data().hash_stable(&mut hcx, &mut hasher);
+            let local_hash: u64 = hasher.finish();
+            debug_assert_eq!(hash.local_hash(), local_hash);
+        }
+
+        Ok(expn_id)
     }
 }
 
@@ -990,8 +1016,7 @@ impl<'a, 'tcx, E> Encodable<CacheEncoder<'a, 'tcx, E>> for ExpnId
 {
     fn encode(&self, s: &mut CacheEncoder<'a, 'tcx, E>) -> Result<(), E::Error> {
         s.hygiene_context.schedule_expn_data_for_encoding(*self);
-        self.krate.encode(s)?;
-        self.local_id.as_u32().encode(s)
+        self.expn_hash().encode(s)
     }
 }
 

compiler/rustc_span/src/def_id.rs

@@ -136,7 +136,7 @@ fn borrow(&self) -> &Fingerprint {
 /// further trouble.
 #[derive(Copy, Clone, Hash, PartialEq, Eq, PartialOrd, Ord, Debug)]
 #[derive(HashStable_Generic, Encodable, Decodable)]
-pub struct StableCrateId(u64);
+pub struct StableCrateId(pub(crate) u64);
 
 impl StableCrateId {
     pub fn to_u64(self) -> u64 {

compiler/rustc_span/src/hygiene.rs

@@ -29,7 +29,7 @@
 use crate::with_session_globals;
 use crate::{HashStableContext, Span, DUMMY_SP};
 
-use crate::def_id::{CrateNum, DefId, CRATE_DEF_ID, LOCAL_CRATE};
+use crate::def_id::{CrateNum, DefId, StableCrateId, CRATE_DEF_ID, LOCAL_CRATE};
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
@@ -92,6 +92,34 @@ pub struct LocalExpnId {
 #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Encodable, Decodable, HashStable_Generic)]
 pub struct ExpnHash(Fingerprint);
 
+impl ExpnHash {
+    /// Returns the [StableCrateId] identifying the crate this [ExpnHash]
+    /// originates from.
+    #[inline]
+    pub fn stable_crate_id(self) -> StableCrateId {
+        StableCrateId(self.0.as_value().0)
+    }
+
+    /// Returns the crate-local part of the [ExpnHash].
+    ///
+    /// Used for tests.
+    #[inline]
+    pub fn local_hash(self) -> u64 {
+        self.0.as_value().1
+    }
+
+    #[inline]
+    pub fn is_root(self) -> bool {
+        self.0 == Fingerprint::ZERO
+    }
+
+    /// Builds a new [ExpnHash] with the given [StableCrateId] and
+    /// `local_hash`, where `local_hash` must be unique within its crate.
+    fn new(stable_crate_id: StableCrateId, local_hash: u64) -> ExpnHash {
+        ExpnHash(Fingerprint::new(stable_crate_id.0, local_hash))
+    }
+}
+
 /// A property of a macro expansion that determines how identifiers
 /// produced by that expansion are resolved.
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Hash, Debug, Encodable, Decodable)]
@@ -268,12 +296,12 @@ pub struct HygieneData {
     expn_hash_to_expn_id: UnhashMap<ExpnHash, ExpnId>,
     syntax_context_data: Vec<SyntaxContextData>,
     syntax_context_map: FxHashMap<(SyntaxContext, ExpnId, Transparency), SyntaxContext>,
-    /// Maps the `Fingerprint` of an `ExpnData` to the next disambiguator value.
+    /// Maps the `local_hash` of an `ExpnData` to the next disambiguator value.
     /// This is used by `update_disambiguator` to keep track of which `ExpnData`s
     /// would have collisions without a disambiguator.
     /// The keys of this map are always computed with `ExpnData.disambiguator`
     /// set to 0.
-    expn_data_disambiguators: FxHashMap<Fingerprint, u32>,
+    expn_data_disambiguators: FxHashMap<u64, u32>,
 }
 
 impl HygieneData {
@@ -981,7 +1009,7 @@ pub fn is_root(&self) -> bool {
     }
 
     #[inline]
-    fn hash_expn(&self, ctx: &mut impl HashStableContext) -> Fingerprint {
+    fn hash_expn(&self, ctx: &mut impl HashStableContext) -> u64 {
         let mut hasher = StableHasher::new();
         self.hash_stable(ctx, &mut hasher);
         hasher.finish()
@@ -1191,75 +1219,46 @@ pub struct HygieneDecodeContext {
     // so that multiple occurrences of the same serialized id are decoded to the same
     // `SyntaxContext`
     remapped_ctxts: Lock<Vec<Option<SyntaxContext>>>,
-    // The same as `remapepd_ctxts`, but for `ExpnId`s
-    remapped_expns: Lock<Vec<Option<LocalExpnId>>>,
 }
 
-pub fn decode_expn_id_incrcomp<E>(
-    krate: CrateNum,
-    index: u32,
-    context: &HygieneDecodeContext,
-    decode_data: impl FnOnce(u32) -> Result<(ExpnData, ExpnHash), E>,
-    decode_foreign: impl FnOnce(ExpnId) -> (ExpnData, ExpnHash),
-) -> Result<ExpnId, E> {
-    // Do this after decoding, so that we decode a `CrateNum`
-    // if necessary
-    if index == 0 {
-        debug!("decode_expn_id: deserialized root");
-        return Ok(ExpnId::root());
-    }
-
-    if krate != LOCAL_CRATE {
-        let expn_id = decode_expn_id(krate, index, decode_foreign);
-        return Ok(expn_id);
-    }
-
-    let outer_expns = &context.remapped_expns;
-
-    // Ensure that the lock() temporary is dropped early
-    {
-        if let Some(expn_id) = outer_expns.lock().get(index as usize).copied().flatten() {
-            return Ok(expn_id.to_expn_id());
-        }
-    }
-
-    // Don't decode the data inside `HygieneData::with`, since we need to recursively decode
-    // other ExpnIds
-    let (mut expn_data, hash) = decode_data(index)?;
-    debug_assert_eq!(krate, expn_data.krate);
-
-    let expn_id = HygieneData::with(|hygiene_data| {
-        if let Some(expn_id) = hygiene_data.expn_hash_to_expn_id.get(&hash) {
-            return *expn_id;
-        }
-
+/// Register an expansion which has been decoded from the on-disk-cache for the local crate.
+pub fn register_local_expn_id(mut data: ExpnData, hash: ExpnHash) -> ExpnId {
+    HygieneData::with(|hygiene_data| {
         // If we just deserialized an `ExpnData` owned by
         // the local crate, its `orig_id` will be stale,
         // so we need to update it to its own value.
         // This only happens when we deserialize the incremental cache,
         // since a crate will never decode its own metadata.
         let expn_id = hygiene_data.local_expn_data.next_index();
-        expn_data.orig_id = Some(expn_id.as_u32());
-        hygiene_data.local_expn_data.push(Some(expn_data));
+        data.orig_id = Some(expn_id.as_u32());
+        hygiene_data.local_expn_data.push(Some(data));
         let _eid = hygiene_data.local_expn_hashes.push(hash);
         debug_assert_eq!(expn_id, _eid);
 
-        let mut expns = outer_expns.lock();
-        let new_len = index as usize + 1;
-        if expns.len() < new_len {
-            expns.resize(new_len, None);
-        }
-        expns[index as usize] = Some(expn_id);
-        drop(expns);
         let expn_id = expn_id.to_expn_id();
 
         let _old_id = hygiene_data.expn_hash_to_expn_id.insert(hash, expn_id);
         debug_assert!(_old_id.is_none());
         expn_id
-    });
-    Ok(expn_id)
+    })
 }
 
+/// Register an expansion which has been decoded from the metadata of a foreign crate.
+pub fn register_expn_id(data: ExpnData, hash: ExpnHash) -> ExpnId {
+    let expn_id =
+        ExpnId { krate: data.krate, local_id: ExpnIndex::from_u32(data.orig_id.unwrap()) };
+    HygieneData::with(|hygiene_data| {
+        let _old_data = hygiene_data.foreign_expn_data.insert(expn_id, data);
+        debug_assert!(_old_data.is_none());
+        let _old_hash = hygiene_data.foreign_expn_hashes.insert(expn_id, hash);
+        debug_assert!(_old_hash.is_none());
+        let _old_id = hygiene_data.expn_hash_to_expn_id.insert(hash, expn_id);
+        debug_assert!(_old_id.is_none());
+    });
+    expn_id
+}
+
+/// Decode an expansion from the metadata of a foreign crate.
 pub fn decode_expn_id(
     krate: CrateNum,
     index: u32,
@@ -1287,16 +1286,7 @@ pub fn decode_expn_id(
     debug_assert_eq!(krate, expn_data.krate);
     debug_assert_eq!(Some(index.as_u32()), expn_data.orig_id);
 
-    HygieneData::with(|hygiene_data| {
-        let _old_data = hygiene_data.foreign_expn_data.insert(expn_id, expn_data);
-        debug_assert!(_old_data.is_none());
-        let _old_hash = hygiene_data.foreign_expn_hashes.insert(expn_id, hash);
-        debug_assert!(_old_hash.is_none());
-        let _old_id = hygiene_data.expn_hash_to_expn_id.insert(hash, expn_id);
-        debug_assert!(_old_id.is_none());
-    });
-
-    expn_id
+    register_expn_id(expn_data, hash)
 }
 
 // Decodes `SyntaxContext`, using the provided `HygieneDecodeContext`
@@ -1493,7 +1483,8 @@ fn update_disambiguator(expn_id: LocalExpnId, mut ctx: impl HashStableContext) {
         });
     }
 
-    let expn_hash = ExpnHash(expn_hash);
+    let expn_hash =
+        ExpnHash::new(ctx.def_path_hash(LOCAL_CRATE.as_def_id()).stable_crate_id(), expn_hash);
     HygieneData::with(|data| {
         data.local_expn_data[expn_id].as_mut().unwrap().disambiguator = disambiguator;
         debug_assert_eq!(data.local_expn_hashes[expn_id].0, Fingerprint::ZERO);