From 041d8e899f5eecd435844ce5ad86dd2aadea3a46 Mon Sep 17 00:00:00 2001 From: Brian Anderson Date: Thu, 22 Aug 2013 13:46:47 -0700 Subject: [PATCH 1/3] Allow metadata to be not compressed --- Makefile.in | 6 ++++++ configure | 4 +++- src/librustc/driver/driver.rs | 7 ++++++- src/librustc/driver/session.rs | 2 ++ src/librustc/metadata/encoder.rs | 16 +++++++++++++--- src/librustc/metadata/loader.rs | 25 +++++++++++++++++++------ src/librustc/middle/trans/base.rs | 1 + 7 files changed, 50 insertions(+), 11 deletions(-) diff --git a/Makefile.in b/Makefile.in index a9a41a073d0..fb0bb6ff379 100644 --- a/Makefile.in +++ b/Makefile.in @@ -107,6 +107,12 @@ else CFG_GCCISH_CFLAGS += -DRUST_NDEBUG endif +ifndef CFG_ENABLE_COMPRESS_METADATA + # XXX: After snapshots extend this to all stages + RUSTFLAGS_STAGE1 += --no-compress-metadata + RUSTFLAGS_STAGE2 += --no-compress-metadata +endif + ifdef SAVE_TEMPS CFG_RUSTC_FLAGS += --save-temps endif diff --git a/configure b/configure index f070ae37dda..8a7a2bae4ea 100755 --- a/configure +++ b/configure @@ -381,7 +381,9 @@ opt mingw-cross 0 "cross-compile for win32 using mingw" opt clang 0 "prefer clang to gcc for building the runtime" opt ccache 0 "invoke gcc/clang via ccache to reuse object files between builds" opt local-rust 0 "use an installed rustc rather than downloading a snapshot" -opt pax-flags 0 "apply PaX flags to rustc binaries (required for GRSecurity/PaX-patched kernels)" +opt pax-flags 0 "apply PaX flags to rustc binaries (required for GRSecurity/PaX-patched + kernels)" +opt compress-metadata 0 "compress crate metadata" valopt prefix "/usr/local" "set installation prefix" valopt local-rust-root "/usr/local" "set prefix for local rust binary" valopt llvm-root "" "set LLVM root" diff --git a/src/librustc/driver/driver.rs b/src/librustc/driver/driver.rs index 1ddebbb4280..bba818c7c9d 100644 --- a/src/librustc/driver/driver.rs +++ b/src/librustc/driver/driver.rs @@ -718,6 +718,8 @@ fn set_llvm_debug() { let android_cross_path = getopts::opt_maybe_str( matches, "android-cross-path"); + let no_compress_metadata = opt_present(matches, "no-compress-metadata"); + let custom_passes = match getopts::opt_maybe_str(matches, "passes") { None => ~[], Some(s) => { @@ -752,7 +754,8 @@ fn set_llvm_debug() { parse_only: parse_only, no_trans: no_trans, debugging_opts: debugging_opts, - android_cross_path: android_cross_path + android_cross_path: android_cross_path, + no_compress_metadata: no_compress_metadata }; return sopts; } @@ -870,6 +873,8 @@ pub fn optgroups() -> ~[getopts::groups::OptGroup] { for details)", "FEATURE"), optopt("", "android-cross-path", "The path to the Android NDK", "PATH"), + optflag("", "no-compress-metadata", + "Do not compress crate metadata (make builds a little faster)"), optflagopt("W", "warn", "Set lint warnings", "OPT"), optmulti("A", "allow", diff --git a/src/librustc/driver/session.rs b/src/librustc/driver/session.rs index 50b29ff16be..e70ac52a0f5 100644 --- a/src/librustc/driver/session.rs +++ b/src/librustc/driver/session.rs @@ -166,6 +166,7 @@ pub struct options { no_trans: bool, debugging_opts: uint, android_cross_path: Option<~str>, + no_compress_metadata: bool } pub struct crate_metadata { @@ -350,6 +351,7 @@ pub fn basic_options() -> @options { no_trans: false, debugging_opts: 0u, android_cross_path: None, + no_compress_metadata: false } } diff --git a/src/librustc/metadata/encoder.rs b/src/librustc/metadata/encoder.rs index 1e9214e666b..3ada5592f75 100644 --- a/src/librustc/metadata/encoder.rs +++ b/src/librustc/metadata/encoder.rs @@ -64,6 +64,7 @@ pub struct EncodeParams<'self> { cstore: @mut cstore::CStore, encode_inlined_item: encode_inlined_item<'self>, reachable: @mut HashSet, + compress: bool } struct Stats { @@ -1567,7 +1568,7 @@ fn encode_hash(ebml_w: &mut writer::Encoder, hash: &str) { 0x75, //'u' as u8, 0x73, //'s' as u8, 0x74, //'t' as u8, - 0, 0, 0, 1 ]; + 0, 0, 0, 2 ]; pub fn encode_metadata(parms: EncodeParams, crate: &Crate) -> ~[u8] { let wr = @io::BytesWriter::new(); @@ -1594,6 +1595,7 @@ pub fn encode_metadata(parms: EncodeParams, crate: &Crate) -> ~[u8] { encode_inlined_item, link_meta, reachable, + compress, _ } = parms; let type_abbrevs = @mut HashMap::new(); @@ -1679,9 +1681,17 @@ pub fn encode_metadata(parms: EncodeParams, crate: &Crate) -> ~[u8] { wr.write(&[0u8, 0u8, 0u8, 0u8]); let writer_bytes: &mut ~[u8] = wr.bytes; + let compression_flag = if compress { [1u8] } else { [0u8] }; - metadata_encoding_version.to_owned() + - flate::deflate_bytes(*writer_bytes) + if compress { + metadata_encoding_version.to_owned() + + compression_flag.to_owned() + + flate::deflate_bytes(*writer_bytes) + } else { + metadata_encoding_version.to_owned() + + compression_flag.to_owned() + + *writer_bytes + } } // Get the encoded string for a type diff --git a/src/librustc/metadata/loader.rs b/src/librustc/metadata/loader.rs index d0060931a66..f374ce2c19a 100644 --- a/src/librustc/metadata/loader.rs +++ b/src/librustc/metadata/loader.rs @@ -228,13 +228,26 @@ fn get_metadata_section(os: os, } if !version_ok { return None; } - let cvbuf1 = ptr::offset(cvbuf, vlen as int); - debug!("inflating %u bytes of compressed metadata", - csz - vlen); - do vec::raw::buf_as_slice(cvbuf1, csz-vlen) |bytes| { - let inflated = flate::inflate_bytes(bytes); - found = Some(@(inflated)); + assert!(csz >= vlen + 1); + + let must_decompress = *ptr::offset(cvbuf, vlen as int) == 1; + let cvbuf1 = ptr::offset(cvbuf, vlen as int + 1); + + do vec::raw::buf_as_slice(cvbuf1, csz-vlen-1) |bytes| { + if must_decompress { + debug!("inflating %u bytes of compressed metadata", + csz - vlen); + let inflated = flate::inflate_bytes(bytes); + found = Some(@(inflated)); + } else { + // Copy the byte vector as fast as possible + let mut buf = vec::with_capacity(bytes.len()); + vec::raw::set_len(&mut buf, bytes.len()); + vec::raw::copy_memory(buf, bytes, bytes.len()); + found = Some(@buf) + } } + if found != None { return found; } diff --git a/src/librustc/middle/trans/base.rs b/src/librustc/middle/trans/base.rs index f19decaa38c..85db79ca54b 100644 --- a/src/librustc/middle/trans/base.rs +++ b/src/librustc/middle/trans/base.rs @@ -2907,6 +2907,7 @@ pub fn crate_ctxt_to_encode_parms<'r>(cx: &'r CrateContext, ie: encoder::encode_ cstore: cx.sess.cstore, encode_inlined_item: ie, reachable: cx.reachable, + compress: !cx.sess.opts.no_compress_metadata } } From 39e8cb6df32afcba3de9a146192046400754d4ea Mon Sep 17 00:00:00 2001 From: Brian Anderson Date: Fri, 23 Aug 2013 18:40:15 -0700 Subject: [PATCH 2/3] Don't copy metadata after loading --- src/libextra/ebml.rs | 77 +++++++++++++++++++++++-------- src/librustc/metadata/creader.rs | 3 +- src/librustc/metadata/csearch.rs | 2 +- src/librustc/metadata/cstore.rs | 3 +- src/librustc/metadata/decoder.rs | 78 ++++++++++++++++++-------------- src/librustc/metadata/loader.rs | 44 +++++++++--------- src/librustc/middle/astencode.rs | 8 ++-- 7 files changed, 132 insertions(+), 83 deletions(-) diff --git a/src/libextra/ebml.rs b/src/libextra/ebml.rs index f66677c21f7..560bcccae80 100644 --- a/src/libextra/ebml.rs +++ b/src/libextra/ebml.rs @@ -12,6 +12,8 @@ use std::str; +use std::cast; +use std::vec; // Simple Extensible Binary Markup Language (ebml) reader and writer on a // cursor model. See the specification here: @@ -29,9 +31,42 @@ struct EbmlState { data_pos: uint, } +#[deriving(Clone)] +pub enum EbmlData { + SafeData(@~[u8]), + UnsafeData(*u8, uint) +} + +impl EbmlData { + #[inline] + pub fn slice<'a>(&'a self, start: uint, end: uint) -> &'a [u8] { + match *self { + SafeData(@ref v) => v.slice(start, end), + UnsafeData(buf, len) => unsafe { + do vec::raw::buf_as_slice(buf, len) |s| { + cast::transmute(s.slice(start, end)) + } + } + } + } + + #[inline] + pub fn as_slice<'a>(&'a self) -> &'a [u8] { + self.slice(0, self.len()) + } + + #[inline] + pub fn len(&self) -> uint { + match *self { + SafeData(@ref v) => v.len(), + UnsafeData(_, len) => len + } + } +} + #[deriving(Clone)] pub struct Doc { - data: @~[u8], + data: EbmlData, start: uint, end: uint, } @@ -185,24 +220,28 @@ pub fn vuint_at(data: &[u8], start: uint) -> Res { } pub fn Doc(data: @~[u8]) -> Doc { - Doc { data: data, start: 0u, end: data.len() } + Doc { data: SafeData(data), start: 0u, end: data.len() } } - pub fn doc_at(data: @~[u8], start: uint) -> TaggedDoc { - let elt_tag = vuint_at(*data, start); - let elt_size = vuint_at(*data, elt_tag.next); + pub fn unsafe_Doc(buf: *u8, len: uint) -> Doc { + Doc { data: UnsafeData(buf, len), start: 0u, end: len } + } + + pub fn doc_at(data: &EbmlData, start: uint) -> TaggedDoc { + let elt_tag = vuint_at(data.as_slice(), start); + let elt_size = vuint_at(data.as_slice(), elt_tag.next); let end = elt_size.next + elt_size.val; TaggedDoc { tag: elt_tag.val, - doc: Doc { data: data, start: elt_size.next, end: end } + doc: Doc { data: data.clone(), start: elt_size.next, end: end } } } pub fn maybe_get_doc(d: Doc, tg: uint) -> Option { let mut pos = d.start; while pos < d.end { - let elt_tag = vuint_at(*d.data, pos); - let elt_size = vuint_at(*d.data, elt_tag.next); + let elt_tag = vuint_at(d.data.as_slice(), pos); + let elt_size = vuint_at(d.data.as_slice(), elt_tag.next); pos = elt_size.next + elt_size.val; if elt_tag.val == tg { return Some(Doc { data: d.data, start: elt_size.next, @@ -225,8 +264,8 @@ pub fn get_doc(d: Doc, tg: uint) -> Doc { pub fn docs(d: Doc, it: &fn(uint, Doc) -> bool) -> bool { let mut pos = d.start; while pos < d.end { - let elt_tag = vuint_at(*d.data, pos); - let elt_size = vuint_at(*d.data, elt_tag.next); + let elt_tag = vuint_at(d.data.as_slice(), pos); + let elt_size = vuint_at(d.data.as_slice(), elt_tag.next); pos = elt_size.next + elt_size.val; let doc = Doc { data: d.data, start: elt_size.next, end: pos }; if !it(elt_tag.val, doc) { @@ -239,8 +278,8 @@ pub fn docs(d: Doc, it: &fn(uint, Doc) -> bool) -> bool { pub fn tagged_docs(d: Doc, tg: uint, it: &fn(Doc) -> bool) -> bool { let mut pos = d.start; while pos < d.end { - let elt_tag = vuint_at(*d.data, pos); - let elt_size = vuint_at(*d.data, elt_tag.next); + let elt_tag = vuint_at(d.data.as_slice(), pos); + let elt_size = vuint_at(d.data.as_slice(), elt_tag.next); pos = elt_size.next + elt_size.val; if elt_tag.val == tg { let doc = Doc { data: d.data, start: elt_size.next, @@ -260,22 +299,22 @@ pub fn with_doc_data(d: Doc, f: &fn(x: &[u8]) -> T) -> T { pub fn doc_as_u8(d: Doc) -> u8 { assert_eq!(d.end, d.start + 1u); - (*d.data)[d.start] + d.data.as_slice()[d.start] } pub fn doc_as_u16(d: Doc) -> u16 { assert_eq!(d.end, d.start + 2u); - io::u64_from_be_bytes(*d.data, d.start, 2u) as u16 + io::u64_from_be_bytes(d.data.as_slice(), d.start, 2u) as u16 } pub fn doc_as_u32(d: Doc) -> u32 { assert_eq!(d.end, d.start + 4u); - io::u64_from_be_bytes(*d.data, d.start, 4u) as u32 + io::u64_from_be_bytes(d.data.as_slice(), d.start, 4u) as u32 } pub fn doc_as_u64(d: Doc) -> u64 { assert_eq!(d.end, d.start + 8u); - io::u64_from_be_bytes(*d.data, d.start, 8u) + io::u64_from_be_bytes(d.data.as_slice(), d.start, 8u) } pub fn doc_as_i8(d: Doc) -> i8 { doc_as_u8(d) as i8 } @@ -298,8 +337,7 @@ pub fn Decoder(d: Doc) -> Decoder { impl Decoder { fn _check_label(&mut self, lbl: &str) { if self.pos < self.parent.end { - let TaggedDoc { tag: r_tag, doc: r_doc } = - doc_at(self.parent.data, self.pos); + let TaggedDoc { tag: r_tag, doc: r_doc } = doc_at(&self.parent.data, self.pos); if r_tag == (EsLabel as uint) { self.pos = r_doc.end; @@ -316,8 +354,7 @@ fn next_doc(&mut self, exp_tag: EbmlEncoderTag) -> Doc { if self.pos >= self.parent.end { fail!("no more documents in current node!"); } - let TaggedDoc { tag: r_tag, doc: r_doc } = - doc_at(self.parent.data, self.pos); + let TaggedDoc { tag: r_tag, doc: r_doc } = doc_at(&self.parent.data, self.pos); debug!("self.parent=%?-%? self.pos=%? r_tag=%? r_doc=%?-%?", self.parent.start, self.parent.end, diff --git a/src/librustc/metadata/creader.rs b/src/librustc/metadata/creader.rs index c8c4a396c87..f43c089d331 100644 --- a/src/librustc/metadata/creader.rs +++ b/src/librustc/metadata/creader.rs @@ -15,6 +15,7 @@ use metadata::decoder; use metadata::filesearch::FileSearch; use metadata::loader; +use metadata::loader::MetadataSection; use std::hashmap::HashMap; use syntax::ast; @@ -308,7 +309,7 @@ fn resolve_crate(e: @mut Env, } // Go through the crate metadata and load any crates that it references -fn resolve_crate_deps(e: @mut Env, cdata: @~[u8]) -> cstore::cnum_map { +fn resolve_crate_deps(e: @mut Env, cdata: MetadataSection) -> cstore::cnum_map { debug!("resolving deps of external crate"); // The map from crate numbers in the crate we're resolving to local crate // numbers diff --git a/src/librustc/metadata/csearch.rs b/src/librustc/metadata/csearch.rs index 317b9cf6ce3..bc6b3d30d43 100644 --- a/src/librustc/metadata/csearch.rs +++ b/src/librustc/metadata/csearch.rs @@ -188,7 +188,7 @@ pub fn get_field_type(tcx: ty::ctxt, class_id: ast::def_id, def: ast::def_id) -> ty::ty_param_bounds_and_ty { let cstore = tcx.cstore; let cdata = cstore::get_crate_data(cstore, class_id.crate); - let all_items = reader::get_doc(reader::Doc(cdata.data), tag_items); + let all_items = reader::get_doc(decoder::section_to_ebml_doc(cdata.data), tag_items); debug!("Looking up %?", class_id); let class_doc = expect(tcx.diag, decoder::maybe_find_item(class_id.node, all_items), diff --git a/src/librustc/metadata/cstore.rs b/src/librustc/metadata/cstore.rs index a5f541412de..4150894e1d2 100644 --- a/src/librustc/metadata/cstore.rs +++ b/src/librustc/metadata/cstore.rs @@ -15,6 +15,7 @@ use metadata::cstore; use metadata::decoder; +use metadata::loader::MetadataSection; use std::hashmap::HashMap; use extra; @@ -29,7 +30,7 @@ pub struct crate_metadata { name: @str, - data: @~[u8], + data: MetadataSection, cnum_map: cnum_map, cnum: ast::CrateNum } diff --git a/src/librustc/metadata/decoder.rs b/src/librustc/metadata/decoder.rs index 81a2e863bde..47ec96ca97e 100644 --- a/src/librustc/metadata/decoder.rs +++ b/src/librustc/metadata/decoder.rs @@ -20,6 +20,7 @@ use metadata::tydecode::{parse_ty_data, parse_def_id, parse_type_param_def_data, parse_bare_fn_ty_data, parse_trait_ref_data}; +use metadata::loader::{MetadataSection, CopiedSection, UnsafeSection}; use middle::ty; use middle::typeck; use middle::astencode::vtable_decoder_helpers; @@ -56,16 +57,16 @@ fn lookup_hash(d: ebml::Doc, eq_fn: &fn(x:&[u8]) -> bool, hash: u64) -> let index = reader::get_doc(d, tag_index); let table = reader::get_doc(index, tag_index_table); let hash_pos = table.start + (hash % 256 * 4) as uint; - let pos = io::u64_from_be_bytes(*d.data, hash_pos, 4) as uint; - let tagged_doc = reader::doc_at(d.data, pos); + let pos = io::u64_from_be_bytes(d.data.as_slice(), hash_pos, 4) as uint; + let tagged_doc = reader::doc_at(&d.data, pos); let belt = tag_index_buckets_bucket_elt; let mut ret = None; do reader::tagged_docs(tagged_doc.doc, belt) |elt| { - let pos = io::u64_from_be_bytes(*elt.data, elt.start, 4) as uint; + let pos = io::u64_from_be_bytes(elt.data.as_slice(), elt.start, 4) as uint; if eq_fn(elt.data.slice(elt.start + 4, elt.end)) { - ret = Some(reader::doc_at(d.data, pos).doc); + ret = Some(reader::doc_at(&d.data, pos).doc); false } else { true @@ -96,8 +97,8 @@ fn find_item(item_id: int, items: ebml::Doc) -> ebml::Doc { // Looks up an item in the given metadata and returns an ebml doc pointing // to the item data. -fn lookup_item(item_id: int, data: @~[u8]) -> ebml::Doc { - let items = reader::get_doc(reader::Doc(data), tag_items); +fn lookup_item(item_id: int, data: MetadataSection) -> ebml::Doc { + let items = reader::get_doc(section_to_ebml_doc(data), tag_items); find_item(item_id, items) } @@ -215,13 +216,13 @@ fn variant_disr_val(d: ebml::Doc) -> Option { fn doc_type(doc: ebml::Doc, tcx: ty::ctxt, cdata: cmd) -> ty::t { let tp = reader::get_doc(doc, tag_items_data_item_type); - parse_ty_data(*tp.data, cdata.cnum, tp.start, tcx, + parse_ty_data(tp.data.as_slice(), cdata.cnum, tp.start, tcx, |_, did| translate_def_id(cdata, did)) } fn doc_method_fty(doc: ebml::Doc, tcx: ty::ctxt, cdata: cmd) -> ty::BareFnTy { let tp = reader::get_doc(doc, tag_item_method_fty); - parse_bare_fn_ty_data(*tp.data, cdata.cnum, tp.start, tcx, + parse_bare_fn_ty_data(tp.data.as_slice(), cdata.cnum, tp.start, tcx, |_, did| translate_def_id(cdata, did)) } @@ -230,7 +231,7 @@ fn doc_transformed_self_ty(doc: ebml::Doc, cdata: cmd) -> Option { do reader::maybe_get_doc(doc, tag_item_method_transformed_self_ty).map |tp| { - parse_ty_data(*tp.data, cdata.cnum, tp.start, tcx, + parse_ty_data(tp.data.as_slice(), cdata.cnum, tp.start, tcx, |_, did| translate_def_id(cdata, did)) } } @@ -241,7 +242,7 @@ pub fn item_type(_item_id: ast::def_id, item: ebml::Doc, } fn doc_trait_ref(doc: ebml::Doc, tcx: ty::ctxt, cdata: cmd) -> ty::TraitRef { - parse_trait_ref_data(*doc.data, cdata.cnum, doc.start, tcx, + parse_trait_ref_data(doc.data.as_slice(), cdata.cnum, doc.start, tcx, |_, did| translate_def_id(cdata, did)) } @@ -256,7 +257,7 @@ fn item_ty_param_defs(item: ebml::Doc, tcx: ty::ctxt, cdata: cmd, let mut bounds = ~[]; do reader::tagged_docs(item, tag) |p| { let bd = parse_type_param_def_data( - *p.data, p.start, cdata.cnum, tcx, + p.data.as_slice(), p.start, cdata.cnum, tcx, |_, did| translate_def_id(cdata, did)); bounds.push(bd); true @@ -359,7 +360,7 @@ fn item_to_def_like(item: ebml::Doc, did: ast::def_id, cnum: ast::CrateNum) } } -pub fn lookup_def(cnum: ast::CrateNum, data: @~[u8], did_: ast::def_id) -> +pub fn lookup_def(cnum: ast::CrateNum, data: MetadataSection, did_: ast::def_id) -> ast::def { let item = lookup_item(did_.node, data); let did = ast::def_id { crate: cnum, node: did_.node }; @@ -406,7 +407,7 @@ pub fn get_region_param(cdata: cmd, id: ast::NodeId) return item_ty_region_param(item); } -pub fn get_type_param_count(data: @~[u8], id: ast::NodeId) -> uint { +pub fn get_type_param_count(data: MetadataSection, id: ast::NodeId) -> uint { item_ty_param_count(lookup_item(id, data)) } @@ -437,7 +438,7 @@ pub fn get_impl_vtables(cdata: cmd, pub fn get_impl_method(intr: @ident_interner, cdata: cmd, id: ast::NodeId, name: ast::ident) -> Option { - let items = reader::get_doc(reader::Doc(cdata.data), tag_items); + let items = reader::get_doc(section_to_ebml_doc(cdata.data), tag_items); let mut found = None; do reader::tagged_docs(find_item(id, items), tag_item_impl_method) |mid| { let m_did = reader::with_doc_data(mid, parse_def_id); @@ -449,7 +450,7 @@ pub fn get_impl_method(intr: @ident_interner, cdata: cmd, id: ast::NodeId, found } -pub fn get_symbol(data: @~[u8], id: ast::NodeId) -> ~str { +pub fn get_symbol(data: MetadataSection, id: ast::NodeId) -> ~str { return item_symbol(lookup_item(id, data)); } @@ -470,7 +471,7 @@ fn def_like_to_def(def_like: def_like) -> ast::def { /// Iterates over the language items in the given crate. pub fn each_lang_item(cdata: cmd, f: &fn(ast::NodeId, uint) -> bool) -> bool { - let root = reader::Doc(cdata.data); + let root = section_to_ebml_doc(cdata.data); let lang_items = reader::get_doc(root, tag_lang_items); do reader::tagged_docs(lang_items, tag_lang_items_item) |item_doc| { let id_doc = reader::get_doc(item_doc, tag_lang_items_item_id); @@ -565,10 +566,10 @@ fn process_item_and_pop_name(&mut self, fn each_item_of_module(&mut self, def_id: ast::def_id) -> bool { // This item might not be in this crate. If it's not, look it up. let items = if def_id.crate == self.cdata.cnum { - reader::get_doc(reader::Doc(self.cdata.data), tag_items) + reader::get_doc(section_to_ebml_doc(self.cdata.data), tag_items) } else { let crate_data = (self.get_crate_data)(def_id.crate); - let root = reader::Doc(crate_data.data); + let root = section_to_ebml_doc(crate_data.data); reader::get_doc(root, tag_items) }; @@ -594,10 +595,10 @@ fn each_child_of_module_or_crate(&mut self, item_doc: ebml::Doc) -> bool { // a reexport. let other_crates_items = if child_def_id.crate == self.cdata.cnum { - reader::get_doc(reader::Doc(self.cdata.data), tag_items) + reader::get_doc(section_to_ebml_doc(self.cdata.data), tag_items) } else { let crate_data = (self.get_crate_data)(child_def_id.crate); - let root = reader::Doc(crate_data.data); + let root = section_to_ebml_doc(crate_data.data); reader::get_doc(root, tag_items) }; @@ -661,10 +662,10 @@ fn each_child_of_module_or_crate(&mut self, item_doc: ebml::Doc) -> bool { // This reexport may be in yet another crate. let other_crates_items = if def_id.crate == self.cdata.cnum { - reader::get_doc(reader::Doc(self.cdata.data), tag_items) + reader::get_doc(section_to_ebml_doc(self.cdata.data), tag_items) } else { let crate_data = (self.get_crate_data)(def_id.crate); - let root = reader::Doc(crate_data.data); + let root = section_to_ebml_doc(crate_data.data); reader::get_doc(root, tag_items) }; @@ -696,7 +697,7 @@ pub fn each_path(intr: @ident_interner, // make fast. It's the source of most of the performance problems when // compiling small crates. - let root_doc = reader::Doc(cdata.data); + let root_doc = section_to_ebml_doc(cdata.data); let misc_info_doc = reader::get_doc(root_doc, tag_misc_info); let crate_items_doc = reader::get_doc(misc_info_doc, tag_misc_info_crate_items); @@ -756,7 +757,7 @@ pub fn maybe_get_item_ast(cdata: cmd, tcx: ty::ctxt, pub fn get_enum_variants(intr: @ident_interner, cdata: cmd, id: ast::NodeId, tcx: ty::ctxt) -> ~[@ty::VariantInfo] { let data = cdata.data; - let items = reader::get_doc(reader::Doc(data), tag_items); + let items = reader::get_doc(section_to_ebml_doc(data), tag_items); let item = find_item(id, items); let mut infos: ~[@ty::VariantInfo] = ~[]; let variant_ids = enum_variant_ids(item, cdata); @@ -1188,8 +1189,15 @@ fn list_crate_attributes(intr: @ident_interner, md: ebml::Doc, hash: &str, out.write_str("\n\n"); } -pub fn get_crate_attributes(data: @~[u8]) -> ~[ast::Attribute] { - return get_attributes(reader::Doc(data)); +pub fn get_crate_attributes(data: MetadataSection) -> ~[ast::Attribute] { + return get_attributes(section_to_ebml_doc(data)); +} + +pub fn section_to_ebml_doc(data: MetadataSection) -> ebml::Doc { + match data { + CopiedSection(data) => reader::Doc(data), + UnsafeSection(_, buf, len) => reader::unsafe_Doc(buf, len) + } } #[deriving(Clone)] @@ -1200,9 +1208,9 @@ pub struct crate_dep { hash: @str } -pub fn get_crate_deps(data: @~[u8]) -> ~[crate_dep] { +pub fn get_crate_deps(data: MetadataSection) -> ~[crate_dep] { let mut deps: ~[crate_dep] = ~[]; - let cratedoc = reader::Doc(data); + let cratedoc = section_to_ebml_doc(data); let depsdoc = reader::get_doc(cratedoc, tag_crate_deps); let mut crate_num = 1; fn docstr(doc: ebml::Doc, tag_: uint) -> @str { @@ -1220,7 +1228,7 @@ fn docstr(doc: ebml::Doc, tag_: uint) -> @str { return deps; } -fn list_crate_deps(data: @~[u8], out: @io::Writer) { +fn list_crate_deps(data: MetadataSection, out: @io::Writer) { out.write_str("=External Dependencies=\n"); let r = get_crate_deps(data); @@ -1233,13 +1241,13 @@ fn list_crate_deps(data: @~[u8], out: @io::Writer) { out.write_str("\n"); } -pub fn get_crate_hash(data: @~[u8]) -> @str { - let cratedoc = reader::Doc(data); +pub fn get_crate_hash(data: MetadataSection) -> @str { + let cratedoc = section_to_ebml_doc(data); let hashdoc = reader::get_doc(cratedoc, tag_crate_hash); hashdoc.as_str_slice().to_managed() } -pub fn get_crate_vers(data: @~[u8]) -> @str { +pub fn get_crate_vers(data: MetadataSection) -> @str { let attrs = decoder::get_crate_attributes(data); let linkage_attrs = attr::find_linkage_metas(attrs); @@ -1264,10 +1272,10 @@ fn iter_crate_items(intr: @ident_interner, cdata: cmd, }; } -pub fn list_crate_metadata(intr: @ident_interner, bytes: @~[u8], +pub fn list_crate_metadata(intr: @ident_interner, bytes: MetadataSection, out: @io::Writer) { let hash = get_crate_hash(bytes); - let md = reader::Doc(bytes); + let md = section_to_ebml_doc(bytes); list_crate_attributes(intr, md, hash, out); list_crate_deps(bytes, out); } @@ -1289,7 +1297,7 @@ pub fn translate_def_id(cdata: cmd, did: ast::def_id) -> ast::def_id { } pub fn get_link_args_for_crate(cdata: cmd) -> ~[~str] { - let link_args = reader::get_doc(reader::Doc(cdata.data), tag_link_args); + let link_args = reader::get_doc(section_to_ebml_doc(cdata.data), tag_link_args); let mut result = ~[]; do reader::tagged_docs(link_args, tag_link_args_arg) |arg_doc| { result.push(arg_doc.as_str()); diff --git a/src/librustc/metadata/loader.rs b/src/librustc/metadata/loader.rs index f374ce2c19a..5f73888bb7d 100644 --- a/src/librustc/metadata/loader.rs +++ b/src/librustc/metadata/loader.rs @@ -11,7 +11,7 @@ //! Finds crate binaries and loads their metadata -use lib::llvm::{False, llvm, mk_object_file, mk_section_iter}; +use lib::llvm::{False, llvm, mk_object_file, mk_section_iter, ObjectFile}; use metadata::decoder; use metadata::encoder; use metadata::filesearch::FileSearch; @@ -54,7 +54,13 @@ pub struct Context { intr: @ident_interner } -pub fn load_library_crate(cx: &Context) -> (~str, @~[u8]) { +#[deriving(Clone)] +pub enum MetadataSection { + CopiedSection(@~[u8]), + UnsafeSection(@ObjectFile, *u8, uint) +} + +pub fn load_library_crate(cx: &Context) -> (~str, MetadataSection) { match find_library_crate(cx) { Some(t) => t, None => { @@ -65,7 +71,7 @@ pub fn load_library_crate(cx: &Context) -> (~str, @~[u8]) { } } -fn find_library_crate(cx: &Context) -> Option<(~str, @~[u8])> { +fn find_library_crate(cx: &Context) -> Option<(~str, MetadataSection)> { attr::require_unique_names(cx.diag, cx.metas); find_library_crate_aux(cx, libname(cx), cx.filesearch) } @@ -87,7 +93,7 @@ fn find_library_crate_aux( cx: &Context, (prefix, suffix): (~str, ~str), filesearch: @filesearch::FileSearch -) -> Option<(~str, @~[u8])> { +) -> Option<(~str, MetadataSection)> { let crate_name = crate_name_from_metas(cx.metas); // want: crate_name.dir_part() + prefix + crate_name.file_part + "-" let prefix = fmt!("%s%s-", prefix, crate_name); @@ -171,7 +177,7 @@ pub fn note_linkage_attrs(intr: @ident_interner, } } -fn crate_matches(crate_data: @~[u8], +fn crate_matches(crate_data: MetadataSection, metas: &[@ast::MetaItem], hash: @str) -> bool { let attrs = decoder::get_crate_attributes(crate_data); @@ -197,17 +203,17 @@ pub fn metadata_matches(extern_metas: &[@ast::MetaItem], } fn get_metadata_section(os: os, - filename: &Path) -> Option<@~[u8]> { + filename: &Path) -> Option { unsafe { let mb = do filename.with_c_str |buf| { llvm::LLVMRustCreateMemoryBufferWithContentsOfFile(buf) }; - if mb as int == 0 { return option::None::<@~[u8]>; } - let of = match mk_object_file(mb) { + if mb as int == 0 { return None; } + let of = @match mk_object_file(mb) { option::Some(of) => of, - _ => return option::None::<@~[u8]> + _ => return None }; - let si = mk_section_iter(of.llof); + let si = mk_section_iter((*of).llof); while llvm::LLVMIsSectionIteratorAtEnd(of.llof, si.llsi) == False { let name_buf = llvm::LLVMGetSectionName(si.llsi); let name = str::raw::from_c_str(name_buf); @@ -233,28 +239,24 @@ fn get_metadata_section(os: os, let must_decompress = *ptr::offset(cvbuf, vlen as int) == 1; let cvbuf1 = ptr::offset(cvbuf, vlen as int + 1); - do vec::raw::buf_as_slice(cvbuf1, csz-vlen-1) |bytes| { - if must_decompress { + if must_decompress { + do vec::raw::buf_as_slice(cvbuf1, csz-vlen-1) |bytes| { debug!("inflating %u bytes of compressed metadata", csz - vlen); let inflated = flate::inflate_bytes(bytes); - found = Some(@(inflated)); - } else { - // Copy the byte vector as fast as possible - let mut buf = vec::with_capacity(bytes.len()); - vec::raw::set_len(&mut buf, bytes.len()); - vec::raw::copy_memory(buf, bytes, bytes.len()); - found = Some(@buf) + found = Some(CopiedSection(@inflated)); } + } else { + found = Some(UnsafeSection(of, cvbuf1, csz-vlen-1)) } - if found != None { + if !found.is_none() { return found; } } llvm::LLVMMoveToNextSection(si.llsi); } - return option::None::<@~[u8]>; + return None; } } diff --git a/src/librustc/middle/astencode.rs b/src/librustc/middle/astencode.rs index a22daac90b5..791be656d4b 100644 --- a/src/librustc/middle/astencode.rs +++ b/src/librustc/middle/astencode.rs @@ -1022,7 +1022,7 @@ fn read_ty_noxcx(&mut self, tcx: ty::ctxt, cdata: @cstore::crate_metadata) -> ty::t { do self.read_opaque |_, doc| { tydecode::parse_ty_data( - *doc.data, + doc.data.as_slice(), cdata.cnum, doc.start, tcx, @@ -1044,7 +1044,7 @@ fn read_ty(&mut self, xcx: @ExtendedDecodeContext) -> ty::t { return do self.read_opaque |this, doc| { let ty = tydecode::parse_ty_data( - *doc.data, + doc.data.as_slice(), xcx.dcx.cdata.cnum, doc.start, xcx.dcx.tcx, @@ -1060,7 +1060,7 @@ fn read_ty(&mut self, xcx: @ExtendedDecodeContext) -> ty::t { fn type_string(doc: ebml::Doc) -> ~str { let mut str = ~""; for i in range(doc.start, doc.end) { - str.push_char(doc.data[i] as char); + str.push_char(doc.data.as_slice()[i] as char); } str } @@ -1074,7 +1074,7 @@ fn read_type_param_def(&mut self, xcx: @ExtendedDecodeContext) -> ty::TypeParameterDef { do self.read_opaque |this, doc| { tydecode::parse_type_param_def_data( - *doc.data, + doc.data.as_slice(), doc.start, xcx.dcx.cdata.cnum, xcx.dcx.tcx, From 022f188a08b11234b2a5fbce3484d233625371c1 Mon Sep 17 00:00:00 2001 From: Brian Anderson Date: Sat, 24 Aug 2013 20:57:35 -0700 Subject: [PATCH 3/3] Don't ever compress metadata --- Makefile.in | 6 ------ configure | 1 - src/librustc/driver/driver.rs | 5 ----- src/librustc/driver/session.rs | 2 -- src/librustc/metadata/decoder.rs | 3 +-- src/librustc/metadata/encoder.rs | 14 +------------- src/librustc/metadata/loader.rs | 25 ++++--------------------- src/librustc/middle/trans/base.rs | 1 - 8 files changed, 6 insertions(+), 51 deletions(-) diff --git a/Makefile.in b/Makefile.in index fb0bb6ff379..a9a41a073d0 100644 --- a/Makefile.in +++ b/Makefile.in @@ -107,12 +107,6 @@ else CFG_GCCISH_CFLAGS += -DRUST_NDEBUG endif -ifndef CFG_ENABLE_COMPRESS_METADATA - # XXX: After snapshots extend this to all stages - RUSTFLAGS_STAGE1 += --no-compress-metadata - RUSTFLAGS_STAGE2 += --no-compress-metadata -endif - ifdef SAVE_TEMPS CFG_RUSTC_FLAGS += --save-temps endif diff --git a/configure b/configure index 8a7a2bae4ea..ed3042a1e71 100755 --- a/configure +++ b/configure @@ -383,7 +383,6 @@ opt ccache 0 "invoke gcc/clang via ccache to reuse object files between builds" opt local-rust 0 "use an installed rustc rather than downloading a snapshot" opt pax-flags 0 "apply PaX flags to rustc binaries (required for GRSecurity/PaX-patched kernels)" -opt compress-metadata 0 "compress crate metadata" valopt prefix "/usr/local" "set installation prefix" valopt local-rust-root "/usr/local" "set prefix for local rust binary" valopt llvm-root "" "set LLVM root" diff --git a/src/librustc/driver/driver.rs b/src/librustc/driver/driver.rs index bba818c7c9d..dd57be823ab 100644 --- a/src/librustc/driver/driver.rs +++ b/src/librustc/driver/driver.rs @@ -718,8 +718,6 @@ fn set_llvm_debug() { let android_cross_path = getopts::opt_maybe_str( matches, "android-cross-path"); - let no_compress_metadata = opt_present(matches, "no-compress-metadata"); - let custom_passes = match getopts::opt_maybe_str(matches, "passes") { None => ~[], Some(s) => { @@ -755,7 +753,6 @@ fn set_llvm_debug() { no_trans: no_trans, debugging_opts: debugging_opts, android_cross_path: android_cross_path, - no_compress_metadata: no_compress_metadata }; return sopts; } @@ -873,8 +870,6 @@ pub fn optgroups() -> ~[getopts::groups::OptGroup] { for details)", "FEATURE"), optopt("", "android-cross-path", "The path to the Android NDK", "PATH"), - optflag("", "no-compress-metadata", - "Do not compress crate metadata (make builds a little faster)"), optflagopt("W", "warn", "Set lint warnings", "OPT"), optmulti("A", "allow", diff --git a/src/librustc/driver/session.rs b/src/librustc/driver/session.rs index e70ac52a0f5..50b29ff16be 100644 --- a/src/librustc/driver/session.rs +++ b/src/librustc/driver/session.rs @@ -166,7 +166,6 @@ pub struct options { no_trans: bool, debugging_opts: uint, android_cross_path: Option<~str>, - no_compress_metadata: bool } pub struct crate_metadata { @@ -351,7 +350,6 @@ pub fn basic_options() -> @options { no_trans: false, debugging_opts: 0u, android_cross_path: None, - no_compress_metadata: false } } diff --git a/src/librustc/metadata/decoder.rs b/src/librustc/metadata/decoder.rs index 47ec96ca97e..39ba64c5e26 100644 --- a/src/librustc/metadata/decoder.rs +++ b/src/librustc/metadata/decoder.rs @@ -20,7 +20,7 @@ use metadata::tydecode::{parse_ty_data, parse_def_id, parse_type_param_def_data, parse_bare_fn_ty_data, parse_trait_ref_data}; -use metadata::loader::{MetadataSection, CopiedSection, UnsafeSection}; +use metadata::loader::{MetadataSection, UnsafeSection}; use middle::ty; use middle::typeck; use middle::astencode::vtable_decoder_helpers; @@ -1195,7 +1195,6 @@ pub fn get_crate_attributes(data: MetadataSection) -> ~[ast::Attribute] { pub fn section_to_ebml_doc(data: MetadataSection) -> ebml::Doc { match data { - CopiedSection(data) => reader::Doc(data), UnsafeSection(_, buf, len) => reader::unsafe_Doc(buf, len) } } diff --git a/src/librustc/metadata/encoder.rs b/src/librustc/metadata/encoder.rs index 3ada5592f75..38d2d3567bb 100644 --- a/src/librustc/metadata/encoder.rs +++ b/src/librustc/metadata/encoder.rs @@ -26,7 +26,6 @@ use std::io; use std::str; use std::vec; -use extra::flate; use extra::serialize::Encodable; use extra; use syntax::abi::AbiSet; @@ -64,7 +63,6 @@ pub struct EncodeParams<'self> { cstore: @mut cstore::CStore, encode_inlined_item: encode_inlined_item<'self>, reachable: @mut HashSet, - compress: bool } struct Stats { @@ -1595,7 +1593,6 @@ pub fn encode_metadata(parms: EncodeParams, crate: &Crate) -> ~[u8] { encode_inlined_item, link_meta, reachable, - compress, _ } = parms; let type_abbrevs = @mut HashMap::new(); @@ -1681,17 +1678,8 @@ pub fn encode_metadata(parms: EncodeParams, crate: &Crate) -> ~[u8] { wr.write(&[0u8, 0u8, 0u8, 0u8]); let writer_bytes: &mut ~[u8] = wr.bytes; - let compression_flag = if compress { [1u8] } else { [0u8] }; - if compress { - metadata_encoding_version.to_owned() + - compression_flag.to_owned() + - flate::deflate_bytes(*writer_bytes) - } else { - metadata_encoding_version.to_owned() + - compression_flag.to_owned() + - *writer_bytes - } + return metadata_encoding_version.to_owned() + *writer_bytes; } // Get the encoded string for a type diff --git a/src/librustc/metadata/loader.rs b/src/librustc/metadata/loader.rs index 5f73888bb7d..182acc68b1e 100644 --- a/src/librustc/metadata/loader.rs +++ b/src/librustc/metadata/loader.rs @@ -32,7 +32,6 @@ use std::ptr; use std::str; use std::vec; -use extra::flate; pub enum os { os_macos, @@ -56,7 +55,8 @@ pub struct Context { #[deriving(Clone)] pub enum MetadataSection { - CopiedSection(@~[u8]), + // A pointer to the object file metadata section, along with + // the ObjectFile handle that keeps it from being destructed UnsafeSection(@ObjectFile, *u8, uint) } @@ -221,7 +221,6 @@ fn get_metadata_section(os: os, if name == read_meta_section_name(os) { let cbuf = llvm::LLVMGetSectionContents(si.llsi); let csz = llvm::LLVMGetSectionSize(si.llsi) as uint; - let mut found = None; let cvbuf: *u8 = cast::transmute(cbuf); let vlen = encoder::metadata_encoding_version.len(); debug!("checking %u bytes of metadata-version stamp", @@ -234,25 +233,9 @@ fn get_metadata_section(os: os, } if !version_ok { return None; } - assert!(csz >= vlen + 1); + let cvbuf1 = ptr::offset(cvbuf, vlen as int); - let must_decompress = *ptr::offset(cvbuf, vlen as int) == 1; - let cvbuf1 = ptr::offset(cvbuf, vlen as int + 1); - - if must_decompress { - do vec::raw::buf_as_slice(cvbuf1, csz-vlen-1) |bytes| { - debug!("inflating %u bytes of compressed metadata", - csz - vlen); - let inflated = flate::inflate_bytes(bytes); - found = Some(CopiedSection(@inflated)); - } - } else { - found = Some(UnsafeSection(of, cvbuf1, csz-vlen-1)) - } - - if !found.is_none() { - return found; - } + return Some(UnsafeSection(of, cvbuf1, csz-vlen)) } llvm::LLVMMoveToNextSection(si.llsi); } diff --git a/src/librustc/middle/trans/base.rs b/src/librustc/middle/trans/base.rs index 85db79ca54b..f19decaa38c 100644 --- a/src/librustc/middle/trans/base.rs +++ b/src/librustc/middle/trans/base.rs @@ -2907,7 +2907,6 @@ pub fn crate_ctxt_to_encode_parms<'r>(cx: &'r CrateContext, ie: encoder::encode_ cstore: cx.sess.cstore, encode_inlined_item: ie, reachable: cx.reachable, - compress: !cx.sess.opts.no_compress_metadata } }