auto merge of #15601 : jbclements/rust/disable-default-macro-behavior, r=alexcrichton

Our AST definition can include macro invocations, which can expand into all kinds of things. Macro invocations are expanded away during expansion time, and the rest of the compiler doesn't have to deal with them. However, we have no way of enforcing this.

This patch adds two protective mechanisms.

First, it adds a (quick) explicit check that ensures there are no macro invocations remaining in the AST after expansion. Second, it updates the visit and fold mechanisms so that by default, they will not traverse macro invocations. It's easy enough to add this, if desired (it's documented in the source, and examples appear, e.g. in the IdentFinder.

Along the way, I also consulted with @sfackler to refactor the macro export mechanism so that it stores macro text spans in a side table, rather than leaving them in the AST.

Author: bors

src/librustc/driver/driver.rs

@@ -270,6 +270,8 @@ pub fn phase_2_configure_and_expand(sess: &Session,
         }
     );
 
+    // JBC: make CFG processing part of expansion to avoid this problem:
+
     // strip again, in case expansion added anything with a #[cfg].
     krate = time(time_passes, "configuration 2", krate, |krate|
                  front::config::strip_unconfigured_items(krate));
@@ -290,6 +292,9 @@ pub fn phase_2_configure_and_expand(sess: &Session,
         krate.encode(&mut json).unwrap();
     }
 
+    time(time_passes, "checking that all macro invocations are gone", &krate, |krate|
+         syntax::ext::expand::check_for_macros(&sess.parse_sess, krate));
+
     Some((krate, map))
 }
 
@@ -302,14 +307,14 @@ pub struct CrateAnalysis {
     pub name: String,
 }
 
+
 /// Run the resolution, typechecking, region checking and other
 /// miscellaneous analysis passes on the crate. Return various
 /// structures carrying the results of the analysis.
 pub fn phase_3_run_analysis_passes(sess: Session,
                                    krate: &ast::Crate,
                                    ast_map: syntax::ast_map::Map,
                                    name: String) -> CrateAnalysis {
-
     let time_passes = sess.time_passes();
 
     time(time_passes, "external crate/lib resolution", (), |_|

src/librustc/driver/session.rs

@@ -29,7 +29,8 @@ use syntax::{ast, codemap};
 use std::os;
 use std::cell::{Cell, RefCell};
 
-
+// Represents the data associated with a compilation
+// session for a single crate.
 pub struct Session {
     pub targ_cfg: config::Config,
     pub opts: config::Options,

src/librustc/front/config.rs

@@ -14,6 +14,8 @@ use syntax::codemap;
 
 use std::gc::{Gc, GC};
 
+/// A folder that strips out items that do not belong in the current
+/// configuration.
 struct Context<'a> {
     in_cfg: |attrs: &[ast::Attribute]|: 'a -> bool,
 }
@@ -41,6 +43,9 @@ impl<'a> fold::Folder for Context<'a> {
     fn fold_expr(&mut self, expr: Gc<ast::Expr>) -> Gc<ast::Expr> {
         fold_expr(self, expr)
     }
+    fn fold_mac(&mut self, mac: &ast::Mac) -> ast::Mac {
+        fold::fold_mac(mac, self)
+    }
 }
 
 pub fn strip_items(krate: ast::Crate,

src/librustc/metadata/encoder.rs

@@ -1588,37 +1588,25 @@ fn encode_plugin_registrar_fn(ecx: &EncodeContext, ebml_w: &mut Encoder) {
     }
 }
 
-struct MacroDefVisitor<'a, 'b, 'c> {
-    ecx: &'a EncodeContext<'b>,
-    ebml_w: &'a mut Encoder<'c>
-}
-
-impl<'a, 'b, 'c> Visitor<()> for MacroDefVisitor<'a, 'b, 'c> {
-    fn visit_item(&mut self, item: &Item, _: ()) {
-        match item.node {
-            ItemMac(..) => {
-                let def = self.ecx.tcx.sess.codemap().span_to_snippet(item.span)
-                    .expect("Unable to find source for macro");
-                self.ebml_w.start_tag(tag_macro_def);
-                self.ebml_w.wr_str(def.as_slice());
-                self.ebml_w.end_tag();
-            }
-            _ => {}
-        }
-        visit::walk_item(self, item, ());
-    }
+/// Given a span, write the text of that span into the output stream
+/// as an exported macro
+fn encode_macro_def(ecx: &EncodeContext,
+                    ebml_w: &mut Encoder,
+                    span: &syntax::codemap::Span) {
+    let def = ecx.tcx.sess.codemap().span_to_snippet(*span)
+        .expect("Unable to find source for macro");
+    ebml_w.start_tag(tag_macro_def);
+    ebml_w.wr_str(def.as_slice());
+    ebml_w.end_tag();
 }
 
-fn encode_macro_defs<'a>(ecx: &'a EncodeContext,
-                         krate: &Crate,
-                         ebml_w: &'a mut Encoder) {
+/// Serialize the text of the exported macros
+fn encode_macro_defs(ecx: &EncodeContext,
+                     krate: &Crate,
+                     ebml_w: &mut Encoder) {
     ebml_w.start_tag(tag_exported_macros);
-    {
-        let mut visitor = MacroDefVisitor {
-            ecx: ecx,
-            ebml_w: ebml_w,
-        };
-        visit::walk_crate(&mut visitor, krate, ());
+    for span in krate.exported_macros.iter() {
+        encode_macro_def(ecx, ebml_w, span);
     }
     ebml_w.end_tag();
 }

src/librustc/plugin/load.rs

@@ -72,6 +72,7 @@ pub fn load_plugins(sess: &Session, krate: &ast::Crate) -> Plugins {
     loader.plugins
 }
 
+// note that macros aren't expanded yet, and therefore macros can't add plugins.
 impl<'a> Visitor<()> for PluginLoader<'a> {
     fn visit_view_item(&mut self, vi: &ast::ViewItem, _: ()) {
         match vi.node {
@@ -109,6 +110,10 @@ impl<'a> Visitor<()> for PluginLoader<'a> {
             _ => (),
         }
     }
+    fn visit_mac(&mut self, _: &ast::Mac, _:()) {
+        // bummer... can't see plugins inside macros.
+        // do nothing.
+    }
 }
 
 impl<'a> PluginLoader<'a> {

src/libsyntax/ast.rs

@@ -249,6 +249,7 @@ pub struct Crate {
     pub attrs: Vec<Attribute>,
     pub config: CrateConfig,
     pub span: Span,
+    pub exported_macros: Vec<Span>
 }
 
 pub type MetaItem = Spanned<MetaItem_>;
@@ -1245,6 +1246,7 @@ mod test {
                 hi: BytePos(20),
                 expn_info: None,
             },
+            exported_macros: Vec::new(),
         };
         // doesn't matter which encoder we use....
         let _f = &e as &serialize::Encodable<json::Encoder, io::IoError>;

src/libsyntax/ast_map.rs

@@ -112,6 +112,7 @@ pub enum Node {
     NodeLifetime(Gc<Lifetime>),
 }
 
+/// Represents an entry and its parent Node ID
 /// The odd layout is to bring down the total size.
 #[deriving(Clone)]
 enum MapEntry {
@@ -184,6 +185,8 @@ impl MapEntry {
     }
 }
 
+/// Represents a mapping from Node IDs to AST elements and their parent
+/// Node IDs
 pub struct Map {
     /// NodeIds are sequential integers from 0, so we can be
     /// super-compact by storing them in a vector. Not everything with
@@ -430,6 +433,8 @@ pub trait FoldOps {
     }
 }
 
+/// A Folder that walks over an AST and constructs a Node ID Map. Its
+/// fold_ops argument has the opportunity to replace Node IDs and spans.
 pub struct Ctx<'a, F> {
     map: &'a Map,
     /// The node in which we are currently mapping (an item or a method).
@@ -584,6 +589,10 @@ impl<'a, F: FoldOps> Folder for Ctx<'a, F> {
         self.insert(lifetime.id, EntryLifetime(self.parent, box(GC) lifetime));
         lifetime
     }
+
+    fn fold_mac(&mut self, mac: &Mac) -> Mac {
+        fold::fold_mac(mac, self)
+    }
 }
 
 pub fn map_crate<F: FoldOps>(krate: Crate, fold_ops: F) -> (Crate, Map) {

src/libsyntax/ext/base.rs

@@ -48,7 +48,8 @@ pub struct BasicMacroExpander {
     pub span: Option<Span>
 }
 
-pub trait MacroExpander {
+/// Represents a thing that maps token trees to Macro Results
+pub trait TTMacroExpander {
     fn expand(&self,
               ecx: &mut ExtCtxt,
               span: Span,
@@ -60,7 +61,7 @@ pub type MacroExpanderFn =
     fn(ecx: &mut ExtCtxt, span: codemap::Span, token_tree: &[ast::TokenTree])
        -> Box<MacResult>;
 
-impl MacroExpander for BasicMacroExpander {
+impl TTMacroExpander for BasicMacroExpander {
     fn expand(&self,
               ecx: &mut ExtCtxt,
               span: Span,
@@ -259,7 +260,7 @@ pub enum SyntaxExtension {
     /// A normal, function-like syntax extension.
     ///
     /// `bytes!` is a `NormalTT`.
-    NormalTT(Box<MacroExpander + 'static>, Option<Span>),
+    NormalTT(Box<TTMacroExpander + 'static>, Option<Span>),
 
     /// A function-like syntax extension that has an extra ident before
     /// the block.
@@ -409,6 +410,7 @@ pub struct ExtCtxt<'a> {
 
     pub mod_path: Vec<ast::Ident> ,
     pub trace_mac: bool,
+    pub exported_macros: Vec<codemap::Span>
 }
 
 impl<'a> ExtCtxt<'a> {
@@ -420,7 +422,8 @@ impl<'a> ExtCtxt<'a> {
             backtrace: None,
             mod_path: Vec::new(),
             ecfg: ecfg,
-            trace_mac: false
+            trace_mac: false,
+            exported_macros: Vec::new(),
         }
     }
 
@@ -538,6 +541,9 @@ impl<'a> ExtCtxt<'a> {
     pub fn name_of(&self, st: &str) -> ast::Name {
         token::intern(st)
     }
+    pub fn push_exported_macro(&mut self, span: codemap::Span) {
+        self.exported_macros.push(span);
+    }
 }
 
 /// Extract a string literal from the macro expanded version of `expr`,

src/libsyntax/ext/expand.rs

@@ -518,10 +518,9 @@ fn expand_item_mac(it: Gc<ast::Item>, fld: &mut MacroExpander)
             // create issue to recommend refactoring here?
             fld.extsbox.insert(intern(name.as_slice()), ext);
             if attr::contains_name(it.attrs.as_slice(), "macro_export") {
-                SmallVector::one(it)
-            } else {
-                SmallVector::zero()
+                fld.cx.push_exported_macro(it.span);
             }
+            SmallVector::zero()
         }
         None => {
             match expanded.make_items() {
@@ -754,7 +753,6 @@ impl Visitor<()> for PatIdentFinder {
             _ => visit::walk_pat(self, pattern, ())
         }
     }
-
 }
 
 /// find the PatIdent paths in a pattern
@@ -903,6 +901,9 @@ impl<'a> Folder for IdentRenamer<'a> {
             ctxt: mtwt::apply_renames(self.renames, id.ctxt),
         }
     }
+    fn fold_mac(&mut self, macro: &ast::Mac) -> ast::Mac {
+        fold::fold_mac(macro, self)
+    }
 }
 
 /// A tree-folder that applies every rename in its list to
@@ -932,6 +933,9 @@ impl<'a> Folder for PatIdentRenamer<'a> {
             _ => noop_fold_pat(pat, self)
         }
     }
+    fn fold_mac(&mut self, macro: &ast::Mac) -> ast::Mac {
+        fold::fold_mac(macro, self)
+    }
 }
 
 // expand a method
@@ -1039,6 +1043,7 @@ pub struct ExportedMacros {
 
 pub fn expand_crate(parse_sess: &parse::ParseSess,
                     cfg: ExpansionConfig,
+                    // these are the macros being imported to this crate:
                     macros: Vec<ExportedMacros>,
                     user_exts: Vec<NamedSyntaxExtension>,
                     c: Crate) -> Crate {
@@ -1066,7 +1071,8 @@ pub fn expand_crate(parse_sess: &parse::ParseSess,
         expander.extsbox.insert(name, extension);
     }
 
-    let ret = expander.fold_crate(c);
+    let mut ret = expander.fold_crate(c);
+    ret.exported_macros = expander.cx.exported_macros.clone();
     parse_sess.span_diagnostic.handler().abort_if_errors();
     return ret;
 }
@@ -1145,6 +1151,25 @@ fn original_span(cx: &ExtCtxt) -> Gc<codemap::ExpnInfo> {
     return einfo;
 }
 
+/// Check that there are no macro invocations left in the AST:
+pub fn check_for_macros(sess: &parse::ParseSess, krate: &ast::Crate) {
+    visit::walk_crate(&mut MacroExterminator{sess:sess}, krate, ());
+}
+
+/// A visitor that ensures that no macro invocations remain in an AST.
+struct MacroExterminator<'a>{
+    sess: &'a parse::ParseSess
+}
+
+impl<'a> visit::Visitor<()> for MacroExterminator<'a> {
+    fn visit_mac(&mut self, macro: &ast::Mac, _:()) {
+        self.sess.span_diagnostic.span_bug(macro.span,
+                                           "macro exterminator: expected AST \
+                                           with no macro invocations");
+    }
+}
+
+
 #[cfg(test)]
 mod test {
     use super::{pattern_bindings, expand_crate, contains_macro_escape};

src/libsyntax/ext/tt/macro_rules.rs

@@ -13,7 +13,7 @@ use ast::{TTDelim};
 use ast;
 use codemap::{Span, Spanned, DUMMY_SP};
 use ext::base::{ExtCtxt, MacResult, MacroDef};
-use ext::base::{NormalTT, MacroExpander};
+use ext::base::{NormalTT, TTMacroExpander};
 use ext::base;
 use ext::tt::macro_parser::{Success, Error, Failure};
 use ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal};
@@ -95,7 +95,7 @@ struct MacroRulesMacroExpander {
     rhses: Vec<Rc<NamedMatch>>,
 }
 
-impl MacroExpander for MacroRulesMacroExpander {
+impl TTMacroExpander for MacroRulesMacroExpander {
     fn expand(&self,
               cx: &mut ExtCtxt,
               sp: Span,