Properly check enum types for recursion

Closes #742

Author: marijnh

src/comp/middle/ty.rs

@@ -172,6 +172,7 @@ export type_is_str;
 export type_is_unique;
 export type_is_enum;
 export type_is_c_like_enum;
+export type_structurally_contains;
 export type_structurally_contains_uniques;
 export type_autoderef;
 export type_param;

src/comp/middle/typeck.rs

@@ -259,10 +259,9 @@ fn ast_ty_to_ty(tcx: ty::ctxt, mode: mode, &&ast_ty: @ast::ty) -> ty::t {
     alt tcx.ast_ty_to_ty_cache.find(ast_ty) {
       some(some(ty)) { ret ty; }
       some(none) {
-        tcx.sess.span_fatal(ast_ty.span,
-                            "illegal recursive type \
-                              insert a enum in the cycle, \
-                              if this is desired)");
+        tcx.sess.span_fatal(ast_ty.span, "illegal recursive type. \
+                                          insert a enum in the cycle, \
+                                          if this is desired)");
       }
       none { }
     } /* go on */
@@ -2490,7 +2489,7 @@ fn check_const(ccx: @crate_ctxt, _sp: span, e: @ast::expr, id: ast::node_id) {
     demand::simple(fcx, e.span, declty, cty);
 }
 
-fn check_enum_variants(ccx: @crate_ctxt, _sp: span, vs: [ast::variant],
+fn check_enum_variants(ccx: @crate_ctxt, sp: span, vs: [ast::variant],
                       id: ast::node_id) {
     // FIXME: this is kinda a kludge; we manufacture a fake function context
     // and statement context for checking the initializer expression.
@@ -2512,7 +2511,7 @@ fn check_enum_variants(ccx: @crate_ctxt, _sp: span, vs: [ast::variant],
           some(e) {
             check_expr(fcx, e);
             let cty = expr_ty(fcx.ccx.tcx, e);
-            let declty =ty::mk_int(fcx.ccx.tcx);
+            let declty = ty::mk_int(fcx.ccx.tcx);
             demand::simple(fcx, e.span, declty, cty);
             // FIXME: issue #1417
             // Also, check_expr (from check_const pass) doesn't guarantee that
@@ -2537,6 +2536,20 @@ fn check_enum_variants(ccx: @crate_ctxt, _sp: span, vs: [ast::variant],
         disr_vals += [disr_val];
         disr_val += 1;
     }
+    let outer = true, did = local_def(id);
+    if ty::type_structurally_contains(ccx.tcx, rty, {|sty|
+        alt sty {
+          ty::ty_enum(id, _) if id == did {
+            if outer { outer = false; false }
+            else { true }
+          }
+          _ { false }
+        }
+    }) {
+        ccx.tcx.sess.span_fatal(sp, "illegal recursive enum type. \
+                                     wrap the inner value in a box to \
+                                     make it represenable");
+    }
 }
 
 // A generic function for checking the pred in a check

src/test/compile-fail/recursive-enum.rs

@@ -0,0 +1,5 @@
+// error-pattern: illegal recursive enum type
+
+enum list<T> { cons(T, list<T>), nil }
+
+fn main() {}