Localize GADTused variable

Author: odersky

compiler/src/dotty/tools/dotc/core/TypeComparer.scala

@@ -133,6 +133,12 @@ class TypeComparer(using val comparerCtx: Context) extends ConstraintHandling wi
     }
   }
 
+  def testSubType(tp1: Type, tp2: Type): CompareResult =
+    GADTused = false
+    if !topLevelSubType(tp1, tp2) then CompareResult.Fail
+    else if GADTused then CompareResult.OKwithGADTUsed
+    else CompareResult.OK
+
   /** The current approximation state. See `ApproxState`. */
   private var approx: ApproxState = FreshApprox
   protected def approxState: ApproxState = approx
@@ -141,7 +147,7 @@ class TypeComparer(using val comparerCtx: Context) extends ConstraintHandling wi
    *  every time we compare components of the previous pair of types.
    *  This type is used for capture conversion in `isSubArgs`.
    */
-  private [this] var leftRoot: Type = _
+  private [this] var leftRoot: Type = null
 
   /** Are we forbidden from recording GADT constraints? */
   private var frozenGadt = false
@@ -2426,6 +2432,9 @@ class TypeComparer(using val comparerCtx: Context) extends ConstraintHandling wi
 
 object TypeComparer {
 
+  enum CompareResult:
+    case OK, Fail, OKwithGADTUsed
+
   /** Class for unification variables used in `natValue`. */
   private class AnyConstantType extends UncachedGroundType with ValueType {
     var tpe: Type = NoType

compiler/src/dotty/tools/dotc/typer/Typer.scala

@@ -26,6 +26,7 @@ import ErrorReporting._
 import Checking._
 import Inferencing._
 import EtaExpansion.etaExpand
+import TypeComparer.CompareResult
 import util.Spans._
 import util.common._
 import util.Property
@@ -3248,23 +3249,22 @@ class Typer extends Namer
               |To turn this error into a warning, pass -Xignore-scala2-macros to the compiler""".stripMargin, tree.sourcePos.startPos)
           tree
         }
-      else if (tree.tpe.widenExpr <:< pt) {
-        if (ctx.typeComparer.GADTused && pt.isValueType)
+      else ctx.typeComparer.testSubType(tree.tpe.widenExpr, pt) match
+        case CompareResult.Fail =>
+          wtp match
+            case wtp: MethodType => missingArgs(wtp)
+            case _ =>
+              typr.println(i"adapt to subtype ${tree.tpe} !<:< $pt")
+              //typr.println(TypeComparer.explained(tree.tpe <:< pt))
+              adaptToSubType(wtp)
+        case CompareResult.OKwithGADTUsed if pt.isValueType =>
           // Insert an explicit cast, so that -Ycheck in later phases succeeds.
           // I suspect, but am not 100% sure that this might affect inferred types,
           // if the expected type is a supertype of the GADT bound. It would be good to come
           // up with a test case for this.
           tree.cast(pt)
-        else
-          tree
-      }
-      else wtp match {
-        case wtp: MethodType => missingArgs(wtp)
         case _ =>
-          typr.println(i"adapt to subtype ${tree.tpe} !<:< $pt")
-          //typr.println(TypeComparer.explained(tree.tpe <:< pt))
-          adaptToSubType(wtp)
-      }
+          tree
     }
 
     // Follow proxies and approximate type paramrefs by their upper bound