[PEP 695] Partial support for new type parameter syntax in Python 3.12 (

#17233)

Add basic support for most features of PEP 695. It's still not generally
useful, but it should be enough for experimentation and testing. I will 
continue working on follow-up PRs after this has been merged.

This is currently behind a feature flag:
`--enable-incomplete-feature=NewGenericSyntax`

These features, among other things, are unimplemented (or at least
untested):
* Recursive type aliases
* Checking for various errors
* Inference of variance in complex cases
* Dealing with unknown variance consistently
* Scoping
* Mypy daemon
* Compilation using mypyc

The trickiest remaining thing is probably variance inference in cases
where some types aren't ready (i.e. not inferred) when we need 
variance. I have some ideas about how to tackle this, but it might 
need significant work. Currently the idea is to infer variance
on demand when we need it, but we may need to defer if variance can't 
be calculated, for example if a type of an attribute is not yet ready. 
The current approach is to fall back to covariance in some cases, 
which is not ideal.

Work on #15238.

mypy/checker.py

@@ -146,6 +146,7 @@
 from mypy.state import state
 from mypy.subtypes import (
     find_member,
+    infer_class_variances,
     is_callable_compatible,
     is_equivalent,
     is_more_precise,
@@ -2374,7 +2375,7 @@ def visit_class_def(self, defn: ClassDef) -> None:
                     self.allow_abstract_call = old_allow_abstract_call
                 # TODO: Apply the sig to the actual TypeInfo so we can handle decorators
                 # that completely swap out the type.  (e.g. Callable[[Type[A]], Type[B]])
-        if typ.defn.type_vars:
+        if typ.defn.type_vars and typ.defn.type_args is None:
             for base_inst in typ.bases:
                 for base_tvar, base_decl_tvar in zip(
                     base_inst.args, base_inst.type.defn.type_vars
@@ -2396,6 +2397,7 @@ def visit_class_def(self, defn: ClassDef) -> None:
             self.check_protocol_variance(defn)
         if not defn.has_incompatible_baseclass and defn.info.is_enum:
             self.check_enum(defn)
+        infer_class_variances(defn.info)
 
     def check_final_deletable(self, typ: TypeInfo) -> None:
         # These checks are only for mypyc. Only perform some checks that are easier
@@ -2566,6 +2568,9 @@ def check_protocol_variance(self, defn: ClassDef) -> None:
         if they are actually covariant/contravariant, since this may break
         transitivity of subtyping, see PEP 544.
         """
+        if defn.type_args is not None:
+            # Using new-style syntax (PEP 695), so variance will be inferred
+            return
         info = defn.info
         object_type = Instance(info.mro[-1], [])
         tvars = info.defn.type_vars
@@ -3412,8 +3417,8 @@ def check_final(self, s: AssignmentStmt | OperatorAssignmentStmt | AssignmentExp
                 if (
                     lv.node.final_unset_in_class
                     and not lv.node.final_set_in_init
-                    and not self.is_stub
-                    and  # It is OK to skip initializer in stub files.
+                    and not self.is_stub  # It is OK to skip initializer in stub files.
+                    and
                     # Avoid extra error messages, if there is no type in Final[...],
                     # then we already reported the error about missing r.h.s.
                     isinstance(s, AssignmentStmt)

mypy/fastparse.py

@@ -17,6 +17,9 @@
     ARG_POS,
     ARG_STAR,
     ARG_STAR2,
+    PARAM_SPEC_KIND,
+    TYPE_VAR_KIND,
+    TYPE_VAR_TUPLE_KIND,
     ArgKind,
     Argument,
     AssertStmt,
@@ -79,6 +82,8 @@
     TempNode,
     TryStmt,
     TupleExpr,
+    TypeAliasStmt,
+    TypeParam,
     UnaryExpr,
     Var,
     WhileStmt,
@@ -87,7 +92,7 @@
     YieldFromExpr,
     check_arg_names,
 )
-from mypy.options import Options
+from mypy.options import NEW_GENERIC_SYNTAX, Options
 from mypy.patterns import (
     AsPattern,
     ClassPattern,
@@ -144,11 +149,6 @@ def ast3_parse(
 NamedExpr = ast3.NamedExpr
 Constant = ast3.Constant
 
-if sys.version_info >= (3, 12):
-    ast_TypeAlias = ast3.TypeAlias
-else:
-    ast_TypeAlias = Any
-
 if sys.version_info >= (3, 10):
     Match = ast3.Match
     MatchValue = ast3.MatchValue
@@ -171,11 +171,21 @@ def ast3_parse(
     MatchAs = Any
     MatchOr = Any
     AstNode = Union[ast3.expr, ast3.stmt, ast3.ExceptHandler]
+
 if sys.version_info >= (3, 11):
     TryStar = ast3.TryStar
 else:
     TryStar = Any
 
+if sys.version_info >= (3, 12):
+    ast_TypeAlias = ast3.TypeAlias
+    ast_ParamSpec = ast3.ParamSpec
+    ast_TypeVarTuple = ast3.TypeVarTuple
+else:
+    ast_TypeAlias = Any
+    ast_ParamSpec = Any
+    ast_TypeVarTuple = Any
+
 N = TypeVar("N", bound=Node)
 
 # There is no way to create reasonable fallbacks at this stage,
@@ -884,6 +894,8 @@ def do_func_def(
 
         arg_kinds = [arg.kind for arg in args]
         arg_names = [None if arg.pos_only else arg.variable.name for arg in args]
+        # Type parameters, if using new syntax for generics (PEP 695)
+        explicit_type_params: list[TypeParam] | None = None
 
         arg_types: list[Type | None] = []
         if no_type_check:
@@ -937,12 +949,17 @@ def do_func_def(
                 return_type = AnyType(TypeOfAny.from_error)
         else:
             if sys.version_info >= (3, 12) and n.type_params:
-                self.fail(
-                    ErrorMessage("PEP 695 generics are not yet supported", code=codes.VALID_TYPE),
-                    n.type_params[0].lineno,
-                    n.type_params[0].col_offset,
-                    blocker=False,
-                )
+                if NEW_GENERIC_SYNTAX in self.options.enable_incomplete_feature:
+                    explicit_type_params = self.translate_type_params(n.type_params)
+                else:
+                    self.fail(
+                        ErrorMessage(
+                            "PEP 695 generics are not yet supported", code=codes.VALID_TYPE
+                        ),
+                        n.type_params[0].lineno,
+                        n.type_params[0].col_offset,
+                        blocker=False,
+                    )
 
             arg_types = [a.type_annotation for a in args]
             return_type = TypeConverter(
@@ -986,7 +1003,7 @@ def do_func_def(
         self.class_and_function_stack.pop()
         self.class_and_function_stack.append("F")
         body = self.as_required_block(n.body, can_strip=True, is_coroutine=is_coroutine)
-        func_def = FuncDef(n.name, args, body, func_type)
+        func_def = FuncDef(n.name, args, body, func_type, explicit_type_params)
         if isinstance(func_def.type, CallableType):
             # semanal.py does some in-place modifications we want to avoid
             func_def.unanalyzed_type = func_def.type.copy_modified()
@@ -1120,13 +1137,19 @@ def visit_ClassDef(self, n: ast3.ClassDef) -> ClassDef:
         self.class_and_function_stack.append("C")
         keywords = [(kw.arg, self.visit(kw.value)) for kw in n.keywords if kw.arg]
 
+        # Type parameters, if using new syntax for generics (PEP 695)
+        explicit_type_params: list[TypeParam] | None = None
+
         if sys.version_info >= (3, 12) and n.type_params:
-            self.fail(
-                ErrorMessage("PEP 695 generics are not yet supported", code=codes.VALID_TYPE),
-                n.type_params[0].lineno,
-                n.type_params[0].col_offset,
-                blocker=False,
-            )
+            if NEW_GENERIC_SYNTAX in self.options.enable_incomplete_feature:
+                explicit_type_params = self.translate_type_params(n.type_params)
+            else:
+                self.fail(
+                    ErrorMessage("PEP 695 generics are not yet supported", code=codes.VALID_TYPE),
+                    n.type_params[0].lineno,
+                    n.type_params[0].col_offset,
+                    blocker=False,
+                )
 
         cdef = ClassDef(
             n.name,
@@ -1135,6 +1158,7 @@ def visit_ClassDef(self, n: ast3.ClassDef) -> ClassDef:
             self.translate_expr_list(n.bases),
             metaclass=dict(keywords).get("metaclass"),
             keywords=keywords,
+            type_args=explicit_type_params,
         )
         cdef.decorators = self.translate_expr_list(n.decorator_list)
         # Set lines to match the old mypy 0.700 lines, in order to keep
@@ -1150,6 +1174,24 @@ def visit_ClassDef(self, n: ast3.ClassDef) -> ClassDef:
         self.class_and_function_stack.pop()
         return cdef
 
+    def translate_type_params(self, type_params: list[Any]) -> list[TypeParam]:
+        explicit_type_params = []
+        for p in type_params:
+            bound = None
+            values: list[Type] = []
+            if isinstance(p, ast_ParamSpec):  # type: ignore[misc]
+                explicit_type_params.append(TypeParam(p.name, PARAM_SPEC_KIND, None, []))
+            elif isinstance(p, ast_TypeVarTuple):  # type: ignore[misc]
+                explicit_type_params.append(TypeParam(p.name, TYPE_VAR_TUPLE_KIND, None, []))
+            else:
+                if isinstance(p.bound, ast3.Tuple):
+                    conv = TypeConverter(self.errors, line=p.lineno)
+                    values = [conv.visit(t) for t in p.bound.elts]
+                elif p.bound is not None:
+                    bound = TypeConverter(self.errors, line=p.lineno).visit(p.bound)
+                explicit_type_params.append(TypeParam(p.name, TYPE_VAR_KIND, bound, values))
+        return explicit_type_params
+
     # Return(expr? value)
     def visit_Return(self, n: ast3.Return) -> ReturnStmt:
         node = ReturnStmt(self.visit(n.value))
@@ -1735,15 +1777,23 @@ def visit_MatchOr(self, n: MatchOr) -> OrPattern:
         node = OrPattern([self.visit(pattern) for pattern in n.patterns])
         return self.set_line(node, n)
 
-    def visit_TypeAlias(self, n: ast_TypeAlias) -> AssignmentStmt:
-        self.fail(
-            ErrorMessage("PEP 695 type aliases are not yet supported", code=codes.VALID_TYPE),
-            n.lineno,
-            n.col_offset,
-            blocker=False,
-        )
-        node = AssignmentStmt([NameExpr(n.name.id)], self.visit(n.value))
-        return self.set_line(node, n)
+    # TypeAlias(identifier name, type_param* type_params, expr value)
+    def visit_TypeAlias(self, n: ast_TypeAlias) -> TypeAliasStmt | AssignmentStmt:
+        node: TypeAliasStmt | AssignmentStmt
+        if NEW_GENERIC_SYNTAX in self.options.enable_incomplete_feature:
+            type_params = self.translate_type_params(n.type_params)
+            value = self.visit(n.value)
+            node = TypeAliasStmt(self.visit_Name(n.name), type_params, value)
+            return self.set_line(node, n)
+        else:
+            self.fail(
+                ErrorMessage("PEP 695 type aliases are not yet supported", code=codes.VALID_TYPE),
+                n.lineno,
+                n.col_offset,
+                blocker=False,
+            )
+            node = AssignmentStmt([NameExpr(n.name.id)], self.visit(n.value))
+            return self.set_line(node, n)
 
 
 class TypeConverter:

mypy/join.py

@@ -6,7 +6,7 @@
 
 import mypy.typeops
 from mypy.maptype import map_instance_to_supertype
-from mypy.nodes import CONTRAVARIANT, COVARIANT, INVARIANT
+from mypy.nodes import CONTRAVARIANT, COVARIANT, INVARIANT, VARIANCE_NOT_READY
 from mypy.state import state
 from mypy.subtypes import (
     SubtypeContext,
@@ -97,7 +97,7 @@ def join_instances(self, t: Instance, s: Instance) -> ProperType:
                 elif isinstance(sa_proper, AnyType):
                     new_type = AnyType(TypeOfAny.from_another_any, sa_proper)
                 elif isinstance(type_var, TypeVarType):
-                    if type_var.variance == COVARIANT:
+                    if type_var.variance in (COVARIANT, VARIANCE_NOT_READY):
                         new_type = join_types(ta, sa, self)
                         if len(type_var.values) != 0 and new_type not in type_var.values:
                             self.seen_instances.pop()

mypy/nodes.py

@@ -653,6 +653,28 @@ def set_line(
         self.variable.set_line(self.line, self.column, self.end_line, self.end_column)
 
 
+# These specify the kind of a TypeParam
+TYPE_VAR_KIND: Final = 0
+PARAM_SPEC_KIND: Final = 1
+TYPE_VAR_TUPLE_KIND: Final = 2
+
+
+class TypeParam:
+    __slots__ = ("name", "kind", "upper_bound", "values")
+
+    def __init__(
+        self,
+        name: str,
+        kind: int,
+        upper_bound: mypy.types.Type | None,
+        values: list[mypy.types.Type],
+    ) -> None:
+        self.name = name
+        self.kind = kind
+        self.upper_bound = upper_bound
+        self.values = values
+
+
 FUNCITEM_FLAGS: Final = FUNCBASE_FLAGS + [
     "is_overload",
     "is_generator",
@@ -672,6 +694,7 @@ class FuncItem(FuncBase):
         "min_args",  # Minimum number of arguments
         "max_pos",  # Maximum number of positional arguments, -1 if no explicit
         # limit (*args not included)
+        "type_args",  # New-style type parameters (PEP 695)
         "body",  # Body of the function
         "is_overload",  # Is this an overload variant of function with more than
         # one overload variant?
@@ -689,12 +712,14 @@ def __init__(
         arguments: list[Argument] | None = None,
         body: Block | None = None,
         typ: mypy.types.FunctionLike | None = None,
+        type_args: list[TypeParam] | None = None,
     ) -> None:
         super().__init__()
         self.arguments = arguments or []
         self.arg_names = [None if arg.pos_only else arg.variable.name for arg in self.arguments]
         self.arg_kinds: list[ArgKind] = [arg.kind for arg in self.arguments]
         self.max_pos: int = self.arg_kinds.count(ARG_POS) + self.arg_kinds.count(ARG_OPT)
+        self.type_args: list[TypeParam] | None = type_args
         self.body: Block = body or Block([])
         self.type = typ
         self.unanalyzed_type = typ
@@ -761,8 +786,9 @@ def __init__(
         arguments: list[Argument] | None = None,
         body: Block | None = None,
         typ: mypy.types.FunctionLike | None = None,
+        type_args: list[TypeParam] | None = None,
     ) -> None:
-        super().__init__(arguments, body, typ)
+        super().__init__(arguments, body, typ, type_args)
         self._name = name
         self.is_decorated = False
         self.is_conditional = False  # Defined conditionally (within block)?
@@ -1070,6 +1096,7 @@ class ClassDef(Statement):
         "name",
         "_fullname",
         "defs",
+        "type_args",
         "type_vars",
         "base_type_exprs",
         "removed_base_type_exprs",
@@ -1089,6 +1116,9 @@ class ClassDef(Statement):
     name: str  # Name of the class without module prefix
     _fullname: str  # Fully qualified name of the class
     defs: Block
+    # New-style type parameters (PEP 695), unanalyzed
+    type_args: list[TypeParam] | None
+    # Semantically analyzed type parameters (all syntax variants)
     type_vars: list[mypy.types.TypeVarLikeType]
     # Base class expressions (not semantically analyzed -- can be arbitrary expressions)
     base_type_exprs: list[Expression]
@@ -1111,12 +1141,14 @@ def __init__(
         base_type_exprs: list[Expression] | None = None,
         metaclass: Expression | None = None,
         keywords: list[tuple[str, Expression]] | None = None,
+        type_args: list[TypeParam] | None = None,
     ) -> None:
         super().__init__()
         self.name = name
         self._fullname = ""
         self.defs = defs
         self.type_vars = type_vars or []
+        self.type_args = type_args
         self.base_type_exprs = base_type_exprs or []
         self.removed_base_type_exprs = []
         self.info = CLASSDEF_NO_INFO
@@ -1607,6 +1639,25 @@ def accept(self, visitor: StatementVisitor[T]) -> T:
         return visitor.visit_match_stmt(self)
 
 
+class TypeAliasStmt(Statement):
+    __slots__ = ("name", "type_args", "value")
+
+    __match_args__ = ("name", "type_args", "value")
+
+    name: NameExpr
+    type_args: list[TypeParam]
+    value: Expression  # Will get translated into a type
+
+    def __init__(self, name: NameExpr, type_args: list[TypeParam], value: Expression) -> None:
+        super().__init__()
+        self.name = name
+        self.type_args = type_args
+        self.value = value
+
+    def accept(self, visitor: StatementVisitor[T]) -> T:
+        return visitor.visit_type_alias_stmt(self)
+
+
 # Expressions
 
 
@@ -2442,6 +2493,7 @@ def accept(self, visitor: ExpressionVisitor[T]) -> T:
 INVARIANT: Final = 0
 COVARIANT: Final = 1
 CONTRAVARIANT: Final = 2
+VARIANCE_NOT_READY: Final = 3  # Variance hasn't been inferred (using Python 3.12 syntax)
 
 
 class TypeVarLikeExpr(SymbolNode, Expression):

mypy/options.py

@@ -73,7 +73,8 @@ class BuildType:
 TYPE_VAR_TUPLE: Final = "TypeVarTuple"
 UNPACK: Final = "Unpack"
 PRECISE_TUPLE_TYPES: Final = "PreciseTupleTypes"
-INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES,))
+NEW_GENERIC_SYNTAX: Final = "NewGenericSyntax"
+INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES, NEW_GENERIC_SYNTAX))
 COMPLETE_FEATURES: Final = frozenset((TYPE_VAR_TUPLE, UNPACK))