Maintenance: Update to mypy v1.13.0

This patch intends to contribute to unlock upgrading to Python 3.11.

Author: amotl

.pre-commit-config.yaml

@@ -18,7 +18,7 @@ repos:
     hooks:
       - id: isort
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v0.770"
+    rev: "v1.13.0"
     hooks:
       - id: mypy
   - repo: https://github.com/adrienverge/yamllint

crate/operator/config.py

@@ -192,8 +192,6 @@ def load(self):
                 f"Invalid {self._prefix}BOOTSTRAP_TIMEOUT="
                 f"'{bootstrap_timeout}'. Needs to be a positive integer or 0."
             )
-        if self.BOOTSTRAP_TIMEOUT == 0:
-            self.BOOTSTRAP_TIMEOUT = None
 
         bootstrap_delay = self.env(
             "BOOTSTRAP_RETRY_DELAY", default=str(self.BOOTSTRAP_RETRY_DELAY)

crate/operator/cratedb.py

@@ -77,7 +77,7 @@ async def create_user(
     name: str,
     username: str,
     password: str,
-    privileges: List[str] = None,
+    privileges: Optional[List[str]] = None,
 ) -> None:
     """
     Create user ``username`` and grant it given privileges.

crate/operator/handlers/handle_create_cratedb.py

@@ -56,6 +56,10 @@ async def create_cratedb(
     logger: logging.Logger,
 ):
     context = status.get(CLUSTER_CREATE_ID)
+    if context is None:
+        raise RuntimeError(
+            f"Operation context is empty or unknown: {CLUSTER_CREATE_ID}"
+        )
     hash = hashlib.md5(str(spec).encode("utf-8")).hexdigest()
     name = meta["name"]
     base_labels = {

crate/operator/operations.py

@@ -1059,7 +1059,7 @@ async def _ensure_no_backup_cronjobs_running(
                 ):
                     await kopf.execute(
                         fns={
-                            "notify_backup_running": subhandler_partial(
+                            "notify_backup_running": subhandler_partial(  # type: ignore[dict-item]  # noqa: E501
                                 self._notify_backup_running, logger
                             )
                         }

crate/operator/prometheus.py

@@ -19,6 +19,8 @@
 # with Crate these terms will supersede the license and you may use the
 # software solely pursuant to the terms of the relevant commercial agreement.
 
+# mypy: disable-error-code="arg-type, attr-defined, list-item, operator"
+
 import enum
 import time
 from datetime import datetime

crate/operator/scale.py

@@ -515,8 +515,8 @@ async def scale_cluster(
                 total_number_of_nodes = (
                     total_number_of_nodes + new_replicas - old_replicas
                 )
-                index, *_ = field_path
-                index = int(index)
+                index_raw, *_ = field_path
+                index = int(index_raw)
                 node_spec = spec["nodes"]["data"][index]
                 node_name = node_spec["name"]
                 sts_name = f"crate-data-{node_name}-{name}"
@@ -559,8 +559,8 @@ async def scale_cluster(
                 total_number_of_nodes = (
                     total_number_of_nodes + new_replicas - old_replicas
                 )
-                index, *_ = field_path
-                index = int(index)
+                index_raw, *_ = field_path
+                index = int(index_raw)
                 node_spec = spec["nodes"]["data"][index]
                 node_name = node_spec["name"]
                 sts_name = f"crate-data-{node_name}-{name}"

crate/operator/utils/kubeapi.py

@@ -44,8 +44,8 @@ async def call_kubeapi(
     *,
     continue_on_absence=False,
     continue_on_conflict=False,
-    namespace: str = None,
-    body: K8sModel = None,
+    namespace: Optional[str] = None,
+    body: Optional[K8sModel] = None,
     **kwargs,
 ) -> Optional[Awaitable[K8sModel]]:
     """

crate/operator/utils/version.py

@@ -21,7 +21,7 @@
 
 import re
 from distutils.version import Version
-from typing import Optional, Tuple
+from typing import Optional, Tuple, Union, cast
 
 
 class CrateVersion(Version):
@@ -63,9 +63,9 @@ class CrateVersion(Version):
 
     version: Optional[Tuple[int, int, int]]
 
-    major: int
-    minor: int
-    hotfix: int
+    major: Union[int, None]
+    minor: Union[int, None]
+    hotfix: Union[int, None]
 
     stable: bool
     snapshot: bool
@@ -130,7 +130,7 @@ def __repr__(self):
 
     def __str__(self):
         if self.stable:
-            return ".".join(map(str, self.version))
+            return ".".join(map(str, cast(list, self.version)))
 
         if self.nightly:
             vstring = "nightly"
@@ -250,14 +250,15 @@ def _parse_regular(self, vstring):
         )
 
         if hotfix:
-            self.version = tuple(map(int, [major, minor, hotfix]))
+            self.version = tuple(map(int, [major, minor, hotfix]))  # type: ignore[assignment]  # noqa: E501
         else:
-            self.version = tuple(map(int, [major, minor])) + (0,)
+            self.version = tuple(map(int, [major, minor])) + (0,)  # type: ignore[assignment]  # noqa: E501
 
         self.stable = False if snapshot else True
         self.nightly = False
         self.snapshot = True if snapshot else False
-        self.major, self.minor, self.hotfix = self.version
+        if self.version is not None:
+            self.major, self.minor, self.hotfix = self.version
 
     def _parse_nightly(self, vstring):
         match = self.nightly_version_re.match(vstring)
@@ -270,7 +271,7 @@ def _parse_nightly(self, vstring):
 
         if has_version:
             self.major, self.minor, self.hotfix = int(major), int(minor), int(hotfix)
-            self.version = tuple(map(int, [self.major, self.minor, self.hotfix]))
+            self.version = tuple(map(int, [self.major, self.minor, self.hotfix]))  # type: ignore[assignment]  # noqa: E501
         else:
             self.major, self.minor, self.hotfix = None, None, None
             self.version = None

docs/source/crate_operator_ext.py

@@ -47,7 +47,7 @@ def run_apidoc(_):
 
 def missing_reference(
     app: Sphinx, env: BuildEnvironment, node: pending_xref, contnode: Element
-) -> Element:
+) -> None:
     """
     Remove or resolve references to third party packages.
 

pyproject.toml

@@ -5,3 +5,21 @@ target-version = ['py38']
 requires = ["setuptools>=58", "wheel", "setuptools_scm>=6.2"]
 
 [tool.setuptools_scm]
+
+[tool.mypy]
+packages = ["crate", "tests"]
+check_untyped_defs = true
+explicit_package_bases = true
+ignore_missing_imports = true
+implicit_optional = false
+install_types = true
+namespace_packages = true
+non_interactive = true
+show_error_codes = true
+strict_equality = true
+warn_unused_ignores = false
+warn_redundant_casts = false
+
+[[tool.mypy.overrides]]
+module = "crate.operator.main"
+ignore_errors = true

setup.cfg

@@ -1,10 +1,3 @@
-[bdist_wheel]
-universal = 1
-
-[mypy]
-ignore_missing_imports = true
-namespace_packages = true
-
 [flake8]
 max-line-length = 88
 ignore = E203 W503

setup.py

@@ -76,7 +76,7 @@ def read(path: str) -> str:
             "black==22.3.0",
             "flake8==3.8.4",
             "isort==5.12.0",
-            "mypy==0.770",
+            "mypy==1.13.0",
         ],
     },
     python_requires=">=3.8",

tests/conftest.py

@@ -19,6 +19,9 @@
 # with Crate these terms will supersede the license and you may use the
 # software solely pursuant to the terms of the relevant commercial agreement.
 
+# Sphinx configuration file does not need type checking.
+# type: ignore
+
 import asyncio
 import os
 import pathlib

tests/test_create.py

@@ -123,6 +123,8 @@ def test_testing_false(self, faker):
         with mock.patch("crate.operator.create.config.TESTING", False):
             affinity = get_statefulset_affinity(name, logging.getLogger(__name__), {})
 
+        assert affinity, "`affinity` is None or empty"
+
         apa = affinity.pod_anti_affinity
         terms = apa.required_during_scheduling_ignored_during_execution[0]
         expressions = terms.label_selector.match_expressions
@@ -168,6 +170,8 @@ def test_dedicated_resources_affinity(self, node_spec, faker):
                 name, logging.getLogger(__name__), node_spec
             )
 
+        assert affinity, "`affinity` is None or empty"
+
         apa = affinity.pod_anti_affinity
         terms = apa.required_during_scheduling_ignored_during_execution[0]
         expressions = terms.label_selector.match_expressions
@@ -202,6 +206,8 @@ def test_shared_resources_affinity(self, node_spec, faker):
                 name, logging.getLogger(__name__), node_spec
             )
 
+        assert affinity, "`affinity` is None or empty"
+
         na = affinity.node_affinity
         selector = na.required_during_scheduling_ignored_during_execution
         terms = selector.node_selector_terms[0]
@@ -223,6 +229,8 @@ def test_cloud_provider(self, provider, faker):
             ):
                 topospread = get_topology_spread(name, logging.getLogger(__name__))
 
+        assert topospread, "`topospread` is None or empty"
+
         terms = topospread[0]
         expressions = terms.label_selector.match_expressions
         assert [e.to_dict() for e in expressions] == [
@@ -274,6 +282,8 @@ def test_dedicated_resources_tolerations(self, node_spec, faker):
         with mock.patch("crate.operator.create.config.TESTING", False):
             tolerations = get_tolerations(name, logging.getLogger(__name__), node_spec)
 
+        assert tolerations, "`tolerations` is None or empty"
+
         assert len(tolerations) == 1
         assert tolerations[0].to_dict() == {
             "effect": "NoSchedule",
@@ -302,6 +312,8 @@ def test_shared_resources_tolerations(self, node_spec, faker):
         with mock.patch("crate.operator.create.config.TESTING", False):
             tolerations = get_tolerations(name, logging.getLogger(__name__), node_spec)
 
+        assert tolerations, "`tolerations` is None or empty"
+
         toleration = tolerations[0]
         expected = {
             "key": "cratedb",
@@ -1009,7 +1021,7 @@ def test_get_data_service(self, provider, dns, faker):
         http = faker.port_number()
         psql = faker.port_number()
         with mock.patch("crate.operator.create.config.CLOUD_PROVIDER", provider):
-            service = get_data_service(None, name, None, http, psql, dns)
+            service = get_data_service(None, name, {}, http, psql, dns)
         annotation_keys = service.metadata.annotations.keys()
         if provider == "aws":
             assert (
@@ -1442,7 +1454,7 @@ async def test_preserve_unknown_object_keys(
 
 
 def test_sql_exporter_config():
-    result = get_sql_exporter_config(None, "test-name", None)
+    result = get_sql_exporter_config(None, "test-name", {})
     assert result.metadata.name == "crate-sql-exporter-test-name"
 
     _, _, filenames = next(walk("crate/operator/data"))

tests/test_metrics.py

@@ -19,6 +19,8 @@
 # with Crate these terms will supersede the license and you may use the
 # software solely pursuant to the terms of the relevant commercial agreement.
 
+# mypy: disable-error-code="attr-defined, arg-type, union-attr"
+
 import logging
 import time
 

tests/test_update_allowed_cidrs.py

@@ -193,6 +193,8 @@ async def _are_source_ranges_updated(core, name, namespace, cidr_list):
     ingress = await read_grand_central_ingress(namespace=namespace, name=name)
     actual = cidr_list if len(cidr_list) > 0 else None
 
+    assert ingress, "`ingress` is None"
+
     return (
         service.spec.load_balancer_source_ranges == actual
         and ingress.metadata.annotations.get(

tests/test_webhooks.py

@@ -48,8 +48,8 @@ def test_payload_serialization_scale():
         namespace="some-namespace",
         cluster="some-cluster",
         scale_data=WebhookScalePayload(
-            old_data_replicas={"a": 1},
-            new_data_replicas={"a": 2},
+            old_data_replicas=[{"name": "a", "replicas": "1"}],
+            new_data_replicas=[{"name": "a", "replicas": "2"}],
             old_master_replicas=3,
             new_master_replicas=4,
         ),
@@ -68,8 +68,8 @@ def test_payload_serialization_scale():
         "namespace": "some-namespace",
         "cluster": "some-cluster",
         "scale_data": {
-            "old_data_replicas": {"a": 1},
-            "new_data_replicas": {"a": 2},
+            "old_data_replicas": [{"name": "a", "replicas": "1"}],
+            "new_data_replicas": [{"name": "a", "replicas": "2"}],
             "old_master_replicas": 3,
             "new_master_replicas": 4,
         },
@@ -136,8 +136,11 @@ async def test_configure():
     assert c._url == "http://localhost:1234/some/path"
     assert c._session._default_headers["Content-Type"] == "application/json"
     assert c._session._default_headers["User-Agent"].startswith("cratedb-operator/")
-    assert c._session._default_auth.login == "itsme"
-    assert c._session._default_auth.password == "secr3t password"
+    assert c._session._default_auth and c._session._default_auth.login == "itsme"
+    assert (
+        c._session._default_auth
+        and c._session._default_auth.password == "secr3t password"
+    )
 
 
 @pytest.mark.asyncio
@@ -173,8 +176,8 @@ async def test_send_scale_notification(self):
             "my-cluster",
             WebhookEvent.SCALE,
             WebhookScalePayload(
-                old_data_replicas={"a": 1},
-                new_data_replicas={"a": 2},
+                old_data_replicas=[{"name": "a", "replicas": "1"}],
+                new_data_replicas=[{"name": "a", "replicas": "2"}],
                 old_master_replicas=3,
                 new_master_replicas=4,
             ),
@@ -192,8 +195,8 @@ async def test_send_scale_notification(self):
                 "namespace": "my-namespace",
                 "cluster": "my-cluster",
                 "scale_data": {
-                    "old_data_replicas": {"a": 1},
-                    "new_data_replicas": {"a": 2},
+                    "old_data_replicas": [{"name": "a", "replicas": "1"}],
+                    "new_data_replicas": [{"name": "a", "replicas": "2"}],
                     "old_master_replicas": 3,
                     "new_master_replicas": 4,
                 },

tests/utils.py

@@ -23,7 +23,7 @@
 import logging
 import os
 from functools import reduce
-from typing import Any, Callable, List, Mapping, Optional, Set, Tuple, Union
+from typing import Any, Callable, Dict, List, Mapping, Optional, Set, Tuple
 from unittest import mock
 
 import psycopg2
@@ -439,7 +439,7 @@ async def cluster_routing_allocation_enable_equals(
 
 
 async def was_notification_sent(
-    mock_send_notification: mock.AsyncMock, call: mock.call
+    mock_send_notification: mock.AsyncMock, call: mock._Call
 ):
     if mock_send_notification.call_count == 0:
         return False
@@ -459,7 +459,7 @@ async def is_cronjob_schedule_matching(
 
 
 async def mocked_coro_func_called_with(
-    mocked_coro_func: mock.AsyncMock, call: mock.call
+    mocked_coro_func: mock.AsyncMock, call: mock._Call
 ) -> bool:
     if mocked_coro_func.call_count == 0:
         return False
@@ -474,7 +474,7 @@ async def mocked_coro_func_called_with(
 async def cluster_setting_equals(
     conn_factory: Callable[[], Connection],
     setting: str,
-    expected_value: Union[str, int],
+    expected_value: Dict[Any, Any],
 ) -> bool:
     try:
         async with conn_factory() as conn: