[Part-1] Refactor k8s related code into a separate module

This commit is first of the chain in refactoring the cluster/
package.

Author: varshaprasad96

.gitignore

@@ -12,3 +12,4 @@ node_modules
 .DS_Store
 ui-tests/playwright-report
 ui-tests/test-results
+/src/codeflare_sdk.egg-info/

poetry.lock

@@ -7,17 +7,19 @@ src/codeflare_sdk.egg-info/SOURCES.txt
 src/codeflare_sdk.egg-info/dependency_links.txt
 src/codeflare_sdk.egg-info/top_level.txt
 src/codeflare_sdk/cluster/__init__.py
-src/codeflare_sdk/cluster/auth.py
 src/codeflare_sdk/cluster/awload.py
 src/codeflare_sdk/cluster/cluster.py
 src/codeflare_sdk/cluster/config.py
 src/codeflare_sdk/cluster/model.py
 src/codeflare_sdk/cluster/widgets.py
+src/codeflare_sdk/common/__init__.py
+src/codeflare_sdk/common/kubernetes_cluster/__init__.py
+src/codeflare_sdk/common/kubernetes_cluster/auth.py
+src/codeflare_sdk/common/kubernetes_cluster/kube_api_helpers.py
 src/codeflare_sdk/job/__init__.py
 src/codeflare_sdk/job/ray_jobs.py
 src/codeflare_sdk/utils/__init__.py
 src/codeflare_sdk/utils/demos.py
 src/codeflare_sdk/utils/generate_cert.py
 src/codeflare_sdk/utils/generate_yaml.py
-src/codeflare_sdk/utils/kube_api_helpers.py
 src/codeflare_sdk/utils/pretty_print.py

src/codeflare_sdk.egg-info/SOURCES.txt

@@ -1,8 +1,4 @@
 from .cluster import (
-    Authentication,
-    KubeConfiguration,
-    TokenAuthentication,
-    KubeConfigFileAuthentication,
     AWManager,
     Cluster,
     ClusterConfiguration,
@@ -17,6 +13,13 @@
     view_clusters,
 )
 
+from .common import (
+    Authentication,
+    KubeConfiguration,
+    TokenAuthentication,
+    KubeConfigFileAuthentication,
+)
+
 from .job import RayJobClient
 
 from .utils import generate_cert

src/codeflare_sdk/__init__.py

@@ -1,10 +1,3 @@
-from .auth import (
-    Authentication,
-    KubeConfiguration,
-    TokenAuthentication,
-    KubeConfigFileAuthentication,
-)
-
 from .model import (
     RayClusterStatus,
     AppWrapperStatus,

src/codeflare_sdk/cluster/__init__.py

@@ -22,9 +22,12 @@
 import os
 import yaml
 
-from kubernetes import client, config
-from ..utils.kube_api_helpers import _kube_api_error_handling
-from .auth import config_check, get_api_client
+from kubernetes import client
+from ..common import _kube_api_error_handling
+from ..common.kubernetes_cluster.auth import (
+    config_check,
+    get_api_client,
+)
 
 
 class AWManager:

src/codeflare_sdk/cluster/awload.py

@@ -18,21 +18,21 @@
 cluster setup queue, a list of all existing clusters, and the user's working namespace.
 """
 
-import re
-import subprocess
 from time import sleep
 from typing import List, Optional, Tuple, Dict
 
-from kubernetes import config
 from ray.job_submission import JobSubmissionClient
 
-from .auth import config_check, get_api_client
+from ..common.kubernetes_cluster.auth import (
+    config_check,
+    get_api_client,
+)
 from ..utils import pretty_print
 from ..utils.generate_yaml import (
     generate_appwrapper,
     head_worker_gpu_count_from_cluster,
 )
-from ..utils.kube_api_helpers import _kube_api_error_handling
+from ..common import _kube_api_error_handling
 from ..utils.generate_yaml import is_openshift_cluster
 
 from .config import ClusterConfiguration
@@ -47,8 +47,7 @@
     cluster_up_down_buttons,
     is_notebook,
 )
-from kubernetes import client, config
-from kubernetes.utils import parse_quantity
+from kubernetes import client
 import yaml
 import os
 import requests

src/codeflare_sdk/cluster/cluster.py

@@ -28,8 +28,11 @@
 import pandas as pd
 from .config import ClusterConfiguration
 from .model import RayClusterStatus
-from ..utils.kube_api_helpers import _kube_api_error_handling
-from .auth import config_check, get_api_client
+from ..common import _kube_api_error_handling
+from ..common.kubernetes_cluster.auth import (
+    config_check,
+    get_api_client,
+)
 
 
 def cluster_up_down_buttons(cluster: "codeflare_sdk.cluster.Cluster") -> widgets.Button:

src/codeflare_sdk/cluster/widgets.py

@@ -0,0 +1,8 @@
+# Importing everything from the kubernetes_cluster module
+from .kubernetes_cluster import (
+    Authentication,
+    KubeConfiguration,
+    TokenAuthentication,
+    KubeConfigFileAuthentication,
+    _kube_api_error_handling,
+)

src/codeflare_sdk/common/__init__.py

@@ -0,0 +1,10 @@
+from .auth import (
+    Authentication,
+    KubeConfiguration,
+    TokenAuthentication,
+    KubeConfigFileAuthentication,
+    config_check,
+    get_api_client,
+)
+
+from .kube_api_helpers import _kube_api_error_handling

src/codeflare_sdk/common/kubernetes_cluster/__init__.py

@@ -23,7 +23,7 @@
 from kubernetes import client, config
 import os
 import urllib3
-from ..utils.kube_api_helpers import _kube_api_error_handling
+from .kube_api_helpers import _kube_api_error_handling
 
 from typing import Optional
 

src/codeflare_sdk/cluster/auth.py renamed to src/codeflare_sdk/common/kubernetes_cluster/auth.py

@@ -19,9 +19,12 @@
 from cryptography import x509
 from cryptography.x509.oid import NameOID
 import datetime
-from ..cluster.auth import config_check, get_api_client
-from kubernetes import client, config
-from .kube_api_helpers import _kube_api_error_handling
+from ..common.kubernetes_cluster.auth import (
+    config_check,
+    get_api_client,
+)
+from kubernetes import client
+from ..common import _kube_api_error_handling
 
 
 def generate_ca_cert(days: int = 30):

src/codeflare_sdk/utils/kube_api_helpers.py renamed to src/codeflare_sdk/common/kubernetes_cluster/kube_api_helpers.py

@@ -21,16 +21,14 @@
 from typing import Optional
 import typing
 import yaml
-import sys
 import os
-import argparse
 import uuid
-from kubernetes import client, config
-from .kube_api_helpers import _kube_api_error_handling
-from ..cluster.auth import get_api_client, config_check
-from os import urandom
-from base64 import b64encode
-from urllib3.util import parse_url
+from kubernetes import client
+from ..common import _kube_api_error_handling
+from ..common.kubernetes_cluster.auth import (
+    get_api_client,
+    config_check,
+)
 from kubernetes.client.exceptions import ApiException
 import codeflare_sdk
 

src/codeflare_sdk/utils/generate_cert.py

@@ -4,7 +4,9 @@
 import subprocess
 from kubernetes import client, config
 import kubernetes.client
-from codeflare_sdk.utils.kube_api_helpers import _kube_api_error_handling
+from codeflare_sdk.common.kubernetes_cluster.kube_api_helpers import (
+    _kube_api_error_handling,
+)
 
 
 def get_ray_image():

src/codeflare_sdk/utils/generate_yaml.py

@@ -39,7 +39,7 @@
     _app_wrapper_status,
     _ray_cluster_status,
 )
-from codeflare_sdk.cluster.auth import (
+from codeflare_sdk.common.kubernetes_cluster import (
     TokenAuthentication,
     Authentication,
     KubeConfigFileAuthentication,
@@ -71,7 +71,7 @@
     get_package_and_version,
 )
 
-import codeflare_sdk.utils.kube_api_helpers
+import codeflare_sdk.common.kubernetes_cluster.kube_api_helpers
 from codeflare_sdk.utils.generate_yaml import (
     gen_names,
     is_openshift_cluster,
@@ -198,8 +198,8 @@ def test_token_auth_login_tls(mocker):
 def test_config_check_no_config_file(mocker):
     mocker.patch("os.path.expanduser", return_value="/mock/home/directory")
     mocker.patch("os.path.isfile", return_value=False)
-    mocker.patch("codeflare_sdk.cluster.auth.config_path", None)
-    mocker.patch("codeflare_sdk.cluster.auth.api_client", None)
+    mocker.patch("codeflare_sdk.common.kubernetes_cluster.auth.config_path", None)
+    mocker.patch("codeflare_sdk.common.kubernetes_cluster.auth.api_client", None)
 
     with pytest.raises(PermissionError) as e:
         config_check()
@@ -210,8 +210,8 @@ def test_config_check_with_incluster_config(mocker):
     mocker.patch("os.path.isfile", return_value=False)
     mocker.patch.dict(os.environ, {"KUBERNETES_PORT": "number"})
     mocker.patch("kubernetes.config.load_incluster_config", side_effect=None)
-    mocker.patch("codeflare_sdk.cluster.auth.config_path", None)
-    mocker.patch("codeflare_sdk.cluster.auth.api_client", None)
+    mocker.patch("codeflare_sdk.common.kubernetes_cluster.auth.config_path", None)
+    mocker.patch("codeflare_sdk.common.kubernetes_cluster.auth.api_client", None)
 
     result = config_check()
     assert result == None
@@ -221,16 +221,18 @@ def test_config_check_with_existing_config_file(mocker):
     mocker.patch("os.path.expanduser", return_value="/mock/home/directory")
     mocker.patch("os.path.isfile", return_value=True)
     mocker.patch("kubernetes.config.load_kube_config", side_effect=None)
-    mocker.patch("codeflare_sdk.cluster.auth.config_path", None)
-    mocker.patch("codeflare_sdk.cluster.auth.api_client", None)
+    mocker.patch("codeflare_sdk.common.kubernetes_cluster.auth.config_path", None)
+    mocker.patch("codeflare_sdk.common.kubernetes_cluster.auth.api_client", None)
 
     result = config_check()
     assert result == None
 
 
 def test_config_check_with_config_path_and_no_api_client(mocker):
-    mocker.patch("codeflare_sdk.cluster.auth.config_path", "/mock/config/path")
-    mocker.patch("codeflare_sdk.cluster.auth.api_client", None)
+    mocker.patch(
+        "codeflare_sdk.common.kubernetes_cluster.auth.config_path", "/mock/config/path"
+    )
+    mocker.patch("codeflare_sdk.common.kubernetes_cluster.auth.api_client", None)
     result = config_check()
     assert result == "/mock/config/path"
 
@@ -2170,7 +2172,8 @@ def test_map_to_ray_cluster(mocker):
 
     mock_api_client = mocker.MagicMock(spec=client.ApiClient)
     mocker.patch(
-        "codeflare_sdk.cluster.auth.get_api_client", return_value=mock_api_client
+        "codeflare_sdk.common.kubernetes_cluster.auth.get_api_client",
+        return_value=mock_api_client,
     )
 
     mock_routes = {

tests/e2e/support.py

@@ -12,7 +12,7 @@
 from tests.e2e.support import *
 
 
-from codeflare_sdk.utils.kube_api_helpers import _kube_api_error_handling
+from codeflare_sdk.common import _kube_api_error_handling
 
 namespace = "test-ns-rayupgrade-sleep"
 # Global variables for kueue resources

tests/unit_test.py

@@ -7,7 +7,7 @@
 from tests.e2e.support import *
 from codeflare_sdk.cluster.cluster import get_cluster
 
-from codeflare_sdk.utils.kube_api_helpers import _kube_api_error_handling
+from codeflare_sdk.common import _kube_api_error_handling
 
 namespace = "test-ns-rayupgrade"
 # Global variables for kueue resources