diff --git a/.github/workflows/additional_demo_notebook_tests.yaml b/.github/workflows/additional_demo_notebook_tests.yaml index 03201e103..a081f9ee2 100644 --- a/.github/workflows/additional_demo_notebook_tests.yaml +++ b/.github/workflows/additional_demo_notebook_tests.yaml @@ -50,7 +50,7 @@ jobs: - name: Set up specific Python version uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: '3.11' cache: 'pip' # caching pip dependencies - name: Setup and start KinD cluster @@ -168,7 +168,7 @@ jobs: - name: Set up specific Python version uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: '3.11' cache: 'pip' # caching pip dependencies - name: Setup and start KinD cluster diff --git a/.github/workflows/coverage-badge.yaml b/.github/workflows/coverage-badge.yaml index af273ce95..e2fdddae1 100644 --- a/.github/workflows/coverage-badge.yaml +++ b/.github/workflows/coverage-badge.yaml @@ -13,10 +13,10 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up Python 3.9 + - name: Set up Python 3.11 uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.11 - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 61781aede..fca6d6e72 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -56,7 +56,7 @@ jobs: - name: Set up specific Python version uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: '3.11' cache: 'pip' # caching pip dependencies - name: Setup NVidia GPU environment for KinD diff --git a/.github/workflows/guided_notebook_tests.yaml b/.github/workflows/guided_notebook_tests.yaml index 7a77d5a3f..71a822845 100644 --- a/.github/workflows/guided_notebook_tests.yaml +++ b/.github/workflows/guided_notebook_tests.yaml @@ -49,7 +49,7 @@ jobs: - name: Set up specific Python version uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: '3.11' cache: 'pip' # caching pip dependencies - name: Setup and start KinD cluster @@ -160,7 +160,7 @@ jobs: - name: Set up specific Python version uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: '3.11' cache: 'pip' # caching pip dependencies - name: Setup NVidia GPU environment for KinD @@ -282,7 +282,7 @@ jobs: - name: Set up specific Python version uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: '3.11' cache: 'pip' # caching pip dependencies - name: Setup NVidia GPU environment for KinD diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ddc23b5ae..50e3f9e19 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -17,7 +17,7 @@ on: default: 'project-codeflare' python_version: type: string - default: "3.8" + default: "3.11" required: true poetry_version: type: string diff --git a/.github/workflows/ui_notebooks_test.yaml b/.github/workflows/ui_notebooks_test.yaml index 28f7e06c3..6d82c49e0 100644 --- a/.github/workflows/ui_notebooks_test.yaml +++ b/.github/workflows/ui_notebooks_test.yaml @@ -49,7 +49,7 @@ jobs: - name: Set up specific Python version uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: "3.11" cache: "pip" # caching pip dependencies - name: Setup and start KinD cluster diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 51934628d..cbe3603d1 100755 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -10,16 +10,16 @@ jobs: unit-tests: runs-on: ubuntu-latest - container: - image: quay.io/project-codeflare/codeflare-sdk-precommit:v0.0.3 steps: - uses: actions/checkout@v4 + + - name: Set up python + uses: actions/setup-python@v5 + with: + python-version: '3.11' - name: Install poetry run: pip install poetry - - uses: actions/setup-python@v5 - with: - python-version: '3.8' - - name: Install dependencies + - name: Install dependencies with poetry run: | poetry config virtualenvs.create false poetry lock --no-update diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 884632da6..8a87bad48 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,7 @@ Thank you for your interest in contributing to the CodeFlare SDK! ### Prerequisites -- Python 3.9 +- Python 3.11 - [Poetry](https://python-poetry.org/) ### Setting Up Your Development Environment diff --git a/docs/sphinx/user-docs/e2e.rst b/docs/sphinx/user-docs/e2e.rst index 846536f11..6f3d1462f 100644 --- a/docs/sphinx/user-docs/e2e.rst +++ b/docs/sphinx/user-docs/e2e.rst @@ -4,7 +4,7 @@ Running e2e tests locally Pre-requisites ^^^^^^^^^^^^^^ -- We recommend using Python 3.9, along with Poetry. +- We recommend using Python 3.11, along with Poetry. On KinD clusters ---------------- diff --git a/src/codeflare_sdk/common/kueue/test_kueue.py b/src/codeflare_sdk/common/kueue/test_kueue.py index 77095d4d9..0093058cd 100644 --- a/src/codeflare_sdk/common/kueue/test_kueue.py +++ b/src/codeflare_sdk/common/kueue/test_kueue.py @@ -11,7 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from ..utils.unit_test_support import get_local_queue, createClusterConfig +from ..utils.unit_test_support import ( + apply_template, + get_local_queue, + createClusterConfig, + get_template_variables, +) from unittest.mock import patch from codeflare_sdk.ray.cluster.cluster import Cluster, ClusterConfiguration import yaml @@ -52,21 +57,21 @@ def test_cluster_creation_no_aw_local_queue(mocker): config.local_queue = "local-queue-default" cluster = Cluster(config) assert cluster.resource_yaml == f"{aw_dir}unit-test-cluster-kueue.yaml" - assert filecmp.cmp( - f"{aw_dir}unit-test-cluster-kueue.yaml", + expected_rc = apply_template( f"{parent}/tests/test_cluster_yamls/kueue/ray_cluster_kueue.yaml", - shallow=True, + get_template_variables(), ) + with open(f"{aw_dir}unit-test-cluster-kueue.yaml", "r") as f: + cluster_kueue = yaml.load(f, Loader=yaml.FullLoader) + assert cluster_kueue == expected_rc + # With resources loaded in memory, no Local Queue specified. config = createClusterConfig() config.name = "unit-test-cluster-kueue" config.write_to_file = False cluster = Cluster(config) - - with open(f"{parent}/tests/test_cluster_yamls/kueue/ray_cluster_kueue.yaml") as f: - expected_rc = yaml.load(f, Loader=yaml.FullLoader) - assert cluster.resource_yaml == expected_rc + assert cluster.resource_yaml == expected_rc def test_aw_creation_local_queue(mocker): @@ -86,12 +91,15 @@ def test_aw_creation_local_queue(mocker): config.local_queue = "local-queue-default" cluster = Cluster(config) assert cluster.resource_yaml == f"{aw_dir}unit-test-aw-kueue.yaml" - assert filecmp.cmp( - f"{aw_dir}unit-test-aw-kueue.yaml", + expected_rc = apply_template( f"{parent}/tests/test_cluster_yamls/kueue/aw_kueue.yaml", - shallow=True, + get_template_variables(), ) + with open(f"{aw_dir}unit-test-aw-kueue.yaml", "r") as f: + aw_kueue = yaml.load(f, Loader=yaml.FullLoader) + assert aw_kueue == expected_rc + # With resources loaded in memory, no Local Queue specified. config = createClusterConfig() config.name = "unit-test-aw-kueue" @@ -99,9 +107,7 @@ def test_aw_creation_local_queue(mocker): config.write_to_file = False cluster = Cluster(config) - with open(f"{parent}/tests/test_cluster_yamls/kueue/aw_kueue.yaml") as f: - expected_rc = yaml.load(f, Loader=yaml.FullLoader) - assert cluster.resource_yaml == expected_rc + assert cluster.resource_yaml == expected_rc def test_get_local_queue_exists_fail(mocker): diff --git a/src/codeflare_sdk/common/utils/unit_test_support.py b/src/codeflare_sdk/common/utils/unit_test_support.py index 8e034378f..4182fc93d 100644 --- a/src/codeflare_sdk/common/utils/unit_test_support.py +++ b/src/codeflare_sdk/common/utils/unit_test_support.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import string +import sys from codeflare_sdk.ray.cluster.cluster import ( Cluster, ClusterConfiguration, @@ -255,6 +257,32 @@ def arg_check_del_effect(group, version, namespace, plural, name, *args): assert name == "ray-dashboard-unit-test-cluster-ray" +def apply_template(yaml_file_path, variables): + with open(yaml_file_path, "r") as file: + yaml_content = file.read() + + # Create a Template instance and substitute the variables + template = string.Template(yaml_content) + filled_yaml = template.substitute(variables) + + # Now load the filled YAML into a Python object + return yaml.load(filled_yaml, Loader=yaml.FullLoader) + + +def get_expected_image(): + python_version = sys.version_info + if python_version.major == 3 and python_version.minor == 9: + return "quay.io/modh/ray@sha256:0d715f92570a2997381b7cafc0e224cfa25323f18b9545acfd23bc2b71576d06" + else: + return "quay.io/modh/ray@sha256:db667df1bc437a7b0965e8031e905d3ab04b86390d764d120e05ea5a5c18d1b4" + + +def get_template_variables(): + return { + "image": get_expected_image(), + } + + def arg_check_apply_effect(group, version, namespace, plural, body, *args): assert namespace == "ns" assert args == tuple() diff --git a/src/codeflare_sdk/ray/cluster/test_config.py b/src/codeflare_sdk/ray/cluster/test_config.py index 3416fc28c..b16cbaf28 100644 --- a/src/codeflare_sdk/ray/cluster/test_config.py +++ b/src/codeflare_sdk/ray/cluster/test_config.py @@ -13,15 +13,15 @@ # limitations under the License. from codeflare_sdk.common.utils.unit_test_support import ( + apply_template, createClusterWrongType, - get_local_queue, create_cluster_all_config_params, + get_template_variables, ) from codeflare_sdk.ray.cluster.cluster import ClusterConfiguration, Cluster from pathlib import Path import filecmp import pytest -import yaml import os parent = Path(__file__).resolve().parents[4] # project directory @@ -36,9 +36,11 @@ def test_default_cluster_creation(mocker): cluster = Cluster(ClusterConfiguration(name="default-cluster", namespace="ns")) - with open(f"{expected_clusters_dir}/ray/default-ray-cluster.yaml") as f: - expected_rc = yaml.load(f, Loader=yaml.FullLoader) - assert cluster.resource_yaml == expected_rc + expected_rc = apply_template( + f"{expected_clusters_dir}/ray/default-ray-cluster.yaml", + get_template_variables(), + ) + assert cluster.resource_yaml == expected_rc def test_default_appwrapper_creation(mocker): @@ -50,9 +52,10 @@ def test_default_appwrapper_creation(mocker): ClusterConfiguration(name="default-appwrapper", namespace="ns", appwrapper=True) ) - with open(f"{expected_clusters_dir}/ray/default-appwrapper.yaml") as f: - expected_aw = yaml.load(f, Loader=yaml.FullLoader) - assert cluster.resource_yaml == expected_aw + expected_aw = apply_template( + f"{expected_clusters_dir}/ray/default-appwrapper.yaml", get_template_variables() + ) + assert cluster.resource_yaml == expected_aw def test_config_creation_all_parameters(mocker): diff --git a/tests/test_cluster_yamls/kueue/aw_kueue.yaml b/tests/test_cluster_yamls/kueue/aw_kueue.yaml index 402ffb6a6..b5b5ae3fe 100644 --- a/tests/test_cluster_yamls/kueue/aw_kueue.yaml +++ b/tests/test_cluster_yamls/kueue/aw_kueue.yaml @@ -38,7 +38,7 @@ spec: template: spec: containers: - - image: quay.io/modh/ray@sha256:0d715f92570a2997381b7cafc0e224cfa25323f18b9545acfd23bc2b71576d06 + - image: "${image}" imagePullPolicy: Always lifecycle: preStop: @@ -103,7 +103,7 @@ spec: template: spec: containers: - - image: quay.io/modh/ray@sha256:0d715f92570a2997381b7cafc0e224cfa25323f18b9545acfd23bc2b71576d06 + - image: "${image}" imagePullPolicy: Always lifecycle: preStop: diff --git a/tests/test_cluster_yamls/kueue/ray_cluster_kueue.yaml b/tests/test_cluster_yamls/kueue/ray_cluster_kueue.yaml index a5cb36164..ad179a0b3 100644 --- a/tests/test_cluster_yamls/kueue/ray_cluster_kueue.yaml +++ b/tests/test_cluster_yamls/kueue/ray_cluster_kueue.yaml @@ -38,7 +38,7 @@ spec: template: spec: containers: - - image: quay.io/modh/ray@sha256:0d715f92570a2997381b7cafc0e224cfa25323f18b9545acfd23bc2b71576d06 + - image: "${image}" imagePullPolicy: Always lifecycle: preStop: @@ -103,7 +103,7 @@ spec: template: spec: containers: - - image: quay.io/modh/ray@sha256:0d715f92570a2997381b7cafc0e224cfa25323f18b9545acfd23bc2b71576d06 + - image: "${image}" imagePullPolicy: Always lifecycle: preStop: diff --git a/tests/test_cluster_yamls/ray/default-appwrapper.yaml b/tests/test_cluster_yamls/ray/default-appwrapper.yaml index 3e97474d4..eadfeaa07 100644 --- a/tests/test_cluster_yamls/ray/default-appwrapper.yaml +++ b/tests/test_cluster_yamls/ray/default-appwrapper.yaml @@ -36,7 +36,7 @@ spec: template: spec: containers: - - image: quay.io/modh/ray@sha256:0d715f92570a2997381b7cafc0e224cfa25323f18b9545acfd23bc2b71576d06 + - image: "${image}" imagePullPolicy: Always lifecycle: preStop: @@ -101,7 +101,7 @@ spec: template: spec: containers: - - image: quay.io/modh/ray@sha256:0d715f92570a2997381b7cafc0e224cfa25323f18b9545acfd23bc2b71576d06 + - image: "${image}" imagePullPolicy: Always lifecycle: preStop: diff --git a/tests/test_cluster_yamls/ray/default-ray-cluster.yaml b/tests/test_cluster_yamls/ray/default-ray-cluster.yaml index 34de53d2b..056479e62 100644 --- a/tests/test_cluster_yamls/ray/default-ray-cluster.yaml +++ b/tests/test_cluster_yamls/ray/default-ray-cluster.yaml @@ -28,7 +28,7 @@ spec: template: spec: containers: - - image: quay.io/modh/ray@sha256:0d715f92570a2997381b7cafc0e224cfa25323f18b9545acfd23bc2b71576d06 + - image: "${image}" imagePullPolicy: Always lifecycle: preStop: @@ -93,7 +93,7 @@ spec: template: spec: containers: - - image: quay.io/modh/ray@sha256:0d715f92570a2997381b7cafc0e224cfa25323f18b9545acfd23bc2b71576d06 + - image: "${image}" imagePullPolicy: Always lifecycle: preStop: