From f472620da6a1bd1a7885946f8d36208533d778a8 Mon Sep 17 00:00:00 2001 From: abhijeet-dhumal Date: Tue, 13 Aug 2024 10:26:17 +0530 Subject: [PATCH 1/2] Update s3 bucket endpoint url to remove https prefix --- tests/e2e/mnist.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/e2e/mnist.py b/tests/e2e/mnist.py index 4c382f67b..e767b755c 100644 --- a/tests/e2e/mnist.py +++ b/tests/e2e/mnist.py @@ -154,11 +154,19 @@ def prepare_data(self): secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY") bucket_name = os.environ.get("AWS_STORAGE_BUCKET") + if endpoint.startswith("https://"): + endpoint = endpoint[len("https://") :] + secure = True + elif endpoint.startswith("http://"): + endpoint = endpoint[len("http://") :] + secure = False + client = Minio( endpoint, access_key=access_key, secret_key=secret_key, cert_check=False, + secure=secure, ) if not os.path.exists(dataset_dir): From 3c7edb0b6e8998b33f4cbba1d922074d384b86f2 Mon Sep 17 00:00:00 2001 From: abhijeet-dhumal Date: Mon, 19 Aug 2024 12:37:53 +0530 Subject: [PATCH 2/2] Update e2e readme wrt secure parameter used for minio client in mnist script --- docs/e2e.md | 1 + tests/e2e/mnist.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/e2e.md b/docs/e2e.md index f3d0c9409..83d8ae4e7 100644 --- a/docs/e2e.md +++ b/docs/e2e.md @@ -130,3 +130,4 @@ Currently the SDK doesn't support tolerations, so e2e tests can't be executed on AWS_STORAGE_BUCKET= AWS_STORAGE_BUCKET_MNIST_DIR= ``` + Note : When using the Python Minio client to connect to a minio storage bucket, the `AWS_DEFAULT_ENDPOINT` environment variable by default expects secure endpoint where user can use endpoint url with https/http prefix for autodetection of secure/insecure endpoint. diff --git a/tests/e2e/mnist.py b/tests/e2e/mnist.py index e767b755c..143a6b6c9 100644 --- a/tests/e2e/mnist.py +++ b/tests/e2e/mnist.py @@ -154,9 +154,10 @@ def prepare_data(self): secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY") bucket_name = os.environ.get("AWS_STORAGE_BUCKET") + # remove prefix if specified in storage bucket endpoint url + secure = True if endpoint.startswith("https://"): endpoint = endpoint[len("https://") :] - secure = True elif endpoint.startswith("http://"): endpoint = endpoint[len("http://") :] secure = False