From b83a2cb1dbf10e33e1429a5f02805b975f1a78b5 Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Tue, 7 Jun 2022 15:43:40 -0500 Subject: [PATCH 1/4] Remove databricks-specific test fixtures --- python-37-slim-buster.dockerfile | 1 - test-container-with-reqs.dockerfile | 3 --- tests/docker-run-unit.sh | 5 ---- thrift_dbr_test/README.md | 6 ----- thrift_dbr_test/__init__.py | 0 thrift_dbr_test/thrift_basic_tests.py | 20 -------------- thrift_dbr_test/thrift_sqlquery_tests.py | 34 ------------------------ 7 files changed, 69 deletions(-) delete mode 100644 python-37-slim-buster.dockerfile delete mode 100644 test-container-with-reqs.dockerfile delete mode 100755 tests/docker-run-unit.sh delete mode 100644 thrift_dbr_test/README.md delete mode 100644 thrift_dbr_test/__init__.py delete mode 100644 thrift_dbr_test/thrift_basic_tests.py delete mode 100644 thrift_dbr_test/thrift_sqlquery_tests.py diff --git a/python-37-slim-buster.dockerfile b/python-37-slim-buster.dockerfile deleted file mode 100644 index dd0b35ab..00000000 --- a/python-37-slim-buster.dockerfile +++ /dev/null @@ -1 +0,0 @@ -FROM python:3.7-slim-buster diff --git a/test-container-with-reqs.dockerfile b/test-container-with-reqs.dockerfile deleted file mode 100644 index 4dc2dd19..00000000 --- a/test-container-with-reqs.dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -RUN pip install -r dev_requirements.txt - -ENTRYPOINT ["./docker-entrypoint.sh"] diff --git a/tests/docker-run-unit.sh b/tests/docker-run-unit.sh deleted file mode 100755 index fbcadfe6..00000000 --- a/tests/docker-run-unit.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -set -e - -docker run $(./cmdexec/clients/pysql_v2-test-container-full_binary_loader) unit diff --git a/thrift_dbr_test/README.md b/thrift_dbr_test/README.md deleted file mode 100644 index 020fc949..00000000 --- a/thrift_dbr_test/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Thrift-dbr-test-classes - -The classes defined in this folder depend on base classes defined in runtime. -(sql/hive-thriftserver/pythontesting/) They are to be combined in the test-docker-image, see e.g. //cmdexec/client/pysql\_v2-test-image. - -These test suites are used for running existing tests that were originally created for ODBC on the python clients that live in universe. diff --git a/thrift_dbr_test/__init__.py b/thrift_dbr_test/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/thrift_dbr_test/thrift_basic_tests.py b/thrift_dbr_test/thrift_basic_tests.py deleted file mode 100644 index c8c99465..00000000 --- a/thrift_dbr_test/thrift_basic_tests.py +++ /dev/null @@ -1,20 +0,0 @@ -import databricks.sql - -from pythontesting.pyhive_cursor_mixin import PyHiveThriftTestCase -from pythontesting.tests.common.test_spark_basictests import SparkBasicSuiteBase - - -# Base classes are defined in runtime:sql/hive-thriftserver/pythontesting/ -# They are to be combined in the test-docker-image (//cmdexec/client/pysql-test-image) -class PySQLBasicSuite(PyHiveThriftTestCase, SparkBasicSuiteBase): - error_type = databricks.sql.Error - - -if __name__ == "__main__": - import unittest - try: - import xmlrunner - testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2) - except ImportError: - testRunner = None - unittest.main(testRunner=testRunner, verbosity=2) diff --git a/thrift_dbr_test/thrift_sqlquery_tests.py b/thrift_dbr_test/thrift_sqlquery_tests.py deleted file mode 100644 index 956b24e7..00000000 --- a/thrift_dbr_test/thrift_sqlquery_tests.py +++ /dev/null @@ -1,34 +0,0 @@ -import databricks.sql - -from pythontesting.pyhive_cursor_mixin import PyHiveThriftTestCase -from pythontesting.tests.common.test_spark_sqlquerytests import SQLQueryTestSuite, retry_error - - -# Base classes are defined in runtime:sql/hive-thriftserver/pythontesting/ -# They are to be combined in the test-docker-image (//cmdexec/client/pysql_v2-test-image) -class PySQLQueryTestSuite(PyHiveThriftTestCase, SQLQueryTestSuite): - error_type = databricks.sql.Error - # 1 (\S+): exception type - # 2 (.+?): exception message - # 3 (:?;\n.+)?: elaborate explanation in AnalysisException (not captured) - exception_re = r"Error running query: (?:\[\S+\])?\s*(\S+):\s(.+?)(?:;\n.+)?$" - - def parse_column(self, name, datatype): - return f"{name}:{datatype}" - - def get_error_from_error_type(self, e): - return str(e) - - @retry_error(90, databricks.sql.Error) - def execute_single_file(self, test, code, set_commands): - return super().execute_single_file(test, code, set_commands) - - -if __name__ == "__main__": - import unittest - try: - import xmlrunner - testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2) - except ImportError: - testRunner = None - unittest.main(testRunner=testRunner, verbosity=2) From 14e8229ba05a6ee9412bc2aa012c10acf1bff415 Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Tue, 7 Jun 2022 15:47:01 -0500 Subject: [PATCH 2/4] Introduce pytest, black, and mypy workflows --- .github/workflows/code-quality-checks.yml | 148 ++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 .github/workflows/code-quality-checks.yml diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml new file mode 100644 index 00000000..b48a2faf --- /dev/null +++ b/.github/workflows/code-quality-checks.yml @@ -0,0 +1,148 @@ +name: Code Quality Checks +on: [push] +jobs: + run-tests: + runs-on: ubuntu-latest + steps: + #---------------------------------------------- + # check-out repo and set-up python + #---------------------------------------------- + - name: Check out repository + uses: actions/checkout@v2 + - name: Set up python + id: setup-python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + #---------------------------------------------- + # ----- install & configure poetry ----- + #---------------------------------------------- + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + #---------------------------------------------- + # load cached venv if cache exists + #---------------------------------------------- + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@v2 + with: + path: .venv + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} + #---------------------------------------------- + # install dependencies if cache does not exist + #---------------------------------------------- + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: poetry install --no-interaction --no-root + #---------------------------------------------- + # install your root project, if required + #---------------------------------------------- + - name: Install library + run: poetry install --no-interaction + #---------------------------------------------- + # run test suite + #---------------------------------------------- + - name: Run tests + run: poetry run pytest tests/ + check-linting: + runs-on: ubuntu-latest + steps: + #---------------------------------------------- + # check-out repo and set-up python + #---------------------------------------------- + - name: Check out repository + uses: actions/checkout@v2 + - name: Set up python + id: setup-python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + #---------------------------------------------- + # ----- install & configure poetry ----- + #---------------------------------------------- + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + #---------------------------------------------- + # load cached venv if cache exists + #---------------------------------------------- + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@v2 + with: + path: .venv + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} + #---------------------------------------------- + # install dependencies if cache does not exist + #---------------------------------------------- + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: poetry install --no-interaction --no-root + #---------------------------------------------- + # install your root project, if required + #---------------------------------------------- + - name: Install library + run: poetry install --no-interaction + #---------------------------------------------- + # black the code + #---------------------------------------------- + - name: Black + run: poetry run black --check src + + check-types: + runs-on: ubuntu-latest + steps: + #---------------------------------------------- + # check-out repo and set-up python + #---------------------------------------------- + - name: Check out repository + uses: actions/checkout@v2 + - name: Set up python + id: setup-python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + #---------------------------------------------- + # ----- install & configure poetry ----- + #---------------------------------------------- + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + #---------------------------------------------- + # load cached venv if cache exists + #---------------------------------------------- + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@v2 + with: + path: .venv + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} + #---------------------------------------------- + # install dependencies if cache does not exist + #---------------------------------------------- + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: poetry install --no-interaction --no-root + #---------------------------------------------- + # install your root project, if required + #---------------------------------------------- + - name: Install library + run: poetry install --no-interaction + #---------------------------------------------- + # black the code + #---------------------------------------------- + - name: Mypy + run: poetry run mypy src \ No newline at end of file From 0226b6157fc3a28580e9dac32b987525ad0d0988 Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Tue, 7 Jun 2022 15:51:32 -0500 Subject: [PATCH 3/4] Skip test_fetches_bench case --- tests/test_fetches_bench.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_fetches_bench.py b/tests/test_fetches_bench.py index 784ef885..e322b44a 100644 --- a/tests/test_fetches_bench.py +++ b/tests/test_fetches_bench.py @@ -4,6 +4,7 @@ import pyarrow as pa import uuid import time +import pytest import databricks.sql.client as client from databricks.sql.utils import ExecuteResponse, ArrowQueue @@ -39,6 +40,7 @@ def make_dummy_result_set_from_initial_results(arrow_table): for col_id in range(arrow_table.num_columns)] return rs + @pytest.mark.skip(reason="Test has not been updated for latest connector API (June 2022)") def test_benchmark_fetchall(self): print("preparing dummy arrow table") arrow_table = FetchBenchmarkTests.make_arrow_table(10, 25000) From 0bd92fafff8c5ffb2ed335cfca04c0490aca0ce5 Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Tue, 7 Jun 2022 15:54:12 -0500 Subject: [PATCH 4/4] Remove references to databricks-specific testing infra --- tests/test_fetches.py | 4 +--- tests/tests.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/test_fetches.py b/tests/test_fetches.py index 73da47d5..a4308d57 100644 --- a/tests/test_fetches.py +++ b/tests/test_fetches.py @@ -9,9 +9,7 @@ class FetchTests(unittest.TestCase): """ - Unit tests for checking the fetch logic. See - qa/test/cmdexec/python/suites/simple_connection_test.py for integration tests that - interact with the server. + Unit tests for checking the fetch logic. """ @staticmethod diff --git a/tests/tests.py b/tests/tests.py index aedf84d7..d5ca2387 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -19,9 +19,7 @@ class ClientTestSuite(unittest.TestCase): """ - Unit tests for isolated client behaviour. See - qa/test/cmdexec/python/suites/simple_connection_test.py for integration tests that - interact with the server. + Unit tests for isolated client behaviour. """ PACKAGE_NAME = "databricks.sql"