diff --git a/dpctl-capi/helper/include/dpctl_utils_helper.h b/dpctl-capi/helper/include/dpctl_utils_helper.h index e9196dac88..1c158c8d51 100644 --- a/dpctl-capi/helper/include/dpctl_utils_helper.h +++ b/dpctl-capi/helper/include/dpctl_utils_helper.h @@ -149,3 +149,32 @@ sycl::aspect DPCTL_DPCTLAspectTypeToSyclAspect(DPCTLSyclAspectType AspectTy); */ DPCTL_API DPCTLSyclAspectType DPCTL_SyclAspectToDPCTLAspectType(sycl::aspect Aspect); + +/*! + * @brief Converts a DPCTLPartitionAffinityDomainType enum value to its + * corresponding sycl::info::partition_affinity_domain enum value. + * + * @param PartitionAffinityDomainTy A + * DPCTLPartitionAffinityDomainType enum value + * @return A sycl::info::partition_affinity_domain enum value for the input + * DPCTLPartitionAffinityDomainType enum value. + * @throws runtime_error + */ +DPCTL_API +sycl::info::partition_affinity_domain +DPCTL_DPCTLPartitionAffinityDomainTypeToSycl( + DPCTLPartitionAffinityDomainType PartitionAffinityDomainTy); + +/*! + * @brief Converts a sycl::info::partition_affinity_domain enum value to + * corresponding DPCTLPartitionAffinityDomainType enum value. + * + * @param PartitionAffinityDomain sycl::info::partition_affinity_domain to be + * converted to DPCTLPartitionAffinityDomainType enum. + * @return A DPCTLPartitionAffinityDomainType enum value for the input + * sycl::info::partition_affinity_domain enum value. + * @throws runtime_error + */ +DPCTL_API +DPCTLPartitionAffinityDomainType DPCTL_SyclPartitionAffinityDomainToDPCTLType( + sycl::info::partition_affinity_domain PartitionAffinityDomain); diff --git a/dpctl-capi/helper/source/dpctl_utils_helper.cpp b/dpctl-capi/helper/source/dpctl_utils_helper.cpp index d90ab3c4ce..bd498951a4 100644 --- a/dpctl-capi/helper/source/dpctl_utils_helper.cpp +++ b/dpctl-capi/helper/source/dpctl_utils_helper.cpp @@ -382,3 +382,49 @@ DPCTLSyclAspectType DPCTL_SyclAspectToDPCTLAspectType(aspect Aspect) throw runtime_error("Unsupported aspect type", -1); } } + +info::partition_affinity_domain DPCTL_DPCTLPartitionAffinityDomainTypeToSycl( + DPCTLPartitionAffinityDomainType PartitionAffinityDomainTy) +{ + switch (PartitionAffinityDomainTy) { + case DPCTLPartitionAffinityDomainType::not_applicable: + return info::partition_affinity_domain::not_applicable; + case DPCTLPartitionAffinityDomainType::numa: + return info::partition_affinity_domain::numa; + case DPCTLPartitionAffinityDomainType::L4_cache: + return info::partition_affinity_domain::L4_cache; + case DPCTLPartitionAffinityDomainType::L3_cache: + return info::partition_affinity_domain::L3_cache; + case DPCTLPartitionAffinityDomainType::L2_cache: + return info::partition_affinity_domain::L2_cache; + case DPCTLPartitionAffinityDomainType::L1_cache: + return info::partition_affinity_domain::L1_cache; + case DPCTLPartitionAffinityDomainType::next_partitionable: + return info::partition_affinity_domain::next_partitionable; + default: + throw runtime_error("Unsupported partition_affinity_domain type", -1); + } +} + +DPCTLPartitionAffinityDomainType DPCTL_SyclPartitionAffinityDomainToDPCTLType( + sycl::info::partition_affinity_domain PartitionAffinityDomain) +{ + switch (PartitionAffinityDomain) { + case info::partition_affinity_domain::not_applicable: + return DPCTLPartitionAffinityDomainType::not_applicable; + case info::partition_affinity_domain::numa: + return DPCTLPartitionAffinityDomainType::numa; + case info::partition_affinity_domain::L4_cache: + return DPCTLPartitionAffinityDomainType::L4_cache; + case info::partition_affinity_domain::L3_cache: + return DPCTLPartitionAffinityDomainType::L3_cache; + case info::partition_affinity_domain::L2_cache: + return DPCTLPartitionAffinityDomainType::L2_cache; + case info::partition_affinity_domain::L1_cache: + return DPCTLPartitionAffinityDomainType::L1_cache; + case info::partition_affinity_domain::next_partitionable: + return DPCTLPartitionAffinityDomainType::next_partitionable; + default: + throw runtime_error("Unsupported partition_affinity_domain type", -1); + } +} diff --git a/dpctl-capi/include/dpctl_sycl_device_interface.h b/dpctl-capi/include/dpctl_sycl_device_interface.h index 293c511f07..a82f3b8130 100644 --- a/dpctl-capi/include/dpctl_sycl_device_interface.h +++ b/dpctl-capi/include/dpctl_sycl_device_interface.h @@ -31,6 +31,7 @@ #include "Support/ExternC.h" #include "Support/MemOwnershipAttrs.h" #include "dpctl_data_types.h" +#include "dpctl_sycl_device_manager.h" #include "dpctl_sycl_enum_types.h" #include "dpctl_sycl_types.h" @@ -265,6 +266,57 @@ DPCTL_API bool DPCTLDevice_HasAspect(__dpctl_keep const DPCTLSyclDeviceRef DRef, DPCTLSyclAspectType AT); +/*! + * @brief Returns a vector of sub devices + * partitioned from this SYCL device based on the count parameter. The returned + * vector contains as many sub devices as can be created such that each sub + * device contains count compute units. If the device’s total number of compute + * units is not evenly divided by count, then the remaining compute units are + * not included in any of the sub devices. + * + * @param DRef Opaque pointer to a sycl::device + * @param count Count compute units that need to contains in + * subdevices + * @return A #DPCTLDeviceVectorRef containing #DPCTLSyclDeviceRef objects + */ +DPCTL_API +__dpctl_give DPCTLDeviceVectorRef +DPCTLDevice_CreateSubDevicesEqually(__dpctl_keep const DPCTLSyclDeviceRef DRef, + size_t count); + +/*! + * @brief Returns a vector of sub devices + * partitioned from this SYCL device based on the counts parameter. For each + * non-zero value M in the counts vector, a sub device with M compute units + * is created. + * + * @param DRef Opaque pointer to a sycl::device + * @param counts Array with count compute units + * that need to contains in subdevices + * @param ncounts Number of counts + * @return A #DPCTLDeviceVectorRef containing #DPCTLSyclDeviceRef objects + */ +DPCTL_API +__dpctl_give DPCTLDeviceVectorRef +DPCTLDevice_CreateSubDevicesByCounts(__dpctl_keep const DPCTLSyclDeviceRef DRef, + __dpctl_keep size_t *counts, + size_t ncounts); + +/*! + * @brief Returns a vector of sub devices + * partitioned from this SYCL device by affinity domain based on the domain + * parameter. + * + * @param DRef Opaque pointer to a sycl::device + * @param DPCTLPartitionAffinityDomainType DPCTLPartitionAffinityDomainType + * of sycl::info::partition_affinity_domain + * @return A #DPCTLDeviceVectorRef containing #DPCTLSyclDeviceRef objects + */ +DPCTL_API +__dpctl_give DPCTLDeviceVectorRef DPCTLDevice_CreateSubDevicesByAffinity( + __dpctl_keep const DPCTLSyclDeviceRef DRef, + DPCTLPartitionAffinityDomainType PartitionAffinityDomainTy); + DPCTL_C_EXTERN_C_END /*! diff --git a/dpctl-capi/include/dpctl_sycl_enum_types.h b/dpctl-capi/include/dpctl_sycl_enum_types.h index 177c928eac..02b81a29a2 100644 --- a/dpctl-capi/include/dpctl_sycl_enum_types.h +++ b/dpctl-capi/include/dpctl_sycl_enum_types.h @@ -122,6 +122,21 @@ enum DPCTLSyclAspectType usm_system_allocator }; +/*! + * @brief DPCTL analogue of sycl::info::partition_affinity_domain enum + * + */ +enum DPCTLPartitionAffinityDomainType +{ + not_applicable, + numa, + L4_cache, + L3_cache, + L2_cache, + L1_cache, + next_partitionable +}; + /*! * @brief Enums to depict the properties that can be passed to a sycl::queue * constructor. diff --git a/dpctl-capi/source/dpctl_sycl_device_interface.cpp b/dpctl-capi/source/dpctl_sycl_device_interface.cpp index dd8937c2de..b6687fce34 100644 --- a/dpctl-capi/source/dpctl_sycl_device_interface.cpp +++ b/dpctl-capi/source/dpctl_sycl_device_interface.cpp @@ -39,6 +39,8 @@ namespace DEFINE_SIMPLE_CONVERSION_FUNCTIONS(device, DPCTLSyclDeviceRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(device_selector, DPCTLSyclDeviceSelectorRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(platform, DPCTLSyclPlatformRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(vector_class, + DPCTLDeviceVectorRef) } /* end of anonymous namespace */ @@ -525,3 +527,89 @@ uint32_t DPCTLDevice_GetPreferredVectorWidthHalf( } return vector_width_half; } + +__dpctl_give DPCTLDeviceVectorRef +DPCTLDevice_CreateSubDevicesEqually(__dpctl_keep const DPCTLSyclDeviceRef DRef, + size_t count) +{ + vector_class *Devices = nullptr; + auto D = unwrap(DRef); + if (D) { + try { + auto subDevices = D->create_sub_devices< + info::partition_property::partition_equally>(count); + Devices = new vector_class(); + for (const auto &sd : subDevices) { + Devices->emplace_back(wrap(new device(sd))); + } + } catch (std::bad_alloc const &ba) { + std::cerr << ba.what() << '\n'; + return nullptr; + } catch (feature_not_supported const &fnse) { + std::cerr << fnse.what() << '\n'; + } catch (runtime_error const &re) { + // \todo log error + std::cerr << re.what() << '\n'; + } + } + return wrap(Devices); +} + +__dpctl_give DPCTLDeviceVectorRef +DPCTLDevice_CreateSubDevicesByCounts(__dpctl_keep const DPCTLSyclDeviceRef DRef, + __dpctl_keep size_t *counts, + size_t ncounts) +{ + vector_class *Devices = nullptr; + std::vector vcounts; + vcounts.assign(counts, counts + ncounts); + auto D = unwrap(DRef); + if (D) { + try { + auto subDevices = D->create_sub_devices< + info::partition_property::partition_by_counts>(vcounts); + Devices = new vector_class(); + for (const auto &sd : subDevices) { + Devices->emplace_back(wrap(new device(sd))); + } + } catch (std::bad_alloc const &ba) { + std::cerr << ba.what() << '\n'; + return nullptr; + } catch (feature_not_supported const &fnse) { + std::cerr << fnse.what() << '\n'; + } catch (runtime_error const &re) { + // \todo log error + std::cerr << re.what() << '\n'; + } + } + return wrap(Devices); +} + +__dpctl_give DPCTLDeviceVectorRef DPCTLDevice_CreateSubDevicesByAffinity( + __dpctl_keep const DPCTLSyclDeviceRef DRef, + DPCTLPartitionAffinityDomainType PartitionAffinityDomainTy) +{ + vector_class *Devices = nullptr; + auto D = unwrap(DRef); + if (D) { + try { + auto domain = DPCTL_DPCTLPartitionAffinityDomainTypeToSycl( + PartitionAffinityDomainTy); + auto subDevices = D->create_sub_devices< + info::partition_property::partition_by_affinity_domain>(domain); + Devices = new vector_class(); + for (const auto &sd : subDevices) { + Devices->emplace_back(wrap(new device(sd))); + } + } catch (std::bad_alloc const &ba) { + std::cerr << ba.what() << '\n'; + return nullptr; + } catch (feature_not_supported const &fnse) { + std::cerr << fnse.what() << '\n'; + } catch (runtime_error const &re) { + // \todo log error + std::cerr << re.what() << '\n'; + } + } + return wrap(Devices); +} diff --git a/dpctl-capi/tests/test_sycl_device_subdevices.cpp b/dpctl-capi/tests/test_sycl_device_subdevices.cpp new file mode 100644 index 0000000000..bab0af5f3a --- /dev/null +++ b/dpctl-capi/tests/test_sycl_device_subdevices.cpp @@ -0,0 +1,370 @@ +//===--- test_sycl_device_interface.cpp - Test cases for device interface ===// +// +// Data Parallel Control (dpCtl) +// +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file has unit test cases for functions defined in +/// dpctl_sycl_device_interface.h. +/// +//===----------------------------------------------------------------------===// + +#include "../helper/include/dpctl_utils_helper.h" +#include "Support/CBindingWrapping.h" +#include "dpctl_sycl_device_interface.h" +#include "dpctl_sycl_device_selector_interface.h" +#include "dpctl_sycl_enum_types.h" +#include "dpctl_sycl_platform_interface.h" +#include "dpctl_utils.h" +#include +#include + +using namespace cl::sycl; + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(device, DPCTLSyclDeviceRef); + +struct TestDPCTLSyclDeviceInterface + : public ::testing::TestWithParam +{ + DPCTLSyclDeviceRef DRef = nullptr; + + TestDPCTLSyclDeviceInterface() + { + auto DS = DPCTLFilterSelector_Create(GetParam()); + if (DS) { + EXPECT_NO_FATAL_FAILURE(DRef = DPCTLDevice_CreateFromSelector(DS)); + } + DPCTLDeviceSelector_Delete(DS); + } + + void SetUp() + { + if (!DRef) { + auto message = "Skipping as no device of type " + + std::string(GetParam()) + "."; + GTEST_SKIP_(message.c_str()); + } + } + + ~TestDPCTLSyclDeviceInterface() + { + DPCTLDevice_Delete(DRef); + } +}; + +TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesEqually) +{ + DPCTLDeviceVectorRef DVRef = nullptr; + uint32_t maxCUs = 0; + + EXPECT_NO_FATAL_FAILURE(maxCUs = DPCTLDevice_GetMaxComputeUnits(DRef)); + if (maxCUs) { + int count = maxCUs / 2; + EXPECT_NO_FATAL_FAILURE( + DVRef = DPCTLDevice_CreateSubDevicesEqually(DRef, count)); + if (DVRef) { + EXPECT_TRUE(DPCTLDeviceVector_Size(DVRef) > 0); + EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); + } + } +} + +TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByCounts) +{ + DPCTLDeviceVectorRef DVRef = nullptr; + uint32_t maxCUs = 0; + + EXPECT_NO_FATAL_FAILURE(maxCUs = DPCTLDevice_GetMaxComputeUnits(DRef)); + if (maxCUs) { + size_t count = maxCUs / 2; + size_t *counts = nullptr; + int n = 2; + counts = new size_t[n]; + for (auto i = 0; i < n; ++i) { + counts[i] = count; + } + EXPECT_NO_FATAL_FAILURE( + DVRef = DPCTLDevice_CreateSubDevicesByCounts(DRef, counts, n)); + if (DVRef) { + EXPECT_TRUE(DPCTLDeviceVector_Size(DVRef) > 0); + EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); + } + } +} + +TEST_P(TestDPCTLSyclDeviceInterface, + Chk_CreateSubDevicesByAffinityNotApplicable) +{ + DPCTLDeviceVectorRef DVRef = nullptr; + + info::partition_affinity_domain domain = + info::partition_affinity_domain::not_applicable; + DPCTLPartitionAffinityDomainType dpctl_domain = + DPCTL_SyclPartitionAffinityDomainToDPCTLType(domain); + + if (dpctl_domain) { + EXPECT_NO_FATAL_FAILURE( + DVRef = DPCTLDevice_CreateSubDevicesByAffinity(DRef, dpctl_domain)); + + auto D = unwrap(DRef); + try { + auto subDevices = D->create_sub_devices< + info::partition_property::partition_by_affinity_domain>(domain); + auto expected_size = subDevices.size(); + + if (DVRef) { + EXPECT_TRUE(DPCTLDeviceVector_Size(DVRef) == expected_size); + EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); + } + } catch (runtime_error const &re) { + } + } +} + +TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByAffinityNuma) +{ + DPCTLDeviceVectorRef DVRef = nullptr; + + info::partition_affinity_domain domain = + info::partition_affinity_domain::numa; + DPCTLPartitionAffinityDomainType dpctl_domain; + EXPECT_NO_FATAL_FAILURE( + dpctl_domain = DPCTL_SyclPartitionAffinityDomainToDPCTLType(domain)); + + if (dpctl_domain) { + EXPECT_NO_FATAL_FAILURE( + DVRef = DPCTLDevice_CreateSubDevicesByAffinity(DRef, dpctl_domain)); + + auto D = unwrap(DRef); + size_t expected_size = 0; + try { + auto subDevices = D->create_sub_devices< + info::partition_property::partition_by_affinity_domain>(domain); + expected_size = subDevices.size(); + } catch (std::bad_alloc const &ba) { + std::cerr << ba.what() << '\n'; + } catch (feature_not_supported const &fnse) { + std::cerr << fnse.what() << '\n'; + } catch (runtime_error const &re) { + // \todo log error + std::cerr << re.what() << '\n'; + } + + if (DVRef && expected_size) { + EXPECT_TRUE(DPCTLDeviceVector_Size(DVRef) == expected_size); + EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); + } + } +} + +TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByAffinityL4Cache) +{ + DPCTLDeviceVectorRef DVRef = nullptr; + + info::partition_affinity_domain domain = + info::partition_affinity_domain::L4_cache; + DPCTLPartitionAffinityDomainType dpctl_domain; + EXPECT_NO_FATAL_FAILURE( + dpctl_domain = DPCTL_SyclPartitionAffinityDomainToDPCTLType(domain)); + + if (dpctl_domain) { + EXPECT_NO_FATAL_FAILURE( + DVRef = DPCTLDevice_CreateSubDevicesByAffinity(DRef, dpctl_domain)); + + auto D = unwrap(DRef); + size_t expected_size = 0; + try { + auto subDevices = D->create_sub_devices< + info::partition_property::partition_by_affinity_domain>(domain); + expected_size = subDevices.size(); + } catch (std::bad_alloc const &ba) { + std::cerr << ba.what() << '\n'; + } catch (feature_not_supported const &fnse) { + std::cerr << fnse.what() << '\n'; + } catch (runtime_error const &re) { + // \todo log error + std::cerr << re.what() << '\n'; + } + + if (DVRef && expected_size) { + EXPECT_TRUE(DPCTLDeviceVector_Size(DVRef) == expected_size); + EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); + } + } +} + +TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByAffinityL3Cache) +{ + DPCTLDeviceVectorRef DVRef = nullptr; + + info::partition_affinity_domain domain = + info::partition_affinity_domain::L3_cache; + DPCTLPartitionAffinityDomainType dpctl_domain; + EXPECT_NO_FATAL_FAILURE( + dpctl_domain = DPCTL_SyclPartitionAffinityDomainToDPCTLType(domain)); + + if (dpctl_domain) { + EXPECT_NO_FATAL_FAILURE( + DVRef = DPCTLDevice_CreateSubDevicesByAffinity(DRef, dpctl_domain)); + + auto D = unwrap(DRef); + size_t expected_size = 0; + try { + auto subDevices = D->create_sub_devices< + info::partition_property::partition_by_affinity_domain>(domain); + expected_size = subDevices.size(); + } catch (std::bad_alloc const &ba) { + std::cerr << ba.what() << '\n'; + } catch (feature_not_supported const &fnse) { + std::cerr << fnse.what() << '\n'; + } catch (runtime_error const &re) { + // \todo log error + std::cerr << re.what() << '\n'; + } + + if (DVRef && expected_size) { + EXPECT_TRUE(DPCTLDeviceVector_Size(DVRef) == expected_size); + EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); + } + } +} + +TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByAffinityL2Cache) +{ + DPCTLDeviceVectorRef DVRef = nullptr; + + info::partition_affinity_domain domain = + info::partition_affinity_domain::L2_cache; + DPCTLPartitionAffinityDomainType dpctl_domain; + EXPECT_NO_FATAL_FAILURE( + dpctl_domain = DPCTL_SyclPartitionAffinityDomainToDPCTLType(domain)); + + if (dpctl_domain) { + EXPECT_NO_FATAL_FAILURE( + DVRef = DPCTLDevice_CreateSubDevicesByAffinity(DRef, dpctl_domain)); + + auto D = unwrap(DRef); + size_t expected_size = 0; + try { + auto subDevices = D->create_sub_devices< + info::partition_property::partition_by_affinity_domain>(domain); + expected_size = subDevices.size(); + } catch (std::bad_alloc const &ba) { + std::cerr << ba.what() << '\n'; + } catch (feature_not_supported const &fnse) { + std::cerr << fnse.what() << '\n'; + } catch (runtime_error const &re) { + // \todo log error + std::cerr << re.what() << '\n'; + } + + if (DVRef && expected_size) { + EXPECT_TRUE(DPCTLDeviceVector_Size(DVRef) == expected_size); + EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); + } + } +} + +TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByAffinityL1Cache) +{ + DPCTLDeviceVectorRef DVRef = nullptr; + + info::partition_affinity_domain domain = + info::partition_affinity_domain::L1_cache; + DPCTLPartitionAffinityDomainType dpctl_domain; + EXPECT_NO_FATAL_FAILURE( + dpctl_domain = DPCTL_SyclPartitionAffinityDomainToDPCTLType(domain)); + + if (dpctl_domain) { + EXPECT_NO_FATAL_FAILURE( + DVRef = DPCTLDevice_CreateSubDevicesByAffinity(DRef, dpctl_domain)); + + auto D = unwrap(DRef); + size_t expected_size = 0; + try { + auto subDevices = D->create_sub_devices< + info::partition_property::partition_by_affinity_domain>(domain); + expected_size = subDevices.size(); + } catch (std::bad_alloc const &ba) { + std::cerr << ba.what() << '\n'; + } catch (feature_not_supported const &fnse) { + std::cerr << fnse.what() << '\n'; + } catch (runtime_error const &re) { + // \todo log error + std::cerr << re.what() << '\n'; + } + + if (DVRef && expected_size) { + EXPECT_TRUE(DPCTLDeviceVector_Size(DVRef) == expected_size); + EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); + } + } +} + +TEST_P(TestDPCTLSyclDeviceInterface, + Chk_CreateSubDevicesByAffinityNextPartitionable) +{ + DPCTLDeviceVectorRef DVRef = nullptr; + + info::partition_affinity_domain domain = + info::partition_affinity_domain::next_partitionable; + DPCTLPartitionAffinityDomainType dpctl_domain; + EXPECT_NO_FATAL_FAILURE( + dpctl_domain = DPCTL_SyclPartitionAffinityDomainToDPCTLType(domain)); + + if (dpctl_domain) { + EXPECT_NO_FATAL_FAILURE( + DVRef = DPCTLDevice_CreateSubDevicesByAffinity(DRef, dpctl_domain)); + + auto D = unwrap(DRef); + size_t expected_size = 0; + try { + auto subDevices = D->create_sub_devices< + info::partition_property::partition_by_affinity_domain>(domain); + expected_size = subDevices.size(); + } catch (std::bad_alloc const &ba) { + std::cerr << ba.what() << '\n'; + } catch (feature_not_supported const &fnse) { + std::cerr << fnse.what() << '\n'; + } catch (runtime_error const &re) { + // \todo log error + std::cerr << re.what() << '\n'; + } + + if (DVRef && expected_size) { + EXPECT_TRUE(DPCTLDeviceVector_Size(DVRef) == expected_size); + EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); + } + } +} + +INSTANTIATE_TEST_SUITE_P(DPCTLDevice_Fns, + TestDPCTLSyclDeviceInterface, + ::testing::Values("opencl", + "opencl:gpu", + "opencl:cpu", + "opencl:gpu:0", + "gpu", + "cpu", + "level_zero", + "level_zero:gpu", + "opencl:cpu:0", + "level_zero:gpu:0", + "gpu:0", + "gpu:1", + "1")); diff --git a/dpctl/_backend.pxd b/dpctl/_backend.pxd index 81078e2762..07b58b1c86 100644 --- a/dpctl/_backend.pxd +++ b/dpctl/_backend.pxd @@ -104,6 +104,18 @@ cdef extern from "dpctl_sycl_enum_types.h": ctypedef _aspect_type DPCTLSyclAspectType + cdef enum _partition_affinity_domain_type 'DPCTLPartitionAffinityDomainType': + _not_applicable 'not_applicable', + _numa 'numa', + _L4_cache 'L4_cache', + _L3_cache 'L3_cache', + _L2_cache 'L2_cache', + _L1_cache 'L1_cache', + _next_partitionable 'next_partitionable', + + ctypedef _partition_affinity_domain_type DPCTLPartitionAffinityDomainType + + cdef extern from "dpctl_sycl_types.h": cdef struct DPCTLOpaqueSyclContext cdef struct DPCTLOpaqueSyclDevice @@ -126,6 +138,14 @@ cdef extern from "dpctl_sycl_types.h": ctypedef DPCTLOpaqueSyclUSM *DPCTLSyclUSMRef +cdef extern from "dpctl_sycl_device_manager.h": + cdef struct DPCTLDeviceVector + ctypedef DPCTLDeviceVector *DPCTLDeviceVectorRef + ctypedef struct DPCTL_DeviceAndContextPair: + DPCTLSyclDeviceRef DRef + DPCTLSyclContextRef CRef + + cdef extern from "dpctl_sycl_device_interface.h": cdef bool DPCTLDevice_AreEq(const DPCTLSyclDeviceRef DRef1, const DPCTLSyclDeviceRef DRef2) @@ -163,11 +183,16 @@ cdef extern from "dpctl_sycl_device_interface.h": cdef uint32_t DPCTLDevice_GetPreferredVectorWidthHalf(const DPCTLSyclDeviceRef DRef) cpdef bool DPCTLDevice_HasAspect( const DPCTLSyclDeviceRef DRef, DPCTLSyclAspectType AT) + cdef DPCTLDeviceVectorRef DPCTLDevice_CreateSubDevicesEqually( + const DPCTLSyclDeviceRef DRef, size_t count) + cdef DPCTLDeviceVectorRef DPCTLDevice_CreateSubDevicesByCounts( + const DPCTLSyclDeviceRef DRef, size_t *counts, size_t ncounts) + cdef DPCTLDeviceVectorRef DPCTLDevice_CreateSubDevicesByAffinity( + const DPCTLSyclDeviceRef DRef, + DPCTLPartitionAffinityDomainType PartitionAffinityDomainTy) cdef extern from "dpctl_sycl_device_manager.h": - cdef struct DPCTLDeviceVector - ctypedef DPCTLDeviceVector *DPCTLDeviceVectorRef cdef void DPCTLDeviceVector_Delete(DPCTLDeviceVectorRef DVRef) cdef void DPCTLDeviceVector_Clear(DPCTLDeviceVectorRef DVRef) cdef size_t DPCTLDeviceVector_Size(DPCTLDeviceVectorRef DVRef) diff --git a/dpctl/_sycl_device.pxd b/dpctl/_sycl_device.pxd index 5293b8fa75..99878c77c5 100644 --- a/dpctl/_sycl_device.pxd +++ b/dpctl/_sycl_device.pxd @@ -23,6 +23,7 @@ from ._backend cimport ( DPCTLSyclDeviceRef, DPCTLSyclDeviceSelectorRef, + _partition_affinity_domain_type ) @@ -44,3 +45,7 @@ cdef class SyclDevice(_SyclDevice): cdef int _init_from__SyclDevice(self, _SyclDevice other) cdef int _init_from_selector(self, DPCTLSyclDeviceSelectorRef DSRef) cdef DPCTLSyclDeviceRef get_device_ref(self) + cdef _raise_sub_devices_creation_error(self, fname, errcode) + cdef list create_sub_devices_equally(self, size_t count) + cdef list create_sub_devices_by_counts(self, list counts) + cdef list create_sub_devices_by_affinity(self, _partition_affinity_domain_type domain) diff --git a/dpctl/_sycl_device.pyx b/dpctl/_sycl_device.pyx index 7a53566707..5a421ace6f 100644 --- a/dpctl/_sycl_device.pyx +++ b/dpctl/_sycl_device.pyx @@ -24,11 +24,16 @@ from ._backend cimport ( _aspect_type, _backend_type, _device_type, + _partition_affinity_domain_type, DPCTLCString_Delete, DPCTLDefaultSelector_Create, DPCTLDevice_Copy, DPCTLDevice_CreateFromSelector, DPCTLDevice_Delete, + DPCTLDeviceVectorRef, + DPCTLDeviceVector_Delete, + DPCTLDeviceVector_GetAt, + DPCTLDeviceVector_Size, DPCTLDevice_GetBackend, DPCTLDevice_GetDeviceType, DPCTLDevice_GetDriverInfo, @@ -61,16 +66,29 @@ from ._backend cimport ( DPCTLDevice_GetPreferredVectorWidthFloat, DPCTLDevice_GetPreferredVectorWidthDouble, DPCTLDevice_GetPreferredVectorWidthHalf, + DPCTLDevice_CreateSubDevicesEqually, + DPCTLDevice_CreateSubDevicesByCounts, + DPCTLDevice_CreateSubDevicesByAffinity, ) from . import backend_type, device_type from libc.stdint cimport uint32_t import warnings +from libc.stdlib cimport malloc, free __all__ = [ "SyclDevice", ] +cdef class SubDeviceCreationError(Exception): + """ + A SubDeviceCreationError exception is raised when + sub-devices were not created. + + """ + pass + + cdef class _SyclDevice: """ A helper metaclass to abstract a cl::sycl::device instance. """ @@ -83,6 +101,19 @@ cdef class _SyclDevice: DPCTLSize_t_Array_Delete(self._max_work_item_sizes) +cdef list _get_devices(DPCTLDeviceVectorRef DVRef): + cdef list devices = [] + cdef size_t nelems = 0 + if DVRef: + nelems = DPCTLDeviceVector_Size(DVRef) + for i in range(0, nelems): + DRef = DPCTLDeviceVector_GetAt(DVRef, i) + D = SyclDevice._create(DRef) + devices.append(D) + + return devices + + cdef class SyclDevice(_SyclDevice): """ Python equivalent for cl::sycl::device class. @@ -157,6 +188,87 @@ cdef class SyclDevice(_SyclDevice): SyclDevice._init_helper(self, DRef) return 0 + cdef _raise_sub_devices_creation_error(self, fname, errcode): + e = SubDeviceCreationError("Sub-devices were not created.") + e.fname = fname + e.code = errcode + raise e + + cdef list create_sub_devices_equally(self, size_t count): + """ Returns a vector of sub devices partitioned from this SYCL device + based on the count parameter. The returned + vector contains as many sub devices as can be created such that each sub + device contains count compute units. If the device’s total number of compute + units is not evenly divided by count, then the remaining compute units are + not included in any of the sub devices. + """ + cdef DPCTLDeviceVectorRef DVRef = NULL + DVRef = DPCTLDevice_CreateSubDevicesEqually(self._device_ref, count) + if DVRef is NULL: + self._raise_sub_devices_creation_error("DPCTLSubDeviceCreationError", -1) + cdef list devices = _get_devices(DVRef) + DPCTLDeviceVector_Delete(DVRef) + return devices + + cdef list create_sub_devices_by_counts(self, list counts): + """ Returns a vector of sub devices + partitioned from this SYCL device based on the counts parameter. For each + non-zero value M in the counts vector, a sub device with M compute units + is created. + """ + cdef size_t ncounts = len(counts) + cdef size_t *counts_buff = malloc(ncounts * sizeof(size_t)) + cdef DPCTLDeviceVectorRef DVRef = NULL + cdef int i + for i in range(ncounts): + counts_buff[i] = counts[i] + DVRef = DPCTLDevice_CreateSubDevicesByCounts(self._device_ref, counts_buff, ncounts) + if DVRef is NULL: + self._raise_sub_devices_creation_error("DPCTLSubDeviceCreationError", -1) + cdef list devices = _get_devices(DVRef) + free(counts_buff) + DPCTLDeviceVector_Delete(DVRef) + return devices + + cdef list create_sub_devices_by_affinity(self, _partition_affinity_domain_type domain): + """ Returns a vector of sub devices + partitioned from this SYCL device by affinity domain based on the domain + parameter. + """ + cdef DPCTLDeviceVectorRef DVRef = NULL + DVRef = DPCTLDevice_CreateSubDevicesByAffinity(self._device_ref, domain) + if DVRef is NULL: + self._raise_sub_devices_creation_error("DPCTLSubDeviceCreationError", -1) + cdef list devices = _get_devices(DVRef) + DPCTLDeviceVector_Delete(DVRef) + return devices + + def create_sub_devices(self, partition): + if isinstance(partition, int) and partition > 0: + return self.create_sub_devices_equally(partition) + elif isinstance(partition, list) and all([i > 0 for i in partition]): + return self.create_sub_devices_by_counts(partition) + elif isinstance(partition, str): + if partition == "not_applicable": + domain_type = _partition_affinity_domain_type._not_applicable + elif partition == "numa": + domain_type = _partition_affinity_domain_type._numa + elif partition == "L4_cache": + domain_type = _partition_affinity_domain_type._L4_cache + elif partition == "L3_cache": + domain_type = _partition_affinity_domain_type._L3_cache + elif partition == "L2_cache": + domain_type = _partition_affinity_domain_type._L2_cache + elif partition == "L1_cache": + domain_type = _partition_affinity_domain_type._L1_cache + elif partition == "next_partitionable": + domain_type = _partition_affinity_domain_type._next_partitionable + else: + raise Exception('Unsupported type of domain') + return self.create_sub_devices_by_affinity(domain_type) + else: + raise Exception('Unsupported type of sub-device argument') + def __cinit__(self, arg=None): cdef DPCTLSyclDeviceSelectorRef DSRef = NULL cdef const char *filter_c_str = NULL diff --git a/dpctl/tests/test_sycl_device.py b/dpctl/tests/test_sycl_device.py index 718e00bbd4..a53ccb395f 100644 --- a/dpctl/tests/test_sycl_device.py +++ b/dpctl/tests/test_sycl_device.py @@ -19,6 +19,7 @@ import dpctl import pytest +from dpctl._sycl_device import SubDeviceCreationError list_of_standard_selectors = [ dpctl.select_accelerator_device, @@ -296,6 +297,89 @@ def check_get_preferred_vector_width_half(device): pytest.fail("preferred_vector_width_half call failed") +def check_create_sub_devices_equally(device): + try: + n = int(device.max_compute_units / 2) + device.create_sub_devices(n) + except SubDeviceCreationError: + pytest.skip("create_sub_devices can't create sub-devices on this device") + except Exception: + pytest.fail("create_sub_devices failed") + + +def check_create_sub_devices_by_counts(device): + try: + n = device.max_compute_units / 2 + device.create_sub_devices([n, n]) + except SubDeviceCreationError: + pytest.skip("create_sub_devices can't create sub-devices on this device") + except Exception: + pytest.fail("create_sub_devices failed") + + +def check_create_sub_devices_by_affinity_not_applicable(device): + try: + device.create_sub_devices("not_applicable") + except SubDeviceCreationError: + pytest.skip("create_sub_devices can't create sub-devices on this device") + except Exception: + pytest.fail("create_sub_devices failed") + + +def check_create_sub_devices_by_affinity_numa(device): + try: + device.create_sub_devices("numa") + except SubDeviceCreationError: + pytest.skip("create_sub_devices can't create sub-devices on this device") + except Exception: + pytest.fail("create_sub_devices failed") + + +def check_create_sub_devices_by_affinity_L4_cache(device): + try: + device.create_sub_devices("L4_cache") + except SubDeviceCreationError: + pytest.skip("create_sub_devices can't create sub-devices on this device") + except Exception: + pytest.fail("create_sub_devices failed") + + +def check_create_sub_devices_by_affinity_L3_cache(device): + try: + device.create_sub_devices("L3_cache") + except SubDeviceCreationError: + pytest.skip("create_sub_devices can't create sub-devices on this device") + except Exception: + pytest.fail("create_sub_devices failed") + + +def check_create_sub_devices_by_affinity_L2_cache(device): + try: + device.create_sub_devices("L2_cache") + except SubDeviceCreationError: + pytest.skip("create_sub_devices can't create sub-devices on this device") + except Exception: + pytest.fail("create_sub_devices failed") + + +def check_create_sub_devices_by_affinity_L1_cache(device): + try: + device.create_sub_devices("L1_cache") + except SubDeviceCreationError: + pytest.skip("create_sub_devices can't create sub-devices on this device") + except Exception: + pytest.fail("create_sub_devices failed") + + +def check_create_sub_devices_by_affinity_next_partitionable(device): + try: + device.create_sub_devices("next_partitionable") + except SubDeviceCreationError: + pytest.skip("create_sub_devices can't create sub-devices on this device") + except Exception: + pytest.fail("create_sub_devices failed") + + def check_print_device_info(device): try: device.print_device_info() @@ -339,6 +423,15 @@ def check_print_device_info(device): check_has_aspect_usm_shared_allocations, check_has_aspect_usm_restricted_shared_allocations, check_has_aspect_usm_system_allocator, + check_create_sub_devices_equally, + check_create_sub_devices_by_counts, + check_create_sub_devices_by_affinity_not_applicable, + check_create_sub_devices_by_affinity_numa, + check_create_sub_devices_by_affinity_L4_cache, + check_create_sub_devices_by_affinity_L3_cache, + check_create_sub_devices_by_affinity_L2_cache, + check_create_sub_devices_by_affinity_L1_cache, + check_create_sub_devices_by_affinity_next_partitionable, check_print_device_info, ]