diff --git a/pandas/core/dtypes/common.py b/pandas/core/dtypes/common.py
index 59550927299fe..e207dac71752e 100644
--- a/pandas/core/dtypes/common.py
+++ b/pandas/core/dtypes/common.py
@@ -128,51 +128,6 @@ def ensure_str(value: Union[bytes, Any]) -> str:
     return value
 
 
-def ensure_int_or_float(arr: ArrayLike, copy: bool = False) -> np.ndarray:
-    """
-    Ensure that an dtype array of some integer dtype
-    has an int64 dtype if possible.
-    If it's not possible, potentially because of overflow,
-    convert the array to float64 instead.
-
-    Parameters
-    ----------
-    arr : array-like
-          The array whose data type we want to enforce.
-    copy: bool
-          Whether to copy the original array or reuse
-          it in place, if possible.
-
-    Returns
-    -------
-    out_arr : The input array cast as int64 if
-              possible without overflow.
-              Otherwise the input array cast to float64.
-
-    Notes
-    -----
-    If the array is explicitly of type uint64 the type
-    will remain unchanged.
-    """
-    # TODO: GH27506 potential bug with ExtensionArrays
-    try:
-        # error: Unexpected keyword argument "casting" for "astype"
-        return arr.astype("int64", copy=copy, casting="safe")  # type: ignore[call-arg]
-    except TypeError:
-        pass
-    try:
-        # error: Unexpected keyword argument "casting" for "astype"
-        return arr.astype("uint64", copy=copy, casting="safe")  # type: ignore[call-arg]
-    except TypeError:
-        if is_extension_array_dtype(arr.dtype):
-            # pandas/core/dtypes/common.py:168: error: Item "ndarray" of
-            # "Union[ExtensionArray, ndarray]" has no attribute "to_numpy"  [union-attr]
-            return arr.to_numpy(  # type: ignore[union-attr]
-                dtype="float64", na_value=np.nan
-            )
-        return arr.astype("float64", copy=copy)
-
-
 def ensure_python_int(value: Union[int, np.integer]) -> int:
     """
     Ensure that a value is a python int.
diff --git a/pandas/core/groupby/ops.py b/pandas/core/groupby/ops.py
index 702d67b198e8d..da4e165dc5ceb 100644
--- a/pandas/core/groupby/ops.py
+++ b/pandas/core/groupby/ops.py
@@ -43,7 +43,6 @@
 from pandas.core.dtypes.common import (
     ensure_float64,
     ensure_int64,
-    ensure_int_or_float,
     ensure_platform_int,
     is_bool_dtype,
     is_categorical_dtype,
@@ -582,7 +581,7 @@ def _ea_wrap_cython_operation(
 
         elif is_integer_dtype(values.dtype) or is_bool_dtype(values.dtype):
             # IntegerArray or BooleanArray
-            values = ensure_int_or_float(values)
+            values = values.to_numpy("float64", na_value=np.nan)
             res_values = self._cython_operation(
                 kind, values, how, axis, min_count, **kwargs
             )
@@ -660,9 +659,11 @@ def _cython_operation(
             values = values.view("int64")
             is_numeric = True
         elif is_bool_dtype(dtype):
-            values = ensure_int_or_float(values)
+            values = values.astype("int64")
         elif is_integer_dtype(dtype):
-            values = ensure_int_or_float(values)
+            # e.g. uint8 -> uint64, int16 -> int64
+            dtype = dtype.kind + "8"
+            values = values.astype(dtype, copy=False)
         elif is_numeric:
             if not is_complex_dtype(dtype):
                 values = ensure_float64(values)