diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py
index 6aac736abf10a..0999a9af3f72a 100644
--- a/pandas/core/internals/blocks.py
+++ b/pandas/core/internals/blocks.py
@@ -1969,6 +1969,9 @@ def convert(
         attempt to cast any object types to better types return a copy of
         the block (if copy = True) by definition we ARE an ObjectBlock!!!!!
         """
+        if self.dtype != object:
+            return [self]
+
         values = self.values
         if values.ndim == 2:
             # maybe_split ensures we only get here with values.shape[0] == 1,
diff --git a/pandas/tests/series/methods/test_infer_objects.py b/pandas/tests/series/methods/test_infer_objects.py
index 4417bdb8e1687..25bff46d682be 100644
--- a/pandas/tests/series/methods/test_infer_objects.py
+++ b/pandas/tests/series/methods/test_infer_objects.py
@@ -1,6 +1,9 @@
 import numpy as np
 
-from pandas import interval_range
+from pandas import (
+    Series,
+    interval_range,
+)
 import pandas._testing as tm
 
 
@@ -44,3 +47,10 @@ def test_infer_objects_interval(self, index_or_series):
 
         result = obj.astype(object).infer_objects()
         tm.assert_equal(result, obj)
+
+    def test_infer_objects_bytes(self):
+        # GH#49650
+        ser = Series([b"a"], dtype="bytes")
+        expected = ser.copy()
+        result = ser.infer_objects()
+        tm.assert_series_equal(result, expected)