ENH: .read_pickle(...) from zip containing hidden OS X/macOS metadata files/folders

doc/source/whatsnew/v1.1.4.rst

@@ -35,7 +35,7 @@ Bug fixes
 
 Other
 ~~~~~
--
+- Added ability to load pickles from ``.zip`` files created by OS X and macOS containing ``__MACOSX/`` or ``.DS_STORE`` hidden folders/files (:issue:`37098`).
 
 .. ---------------------------------------------------------------------------
 

pandas/io/common.py

@@ -564,7 +564,12 @@ def get_handle(
             if zf.mode == "w":
                 f = zf
             elif zf.mode == "r":
-                zip_names = zf.namelist()
+                # Ignore hidden folders added by OS X/macOS on .zip creation
+                zip_names = [
+                    _
+                    for _ in zf.namelist()
+                    if not (_.startswith("__MACOSX/") or _.startswith(".DS_STORE"))
+                ]
                 if len(zip_names) == 1:
                     f = zf.open(zip_names.pop())
                 elif len(zip_names) == 0:

pandas/tests/io/test_pickle.py

@@ -376,6 +376,40 @@ def test_read_infer(self, ext, get_random_path):
 
             tm.assert_frame_equal(df, df2)
 
+    @pytest.mark.parametrize("cruft", ["__MACOSX/", ".DS_STORE"])
+    def test_load_zip_with_hidden_folders(self, cruft, get_random_path):
+        """Test loading .zip files that have extraneous metadata in hidden folders. """
+        base = get_random_path
+        path1 = base + ".raw"
+        path2 = base + ".zip"
+        dummy = base + ".dummy"
+        compression = "zip"
+
+        with tm.ensure_clean(path1) as p1, tm.ensure_clean(
+            path2
+        ) as p2, tm.ensure_clean(dummy) as d:
+            df = tm.makeDataFrame()
+
+            # write to uncompressed file
+            df.to_pickle(p1, compression=None)
+
+            # compress dataframe normally
+            self.compress_file(p1, p2, compression=compression)
+
+            # add dummy file `{cruft}{dummy}` to the archive
+            with zipfile.ZipFile(p2, "a", compression=zipfile.ZIP_DEFLATED) as f:
+                f.write(d, f"{cruft}{dummy}")
+
+            # check the file was definitely added to the archive
+            with zipfile.ZipFile(p2, "r") as f:
+                assert f"{cruft}{dummy}" in f.namelist()
+
+            # compressed file by inferred compression method,
+            # dummy file should have been ignored
+            df2 = pd.read_pickle(p2)
+
+            tm.assert_frame_equal(df, df2)
+
 
 # ---------------------
 # test pickle compression