JsonReader should only close if it opened

Author: louispotok

pandas/io/json/json.py

@@ -377,13 +377,13 @@ def __init__(self, filepath_or_buffer, orient, typ, dtype, convert_axes,
         self.lines = lines
         self.chunksize = chunksize
         self.nrows_seen = 0
+        self.should_close = False
 
         if self.chunksize is not None:
             self.chunksize = _validate_integer("chunksize", self.chunksize, 1)
             if not self.lines:
                 raise ValueError("chunksize can only be passed if lines=True")
 
-        self.fp_or_buf = filepath_or_buffer
         data = self._get_data_from_filepath(filepath_or_buffer)
         self.data = self._preprocess_data(data)
 
@@ -427,6 +427,8 @@ def _get_data_from_filepath(self, filepath_or_buffer):
                 if exists:
                     data, _ = _get_handle(filepath_or_buffer, 'r',
                                           encoding=self.encoding)
+                    self.should_close = True
+                    self.open_stream = data
 
         return data
 
@@ -472,18 +474,14 @@ def _get_object_parser(self, json):
 
     def close(self):
         """
-        If self.chunksize, self.data may need closing.
-        If not, self.fp_or_buff may need closing.
+        If we opened a  stream earlier, in _get_data_from_filepath, we should
+        close it. If an open stream or file was passed, we leave it open.
         """
-        try:
-            self.data.close()
-        except (IOError, AttributeError):
-            pass
-
-        try:
-            self.fp_or_buf.close()
-        except(IOError, AttributeError):
-            pass
+        if self.should_close:
+            try:
+                self.open_stream.close()
+            except (IOError, AttributeError):
+                pass
 
     def __next__(self):
         lines = list(islice(self.data, self.chunksize))

pandas/tests/io/json/test_pandas.py

@@ -10,6 +10,7 @@
                     read_json, compat)
 from datetime import timedelta
 import pandas as pd
+from pandas.io.json.json import JsonReader
 
 from pandas.util.testing import (assert_almost_equal, assert_frame_equal,
                                  assert_series_equal, network,
@@ -1148,13 +1149,14 @@ def test_readjson_chunks_closes(self, chunksize):
         with ensure_clean('test.json') as path:
             df = pd.DataFrame({'A': [1, 2, 3], 'B': [4, 5, 6]})
             df.to_json(path, lines=True, orient="records")
-            f = open(path, 'r')
-            if chunksize is not None:
-                pd.concat(pd.read_json(f, lines=True, chunksize=chunksize))
-            else:
-                pd.read_json(f, lines=True)
-            assert f.closed, \
-                "didn't close file with chunksize = %s" % chunksize
+            reader = JsonReader(
+                path, orient=None, typ="frame", dtype=True, convert_axes=True,
+                convert_dates=True, keep_default_dates=True, numpy=False,
+                precise_float=False, date_unit=None, encoding=None,
+                lines=True, chunksize=chunksize)
+            reader.read()
+            assert reader.open_stream.closed, "didn't close stream with \
+                chunksize = %s" % chunksize
 
     @pytest.mark.parametrize("chunksize", [0, -1, 2.2, "foo"])
     def test_readjson_invalid_chunksize(self, lines_json_df, chunksize):