add read pvgis tmy (#907)

* add read_pvgis_tmy

* for files downloaded from pvgis tool
* update tests, api, and docs
* allow commit .csv to pvlib/data

* update what's new with read_pvgis_tmy

* closes #880
* test import from api vs. full namespace path

* py35 no likey fstring in read_pvgis_tmy

* don't use is for string comparison, ditto
* state returns, add see also

* use str in json.load in read_pvgis

* test read_pvgis_tmy_json, don't use binary buffer

* try parse_epw in read_pvgis_tmy first

* since attribute error only occurs for str/path

* more comments, space between pvgis outputformats

* raise ValueError if unknown output format
* test if ValueError raised for bad output format

* trailing new line in pvgis.py

* use `pvgis_format` instead of confusing arg

- infer parser format from file extension for csv, epw, and json
- much more detailed docstring for `pvgis_format`
- update tests for inferring format from extensions as well as failing
 if basic output format or file buffer isn't explicit

* Apply pvgis suggestions from code review to docstring

thanks Cliff!

Co-Authored-By: Cliff Hansen <[email protected]>

* wrap long lines in read pvgis docstring

* wordsmithing and formatting, hope it's okay
* don't need to index filename to get pvgis_format,
 just use rsplit(',', 1)[-1] to file extension

* use pathlib instead of rsplit in read_pvgis

- TypeError raised if filename is buffer, not ValueError

* python 3.5 has differnet messae for pathlib

- py>3.5: expected str, bytes or os.PathLike object
- py=3.5: argument should be a path or str object

Co-authored-by: Cliff Hansen <[email protected]>

Author: mikofski

.gitignore

@@ -78,15 +78,17 @@ coverage.xml
 *.h5
 
 
-#Datafiles
+# Datafiles
 *.csv
+# Ignore some csv 
+!pvlib/data/*.csv 
 
 # vi
 *.swp
 
-#Ignore some notebooks
+# Ignore some notebooks
 *.ipynb
 !docs/tutorials/*.ipynb
 
-#Ignore Mac DS_store files
+# Ignore Mac DS_store files
 *.DS_Store

docs/sphinx/source/api.rst

@@ -384,6 +384,7 @@ relevant to solar energy modeling.
    iotools.read_psm3
    iotools.parse_psm3
    iotools.get_pvgis_tmy
+   iotools.read_pvgis_tmy
 
 A :py:class:`~pvlib.location.Location` object may be created from metadata
 in some files.

docs/sphinx/source/whatsnew/v0.7.2.rst

@@ -20,6 +20,8 @@ Enhancements
   ``temperature_model='faiman'`` option to :py:class:`~pvlib.modelchain.ModelChain`
   (:pull:`897`) (:issue:`836`).
 * Add Kimber soiling model :py:func:`pvlib.losses.soiling_kimber`. (:pull:`860`)
+* Add :func:`~pvlib.iotools.read_pvgis_tmy` for files downloaded using the
+  PVGIS tool. (:issue:`880`)
 
 Bug fixes
 ~~~~~~~~~

pvlib/data/tmy_45.000_8.000_2005_2016.csv

@@ -12,4 +12,4 @@
 from pvlib.iotools.psm3 import get_psm3  # noqa: F401
 from pvlib.iotools.psm3 import read_psm3  # noqa: F401
 from pvlib.iotools.psm3 import parse_psm3  # noqa: F401
-from pvlib.iotools.pvgis import get_pvgis_tmy  # noqa: F401
+from pvlib.iotools.pvgis import get_pvgis_tmy, read_pvgis_tmy  # noqa: F401

pvlib/data/tmy_45.000_8.000_2005_2016.epw

@@ -15,9 +15,11 @@
   <https://ec.europa.eu/jrc/en/PVGIS/tools/monthly-radiation>`_
 """
 import io
+import json
+from pathlib import Path
 import requests
 import pandas as pd
-from pvlib.iotools import parse_epw
+from pvlib.iotools import read_epw, parse_epw
 
 URL = 'https://re.jrc.ec.europa.eu/api/'
 
@@ -72,6 +74,10 @@ def get_pvgis_tmy(lat, lon, outputformat='json', usehorizon=True,
         the error message in the response will be raised as an exception,
         otherwise raise whatever ``HTTP/1.1`` error occurred
 
+    See also
+    --------
+    read_pvgis_tmy
+
     References
     ----------
 
@@ -174,3 +180,99 @@ def _parse_pvgis_tmy_basic(src):
         data['time(UTC)'], format='%Y%m%d:%H%M', utc=True)
     data = data.drop('time(UTC)', axis=1)
     return data, None, None, None
+
+
+def read_pvgis_tmy(filename, pvgis_format=None):
+    """
+    Read a file downloaded from PVGIS.
+
+    Parameters
+    ----------
+    filename : str, pathlib.Path, or file-like buffer
+        Name, path, or buffer of file downloaded from PVGIS.
+    pvgis_format : str, default None
+        Format of PVGIS file or buffer. Equivalent to the ``outputformat``
+        parameter in the PVGIS TMY API. If `filename` is a file and
+        `pvgis_format` is ``None`` then the file extension will be used to
+        determine the PVGIS format to parse. For PVGIS files from the API with
+        ``outputformat='basic'``, please set `pvgis_format` to ``'basic'``. If
+        `filename` is a buffer, then `pvgis_format` is required and must be in
+        ``['csv', 'epw', 'json', 'basic']``.
+
+    Returns
+    -------
+    data : pandas.DataFrame
+        the weather data
+    months_selected : list
+        TMY year for each month, ``None`` for basic and EPW
+    inputs : dict
+        the inputs, ``None`` for basic and EPW
+    meta : list or dict
+        meta data, ``None`` for basic
+
+    Raises
+    ------
+    ValueError
+        if `pvgis_format` is ``None`` and the file extension is neither
+        ``.csv``, ``.json``, nor ``.epw``, or if `pvgis_format` is provided as
+        input but isn't in ``['csv', 'epw', 'json', 'basic']``
+    TypeError
+        if `pvgis_format` is ``None`` and `filename` is a buffer
+
+    See also
+    --------
+    get_pvgis_tmy
+    """
+    # get the PVGIS outputformat
+    if pvgis_format is None:
+        # get the file extension from suffix, but remove the dot and make sure
+        # it's lower case to compare with epw, csv, or json
+        # NOTE: raises TypeError if filename is a buffer
+        outputformat = Path(filename).suffix[1:].lower()
+    else:
+        outputformat = pvgis_format
+
+    # parse the pvgis file based on the output format, either 'epw', 'json',
+    # 'csv', or 'basic'
+
+    # EPW: use the EPW parser from the pvlib.iotools epw.py module
+    if outputformat == 'epw':
+        try:
+            data, meta = parse_epw(filename)
+        except AttributeError:  # str/path has no .read() attribute
+            data, meta = read_epw(filename)
+        return data, None, None, meta
+
+    # NOTE: json, csv, and basic output formats have parsers defined as private
+    # functions in this module
+
+    # JSON: use Python built-in json module to convert file contents to a
+    # Python dictionary, and pass the dictionary to the _parse_pvgis_tmy_json()
+    # function from this module
+    if outputformat == 'json':
+        try:
+            src = json.load(filename)
+        except AttributeError:  # str/path has no .read() attribute
+            with open(str(filename), 'r') as fbuf:
+                src = json.load(fbuf)
+        return _parse_pvgis_tmy_json(src)
+
+    # CSV or basic: use the correct parser from this module
+    # eg: _parse_pvgis_tmy_csv() or _parse_pvgist_tmy_basic()
+    if outputformat in ['csv', 'basic']:
+        # get the correct parser function for this output format from globals()
+        pvgis_parser = globals()['_parse_pvgis_tmy_{:s}'.format(outputformat)]
+        # NOTE: pvgis_parse() is a pvgis parser function from this module,
+        # either _parse_pvgis_tmy_csv() or _parse_pvgist_tmy_basic()
+        try:
+            pvgis_data = pvgis_parser(filename)
+        except AttributeError:  # str/path has no .read() attribute
+            with open(str(filename), 'rb') as fbuf:
+                pvgis_data = pvgis_parser(fbuf)
+        return pvgis_data
+
+    # raise exception if pvgis format isn't in ['csv', 'basic', 'epw', 'json']
+    err_msg = (
+        "pvgis format '{:s}' was unknown, must be either 'epw', 'json', 'csv'"
+        ", or 'basic'").format(outputformat)
+    raise ValueError(err_msg)

pvlib/data/tmy_45.000_8.000_2005_2016.json

@@ -6,7 +6,7 @@
 import pandas as pd
 import pytest
 import requests
-from pvlib.iotools import get_pvgis_tmy
+from pvlib.iotools import get_pvgis_tmy, read_pvgis_tmy
 from conftest import DATA_DIR
 
 
@@ -70,7 +70,14 @@ def csv_meta(meta_expected):
 @pytest.mark.remote_data
 def test_get_pvgis_tmy(expected, month_year_expected, inputs_expected,
                        meta_expected):
-    data, months_selected, inputs, meta = get_pvgis_tmy(45, 8)
+    pvgis_data = get_pvgis_tmy(45, 8)
+    _compare_pvgis_tmy_json(expected, month_year_expected, inputs_expected,
+                            meta_expected, pvgis_data)
+
+
+def _compare_pvgis_tmy_json(expected, month_year_expected, inputs_expected,
+                            meta_expected, pvgis_data):
+    data, months_selected, inputs, meta = pvgis_data
     # check each column of output separately
     for outvar in meta_expected['outputs']['tmy_hourly']['variables'].keys():
         assert np.allclose(data[outvar], expected[outvar])
@@ -113,7 +120,12 @@ def test_get_pvgis_tmy_kwargs(userhorizon_expected):
 
 @pytest.mark.remote_data
 def test_get_pvgis_tmy_basic(expected, meta_expected):
-    data, _, _, _ = get_pvgis_tmy(45, 8, outputformat='basic')
+    pvgis_data = get_pvgis_tmy(45, 8, outputformat='basic')
+    _compare_pvgis_tmy_basic(expected, meta_expected, pvgis_data)
+
+
+def _compare_pvgis_tmy_basic(expected, meta_expected, pvgis_data):
+    data, _, _, _ = pvgis_data
     # check each column of output separately
     for outvar in meta_expected['outputs']['tmy_hourly']['variables'].keys():
         assert np.allclose(data[outvar], expected[outvar])
@@ -122,8 +134,14 @@ def test_get_pvgis_tmy_basic(expected, meta_expected):
 @pytest.mark.remote_data
 def test_get_pvgis_tmy_csv(expected, month_year_expected, inputs_expected,
                            meta_expected, csv_meta):
-    data, months_selected, inputs, meta = get_pvgis_tmy(
-        45, 8, outputformat='csv')
+    pvgis_data = get_pvgis_tmy(45, 8, outputformat='csv')
+    _compare_pvgis_tmy_csv(expected, month_year_expected, inputs_expected,
+                           meta_expected, csv_meta, pvgis_data)
+
+
+def _compare_pvgis_tmy_csv(expected, month_year_expected, inputs_expected,
+                           meta_expected, csv_meta, pvgis_data):
+    data, months_selected, inputs, meta = pvgis_data
     # check each column of output separately
     for outvar in meta_expected['outputs']['tmy_hourly']['variables'].keys():
         assert np.allclose(data[outvar], expected[outvar])
@@ -144,8 +162,12 @@ def test_get_pvgis_tmy_csv(expected, month_year_expected, inputs_expected,
 
 @pytest.mark.remote_data
 def test_get_pvgis_tmy_epw(expected, epw_meta):
-    data, _, _, meta = get_pvgis_tmy(
-        45, 8, outputformat='epw')
+    pvgis_data = get_pvgis_tmy(45, 8, outputformat='epw')
+    _compare_pvgis_tmy_epw(expected, epw_meta, pvgis_data)
+
+
+def _compare_pvgis_tmy_epw(expected, epw_meta, pvgis_data):
+    data, _, _, meta = pvgis_data
     assert np.allclose(data.ghi, expected['G(h)'])
     assert np.allclose(data.dni, expected['Gb(n)'])
     assert np.allclose(data.dhi, expected['Gd(h)'])
@@ -160,3 +182,73 @@ def test_get_pvgis_tmy_error():
         get_pvgis_tmy(45, 8, outputformat='bad')
     with pytest.raises(requests.HTTPError, match='404 Client Error'):
         get_pvgis_tmy(45, 8, url='https://re.jrc.ec.europa.eu/')
+
+
+def test_read_pvgis_tmy_json(expected, month_year_expected, inputs_expected,
+                             meta_expected):
+    fn = DATA_DIR / 'tmy_45.000_8.000_2005_2016.json'
+    # infer outputformat from file extensions
+    pvgis_data = read_pvgis_tmy(fn)
+    _compare_pvgis_tmy_json(expected, month_year_expected, inputs_expected,
+                            meta_expected, pvgis_data)
+    # explicit pvgis outputformat
+    pvgis_data = read_pvgis_tmy(fn, pvgis_format='json')
+    _compare_pvgis_tmy_json(expected, month_year_expected, inputs_expected,
+                            meta_expected, pvgis_data)
+    with fn.open('r') as fbuf:
+        pvgis_data = read_pvgis_tmy(fbuf, pvgis_format='json')
+        _compare_pvgis_tmy_json(expected, month_year_expected, inputs_expected,
+                                meta_expected, pvgis_data)
+
+
+def test_read_pvgis_tmy_epw(expected, epw_meta):
+    fn = DATA_DIR / 'tmy_45.000_8.000_2005_2016.epw'
+    # infer outputformat from file extensions
+    pvgis_data = read_pvgis_tmy(fn)
+    _compare_pvgis_tmy_epw(expected, epw_meta, pvgis_data)
+    # explicit pvgis outputformat
+    pvgis_data = read_pvgis_tmy(fn, pvgis_format='epw')
+    _compare_pvgis_tmy_epw(expected, epw_meta, pvgis_data)
+    with fn.open('r') as fbuf:
+        pvgis_data = read_pvgis_tmy(fbuf, pvgis_format='epw')
+        _compare_pvgis_tmy_epw(expected, epw_meta, pvgis_data)
+
+
+def test_read_pvgis_tmy_csv(expected, month_year_expected, inputs_expected,
+                            meta_expected, csv_meta):
+    fn = DATA_DIR / 'tmy_45.000_8.000_2005_2016.csv'
+    # infer outputformat from file extensions
+    pvgis_data = read_pvgis_tmy(fn)
+    _compare_pvgis_tmy_csv(expected, month_year_expected, inputs_expected,
+                           meta_expected, csv_meta, pvgis_data)
+    # explicit pvgis outputformat
+    pvgis_data = read_pvgis_tmy(fn, pvgis_format='csv')
+    _compare_pvgis_tmy_csv(expected, month_year_expected, inputs_expected,
+                           meta_expected, csv_meta, pvgis_data)
+    with fn.open('rb') as fbuf:
+        pvgis_data = read_pvgis_tmy(fbuf, pvgis_format='csv')
+        _compare_pvgis_tmy_csv(expected, month_year_expected, inputs_expected,
+                               meta_expected, csv_meta, pvgis_data)
+
+
+def test_read_pvgis_tmy_basic(expected, meta_expected):
+    fn = DATA_DIR / 'tmy_45.000_8.000_2005_2016.txt'
+    # XXX: can't infer outputformat from file extensions for basic
+    with pytest.raises(ValueError, match="pvgis format 'txt' was unknown"):
+        read_pvgis_tmy(fn)
+    # explicit pvgis outputformat
+    pvgis_data = read_pvgis_tmy(fn, pvgis_format='basic')
+    _compare_pvgis_tmy_basic(expected, meta_expected, pvgis_data)
+    with fn.open('rb') as fbuf:
+        pvgis_data = read_pvgis_tmy(fbuf, pvgis_format='basic')
+        _compare_pvgis_tmy_basic(expected, meta_expected, pvgis_data)
+        # file buffer raises TypeError if passed to pathlib.Path()
+        with pytest.raises(TypeError):
+            read_pvgis_tmy(fbuf)
+
+
+def test_read_pvgis_tmy_exception():
+    bad_outputformat = 'bad'
+    err_msg = "pvgis format '{:s}' was unknown".format(bad_outputformat)
+    with pytest.raises(ValueError, match=err_msg):
+        read_pvgis_tmy('filename', pvgis_format=bad_outputformat)