From 19fab842d544acda45e465deb1eda35c28667b11 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Sun, 17 Dec 2017 14:19:30 -0500 Subject: [PATCH 01/14] Port importlib_resources to importlib.resources --- Lib/importlib/resources.py | 298 ++++++++++++++++++ Lib/test/test_importlib/data01/__init__.py | 0 Lib/test/test_importlib/data01/binary.file | Bin 0 -> 4 bytes .../data01/subdirectory/__init__.py | 0 .../data01/subdirectory/binary.file | Bin 0 -> 4 bytes Lib/test/test_importlib/data01/utf-16.file | Bin 0 -> 44 bytes Lib/test/test_importlib/data01/utf-8.file | 1 + Lib/test/test_importlib/data02/__init__.py | 0 .../test_importlib/data02/one/__init__.py | 0 .../test_importlib/data02/one/resource1.txt | 1 + .../test_importlib/data02/two/__init__.py | 0 .../test_importlib/data02/two/resource2.txt | 1 + Lib/test/test_importlib/data03/__init__.py | 0 .../data03/namespace/portion1/__init__.py | 0 .../data03/namespace/portion2/__init__.py | 0 .../data03/namespace/resource1.txt | 0 Lib/test/test_importlib/test_open.py | 72 +++++ Lib/test/test_importlib/test_path.py | 39 +++ Lib/test/test_importlib/test_read.py | 53 ++++ Lib/test/test_importlib/test_resource.py | 143 +++++++++ Lib/test/test_importlib/util.py | 163 ++++++++++ Lib/test/test_importlib/zipdata01/__init__.py | 0 .../test_importlib/zipdata01/ziptestdata.zip | Bin 0 -> 876 bytes Lib/test/test_importlib/zipdata02/__init__.py | 0 .../test_importlib/zipdata02/ziptestdata.zip | Bin 0 -> 698 bytes 25 files changed, 771 insertions(+) create mode 100644 Lib/importlib/resources.py create mode 100644 Lib/test/test_importlib/data01/__init__.py create mode 100644 Lib/test/test_importlib/data01/binary.file create mode 100644 Lib/test/test_importlib/data01/subdirectory/__init__.py create mode 100644 Lib/test/test_importlib/data01/subdirectory/binary.file create mode 100644 Lib/test/test_importlib/data01/utf-16.file create mode 100644 Lib/test/test_importlib/data01/utf-8.file create mode 100644 Lib/test/test_importlib/data02/__init__.py create mode 100644 Lib/test/test_importlib/data02/one/__init__.py create mode 100644 Lib/test/test_importlib/data02/one/resource1.txt create mode 100644 Lib/test/test_importlib/data02/two/__init__.py create mode 100644 Lib/test/test_importlib/data02/two/resource2.txt create mode 100644 Lib/test/test_importlib/data03/__init__.py create mode 100644 Lib/test/test_importlib/data03/namespace/portion1/__init__.py create mode 100644 Lib/test/test_importlib/data03/namespace/portion2/__init__.py create mode 100644 Lib/test/test_importlib/data03/namespace/resource1.txt create mode 100644 Lib/test/test_importlib/test_open.py create mode 100644 Lib/test/test_importlib/test_path.py create mode 100644 Lib/test/test_importlib/test_read.py create mode 100644 Lib/test/test_importlib/test_resource.py create mode 100644 Lib/test/test_importlib/zipdata01/__init__.py create mode 100644 Lib/test/test_importlib/zipdata01/ziptestdata.zip create mode 100644 Lib/test/test_importlib/zipdata02/__init__.py create mode 100644 Lib/test/test_importlib/zipdata02/ziptestdata.zip diff --git a/Lib/importlib/resources.py b/Lib/importlib/resources.py new file mode 100644 index 00000000000000..3a637f98208a8b --- /dev/null +++ b/Lib/importlib/resources.py @@ -0,0 +1,298 @@ +import os +import tempfile + +from . import abc as resources_abc +from builtins import open as builtins_open +from contextlib import contextmanager, suppress +from importlib import import_module +from importlib.abc import ResourceLoader +from io import BytesIO, TextIOWrapper +from pathlib import Path +from types import ModuleType +from typing import Iterator, Optional, Set, Union # noqa: F401 +from typing import cast +from typing.io import BinaryIO, TextIO +from zipfile import ZipFile + + +Package = Union[str, ModuleType] +Resource = Union[str, os.PathLike] + + +def _get_package(package) -> ModuleType: + if hasattr(package, '__spec__'): + if package.__spec__.submodule_search_locations is None: + raise TypeError('{!r} is not a package'.format( + package.__spec__.name)) + else: + return package + else: + module = import_module(package) + if module.__spec__.submodule_search_locations is None: + raise TypeError('{!r} is not a package'.format(package)) + else: + return module + + +def _normalize_path(path) -> str: + str_path = str(path) + parent, file_name = os.path.split(str_path) + if parent: + raise ValueError('{!r} must be only a file name'.format(path)) + else: + return file_name + + +def _get_resource_reader( + package: ModuleType) -> Optional[resources_abc.ResourceReader]: + # Return the package's loader if it's a ResourceReader. We can't use + # a issubclass() check here because apparently abc.'s __subclasscheck__() + # hook wants to create a weak reference to the object, but + # zipimport.zipimporter does not support weak references, resulting in a + # TypeError. That seems terrible. + if hasattr(package.__spec__.loader, 'open_resource'): + return cast(resources_abc.ResourceReader, package.__spec__.loader) + return None + + +def open_binary(package: Package, resource: Resource) -> BinaryIO: + """Return a file-like object opened for binary reading of the resource.""" + resource = _normalize_path(resource) + package = _get_package(package) + reader = _get_resource_reader(package) + if reader is not None: + return reader.open_resource(resource) + # Using pathlib doesn't work well here due to the lack of 'strict' + # argument for pathlib.Path.resolve() prior to Python 3.6. + absolute_package_path = os.path.abspath(package.__spec__.origin) + package_path = os.path.dirname(absolute_package_path) + full_path = os.path.join(package_path, resource) + try: + return builtins_open(full_path, mode='rb') + except IOError: + # Just assume the loader is a resource loader; all the relevant + # importlib.machinery loaders are and an AttributeError for + # get_data() will make it clear what is needed from the loader. + loader = cast(ResourceLoader, package.__spec__.loader) + data = None + if hasattr(package.__spec__.loader, 'get_data'): + with suppress(IOError): + data = loader.get_data(full_path) + if data is None: + package_name = package.__spec__.name + message = '{!r} resource not found in {!r}'.format( + resource, package_name) + raise FileNotFoundError(message) + else: + return BytesIO(data) + + +def open_text(package: Package, + resource: Resource, + encoding: str = 'utf-8', + errors: str = 'strict') -> TextIO: + """Return a file-like object opened for text reading of the resource.""" + resource = _normalize_path(resource) + package = _get_package(package) + reader = _get_resource_reader(package) + if reader is not None: + return TextIOWrapper(reader.open_resource(resource), encoding, errors) + # Using pathlib doesn't work well here due to the lack of 'strict' + # argument for pathlib.Path.resolve() prior to Python 3.6. + absolute_package_path = os.path.abspath(package.__spec__.origin) + package_path = os.path.dirname(absolute_package_path) + full_path = os.path.join(package_path, resource) + try: + return builtins_open( + full_path, mode='r', encoding=encoding, errors=errors) + except IOError: + # Just assume the loader is a resource loader; all the relevant + # importlib.machinery loaders are and an AttributeError for + # get_data() will make it clear what is needed from the loader. + loader = cast(ResourceLoader, package.__spec__.loader) + data = None + if hasattr(package.__spec__.loader, 'get_data'): + with suppress(IOError): + data = loader.get_data(full_path) + if data is None: + package_name = package.__spec__.name + message = '{!r} resource not found in {!r}'.format( + resource, package_name) + raise FileNotFoundError(message) + else: + return TextIOWrapper(BytesIO(data), encoding, errors) + + +def read_binary(package: Package, resource: Resource) -> bytes: + """Return the binary contents of the resource.""" + resource = _normalize_path(resource) + package = _get_package(package) + with open_binary(package, resource) as fp: + return fp.read() + + +def read_text(package: Package, + resource: Resource, + encoding: str = 'utf-8', + errors: str = 'strict') -> str: + """Return the decoded string of the resource. + + The decoding-related arguments have the same semantics as those of + bytes.decode(). + """ + resource = _normalize_path(resource) + package = _get_package(package) + with open_text(package, resource, encoding, errors) as fp: + return fp.read() + + +@contextmanager +def path(package: Package, resource: Resource) -> Iterator[Path]: + """A context manager providing a file path object to the resource. + + If the resource does not already exist on its own on the file system, + a temporary file will be created. If the file was created, the file + will be deleted upon exiting the context manager (no exception is + raised if the file was deleted prior to the context manager + exiting). + """ + resource = _normalize_path(resource) + package = _get_package(package) + reader = _get_resource_reader(package) + if reader is not None: + try: + yield Path(reader.resource_path(resource)) + return + except FileNotFoundError: + pass + # Fall-through for both the lack of resource_path() *and* if + # resource_path() raises FileNotFoundError. + package_directory = Path(package.__spec__.origin).parent + file_path = package_directory / resource + if file_path.exists(): + yield file_path + else: + with open_binary(package, resource) as fp: + data = fp.read() + # Not using tempfile.NamedTemporaryFile as it leads to deeper 'try' + # blocks due to the need to close the temporary file to work on + # Windows properly. + fd, raw_path = tempfile.mkstemp() + try: + os.write(fd, data) + os.close(fd) + yield Path(raw_path) + finally: + try: + os.remove(raw_path) + except FileNotFoundError: + pass + + +def is_resource(package: Package, name: str) -> bool: + """True if `name` is a resource inside `package`. + + Directories are *not* resources. + """ + package = _get_package(package) + _normalize_path(name) + reader = _get_resource_reader(package) + if reader is not None: + return reader.is_resource(name) + try: + package_contents = set(contents(package)) + except (NotADirectoryError, FileNotFoundError): + return False + if name not in package_contents: + return False + # Just because the given file_name lives as an entry in the package's + # contents doesn't necessarily mean it's a resource. Directories are not + # resources, so let's try to find out if it's a directory or not. + path = Path(package.__spec__.origin).parent / name + if path.is_file(): + return True + if path.is_dir(): + return False + # If it's not a file and it's not a directory, what is it? Well, this + # means the file doesn't exist on the file system, so it probably lives + # inside a zip file. We have to crack open the zip, look at its table of + # contents, and make sure that this entry doesn't have sub-entries. + archive_path = package.__spec__.loader.archive # type: ignore + package_directory = Path(package.__spec__.origin).parent + with ZipFile(archive_path) as zf: + toc = zf.namelist() + relpath = package_directory.relative_to(archive_path) + candidate_path = relpath / name + for entry in toc: # pragma: nobranch + try: + relative_to_candidate = Path(entry).relative_to(candidate_path) + except ValueError: + # The two paths aren't relative to each other so we can ignore it. + continue + # Since directories aren't explicitly listed in the zip file, we must + # infer their 'directory-ness' by looking at the number of path + # components in the path relative to the package resource we're + # looking up. If there are zero additional parts, it's a file, i.e. a + # resource. If there are more than zero it's a directory, i.e. not a + # resource. It has to be one of these two cases. + return len(relative_to_candidate.parts) == 0 + # I think it's impossible to get here. It would mean that we are looking + # for a resource in a zip file, there's an entry matching it in the return + # value of contents(), but we never actually found it in the zip's table of + # contents. + raise AssertionError('Impossible situation') + + +def contents(package: Package) -> Iterator[str]: + """Return the list of entries in `package`. + + Note that not all entries are resources. Specifically, directories are + not considered resources. Use `is_resource()` on each entry returned here + to check if it is a resource or not. + """ + package = _get_package(package) + reader = _get_resource_reader(package) + if reader is not None: + yield from reader.contents() + return + # Is the package a namespace package? By definition, namespace packages + # cannot have resources. + if (package.__spec__.origin == 'namespace' and + not package.__spec__.has_location): + return [] + package_directory = Path(package.__spec__.origin).parent + try: + yield from os.listdir(str(package_directory)) + except (NotADirectoryError, FileNotFoundError): + # The package is probably in a zip file. + archive_path = getattr(package.__spec__.loader, 'archive', None) + if archive_path is None: + raise + relpath = package_directory.relative_to(archive_path) + with ZipFile(archive_path) as zf: + toc = zf.namelist() + subdirs_seen = set() # type: Set + for filename in toc: + path = Path(filename) + # Strip off any path component parts that are in common with the + # package directory, relative to the zip archive's file system + # path. This gives us all the parts that live under the named + # package inside the zip file. If the length of these subparts is + # exactly 1, then it is situated inside the package. The resulting + # length will be 0 if it's above the package, and it will be + # greater than 1 if it lives in a subdirectory of the package + # directory. + # + # However, since directories themselves don't appear in the zip + # archive as a separate entry, we need to return the first path + # component for any case that has > 1 subparts -- but only once! + if path.parts[:len(relpath.parts)] != relpath.parts: + continue + subparts = path.parts[len(relpath.parts):] + if len(subparts) == 1: + yield subparts[0] + elif len(subparts) > 1: # pragma: nobranch + subdir = subparts[0] + if subdir not in subdirs_seen: + subdirs_seen.add(subdir) + yield subdir diff --git a/Lib/test/test_importlib/data01/__init__.py b/Lib/test/test_importlib/data01/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_importlib/data01/binary.file b/Lib/test/test_importlib/data01/binary.file new file mode 100644 index 0000000000000000000000000000000000000000..eaf36c1daccfdf325514461cd1a2ffbc139b5464 GIT binary patch literal 4 LcmZQzWMT#Y01f~L literal 0 HcmV?d00001 diff --git a/Lib/test/test_importlib/data01/subdirectory/__init__.py b/Lib/test/test_importlib/data01/subdirectory/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_importlib/data01/subdirectory/binary.file b/Lib/test/test_importlib/data01/subdirectory/binary.file new file mode 100644 index 0000000000000000000000000000000000000000..eaf36c1daccfdf325514461cd1a2ffbc139b5464 GIT binary patch literal 4 LcmZQzWMT#Y01f~L literal 0 HcmV?d00001 diff --git a/Lib/test/test_importlib/data01/utf-16.file b/Lib/test/test_importlib/data01/utf-16.file new file mode 100644 index 0000000000000000000000000000000000000000..2cb772295ef4b480a8d83725bd5006a0236d8f68 GIT binary patch literal 44 ucmezW&x0YAAqNQa8FUyF7(y9B7~B|i84MZBfV^^`Xc15@g+Y;liva-T)Ce>H literal 0 HcmV?d00001 diff --git a/Lib/test/test_importlib/data01/utf-8.file b/Lib/test/test_importlib/data01/utf-8.file new file mode 100644 index 00000000000000..1c0132ad90a192 --- /dev/null +++ b/Lib/test/test_importlib/data01/utf-8.file @@ -0,0 +1 @@ +Hello, UTF-8 world! diff --git a/Lib/test/test_importlib/data02/__init__.py b/Lib/test/test_importlib/data02/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_importlib/data02/one/__init__.py b/Lib/test/test_importlib/data02/one/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_importlib/data02/one/resource1.txt b/Lib/test/test_importlib/data02/one/resource1.txt new file mode 100644 index 00000000000000..61a813e40174a6 --- /dev/null +++ b/Lib/test/test_importlib/data02/one/resource1.txt @@ -0,0 +1 @@ +one resource diff --git a/Lib/test/test_importlib/data02/two/__init__.py b/Lib/test/test_importlib/data02/two/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_importlib/data02/two/resource2.txt b/Lib/test/test_importlib/data02/two/resource2.txt new file mode 100644 index 00000000000000..a80ce46ea362e2 --- /dev/null +++ b/Lib/test/test_importlib/data02/two/resource2.txt @@ -0,0 +1 @@ +two resource diff --git a/Lib/test/test_importlib/data03/__init__.py b/Lib/test/test_importlib/data03/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_importlib/data03/namespace/portion1/__init__.py b/Lib/test/test_importlib/data03/namespace/portion1/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_importlib/data03/namespace/portion2/__init__.py b/Lib/test/test_importlib/data03/namespace/portion2/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_importlib/data03/namespace/resource1.txt b/Lib/test/test_importlib/data03/namespace/resource1.txt new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_importlib/test_open.py b/Lib/test/test_importlib/test_open.py new file mode 100644 index 00000000000000..ad236c617169fd --- /dev/null +++ b/Lib/test/test_importlib/test_open.py @@ -0,0 +1,72 @@ +import unittest + +from importlib import resources +from . import data01 +from . import util + + +class CommonBinaryTests(util.CommonResourceTests, unittest.TestCase): + def execute(self, package, path): + with resources.open_binary(package, path): + pass + + +class CommonTextTests(util.CommonResourceTests, unittest.TestCase): + def execute(self, package, path): + with resources.open_text(package, path): + pass + + +class OpenTests: + def test_open_binary(self): + with resources.open_binary(self.data, 'utf-8.file') as fp: + result = fp.read() + self.assertEqual(result, b'Hello, UTF-8 world!\n') + + def test_open_text_default_encoding(self): + with resources.open_text(self.data, 'utf-8.file') as fp: + result = fp.read() + self.assertEqual(result, 'Hello, UTF-8 world!\n') + + def test_open_text_given_encoding(self): + with resources.open_text( + self.data, 'utf-16.file', 'utf-16', 'strict') as fp: + result = fp.read() + self.assertEqual(result, 'Hello, UTF-16 world!\n') + + def test_open_text_with_errors(self): + # Raises UnicodeError without the 'errors' argument. + with resources.open_text( + self.data, 'utf-16.file', 'utf-8', 'strict') as fp: + self.assertRaises(UnicodeError, fp.read) + with resources.open_text( + self.data, 'utf-16.file', 'utf-8', 'ignore') as fp: + result = fp.read() + self.assertEqual( + result, + 'H\x00e\x00l\x00l\x00o\x00,\x00 ' + '\x00U\x00T\x00F\x00-\x001\x006\x00 ' + '\x00w\x00o\x00r\x00l\x00d\x00!\x00\n\x00') + + def test_open_binary_FileNotFoundError(self): + self.assertRaises( + FileNotFoundError, + resources.open_binary, self.data, 'does-not-exist') + + def test_open_text_FileNotFoundError(self): + self.assertRaises( + FileNotFoundError, + resources.open_text, self.data, 'does-not-exist') + + +class OpenDiskTests(OpenTests, unittest.TestCase): + def setUp(self): + self.data = data01 + + +class OpenZipTests(OpenTests, util.ZipSetup, unittest.TestCase): + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/test_path.py b/Lib/test/test_importlib/test_path.py new file mode 100644 index 00000000000000..2d3dcda7ed2e79 --- /dev/null +++ b/Lib/test/test_importlib/test_path.py @@ -0,0 +1,39 @@ +import unittest + +from importlib import resources +from . import data01 +from . import util + + +class CommonTests(util.CommonResourceTests, unittest.TestCase): + def execute(self, package, path): + with resources.path(package, path): + pass + + +class PathTests: + def test_reading(self): + # Path should be readable. + # Test also implicitly verifies the returned object is a pathlib.Path + # instance. + with resources.path(self.data, 'utf-8.file') as path: + # pathlib.Path.read_text() was introduced in Python 3.5. + with path.open('r', encoding='utf-8') as file: + text = file.read() + self.assertEqual('Hello, UTF-8 world!\n', text) + + +class PathDiskTests(PathTests, unittest.TestCase): + data = data01 + + +class PathZipTests(PathTests, util.ZipSetup, unittest.TestCase): + def test_remove_in_context_manager(self): + # It is not an error if the file that was temporarily stashed on the + # file system is removed inside the `with` stanza. + with resources.path(self.data, 'utf-8.file') as path: + path.unlink() + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/test_read.py b/Lib/test/test_importlib/test_read.py new file mode 100644 index 00000000000000..231f5017b688ee --- /dev/null +++ b/Lib/test/test_importlib/test_read.py @@ -0,0 +1,53 @@ +import unittest + +from importlib import resources +from . import data01 +from . import util + + +class CommonBinaryTests(util.CommonResourceTests, unittest.TestCase): + def execute(self, package, path): + resources.read_binary(package, path) + + +class CommonTextTests(util.CommonResourceTests, unittest.TestCase): + def execute(self, package, path): + resources.read_text(package, path) + + +class ReadTests: + def test_read_binary(self): + result = resources.read_binary(self.data, 'binary.file') + self.assertEqual(result, b'\0\1\2\3') + + def test_read_text_default_encoding(self): + result = resources.read_text(self.data, 'utf-8.file') + self.assertEqual(result, 'Hello, UTF-8 world!\n') + + def test_read_text_given_encoding(self): + result = resources.read_text( + self.data, 'utf-16.file', encoding='utf-16') + self.assertEqual(result, 'Hello, UTF-16 world!\n') + + def test_read_text_with_errors(self): + # Raises UnicodeError without the 'errors' argument. + self.assertRaises( + UnicodeError, resources.read_text, self.data, 'utf-16.file') + result = resources.read_text(self.data, 'utf-16.file', errors='ignore') + self.assertEqual( + result, + 'H\x00e\x00l\x00l\x00o\x00,\x00 ' + '\x00U\x00T\x00F\x00-\x001\x006\x00 ' + '\x00w\x00o\x00r\x00l\x00d\x00!\x00\n\x00') + + +class ReadDiskTests(ReadTests, unittest.TestCase): + data = data01 + + +class ReadZipTests(ReadTests, util.ZipSetup, unittest.TestCase): + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/test_resource.py b/Lib/test/test_importlib/test_resource.py new file mode 100644 index 00000000000000..c35f78974857ed --- /dev/null +++ b/Lib/test/test_importlib/test_resource.py @@ -0,0 +1,143 @@ +import sys +import unittest + +from importlib import resources +from . import data01 +from . import zipdata02 +from . import util + + +class ResourceTests: + # Subclasses are expected to set the `data` attribute. + + def test_is_resource_good_path(self): + self.assertTrue(resources.is_resource(self.data, 'binary.file')) + + def test_is_resource_missing(self): + self.assertFalse(resources.is_resource(self.data, 'not-a-file')) + + def test_is_resource_subresource_directory(self): + # Directories are not resources. + self.assertFalse(resources.is_resource(self.data, 'subdirectory')) + + def test_contents(self): + contents = set(resources.contents(self.data)) + # There may be cruft in the directory listing of the data directory. + # Under Python 3 we could have a __pycache__ directory, and under + # Python 2 we could have .pyc files. These are both artifacts of the + # test suite importing these modules and writing these caches. They + # aren't germane to this test, so just filter them out. + contents.discard('__pycache__') + contents.discard('__init__.pyc') + contents.discard('__init__.pyo') + self.assertEqual(contents, { + '__init__.py', + 'subdirectory', + 'utf-8.file', + 'binary.file', + 'utf-16.file', + }) + + +class ResourceDiskTests(ResourceTests, unittest.TestCase): + def setUp(self): + self.data = data01 + + +class ResourceZipTests(ResourceTests, util.ZipSetup, unittest.TestCase): + pass + + +class ResourceLoaderTests(unittest.TestCase): + def test_resource_contents(self): + package = util.create_package( + file=data01, path=data01.__file__, contents=['A', 'B', 'C']) + self.assertEqual( + set(resources.contents(package)), + {'A', 'B', 'C'}) + + def test_resource_is_resource(self): + package = util.create_package( + file=data01, path=data01.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F']) + self.assertTrue(resources.is_resource(package, 'B')) + + def test_resource_directory_is_not_resource(self): + package = util.create_package( + file=data01, path=data01.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F']) + self.assertFalse(resources.is_resource(package, 'D')) + + def test_resource_missing_is_not_resource(self): + package = util.create_package( + file=data01, path=data01.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F']) + self.assertFalse(resources.is_resource(package, 'Z')) + + +class ResourceCornerCaseTests(unittest.TestCase): + def test_package_has_no_reader_fallback(self): + # Test odd ball packages which: + # 1. Do not have a ResourceReader as a loader + # 2. Are not on the file system + # 3. Are not in a zip file + module = util.create_package( + file=data01, path=data01.__file__, contents=['A', 'B', 'C']) + # Give the module a dummy loader. + module.__loader__ = object() + # Give the module a dummy origin. + module.__file__ = '/path/which/shall/not/be/named' + if sys.version_info >= (3,): + module.__spec__.loader = module.__loader__ + module.__spec__.origin = module.__file__ + self.assertFalse(resources.is_resource(module, 'A')) + + +class ResourceFromZipsTest(util.ZipSetupBase, unittest.TestCase): + ZIP_MODULE = zipdata02 # type: ignore + + def test_unrelated_contents(self): + # https://gitlab.com/python-devs/importlib_resources/issues/44 + # + # Here we have a zip file with two unrelated subpackages. The bug + # reports that getting the contents of a resource returns unrelated + # files. + self.assertEqual( + set(resources.contents('ziptestdata.one')), + {'__init__.py', 'resource1.txt'}) + self.assertEqual( + set(resources.contents('ziptestdata.two')), + {'__init__.py', 'resource2.txt'}) + + +class NamespaceTest(unittest.TestCase): + def test_namespaces_cant_have_resources(self): + contents = set(resources.contents( + 'test.test_importlib.data03.namespace')) + self.assertEqual(len(contents), 0) + # Even though there is a file in the namespace directory, it is not + # considered a resource, since namespace packages can't have them. + self.assertFalse(resources.is_resource( + 'test.test_importlib.data03.namespace', + 'resource1.txt')) + # We should get an exception if we try to read it or open it. + self.assertRaises( + FileNotFoundError, + resources.open_text, + 'test.test_importlib.data03.namespace', 'resource1.txt') + self.assertRaises( + FileNotFoundError, + resources.open_binary, + 'test.test_importlib.data03.namespace', 'resource1.txt') + self.assertRaises( + FileNotFoundError, + resources.read_text, + 'test.test_importlib.data03.namespace', 'resource1.txt') + self.assertRaises( + FileNotFoundError, + resources.read_binary, + 'test.test_importlib.data03.namespace', 'resource1.txt') + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index 64e039e00feaae..bfb7cad6e59b46 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -1,17 +1,24 @@ +import abc import builtins import contextlib import errno import functools import importlib from importlib import machinery, util, invalidate_caches +from importlib.abc import ResourceReader +import io import os import os.path +from pathlib import Path, PurePath from test import support import unittest import sys import tempfile import types +from . import data01 +from . import zipdata01 + BUILTINS = types.SimpleNamespace() BUILTINS.good_name = None @@ -386,3 +393,159 @@ def caseok_env_changed(self, *, should_exist): if any(x in self.importlib._bootstrap_external._os.environ for x in possibilities) != should_exist: self.skipTest('os.environ changes not reflected in _os.environ') + + +def create_package(file, path, is_package=True, contents=()): + class Reader(ResourceReader): + def open_resource(self, path): + self._path = path + if isinstance(file, Exception): + raise file + else: + return file + + def resource_path(self, path_): + self._path = path_ + if isinstance(path, Exception): + raise path + else: + return path + + def is_resource(self, path_): + self._path = path_ + if isinstance(path, Exception): + raise path + for entry in contents: + parts = entry.split('/') + if len(parts) == 1 and parts[0] == path_: + return True + return False + + def contents(self): + if isinstance(path, Exception): + raise path + # There's no yield from in baseball, er, Python 2. + for entry in contents: + yield entry + + name = 'testingpackage' + # Unforunately importlib.util.module_from_spec() was not introduced until + # Python 3.5. + module = types.ModuleType(name) + loader = Reader() + spec = machinery.ModuleSpec( + name, loader, + origin='does-not-exist', + is_package=is_package) + module.__spec__ = spec + module.__loader__ = loader + return module + + +class CommonResourceTests(abc.ABC): + @abc.abstractmethod + def execute(self, package, path): + raise NotImplementedError + + def test_package_name(self): + # Passing in the package name should succeed. + self.execute(data01.__name__, 'utf-8.file') + + def test_package_object(self): + # Passing in the package itself should succeed. + self.execute(data01, 'utf-8.file') + + def test_string_path(self): + # Passing in a string for the path should succeed. + path = 'utf-8.file' + self.execute(data01, path) + + @unittest.skipIf(sys.version_info < (3, 6), 'requires os.PathLike support') + def test_pathlib_path(self): + # Passing in a pathlib.PurePath object for the path should succeed. + path = PurePath('utf-8.file') + self.execute(data01, path) + + def test_absolute_path(self): + # An absolute path is a ValueError. + path = Path(__file__) + full_path = path.parent/'utf-8.file' + with self.assertRaises(ValueError): + self.execute(data01, full_path) + + def test_relative_path(self): + # A reative path is a ValueError. + with self.assertRaises(ValueError): + self.execute(data01, '../data01/utf-8.file') + + def test_importing_module_as_side_effect(self): + # The anchor package can already be imported. + del sys.modules[data01.__name__] + self.execute(data01.__name__, 'utf-8.file') + + def test_non_package_by_name(self): + # The anchor package cannot be a module. + with self.assertRaises(TypeError): + self.execute(__name__, 'utf-8.file') + + def test_non_package_by_package(self): + # The anchor package cannot be a module. + with self.assertRaises(TypeError): + module = sys.modules['test.test_importlib.util'] + self.execute(module, 'utf-8.file') + + @unittest.skipIf(sys.version_info < (3,), 'No ResourceReader in Python 2') + def test_resource_opener(self): + bytes_data = io.BytesIO(b'Hello, world!') + package = create_package(file=bytes_data, path=FileNotFoundError()) + self.execute(package, 'utf-8.file') + self.assertEqual(package.__loader__._path, 'utf-8.file') + + @unittest.skipIf(sys.version_info < (3,), 'No ResourceReader in Python 2') + def test_resource_path(self): + bytes_data = io.BytesIO(b'Hello, world!') + path = __file__ + package = create_package(file=bytes_data, path=path) + self.execute(package, 'utf-8.file') + self.assertEqual(package.__loader__._path, 'utf-8.file') + + def test_useless_loader(self): + package = create_package(file=FileNotFoundError(), + path=FileNotFoundError()) + with self.assertRaises(FileNotFoundError): + self.execute(package, 'utf-8.file') + + +class ZipSetupBase: + ZIP_MODULE = None + + @classmethod + def setUpClass(cls): + data_path = Path(cls.ZIP_MODULE.__file__) + data_dir = data_path.parent + cls._zip_path = str(data_dir / 'ziptestdata.zip') + sys.path.append(cls._zip_path) + cls.data = importlib.import_module('ziptestdata') + + @classmethod + def tearDownClass(cls): + try: + sys.path.remove(cls._zip_path) + except ValueError: + pass + + try: + del sys.path_importer_cache[cls._zip_path] + del sys.modules[cls.data.__name__] + except KeyError: + pass + + try: + del cls.data + del cls._zip_path + except AttributeError: + pass + + +class ZipSetup(ZipSetupBase): + ZIP_MODULE = zipdata01 # type: ignore diff --git a/Lib/test/test_importlib/zipdata01/__init__.py b/Lib/test/test_importlib/zipdata01/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_importlib/zipdata01/ziptestdata.zip b/Lib/test/test_importlib/zipdata01/ziptestdata.zip new file mode 100644 index 0000000000000000000000000000000000000000..8d8fa97f199cc29f6905404ea05f88926658ee2b GIT binary patch literal 876 zcmWIWW@Zs#0D-o$9&a!MN{9pLs?36t)Z&tq#F9k)`1s7c%#!$cy@JXB6ivcyJG)tc zia{7%Q&MJLVo{}DT4qiv10xeNs>aRTN;Wz`O(2Y}v9u&j*U${C@&7*$hE#?eAj}6U zRbU8Z2w`wz&}A@WFaz?+fucn~xfBLP1}+9v3r_zyGz;W85EcVsWbc41umGFyk(!f} zucHte;-+h%P@Z3ulcLClY7S5<9H;&?E(z02lENC@Ig!?SR;Polaa10BMBT zX9RKpPypEhj7%cTxC0L88!*_?2%>NXBW^=L0SN Date: Sun, 17 Dec 2017 14:19:30 -0500 Subject: [PATCH 02/14] Port importlib_resources to importlib.resources --- Lib/importlib/resources.py | 298 ++++++++++++++++++ Lib/test/test_importlib/data01/__init__.py | 0 Lib/test/test_importlib/data01/binary.file | Bin 0 -> 4 bytes .../data01/subdirectory/__init__.py | 0 .../data01/subdirectory/binary.file | Bin 0 -> 4 bytes Lib/test/test_importlib/data01/utf-16.file | Bin 0 -> 44 bytes Lib/test/test_importlib/data01/utf-8.file | 1 + Lib/test/test_importlib/data02/__init__.py | 0 .../test_importlib/data02/one/__init__.py | 0 .../test_importlib/data02/one/resource1.txt | 1 + .../test_importlib/data02/two/__init__.py | 0 .../test_importlib/data02/two/resource2.txt | 1 + Lib/test/test_importlib/data03/__init__.py | 0 .../data03/namespace/portion1/__init__.py | 0 .../data03/namespace/portion2/__init__.py | 0 .../data03/namespace/resource1.txt | 0 Lib/test/test_importlib/test_open.py | 72 +++++ Lib/test/test_importlib/test_path.py | 39 +++ Lib/test/test_importlib/test_read.py | 53 ++++ Lib/test/test_importlib/test_resource.py | 143 +++++++++ Lib/test/test_importlib/util.py | 163 ++++++++++ Lib/test/test_importlib/zipdata01/__init__.py | 0 .../test_importlib/zipdata01/ziptestdata.zip | Bin 0 -> 876 bytes Lib/test/test_importlib/zipdata02/__init__.py | 0 .../test_importlib/zipdata02/ziptestdata.zip | Bin 0 -> 698 bytes 25 files changed, 771 insertions(+) create mode 100644 Lib/importlib/resources.py create mode 100644 Lib/test/test_importlib/data01/__init__.py create mode 100644 Lib/test/test_importlib/data01/binary.file create mode 100644 Lib/test/test_importlib/data01/subdirectory/__init__.py create mode 100644 Lib/test/test_importlib/data01/subdirectory/binary.file create mode 100644 Lib/test/test_importlib/data01/utf-16.file create mode 100644 Lib/test/test_importlib/data01/utf-8.file create mode 100644 Lib/test/test_importlib/data02/__init__.py create mode 100644 Lib/test/test_importlib/data02/one/__init__.py create mode 100644 Lib/test/test_importlib/data02/one/resource1.txt create mode 100644 Lib/test/test_importlib/data02/two/__init__.py create mode 100644 Lib/test/test_importlib/data02/two/resource2.txt create mode 100644 Lib/test/test_importlib/data03/__init__.py create mode 100644 Lib/test/test_importlib/data03/namespace/portion1/__init__.py create mode 100644 Lib/test/test_importlib/data03/namespace/portion2/__init__.py create mode 100644 Lib/test/test_importlib/data03/namespace/resource1.txt create mode 100644 Lib/test/test_importlib/test_open.py create mode 100644 Lib/test/test_importlib/test_path.py create mode 100644 Lib/test/test_importlib/test_read.py create mode 100644 Lib/test/test_importlib/test_resource.py create mode 100644 Lib/test/test_importlib/zipdata01/__init__.py create mode 100644 Lib/test/test_importlib/zipdata01/ziptestdata.zip create mode 100644 Lib/test/test_importlib/zipdata02/__init__.py create mode 100644 Lib/test/test_importlib/zipdata02/ziptestdata.zip diff --git a/Lib/importlib/resources.py b/Lib/importlib/resources.py new file mode 100644 index 00000000000000..3a637f98208a8b --- /dev/null +++ b/Lib/importlib/resources.py @@ -0,0 +1,298 @@ +import os +import tempfile + +from . import abc as resources_abc +from builtins import open as builtins_open +from contextlib import contextmanager, suppress +from importlib import import_module +from importlib.abc import ResourceLoader +from io import BytesIO, TextIOWrapper +from pathlib import Path +from types import ModuleType +from typing import Iterator, Optional, Set, Union # noqa: F401 +from typing import cast +from typing.io import BinaryIO, TextIO +from zipfile import ZipFile + + +Package = Union[str, ModuleType] +Resource = Union[str, os.PathLike] + + +def _get_package(package) -> ModuleType: + if hasattr(package, '__spec__'): + if package.__spec__.submodule_search_locations is None: + raise TypeError('{!r} is not a package'.format( + package.__spec__.name)) + else: + return package + else: + module = import_module(package) + if module.__spec__.submodule_search_locations is None: + raise TypeError('{!r} is not a package'.format(package)) + else: + return module + + +def _normalize_path(path) -> str: + str_path = str(path) + parent, file_name = os.path.split(str_path) + if parent: + raise ValueError('{!r} must be only a file name'.format(path)) + else: + return file_name + + +def _get_resource_reader( + package: ModuleType) -> Optional[resources_abc.ResourceReader]: + # Return the package's loader if it's a ResourceReader. We can't use + # a issubclass() check here because apparently abc.'s __subclasscheck__() + # hook wants to create a weak reference to the object, but + # zipimport.zipimporter does not support weak references, resulting in a + # TypeError. That seems terrible. + if hasattr(package.__spec__.loader, 'open_resource'): + return cast(resources_abc.ResourceReader, package.__spec__.loader) + return None + + +def open_binary(package: Package, resource: Resource) -> BinaryIO: + """Return a file-like object opened for binary reading of the resource.""" + resource = _normalize_path(resource) + package = _get_package(package) + reader = _get_resource_reader(package) + if reader is not None: + return reader.open_resource(resource) + # Using pathlib doesn't work well here due to the lack of 'strict' + # argument for pathlib.Path.resolve() prior to Python 3.6. + absolute_package_path = os.path.abspath(package.__spec__.origin) + package_path = os.path.dirname(absolute_package_path) + full_path = os.path.join(package_path, resource) + try: + return builtins_open(full_path, mode='rb') + except IOError: + # Just assume the loader is a resource loader; all the relevant + # importlib.machinery loaders are and an AttributeError for + # get_data() will make it clear what is needed from the loader. + loader = cast(ResourceLoader, package.__spec__.loader) + data = None + if hasattr(package.__spec__.loader, 'get_data'): + with suppress(IOError): + data = loader.get_data(full_path) + if data is None: + package_name = package.__spec__.name + message = '{!r} resource not found in {!r}'.format( + resource, package_name) + raise FileNotFoundError(message) + else: + return BytesIO(data) + + +def open_text(package: Package, + resource: Resource, + encoding: str = 'utf-8', + errors: str = 'strict') -> TextIO: + """Return a file-like object opened for text reading of the resource.""" + resource = _normalize_path(resource) + package = _get_package(package) + reader = _get_resource_reader(package) + if reader is not None: + return TextIOWrapper(reader.open_resource(resource), encoding, errors) + # Using pathlib doesn't work well here due to the lack of 'strict' + # argument for pathlib.Path.resolve() prior to Python 3.6. + absolute_package_path = os.path.abspath(package.__spec__.origin) + package_path = os.path.dirname(absolute_package_path) + full_path = os.path.join(package_path, resource) + try: + return builtins_open( + full_path, mode='r', encoding=encoding, errors=errors) + except IOError: + # Just assume the loader is a resource loader; all the relevant + # importlib.machinery loaders are and an AttributeError for + # get_data() will make it clear what is needed from the loader. + loader = cast(ResourceLoader, package.__spec__.loader) + data = None + if hasattr(package.__spec__.loader, 'get_data'): + with suppress(IOError): + data = loader.get_data(full_path) + if data is None: + package_name = package.__spec__.name + message = '{!r} resource not found in {!r}'.format( + resource, package_name) + raise FileNotFoundError(message) + else: + return TextIOWrapper(BytesIO(data), encoding, errors) + + +def read_binary(package: Package, resource: Resource) -> bytes: + """Return the binary contents of the resource.""" + resource = _normalize_path(resource) + package = _get_package(package) + with open_binary(package, resource) as fp: + return fp.read() + + +def read_text(package: Package, + resource: Resource, + encoding: str = 'utf-8', + errors: str = 'strict') -> str: + """Return the decoded string of the resource. + + The decoding-related arguments have the same semantics as those of + bytes.decode(). + """ + resource = _normalize_path(resource) + package = _get_package(package) + with open_text(package, resource, encoding, errors) as fp: + return fp.read() + + +@contextmanager +def path(package: Package, resource: Resource) -> Iterator[Path]: + """A context manager providing a file path object to the resource. + + If the resource does not already exist on its own on the file system, + a temporary file will be created. If the file was created, the file + will be deleted upon exiting the context manager (no exception is + raised if the file was deleted prior to the context manager + exiting). + """ + resource = _normalize_path(resource) + package = _get_package(package) + reader = _get_resource_reader(package) + if reader is not None: + try: + yield Path(reader.resource_path(resource)) + return + except FileNotFoundError: + pass + # Fall-through for both the lack of resource_path() *and* if + # resource_path() raises FileNotFoundError. + package_directory = Path(package.__spec__.origin).parent + file_path = package_directory / resource + if file_path.exists(): + yield file_path + else: + with open_binary(package, resource) as fp: + data = fp.read() + # Not using tempfile.NamedTemporaryFile as it leads to deeper 'try' + # blocks due to the need to close the temporary file to work on + # Windows properly. + fd, raw_path = tempfile.mkstemp() + try: + os.write(fd, data) + os.close(fd) + yield Path(raw_path) + finally: + try: + os.remove(raw_path) + except FileNotFoundError: + pass + + +def is_resource(package: Package, name: str) -> bool: + """True if `name` is a resource inside `package`. + + Directories are *not* resources. + """ + package = _get_package(package) + _normalize_path(name) + reader = _get_resource_reader(package) + if reader is not None: + return reader.is_resource(name) + try: + package_contents = set(contents(package)) + except (NotADirectoryError, FileNotFoundError): + return False + if name not in package_contents: + return False + # Just because the given file_name lives as an entry in the package's + # contents doesn't necessarily mean it's a resource. Directories are not + # resources, so let's try to find out if it's a directory or not. + path = Path(package.__spec__.origin).parent / name + if path.is_file(): + return True + if path.is_dir(): + return False + # If it's not a file and it's not a directory, what is it? Well, this + # means the file doesn't exist on the file system, so it probably lives + # inside a zip file. We have to crack open the zip, look at its table of + # contents, and make sure that this entry doesn't have sub-entries. + archive_path = package.__spec__.loader.archive # type: ignore + package_directory = Path(package.__spec__.origin).parent + with ZipFile(archive_path) as zf: + toc = zf.namelist() + relpath = package_directory.relative_to(archive_path) + candidate_path = relpath / name + for entry in toc: # pragma: nobranch + try: + relative_to_candidate = Path(entry).relative_to(candidate_path) + except ValueError: + # The two paths aren't relative to each other so we can ignore it. + continue + # Since directories aren't explicitly listed in the zip file, we must + # infer their 'directory-ness' by looking at the number of path + # components in the path relative to the package resource we're + # looking up. If there are zero additional parts, it's a file, i.e. a + # resource. If there are more than zero it's a directory, i.e. not a + # resource. It has to be one of these two cases. + return len(relative_to_candidate.parts) == 0 + # I think it's impossible to get here. It would mean that we are looking + # for a resource in a zip file, there's an entry matching it in the return + # value of contents(), but we never actually found it in the zip's table of + # contents. + raise AssertionError('Impossible situation') + + +def contents(package: Package) -> Iterator[str]: + """Return the list of entries in `package`. + + Note that not all entries are resources. Specifically, directories are + not considered resources. Use `is_resource()` on each entry returned here + to check if it is a resource or not. + """ + package = _get_package(package) + reader = _get_resource_reader(package) + if reader is not None: + yield from reader.contents() + return + # Is the package a namespace package? By definition, namespace packages + # cannot have resources. + if (package.__spec__.origin == 'namespace' and + not package.__spec__.has_location): + return [] + package_directory = Path(package.__spec__.origin).parent + try: + yield from os.listdir(str(package_directory)) + except (NotADirectoryError, FileNotFoundError): + # The package is probably in a zip file. + archive_path = getattr(package.__spec__.loader, 'archive', None) + if archive_path is None: + raise + relpath = package_directory.relative_to(archive_path) + with ZipFile(archive_path) as zf: + toc = zf.namelist() + subdirs_seen = set() # type: Set + for filename in toc: + path = Path(filename) + # Strip off any path component parts that are in common with the + # package directory, relative to the zip archive's file system + # path. This gives us all the parts that live under the named + # package inside the zip file. If the length of these subparts is + # exactly 1, then it is situated inside the package. The resulting + # length will be 0 if it's above the package, and it will be + # greater than 1 if it lives in a subdirectory of the package + # directory. + # + # However, since directories themselves don't appear in the zip + # archive as a separate entry, we need to return the first path + # component for any case that has > 1 subparts -- but only once! + if path.parts[:len(relpath.parts)] != relpath.parts: + continue + subparts = path.parts[len(relpath.parts):] + if len(subparts) == 1: + yield subparts[0] + elif len(subparts) > 1: # pragma: nobranch + subdir = subparts[0] + if subdir not in subdirs_seen: + subdirs_seen.add(subdir) + yield subdir diff --git a/Lib/test/test_importlib/data01/__init__.py b/Lib/test/test_importlib/data01/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_importlib/data01/binary.file b/Lib/test/test_importlib/data01/binary.file new file mode 100644 index 0000000000000000000000000000000000000000..eaf36c1daccfdf325514461cd1a2ffbc139b5464 GIT binary patch literal 4 LcmZQzWMT#Y01f~L literal 0 HcmV?d00001 diff --git a/Lib/test/test_importlib/data01/subdirectory/__init__.py b/Lib/test/test_importlib/data01/subdirectory/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_importlib/data01/subdirectory/binary.file b/Lib/test/test_importlib/data01/subdirectory/binary.file new file mode 100644 index 0000000000000000000000000000000000000000..eaf36c1daccfdf325514461cd1a2ffbc139b5464 GIT binary patch literal 4 LcmZQzWMT#Y01f~L literal 0 HcmV?d00001 diff --git a/Lib/test/test_importlib/data01/utf-16.file b/Lib/test/test_importlib/data01/utf-16.file new file mode 100644 index 0000000000000000000000000000000000000000..2cb772295ef4b480a8d83725bd5006a0236d8f68 GIT binary patch literal 44 ucmezW&x0YAAqNQa8FUyF7(y9B7~B|i84MZBfV^^`Xc15@g+Y;liva-T)Ce>H literal 0 HcmV?d00001 diff --git a/Lib/test/test_importlib/data01/utf-8.file b/Lib/test/test_importlib/data01/utf-8.file new file mode 100644 index 00000000000000..1c0132ad90a192 --- /dev/null +++ b/Lib/test/test_importlib/data01/utf-8.file @@ -0,0 +1 @@ +Hello, UTF-8 world! diff --git a/Lib/test/test_importlib/data02/__init__.py b/Lib/test/test_importlib/data02/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_importlib/data02/one/__init__.py b/Lib/test/test_importlib/data02/one/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_importlib/data02/one/resource1.txt b/Lib/test/test_importlib/data02/one/resource1.txt new file mode 100644 index 00000000000000..61a813e40174a6 --- /dev/null +++ b/Lib/test/test_importlib/data02/one/resource1.txt @@ -0,0 +1 @@ +one resource diff --git a/Lib/test/test_importlib/data02/two/__init__.py b/Lib/test/test_importlib/data02/two/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_importlib/data02/two/resource2.txt b/Lib/test/test_importlib/data02/two/resource2.txt new file mode 100644 index 00000000000000..a80ce46ea362e2 --- /dev/null +++ b/Lib/test/test_importlib/data02/two/resource2.txt @@ -0,0 +1 @@ +two resource diff --git a/Lib/test/test_importlib/data03/__init__.py b/Lib/test/test_importlib/data03/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_importlib/data03/namespace/portion1/__init__.py b/Lib/test/test_importlib/data03/namespace/portion1/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_importlib/data03/namespace/portion2/__init__.py b/Lib/test/test_importlib/data03/namespace/portion2/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_importlib/data03/namespace/resource1.txt b/Lib/test/test_importlib/data03/namespace/resource1.txt new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_importlib/test_open.py b/Lib/test/test_importlib/test_open.py new file mode 100644 index 00000000000000..ad236c617169fd --- /dev/null +++ b/Lib/test/test_importlib/test_open.py @@ -0,0 +1,72 @@ +import unittest + +from importlib import resources +from . import data01 +from . import util + + +class CommonBinaryTests(util.CommonResourceTests, unittest.TestCase): + def execute(self, package, path): + with resources.open_binary(package, path): + pass + + +class CommonTextTests(util.CommonResourceTests, unittest.TestCase): + def execute(self, package, path): + with resources.open_text(package, path): + pass + + +class OpenTests: + def test_open_binary(self): + with resources.open_binary(self.data, 'utf-8.file') as fp: + result = fp.read() + self.assertEqual(result, b'Hello, UTF-8 world!\n') + + def test_open_text_default_encoding(self): + with resources.open_text(self.data, 'utf-8.file') as fp: + result = fp.read() + self.assertEqual(result, 'Hello, UTF-8 world!\n') + + def test_open_text_given_encoding(self): + with resources.open_text( + self.data, 'utf-16.file', 'utf-16', 'strict') as fp: + result = fp.read() + self.assertEqual(result, 'Hello, UTF-16 world!\n') + + def test_open_text_with_errors(self): + # Raises UnicodeError without the 'errors' argument. + with resources.open_text( + self.data, 'utf-16.file', 'utf-8', 'strict') as fp: + self.assertRaises(UnicodeError, fp.read) + with resources.open_text( + self.data, 'utf-16.file', 'utf-8', 'ignore') as fp: + result = fp.read() + self.assertEqual( + result, + 'H\x00e\x00l\x00l\x00o\x00,\x00 ' + '\x00U\x00T\x00F\x00-\x001\x006\x00 ' + '\x00w\x00o\x00r\x00l\x00d\x00!\x00\n\x00') + + def test_open_binary_FileNotFoundError(self): + self.assertRaises( + FileNotFoundError, + resources.open_binary, self.data, 'does-not-exist') + + def test_open_text_FileNotFoundError(self): + self.assertRaises( + FileNotFoundError, + resources.open_text, self.data, 'does-not-exist') + + +class OpenDiskTests(OpenTests, unittest.TestCase): + def setUp(self): + self.data = data01 + + +class OpenZipTests(OpenTests, util.ZipSetup, unittest.TestCase): + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/test_path.py b/Lib/test/test_importlib/test_path.py new file mode 100644 index 00000000000000..2d3dcda7ed2e79 --- /dev/null +++ b/Lib/test/test_importlib/test_path.py @@ -0,0 +1,39 @@ +import unittest + +from importlib import resources +from . import data01 +from . import util + + +class CommonTests(util.CommonResourceTests, unittest.TestCase): + def execute(self, package, path): + with resources.path(package, path): + pass + + +class PathTests: + def test_reading(self): + # Path should be readable. + # Test also implicitly verifies the returned object is a pathlib.Path + # instance. + with resources.path(self.data, 'utf-8.file') as path: + # pathlib.Path.read_text() was introduced in Python 3.5. + with path.open('r', encoding='utf-8') as file: + text = file.read() + self.assertEqual('Hello, UTF-8 world!\n', text) + + +class PathDiskTests(PathTests, unittest.TestCase): + data = data01 + + +class PathZipTests(PathTests, util.ZipSetup, unittest.TestCase): + def test_remove_in_context_manager(self): + # It is not an error if the file that was temporarily stashed on the + # file system is removed inside the `with` stanza. + with resources.path(self.data, 'utf-8.file') as path: + path.unlink() + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/test_read.py b/Lib/test/test_importlib/test_read.py new file mode 100644 index 00000000000000..231f5017b688ee --- /dev/null +++ b/Lib/test/test_importlib/test_read.py @@ -0,0 +1,53 @@ +import unittest + +from importlib import resources +from . import data01 +from . import util + + +class CommonBinaryTests(util.CommonResourceTests, unittest.TestCase): + def execute(self, package, path): + resources.read_binary(package, path) + + +class CommonTextTests(util.CommonResourceTests, unittest.TestCase): + def execute(self, package, path): + resources.read_text(package, path) + + +class ReadTests: + def test_read_binary(self): + result = resources.read_binary(self.data, 'binary.file') + self.assertEqual(result, b'\0\1\2\3') + + def test_read_text_default_encoding(self): + result = resources.read_text(self.data, 'utf-8.file') + self.assertEqual(result, 'Hello, UTF-8 world!\n') + + def test_read_text_given_encoding(self): + result = resources.read_text( + self.data, 'utf-16.file', encoding='utf-16') + self.assertEqual(result, 'Hello, UTF-16 world!\n') + + def test_read_text_with_errors(self): + # Raises UnicodeError without the 'errors' argument. + self.assertRaises( + UnicodeError, resources.read_text, self.data, 'utf-16.file') + result = resources.read_text(self.data, 'utf-16.file', errors='ignore') + self.assertEqual( + result, + 'H\x00e\x00l\x00l\x00o\x00,\x00 ' + '\x00U\x00T\x00F\x00-\x001\x006\x00 ' + '\x00w\x00o\x00r\x00l\x00d\x00!\x00\n\x00') + + +class ReadDiskTests(ReadTests, unittest.TestCase): + data = data01 + + +class ReadZipTests(ReadTests, util.ZipSetup, unittest.TestCase): + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/test_resource.py b/Lib/test/test_importlib/test_resource.py new file mode 100644 index 00000000000000..c35f78974857ed --- /dev/null +++ b/Lib/test/test_importlib/test_resource.py @@ -0,0 +1,143 @@ +import sys +import unittest + +from importlib import resources +from . import data01 +from . import zipdata02 +from . import util + + +class ResourceTests: + # Subclasses are expected to set the `data` attribute. + + def test_is_resource_good_path(self): + self.assertTrue(resources.is_resource(self.data, 'binary.file')) + + def test_is_resource_missing(self): + self.assertFalse(resources.is_resource(self.data, 'not-a-file')) + + def test_is_resource_subresource_directory(self): + # Directories are not resources. + self.assertFalse(resources.is_resource(self.data, 'subdirectory')) + + def test_contents(self): + contents = set(resources.contents(self.data)) + # There may be cruft in the directory listing of the data directory. + # Under Python 3 we could have a __pycache__ directory, and under + # Python 2 we could have .pyc files. These are both artifacts of the + # test suite importing these modules and writing these caches. They + # aren't germane to this test, so just filter them out. + contents.discard('__pycache__') + contents.discard('__init__.pyc') + contents.discard('__init__.pyo') + self.assertEqual(contents, { + '__init__.py', + 'subdirectory', + 'utf-8.file', + 'binary.file', + 'utf-16.file', + }) + + +class ResourceDiskTests(ResourceTests, unittest.TestCase): + def setUp(self): + self.data = data01 + + +class ResourceZipTests(ResourceTests, util.ZipSetup, unittest.TestCase): + pass + + +class ResourceLoaderTests(unittest.TestCase): + def test_resource_contents(self): + package = util.create_package( + file=data01, path=data01.__file__, contents=['A', 'B', 'C']) + self.assertEqual( + set(resources.contents(package)), + {'A', 'B', 'C'}) + + def test_resource_is_resource(self): + package = util.create_package( + file=data01, path=data01.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F']) + self.assertTrue(resources.is_resource(package, 'B')) + + def test_resource_directory_is_not_resource(self): + package = util.create_package( + file=data01, path=data01.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F']) + self.assertFalse(resources.is_resource(package, 'D')) + + def test_resource_missing_is_not_resource(self): + package = util.create_package( + file=data01, path=data01.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F']) + self.assertFalse(resources.is_resource(package, 'Z')) + + +class ResourceCornerCaseTests(unittest.TestCase): + def test_package_has_no_reader_fallback(self): + # Test odd ball packages which: + # 1. Do not have a ResourceReader as a loader + # 2. Are not on the file system + # 3. Are not in a zip file + module = util.create_package( + file=data01, path=data01.__file__, contents=['A', 'B', 'C']) + # Give the module a dummy loader. + module.__loader__ = object() + # Give the module a dummy origin. + module.__file__ = '/path/which/shall/not/be/named' + if sys.version_info >= (3,): + module.__spec__.loader = module.__loader__ + module.__spec__.origin = module.__file__ + self.assertFalse(resources.is_resource(module, 'A')) + + +class ResourceFromZipsTest(util.ZipSetupBase, unittest.TestCase): + ZIP_MODULE = zipdata02 # type: ignore + + def test_unrelated_contents(self): + # https://gitlab.com/python-devs/importlib_resources/issues/44 + # + # Here we have a zip file with two unrelated subpackages. The bug + # reports that getting the contents of a resource returns unrelated + # files. + self.assertEqual( + set(resources.contents('ziptestdata.one')), + {'__init__.py', 'resource1.txt'}) + self.assertEqual( + set(resources.contents('ziptestdata.two')), + {'__init__.py', 'resource2.txt'}) + + +class NamespaceTest(unittest.TestCase): + def test_namespaces_cant_have_resources(self): + contents = set(resources.contents( + 'test.test_importlib.data03.namespace')) + self.assertEqual(len(contents), 0) + # Even though there is a file in the namespace directory, it is not + # considered a resource, since namespace packages can't have them. + self.assertFalse(resources.is_resource( + 'test.test_importlib.data03.namespace', + 'resource1.txt')) + # We should get an exception if we try to read it or open it. + self.assertRaises( + FileNotFoundError, + resources.open_text, + 'test.test_importlib.data03.namespace', 'resource1.txt') + self.assertRaises( + FileNotFoundError, + resources.open_binary, + 'test.test_importlib.data03.namespace', 'resource1.txt') + self.assertRaises( + FileNotFoundError, + resources.read_text, + 'test.test_importlib.data03.namespace', 'resource1.txt') + self.assertRaises( + FileNotFoundError, + resources.read_binary, + 'test.test_importlib.data03.namespace', 'resource1.txt') + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index 64e039e00feaae..bfb7cad6e59b46 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -1,17 +1,24 @@ +import abc import builtins import contextlib import errno import functools import importlib from importlib import machinery, util, invalidate_caches +from importlib.abc import ResourceReader +import io import os import os.path +from pathlib import Path, PurePath from test import support import unittest import sys import tempfile import types +from . import data01 +from . import zipdata01 + BUILTINS = types.SimpleNamespace() BUILTINS.good_name = None @@ -386,3 +393,159 @@ def caseok_env_changed(self, *, should_exist): if any(x in self.importlib._bootstrap_external._os.environ for x in possibilities) != should_exist: self.skipTest('os.environ changes not reflected in _os.environ') + + +def create_package(file, path, is_package=True, contents=()): + class Reader(ResourceReader): + def open_resource(self, path): + self._path = path + if isinstance(file, Exception): + raise file + else: + return file + + def resource_path(self, path_): + self._path = path_ + if isinstance(path, Exception): + raise path + else: + return path + + def is_resource(self, path_): + self._path = path_ + if isinstance(path, Exception): + raise path + for entry in contents: + parts = entry.split('/') + if len(parts) == 1 and parts[0] == path_: + return True + return False + + def contents(self): + if isinstance(path, Exception): + raise path + # There's no yield from in baseball, er, Python 2. + for entry in contents: + yield entry + + name = 'testingpackage' + # Unforunately importlib.util.module_from_spec() was not introduced until + # Python 3.5. + module = types.ModuleType(name) + loader = Reader() + spec = machinery.ModuleSpec( + name, loader, + origin='does-not-exist', + is_package=is_package) + module.__spec__ = spec + module.__loader__ = loader + return module + + +class CommonResourceTests(abc.ABC): + @abc.abstractmethod + def execute(self, package, path): + raise NotImplementedError + + def test_package_name(self): + # Passing in the package name should succeed. + self.execute(data01.__name__, 'utf-8.file') + + def test_package_object(self): + # Passing in the package itself should succeed. + self.execute(data01, 'utf-8.file') + + def test_string_path(self): + # Passing in a string for the path should succeed. + path = 'utf-8.file' + self.execute(data01, path) + + @unittest.skipIf(sys.version_info < (3, 6), 'requires os.PathLike support') + def test_pathlib_path(self): + # Passing in a pathlib.PurePath object for the path should succeed. + path = PurePath('utf-8.file') + self.execute(data01, path) + + def test_absolute_path(self): + # An absolute path is a ValueError. + path = Path(__file__) + full_path = path.parent/'utf-8.file' + with self.assertRaises(ValueError): + self.execute(data01, full_path) + + def test_relative_path(self): + # A reative path is a ValueError. + with self.assertRaises(ValueError): + self.execute(data01, '../data01/utf-8.file') + + def test_importing_module_as_side_effect(self): + # The anchor package can already be imported. + del sys.modules[data01.__name__] + self.execute(data01.__name__, 'utf-8.file') + + def test_non_package_by_name(self): + # The anchor package cannot be a module. + with self.assertRaises(TypeError): + self.execute(__name__, 'utf-8.file') + + def test_non_package_by_package(self): + # The anchor package cannot be a module. + with self.assertRaises(TypeError): + module = sys.modules['test.test_importlib.util'] + self.execute(module, 'utf-8.file') + + @unittest.skipIf(sys.version_info < (3,), 'No ResourceReader in Python 2') + def test_resource_opener(self): + bytes_data = io.BytesIO(b'Hello, world!') + package = create_package(file=bytes_data, path=FileNotFoundError()) + self.execute(package, 'utf-8.file') + self.assertEqual(package.__loader__._path, 'utf-8.file') + + @unittest.skipIf(sys.version_info < (3,), 'No ResourceReader in Python 2') + def test_resource_path(self): + bytes_data = io.BytesIO(b'Hello, world!') + path = __file__ + package = create_package(file=bytes_data, path=path) + self.execute(package, 'utf-8.file') + self.assertEqual(package.__loader__._path, 'utf-8.file') + + def test_useless_loader(self): + package = create_package(file=FileNotFoundError(), + path=FileNotFoundError()) + with self.assertRaises(FileNotFoundError): + self.execute(package, 'utf-8.file') + + +class ZipSetupBase: + ZIP_MODULE = None + + @classmethod + def setUpClass(cls): + data_path = Path(cls.ZIP_MODULE.__file__) + data_dir = data_path.parent + cls._zip_path = str(data_dir / 'ziptestdata.zip') + sys.path.append(cls._zip_path) + cls.data = importlib.import_module('ziptestdata') + + @classmethod + def tearDownClass(cls): + try: + sys.path.remove(cls._zip_path) + except ValueError: + pass + + try: + del sys.path_importer_cache[cls._zip_path] + del sys.modules[cls.data.__name__] + except KeyError: + pass + + try: + del cls.data + del cls._zip_path + except AttributeError: + pass + + +class ZipSetup(ZipSetupBase): + ZIP_MODULE = zipdata01 # type: ignore diff --git a/Lib/test/test_importlib/zipdata01/__init__.py b/Lib/test/test_importlib/zipdata01/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_importlib/zipdata01/ziptestdata.zip b/Lib/test/test_importlib/zipdata01/ziptestdata.zip new file mode 100644 index 0000000000000000000000000000000000000000..8d8fa97f199cc29f6905404ea05f88926658ee2b GIT binary patch literal 876 zcmWIWW@Zs#0D-o$9&a!MN{9pLs?36t)Z&tq#F9k)`1s7c%#!$cy@JXB6ivcyJG)tc zia{7%Q&MJLVo{}DT4qiv10xeNs>aRTN;Wz`O(2Y}v9u&j*U${C@&7*$hE#?eAj}6U zRbU8Z2w`wz&}A@WFaz?+fucn~xfBLP1}+9v3r_zyGz;W85EcVsWbc41umGFyk(!f} zucHte;-+h%P@Z3ulcLClY7S5<9H;&?E(z02lENC@Ig!?SR;Polaa10BMBT zX9RKpPypEhj7%cTxC0L88!*_?2%>NXBW^=L0SN Date: Sun, 17 Dec 2017 14:22:53 -0500 Subject: [PATCH 03/14] Update blurb --- .../next/Library/2017-12-15-15-34-12.bpo-32248.zmO8G2.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2017-12-15-15-34-12.bpo-32248.zmO8G2.rst b/Misc/NEWS.d/next/Library/2017-12-15-15-34-12.bpo-32248.zmO8G2.rst index f77cdb03dde60d..02b7e5fef1109d 100644 --- a/Misc/NEWS.d/next/Library/2017-12-15-15-34-12.bpo-32248.zmO8G2.rst +++ b/Misc/NEWS.d/next/Library/2017-12-15-15-34-12.bpo-32248.zmO8G2.rst @@ -1,2 +1,3 @@ Add :class:`importlib.abc.ResourceReader` as an ABC for loaders to provide a -unified API for reading resources contained within packages. +unified API for reading resources contained within packages. Also add +:mod:`importlib.resources` as the port of ``importlib_resources``. From 0da9744fb5b2d6b7fef9829ddd688188f2c6f486 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Sun, 17 Dec 2017 15:37:34 -0500 Subject: [PATCH 04/14] Port the documentation --- Doc/library/importlib.rst | 162 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index eeccc9d40e6327..88ed80e93347a1 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -775,6 +775,168 @@ ABC hierarchy:: itself does not end in ``__init__``. +:mod:`importlib.resources` -- Resources +--------------------------------------- + +.. module:: importlib.resources + :synopsis: Package resource reading, opening, and access + +**Source code:** :source:`Lib/importlib/resources.py` + +-------------- + +.. versionadded:: 3.7 + +This module leverages Python's import system to provide access to *resources* +within *packages*. If you can import a package, you can access resources +within that package. Resources can be opened or read, in either binary or +text mode. + +Resources are roughly akin to files inside directories, though it's important +to keep in mind that this is just a metaphor. Resources and packages **do +not** have to exist as physical files and directories on the file system. + +Loaders can support resources by implementing the :class:`ResourceReader` +abstract base class. + +The following types are defined. + +.. class:: Package + + ``Package`` types are defined as ``Union[str, ModuleType]``. This means + that where the function describes accepting a ``Package``, you can pass in + either a string or a module. Module objects must have a resolvable + ``__spec__.submodule_search_locations`` that is not ``None``. + +.. class:: Resource + + This type describes the resource names passed into the various functions + in this package. This is defined as ``Union[str, os.PathLike]``. + + +The following functions are available. + +.. function:: importlib.resources.open_binary(package, resource) + + Open for binary reading the *resource* within *package*. + + :param package: A package name or module object. See above for the API + that such module objects must support. + :type package: ``Package`` + :param resource: The name of the resource to open within *package*. + *resource* may not contain path separators and it may + not have sub-resources (i.e. it cannot be a directory). + :type resource: ``Resource`` + :returns: a binary I/O stream open for reading. + :rtype: ``typing.io.BinaryIO`` + + +.. function:: importlib.resources.open_text(package, resource, encoding='utf-8', errors='strict') + + Open for text reading the *resource* within *package*. By default, the + resource is opened for reading as UTF-8. + + :param package: A package name or module object. See above for the API + that such module objects must support. + :type package: ``Package`` + :param resource: The name of the resource to open within *package*. + *resource* may not contain path separators and it may + not have sub-resources (i.e. it cannot be a directory). + :type resource: ``Resource`` + :param encoding: The encoding to open the resource in. *encoding* has + the same meaning as with :func:`open`. + :type encoding: str + :param errors: This parameter has the same meaning as with :func:`open`. + :type errors: str + :returns: an I/O stream open for reading. + :rtype: ``typing.TextIO`` + +.. function:: importlib.resources.read_binary(package, resource) + + Read and return the contents of the *resource* within *package* as + ``bytes``. + + :param package: A package name or module object. See above for the API + that such module objects must support. + :type package: ``Package`` + :param resource: The name of the resource to read within *package*. + *resource* may not contain path separators and it may + not have sub-resources (i.e. it cannot be a directory). + :type resource: ``Resource`` + :returns: the contents of the resource. + :rtype: ``bytes`` + +.. function:: importlib.resources.read_text(package, resource, encoding='utf-8', errors='strict') + + Read and return the contents of *resource* within *package* as a ``str``. + By default, the contents are read as strict UTF-8. + + :param package: A package name or module object. See above for the API + that such module objects must support. + :type package: ``Package`` + :param resource: The name of the resource to read within *package*. + *resource* may not contain path separators and it may + not have sub-resources (i.e. it cannot be a directory). + :type resource: ``Resource`` + :param encoding: The encoding to read the contents of the resource in. + *encoding* has the same meaning as with :func:`open`. + :type encoding: str + :param errors: This parameter has the same meaning as with :func:`open`. + :type errors: str + :returns: the contents of the resource. + :rtype: ``str`` + +.. function:: importlib.resources.path(package, resource) + + Return the path to the *resource* as an actual file system path. This + function returns a context manager for use in a :keyword:`with` statement. + The context manager provides a :class:`pathlib.Path` object. + + Exiting the context manager cleans up any temporary file created when the + resource needs to be extracted from e.g. a zip file. + + :param package: A package name or module object. See above for the API + that such module objects must support. + :type package: ``Package`` + :param resource: The name of the resource to read within *package*. + *resource* may not contain path separators and it may + not have sub-resources (i.e. it cannot be a directory). + :type resource: ``Resource`` + :returns: A context manager for use in a :keyword`with` statement. + Entering the context manager provides a :class:`pathlib.Path` + object. + :rtype: context manager providing a :class:`pathlib.Path` object + + +.. function:: importlib.resources.is_resource(package, name) + + Return ``True`` if there is a resource named *name* in the package, + otherwise ``False``. Remember that directories are *not* resources! + + :param package: A package name or module object. See above for the API + that such module objects must support. + :type package: ``Package`` + :param name: The name of the resource to read within *package*. + *resource* may not contain path separators and it may + not have sub-resources (i.e. it cannot be a directory). + :type name: ``str`` + :returns: A flag indicating whether the resource exists or not. + :rtype: ``bool`` + + +.. function:: importlib.resources.contents(package) + + Return an iterator over the contents of the package. The iterator can + return resources (e.g. files) and non-resources (e.g. directories). The + iterator does not recurse into subdirectories. + + :param package: A package name or module object. See above for the API + that such module objects must support. + :type package: ``Package`` + :returns: The contents of the package, both resources and non-resources. + :rtype: An iterator over ``str`` + + :mod:`importlib.machinery` -- Importers and path hooks ------------------------------------------------------ From 871367d32d1e239d1696db277df284db2e2b5087 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Sun, 17 Dec 2017 15:44:11 -0500 Subject: [PATCH 05/14] Add a What's New for importlib.resources --- Doc/whatsnew/3.7.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 3574b53ad18009..867cdea926962e 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -282,7 +282,14 @@ Other Language Changes New Modules =========== -* None yet. +importlib.resource +------------------ + +This module provides several new APIs and one new ABC for access to, opening, +and reading *resources* inside packages. Resources are roughly akin to files +inside of packages, but they needn't be actual files on the physical file +system. Module loaders can implement the +:class:`importlib.abc.ResourceReader` ABC to support this new module's API. Improved Modules From d41bac368765df1a3a491f526d26a5fa4916799c Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Sun, 17 Dec 2017 15:47:49 -0500 Subject: [PATCH 06/14] Clean up --- Doc/library/importlib.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 88ed80e93347a1..3e729e848ec0bd 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -816,7 +816,7 @@ The following types are defined. The following functions are available. -.. function:: importlib.resources.open_binary(package, resource) +.. function:: open_binary(package, resource) Open for binary reading the *resource* within *package*. @@ -831,7 +831,7 @@ The following functions are available. :rtype: ``typing.io.BinaryIO`` -.. function:: importlib.resources.open_text(package, resource, encoding='utf-8', errors='strict') +.. function:: open_text(package, resource, encoding='utf-8', errors='strict') Open for text reading the *resource* within *package*. By default, the resource is opened for reading as UTF-8. @@ -851,7 +851,7 @@ The following functions are available. :returns: an I/O stream open for reading. :rtype: ``typing.TextIO`` -.. function:: importlib.resources.read_binary(package, resource) +.. function:: read_binary(package, resource) Read and return the contents of the *resource* within *package* as ``bytes``. @@ -866,7 +866,7 @@ The following functions are available. :returns: the contents of the resource. :rtype: ``bytes`` -.. function:: importlib.resources.read_text(package, resource, encoding='utf-8', errors='strict') +.. function:: read_text(package, resource, encoding='utf-8', errors='strict') Read and return the contents of *resource* within *package* as a ``str``. By default, the contents are read as strict UTF-8. @@ -886,7 +886,7 @@ The following functions are available. :returns: the contents of the resource. :rtype: ``str`` -.. function:: importlib.resources.path(package, resource) +.. function:: path(package, resource) Return the path to the *resource* as an actual file system path. This function returns a context manager for use in a :keyword:`with` statement. @@ -908,7 +908,7 @@ The following functions are available. :rtype: context manager providing a :class:`pathlib.Path` object -.. function:: importlib.resources.is_resource(package, name) +.. function:: is_resource(package, name) Return ``True`` if there is a resource named *name* in the package, otherwise ``False``. Remember that directories are *not* resources! @@ -924,7 +924,7 @@ The following functions are available. :rtype: ``bool`` -.. function:: importlib.resources.contents(package) +.. function:: contents(package) Return an iterator over the contents of the package. The iterator can return resources (e.g. files) and non-resources (e.g. directories). The From d7ec3391758e514d1905f8b5c9ed635bdb775ab1 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Mon, 18 Dec 2017 13:27:22 -0500 Subject: [PATCH 07/14] Fix two typos --- Doc/library/importlib.rst | 2 +- Doc/whatsnew/3.7.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 3e729e848ec0bd..fe2e0069b7cd2f 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -902,7 +902,7 @@ The following functions are available. *resource* may not contain path separators and it may not have sub-resources (i.e. it cannot be a directory). :type resource: ``Resource`` - :returns: A context manager for use in a :keyword`with` statement. + :returns: A context manager for use in a :keyword:`with` statement. Entering the context manager provides a :class:`pathlib.Path` object. :rtype: context manager providing a :class:`pathlib.Path` object diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 867cdea926962e..6fde8e458c7f91 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -282,8 +282,8 @@ Other Language Changes New Modules =========== -importlib.resource ------------------- +importlib.resources +------------------- This module provides several new APIs and one new ABC for access to, opening, and reading *resources* inside packages. Resources are roughly akin to files From 08859e275d0fc8aa51f990ae8e89fcf77af42908 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Mon, 18 Dec 2017 13:33:43 -0500 Subject: [PATCH 08/14] Fix another typo from a previous branch --- Doc/library/importlib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index fe2e0069b7cd2f..ff2c5cd1b7f253 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -484,7 +484,7 @@ ABC hierarchy:: versus on the file system. For any of methods of this class, a *resource* argument is - expected to be a :term:`file-like object` which represents + expected to be a :term:`path-like object` which represents conceptually just a file name. This means that no subdirectory paths should be included in the *resource* argument. This is because the location of the package that the loader is for acts From 05353ed5fe55402ae49e41cba1008fe9c18a3d44 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Tue, 26 Dec 2017 22:27:06 -0500 Subject: [PATCH 09/14] Respond to review --- Doc/library/importlib.rst | 113 +++++++++++++------------------------ Lib/importlib/resources.py | 17 ++++-- 2 files changed, 51 insertions(+), 79 deletions(-) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index ff2c5cd1b7f253..e6b02e95ca9dda 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -820,15 +820,11 @@ The following functions are available. Open for binary reading the *resource* within *package*. - :param package: A package name or module object. See above for the API - that such module objects must support. - :type package: ``Package`` - :param resource: The name of the resource to open within *package*. - *resource* may not contain path separators and it may - not have sub-resources (i.e. it cannot be a directory). - :type resource: ``Resource`` - :returns: a binary I/O stream open for reading. - :rtype: ``typing.io.BinaryIO`` + *package* is either a name or a module object which conforms to the + :class:`Package` requirements. *resource* is the name of the resource to + open within *package*; it may not contain path separators and it may not + have sub-resources (i.e. it cannot be a directory). This function returns + a ``typing.BinaryIO`` instance, a binary I/O stream open for reading. .. function:: open_text(package, resource, encoding='utf-8', errors='strict') @@ -836,55 +832,40 @@ The following functions are available. Open for text reading the *resource* within *package*. By default, the resource is opened for reading as UTF-8. - :param package: A package name or module object. See above for the API - that such module objects must support. - :type package: ``Package`` - :param resource: The name of the resource to open within *package*. - *resource* may not contain path separators and it may - not have sub-resources (i.e. it cannot be a directory). - :type resource: ``Resource`` - :param encoding: The encoding to open the resource in. *encoding* has - the same meaning as with :func:`open`. - :type encoding: str - :param errors: This parameter has the same meaning as with :func:`open`. - :type errors: str - :returns: an I/O stream open for reading. - :rtype: ``typing.TextIO`` + *package* is either a name or a module object which conforms to the + :class:`Package` requirements. *resource* is the name of the resource to + open within *package*; it may not contain path separators and it may not + have sub-resources (i.e. it cannot be a directory). *encoding* and + *errors* have the same meaning as with built-in :func:`open`. + + This function returns a ``typing.TextIO`` instance, a text I/O stream open + for reading. + .. function:: read_binary(package, resource) Read and return the contents of the *resource* within *package* as ``bytes``. - :param package: A package name or module object. See above for the API - that such module objects must support. - :type package: ``Package`` - :param resource: The name of the resource to read within *package*. - *resource* may not contain path separators and it may - not have sub-resources (i.e. it cannot be a directory). - :type resource: ``Resource`` - :returns: the contents of the resource. - :rtype: ``bytes`` + *package* is either a name or a module object which conforms to the + :class:`Package` requirements. *resource* is the name of the resource to + open within *package*; it may not contain path separators and it may not + have sub-resources (i.e. it cannot be a directory). This function returns + the contents of the resource as :class:`bytes`. + .. function:: read_text(package, resource, encoding='utf-8', errors='strict') Read and return the contents of *resource* within *package* as a ``str``. By default, the contents are read as strict UTF-8. - :param package: A package name or module object. See above for the API - that such module objects must support. - :type package: ``Package`` - :param resource: The name of the resource to read within *package*. - *resource* may not contain path separators and it may - not have sub-resources (i.e. it cannot be a directory). - :type resource: ``Resource`` - :param encoding: The encoding to read the contents of the resource in. - *encoding* has the same meaning as with :func:`open`. - :type encoding: str - :param errors: This parameter has the same meaning as with :func:`open`. - :type errors: str - :returns: the contents of the resource. - :rtype: ``str`` + *package* is either a name or a module object which conforms to the + :class:`Package` requirements. *resource* is the name of the resource to + open within *package*; it may not contain path separators and it may not + have sub-resources (i.e. it cannot be a directory). *encoding* and + *errors* have the same meaning as with built-in :func:`open`. This + function returns the contents of the resource as :class:`str`. + .. function:: path(package, resource) @@ -895,46 +876,28 @@ The following functions are available. Exiting the context manager cleans up any temporary file created when the resource needs to be extracted from e.g. a zip file. - :param package: A package name or module object. See above for the API - that such module objects must support. - :type package: ``Package`` - :param resource: The name of the resource to read within *package*. - *resource* may not contain path separators and it may - not have sub-resources (i.e. it cannot be a directory). - :type resource: ``Resource`` - :returns: A context manager for use in a :keyword:`with` statement. - Entering the context manager provides a :class:`pathlib.Path` - object. - :rtype: context manager providing a :class:`pathlib.Path` object + *package* is either a name or a module object which conforms to the + :class:`Package` requirements. *resource* is the name of the resource to + open within *package*; it may not contain path separators and it may not + have sub-resources (i.e. it cannot be a directory). .. function:: is_resource(package, name) Return ``True`` if there is a resource named *name* in the package, otherwise ``False``. Remember that directories are *not* resources! - - :param package: A package name or module object. See above for the API - that such module objects must support. - :type package: ``Package`` - :param name: The name of the resource to read within *package*. - *resource* may not contain path separators and it may - not have sub-resources (i.e. it cannot be a directory). - :type name: ``str`` - :returns: A flag indicating whether the resource exists or not. - :rtype: ``bool`` + *package* is either a name or a module object which conforms to the + :class:`Package` requirements. .. function:: contents(package) - Return an iterator over the contents of the package. The iterator can - return resources (e.g. files) and non-resources (e.g. directories). The - iterator does not recurse into subdirectories. + Return an iterator over the named items within the package. The iterator + returns :class:`str` resources (e.g. files) and non-resources + (e.g. directories). The iterator does not recurse into subdirectories. - :param package: A package name or module object. See above for the API - that such module objects must support. - :type package: ``Package`` - :returns: The contents of the package, both resources and non-resources. - :rtype: An iterator over ``str`` + *package* is either a name or a module object which conforms to the + :class:`Package` requirements. :mod:`importlib.machinery` -- Importers and path hooks diff --git a/Lib/importlib/resources.py b/Lib/importlib/resources.py index 3a637f98208a8b..6eac84c31b8f99 100644 --- a/Lib/importlib/resources.py +++ b/Lib/importlib/resources.py @@ -20,6 +20,11 @@ def _get_package(package) -> ModuleType: + """Take a package name or module object and return the module. + + If a name, the module is imported. If the passed or imported module + object is not a package, raise an exception. + """ if hasattr(package, '__spec__'): if package.__spec__.submodule_search_locations is None: raise TypeError('{!r} is not a package'.format( @@ -35,6 +40,10 @@ def _get_package(package) -> ModuleType: def _normalize_path(path) -> str: + """Normalize a path by ensuring it is a string. + + If the resulting string contains path separators, an exception is raised. + """ str_path = str(path) parent, file_name = os.path.split(str_path) if parent: @@ -69,14 +78,14 @@ def open_binary(package: Package, resource: Resource) -> BinaryIO: full_path = os.path.join(package_path, resource) try: return builtins_open(full_path, mode='rb') - except IOError: + except OSError: # Just assume the loader is a resource loader; all the relevant # importlib.machinery loaders are and an AttributeError for # get_data() will make it clear what is needed from the loader. loader = cast(ResourceLoader, package.__spec__.loader) data = None if hasattr(package.__spec__.loader, 'get_data'): - with suppress(IOError): + with suppress(OSError): data = loader.get_data(full_path) if data is None: package_name = package.__spec__.name @@ -105,14 +114,14 @@ def open_text(package: Package, try: return builtins_open( full_path, mode='r', encoding=encoding, errors=errors) - except IOError: + except OSError: # Just assume the loader is a resource loader; all the relevant # importlib.machinery loaders are and an AttributeError for # get_data() will make it clear what is needed from the loader. loader = cast(ResourceLoader, package.__spec__.loader) data = None if hasattr(package.__spec__.loader, 'get_data'): - with suppress(IOError): + with suppress(OSError): data = loader.get_data(full_path) if data is None: package_name = package.__spec__.name From e143941a10d64285ff0bfed2812016509523efaa Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Thu, 28 Dec 2017 10:17:26 -0500 Subject: [PATCH 10/14] Use data:: for types as per typing.rst --- Doc/library/importlib.rst | 50 +++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index e6b02e95ca9dda..e99c6067a3d69c 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -801,14 +801,14 @@ abstract base class. The following types are defined. -.. class:: Package +.. data:: Package - ``Package`` types are defined as ``Union[str, ModuleType]``. This means + The ``Package`` type is defined as ``Union[str, ModuleType]``. This means that where the function describes accepting a ``Package``, you can pass in either a string or a module. Module objects must have a resolvable ``__spec__.submodule_search_locations`` that is not ``None``. -.. class:: Resource +.. data:: Resource This type describes the resource names passed into the various functions in this package. This is defined as ``Union[str, os.PathLike]``. @@ -821,10 +821,10 @@ The following functions are available. Open for binary reading the *resource* within *package*. *package* is either a name or a module object which conforms to the - :class:`Package` requirements. *resource* is the name of the resource to - open within *package*; it may not contain path separators and it may not - have sub-resources (i.e. it cannot be a directory). This function returns - a ``typing.BinaryIO`` instance, a binary I/O stream open for reading. + ``Package`` requirements. *resource* is the name of the resource to open + within *package*; it may not contain path separators and it may not have + sub-resources (i.e. it cannot be a directory). This function returns a + ``typing.BinaryIO`` instance, a binary I/O stream open for reading. .. function:: open_text(package, resource, encoding='utf-8', errors='strict') @@ -833,10 +833,10 @@ The following functions are available. resource is opened for reading as UTF-8. *package* is either a name or a module object which conforms to the - :class:`Package` requirements. *resource* is the name of the resource to - open within *package*; it may not contain path separators and it may not - have sub-resources (i.e. it cannot be a directory). *encoding* and - *errors* have the same meaning as with built-in :func:`open`. + ``Package`` requirements. *resource* is the name of the resource to open + within *package*; it may not contain path separators and it may not have + sub-resources (i.e. it cannot be a directory). *encoding* and *errors* + have the same meaning as with built-in :func:`open`. This function returns a ``typing.TextIO`` instance, a text I/O stream open for reading. @@ -848,10 +848,10 @@ The following functions are available. ``bytes``. *package* is either a name or a module object which conforms to the - :class:`Package` requirements. *resource* is the name of the resource to - open within *package*; it may not contain path separators and it may not - have sub-resources (i.e. it cannot be a directory). This function returns - the contents of the resource as :class:`bytes`. + ``Package`` requirements. *resource* is the name of the resource to open + within *package*; it may not contain path separators and it may not have + sub-resources (i.e. it cannot be a directory). This function returns the + contents of the resource as :class:`bytes`. .. function:: read_text(package, resource, encoding='utf-8', errors='strict') @@ -860,11 +860,11 @@ The following functions are available. By default, the contents are read as strict UTF-8. *package* is either a name or a module object which conforms to the - :class:`Package` requirements. *resource* is the name of the resource to - open within *package*; it may not contain path separators and it may not - have sub-resources (i.e. it cannot be a directory). *encoding* and - *errors* have the same meaning as with built-in :func:`open`. This - function returns the contents of the resource as :class:`str`. + ``Package`` requirements. *resource* is the name of the resource to open + within *package*; it may not contain path separators and it may not have + sub-resources (i.e. it cannot be a directory). *encoding* and *errors* + have the same meaning as with built-in :func:`open`. This function + returns the contents of the resource as :class:`str`. .. function:: path(package, resource) @@ -877,9 +877,9 @@ The following functions are available. resource needs to be extracted from e.g. a zip file. *package* is either a name or a module object which conforms to the - :class:`Package` requirements. *resource* is the name of the resource to - open within *package*; it may not contain path separators and it may not - have sub-resources (i.e. it cannot be a directory). + ``Package`` requirements. *resource* is the name of the resource to open + within *package*; it may not contain path separators and it may not have + sub-resources (i.e. it cannot be a directory). .. function:: is_resource(package, name) @@ -887,7 +887,7 @@ The following functions are available. Return ``True`` if there is a resource named *name* in the package, otherwise ``False``. Remember that directories are *not* resources! *package* is either a name or a module object which conforms to the - :class:`Package` requirements. + ``Package`` requirements. .. function:: contents(package) @@ -897,7 +897,7 @@ The following functions are available. (e.g. directories). The iterator does not recurse into subdirectories. *package* is either a name or a module object which conforms to the - :class:`Package` requirements. + ``Package`` requirements. :mod:`importlib.machinery` -- Importers and path hooks From 360784621da2a903aa8e7d5114469782b69d118c Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Fri, 29 Dec 2017 10:56:55 -0500 Subject: [PATCH 11/14] This is 3.7 so we can use Path.resolve() --- Lib/importlib/resources.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/Lib/importlib/resources.py b/Lib/importlib/resources.py index 6eac84c31b8f99..1f273652e48c2f 100644 --- a/Lib/importlib/resources.py +++ b/Lib/importlib/resources.py @@ -71,11 +71,9 @@ def open_binary(package: Package, resource: Resource) -> BinaryIO: reader = _get_resource_reader(package) if reader is not None: return reader.open_resource(resource) - # Using pathlib doesn't work well here due to the lack of 'strict' - # argument for pathlib.Path.resolve() prior to Python 3.6. - absolute_package_path = os.path.abspath(package.__spec__.origin) - package_path = os.path.dirname(absolute_package_path) - full_path = os.path.join(package_path, resource) + absolute_package_path = Path(package.__spec__.origin).resolve() + package_path = absolute_package_path.parent + full_path = package_path / resource try: return builtins_open(full_path, mode='rb') except OSError: @@ -86,7 +84,7 @@ def open_binary(package: Package, resource: Resource) -> BinaryIO: data = None if hasattr(package.__spec__.loader, 'get_data'): with suppress(OSError): - data = loader.get_data(full_path) + data = loader.get_data(str(full_path)) if data is None: package_name = package.__spec__.name message = '{!r} resource not found in {!r}'.format( @@ -106,11 +104,9 @@ def open_text(package: Package, reader = _get_resource_reader(package) if reader is not None: return TextIOWrapper(reader.open_resource(resource), encoding, errors) - # Using pathlib doesn't work well here due to the lack of 'strict' - # argument for pathlib.Path.resolve() prior to Python 3.6. - absolute_package_path = os.path.abspath(package.__spec__.origin) - package_path = os.path.dirname(absolute_package_path) - full_path = os.path.join(package_path, resource) + absolute_package_path = Path(package.__spec__.origin).resolve() + package_path = absolute_package_path.parent + full_path = package_path / resource try: return builtins_open( full_path, mode='r', encoding=encoding, errors=errors) @@ -122,7 +118,7 @@ def open_text(package: Package, data = None if hasattr(package.__spec__.loader, 'get_data'): with suppress(OSError): - data = loader.get_data(full_path) + data = loader.get_data(str(full_path)) if data is None: package_name = package.__spec__.name message = '{!r} resource not found in {!r}'.format( From 1205e2a0f7808699996f9792d5d09f3ae652806a Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Fri, 29 Dec 2017 17:30:15 -0500 Subject: [PATCH 12/14] A couple of last minute review comments --- Lib/importlib/resources.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/importlib/resources.py b/Lib/importlib/resources.py index 1f273652e48c2f..5943d691800539 100644 --- a/Lib/importlib/resources.py +++ b/Lib/importlib/resources.py @@ -195,7 +195,7 @@ def path(package: Package, resource: Resource) -> Iterator[Path]: def is_resource(package: Package, name: str) -> bool: - """True if `name` is a resource inside `package`. + """True if 'name' is a resource inside 'package'. Directories are *not* resources. """ @@ -228,7 +228,7 @@ def is_resource(package: Package, name: str) -> bool: toc = zf.namelist() relpath = package_directory.relative_to(archive_path) candidate_path = relpath / name - for entry in toc: # pragma: nobranch + for entry in toc: try: relative_to_candidate = Path(entry).relative_to(candidate_path) except ValueError: @@ -249,7 +249,7 @@ def is_resource(package: Package, name: str) -> bool: def contents(package: Package) -> Iterator[str]: - """Return the list of entries in `package`. + """Return the list of entries in 'package'. Note that not all entries are resources. Specifically, directories are not considered resources. Use `is_resource()` on each entry returned here @@ -296,7 +296,7 @@ def contents(package: Package) -> Iterator[str]: subparts = path.parts[len(relpath.parts):] if len(subparts) == 1: yield subparts[0] - elif len(subparts) > 1: # pragma: nobranch + elif len(subparts) > 1: subdir = subparts[0] if subdir not in subdirs_seen: subdirs_seen.add(subdir) From 1a68c681da31a9e15c6120a9b803b6cb92a96a7f Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Sat, 30 Dec 2017 10:52:14 -0500 Subject: [PATCH 13/14] Revert "This is 3.7 so we can use Path.resolve()" This reverts commit 360784621da2a903aa8e7d5114469782b69d118c. Let's see if this is the cause of the Windows failures. --- Lib/importlib/resources.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/Lib/importlib/resources.py b/Lib/importlib/resources.py index 5943d691800539..31be6c4c51c2cd 100644 --- a/Lib/importlib/resources.py +++ b/Lib/importlib/resources.py @@ -71,9 +71,11 @@ def open_binary(package: Package, resource: Resource) -> BinaryIO: reader = _get_resource_reader(package) if reader is not None: return reader.open_resource(resource) - absolute_package_path = Path(package.__spec__.origin).resolve() - package_path = absolute_package_path.parent - full_path = package_path / resource + # Using pathlib doesn't work well here due to the lack of 'strict' + # argument for pathlib.Path.resolve() prior to Python 3.6. + absolute_package_path = os.path.abspath(package.__spec__.origin) + package_path = os.path.dirname(absolute_package_path) + full_path = os.path.join(package_path, resource) try: return builtins_open(full_path, mode='rb') except OSError: @@ -84,7 +86,7 @@ def open_binary(package: Package, resource: Resource) -> BinaryIO: data = None if hasattr(package.__spec__.loader, 'get_data'): with suppress(OSError): - data = loader.get_data(str(full_path)) + data = loader.get_data(full_path) if data is None: package_name = package.__spec__.name message = '{!r} resource not found in {!r}'.format( @@ -104,9 +106,11 @@ def open_text(package: Package, reader = _get_resource_reader(package) if reader is not None: return TextIOWrapper(reader.open_resource(resource), encoding, errors) - absolute_package_path = Path(package.__spec__.origin).resolve() - package_path = absolute_package_path.parent - full_path = package_path / resource + # Using pathlib doesn't work well here due to the lack of 'strict' + # argument for pathlib.Path.resolve() prior to Python 3.6. + absolute_package_path = os.path.abspath(package.__spec__.origin) + package_path = os.path.dirname(absolute_package_path) + full_path = os.path.join(package_path, resource) try: return builtins_open( full_path, mode='r', encoding=encoding, errors=errors) @@ -118,7 +122,7 @@ def open_text(package: Package, data = None if hasattr(package.__spec__.loader, 'get_data'): with suppress(OSError): - data = loader.get_data(str(full_path)) + data = loader.get_data(full_path) if data is None: package_name = package.__spec__.name message = '{!r} resource not found in {!r}'.format( From 535da74df982bb2a702bb699c8553bef9c61a573 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Sat, 30 Dec 2017 13:37:39 -0500 Subject: [PATCH 14/14] Remove some obsolete comments --- Lib/importlib/resources.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Lib/importlib/resources.py b/Lib/importlib/resources.py index 31be6c4c51c2cd..8511f24d8e7c63 100644 --- a/Lib/importlib/resources.py +++ b/Lib/importlib/resources.py @@ -71,8 +71,6 @@ def open_binary(package: Package, resource: Resource) -> BinaryIO: reader = _get_resource_reader(package) if reader is not None: return reader.open_resource(resource) - # Using pathlib doesn't work well here due to the lack of 'strict' - # argument for pathlib.Path.resolve() prior to Python 3.6. absolute_package_path = os.path.abspath(package.__spec__.origin) package_path = os.path.dirname(absolute_package_path) full_path = os.path.join(package_path, resource) @@ -106,8 +104,6 @@ def open_text(package: Package, reader = _get_resource_reader(package) if reader is not None: return TextIOWrapper(reader.open_resource(resource), encoding, errors) - # Using pathlib doesn't work well here due to the lack of 'strict' - # argument for pathlib.Path.resolve() prior to Python 3.6. absolute_package_path = os.path.abspath(package.__spec__.origin) package_path = os.path.dirname(absolute_package_path) full_path = os.path.join(package_path, resource)