Merge pull request #99 from mrocklin/lock-it-all

Add file based locks

Author: martindurant

continuous_integration/Dockerfile

@@ -9,10 +9,10 @@ ENV PATH /opt/conda/bin:$PATH
 
 # hdfs3 - python
 ENV LIBHDFS3_CONF /etc/hadoop/conf/hdfs-site.xml
-RUN conda install -y -q ipython pytest
+RUN conda install -y -q ipython pytest locket
 RUN conda install -y -q libhdfs3 -c conda-forge
 RUN conda create -y -n py3 python=3
-RUN conda install -y -n py3 ipython pytest
+RUN conda install -y -n py3 ipython pytest locket
 RUN conda install -y -n py3 libhdfs3 -c conda-forge
 
 # Cloudera repositories

docs/source/limitations.rst

@@ -6,24 +6,6 @@ Forked processes
 
 The ``libhdfs3`` library may fail when an ``HDFileSystem`` is copied to a new
 forked process.  This happens in some cases when using ``hdfs3`` with
-``multiprocessing`` in Python 2.  Common solutions include the following:
+``multiprocessing`` in Python 2.
 
-*  Use threads
-*  Use Python 3 and a multiprocessing context using spawn with
-   ``multiprocessing.get_context(method='spawn')`` see `multiprocessing docs`_
-*  Only instantiate ``HDFileSystem`` within the forked processes, do not start
-   an ``HDFileSystem`` within the parent processes or do not use that
-   ``HDFileSystem`` within the child processes.
-*  Use a file based lock.  We recommend ``locket``::
-
-        $ pip install locket
-
-    .. code-block:: python
-
-        import locket
-
-        with locket.lock_file('.lock'):
-            # do hdfs3 work
-
-
-.. _`multiprocessing docs`: https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods
+We get around this by using file-based locks, which slightly limit concurrency.