Do not swallow PicklingError

joblib/_store_backends.py

@@ -1,5 +1,6 @@
 """Storage providers backends for Memory caching."""
 
+from pickle import PicklingError
 import re
 import os
 import os.path
@@ -185,12 +186,24 @@ def dump_item(self, path, item, verbose=1):
 
             def write_func(to_write, dest_filename):
                 with self._open_item(dest_filename, "wb") as f:
-                    numpy_pickle.dump(to_write, f,
-                                      compress=self.compress)
+                    try:
+                        numpy_pickle.dump(to_write, f, compress=self.compress)
+                    except PicklingError as e:
+                        warnings.warn(
+                            'Unable to cache to disk: '
+                            'failed to pickle output. '
+                            'In future versions of joblib '
+                            'this will raise an exception.'
+                            'Exception: %s.' % e,
+                            FutureWarning
+                        )
 
             self._concurrency_safe_write(item, filename, write_func)
-        except:  # noqa: E722
-            " Race condition in the creation of the directory "
+        except Exception as e:  # noqa: E722
+            warnings.warn(
+                'Unable to cache to disk. '
+                'Possibly a race condition in the creation of the directory. '
+                'Exception: %s' % e)
 
     def clear_item(self, path):
         """Clear the item at the path, given as a list of strings."""

joblib/test/test_store_backends.py

@@ -1,3 +1,7 @@
+import warnings
+from pickle import PicklingError
+from unittest.mock import MagicMock
+
 try:
     # Python 2.7: use the C pickle to speed up
     # test_concurrency_safe_write which pickles big python objects
@@ -10,8 +14,9 @@
 from joblib.testing import parametrize, timeout
 from joblib.test.common import with_multiprocessing
 from joblib.backports import concurrency_safe_rename
-from joblib import Parallel, delayed
-from joblib._store_backends import concurrency_safe_write
+from joblib import Parallel, delayed, numpy_pickle
+from joblib._store_backends import concurrency_safe_write, \
+    FileSystemStoreBackend
 
 
 def write_func(output, filename):
@@ -54,3 +59,41 @@ def test_concurrency_safe_write(tmpdir, backend):
              if i % 3 != 2 else load_func for i in range(12)]
     Parallel(n_jobs=2, backend=backend)(
         delayed(func)(obj, filename) for func in funcs)
+
+
+@with_multiprocessing
+def test_warning_on_dump_failure(tmpdir, monkeypatch):
+    backend = FileSystemStoreBackend()
+    backend.location = tmpdir.join('test_warning_on_pickling_error').strpath
+    backend.compress = None
+
+    def monkeypatched_pickle_dump(*args, **kwargs):
+        raise Exception()
+
+    warnings_mock = MagicMock()
+    monkeypatch.setattr(numpy_pickle, "dump", monkeypatched_pickle_dump)
+    monkeypatch.setattr(warnings, "warn", warnings_mock)
+
+    backend.dump_item("testpath", "testitem")
+
+    warnings_mock.assert_called_once()
+
+
+@with_multiprocessing
+def test_warning_on_pickling_error(tmpdir, monkeypatch):
+    # This is separate from test_warning_on_dump_failure because in the
+    # future we will turn this into an exception.
+    backend = FileSystemStoreBackend()
+    backend.location = tmpdir.join('test_warning_on_pickling_error').strpath
+    backend.compress = None
+
+    def monkeypatched_pickle_dump(*args, **kwargs):
+        raise PicklingError()
+
+    warnings_mock = MagicMock()
+    monkeypatch.setattr(numpy_pickle, "dump", monkeypatched_pickle_dump)
+    monkeypatch.setattr(warnings, "warn", warnings_mock)
+
+    backend.dump_item("testpath", "testitem")
+
+    warnings_mock.assert_called_once()