""" Utilities for file-based Contents/Checkpoints managers. """ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. from contextlib import contextmanager import errno import io import os import shutil from tornado.web import HTTPError from notebook.utils import ( to_api_path, to_os_path, ) import nbformat from ipython_genutils.py3compat import str_to_unicode from traitlets.config import Configurable from traitlets import Bool from base64 import encodebytes, decodebytes def replace_file(src, dst): """ replace dst with src switches between os.replace or os.rename based on python 2.7 or python 3 """ if hasattr(os, 'replace'): # PY3 os.replace(src, dst) else: if os.name == 'nt' and os.path.exists(dst): # Rename over existing file doesn't work on Windows os.remove(dst) os.rename(src, dst) def copy2_safe(src, dst, log=None): """copy src to dst like shutil.copy2, but log errors in copystat instead of raising """ shutil.copyfile(src, dst) try: shutil.copystat(src, dst) except OSError: if log: log.debug("copystat on %s failed", dst, exc_info=True) def path_to_intermediate(path): '''Name of the intermediate file used in atomic writes. The .~ prefix will make Dropbox ignore the temporary file.''' dirname, basename = os.path.split(path) return os.path.join(dirname, '.~'+basename) def path_to_invalid(path): '''Name of invalid file after a failed atomic write and subsequent read.''' dirname, basename = os.path.split(path) return os.path.join(dirname, basename+'.invalid') @contextmanager def atomic_writing(path, text=True, encoding='utf-8', log=None, **kwargs): """Context manager to write to a file only if the entire write is successful. This works by copying the previous file contents to a temporary file in the same directory, and renaming that file back to the target if the context exits with an error. If the context is successful, the new data is synced to disk and the temporary file is removed. Parameters ---------- path : str The target file to write to. text : bool, optional Whether to open the file in text mode (i.e. to write unicode). Default is True. encoding : str, optional The encoding to use for files opened in text mode. Default is UTF-8. **kwargs Passed to :func:`io.open`. """ # realpath doesn't work on Windows: https://bugs.python.org/issue9949 # Luckily, we only need to resolve the file itself being a symlink, not # any of its directories, so this will suffice: if os.path.islink(path): path = os.path.join(os.path.dirname(path), os.readlink(path)) tmp_path = path_to_intermediate(path) if os.path.isfile(path): copy2_safe(path, tmp_path, log=log) if text: # Make sure that text files have Unix linefeeds by default kwargs.setdefault('newline', '\n') fileobj = io.open(path, 'w', encoding=encoding, **kwargs) else: fileobj = io.open(path, 'wb', **kwargs) try: yield fileobj except: # Failed! Move the backup file back to the real path to avoid corruption fileobj.close() replace_file(tmp_path, path) raise # Flush to disk fileobj.flush() os.fsync(fileobj.fileno()) fileobj.close() # Written successfully, now remove the backup copy if os.path.isfile(tmp_path): os.remove(tmp_path) @contextmanager def _simple_writing(path, text=True, encoding='utf-8', log=None, **kwargs): """Context manager to write file without doing atomic writing ( for weird filesystem eg: nfs). Parameters ---------- path : str The target file to write to. text : bool, optional Whether to open the file in text mode (i.e. to write unicode). Default is True. encoding : str, optional The encoding to use for files opened in text mode. Default is UTF-8. **kwargs Passed to :func:`io.open`. """ # realpath doesn't work on Windows: https://bugs.python.org/issue9949 # Luckily, we only need to resolve the file itself being a symlink, not # any of its directories, so this will suffice: if os.path.islink(path): path = os.path.join(os.path.dirname(path), os.readlink(path)) if text: # Make sure that text files have Unix linefeeds by default kwargs.setdefault('newline', '\n') fileobj = io.open(path, 'w', encoding=encoding, **kwargs) else: fileobj = io.open(path, 'wb', **kwargs) try: yield fileobj except: fileobj.close() raise fileobj.close() class FileManagerMixin(Configurable): """ Mixin for ContentsAPI classes that interact with the filesystem. Provides facilities for reading, writing, and copying both notebooks and generic files. Shared by FileContentsManager and FileCheckpoints. Note ---- Classes using this mixin must provide the following attributes: root_dir : unicode A directory against against which API-style paths are to be resolved. log : logging.Logger """ use_atomic_writing = Bool(True, config=True, help= """By default notebooks are saved on disk on a temporary file and then if successfully written, it replaces the old ones. This procedure, namely 'atomic_writing', causes some bugs on file system without operation order enforcement (like some networked fs). If set to False, the new notebook is written directly on the old one which could fail (eg: full filesystem or quota )""") @contextmanager def open(self, os_path, *args, **kwargs): """wrapper around io.open that turns permission errors into 403""" with self.perm_to_403(os_path): with io.open(os_path, *args, **kwargs) as f: yield f @contextmanager def atomic_writing(self, os_path, *args, **kwargs): """wrapper around atomic_writing that turns permission errors to 403. Depending on flag 'use_atomic_writing', the wrapper perform an actual atomic writing or simply writes the file (whatever an old exists or not)""" with self.perm_to_403(os_path): if self.use_atomic_writing: with atomic_writing(os_path, *args, log=self.log, **kwargs) as f: yield f else: with _simple_writing(os_path, *args, log=self.log, **kwargs) as f: yield f @contextmanager def perm_to_403(self, os_path=''): """context manager for turning permission errors into 403.""" try: yield except (OSError, IOError) as e: if e.errno in {errno.EPERM, errno.EACCES}: # make 403 error message without root prefix # this may not work perfectly on unicode paths on Python 2, # but nobody should be doing that anyway. if not os_path: os_path = str_to_unicode(e.filename or 'unknown file') path = to_api_path(os_path, root=self.root_dir) raise HTTPError(403, u'Permission denied: %s' % path) else: raise def _copy(self, src, dest): """copy src to dest like shutil.copy2, but log errors in copystat """ copy2_safe(src, dest, log=self.log) def _get_os_path(self, path): """Given an API path, return its file system path. Parameters ---------- path : string The relative API path to the named file. Returns ------- path : string Native, absolute OS path to for a file. Raises ------ 404: if path is outside root """ root = os.path.abspath(self.root_dir) os_path = to_os_path(path, root) if not (os.path.abspath(os_path) + os.path.sep).startswith(root): raise HTTPError(404, "%s is outside root contents directory" % path) return os_path def _read_notebook(self, os_path, as_version=4): """Read a notebook from an os path.""" with self.open(os_path, 'r', encoding='utf-8') as f: try: return nbformat.read(f, as_version=as_version) except Exception as e: e_orig = e # If use_atomic_writing is enabled, we'll guess that it was also # enabled when this notebook was written and look for a valid # atomic intermediate. tmp_path = path_to_intermediate(os_path) if not self.use_atomic_writing or not os.path.exists(tmp_path): raise HTTPError( 400, u"Unreadable Notebook: %s %r" % (os_path, e_orig), ) # Move the bad file aside, restore the intermediate, and try again. invalid_file = path_to_invalid(os_path) replace_file(os_path, invalid_file) replace_file(tmp_path, os_path) return self._read_notebook(os_path, as_version) def _save_notebook(self, os_path, nb): """Save a notebook to an os_path.""" with self.atomic_writing(os_path, encoding='utf-8') as f: nbformat.write(nb, f, version=nbformat.NO_CONVERT) def _read_file(self, os_path, format): """Read a non-notebook file. os_path: The path to be read. format: If 'text', the contents will be decoded as UTF-8. If 'base64', the raw bytes contents will be encoded as base64. If not specified, try to decode as UTF-8, and fall back to base64 """ if not os.path.isfile(os_path): raise HTTPError(400, "Cannot read non-file %s" % os_path) with self.open(os_path, 'rb') as f: bcontent = f.read() if format is None or format == 'text': # Try to interpret as unicode if format is unknown or if unicode # was explicitly requested. try: return bcontent.decode('utf8'), 'text' except UnicodeError: if format == 'text': raise HTTPError( 400, "%s is not UTF-8 encoded" % os_path, reason='bad format', ) return encodebytes(bcontent).decode('ascii'), 'base64' def _save_file(self, os_path, content, format): """Save content of a generic file.""" if format not in {'text', 'base64'}: raise HTTPError( 400, "Must specify format of file contents as 'text' or 'base64'", ) try: if format == 'text': bcontent = content.encode('utf8') else: b64_bytes = content.encode('ascii') bcontent = decodebytes(b64_bytes) except Exception as e: raise HTTPError( 400, u'Encoding error saving %s: %s' % (os_path, e) ) with self.atomic_writing(os_path, text=False) as f: f.write(bcontent)