atomicwrites.py 6.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236
  1. """
  2. Copy-pasted from
  3. https://github.com/untitaker/python-atomicwrites
  4. """
  5. import contextlib
  6. import io
  7. import os
  8. import sys
  9. import tempfile
  10. try:
  11. import fcntl
  12. except ImportError:
  13. fcntl = None
  14. # `fspath` was added in Python 3.6
  15. try:
  16. from os import fspath
  17. except ImportError:
  18. fspath = None
  19. __version__ = '1.4.1'
  20. PY2 = sys.version_info[0] == 2
  21. text_type = unicode if PY2 else str # noqa
  22. def _path_to_unicode(x):
  23. if not isinstance(x, text_type):
  24. return x.decode(sys.getfilesystemencoding())
  25. return x
  26. DEFAULT_MODE = "wb" if PY2 else "w"
  27. _proper_fsync = os.fsync
  28. if sys.platform != 'win32':
  29. if hasattr(fcntl, 'F_FULLFSYNC'):
  30. def _proper_fsync(fd):
  31. # https://lists.apple.com/archives/darwin-dev/2005/Feb/msg00072.html
  32. # https://developer.apple.com/library/mac/documentation/Darwin/Reference/ManPages/man2/fsync.2.html
  33. # https://github.com/untitaker/python-atomicwrites/issues/6
  34. fcntl.fcntl(fd, fcntl.F_FULLFSYNC)
  35. def _sync_directory(directory):
  36. # Ensure that filenames are written to disk
  37. fd = os.open(directory, 0)
  38. try:
  39. _proper_fsync(fd)
  40. finally:
  41. os.close(fd)
  42. def _replace_atomic(src, dst):
  43. os.rename(src, dst)
  44. _sync_directory(os.path.normpath(os.path.dirname(dst)))
  45. def _move_atomic(src, dst):
  46. os.link(src, dst)
  47. os.unlink(src)
  48. src_dir = os.path.normpath(os.path.dirname(src))
  49. dst_dir = os.path.normpath(os.path.dirname(dst))
  50. _sync_directory(dst_dir)
  51. if src_dir != dst_dir:
  52. _sync_directory(src_dir)
  53. else:
  54. from ctypes import windll, WinError
  55. _MOVEFILE_REPLACE_EXISTING = 0x1
  56. _MOVEFILE_WRITE_THROUGH = 0x8
  57. _windows_default_flags = _MOVEFILE_WRITE_THROUGH
  58. def _handle_errors(rv):
  59. if not rv:
  60. raise WinError()
  61. def _replace_atomic(src, dst):
  62. _handle_errors(windll.kernel32.MoveFileExW(
  63. _path_to_unicode(src), _path_to_unicode(dst),
  64. _windows_default_flags | _MOVEFILE_REPLACE_EXISTING
  65. ))
  66. def _move_atomic(src, dst):
  67. _handle_errors(windll.kernel32.MoveFileExW(
  68. _path_to_unicode(src), _path_to_unicode(dst),
  69. _windows_default_flags
  70. ))
  71. def replace_atomic(src, dst):
  72. '''
  73. Move ``src`` to ``dst``. If ``dst`` exists, it will be silently
  74. overwritten.
  75. Both paths must reside on the same filesystem for the operation to be
  76. atomic.
  77. '''
  78. return _replace_atomic(src, dst)
  79. def move_atomic(src, dst):
  80. '''
  81. Move ``src`` to ``dst``. There might a timewindow where both filesystem
  82. entries exist. If ``dst`` already exists, :py:exc:`FileExistsError` will be
  83. raised.
  84. Both paths must reside on the same filesystem for the operation to be
  85. atomic.
  86. '''
  87. return _move_atomic(src, dst)
  88. class AtomicWriter(object):
  89. '''
  90. A helper class for performing atomic writes. Usage::
  91. with AtomicWriter(path).open() as f:
  92. f.write(...)
  93. :param path: The destination filepath. May or may not exist.
  94. :param mode: The filemode for the temporary file. This defaults to `wb` in
  95. Python 2 and `w` in Python 3.
  96. :param overwrite: If set to false, an error is raised if ``path`` exists.
  97. Errors are only raised after the file has been written to. Either way,
  98. the operation is atomic.
  99. :param open_kwargs: Keyword-arguments to pass to the underlying
  100. :py:func:`open` call. This can be used to set the encoding when opening
  101. files in text-mode.
  102. If you need further control over the exact behavior, you are encouraged to
  103. subclass.
  104. '''
  105. def __init__(self, path, mode=DEFAULT_MODE, overwrite=False,
  106. **open_kwargs):
  107. if 'a' in mode:
  108. raise ValueError(
  109. 'Appending to an existing file is not supported, because that '
  110. 'would involve an expensive `copy`-operation to a temporary '
  111. 'file. Open the file in normal `w`-mode and copy explicitly '
  112. 'if that\'s what you\'re after.'
  113. )
  114. if 'x' in mode:
  115. raise ValueError('Use the `overwrite`-parameter instead.')
  116. if 'w' not in mode:
  117. raise ValueError('AtomicWriters can only be written to.')
  118. # Attempt to convert `path` to `str` or `bytes`
  119. if fspath is not None:
  120. path = fspath(path)
  121. self._path = path
  122. self._mode = mode
  123. self._overwrite = overwrite
  124. self._open_kwargs = open_kwargs
  125. def open(self):
  126. '''
  127. Open the temporary file.
  128. '''
  129. return self._open(self.get_fileobject)
  130. @contextlib.contextmanager
  131. def _open(self, get_fileobject):
  132. f = None # make sure f exists even if get_fileobject() fails
  133. try:
  134. success = False
  135. with get_fileobject(**self._open_kwargs) as f:
  136. yield f
  137. self.sync(f)
  138. self.commit(f)
  139. success = True
  140. finally:
  141. if not success:
  142. try:
  143. self.rollback(f)
  144. except Exception:
  145. pass
  146. def get_fileobject(self, suffix="", prefix=tempfile.gettempprefix(),
  147. dir=None, **kwargs):
  148. '''Return the temporary file to use.'''
  149. if dir is None:
  150. dir = os.path.normpath(os.path.dirname(self._path))
  151. descriptor, name = tempfile.mkstemp(suffix=suffix, prefix=prefix,
  152. dir=dir)
  153. # io.open() will take either the descriptor or the name, but we need
  154. # the name later for commit()/replace_atomic() and couldn't find a way
  155. # to get the filename from the descriptor.
  156. os.close(descriptor)
  157. kwargs['mode'] = self._mode
  158. kwargs['file'] = name
  159. return io.open(**kwargs)
  160. def sync(self, f):
  161. '''responsible for clearing as many file caches as possible before
  162. commit'''
  163. f.flush()
  164. _proper_fsync(f.fileno())
  165. def commit(self, f):
  166. '''Move the temporary file to the target location.'''
  167. if self._overwrite:
  168. replace_atomic(f.name, self._path)
  169. else:
  170. move_atomic(f.name, self._path)
  171. def rollback(self, f):
  172. '''Clean up all temporary resources.'''
  173. os.unlink(f.name)
  174. def atomic_write(path, writer_cls=AtomicWriter, **cls_kwargs):
  175. '''
  176. Simple atomic writes. This wraps :py:class:`AtomicWriter`::
  177. with atomic_write(path) as f:
  178. f.write(...)
  179. :param path: The target path to write to.
  180. :param writer_cls: The writer class to use. This parameter is useful if you
  181. subclassed :py:class:`AtomicWriter` to change some behavior and want to
  182. use that new subclass.
  183. Additional keyword arguments are passed to the writer class. See
  184. :py:class:`AtomicWriter`.
  185. '''
  186. return writer_cls(path, **cls_kwargs).open()