You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
555 lines
19 KiB
Python
555 lines
19 KiB
Python
"""
|
|
gevent internals.
|
|
"""
|
|
from __future__ import absolute_import, print_function, division
|
|
|
|
try:
|
|
from errno import EBADF
|
|
except ImportError:
|
|
EBADF = 9
|
|
|
|
import io
|
|
import functools
|
|
import sys
|
|
import os
|
|
|
|
from gevent.hub import _get_hub_noargs as get_hub
|
|
from gevent._compat import PY2
|
|
from gevent._compat import integer_types
|
|
from gevent._compat import reraise
|
|
from gevent._compat import fspath
|
|
from gevent.lock import Semaphore, DummySemaphore
|
|
|
|
class cancel_wait_ex(IOError):
|
|
|
|
def __init__(self):
|
|
IOError.__init__(
|
|
self,
|
|
EBADF, 'File descriptor was closed in another greenlet')
|
|
|
|
class FileObjectClosed(IOError):
|
|
|
|
def __init__(self):
|
|
IOError.__init__(
|
|
self,
|
|
EBADF, 'Bad file descriptor (FileObject was closed)')
|
|
|
|
class UniversalNewlineBytesWrapper(io.TextIOWrapper):
|
|
"""
|
|
Uses TextWrapper to decode universal newlines, but returns the
|
|
results as bytes.
|
|
|
|
This is for Python 2 where the 'rU' mode did that.
|
|
"""
|
|
mode = None
|
|
def __init__(self, fobj, line_buffering):
|
|
# latin-1 has the ability to round-trip arbitrary bytes.
|
|
io.TextIOWrapper.__init__(self, fobj, encoding='latin-1',
|
|
newline=None,
|
|
line_buffering=line_buffering)
|
|
|
|
def read(self, *args, **kwargs):
|
|
result = io.TextIOWrapper.read(self, *args, **kwargs)
|
|
return result.encode('latin-1')
|
|
|
|
def readline(self, limit=-1):
|
|
result = io.TextIOWrapper.readline(self, limit)
|
|
return result.encode('latin-1')
|
|
|
|
def __iter__(self):
|
|
# readlines() is implemented in terms of __iter__
|
|
# and TextIOWrapper.__iter__ checks that readline returns
|
|
# a unicode object, which we don't, so we override
|
|
return self
|
|
|
|
def __next__(self):
|
|
line = self.readline()
|
|
if not line:
|
|
raise StopIteration
|
|
return line
|
|
|
|
next = __next__
|
|
|
|
class FlushingBufferedWriter(io.BufferedWriter):
|
|
|
|
def write(self, b):
|
|
ret = io.BufferedWriter.write(self, b)
|
|
self.flush()
|
|
return ret
|
|
|
|
class OpenDescriptor(object): # pylint:disable=too-many-instance-attributes
|
|
"""
|
|
Interprets the arguments to `open`. Internal use only.
|
|
|
|
Originally based on code in the stdlib's _pyio.py (Python implementation of
|
|
the :mod:`io` module), but modified for gevent:
|
|
|
|
- Native strings are returned on Python 2 when neither
|
|
'b' nor 't' are in the mode string and no encoding is specified.
|
|
- Universal newlines work in that mode.
|
|
- Allows unbuffered text IO.
|
|
"""
|
|
|
|
@staticmethod
|
|
def _collapse_arg(preferred_val, old_val, default):
|
|
if preferred_val is not None and old_val is not None:
|
|
raise TypeError
|
|
if preferred_val is None and old_val is None:
|
|
return default
|
|
return preferred_val if preferred_val is not None else old_val
|
|
|
|
def __init__(self, fobj, mode='r', bufsize=None, close=None,
|
|
encoding=None, errors=None, newline=None,
|
|
buffering=None, closefd=None):
|
|
# Based on code in the stdlib's _pyio.py from 3.8.
|
|
# pylint:disable=too-many-locals,too-many-branches,too-many-statements
|
|
closefd = self._collapse_arg(closefd, close, True)
|
|
del close
|
|
buffering = self._collapse_arg(buffering, bufsize, -1)
|
|
del bufsize
|
|
|
|
if not hasattr(fobj, 'fileno'):
|
|
if not isinstance(fobj, integer_types):
|
|
# Not a fd. Support PathLike on Python 2 and Python <= 3.5.
|
|
fobj = fspath(fobj)
|
|
if not isinstance(fobj, (str, bytes) + integer_types): # pragma: no cover
|
|
raise TypeError("invalid file: %r" % fobj)
|
|
if isinstance(fobj, (str, bytes)):
|
|
closefd = True
|
|
|
|
if not isinstance(mode, str):
|
|
raise TypeError("invalid mode: %r" % mode)
|
|
if not isinstance(buffering, integer_types):
|
|
raise TypeError("invalid buffering: %r" % buffering)
|
|
if encoding is not None and not isinstance(encoding, str):
|
|
raise TypeError("invalid encoding: %r" % encoding)
|
|
if errors is not None and not isinstance(errors, str):
|
|
raise TypeError("invalid errors: %r" % errors)
|
|
|
|
modes = set(mode)
|
|
if modes - set("axrwb+tU") or len(mode) > len(modes):
|
|
raise ValueError("invalid mode: %r" % mode)
|
|
|
|
creating = "x" in modes
|
|
reading = "r" in modes
|
|
writing = "w" in modes
|
|
appending = "a" in modes
|
|
updating = "+" in modes
|
|
text = "t" in modes
|
|
binary = "b" in modes
|
|
universal = 'U' in modes
|
|
|
|
can_write = creating or writing or appending or updating
|
|
|
|
if universal:
|
|
if can_write:
|
|
raise ValueError("mode U cannot be combined with 'x', 'w', 'a', or '+'")
|
|
# Just because the stdlib deprecates this, no need for us to do so as well.
|
|
# Especially not while we still support Python 2.
|
|
# import warnings
|
|
# warnings.warn("'U' mode is deprecated",
|
|
# DeprecationWarning, 4)
|
|
reading = True
|
|
if text and binary:
|
|
raise ValueError("can't have text and binary mode at once")
|
|
if creating + reading + writing + appending > 1:
|
|
raise ValueError("can't have read/write/append mode at once")
|
|
if not (creating or reading or writing or appending):
|
|
raise ValueError("must have exactly one of read/write/append mode")
|
|
if binary and encoding is not None:
|
|
raise ValueError("binary mode doesn't take an encoding argument")
|
|
if binary and errors is not None:
|
|
raise ValueError("binary mode doesn't take an errors argument")
|
|
if binary and newline is not None:
|
|
raise ValueError("binary mode doesn't take a newline argument")
|
|
if binary and buffering == 1:
|
|
import warnings
|
|
warnings.warn("line buffering (buffering=1) isn't supported in binary "
|
|
"mode, the default buffer size will be used",
|
|
RuntimeWarning, 4)
|
|
|
|
self.fobj = fobj
|
|
self.fileio_mode = (
|
|
(creating and "x" or "")
|
|
+ (reading and "r" or "")
|
|
+ (writing and "w" or "")
|
|
+ (appending and "a" or "")
|
|
+ (updating and "+" or "")
|
|
)
|
|
self.mode = self.fileio_mode + ('t' if text else '') + ('b' if binary else '')
|
|
|
|
self.creating = creating
|
|
self.reading = reading
|
|
self.writing = writing
|
|
self.appending = appending
|
|
self.updating = updating
|
|
self.text = text
|
|
self.binary = binary
|
|
self.can_write = can_write
|
|
self.can_read = reading or updating
|
|
self.native = (
|
|
not self.text and not self.binary # Neither t nor b given.
|
|
and not encoding and not errors # And no encoding or error handling either.
|
|
)
|
|
self.universal = universal
|
|
|
|
self.buffering = buffering
|
|
self.encoding = encoding
|
|
self.errors = errors
|
|
self.newline = newline
|
|
self.closefd = closefd
|
|
|
|
default_buffer_size = io.DEFAULT_BUFFER_SIZE
|
|
|
|
def is_fd(self):
|
|
return isinstance(self.fobj, integer_types)
|
|
|
|
def open(self):
|
|
return self.open_raw_and_wrapped()[1]
|
|
|
|
def open_raw_and_wrapped(self):
|
|
raw = self.open_raw()
|
|
try:
|
|
return raw, self.wrapped(raw)
|
|
except:
|
|
raw.close()
|
|
raise
|
|
|
|
def open_raw(self):
|
|
if hasattr(self.fobj, 'fileno'):
|
|
return self.fobj
|
|
return io.FileIO(self.fobj, self.fileio_mode, self.closefd)
|
|
|
|
def wrapped(self, raw):
|
|
"""
|
|
Wraps the raw IO object (`RawIOBase` or `io.TextIOBase`) in
|
|
buffers, text decoding, and newline handling.
|
|
"""
|
|
# pylint:disable=too-many-branches
|
|
result = raw
|
|
buffering = self.buffering
|
|
|
|
line_buffering = False
|
|
if buffering == 1 or buffering < 0 and raw.isatty():
|
|
buffering = -1
|
|
line_buffering = True
|
|
if buffering < 0:
|
|
buffering = self.default_buffer_size
|
|
try:
|
|
bs = os.fstat(raw.fileno()).st_blksize
|
|
except (OSError, AttributeError):
|
|
pass
|
|
else:
|
|
if bs > 1:
|
|
buffering = bs
|
|
if buffering < 0: # pragma: no cover
|
|
raise ValueError("invalid buffering size")
|
|
|
|
if not isinstance(raw, io.BufferedIOBase) and \
|
|
(not hasattr(raw, 'buffer') or raw.buffer is None):
|
|
# Need to wrap our own buffering around it. If it
|
|
# is already buffered, don't do so.
|
|
if buffering != 0:
|
|
if self.updating:
|
|
Buffer = io.BufferedRandom
|
|
elif self.creating or self.writing or self.appending:
|
|
Buffer = io.BufferedWriter
|
|
elif self.reading:
|
|
Buffer = io.BufferedReader
|
|
else: # prgama: no cover
|
|
raise ValueError("unknown mode: %r" % self.mode)
|
|
|
|
try:
|
|
result = Buffer(raw, buffering)
|
|
except AttributeError:
|
|
# Python 2 file() objects don't have the readable/writable
|
|
# attributes. But they handle their own buffering.
|
|
result = raw
|
|
|
|
if self.binary:
|
|
if isinstance(raw, io.TextIOBase):
|
|
# Can't do it. The TextIO object will have its own buffer, and
|
|
# trying to read from the raw stream or the buffer without going through
|
|
# the TextIO object is likely to lead to problems with the codec.
|
|
raise ValueError("Unable to perform binary IO on top of text IO stream")
|
|
return result
|
|
|
|
# Either native or text at this point.
|
|
if PY2 and self.native:
|
|
# Neither text mode nor binary mode specified.
|
|
if self.universal:
|
|
# universal was requested, e.g., 'rU'
|
|
result = UniversalNewlineBytesWrapper(result, line_buffering)
|
|
else:
|
|
# Python 2 and text mode, or Python 3 and either text or native (both are the same)
|
|
if not isinstance(raw, io.TextIOBase):
|
|
# Avoid double-wrapping a TextIOBase in another TextIOWrapper.
|
|
# That tends not to work. See https://github.com/gevent/gevent/issues/1542
|
|
result = io.TextIOWrapper(result, self.encoding, self.errors, self.newline,
|
|
line_buffering)
|
|
|
|
if result is not raw:
|
|
# Set the mode, if possible, but only if we created a new
|
|
# object.
|
|
try:
|
|
result.mode = self.mode
|
|
except (AttributeError, TypeError):
|
|
# AttributeError: No such attribute
|
|
# TypeError: Readonly attribute (py2)
|
|
pass
|
|
|
|
return result
|
|
|
|
|
|
class FileObjectBase(object):
|
|
"""
|
|
Internal base class to ensure a level of consistency
|
|
between :class:`~.FileObjectPosix`, :class:`~.FileObjectThread`
|
|
and :class:`~.FileObjectBlock`.
|
|
"""
|
|
|
|
# List of methods we delegate to the wrapping IO object, if they
|
|
# implement them and we do not.
|
|
_delegate_methods = (
|
|
# General methods
|
|
'flush',
|
|
'fileno',
|
|
'writable',
|
|
'readable',
|
|
'seek',
|
|
'seekable',
|
|
'tell',
|
|
|
|
# Read
|
|
'read',
|
|
'readline',
|
|
'readlines',
|
|
'read1',
|
|
|
|
# Write
|
|
'write',
|
|
'writelines',
|
|
'truncate',
|
|
)
|
|
|
|
|
|
_io = None
|
|
|
|
def __init__(self, fobj, closefd):
|
|
self._io = fobj
|
|
# We don't actually use this property ourself, but we save it (and
|
|
# pass it along) for compatibility.
|
|
self._close = closefd
|
|
|
|
self._do_delegate_methods()
|
|
|
|
|
|
io = property(lambda s: s._io,
|
|
# Historically we either hand-wrote all the delegation methods
|
|
# to use self.io, or we simply used __getattr__ to look them up at
|
|
# runtime. This meant people could change the io attribute on the fly
|
|
# and it would mostly work (subprocess.py used to do that). We don't recommend
|
|
# that, but we still support it.
|
|
lambda s, nv: setattr(s, '_io', nv) or s._do_delegate_methods())
|
|
|
|
def _do_delegate_methods(self):
|
|
for meth_name in self._delegate_methods:
|
|
meth = getattr(self._io, meth_name, None)
|
|
implemented_by_class = hasattr(type(self), meth_name)
|
|
if meth and not implemented_by_class:
|
|
setattr(self, meth_name, self._wrap_method(meth))
|
|
elif hasattr(self, meth_name) and not implemented_by_class:
|
|
delattr(self, meth_name)
|
|
|
|
def _wrap_method(self, method):
|
|
"""
|
|
Wrap a method we're copying into our dictionary from the underlying
|
|
io object to do something special or different, if necessary.
|
|
"""
|
|
return method
|
|
|
|
@property
|
|
def closed(self):
|
|
"""True if the file is closed"""
|
|
return self._io is None
|
|
|
|
def close(self):
|
|
if self._io is None:
|
|
return
|
|
|
|
fobj = self._io
|
|
self._io = None
|
|
self._do_close(fobj, self._close)
|
|
|
|
def _do_close(self, fobj, closefd):
|
|
raise NotImplementedError()
|
|
|
|
def __getattr__(self, name):
|
|
if self._io is None:
|
|
raise FileObjectClosed()
|
|
return getattr(self._io, name)
|
|
|
|
def __repr__(self):
|
|
return '<%s at 0x%x %s_fobj=%r%s>' % (
|
|
self.__class__.__name__,
|
|
id(self),
|
|
'closed' if self.closed else '',
|
|
self.io,
|
|
self._extra_repr()
|
|
)
|
|
|
|
def _extra_repr(self):
|
|
return ''
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *args):
|
|
self.close()
|
|
|
|
def __iter__(self):
|
|
return self
|
|
|
|
def __next__(self):
|
|
line = self.readline()
|
|
if not line:
|
|
raise StopIteration
|
|
return line
|
|
|
|
next = __next__
|
|
|
|
def __bool__(self):
|
|
return True
|
|
|
|
__nonzero__ = __bool__
|
|
|
|
|
|
class FileObjectBlock(FileObjectBase):
|
|
"""
|
|
FileObjectBlock()
|
|
|
|
A simple synchronous wrapper around a file object.
|
|
|
|
Adds no concurrency or gevent compatibility.
|
|
"""
|
|
|
|
def __init__(self, fobj, *args, **kwargs):
|
|
descriptor = OpenDescriptor(fobj, *args, **kwargs)
|
|
FileObjectBase.__init__(self, descriptor.open(), descriptor.closefd)
|
|
|
|
def _do_close(self, fobj, closefd):
|
|
fobj.close()
|
|
|
|
|
|
class FileObjectThread(FileObjectBase):
|
|
"""
|
|
FileObjectThread()
|
|
|
|
A file-like object wrapping another file-like object, performing all blocking
|
|
operations on that object in a background thread.
|
|
|
|
.. caution::
|
|
Attempting to change the threadpool or lock of an existing FileObjectThread
|
|
has undefined consequences.
|
|
|
|
.. versionchanged:: 1.1b1
|
|
The file object is closed using the threadpool. Note that whether or
|
|
not this action is synchronous or asynchronous is not documented.
|
|
"""
|
|
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
"""
|
|
:keyword bool lock: If True (the default) then all operations will
|
|
be performed one-by-one. Note that this does not guarantee that, if using
|
|
this file object from multiple threads/greenlets, operations will be performed
|
|
in any particular order, only that no two operations will be attempted at the
|
|
same time. You can also pass your own :class:`gevent.lock.Semaphore` to synchronize
|
|
file operations with an external resource.
|
|
:keyword bool closefd: If True (the default) then when this object is closed,
|
|
the underlying object is closed as well. If *fobj* is a path, then
|
|
*closefd* must be True.
|
|
"""
|
|
lock = kwargs.pop('lock', True)
|
|
threadpool = kwargs.pop('threadpool', None)
|
|
descriptor = OpenDescriptor(*args, **kwargs)
|
|
|
|
self.threadpool = threadpool or get_hub().threadpool
|
|
self.lock = lock
|
|
if self.lock is True:
|
|
self.lock = Semaphore()
|
|
elif not self.lock:
|
|
self.lock = DummySemaphore()
|
|
if not hasattr(self.lock, '__enter__'):
|
|
raise TypeError('Expected a Semaphore or boolean, got %r' % type(self.lock))
|
|
|
|
self.__io_holder = [descriptor.open()] # signal for _wrap_method
|
|
FileObjectBase.__init__(self, self.__io_holder[0], descriptor.closefd)
|
|
|
|
def _do_close(self, fobj, closefd):
|
|
self.__io_holder[0] = None # for _wrap_method
|
|
try:
|
|
with self.lock:
|
|
self.threadpool.apply(fobj.flush)
|
|
finally:
|
|
if closefd:
|
|
# Note that we're not taking the lock; older code
|
|
# did fobj.close() without going through the threadpool at all,
|
|
# so acquiring the lock could potentially introduce deadlocks
|
|
# that weren't present before. Avoiding the lock doesn't make
|
|
# the existing race condition any worse.
|
|
# We wrap the close in an exception handler and re-raise directly
|
|
# to avoid the (common, expected) IOError from being logged by the pool
|
|
def close(_fobj=fobj):
|
|
try:
|
|
_fobj.close()
|
|
except: # pylint:disable=bare-except
|
|
return sys.exc_info()
|
|
finally:
|
|
_fobj = None
|
|
del fobj
|
|
|
|
exc_info = self.threadpool.apply(close)
|
|
del close
|
|
|
|
if exc_info:
|
|
reraise(*exc_info)
|
|
|
|
def _do_delegate_methods(self):
|
|
FileObjectBase._do_delegate_methods(self)
|
|
# if not hasattr(self, 'read1') and 'r' in getattr(self._io, 'mode', ''):
|
|
# self.read1 = self.read
|
|
self.__io_holder[0] = self._io
|
|
|
|
def _extra_repr(self):
|
|
return ' threadpool=%r' % (self.threadpool,)
|
|
|
|
def __iter__(self):
|
|
return self
|
|
|
|
def next(self):
|
|
line = self.readline()
|
|
if line:
|
|
return line
|
|
raise StopIteration
|
|
__next__ = next
|
|
|
|
def _wrap_method(self, method):
|
|
# NOTE: We are careful to avoid introducing a refcycle
|
|
# within self. Our wrapper cannot refer to self.
|
|
io_holder = self.__io_holder
|
|
lock = self.lock
|
|
threadpool = self.threadpool
|
|
|
|
@functools.wraps(method)
|
|
def thread_method(*args, **kwargs):
|
|
if io_holder[0] is None:
|
|
# This is different than FileObjectPosix, etc,
|
|
# because we want to save the expensive trip through
|
|
# the threadpool.
|
|
raise FileObjectClosed()
|
|
with lock:
|
|
return threadpool.apply(method, args, kwargs)
|
|
|
|
return thread_method
|