You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

161 lines
4.7 KiB
Python

5 years ago
"""
A small selection of primitives that always work with
native threads. This has very limited utility and is
targeted only for the use of gevent's threadpool.
"""
from __future__ import absolute_import
from collections import deque
from gevent import monkey
from gevent._compat import thread_mod_name
__all__ = [
'Lock',
'Queue',
]
start_new_thread, Lock, get_thread_ident, = monkey.get_original(thread_mod_name, [
'start_new_thread', 'allocate_lock', 'get_ident',
])
# pylint 2.0.dev2 things collections.dequeue.popleft() doesn't return
# pylint:disable=assignment-from-no-return
class _Condition(object):
# pylint:disable=method-hidden
def __init__(self, lock):
self.__lock = lock
self.__waiters = []
# No need to special case for _release_save and
# _acquire_restore; those are only used for RLock, and
# we don't use those.
def __enter__(self):
return self.__lock.__enter__()
def __exit__(self, t, v, tb):
return self.__lock.__exit__(t, v, tb)
def __repr__(self):
return "<Condition(%s, %d)>" % (self.__lock, len(self.__waiters))
def wait(self):
# This variable is for the monitoring utils to know that
# this is an idle frame and shouldn't be counted.
gevent_threadpool_worker_idle = True # pylint:disable=unused-variable
# Our __lock MUST be owned, but we don't check that.
waiter = Lock()
waiter.acquire()
self.__waiters.append(waiter)
self.__lock.release()
try:
waiter.acquire() # Block on the native lock
finally:
self.__lock.acquire()
# just good form to release the lock we're holding before it goes
# out of scope
waiter.release()
def notify_one(self):
# The lock SHOULD be owned, but we don't check that.
try:
waiter = self.__waiters.pop()
except IndexError:
# Nobody around
pass
else:
waiter.release()
class Queue(object):
"""Create a queue object.
The queue is always infinite size.
"""
__slots__ = ('_queue', '_mutex', '_not_empty', 'unfinished_tasks')
def __init__(self):
self._queue = deque()
# mutex must be held whenever the queue is mutating. All methods
# that acquire mutex must release it before returning. mutex
# is shared between the three conditions, so acquiring and
# releasing the conditions also acquires and releases mutex.
self._mutex = Lock()
# Notify not_empty whenever an item is added to the queue; a
# thread waiting to get is notified then.
self._not_empty = _Condition(self._mutex)
self.unfinished_tasks = 0
def task_done(self):
"""Indicate that a formerly enqueued task is complete.
Used by Queue consumer threads. For each get() used to fetch a task,
a subsequent call to task_done() tells the queue that the processing
on the task is complete.
If a join() is currently blocking, it will resume when all items
have been processed (meaning that a task_done() call was received
for every item that had been put() into the queue).
Raises a ValueError if called more times than there were items
placed in the queue.
"""
with self._mutex:
unfinished = self.unfinished_tasks - 1
if unfinished <= 0:
if unfinished < 0:
raise ValueError('task_done() called too many times')
self.unfinished_tasks = unfinished
def qsize(self, len=len):
"""Return the approximate size of the queue (not reliable!)."""
return len(self._queue)
def empty(self):
"""Return True if the queue is empty, False otherwise (not reliable!)."""
return not self.qsize()
def full(self):
"""Return True if the queue is full, False otherwise (not reliable!)."""
return False
def put(self, item):
"""Put an item into the queue.
"""
with self._mutex:
self._queue.append(item)
self.unfinished_tasks += 1
self._not_empty.notify_one()
def get(self):
"""Remove and return an item from the queue.
"""
with self._mutex:
while not self._queue:
self._not_empty.wait()
item = self._queue.popleft()
return item
def kill(self):
"""
Call to destroy this object.
Use this when it's not possible to safely drain the queue, e.g.,
after a fork when the locks are in an uncertain state.
"""
self._queue = None
self._mutex = None
self._not_empty = None
self.unfinished_tasks = None