You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
161 lines
4.7 KiB
Python
161 lines
4.7 KiB
Python
"""
|
|
A small selection of primitives that always work with
|
|
native threads. This has very limited utility and is
|
|
targeted only for the use of gevent's threadpool.
|
|
"""
|
|
from __future__ import absolute_import
|
|
|
|
from collections import deque
|
|
|
|
from gevent import monkey
|
|
from gevent._compat import thread_mod_name
|
|
|
|
|
|
__all__ = [
|
|
'Lock',
|
|
'Queue',
|
|
]
|
|
|
|
|
|
start_new_thread, Lock, get_thread_ident, = monkey.get_original(thread_mod_name, [
|
|
'start_new_thread', 'allocate_lock', 'get_ident',
|
|
])
|
|
|
|
|
|
# pylint 2.0.dev2 things collections.dequeue.popleft() doesn't return
|
|
# pylint:disable=assignment-from-no-return
|
|
|
|
class _Condition(object):
|
|
# pylint:disable=method-hidden
|
|
|
|
def __init__(self, lock):
|
|
self.__lock = lock
|
|
self.__waiters = []
|
|
|
|
# No need to special case for _release_save and
|
|
# _acquire_restore; those are only used for RLock, and
|
|
# we don't use those.
|
|
|
|
def __enter__(self):
|
|
return self.__lock.__enter__()
|
|
|
|
def __exit__(self, t, v, tb):
|
|
return self.__lock.__exit__(t, v, tb)
|
|
|
|
def __repr__(self):
|
|
return "<Condition(%s, %d)>" % (self.__lock, len(self.__waiters))
|
|
|
|
def wait(self):
|
|
# This variable is for the monitoring utils to know that
|
|
# this is an idle frame and shouldn't be counted.
|
|
gevent_threadpool_worker_idle = True # pylint:disable=unused-variable
|
|
|
|
# Our __lock MUST be owned, but we don't check that.
|
|
waiter = Lock()
|
|
waiter.acquire()
|
|
self.__waiters.append(waiter)
|
|
self.__lock.release()
|
|
|
|
try:
|
|
waiter.acquire() # Block on the native lock
|
|
finally:
|
|
self.__lock.acquire()
|
|
|
|
# just good form to release the lock we're holding before it goes
|
|
# out of scope
|
|
waiter.release()
|
|
|
|
def notify_one(self):
|
|
# The lock SHOULD be owned, but we don't check that.
|
|
try:
|
|
waiter = self.__waiters.pop()
|
|
except IndexError:
|
|
# Nobody around
|
|
pass
|
|
else:
|
|
waiter.release()
|
|
|
|
|
|
class Queue(object):
|
|
"""Create a queue object.
|
|
|
|
The queue is always infinite size.
|
|
"""
|
|
|
|
__slots__ = ('_queue', '_mutex', '_not_empty', 'unfinished_tasks')
|
|
|
|
def __init__(self):
|
|
self._queue = deque()
|
|
# mutex must be held whenever the queue is mutating. All methods
|
|
# that acquire mutex must release it before returning. mutex
|
|
# is shared between the three conditions, so acquiring and
|
|
# releasing the conditions also acquires and releases mutex.
|
|
self._mutex = Lock()
|
|
# Notify not_empty whenever an item is added to the queue; a
|
|
# thread waiting to get is notified then.
|
|
self._not_empty = _Condition(self._mutex)
|
|
|
|
self.unfinished_tasks = 0
|
|
|
|
def task_done(self):
|
|
"""Indicate that a formerly enqueued task is complete.
|
|
|
|
Used by Queue consumer threads. For each get() used to fetch a task,
|
|
a subsequent call to task_done() tells the queue that the processing
|
|
on the task is complete.
|
|
|
|
If a join() is currently blocking, it will resume when all items
|
|
have been processed (meaning that a task_done() call was received
|
|
for every item that had been put() into the queue).
|
|
|
|
Raises a ValueError if called more times than there were items
|
|
placed in the queue.
|
|
"""
|
|
with self._mutex:
|
|
unfinished = self.unfinished_tasks - 1
|
|
if unfinished <= 0:
|
|
if unfinished < 0:
|
|
raise ValueError('task_done() called too many times')
|
|
self.unfinished_tasks = unfinished
|
|
|
|
def qsize(self, len=len):
|
|
"""Return the approximate size of the queue (not reliable!)."""
|
|
return len(self._queue)
|
|
|
|
def empty(self):
|
|
"""Return True if the queue is empty, False otherwise (not reliable!)."""
|
|
return not self.qsize()
|
|
|
|
def full(self):
|
|
"""Return True if the queue is full, False otherwise (not reliable!)."""
|
|
return False
|
|
|
|
def put(self, item):
|
|
"""Put an item into the queue.
|
|
"""
|
|
with self._mutex:
|
|
self._queue.append(item)
|
|
self.unfinished_tasks += 1
|
|
self._not_empty.notify_one()
|
|
|
|
def get(self):
|
|
"""Remove and return an item from the queue.
|
|
"""
|
|
with self._mutex:
|
|
while not self._queue:
|
|
self._not_empty.wait()
|
|
item = self._queue.popleft()
|
|
return item
|
|
|
|
def kill(self):
|
|
"""
|
|
Call to destroy this object.
|
|
|
|
Use this when it's not possible to safely drain the queue, e.g.,
|
|
after a fork when the locks are in an uncertain state.
|
|
"""
|
|
self._queue = None
|
|
self._mutex = None
|
|
self._not_empty = None
|
|
self.unfinished_tasks = None
|