Source code for google.cloud.pubsub_v1.subscriber.scheduler
# Copyright 2018, Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Schedulers provide means to *schedule* callbacks asynchronously.
These are used by the subscriber to call the user-provided callback to process
each message.
"""
import abc
import concurrent.futures
import sys
import six
from six.moves import queue
[docs]@six.add_metaclass(abc.ABCMeta)
class Scheduler(object):
"""Abstract base class for schedulers.
Schedulers are used to schedule callbacks asynchronously.
"""
@property
@abc.abstractmethod
def queue(self):
"""Queue: A concurrency-safe queue specific to the underlying
concurrency implementation.
This queue is used to send messages *back* to the scheduling actor.
"""
raise NotImplementedError
[docs] @abc.abstractmethod
def schedule(self, callback, *args, **kwargs):
"""Schedule the callback to be called asynchronously.
Args:
callback (Callable): The function to call.
args: Positional arguments passed to the function.
kwargs: Key-word arguments passed to the function.
Returns:
None
"""
raise NotImplementedError
[docs] @abc.abstractmethod
def shutdown(self):
"""Shuts down the scheduler and immediately end all pending callbacks.
"""
raise NotImplementedError
def _make_default_thread_pool_executor():
# Python 2.7 and 3.6+ have the thread_name_prefix argument, which is useful
# for debugging.
executor_kwargs = {}
if sys.version_info[:2] == (2, 7) or sys.version_info >= (3, 6):
executor_kwargs["thread_name_prefix"] = "ThreadPoolExecutor-ThreadScheduler"
return concurrent.futures.ThreadPoolExecutor(max_workers=10, **executor_kwargs)
[docs]class ThreadScheduler(Scheduler):
"""A thread pool-based scheduler.
This scheduler is useful in typical I/O-bound message processing.
Args:
executor(concurrent.futures.ThreadPoolExecutor): An optional executor
to use. If not specified, a default one will be created.
"""
def __init__(self, executor=None):
self._queue = queue.Queue()
if executor is None:
self._executor = _make_default_thread_pool_executor()
else:
self._executor = executor
@property
def queue(self):
"""Queue: A thread-safe queue used for communication between callbacks
and the scheduling thread."""
return self._queue
[docs] def schedule(self, callback, *args, **kwargs):
"""Schedule the callback to be called asynchronously in a thread pool.
Args:
callback (Callable): The function to call.
args: Positional arguments passed to the function.
kwargs: Key-word arguments passed to the function.
Returns:
None
"""
self._executor.submit(callback, *args, **kwargs)
[docs] def shutdown(self):
"""Shuts down the scheduler and immediately end all pending callbacks.
"""
# Drop all pending item from the executor. Without this, the executor
# will block until all pending items are complete, which is
# undesirable.
try:
while True:
self._executor._work_queue.get(block=False)
except queue.Empty:
pass
self._executor.shutdown()