Source code for taskflow.engines.worker_based.worker
# -*- coding: utf-8 -*-
# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import platform
import socket
import string
import sys
import futurist
from oslo_utils import reflection
from taskflow.engines.worker_based import endpoint
from taskflow.engines.worker_based import server
from taskflow import logging
from taskflow import task as t_task
from taskflow.utils import misc
from taskflow.utils import threading_utils as tu
from taskflow import version
LOG = logging.getLogger(__name__)
[docs]class Worker(object):
"""Worker that can be started on a remote host for handling tasks requests.
:param url: broker url
:param exchange: broker exchange name
:param topic: topic name under which worker is stated
:param tasks: task list that worker is capable of performing, items in
the list can be one of the following types; 1, a string naming the
python module name to search for tasks in or the task class name; 2, a
python module to search for tasks in; 3, a task class object that
will be used to create tasks from.
:param executor: custom executor object that can used for processing
requests in separate threads (if not provided one will be created)
:param threads_count: threads count to be passed to the
default executor (used only if an executor is not
passed in)
:param transport: transport to be used (e.g. amqp, memory, etc.)
:param transport_options: transport specific options (see:
http://kombu.readthedocs.org/ for what these
options imply and are expected to be)
:param retry_options: retry specific options
(see: :py:attr:`~.proxy.Proxy.DEFAULT_RETRY_OPTIONS`)
"""
BANNER_TEMPLATE = string.Template("""
TaskFlow v${version} WBE worker.
Connection details:
Driver = $transport_driver
Exchange = $exchange
Topic = $topic
Transport = $transport_type
Uri = $connection_uri
Powered by:
Executor = $executor_type
Thread count = $executor_thread_count
Supported endpoints:$endpoints
System details:
Hostname = $hostname
Pid = $pid
Platform = $platform
Python = $python
Thread id = $thread_id
""".strip())
# See: http://bugs.python.org/issue13173 for why we are doing this...
BANNER_TEMPLATE.defaults = {
# These values may not be possible to fetch/known, default
# to ??? to represent that they are unknown...
'pid': '???',
'hostname': '???',
'executor_thread_count': '???',
'endpoints': ' %s' % ([]),
# These are static (avoid refetching...)
'version': version.version_string(),
'python': sys.version.split("\n", 1)[0].strip(),
}
def __init__(self, exchange, topic, tasks,
executor=None, threads_count=None, url=None,
transport=None, transport_options=None,
retry_options=None):
self._topic = topic
self._executor = executor
self._owns_executor = False
if self._executor is None:
self._executor = futurist.ThreadPoolExecutor(
max_workers=threads_count)
self._owns_executor = True
self._endpoints = self._derive_endpoints(tasks)
self._exchange = exchange
self._server = server.Server(topic, exchange, self._executor,
self._endpoints, url=url,
transport=transport,
transport_options=transport_options,
retry_options=retry_options)
@staticmethod
def _derive_endpoints(tasks):
"""Derive endpoints from list of strings, classes or packages."""
derived_tasks = misc.find_subclasses(tasks, t_task.BaseTask)
return [endpoint.Endpoint(task) for task in derived_tasks]
def _generate_banner(self):
"""Generates a banner that can be useful to display before running."""
try:
tpl_params = dict(self.BANNER_TEMPLATE.defaults)
except AttributeError:
tpl_params = {}
connection_details = self._server.connection_details
transport = connection_details.transport
if transport.driver_version:
transport_driver = "%s v%s" % (transport.driver_name,
transport.driver_version)
else:
transport_driver = transport.driver_name
tpl_params['transport_driver'] = transport_driver
tpl_params['exchange'] = self._exchange
tpl_params['topic'] = self._topic
tpl_params['transport_type'] = transport.driver_type
tpl_params['connection_uri'] = connection_details.uri
tpl_params['executor_type'] = reflection.get_class_name(self._executor)
threads_count = getattr(self._executor, 'max_workers', None)
if threads_count is not None:
tpl_params['executor_thread_count'] = threads_count
if self._endpoints:
pretty_endpoints = []
for ep in self._endpoints:
pretty_endpoints.append(" - %s" % ep)
# This ensures there is a newline before the list...
tpl_params['endpoints'] = "\n" + "\n".join(pretty_endpoints)
try:
tpl_params['hostname'] = socket.getfqdn()
except socket.error:
pass
try:
tpl_params['pid'] = os.getpid()
except OSError:
pass
tpl_params['platform'] = platform.platform()
tpl_params['thread_id'] = tu.get_ident()
banner = self.BANNER_TEMPLATE.substitute(**tpl_params)
# NOTE(harlowja): this is needed since the template in this file
# will always have newlines that end with '\n' (even on different
# platforms due to the way this source file is encoded) so we have
# to do this little dance to make it platform neutral...
return misc.fix_newlines(banner)
[docs] def run(self, display_banner=True, banner_writer=None):
"""Runs the worker."""
if display_banner:
banner = self._generate_banner()
if banner_writer is None:
for line in banner.splitlines():
LOG.info(line)
else:
banner_writer(banner)
self._server.start()
[docs] def wait(self):
"""Wait until worker is started."""
self._server.wait()
[docs] def stop(self):
"""Stop worker."""
self._server.stop()
if self._owns_executor:
self._executor.shutdown()