Source code for taskflow.utils.kazoo_utils

# -*- coding: utf-8 -*-

#    Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from kazoo import client
from kazoo import exceptions as k_exc
from oslo_utils import reflection
from oslo_utils import strutils

from taskflow import exceptions as exc
from taskflow import logging


LOG = logging.getLogger(__name__)

CONF_TRANSFERS = (
    ('read_only', strutils.bool_from_string, False),
    ('randomize_hosts', strutils.bool_from_string, False),
    ('keyfile', None, None),
    ('keyfile_password', None, None),
    ('certfile', None, None),
    ('use_ssl', strutils.bool_from_string, False),
    ('verify_certs', strutils.bool_from_string, True))


def _parse_hosts(hosts):
    if isinstance(hosts, str):
        return hosts.strip()
    if isinstance(hosts, (dict)):
        host_ports = []
        for (k, v) in hosts.items():
            host_ports.append("%s:%s" % (k, v))
        hosts = host_ports
    if isinstance(hosts, (list, set, tuple)):
        return ",".join([str(h) for h in hosts])
    return hosts


[docs] def prettify_failures(failures, limit=-1): """Prettifies a checked commits failures (ignores sensitive data...).""" prettier = [] for (op, r) in failures: pretty_op = reflection.get_class_name(op, fully_qualified=False) # Pick off a few attributes that are meaningful (but one that don't # show actual data, which might not be desired to show...). selected_attrs = [ "path=%r" % op.path, ] try: if op.version != -1: selected_attrs.append("version=%s" % op.version) except AttributeError: pass pretty_op += "(%s)" % (", ".join(selected_attrs)) pretty_cause = reflection.get_class_name(r, fully_qualified=False) prettier.append("%s@%s" % (pretty_cause, pretty_op)) if limit <= 0 or len(prettier) <= limit: return ", ".join(prettier) else: leftover = prettier[limit:] prettier = prettier[0:limit] return ", ".join(prettier) + " and %s more..." % len(leftover)
[docs] class KazooTransactionException(k_exc.KazooException): """Exception raised when a checked commit fails.""" def __init__(self, message, failures): super(KazooTransactionException, self).__init__(message) self._failures = tuple(failures) @property def failures(self): return self._failures
[docs] def checked_commit(txn): """Commits a kazoo transcation and validates the result. NOTE(harlowja): Until https://github.com/python-zk/kazoo/pull/224 is fixed or a similar pull request is merged we have to workaround the transaction failing silently. """ if not txn.operations: return [] results = txn.commit() failures = [] for op, result in zip(txn.operations, results): if isinstance(result, k_exc.KazooException): failures.append((op, result)) if len(results) < len(txn.operations): raise KazooTransactionException( "Transaction returned %s results, this is less than" " the number of expected transaction operations %s" % (len(results), len(txn.operations)), failures) if len(results) > len(txn.operations): raise KazooTransactionException( "Transaction returned %s results, this is greater than" " the number of expected transaction operations %s" % (len(results), len(txn.operations)), failures) if failures: raise KazooTransactionException( "Transaction with %s operations failed: %s" % (len(txn.operations), prettify_failures(failures, limit=1)), failures) return results
[docs] def finalize_client(client): """Stops and closes a client, even if it wasn't started.""" client.stop() client.close()
[docs] def check_compatible(client, min_version=None, max_version=None): """Checks if a kazoo client is backed by a zookeeper server version. This check will verify that the zookeeper server version that the client is connected to satisfies a given minimum version (inclusive) and maximum (inclusive) version range. If the server is not in the provided version range then a exception is raised indiciating this. """ server_version = None if min_version: server_version = tuple((int(a) for a in client.server_version())) min_version = tuple((int(a) for a in min_version)) if server_version < min_version: pretty_server_version = ".".join([str(a) for a in server_version]) min_version = ".".join([str(a) for a in min_version]) raise exc.IncompatibleVersion("Incompatible zookeeper version" " %s detected, zookeeper >= %s" " required" % (pretty_server_version, min_version)) if max_version: if server_version is None: server_version = tuple((int(a) for a in client.server_version())) max_version = tuple((int(a) for a in max_version)) if server_version > max_version: pretty_server_version = ".".join([str(a) for a in server_version]) max_version = ".".join([str(a) for a in max_version]) raise exc.IncompatibleVersion("Incompatible zookeeper version" " %s detected, zookeeper <= %s" " required" % (pretty_server_version, max_version))
[docs] def make_client(conf): """Creates a `kazoo`_ `client`_ given a configuration dictionary. :param conf: configuration dictionary that will be used to configure the created client :type conf: dict The keys that will be extracted are: - ``read_only``: boolean that specifies whether to allow connections to read only servers, defaults to ``False`` - ``randomize_hosts``: boolean that specifies whether to randomize host lists provided, defaults to ``False`` - ``command_retry``: a kazoo `retry`_ object (or dict of options which will be used for creating one) that will be used for retrying commands that are executed - ``connection_retry``: a kazoo `retry`_ object (or dict of options which will be used for creating one) that will be used for retrying connection failures that occur - ``hosts``: a string, list, set (or dict with host keys) that will specify the hosts the kazoo client should be connected to, if none is provided then ``localhost:2181`` will be used by default - ``timeout``: a float value that specifies the default timeout that the kazoo client will use - ``handler``: a kazoo handler object that can be used to provide the client with alternate async strategies (the default is `thread`_ based, but `gevent`_, or `eventlet`_ ones can be provided as needed) - ``keyfile`` : SSL keyfile to use for authentication - ``keyfile_password``: SSL keyfile password - ``certfile``: SSL certfile to use for authentication - ``ca``: SSL CA file to use for authentication - ``use_ssl``: argument to control whether SSL is used or not - ``verify_certs``: when using SSL, argument to bypass certs verification .. _client: https://kazoo.readthedocs.io/en/latest/api/client.html .. _kazoo: https://kazoo.readthedocs.io/ .. _retry: https://kazoo.readthedocs.io/en/latest/api/retry.html .. _gevent: https://kazoo.readthedocs.io/en/latest/api/\ handlers/gevent.html .. _eventlet: https://kazoo.readthedocs.io/en/latest/api/\ handlers/eventlet.html .. _thread: https://kazoo.readthedocs.io/en/latest/api/\ handlers/threading.html """ # See: https://kazoo.readthedocs.io/en/latest/api/client.html client_kwargs = { 'logger': LOG, } for key, value_type_converter, default in CONF_TRANSFERS: if key in conf: if value_type_converter is not None: client_kwargs[key] = value_type_converter(conf[key], default=default) else: client_kwargs[key] = conf[key] else: client_kwargs[key] = default # See: https://kazoo.readthedocs.io/en/latest/api/retry.html if 'command_retry' in conf: client_kwargs['command_retry'] = conf['command_retry'] if 'connection_retry' in conf: client_kwargs['connection_retry'] = conf['connection_retry'] hosts = _parse_hosts(conf.get("hosts", "localhost:2181")) if not hosts or not isinstance(hosts, str): raise TypeError("Invalid hosts format, expected " "non-empty string/list, not '%s' (%s)" % (hosts, type(hosts))) client_kwargs['hosts'] = hosts if 'timeout' in conf: client_kwargs['timeout'] = float(conf['timeout']) # Kazoo supports various handlers, gevent, threading, eventlet... # allow the user of this client object to optionally specify one to be # used. if 'handler' in conf: client_kwargs['handler'] = conf['handler'] return client.KazooClient(**client_kwargs)