commit 1643979f79a967c5eaae9c267b5998bc2c6d8fdb Author: Alex Kavanagh Date: Sat Sep 26 18:27:02 2020 +0100 Sync libraries & common files prior to freeze * charm-helpers sync for classic charms * charms.ceph sync for ceph charms * rebuild for reactive charms * sync tox.ini files as needed * sync requirements.txt files to sync to standard Change-Id: I79af80dd7a0faa9175c9dca1fac669f8c187e3f5 diff --git a/hooks/charmhelpers/contrib/network/ovs/__init__.py b/hooks/charmhelpers/contrib/network/ovs/__init__.py index acb503b..3004125 100644 --- a/hooks/charmhelpers/contrib/network/ovs/__init__.py +++ b/hooks/charmhelpers/contrib/network/ovs/__init__.py @@ -13,6 +13,7 @@ # limitations under the License. ''' Helpers for interacting with OpenvSwitch ''' +import collections import hashlib import os import re @@ -20,9 +21,9 @@ import six import subprocess from charmhelpers import deprecate +from charmhelpers.contrib.network.ovs import ovsdb as ch_ovsdb from charmhelpers.fetch import apt_install - from charmhelpers.core.hookenv import ( log, WARNING, INFO, DEBUG ) @@ -592,3 +593,76 @@ def ovs_appctl(target, args): cmd = ['ovs-appctl', '-t', target] cmd.extend(args) return subprocess.check_output(cmd, universal_newlines=True) + + +def uuid_for_port(port_name): + """Get UUID of named port. + + :param port_name: Name of port. + :type port_name: str + :returns: Port UUID. + :rtype: Optional[uuid.UUID] + """ + for port in ch_ovsdb.SimpleOVSDB( + 'ovs-vsctl').port.find('name={}'.format(port_name)): + return port['_uuid'] + + +def bridge_for_port(port_uuid): + """Find which bridge a port is on. + + :param port_uuid: UUID of port. + :type port_uuid: uuid.UUID + :returns: Name of bridge or None. + :rtype: Optional[str] + """ + for bridge in ch_ovsdb.SimpleOVSDB( + 'ovs-vsctl').bridge: + # If there is a single port on a bridge the ports property will not be + # a list. ref: juju/charm-helpers#510 + if (isinstance(bridge['ports'], list) and + port_uuid in bridge['ports'] or + port_uuid == bridge['ports']): + return bridge['name'] + + +PatchPort = collections.namedtuple('PatchPort', ('bridge', 'port')) +Patch = collections.namedtuple('Patch', ('this_end', 'other_end')) + + +def patch_ports_on_bridge(bridge): + """Find patch ports on a bridge. + + :param bridge: Name of bridge + :type bridge: str + :returns: Iterator with bridge and port name for both ends of a patch. + :rtype: Iterator[Patch[PatchPort[str,str],PatchPort[str,str]]] + :raises: ValueError + """ + # On any given vSwitch there will be a small number of patch ports, so we + # start by iterating over ports with type `patch` then look up which bridge + # they belong to and act on any ports that match the criteria. + for interface in ch_ovsdb.SimpleOVSDB( + 'ovs-vsctl').interface.find('type=patch'): + for port in ch_ovsdb.SimpleOVSDB( + 'ovs-vsctl').port.find('name={}'.format(interface['name'])): + if bridge_for_port(port['_uuid']) == bridge: + this_end = PatchPort(bridge, port['name']) + other_end = PatchPort(bridge_for_port( + uuid_for_port( + interface['options']['peer'])), + interface['options']['peer']) + yield(Patch(this_end, other_end)) + # We expect one result and it is ok if it turns out to be a port + # for a different bridge. However we need a break here to satisfy + # the for/else check which is in place to detect interface refering + # to non-existent port. + break + else: + raise ValueError('Port for interface named "{}" does unexpectedly ' + 'not exist.'.format(interface['name'])) + else: + # Allow our caller to handle no patch ports found gracefully, in + # reference to PEP479 just doing a return will provide a emtpy iterator + # and not None. + return diff --git a/hooks/charmhelpers/contrib/network/ovs/ovsdb.py b/hooks/charmhelpers/contrib/network/ovs/ovsdb.py index 5e50bc3..2f1e53d 100644 --- a/hooks/charmhelpers/contrib/network/ovs/ovsdb.py +++ b/hooks/charmhelpers/contrib/network/ovs/ovsdb.py @@ -36,6 +36,11 @@ class SimpleOVSDB(object): for br in ovsdb.bridge: if br['name'] == 'br-test': ovsdb.bridge.set(br['uuid'], 'external_ids:charm', 'managed') + + WARNING: If a list type field only have one item `ovs-vsctl` will present + it as a single item. Since we do not know the schema we have no way of + knowing what fields should be de-serialized as lists so the caller has + to be careful of checking the type of values returned from this library. """ # For validation we keep a complete map of currently known good tool and @@ -157,23 +162,59 @@ class SimpleOVSDB(object): self._tool = tool self._table = table - def _find_tbl(self, condition=None): - """Run and parse output of OVSDB `find` command. + def _deserialize_ovsdb(self, data): + """Deserialize OVSDB RFC7047 section 5.1 data. - :param condition: An optional RFC 7047 5.1 match condition - :type condition: Optional[str] - :returns: Dictionary with data - :rtype: Dict[str, any] + :param data: Multidimensional list where first row contains RFC7047 + type information + :type data: List[str,any] + :returns: Deserialized data. + :rtype: any """ # When using json formatted output to OVS commands Internal OVSDB # notation may occur that require further deserializing. # Reference: https://tools.ietf.org/html/rfc7047#section-5.1 ovs_type_cb_map = { 'uuid': uuid.UUID, - # FIXME sets also appear to sometimes contain type/value tuples + # NOTE: OVSDB sets have overloaded type + # see special handling below 'set': list, 'map': dict, } + assert len(data) > 1, ('Invalid data provided, expecting list ' + 'with at least two elements.') + if data[0] == 'set': + # special handling for set + # + # it is either a list of strings or a list of typed lists. + # taste first element to see which it is + for el in data[1]: + # NOTE: We lock this handling down to the `uuid` type as + # that is the only one we have a practical example of. + # We could potentially just handle this generally based on + # the types listed in `ovs_type_cb_map` but let's open for + # that as soon as we have a concrete example to validate on + if isinstance( + el, list) and len(el) and el[0] == 'uuid': + decoded_set = [] + for el in data[1]: + decoded_set.append(self._deserialize_ovsdb(el)) + return(decoded_set) + # fall back to normal processing below + break + + # Use map to deserialize data with fallback to `str` + f = ovs_type_cb_map.get(data[0], str) + return f(data[1]) + + def _find_tbl(self, condition=None): + """Run and parse output of OVSDB `find` command. + + :param condition: An optional RFC 7047 5.1 match condition + :type condition: Optional[str] + :returns: Dictionary with data + :rtype: Dict[str, any] + """ cmd = [self._tool, '-f', 'json', 'find', self._table] if condition: cmd.append(condition) @@ -182,9 +223,8 @@ class SimpleOVSDB(object): for row in data['data']: values = [] for col in row: - if isinstance(col, list): - f = ovs_type_cb_map.get(col[0], str) - values.append(f(col[1])) + if isinstance(col, list) and len(col) > 1: + values.append(self._deserialize_ovsdb(col)) else: values.append(col) yield dict(zip(data['headings'], values)) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 0e41a9f..54aed7f 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -3245,6 +3245,18 @@ class CephBlueStoreCompressionContext(OSContextGenerator): """ return self.op + def get_kwargs(self): + """Get values for use as keyword arguments. + + :returns: Context values with key suitable for use as kwargs to + CephBrokerRq add_op_create_*_pool methods. + :rtype: Dict[str,any] + """ + return { + k.replace('-', '_'): v + for k, v in self.op.items() + } + def validate(self): """Validate options. diff --git a/hooks/charmhelpers/contrib/python/__init__.py b/hooks/charmhelpers/contrib/python/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index d9d4357..526b95a 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -705,12 +705,12 @@ class ErasurePool(BasePool): # from different handling of this in the `charms.ceph` library. self.erasure_code_profile = op.get('erasure-profile', 'default-canonical') + self.allow_ec_overwrites = op.get('allow-ec-overwrites') else: # We keep the class default when initialized from keyword arguments # to not break the API for any other consumers. self.erasure_code_profile = erasure_code_profile or 'default' - - self.allow_ec_overwrites = allow_ec_overwrites + self.allow_ec_overwrites = allow_ec_overwrites def _create(self): # Try to find the erasure profile information in order to properly @@ -1972,12 +1972,14 @@ class CephBrokerRq(object): 'request-id': self.request_id}) def _ops_equal(self, other): + keys_to_compare = [ + 'replicas', 'name', 'op', 'pg_num', 'group-permission', + 'object-prefix-permissions', + ] + keys_to_compare += list(self._partial_build_common_op_create().keys()) if len(self.ops) == len(other.ops): for req_no in range(0, len(self.ops)): - for key in [ - 'replicas', 'name', 'op', 'pg_num', 'weight', - 'group', 'group-namespace', 'group-permission', - 'object-prefix-permissions']: + for key in keys_to_compare: if self.ops[req_no].get(key) != other.ops[req_no].get(key): return False else: diff --git a/requirements.txt b/requirements.txt index 2316401..8ba1941 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,7 @@ # requirements. They are intertwined. Also, Zaza itself should specify # all of its own requirements and if it doesn't, fix it there. # +setuptools<50.0.0 # https://github.com/pypa/setuptools/commit/04e3df22df840c6bb244e9b27bc56750c44b7c85 pbr>=1.8.0,<1.9.0 simplejson>=2.2.0 netifaces>=0.10.4 diff --git a/test-requirements.txt b/test-requirements.txt index 44b5023..56fbf92 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -7,6 +7,7 @@ # requirements. They are intertwined. Also, Zaza itself should specify # all of its own requirements and if it doesn't, fix it there. # +setuptools<50.0.0 # https://github.com/pypa/setuptools/commit/04e3df22df840c6bb244e9b27bc56750c44b7c85 charm-tools>=2.4.4 requests>=2.18.4 mock>=1.2 diff --git a/tox.ini b/tox.ini index 8080ba6..e2d58f5 100644 --- a/tox.ini +++ b/tox.ini @@ -116,5 +116,5 @@ commands = functest-run-suite --keep-model --bundle {posargs} [flake8] -ignore = E402,E226,W504 +ignore = E402,E226,W503,W504 exclude = */charmhelpers