Source code for watcher.decision_engine.sync

# -*- encoding: utf-8 -*-
# Copyright (c) 2016 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import ast
import collections

from oslo_log import log

from watcher._i18n import _LI, _LW
from watcher.common import context
from watcher.decision_engine.loading import default
from watcher.decision_engine.scoring import scoring_factory
from watcher import objects

LOG = log.getLogger(__name__)

GoalMapping = collections.namedtuple(
    'GoalMapping', ['name', 'display_name', 'efficacy_specification'])
StrategyMapping = collections.namedtuple(
    'StrategyMapping',
    ['name', 'goal_name', 'display_name', 'parameters_spec'])
ScoringEngineMapping = collections.namedtuple(
    'ScoringEngineMapping',
    ['name', 'description', 'metainfo'])

IndicatorSpec = collections.namedtuple(
    'IndicatorSpec', ['name', 'description', 'unit', 'schema'])


[docs]class Syncer(object): """Syncs all available goals and strategies with the Watcher DB""" def __init__(self): self.ctx = context.make_context() self.discovered_map = None self._available_goals = None self._available_goals_map = None self._available_strategies = None self._available_strategies_map = None self._available_scoringengines = None self._available_scoringengines_map = None # This goal mapping maps stale goal IDs to the synced goal self.goal_mapping = dict() # This strategy mapping maps stale strategy IDs to the synced goal self.strategy_mapping = dict() # Maps stale scoring engine IDs to the synced scoring engines self.se_mapping = dict() self.stale_audit_templates_map = {} self.stale_audits_map = {} self.stale_action_plans_map = {} @property def available_goals(self): """Goals loaded from DB""" if self._available_goals is None: self._available_goals = objects.Goal.list(self.ctx) return self._available_goals @property def available_strategies(self): """Strategies loaded from DB""" if self._available_strategies is None: self._available_strategies = objects.Strategy.list(self.ctx) return self._available_strategies @property def available_scoringengines(self): """Scoring Engines loaded from DB""" if self._available_scoringengines is None: self._available_scoringengines = (objects.ScoringEngine .list(self.ctx)) return self._available_scoringengines @property def available_goals_map(self): """Mapping of goals loaded from DB""" if self._available_goals_map is None: self._available_goals_map = { GoalMapping( name=g.name, display_name=g.display_name, efficacy_specification=tuple( IndicatorSpec(**item) for item in g.efficacy_specification)): g for g in self.available_goals } return self._available_goals_map @property def available_strategies_map(self): if self._available_strategies_map is None: goals_map = {g.id: g.name for g in self.available_goals} self._available_strategies_map = { StrategyMapping( name=s.name, goal_name=goals_map[s.goal_id], display_name=s.display_name, parameters_spec=str(s.parameters_spec)): s for s in self.available_strategies } return self._available_strategies_map @property def available_scoringengines_map(self): if self._available_scoringengines_map is None: self._available_scoringengines_map = { ScoringEngineMapping( name=s.id, description=s.description, metainfo=s.metainfo): s for s in self.available_scoringengines } return self._available_scoringengines_map
[docs] def sync(self): self.discovered_map = self._discover() goals_map = self.discovered_map["goals"] strategies_map = self.discovered_map["strategies"] scoringengines_map = self.discovered_map["scoringengines"] for goal_name, goal_map in goals_map.items(): if goal_map in self.available_goals_map: LOG.info(_LI("Goal %s already exists"), goal_name) continue self.goal_mapping.update(self._sync_goal(goal_map)) for strategy_name, strategy_map in strategies_map.items(): if (strategy_map in self.available_strategies_map and strategy_map.goal_name not in [g.name for g in self.goal_mapping.values()]): LOG.info(_LI("Strategy %s already exists"), strategy_name) continue self.strategy_mapping.update(self._sync_strategy(strategy_map)) for se_name, se_map in scoringengines_map.items(): if se_map in self.available_scoringengines_map: LOG.info(_LI("Scoring Engine %s already exists"), se_name) continue self.se_mapping.update(self._sync_scoringengine(se_map)) self._sync_objects() self._soft_delete_removed_scoringengines()
def _sync_goal(self, goal_map): goal_name = goal_map.name goal_mapping = dict() # Goals that are matching by name with the given discovered goal name matching_goals = [g for g in self.available_goals if g.name == goal_name] stale_goals = self._soft_delete_stale_goals(goal_map, matching_goals) if stale_goals or not matching_goals: goal = objects.Goal(self.ctx) goal.name = goal_name goal.display_name = goal_map.display_name goal.efficacy_specification = [ indicator._asdict() for indicator in goal_map.efficacy_specification] goal.create() LOG.info(_LI("Goal %s created"), goal_name) # Updating the internal states self.available_goals_map[goal] = goal_map # Map the old goal IDs to the new (equivalent) goal for matching_goal in matching_goals: goal_mapping[matching_goal.id] = goal return goal_mapping def _sync_strategy(self, strategy_map): strategy_name = strategy_map.name strategy_display_name = strategy_map.display_name goal_name = strategy_map.goal_name parameters_spec = strategy_map.parameters_spec strategy_mapping = dict() # Strategies that are matching by name with the given # discovered strategy name matching_strategies = [s for s in self.available_strategies if s.name == strategy_name] stale_strategies = self._soft_delete_stale_strategies( strategy_map, matching_strategies) if stale_strategies or not matching_strategies: strategy = objects.Strategy(self.ctx) strategy.name = strategy_name strategy.display_name = strategy_display_name strategy.goal_id = objects.Goal.get_by_name(self.ctx, goal_name).id strategy.parameters_spec = parameters_spec strategy.create() LOG.info(_LI("Strategy %s created"), strategy_name) # Updating the internal states self.available_strategies_map[strategy] = strategy_map # Map the old strategy IDs to the new (equivalent) strategy for matching_strategy in matching_strategies: strategy_mapping[matching_strategy.id] = strategy return strategy_mapping def _sync_scoringengine(self, scoringengine_map): scoringengine_name = scoringengine_map.name se_mapping = dict() # Scoring Engines matching by id with discovered Scoring engine matching_scoringengines = [se for se in self.available_scoringengines if se.name == scoringengine_name] stale_scoringengines = self._soft_delete_stale_scoringengines( scoringengine_map, matching_scoringengines) if stale_scoringengines or not matching_scoringengines: scoringengine = objects.ScoringEngine(self.ctx) scoringengine.name = scoringengine_name scoringengine.description = scoringengine_map.description scoringengine.metainfo = scoringengine_map.metainfo scoringengine.create() LOG.info(_LI("Scoring Engine %s created"), scoringengine_name) # Updating the internal states self.available_scoringengines_map[scoringengine] = \ scoringengine_map # Map the old scoring engine names to the new (equivalent) SE for matching_scoringengine in matching_scoringengines: se_mapping[matching_scoringengine.name] = scoringengine return se_mapping def _sync_objects(self): # First we find audit templates, audits and action plans that are stale # because their associated goal or strategy has been modified and we # update them in-memory self._find_stale_audit_templates_due_to_goal() self._find_stale_audit_templates_due_to_strategy() self._find_stale_audits_due_to_goal() self._find_stale_audits_due_to_strategy() self._find_stale_action_plans_due_to_strategy() self._find_stale_action_plans_due_to_audit() # Then we handle the case where an audit template, an audit or an # action plan becomes stale because its related goal does not # exist anymore. self._soft_delete_removed_goals() # Then we handle the case where an audit template, an audit or an # action plan becomes stale because its related strategy does not # exist anymore. self._soft_delete_removed_strategies() # Finally, we save into the DB the updated stale audit templates # and soft delete stale audits and action plans for stale_audit_template in self.stale_audit_templates_map.values(): stale_audit_template.save() LOG.info(_LI("Audit Template '%s' synced"), stale_audit_template.name) for stale_audit in self.stale_audits_map.values(): stale_audit.save() LOG.info(_LI("Stale audit '%s' synced and cancelled"), stale_audit.uuid) for stale_action_plan in self.stale_action_plans_map.values(): stale_action_plan.save() LOG.info(_LI("Stale action plan '%s' synced and cancelled"), stale_action_plan.uuid) def _find_stale_audit_templates_due_to_goal(self): for goal_id, synced_goal in self.goal_mapping.items(): filters = {"goal_id": goal_id} stale_audit_templates = objects.AuditTemplate.list( self.ctx, filters=filters) # Update the goal ID for the stale audit templates (w/o saving) for audit_template in stale_audit_templates: if audit_template.id not in self.stale_audit_templates_map: audit_template.goal_id = synced_goal.id self.stale_audit_templates_map[audit_template.id] = ( audit_template) else: self.stale_audit_templates_map[ audit_template.id].goal_id = synced_goal.id def _find_stale_audit_templates_due_to_strategy(self): for strategy_id, synced_strategy in self.strategy_mapping.items(): filters = {"strategy_id": strategy_id} stale_audit_templates = objects.AuditTemplate.list( self.ctx, filters=filters) # Update strategy IDs for all stale audit templates (w/o saving) for audit_template in stale_audit_templates: if audit_template.id not in self.stale_audit_templates_map: audit_template.strategy_id = synced_strategy.id self.stale_audit_templates_map[audit_template.id] = ( audit_template) else: self.stale_audit_templates_map[ audit_template.id].strategy_id = synced_strategy.id def _find_stale_audits_due_to_goal(self): for goal_id, synced_goal in self.goal_mapping.items(): filters = {"goal_id": goal_id} stale_audits = objects.Audit.list( self.ctx, filters=filters, eager=True) # Update the goal ID for the stale audits (w/o saving) for audit in stale_audits: if audit.id not in self.stale_audits_map: audit.goal_id = synced_goal.id self.stale_audits_map[audit.id] = audit else: self.stale_audits_map[audit.id].goal_id = synced_goal.id def _find_stale_audits_due_to_strategy(self): for strategy_id, synced_strategy in self.strategy_mapping.items(): filters = {"strategy_id": strategy_id} stale_audits = objects.Audit.list( self.ctx, filters=filters, eager=True) # Update strategy IDs for all stale audits (w/o saving) for audit in stale_audits: if audit.id not in self.stale_audits_map: audit.strategy_id = synced_strategy.id audit.state = objects.audit.State.CANCELLED self.stale_audits_map[audit.id] = audit else: self.stale_audits_map[ audit.id].strategy_id = synced_strategy.id self.stale_audits_map[ audit.id].state = objects.audit.State.CANCELLED def _find_stale_action_plans_due_to_strategy(self): for strategy_id, synced_strategy in self.strategy_mapping.items(): filters = {"strategy_id": strategy_id} stale_action_plans = objects.ActionPlan.list( self.ctx, filters=filters) # Update strategy IDs for all stale action plans (w/o saving) for action_plan in stale_action_plans: if action_plan.id not in self.stale_action_plans_map: action_plan.strategy_id = synced_strategy.id action_plan.state = objects.action_plan.State.CANCELLED self.stale_action_plans_map[action_plan.id] = action_plan else: self.stale_action_plans_map[ action_plan.id].strategy_id = synced_strategy.id self.stale_action_plans_map[ action_plan.id].state = ( objects.action_plan.State.CANCELLED) def _find_stale_action_plans_due_to_audit(self): for audit_id, synced_audit in self.stale_audits_map.items(): filters = {"audit_id": audit_id} stale_action_plans = objects.ActionPlan.list( self.ctx, filters=filters) # Update audit IDs for all stale action plans (w/o saving) for action_plan in stale_action_plans: if action_plan.id not in self.stale_action_plans_map: action_plan.audit_id = synced_audit.id action_plan.state = objects.action_plan.State.CANCELLED self.stale_action_plans_map[action_plan.id] = action_plan else: self.stale_action_plans_map[ action_plan.id].audit_id = synced_audit.id self.stale_action_plans_map[ action_plan.id].state = ( objects.action_plan.State.CANCELLED) def _soft_delete_removed_goals(self): removed_goals = [ g for g in self.available_goals if g.name not in self.discovered_map['goals']] for removed_goal in removed_goals: removed_goal.soft_delete() filters = {"goal_id": removed_goal.id} invalid_ats = objects.AuditTemplate.list(self.ctx, filters=filters) for at in invalid_ats: LOG.warning( _LW("Audit Template '%(audit_template)s' references a " "goal that does not exist"), audit_template=at.uuid) stale_audits = objects.Audit.list( self.ctx, filters=filters, eager=True) for audit in stale_audits: LOG.warning( _LW("Audit '%(audit)s' references a " "goal that does not exist"), audit=audit.uuid) if audit.id not in self.stale_audits_map: audit.state = objects.audit.State.CANCELLED self.stale_audits_map[audit.id] = audit else: self.stale_audits_map[ audit.id].state = objects.audit.State.CANCELLED def _soft_delete_removed_strategies(self): removed_strategies = [ s for s in self.available_strategies if s.name not in self.discovered_map['strategies']] for removed_strategy in removed_strategies: removed_strategy.soft_delete() filters = {"strategy_id": removed_strategy.id} invalid_ats = objects.AuditTemplate.list(self.ctx, filters=filters) for at in invalid_ats: LOG.info( _LI("Audit Template '%(audit_template)s' references a " "strategy that does not exist"), audit_template=at.uuid) # In this case we can reset the strategy ID to None # so the audit template can still achieve the same goal # but with a different strategy if at.id not in self.stale_audit_templates_map: at.strategy_id = None self.stale_audit_templates_map[at.id] = at else: self.stale_audit_templates_map[at.id].strategy_id = None stale_audits = objects.Audit.list( self.ctx, filters=filters, eager=True) for audit in stale_audits: LOG.warning( _LW("Audit '%(audit)s' references a " "strategy that does not exist"), audit=audit.uuid) if audit.id not in self.stale_audits_map: audit.state = objects.audit.State.CANCELLED self.stale_audits_map[audit.id] = audit else: self.stale_audits_map[ audit.id].state = objects.audit.State.CANCELLED stale_action_plans = objects.ActionPlan.list( self.ctx, filters=filters) for action_plan in stale_action_plans: LOG.warning( _LW("Action Plan '%(action_plan)s' references a " "strategy that does not exist"), action_plan=action_plan.uuid) if action_plan.id not in self.stale_action_plans_map: action_plan.state = objects.action_plan.State.CANCELLED self.stale_action_plans_map[action_plan.id] = action_plan else: self.stale_action_plans_map[ action_plan.id].state = ( objects.action_plan.State.CANCELLED) def _soft_delete_removed_scoringengines(self): removed_se = [ se for se in self.available_scoringengines if se.name not in self.discovered_map['scoringengines']] for se in removed_se: LOG.info(_LI("Scoring Engine %s removed"), se.name) se.soft_delete() def _discover(self): strategies_map = {} goals_map = {} scoringengines_map = {} discovered_map = { "goals": goals_map, "strategies": strategies_map, "scoringengines": scoringengines_map} goal_loader = default.DefaultGoalLoader() implemented_goals = goal_loader.list_available() strategy_loader = default.DefaultStrategyLoader() implemented_strategies = strategy_loader.list_available() for goal_cls in implemented_goals.values(): goals_map[goal_cls.get_name()] = GoalMapping( name=goal_cls.get_name(), display_name=goal_cls.get_translatable_display_name(), efficacy_specification=tuple( IndicatorSpec(**indicator.to_dict()) for indicator in goal_cls.get_efficacy_specification( ).get_indicators_specifications())) for strategy_cls in implemented_strategies.values(): strategies_map[strategy_cls.get_name()] = StrategyMapping( name=strategy_cls.get_name(), goal_name=strategy_cls.get_goal_name(), display_name=strategy_cls.get_translatable_display_name(), parameters_spec=str(strategy_cls.get_schema())) for se in scoring_factory.get_scoring_engine_list(): scoringengines_map[se.get_name()] = ScoringEngineMapping( name=se.get_name(), description=se.get_description(), metainfo=se.get_metainfo()) return discovered_map def _soft_delete_stale_goals(self, goal_map, matching_goals): """Soft delete the stale goals :param goal_map: discovered goal map :type goal_map: :py:class:`~.GoalMapping` instance :param matching_goals: list of DB goals matching the goal_map :type matching_goals: list of :py:class:`~.objects.Goal` instances :returns: A list of soft deleted DB goals (subset of matching goals) :rtype: list of :py:class:`~.objects.Goal` instances """ goal_display_name = goal_map.display_name goal_name = goal_map.name goal_efficacy_spec = goal_map.efficacy_specification stale_goals = [] for matching_goal in matching_goals: if (matching_goal.efficacy_specification == goal_efficacy_spec and matching_goal.display_name == goal_display_name): LOG.info(_LI("Goal %s unchanged"), goal_name) else: LOG.info(_LI("Goal %s modified"), goal_name) matching_goal.soft_delete() stale_goals.append(matching_goal) return stale_goals def _soft_delete_stale_strategies(self, strategy_map, matching_strategies): strategy_name = strategy_map.name strategy_display_name = strategy_map.display_name parameters_spec = strategy_map.parameters_spec stale_strategies = [] for matching_strategy in matching_strategies: if (matching_strategy.display_name == strategy_display_name and matching_strategy.goal_id not in self.goal_mapping and matching_strategy.parameters_spec == ast.literal_eval(parameters_spec)): LOG.info(_LI("Strategy %s unchanged"), strategy_name) else: LOG.info(_LI("Strategy %s modified"), strategy_name) matching_strategy.soft_delete() stale_strategies.append(matching_strategy) return stale_strategies def _soft_delete_stale_scoringengines( self, scoringengine_map, matching_scoringengines): se_name = scoringengine_map.name se_description = scoringengine_map.description se_metainfo = scoringengine_map.metainfo stale_scoringengines = [] for matching_scoringengine in matching_scoringengines: if (matching_scoringengine.description == se_description and matching_scoringengine.metainfo == se_metainfo): LOG.info(_LI("Scoring Engine %s unchanged"), se_name) else: LOG.info(_LI("Scoring Engine %s modified"), se_name) matching_scoringengine.soft_delete() stale_scoringengines.append(matching_scoringengine) return stale_scoringengines