commit e8c1e33c1672df64e2645a17c55a8ec8dd1686d1 Author: Michal Piotrowski Date: Tue Aug 25 09:26:24 2020 +0200 Remove deprecated YAML config Change-Id: I3ea571c668b45fa0c8d874e0ea8ac355a98e072a Story: 2003180 Task: 39090 diff --git a/docker/Dockerfile b/docker/Dockerfile index bf0f36e..15d5b46 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,18 +1,14 @@ ARG DOCKER_IMAGE=monasca/notification ARG APP_REPO=https://review.opendev.org/openstack/monasca-notification - # Branch, tag or git hash to build from. ARG REPO_VERSION=master ARG CONSTRAINTS_BRANCH=master ARG COMMON_VERSION=master - # Extra Python3 dependencies. ARG EXTRA_DEPS="netaddr gevent==1.3.5 greenlet" - # Always start from `monasca-base` image and use specific tag of it. ARG BASE_TAG=master FROM monasca/base:$BASE_TAG - # Environment variables used for our service or wait scripts. ENV \ KAFKA_URI=kafka:9092 \ @@ -31,9 +27,7 @@ ENV \ STATSD_HOST=monasca-statsd \ STATSD_PORT=8125 \ STAY_ALIVE_ON_FAILURE="false" - # Copy all necessary files to proper locations. -COPY notification.yaml.j2 /etc/monasca/ - +COPY monasca-notification.conf.j2 /etc/monasca/ # Implement start script in `start.sh` file. -CMD ["/start.sh"] +CMD ["/start.sh"] \ No newline at end of file diff --git a/docker/build_image.sh b/docker/build_image.sh index 2c35055..59f2325 100755 --- a/docker/build_image.sh +++ b/docker/build_image.sh @@ -45,7 +45,7 @@ set -eo pipefail # Exit the script if any statement returns error. # $ ./build_image.sh master master refs/changes/19/595719/3 # Go to folder with Docker files. -REAL_PATH=$(python -c "import os,sys; print(os.path.realpath('$0'))") +REAL_PATH=$(python3 -c "import os,sys; print(os.path.realpath('$0'))") cd "$(dirname "$REAL_PATH")/../docker/" [ -z "$DOCKER_IMAGE" ] && \ diff --git a/docker/monasca-notification.conf.j2 b/docker/monasca-notification.conf.j2 new file mode 100644 index 0000000..b7f7256 --- /dev/null +++ b/docker/monasca-notification.conf.j2 @@ -0,0 +1,584 @@ +[DEFAULT] + +# +# From oslo.log +# + +# If set to true, the logging level will be set to DEBUG instead of the default +# INFO level. (boolean value) +# Note: This option can be changed without restarting. +#debug = false + +# The name of a logging configuration file. This file is appended to any +# existing logging configuration files. For details about logging configuration +# files, see the Python logging module documentation. Note that when logging +# configuration files are used then all logging configuration is set in the +# configuration file and other logging configuration options are ignored (for +# example, log-date-format). (string value) +# Note: This option can be changed without restarting. +# Deprecated group/name - [DEFAULT]/log_config +#log_config_append = + +# Defines the format string for %%(asctime)s in log records. Default: +# %(default)s . This option is ignored if log_config_append is set. (string +# value) +#log_date_format = %Y-%m-%d %H:%M:%S + +# (Optional) Name of log file to send logging output to. If no default is set, +# logging will go to stderr as defined by use_stderr. This option is ignored if +# log_config_append is set. (string value) +# Deprecated group/name - [DEFAULT]/logfile +#log_file = + +# (Optional) The base directory used for relative log_file paths. This option +# is ignored if log_config_append is set. (string value) +# Deprecated group/name - [DEFAULT]/logdir +#log_dir = + +# Uses logging handler designed to watch file system. When log file is moved or +# removed this handler will open a new log file with specified path +# instantaneously. It makes sense only if log_file option is specified and Linux +# platform is used. This option is ignored if log_config_append is set. (boolean +# value) +#watch_log_file = false + +# Use syslog for logging. Existing syslog format is DEPRECATED and will be +# changed later to honor RFC5424. This option is ignored if log_config_append is +# set. (boolean value) +#use_syslog = false + +# Enable journald for logging. If running in a systemd environment you may wish +# to enable journal support. Doing so will use the journal native protocol which +# includes structured metadata in addition to log messages.This option is +# ignored if log_config_append is set. (boolean value) +#use_journal = false + +# Syslog facility to receive log lines. This option is ignored if +# log_config_append is set. (string value) +#syslog_log_facility = LOG_USER + +# Use JSON formatting for logging. This option is ignored if log_config_append +# is set. (boolean value) +#use_json = false + +# Log output to standard error. This option is ignored if log_config_append is +# set. (boolean value) +#use_stderr = false + +# Log output to Windows Event Log. (boolean value) +#use_eventlog = false + +# The amount of time before the log files are rotated. This option is ignored +# unless log_rotation_type is setto "interval". (integer value) +#log_rotate_interval = 1 + +# Rotation interval type. The time of the last file change (or the time when the +# service was started) is used when scheduling the next rotation. (string value) +# Possible values: +# Seconds - +# Minutes - +# Hours - +# Days - +# Weekday - +# Midnight - +#log_rotate_interval_type = days + +# Maximum number of rotated log files. (integer value) +#max_logfile_count = 30 + +# Log file maximum size in MB. This option is ignored if "log_rotation_type" is +# not set to "size". (integer value) +#max_logfile_size_mb = 200 + +# Log rotation type. (string value) +# Possible values: +# interval - Rotate logs at predefined time intervals. +# size - Rotate logs once they reach a predefined size. +# none - Do not rotate log files. +#log_rotation_type = none + +# Format string to use for log messages with context. Used by +# oslo_log.formatters.ContextFormatter (string value) +#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s + +# Format string to use for log messages when context is undefined. Used by +# oslo_log.formatters.ContextFormatter (string value) +#logging_default_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [-] %(instance)s%(message)s + +# Additional data to append to log message when logging level for the message is +# DEBUG. Used by oslo_log.formatters.ContextFormatter (string value) +#logging_debug_format_suffix = %(funcName)s %(pathname)s:%(lineno)d + +# Prefix each line of exception output with this format. Used by +# oslo_log.formatters.ContextFormatter (string value) +#logging_exception_prefix = %(asctime)s.%(msecs)03d %(process)d ERROR %(name)s %(instance)s + +# Defines the format string for %(user_identity)s that is used in +# logging_context_format_string. Used by oslo_log.formatters.ContextFormatter +# (string value) +#logging_user_identity_format = %(user)s %(tenant)s %(domain)s %(user_domain)s %(project_domain)s + +# List of package logging levels in logger=LEVEL pairs. This option is ignored +# if log_config_append is set. (list value) +#default_log_levels = amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,oslo_messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,requests.packages.urllib3.util.retry=WARN,urllib3.util.retry=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN,taskflow=WARN,keystoneauth=WARN,oslo.cache=INFO,oslo_policy=INFO,dogpile.core.dogpile=INFO + +# Enables or disables publication of error events. (boolean value) +#publish_errors = false + +# The format for an instance that is passed with the log message. (string value) +#instance_format = "[instance: %(uuid)s] " + +# The format for an instance UUID that is passed with the log message. (string +# value) +#instance_uuid_format = "[instance: %(uuid)s] " + +# Interval, number of seconds, of log rate limiting. (integer value) +#rate_limit_interval = 0 + +# Maximum number of logged messages per rate_limit_interval. (integer value) +#rate_limit_burst = 0 + +# Log level name used by rate limiting: CRITICAL, ERROR, INFO, WARNING, DEBUG or +# empty string. Logs with level greater or equal to rate_limit_except_level are +# not filtered. An empty string means that all levels are filtered. (string +# value) +#rate_limit_except_level = CRITICAL + +# Enables or disables fatal status of deprecations. (boolean value) +#fatal_deprecations = false + + +[alarm_processor] +# Options to configure alarm processor. + +# +# From monasca_notification +# + +# DEPRECATED: Number of alarm processors to spawn (integer value) +# Minimum value: 1 +# This option is deprecated for removal since 1.8.0. +# Its value may be silently ignored in the future. +# Reason: Options is not used in the current code and will be removed in future +# releases. +number = {{ ALARM_PROCESSORS | default(2) }} + +# Alarms older than TTL are not processed by notification engine. (integer +# value) +# Advanced Option: intended for advanced users and not used +# by the majority of users, and might have a significant +# effect on stability and/or performance. +#ttl = 14400 + + +[database] +# Driver configuration for database connectivity. + +# +# From monasca_notification +# + +# Driver name (or full class path) that should be used to handle RDB +# connections. Accepts either short labels dict_keys(['postgresql', 'mysql', +# 'orm']) or full class names +# dict_values(['monasca_notification.common.repositories.postgres.pgsql_repo:PostgresqlRepo', +# 'monasca_notification.common.repositories.mysql.mysql_repo:MysqlRepo', +# 'monasca_notification.common.repositories.orm.orm_repo:OrmRepo']). Configuring +# either of them will require presence of one of following sections +# dict_keys(['postgresql', 'mysql', 'orm']) inside configuration file. (plugin +# value) +# Possible values: +# postgresql - +# monasca_notification.common.repositories.orm.orm_repo:OrmRepo - +# mysql - +# monasca_notification.common.repositories.mysql.mysql_repo:MysqlRepo - +# orm - + +# (string value) +#password = + +# (string value) +#custom_formatter = + +# (string value) +#proxy = + + +[kafka] +# Options under this group allow to configure valid connection or Kafka queue. + +# +# From monasca_notification +# + +# +# Comma separated list of Kafka broker host:port +# (list value) +#uri = {{ KAFKA_URI }} + + +# List of addresses (with ports) pointing at Kafka cluster. (list value) +url = {{ KAFKA_URI }} + +# Consumer's group for monasca-notification client. (string value) +# Advanced Option: intended for advanced users and not used +# by the majority of users, and might have a significant +# effect on stability and/or performance. +group = monasca-notification + +# Topic name in kafka where alarm transitions are stored. (string value) +# Advanced Option: intended for advanced users and not used +# by the majority of users, and might have a significant +# effect on stability and/or performance. +alarm_topic = alarm-state-transitions + +# Topic name in kafka where alarm notifications are stored. (string value) +# Advanced Option: intended for advanced users and not used +# by the majority of users, and might have a significant +# effect on stability and/or performance. +notification_topic = alarm-notifications + +# Topic name in kafka where notifications, that have failed to be sent and are +# waiting for retry operations, are stored. (string value) +# Advanced Option: intended for advanced users and not used +# by the majority of users, and might have a significant +# effect on stability and/or performance. +notification_retry_topic = retry-notifications + +# Dict of periodic topics. Keys are the period and values the actual topic names +# in kafka where notifications are stored. (dict value) +# Advanced Option: intended for advanced users and not used +# by the majority of users, and might have a significant +# effect on stability and/or performance. +periodic = 60:60-seconds-notifications + +# Maximum lag for topic that is acceptable by the monasca-notification. +# Notifications that are older than this offset are skipped. (integer value) +# Advanced Option: intended for advanced users and not used +# by the majority of users, and might have a significant +# effect on stability and/or performance. +max_offset_lag = 600 + +# Enable legacy Kafka client. When set old version of kafka-python library is +# used. Message format version for the brokers should be set to 0.9.0.0 to avoid +# performance issues until all consumers are upgraded. (boolean value) +# Advanced Option: intended for advanced users and not used +# by the majority of users, and might have a significant +# effect on stability and/or performance. +#legacy_kafka_client_enabled = true + + +[keystone] +# Options under this group allow to configure valid connection via +# Keystoneauthentication. + +# +# From monasca_notification +# + +# This option enable or disable authentication using keystone (boolean value) +#auth_required = false + +# URL of identity service (string value) +#auth_url = http://127.0.0.1/identity/v3 + +# Username (string value) +#username = admin + +# Password of identity service (string value) +#password = password + +# Name of project (string value) +#project_name = admin + +# User domain name (string value) +#user_domain_name = default + +# Project domain name (string value) +#project_domain_name = default + +# Type of authentication (string value) +#auth_type = password + + +[mysql] +# Configuration options to configure plain MySQL RBD driver. + +# +# From monasca_notification +# + +# IP address of MySQL instance. (host address value) +host = {{ MYSQL_HOST }} + +# Port number of MySQL instance. (port value) +# Minimum value: 0 +# Maximum value: 65535 +port = {{ MYSQL_PORT }} + +# Username to connect to MySQL instance and given database. (string value) +user = {{ MYSQL_USER }} + +# Password to connect to MySQL instance and given database. (string value) +passwd = {{ MYSQL_PASSWORD }} + +# A dict of arguments similar to mysql_ssl_set parameters. (dict value) +#ssl = + +# Database name available in given MySQL instance. (string value) +db = {{ MYSQL_DB }} + + +[notification_processor] +# Options to configure notification processor. + +# +# From monasca_notification +# + +# Number of notification processors to spawn. (integer value) +# Minimum value: 1 +number = {{ NOTIFICATION_PROCESSORS | default(2) }} + + +[orm] +# Configuration options to configure ORM RBD driver. + +# +# From monasca_notification +# + +# Connection string for sqlalchemy. (string value) +# url = + + +[postgresql] +# Configuration options to configure plain PostgreSQL RBD driver. + +# +# From monasca_notification +# + +# IP address of PostgreSQL instance. (host address value) +#host = 127.0.0.1 + +# Port number of PostgreSQL instance. (port value) +# Minimum value: 0 +# Maximum value: 65535 +#port = 5432 + +# Username to connect to PostgreSQL instance and given database. (string value) +#user = notification + +# Password to connect to PostgreSQL instance and given database (string value) +#password = password + +# Database name available in given PostgreSQL instance. (string value) +#database = mon + + +[queues] +# Options under this group allow to configure valid connection sizes of all +# queues. + +# +# From monasca_notification +# + +# Size of the alarms queue. (integer value) +# Minimum value: 1 +alarms_size = 256 + +# Size of the finished alarms queue. (integer value) +# Minimum value: 1 +finished_size = 256 + +# Size of notifications queue. (integer value) +# Minimum value: 1 +notifications_size = 256 + +# Size of sent notifications queue. Limiting this size reduces potential or re- +# sent notifications after a failure. (integer value) +# Minimum value: 1 +sent_notifications_size = 50 + + +[retry_engine] +# Options under this group allow to configure valid connection for retry engine. + +# +# From monasca_notification +# + +# How often should retry happen. (integer value) +# Minimum value: 30 +# Advanced Option: intended for advanced users and not used +# by the majority of users, and might have a significant +# effect on stability and/or performance. +# interval = {{ RETRY_INTERVAL | default(30) }} + +# How many times should retrying be tried. (integer value) +# Advanced Option: intended for advanced users and not used +# by the majority of users, and might have a significant +# effect on stability and/or performance. +# max_attempts = {{ RETRY_MAX_ATTEMPTS | default(5) }} + + +[slack_notifier] + +# +# From monasca_notification +# + +# (integer value) +# Minimum value: 1 +#timeout = 5 + +# (boolean value) +#insecure = true + +# (string value) +#ca_certs = + +# (string value) +#proxy = + +# (string value) +#message_template = + + +[statsd] +# Options under this group allow to configure valid connection to statsd server +# launched by monasca-agent. + +# +# From monasca_notification +# + +# Enable or disable self monitoring. (boolean value) +#enable = true + +# IP address of statsd server. (host address value) +#host = 127.0.0.1 + +# Port of statsd server. (port value) +# Minimum value: 0 +# Maximum value: 65535 +#port = 8125 + + +[zookeeper] +# Options under this group allow to configure settings for zookeeper handling. + +# +# From monasca_notification +# + +# List of addresses (with ports) pointing at zookeeper cluster. (list value) +url = {{ ZOOKEEPER_URL }} + +# Path in zookeeper tree to track notification offsets. (string value) +# Advanced Option: intended for advanced users and not used +# by the majority of users, and might have a significant +# effect on stability and/or performance. +notification_path = /notification/alarms + +# Path in zookeeper tree to track notification retries offsets. (string value) +# Advanced Option: intended for advanced users and not used +# by the majority of users, and might have a significant +# effect on stability and/or performance. +notification_retry_path = /notification/retry + +# Paths in zookeeper tree to track periodic offsets. Keys must be integers +# describing the interval of periodic notification. Values are actual paths +# inside zookeeper tree. (dict value) +# Advanced Option: intended for advanced users and not used +# by the majority of users, and might have a significant +# effect on stability and/or performance. +periodic_path = 60:/notification/60_seconds diff --git a/docker/notification.yaml.j2 b/docker/notification.yaml.j2 deleted file mode 100644 index 9c9ac8d..0000000 --- a/docker/notification.yaml.j2 +++ /dev/null @@ -1,153 +0,0 @@ -# (C) Copyright 2017 Hewlett Packard Enterprise Development LP -# Copyright 2018 Fujitsu LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -kafka: - url: {{ KAFKA_URI | default('kafka:9092') }} - group: "monasca-notification" - alarm_topic: "alarm-state-transitions" - notification_topic: "alarm-notifications" - notification_retry_topic: "retry-notifications" - periodic: - 60: 60-seconds-notifications - max_offset_lag: 600 # In seconds, undefined for none - -mysql: - host: "{{ MYSQL_HOST | default('mysql') }}" - port: {{ MYSQL_PORT | default('3306') }} - user: "{{ MYSQL_USER | default('notification') }}" - passwd: "{{ MYSQL_PASSWORD | default('password') }}" - db: "{{ MYSQL_DB | default('mon') }}" - -notification_types: -{% if NF_PLUGINS %} - {% set plugins = NF_PLUGINS.split(',')|map('trim')|list %} - plugins: - {% if 'email' in plugins %} - - monasca_notification.plugins.email_notifier:EmailNotifier - {% endif %} - {% if 'webhook' in plugins %} - - monasca_notification.plugins.webhook_notifier:WebhookNotifier - {% endif %} - {% if 'pagerduty' in plugins %} - - monasca_notification.plugins.pagerduty_notifier:PagerdutyNotifier - {% endif %} - {% if 'hipchat' in plugins %} - - monasca_notification.plugins.hipchat_notifier:HipChatNotifier - {% endif %} - {% if 'slack' in plugins %} - - monasca_notification.plugins.slack_notifier:SlackNotifier - {% endif %} -{% else %} - plugins: [] - {% set plugins = [] %} -{% endif %} - -{% if 'email' in plugins %} - email: - server: "{{ NF_EMAIL_SERVER }}" - port: {{ NF_EMAIL_PORT | default(25) }} - user: "{{ NF_EMAIL_USER }}" - password: "{{ NF_EMAIL_PASSWORD }}" - timeout: {{ NF_EMAIL_TIMEOUT | default(15) }} - from_addr: "{{ NF_EMAIL_FROM_ADDR }}" - grafana_url: "{{ NF_EMAIL_GRAFANA_URL }}" -{% endif -%} - -{% if 'webhook' in plugins %} - webhook: - timeout: {{ NF_WEBHOOK_TIMEOUT | default(5) }} -{% endif -%} - -{% if 'pagerduty' in plugins %} - pagerduty: - timeout: {{ NF_PAGERDUTY_TIMEOUT | default(5) }} - url: "{{ NF_PAGERDUTY_URL | default('https://events.pagerduty.com/generic/2010-04-15/create_event.json') }}" -{% endif -%} - -{% if 'hipchat' in plugins %} - hipchat: - timeout: {{ NF_HIPCHAT_TIMEOUT | default(5) }} - ca_certs: "{{ NF_HIPCHAT_SSL_CERTS | default('/etc/ssl/certs/ca-certificates.crt') }}" - insecure: {{ NF_HIPCHAT_INSECURE | default('false') }} - {% if NF_HIPCHAT_PROXY %} - proxy: {{ NF_HIPCHAT_PROXY }} - {% endif %} - {% if NF_HIPCHAT_TEMPLATE %} - template: "{{ NF_HIPCHAT_TEMPLATE }}" - {% endif %} -{% endif -%} - -{% if 'slack' in plugins %} - slack: - timeout: {{ NF_SLACK_TIMEOUT | default(5) }} - ca_certs: "{{ NF_SLACK_CA_CERTS | default('/etc/ssl/certs/ca-certificates.crt') }}" - insecure: {{ NF_SLACK_INSECURE | default('false') }} - {% if NF_SLACK_PROXY %} - proxy: {{ NF_SLACK_PROXY }} - {% endif %} -{% endif %} - -processors: - alarm: - number: {{ ALARM_PROCESSORS | default(2) }} - - # In seconds, undefined for none. Alarms older than this are not processed - ttl: 14400 - notification: - number: {{ NOTIFICATION_PROCESSORS | default(2) }} - -retry: - interval: {{ RETRY_INTERVAL | default(30) }} - max_attempts: {{ RETRY_MAX_ATTEMPTS | default(5) }} - -queues: - alarms_size: 256 - finished_size: 256 - notifications_size: 256 - sent_notifications_size: 50 - -zookeeper: - url: "{{ ZOOKEEPER_URL | default('zookeeper:2181') }}" - notification_path: "/notification/alarms" - notification_retry_path: "/notification/retry" - periodic_path: - 60: /notification/60_seconds - -logging: # Used in logging.dictConfig - version: 1 - disable_existing_loggers: False - formatters: - default: - format: "%(asctime)s %(levelname)s %(name)s %(message)s" - handlers: - console: - class: logging.StreamHandler - formatter: default - loggers: - kazoo: - level: WARN - kafka: - level: WARN - statsd: - level: WARN - root: - handlers: - - console - level: {{ LOG_LEVEL | default('WARN') }} - -statsd: - enable: {{ STATSD_ENABLE }} - host: "{{ STATSD_HOST }}" - port: {{ STATSD_PORT }} diff --git a/docker/start.sh b/docker/start.sh index 9a72353..5391393 100644 --- a/docker/start.sh +++ b/docker/start.sh @@ -26,11 +26,17 @@ python3 /mysql_check.py # Template all config files before start, it will use env variables. # Read usage examples: https://pypi.org/project/Templer/ echo "Start script: creating config files from templates" -templer -v -f /etc/monasca/notification.yaml.j2 /etc/monasca/notification.yaml +templer -v -f /etc/monasca/monasca-notification.conf.j2 /etc/monasca/monasca-notification.conf + + +echo "Checking files" +cd /etc/monasca/ +ls -l +echo "Checked already" # Start our service. echo "Start script: starting container" -monasca-notification /etc/monasca/notification.yaml +monasca-notification --config-file /etc/monasca/monasca-notification.conf # Allow server to stay alive in case of failure for 2 hours for debugging. EXIT_CODE=$?