#!/bin/sh

set -e
#set -x

usage () {
	echo "Usage: $0 <CLUSTER_NAME> <FROM_OS_RELEASE> <TO_OS_RELEASE>"
	exit 1
}

if [ $# != 3 ] ; then
	usage
fi

RED="\033[1;31m"
NO_COL="\033[0m"
GREEN="\033[1;32m"
green_echo () {
	echo ${GREEN}${1}${NO_COL}
}
red_echo () {
	echo ${RED}${1}${NO_COL}
}

ALL_SERVICES_CTRL="aodh-api aodh-evaluator aodh-expirer aodh-listener aodh-notifier barbican-api barbican-keystone-listener barbican-worker ceilometer-agent-central ceilometer-agent-notification ceilometer-polling cinder-api cinder-backup cinder-scheduler cinder-volume cloudkitty-api cloudkitty-processor glance-api gnocchi-api gnocchi-metricd gnocchi-statsd heat-api-cfn heat-api heat-engine keystone magnum-api magnum-conductor neutron-api neutron-dhcp-agent neutron-l3-agent neutron-metadata-agent neutron-metering-agent neutron-openvswitch-agent neutron-rpc-server nova-api nova-api-metadata nova-compute nova-conductor nova-novncproxy nova-scheduler octavia-api octavia-health-manager octavia-housekeeping octavia-worker panko-api placement-api neutron-api neutron-rpc-server"
ALL_SERVICES_VOL="cinder-backup cinder-volume"
ALL_SERVICES_NET="neutron-dhcp-agent neutron-l3-agent neutron-metadata-agent neutron-metering-agent neutron-openvswitch-agent"
ALL_SERVICES_COMP="ceilometer-agent-compute cinder-backup cinder-volume neutron-dhcp-agent neutron-l3-agent neutron-metadata-agent neutron-metering-agent neutron-openvswitch-agent nova-compute"
ALL_SERVICES_MSG="gnocchi-api gnocchi-metricd gnocchi-statsd cloudkitty-api cloudkitty-processor"

CLUSTER_NAME=${1}
FROM_OS_RELEASE=${2}
TO_OS_RELEASE=${3}
CUR_REL=$(lsb_release -c -s)
FROM_DEB_RELEASE=${CUR_REL}
TO_DEB_RELEASE=${CUR_REL}

ME=$(basename $0)

green_echo "---> Fetching cluster info"
TMP_CLUSTER_SHOW=$(mktemp -t ${ME}-cluster-show.XXXXXX)
ocicli -csv cluster-show ${CLUSTER_NAME} >${TMP_CLUSTER_SHOW}

DOMAIN_NAME=$(cat ${TMP_CLUSTER_SHOW} | grep Domain: | cut -d, -f2)
HAS_BARBICAN=$(cat ${TMP_CLUSTER_SHOW} | grep "Install Barbican:" | cut -d, -f2)
HAS_HEAT=$(cat ${TMP_CLUSTER_SHOW} | grep "Install Heat:" | cut -d, -f2)
HAS_DESIGNATE=$(cat ${TMP_CLUSTER_SHOW} | grep "Install designate:" | cut -d, -f2)
HAS_OCTAVIA=$(cat ${TMP_CLUSTER_SHOW} | grep "Install octavia:" | cut -d, -f2)
HAS_TELEMETRY=$(cat ${TMP_CLUSTER_SHOW} | grep "Install telemetry:" | cut -d, -f2)
HAS_CLOUDKITTY_DASHBOARD=$(cat ${TMP_CLUSTER_SHOW} | grep "Install cloudkitty dashboard:" | cut -d, -f2)
USE_EXTERNAL_KEYSTONE=$(cat ${TMP_CLUSTER_SHOW} | grep "External keystone activate:" | cut -d, -f2)

# Make a survey of all the hosts we have in the cluster
green_echo "---> Fetching nodes info"
TMP_MACHINE_LIST=$(mktemp -t ${ME}-machine-list.XXXXXX)
ocicli -csv machine-list -a >${TMP_MACHINE_LIST}

ALL_CTRL=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='controller' AND cluster='${CLUSTER_NAME}' AND status='installed' ORDER BY hostname" | tr '\n' ' ')
ONE_CTRL=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='controller' AND cluster='${CLUSTER_NAME}' AND status='installed' ORDER BY hostname LIMIT 1" | tr '\n' ' ')
OTHER_CTRL=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='controller' AND cluster='${CLUSTER_NAME}' AND status='installed' AND hostname NOT LIKE '${CLUSTER_NAME}-controller-1.%' ORDER BY hostname" | tr '\n' ' ')
ONE_CTRL_IP=$(echo ${ONE_CTRL} | cut -d, -f1)
ONE_CTRL_HOST=$(echo ${ONE_CTRL} | cut -d, -f2)
ALL_COMP=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='compute' AND cluster='${CLUSTER_NAME}' AND status='installed' ORDER BY hostname" | tr '\n' ' ')
ALL_VOL=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='volume' AND cluster='${CLUSTER_NAME}' AND status='installed' ORDER BY hostname" | tr '\n' ' ')
ALL_NET=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='network' AND cluster='${CLUSTER_NAME}' AND status='installed' ORDER BY hostname" | tr '\n' ' ')
ALL_MSG=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='messaging' AND cluster='${CLUSTER_NAME}' AND status='installed' ORDER BY hostname" | tr '\n' ' ')
ONE_MSG=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='messaging' AND cluster='${CLUSTER_NAME}' AND status='installed' ORDER BY hostname LIMIT 1" | tr '\n' ' ')
ONE_MSG_IP=$(echo ${ONE_MSG} | cut -d, -f1)
ONE_MSG_HOST=$(echo ${ONE_MSG} | cut -d, -f2)
OTHER_MSG=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='messaging' AND cluster='${CLUSTER_NAME}' AND status='installed' AND hostname NOT LIKE '${CLUSTER_NAME}-messaging-1.%' ORDER BY hostname" | tr '\n' ' ')
ALL_SWIFTPRX=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='swiftproxy' AND cluster='${CLUSTER_NAME}' AND status='installed' ORDER BY hostname" | tr '\n' ' ')
ALL_SWIFTSTR=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='swiftstore' AND cluster='${CLUSTER_NAME}' AND status='installed' ORDER BY hostname" | tr '\n' ' ')
ALL_NODES=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE cluster='${CLUSTER_NAME}' AND status='installed' ORDER BY hostname" | tr '\n' ' ')
ALL_NODES_BUT_CEPH=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE cluster='${CLUSTER_NAME}' AND status='installed' AND role NOT LIKE 'ceph%' AND role NOT LIKE 'bill%' ORDER BY hostname" | tr '\n' ' ')
ALL_TEMPEST=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='tempest' AND cluster='${CLUSTER_NAME}' AND status='installed' ORDER BY hostname" | tr '\n' ' ')

# Get to know if we have specific nodes for:
# - volume
# - network
# - cephosd
if [ $(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT COUNT(serial) FROM - WHERE role='cephosd'") = "0" ] ; then
	HAS_OSD="no"
else
	HAS_OSD="yes"
fi

if [ $(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT COUNT(serial) FROM - WHERE role='volume'") = "0" ] ; then
	HAS_VOL="no"
else
	HAS_VOL="yes"
fi

if [ $(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT COUNT(serial) FROM - WHERE role='network'") = "0" ] ; then
	HAS_NET="no"
else
	HAS_NET="yes"
fi

rm -f ${TMP_MACHINE_LIST}

green_echo "---> Checking if cluster uses self-signed cert"
if [ $(ocicli cluster-show ${CLUSTER_NAME} | grep "Self signed API cert:" | awk '{print $5}')"" = "yes" ] ; then
	SELF_SIGNED_API_CERT=yes
else
	SELF_SIGNED_API_CERT=no
fi

rm -f ${TMP_CLUSTER_SHOW}

#########################
### GENERIC FUNCTIONS ###
#########################

sshi () {
	unset SSH_AUTH_SOCK
	ssh -i /etc/openstack-cluster-installer/id_rsa -o ConnectTimeout=10 $@
}

scpi () {
	scp -i /etc/openstack-cluster-installer/id_rsa $@
}

wait_for_ssh () {
	local COUNT CYCLES OTCI_CAN_SSH SSH_HOST
	SYSUSERNAME=root
	# This is 15 minutes
	COUNT=900
	CYCLES=0
	OTCI_CAN_SSH=no
	SSH_HOST=${1}
	echo -n "${GREEN}---> Waiting for ssh to be up for ${SSH_HOST}:${NO_COL}"
	ssh-keygen -f ~/.ssh/known_hosts -R ${SSH_HOST} 1>/dev/null 2>/dev/null || true
	while [ "${OTCI_CAN_SSH}" != "yes" ] && [ ${COUNT} != 0 ] ; do
		if ssh -i /etc/openstack-cluster-installer/id_rsa -o "ConnectTimeout 2" ${SYSUSERNAME}@${SSH_HOST} 'echo -n ""' 2>/dev/null ; then
			OTCI_CAN_SSH=yes
			echo "${GREEN}ok.${NO_COL}"
		else
			COUNT=$(( ${COUNT} - 1 ))
			CYCLES=$(( ${CYCLES} + 1 ))
			sleep 2
			echo -n "."
		fi
	done
	if [ ${OTCI_CAN_SSH} = "yes" ] ; then
		sleep 1
	else
		echo "${RED}timeout.${NO_COL}"
		exit 1
	fi
}

disable_puppet () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)

	green_echo "-> ${HOST_NAME}"
	sshi root@${HOST_IP} "systemctl stop puppet"
	sshi root@${HOST_IP} "systemctl disable puppet"
	sshi root@${HOST_IP} "systemctl mask puppet"
	sshi root@${HOST_IP} "rm -f /usr/sbin/policy-rc.d"
}

enable_puppet () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)

	green_echo "-> ${HOST_NAME}"
	sshi root@${HOST_IP} "systemctl unmask puppet"
	sshi root@${HOST_IP} "systemctl enable puppet"
	sshi root@${HOST_IP} "systemctl start puppet"
}

switch_to_release () {
	local HOST HOST_IP HOST_NAME FROM_DEB_RELEASE FROM_OS_RELEASE TO_DEB_RELEASE TO_OS_RELEASE
	HOST=${1}
	FROM_DEB_RELEASE=${2}
	FROM_OS_RELEASE=${3}
	TO_DEB_RELEASE=${4}
	TO_OS_RELEASE=${5}

	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)

	green_echo "-> Switching ${HOST_NAME} / ${HOST_IP} to ${TO_DEB_RELEASE}-${TO_OS_RELEASE}"
	sshi root@${HOST_IP} "if [ -e /etc/apt/sources.list.d/${FROM_DEB_RELEASE}-${FROM_OS_RELEASE}.list ] ; then mv /etc/apt/sources.list.d/${FROM_DEB_RELEASE}-${FROM_OS_RELEASE}.list /etc/apt/sources.list.d/${TO_DEB_RELEASE}-${TO_OS_RELEASE}.list ; fi"
	sshi root@${HOST_IP} "sed -i \"s/${FROM_OS_RELEASE}/${TO_OS_RELEASE}/g\" /etc/apt/sources.list.d/${TO_DEB_RELEASE}-${TO_OS_RELEASE}.list"
	sshi root@${HOST_IP} "curl http://${TO_DEB_RELEASE}-${TO_OS_RELEASE}.debian.net/debian/dists/pubkey.gpg >pubkey.txt ; apt-key add pubkey.txt ; rm pubkey.txt"
	sshi root@${HOST_IP} "apt-get update"
}

# Example call:
# iterate_on_hosts "192.168.101.2,z-controller-1 192.168.101.3,z-controller-2" switch_to_release buster rocky buster stein
# Example transformation for the first host:
# switch_to_release 192.168.101.2,z-controller-1 buster rocky buster stein
iterate_on_hosts () {
	local HOST_LIST FUNCTION USE_PARALLEL
	if [ "${1}" = "-p" ] || [ "${1}" = "--parallel" ] ; then
		USE_PARALLEL=yes
		shift
	else
		USE_PARALLEL=no
	fi
	HOST_LIST=${1}
	FUNCTION=${2}
	shift
	shift

	MAX_PARALLEL=8

	if [ "${USE_PARALLEL}" = "yes" ] ; then
		CNT=1
		for i in ${HOST_LIST} ; do
			${FUNCTION} ${i} $@ &
			CNT=$(( ${CNT} + 1))
			if [ ${CNT} -gt ${MAX_PARALLEL} ] ; then
				CNT=1
				wait
			fi
		done
		wait
	else
		for i in ${HOST_LIST} ; do
			${FUNCTION} ${i} $@
		done
	fi
}

#####################
### OCI FUNCTIONS ###
#####################

switch_oci_to_release () {
	local FROM_DEB_RELEASE FROM_OS_RELEASE TO_DEB_RELEASE TO_OS_RELEASE
	FROM_DEB_RELEASE=${1}
	FROM_OS_RELEASE=${2}
	TO_DEB_RELEASE=${3}
	TO_OS_RELEASE=${4}

	green_echo "===> Switch OCI to use ${TO_OS_RELEASE}"
	if [ -e /etc/apt/sources.list.d/${FROM_DEB_RELEASE}-${FROM_OS_RELEASE}.list ] ; then
		mv /etc/apt/sources.list.d/${FROM_DEB_RELEASE}-${FROM_OS_RELEASE}.list /etc/apt/sources.list.d/${TO_DEB_RELEASE}-${TO_OS_RELEASE}.list
	fi
	# That one should be the standard on all OCI setup
	if [ -e /etc/apt/sources.list.d/${TO_DEB_RELEASE}-${TO_OS_RELEASE}.list ] ; then
		sed -i "s/${FROM_OS_RELEASE}/${TO_OS_RELEASE}/g" /etc/apt/sources.list.d/${TO_DEB_RELEASE}-${TO_OS_RELEASE}.list
	fi
	# That one is used in the PoC
	if [ -e /etc/apt/sources.list.d/openstack.list ] ; then
		sed -i "s/${FROM_OS_RELEASE}/${TO_OS_RELEASE}/g" /etc/apt/sources.list.d/openstack.list
	fi
	wget http://${TO_DEB_RELEASE}-${TO_OS_RELEASE}.debian.net/debian/dists/pubkey.gpg
	apt-key add pubkey.gpg
	rm pubkey.gpg
	apt-get update
	DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y -o Dpkg::Options::=--force-confold --allow-downgrades
	if [ "${FROM_OS_RELEASE}" = "stein" ] ; then
		DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold puppet-module-placement
	fi
	if [ -x /etc/init.d/puppet-master ] ; then
		/etc/init.d/puppet-master restart
	else
		service puppetserver restart
	fi
	sed -i s/openstack_release=${FROM_OS_RELEASE}/openstack_release=${TO_OS_RELEASE}/ /etc/openstack-cluster-installer/openstack-cluster-installer.conf
}

########################
### KEYSTONE UPGRADE ###
########################
stop_keystone_services () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Stopping keystone on ${HOST_NAME}"
	if [ "${FROM_OS_RELEASE}" = "rocky" ] ; then
		# If we're with Train, Keystone is still running on Apache, so we disable it there and restart apache.
		sshi root@${HOST_IP} "sed -i \"s/Listen ${HOST_IP}:5000//\" /etc/apache2/ports.conf"
		sshi root@${HOST_IP} "sed -i \"s/Listen ${HOST_IP}:35357//\" /etc/apache2/ports.conf"
		sshi root@${HOST_IP} "a2dissite 10-keystone_wsgi_admin.conf ; a2dissite 10-keystone_wsgi_main.conf"
		sshi root@${HOST_IP} "systemctl restart apache2"
		sshi root@${HOST_IP} "HOSTNAME=\$(hostname --fqdn) ; if [ -e /etc/keystone/ssl/private/\${HOSTNAME}.key ] ; then mv /etc/keystone/ssl/private/\${HOSTNAME}.key /etc/keystone/ssl/private/\${HOSTNAME}.pem ; fi"
	fi
	sshi root@${HOST_IP} "systemctl stop keystone || true ; systemctl disable keystone || true ; systemctl mask keystone || true"
}

upgrade_keystone () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Upgrading keystone on ${HOST_NAME}"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold keystone"
}

start_keystone_services () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Starting keystone on ${HOST_NAME}"
	sshi root@${HOST_IP} "systemctl unmask keystone ; systemctl enable keystone ; systemctl start keystone"
}

drain_keystone_backend () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Draining ${HOST_NAME} from keystonepubbe backend"
	sshi root@${ONE_CTRL_IP} "hapc drain-server --backend keystonepubbe --server ${HOST_NAME} --verbose --wait"
}

enable_keystone_backend () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Emabling Draining ${HOST_NAME} in keystonepubbe backend"
	sshi root@${ONE_CTRL_IP} "hapc enable-server --backend keystonepubbe --server ${HOST_NAME} --verbose"
}

cluster_upgrade_keystone () {
	green_echo "===> Upgrading Keystone"

	# Upgrade frist controller
	green_echo "-> Draining ${ONE_CTRL_HOST} keystone backend"
	sshi root@${ONE_CTRL_IP} "hapc drain-server --backend keystonepubbe --server ${ONE_CTRL_HOST} --verbose --wait"

# This fails for keystone, so we wont stop it, drain is enough...
#	green_echo "-> Stopping keystone on ${ONE_CTRL_HOST}"
#	sshi root@${ONE_CTRL_IP} "systemctl stop keystone || true ; systemctl disable keystone || true ; systemctl mask keystone || true"

	green_echo "-> Upgrading keystone on ${ONE_CTRL_HOST}"
	sshi root@${ONE_CTRL_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold keystone"

# This fails for keystone, so we wont stop it, drain is enough...
#	green_echo "-> Starting keystone on ${ONE_CTRL_HOST}"
#	sshi root@${ONE_CTRL_IP} "systemctl unmask keystone ; systemctl enable keystone ; systemctl start keystone"

	sleep 2

	# Do the db_sync
	green_echo "===> Running keystone-manage db_sync"
	sshi root@${ONE_CTRL_IP} "keystone-manage db_sync"

	green_echo "===> Restart Keystone on ${ONE_CTRL_HOST}"
	sshi root@${ONE_CTRL_IP} "systemctl restart keystone"

	green_echo "-> Activating ${ONE_CTRL_HOST} keystone backend"
	sshi root@${ONE_CTRL_IP} "hapc enable-server --backend keystonepubbe --server ${ONE_CTRL_HOST} --verbose"

	# Upgrade all other controllers
	green_echo "-> Draining all other Keystone backends"
	iterate_on_hosts "${OTHER_CTRL}" drain_keystone_backend

# Same here.
#	green_echo "-> Stopping Keystone on all other controllers"
#	iterate_on_hosts "${OTHER_CTRL}" stop_keystone_services

	green_echo "-> Upgrading Keystone on all controllers"
	iterate_on_hosts "${OTHER_CTRL}" upgrade_keystone

# And therefore here.
#	green_echo "-> Starting Keystone on all other controllers"
#	iterate_on_hosts "${OTHER_CTRL}" start_keystone_services

	sleep 2

	green_echo "-> Activating all other controllers backend"
	iterate_on_hosts "${OTHER_CTRL}" enable_keystone_backend
}

#######################
### NEUTRON UPGRADE ###
#######################
upgrade_ovs () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Upgrading OVS on ${HOST_NAME}"
	# Because we're rebooting just after this, let's install latest kernel at the same time...

	if sshi root@${HOST_IP} "dpkg-query -W openvswitch-common" ; then
		CUR_OVS_VERSION=$(sshi root@${HOST_IP} "dpkg-query -W openvswitch-common | awk '{print \$2}'" 2>/dev/null)
		NEXT_OVS_VERSION=$(sshi root@${HOST_IP} "apt-cache policy openvswitch-common | grep Candidate: | awk '{print \$2}'" 2>/dev/null)

		if dpkg --compare-versions ${NEXT_OVS_VERSION} gt ${CUR_OVS_VERSION} ; then
			sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold openvswitch-common openvswitch-switch python3-openvswitch linux-image-amd64"

			green_echo "-> Rebooting ${HOST_NAME} after OVS upgrade"
			sshi root@${HOST_IP} "reboot" || true
			sleep 5
			wait_for_ssh ${HOST_IP}

			green_echo "-> Waiting 10 seconds for network to settle on ${HOST_NAME} after OVS upgrade"
			sleep 10
			green_echo "-> Restarting neutron-openvswitch-agent on ${HOST_NAME}"
			sshi root@${HOST_IP} "if [ -e /etc/init.d/neutron-openvswitch-agent ] ; then /etc/init.d/neutron-openvswitch-agent stop ; sleep 1 ; /etc/init.d/neutron-openvswitch-agent start ; fi"

			green_echo "-> Waiting 20 seconds for OVS agent to settle on ${HOST_NAME}"
			sleep 10
			green_echo "-> Restarting neutron-l3-agent on ${HOST_NAME}"
			sshi root@${HOST_IP} "if [ -e /etc/init.d/neutron-l3-agent ] ; then /etc/init.d/neutron-l3-agent stop ; sleep 1 ; /etc/init.d/neutron-l3-agent start ; fi"
		fi
	fi
}

stop_neutron_server () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Stopping Neutron on ${HOST_NAME}"
	sshi root@${HOST_IP} "systemctl stop neutron-api ; systemctl stop neutron-rpc-server ; systemctl disable neutron-api ; systemctl disable neutron-rpc-server ; systemctl mask neutron-api ; systemctl mask neutron-rpc-server"
}

start_neutron_server () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Starting Neutron on ${HOST_NAME}"
	sshi root@${HOST_IP} "systemctl unmask neutron-api ; systemctl unmask neutron-rpc-server ; systemctl enable neutron-api ; systemctl enable neutron-rpc-server ; systemctl start neutron-api ; systemctl start neutron-rpc-server"
}

upgrade_neutron_server () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Upgrade Neutron on ${HOST_NAME}"
	# Fix [nova]username is defined twice in neutron.conf
	sshi root@${HOST_IP} "sed -i 's/^username = nova//' /etc/neutron/neutron.conf"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold neutron-common python3-neutron neutron-api"
	green_echo "-> Upgrade python3-neutron-fwaas on ${HOST_NAME}"
	sshi root@${HOST_IP} "if dpkg-query -W python3-neutron-fwaas 2>&1 ; then DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold neutron-common python3-neutron-fwaas || true ; fi"
	green_echo "-> Upgrading python3-neutron-dynamic-routing if exists on ${HOST_NAME}"
	sshi root@${HOST_IP} "if dpkg-query -W python3-neutron-dynamic-routing 2>/dev/null 1>/dev/null ; then DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold python3-neutron-dynamic-routing ; fi"
}

upgrade_network_node () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Upgrading neutron on ${HOST_NAME}"
	# Fix [nova]username is defined twice in neutron.conf
#	sshi root@${HOST_IP} "sed -i 's/^username = nova//' /etc/neutron/neutron.conf"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold python3-neutron"
	green_echo "-> Upgrading python3-neutron-dynamic-routing if exists on ${HOST_NAME}"
	sshi root@${HOST_IP} "if dpkg-query -W python3-neutron-dynamic-routing 2>/dev/null 1>/dev/null ; then DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold python3-neutron-dynamic-routing ; fi"
}

upgrade_compute_neutron_agents () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Upgrading Neutron on ${HOST_NAME}"
	# Fix [nova]username is defined twice in neutron.conf
#	sshi root@${HOST_IP} "sed -i 's/^username = nova//' /etc/neutron/neutron.conf"
	sshi root@${HOST_IP} "if dpkg-query -W python3-neutron-dynamic-routing 2>/dev/null 1>/dev/null ; then DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold neutron-common python3-neutron neutron-l3-agent neutron-metadata-agent neutron-metering-agent neutron-openvswitch-agent python3-neutron-dynamic-routing ; else DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold neutron-common python3-neutron neutron-l3-agent neutron-metadata-agent neutron-metering-agent neutron-openvswitch-agent ; fi"
}

upgrade_network_agents_on_controller () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Upgrading Neutron on ${HOST_NAME}"
	# Fix [nova]username is defined twice in neutron.conf
	sshi root@${HOST_IP} "sed -i 's/^username = nova//' /etc/neutron/neutron.conf"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold neutron-common python3-neutron neutron-dhcp-agent neutron-l3-agent neutron-metadata-agent neutron-metering-agent neutron-openvswitch-agent"
	green_echo "-> Upgrading python3-neutron-dynamic-routing if exists on ${ONE_CTRL_HOST}"
	sshi root@${HOST_IP} "if dpkg-query -W python3-neutron-dynamic-routing 2>/dev/null 1>/dev/null ; then DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold python3-neutron-dynamic-routing ; fi"
}

drain_neutron_host_backend () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Draining ${HOST_NAME} Neutron API from possible backends"
	sshi root@${ONE_CTRL_IP} "hapc drain-server --backend neutronbe --server ${HOST_NAME} --verbose --wait"
}

enable_neutron_host_backend () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Enabling ${HOST_NAME} Neutron API as possible backends"
	sshi root@${ONE_CTRL_IP} "hapc enable-server --backend neutronbe --server ${HOST_NAME} --verbose"
}

cluster_upgrade_neutron () {
	green_echo "===> Upgrading neutron API and RPC servers on all controllers"

	# Upgrading controller 1
	green_echo "-> Draining ${ONE_CTRL_HOST} neutron-api backend"
	sshi root@${ONE_CTRL_IP} "hapc drain-server --backend neutronbe --server ${ONE_CTRL_HOST} --verbose --wait"
	green_echo "-> Stopping all neutron services on ${ONE_CTRL_HOST}"
	sshi root@${ONE_CTRL_IP} "systemctl stop neutron-api ; systemctl stop neutron-rpc-server ; systemctl disable neutron-api ; systemctl disable neutron-rpc-server ; systemctl mask neutron-api ; systemctl mask neutron-rpc-server"
	green_echo "-> Upgrading all neutron services on ${ONE_CTRL_HOST}"
	sshi root@${ONE_CTRL_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold neutron-common python3-neutron neutron-api"
	green_echo "===> Upgrading neutron db"
	sshi root@${ONE_CTRL_IP} "neutron-db-manage --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/plugins/ml2/ml2_conf.ini upgrade head"
	green_echo "-> Upgrading python3-neutron-dynamic-routing if exists on ${ONE_CTRL_HOST}"
	sshi root@${ONE_CTRL_IP} "if dpkg-query -W python3-neutron-dynamic-routing 2>/dev/null 1>/dev/null ; then DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold python3-neutron-dynamic-routing ; fi"
	green_echo "-> Starting all neutron services on ${ONE_CTRL_HOST}"
	sshi root@${ONE_CTRL_IP} "systemctl unmask neutron-api ; systemctl unmask neutron-rpc-server ; systemctl enable neutron-api ; systemctl enable neutron-rpc-server ; systemctl start neutron-api ; systemctl start neutron-rpc-server"
	green_echo "-> Enabling ${ONE_CTRL_HOST} neutron-api backend"
	sshi root@${ONE_CTRL_IP} "hapc enable-server --backend neutronbe --server ${ONE_CTRL_HOST} --verbose"

	# Upgrading other controllers
	green_echo "===> Upgrading neutron API and RPC servers on all other controllers"
	iterate_on_hosts "${OTHER_CTRL}" drain_neutron_host_backend
	iterate_on_hosts "${OTHER_CTRL}" stop_neutron_server
	iterate_on_hosts "${OTHER_CTRL}" upgrade_neutron_server
	iterate_on_hosts "${OTHER_CTRL}" start_neutron_server
	iterate_on_hosts "${OTHER_CTRL}" enable_neutron_host_backend

	if [ "${HAS_NET}" = "yes" ] ; then
		green_echo "===> Upgrading Neutron agents on network nodes"
		iterate_on_hosts "${ALL_NET}" upgrade_network_node
	else
		green_echo "===> Upgrading Neutron agents on network nodes"
		iterate_on_hosts "${ALL_CTRL}" upgrade_network_agents_on_controller
	fi

	green_echo "===> Upgrading Neutron agents on compute nodes"
	iterate_on_hosts -p "${ALL_COMP}" upgrade_compute_neutron_agents
}

########################
### Common functions ###
########################
drain_backend () {
	local HOST HOST_IP HOST_NAME BACKEND
	HOST=${1}
	BACKEND=${2}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Draining ${HOST_NAME} backend from ${BACKEND}"
	sshi root@${ONE_CTRL_IP} "hapc drain-server --backend ${BACKEND} --server ${HOST_NAME} --verbose --wait"
}

enable_backend () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	BACKEND=${2}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Enabling ${HOST_NAME} in ${BACKEND} backend"
	sshi root@${ONE_CTRL_IP} "hapc enable-server --backend ${BACKEND} --server ${HOST_NAME} --verbose"
}

stop_service () {
	local HOST HOST_IP HOST_NAME SERVICE
	HOST=${1}
	SERVICE=${2}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Stopping ${SERVICE} on ${HOST_NAME}"
	sshi root@${HOST_IP} "systemctl stop ${SERVICE} || true ; systemctl disable ${SERVICE} || true ; systemctl mask ${SERVICE} || true"
}

start_service () {
	local HOST HOST_IP HOST_NAME SERVICE
	HOST=${1}
	SERVICE=${2}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Starting ${SERVICE} on ${HOST_NAME}"
	sshi root@${HOST_IP} "systemctl unmask ${SERVICE} || true ; systemctl enable ${SERVICE} || true ; systemctl start ${SERVICE} || true"
}

upgrade_package () {
	local HOST HOST_IP HOST_NAME PACKAGE
	HOST=${1}
	PACKAGE=${2}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Upgrading ${PACKAGE} on ${HOST_NAME}"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold ${PACKAGE}"
}

# This is a generic fonction that upgrades a project in the controllers.
# The project must have a unique API backend and one or multiple services
# running on the controllers.
cluster_upgrade_project () {
	local PROJECT_NICE_NAME PROJECT_BACKEND PROJECT_SERVICE_LIST PROJECT_PACKAGE
	PROJECT_NICE_NAME=${1}
	PROJECT_BACKEND=${2}
	PROJECT_SERVICE_LIST=${3}
	PROJECT_PACKAGE=${4}
	PROJECT_UPGRADE_DB_CMD=${5}

	green_echo "===> Upgrading ${PROJECT_NICE_NAME}"
	drain_backend ${ONE_CTRL_IP},${ONE_CTRL_HOST} ${PROJECT_BACKEND}
	stop_service ${ONE_CTRL_IP},${ONE_CTRL_HOST} "${PROJECT_SERVICE_LIST}"
	upgrade_package ${ONE_CTRL_IP},${ONE_CTRL_HOST} "${PROJECT_PACKAGE}"
	if [ -n "${PROJECT_UPGRADE_DB_CMD}" ] ; then
		green_echo "-> Running: ${PROJECT_UPGRADE_DB_CMD}"
		sshi root@${ONE_CTRL_IP} "${PROJECT_UPGRADE_DB_CMD}"
	fi
	start_service ${ONE_CTRL_IP},${ONE_CTRL_HOST} "${PROJECT_SERVICE_LIST}"
	enable_backend ${ONE_CTRL_IP},${ONE_CTRL_HOST} ${PROJECT_BACKEND}

	green_echo "===> Upgrading ${PROJECT_NICE_NAME} on all other controllers"
	iterate_on_hosts "${OTHER_CTRL}" drain_backend ${PROJECT_BACKEND}
	for SERVICE in ${PROJECT_SERVICE_LIST} ; do
		iterate_on_hosts "${OTHER_CTRL}" stop_service ${SERVICE}
	done
	iterate_on_hosts "${OTHER_CTRL}" upgrade_package ${PROJECT_PACKAGE}
	for SERVICE in ${PROJECT_SERVICE_LIST} ; do
		iterate_on_hosts "${OTHER_CTRL}" start_service ${SERVICE}
	done
	iterate_on_hosts "${OTHER_CTRL}" enable_backend ${PROJECT_BACKEND}
}

############
### Heat ###
############
cluster_upgrade_heat () {
	green_echo "===> Upgrading Heat"
	drain_backend ${ONE_CTRL_IP},${ONE_CTRL_HOST} heatbe
	drain_backend ${ONE_CTRL_IP},${ONE_CTRL_HOST} heatcfnbe
	stop_service ${ONE_CTRL_IP},${ONE_CTRL_HOST} "heat-api heat-api-cfn heat-engine"
	upgrade_package ${ONE_CTRL_IP},${ONE_CTRL_HOST} "heat-api heat-api-cfn heat-engine"
	green_echo "===> Running heat-manage db_sync"
	sshi root@${ONE_CTRL_IP} "su heat -s /bin/sh -c 'heat-manage db_sync'"
	start_service ${ONE_CTRL_IP},${ONE_CTRL_HOST} "heat-api heat-api-cfn heat-engine"
	enable_backend ${ONE_CTRL_IP},${ONE_CTRL_HOST} heatbe
	enable_backend ${ONE_CTRL_IP},${ONE_CTRL_HOST} heatcfnbe

	green_echo "===> Upgrading Heat on all other controllers"
	iterate_on_hosts "${OTHER_CTRL}" drain_backend heatbe
	iterate_on_hosts "${OTHER_CTRL}" drain_backend heatcfnbe
	iterate_on_hosts "${OTHER_CTRL}" stop_service heat-api
	iterate_on_hosts "${OTHER_CTRL}" stop_service heat-api-cfn
	iterate_on_hosts "${OTHER_CTRL}" stop_service heat-engine
	iterate_on_hosts "${OTHER_CTRL}" upgrade_package heat-api
	iterate_on_hosts "${OTHER_CTRL}" start_service heat-api
	iterate_on_hosts "${OTHER_CTRL}" start_service heat-api-cfn
	iterate_on_hosts "${OTHER_CTRL}" start_service heat-engine
	iterate_on_hosts "${OTHER_CTRL}" enable_backend heatbe
	iterate_on_hosts "${OTHER_CTRL}" enable_backend heatcfnbe
}

#########################
### PLACEMENT UPGRADE ###
#########################
cluster_upgrade_placement () {
	cluster_upgrade_project Placement placementbe placement-api placement-api "su placement -s /bin/sh -c 'placement-manage db sync'"
}

######################
### CINDER UPGRADE ###
######################
stop_cinder_volume_and_backup () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Stopping Cinder backup and volume on ${HOST_NAME}"
	for i in cinder-volume cinder-backup ; do
		sshi root@${HOST_IP} "systemctl stop $i ; systemctl disable $i ; systemctl mask $i"
	done
}

start_cinder_volume_and_backup () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Starting Cinder backup and volume on ${HOST_NAME}"
	for i in cinder-volume cinder-backup ; do
		sshi root@${HOST_IP} "systemctl unmask $i ; systemctl enable $i ; systemctl start $i"
	done
}

stop_apache_cinder_servers () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Stopping Cinder API on apache on ${HOST_NAME}"
	if [ "${FROM_OS_RELEASE}" = "rocky" ] ; then
		# If we're with Train, Cinder is still running on Apache, so we disable it there and restart apache.
		sshi root@${HOST_IP} "sed -i \"s/Listen ${HOST_IP}:8776//\" /etc/apache2/ports.conf"
		sshi root@${HOST_IP} "a2dissite 10-cinder_wsgi.conf"
		sshi root@${HOST_IP} "systemctl restart apache2"
	fi
}

upgrade_cinder_controller () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Upgrading Cinder on ${HOST_NAME}"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold python3-cinder cinder-scheduler cinder-common cinder-api"
}

start_cinder_volume_and_backup () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Starting Cinder Volume and Backup on ${HOST_NAME}"
	for i in cinder-volume cinder-backup ; do
		sshi root@${HOST_IP} "if dpkg-query -W $i ; then systemctl unmask $i ; systemctl enable $i ; systemctl start $i ; fi"
	done
}

upgrade_volume_services () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Upgrading python3-cinder on ${HOST_NAME}"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold python3-cinder"
#	sshi root@${HOST_IP} "oci-puppet || true"
}

cluster_upgrade_cinder () {
	if [ "${FROM_OS_RELEASE}" = "rocky" ] ; then
		green_echo "===> Disabling cinder-api on Apache"
		iterate_on_hosts "${ALL_CTRL}" stop_apache_cinder_servers
	fi

	cluster_upgrade_project Cinder cinderbe "cinder-api cinder-scheduler" cinder-api "su -s /bin/sh -c 'cinder-manage db sync' cinder"

	green_echo "===> Upgrading cinder-volume and cinder-backup in all nodes of the cluster"
	if [ "${HAS_VOL}" = "yes" ] ; then
		green_echo "===> Stopping all cinder-volume services on volume nodes"
		iterate_on_hosts -p "${ALL_VOL}" stop_cinder_volume_and_backup
		green_echo "===> Upgrade cinder on volume nodes"
		iterate_on_hosts -p "${ALL_VOL}" upgrade_volume_services
		green_echo "===> Starting cinder services on volume nodes"
		iterate_on_hosts -p "${ALL_VOL}" start_cinder_volume_and_backup
	fi
	if [ "${HAS_OSD}" = "yes" ] ; then
		green_echo "===> Stopping all cinder-volume services on compute nodes"
		iterate_on_hosts -p "${ALL_COMP}" stop_cinder_volume_and_backup
		green_echo "===> Upgrade cinder on compute nodes"
		iterate_on_hosts -p "${ALL_COMP}" upgrade_volume_services
		green_echo "===> Starting cinder-volume services on compute nodes"
		iterate_on_hosts -p "${ALL_COMP}" start_cinder_volume_and_backup
	fi
}

####################
### NOVA UPGRADE ###
####################
migrate_from_old_nova_stuff () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	if [ "${FROM_OS_RELEASE}" = "rocky" ] ; then
		# If we're with Train, Nova is still running on Apache, so we disable it there and restart apache.
		green_echo "-> Removing nova-api from Apache on ${HOST_NAME}"
		sshi root@${HOST_IP} "sed -i \"s/Listen ${HOST_IP}:8774//\" /etc/apache2/ports.conf"
		sshi root@${HOST_IP} "a2dissite 10-nova_api_wsgi.conf"
		sshi root@${HOST_IP} "systemctl restart apache2"
	fi
	if [ "${FROM_OS_RELEASE}" = "stein" ] ; then
		green_echo "-> Purging nova-consoleauth and nova-placement-api on ${HOST_NAME}"
		sshi root@${HOST_IP} "apt-get purge -y nova-consoleauth nova-placement-api"
	fi
}

upgrade_compute (){
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Upgrading ${HOST_NAME}"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold python3-nova"
#	sshi root@${HOST_IP} "oci-puppet || true"
}

install_placement (){
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Installing placement on ${HOST_NAME}"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold placement-api python3-osc-placement python3-placement placement-common"
	# Get the placement connection from nova and set it for placement
	sshi root@${HOST_IP} "if [ -d /etc/placement ] && ! [ -r /etc/placement/placement.conf ] && [ -r /usr/share/placement-common/placement.conf ] ; then cp /usr/share/placement-common/placement.conf /etc/placement ; chown placement:placement /etc/placement/placement.conf ; chmod 640 /etc/placement/placement.conf ; fi"
	sshi root@${HOST_IP} ". /usr/share/openstack-pkg-tools/pkgos_func ; TMPFILE=\$(mktemp); cat /etc/nova/nova.conf | grep -v '^#' >\${TMPFILE} ; pkgos_inifile get \${TMPFILE} placement_database connection ; rm -f \${TMPFILE} ; pkgos_inifile set /etc/placement/placement.conf placement_database connection \${RET}"
}

cluster_upgrade_nova () {
	iterate_on_hosts "${ALL_CTRL}" migrate_from_old_nova_stuff

	# Todo: need to fix-up so that we have graceful upgrade of nova-api-metadata
	# which means disabling backends in compute and/or network nodes.
	cluster_upgrade_project Nova novabe "nova-api nova-api-metadata nova-conductor nova-novncproxy nova-scheduler" nova-api "su nova -s /bin/sh -c 'nova-manage db sync' && su nova -s /bin/sh -c 'nova-manage api_db sync'"

	if [ "${FROM_OS_RELEASE}" = "rocky" ] ; then
		iterate_on_hosts "${ALL_CTRL}" install_placement
		sshi root@${ONE_CTRL_IP} "placement-manage db stamp b4ed3a175331"
		sshi root@${ONE_CTRL_IP} "placement-manage db sync"
	fi

	green_echo "===> Nova upgrade of all computes"
	iterate_on_hosts -p "${ALL_COMP}" upgrade_compute

	green_echo "===> Running nova-manage db online_data_migrations until completed"
	sshi root@${ONE_CTRL_IP} "nova-manage db online_data_migrations; RET=\$? ; while [ \"\$RET\" != 0 ] ; do if [ \"\$RET\" == 1 ] ; then exit 1 ; else nova-manage db online_data_migrations; RET=\$? ; fi ; done"
}

#################
### Designate ###
#################
fix_designate_log_rights (){
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Fixing designate-manage.log ownership in ${HOST_NAME}"
	# Without this, running "designate-manage  database sync" as designate user during the puppet run will fail.
	sshi root@${HOST_IP} "chown designate:designate /var/log/designate/designate-manage.log || true"
}

cluster_upgrade_designate () {
	iterate_on_hosts "${ALL_CTRL}" fix_designate_log_rights

	cluster_upgrade_project Designate designatebe "designate-api designate-central designate-mdns designate-producer designate-sink designate-worker" designate-api "su designate -s /bin/sh -c 'cd /tmp && designate-manage database sync'"
}

############
### AODH ###
############
cluster_upgrade_aodh () {
	cluster_upgrade_project Aodh aodhbe "aodh-api aodh-notifier aodh-listener aodh-evaluator" aodh-api "su aodh -s /bin/sh -c 'aodh-dbsync'"
}

###############
### Octavia ###
###############
cluster_upgrade_octavia () {
	cluster_upgrade_project Octavia octaviabe "octavia-api octavia-worker octavia-health-manager octavia-housekeeping" octavia-api "su octavia -s /bin/sh -c 'octavia-db-manage upgrade head'"
}

##############
### Glance ###
##############
cluster_upgrade_glance () {
	cluster_upgrade_project Glance glancebe glance-api glance-api "su glance -s /bin/sh -c 'glance-manage db_sync'"
}

################
### Barbican ###
################
cluster_upgrade_barbican () {
	cluster_upgrade_project Barbican barbicanbe "barbican-api barbican-worker" barbican-api "su barbican -s /bin/sh -c 'barbican-db-manage upgrade'"
}



###############
### Gnocchi ###
###############
controller_upgrade_gnocchi_if_exists () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Upgrading python3-gnocchi on ${HOST_NAME} if exists"
	sshi root@${HOST_IP} "if dpkg-query -W python3-gnocchi 2>/dev/null 1>/dev/null ; then DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold python3-gnocchi ; fi"
}

cluster_upgrade_gnocchi () {
	green_echo "===> Upgrading Gnocchi"

	# If we have MSG nodes, all is on them, otherwise, all on the controllers
	if [ -n "${ALL_MSG}" ] ; then
		green_echo "===> Upgrading Gnocchi on ${ONE_MSG_HOST}"
		drain_backend ${ONE_MSG_IP},${ONE_MSG_HOST} gnocchibe
		stop_service ${ONE_MSG_IP},${ONE_MSG_HOST} "gnocchi-api gnocchi-metricd gnocchi-statsd"
		upgrade_package ${ONE_MSG_IP},${ONE_MSG_HOST} gnocchi-api
		start_service ${ONE_MSG_IP},${ONE_MSG_HOST} "gnocchi-api gnocchi-metricd gnocchi-statsd"
		enable_backend ${ONE_MSG_IP},${ONE_MSG_HOST} gnocchibe

		green_echo "===> Upgrading Gnocchi on all other messaging nodes"
		iterate_on_hosts "${OTHER_MSG}" drain_backend gnocchibe
		for SERVICE in gnocchi-api gnocchi-metricd gnocchi-statsd ; do
			iterate_on_hosts "${OTHER_MSG}" stop_service ${SERVICE}
		done
		iterate_on_hosts "${OTHER_MSG}" upgrade_package gnocchi-api
		for SERVICE in gnocchi-api gnocchi-metricd gnocchi-statsd ; do
			iterate_on_hosts "${OTHER_MSG}" start_service ${SERVICE}
		done
		iterate_on_hosts "${OTHER_MSG}" enable_backend gnocchibe

		iterate_on_hosts "${ALL_CTRL}" controller_upgrade_gnocchi_if_exists

		green_echo "===> Running gnocchi-upgrade"
		sshi root@${ONE_MSG_IP} "su gnocchi -s /bin/sh -c 'gnocchi-upgrade'"
	else
		green_echo "===> Upgrading Gnocchi on ${ONE_CTRL_HOST}"
		drain_backend ${ONE_CTRL_IP},${ONE_CTRL_HOST} gnocchibe
		stop_service ${ONE_CTRL_IP},${ONE_CTRL_HOST} "gnocchi-api gnocchi-metricd gnocchi-statsd"
		upgrade_package ${ONE_CTRL_IP},${ONE_CTRL_HOST} gnocchi-api
		start_service ${ONE_CTRL_IP},${ONE_CTRL_HOST} "gnocchi-api gnocchi-metricd gnocchi-statsd"
		enable_backend ${ONE_CTRL_IP},${ONE_CTRL_HOST} gnocchibe

		green_echo "===> Upgrading Gnocchi on all other controller nodes"
		iterate_on_hosts "${OTHER_CTRL}" drain_backend gnocchibe
		for SERVICE in gnocchi-api gnocchi-metricd gnocchi-statsd ; do
			iterate_on_hosts "${OTHER_CTRL}" stop_service ${SERVICE}
		done
		iterate_on_hosts "${OTHER_CTRL}" upgrade_package gnocchi-api
		for SERVICE in gnocchi-api gnocchi-metricd gnocchi-statsd ; do
			iterate_on_hosts "${OTHER_CTRL}" start_service ${SERVICE}
		done
		iterate_on_hosts "${OTHER_CTRL}" enable_backend gnocchibe

		green_echo "===> Running gnocchi-upgrade"
		sshi root@${ONE_CTRL_IP} "su gnocchi -s /bin/sh -c 'gnocchi-upgrade'"
	fi
}

##################
### Ceilometer ###
##################
cluster_upgrade_ceilometer () {
	green_echo "===> Upgrading Ceilometer"

	# Controller: ceilometer-agent-notification ceilometer-polling
	green_echo "-> Upgrading all controllers"
	iterate_on_hosts "${ALL_CTRL}" upgrade_package "ceilometer-agent-notification ceilometer-polling"

	# Compute: ceilometer-polling
	green_echo "-> Upgrading ceilometer-polling on all compute nodes"
	iterate_on_hosts -p "${ALL_COMP}" upgrade_package "ceilometer-polling"
}

##################
### Cloudkitty ###
##################
upgrade_python3_cloudkitty_on_ctrl () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Upgrading python3-cloudkitty on controller ${HOST_NAME} if exists"
	sshi root@${HOST_IP} "if dpkg-query -W python3-cloudkitty 2>/dev/null 1>/dev/null ; then DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold python3-cloudkitty ; fi"
}

cluster_upgrade_cloudkitty () {
	green_echo "===> Upgrading Cloudkitty"

	# Messaging: cloudkitty-api cloudkitty-processor

	# If we have MSG nodes, all is on them, otherwise, all on the controllers
	if [ -n "${ALL_MSG}" ] ; then
		green_echo "===> Upgrading Cloudkitty on ${ONE_MSG_HOST}"
		drain_backend ${ONE_MSG_IP},${ONE_MSG_HOST} cloudkittybe
		stop_service ${ONE_MSG_IP},${ONE_MSG_HOST} "cloudkitty-api cloudkitty-processor"
		upgrade_package ${ONE_MSG_IP},${ONE_MSG_HOST} cloudkitty-api
		green_echo "===> Running cloudkitty-dbsync upgrade"
		sshi root@${ONE_MSG_IP} "su cloudkitty -s /bin/sh -c 'cloudkitty-dbsync upgrade'"
		start_service ${ONE_MSG_IP},${ONE_MSG_HOST} "cloudkitty-api cloudkitty-processor"
		enable_backend ${ONE_MSG_IP},${ONE_MSG_HOST} cloudkittybe

		green_echo "===> Upgrading Cloudkitty on all other messaging nodes"
		iterate_on_hosts "${OTHER_MSG}" drain_backend cloudkittybe
		for SERVICE in cloudkitty-api cloudkitty-processor ; do
			iterate_on_hosts "${OTHER_MSG}" stop_service ${SERVICE}
		done
		iterate_on_hosts "${OTHER_MSG}" upgrade_package cloudkitty-api
		for SERVICE in cloudkitty-api cloudkitty-processor ; do
			iterate_on_hosts "${OTHER_MSG}" start_service ${SERVICE}
		done
		iterate_on_hosts "${OTHER_MSG}" enable_backend cloudkittybe

		iterate_on_hosts "${ALL_CTRL}" upgrade_python3_cloudkitty_on_ctrl
	else
		green_echo "===> Upgrading Cloudkitty on ${ONE_CTRL_HOST}"
		drain_backend ${ONE_CTRL_IP},${ONE_CTRL_HOST} cloudkittybe
		stop_service ${ONE_CTRL_IP},${ONE_CTRL_HOST} "cloudkitty-api cloudkitty-processor"
		upgrade_package ${ONE_CTRL_IP},${ONE_CTRL_HOST} cloudkitty-api
		green_echo "===> Running cloudkitty-dbsync upgrade"
		sshi root@${ONE_CTRL_IP} "su cloudkitty -s /bin/sh -c 'cloudkitty-dbsync upgrade'"
		start_service ${ONE_CTRL_IP},${ONE_CTRL_HOST} "cloudkitty-api cloudkitty-processor"
		enable_backend ${ONE_CTRL_IP},${ONE_CTRL_HOST} cloudkittybe

		green_echo "===> Upgrading Cloudkitty on all other controller nodes"
		iterate_on_hosts "${OTHER_CTRL}" drain_backend cloudkittybe
		for SERVICE in cloudkitty-api cloudkitty-processor ; do
			iterate_on_hosts "${OTHER_CTRL}" stop_service ${SERVICE}
		done
		iterate_on_hosts "${OTHER_CTRL}" upgrade_package cloudkitty-api
		for SERVICE in cloudkitty-api cloudkitty-processor ; do
			iterate_on_hosts "${OTHER_CTRL}" start_service ${SERVICE}
		done
		iterate_on_hosts "${OTHER_CTRL}" enable_backend cloudkittybe
	fi
}

#############
### Swift ###
#############
upgrade_python3_swift () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Upgrading ${HOST_NAME}"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold python3-swift"
}

upgrade_swift_proxy () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Draining ${HOST_NAME}"
	sshi root@${HOST_IP} "hapc drain-server --backend swiftbe --server ${HOST_NAME} --verbose --wait"
	green_echo "-> Upgrading ${HOST_NAME}"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold python3-swift"
	green_echo "-> Enabling ${HOST_NAME}"
	sshi root@${HOST_IP} "hapc enable-server --backend swiftbe --server ${HOST_NAME} --verbose"
}

cluster_upgrade_swift () {
	green_echo "===> Upgrading Swift"
	iterate_on_hosts -p "${ALL_SWIFTSTR}" upgrade_python3_swift
	iterate_on_hosts -p "${ALL_SWIFTPRX}" upgrade_swift_proxy
}

#######################
### HORIZON REMOVAL ###
#######################
upgrade_horizon () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Upgrading horizon on ${HOST_NAME}"
	case "${TO_OS_RELEASE}" in
	# No new deps for "wallaby", so only python3-django-horizon like default.
	"xena")
		HORIZON_DEPENDS="python3-django-horizon python3-xstatic-angular-bootstrap python3-xstatic-angular-gettext python3-xstatic-bootstrap-scss"
	;;
	# No new deps for "yoga", so only python3-django-horizon like default.
	"zed")
		HORIZON_DEPENDS="python3-django-horizon python3-xstatic-angular python3-xstatic-mdi python3-heat-dashboard"
		if [ "${HAS_DESIGNATE}" = "yes" ] ; then
			HORIZON_DEPENDS="${HORIZON_DEPENDS} python3-designate-dashboard"
		fi
		if [ "${HAS_OCTAVIA}" ] ; then
			HORIZON_DEPENDS="${HORIZON_DEPENDS} python3-octavia-dashboard"
		fi
		if [ "${HAS_TELEMETRY}" = "yes" ] && [ "${HAS_CLOUDKITTY_DASHBOARD}" = "yes" ] ; then
			HORIZON_DEPENDS="${HORIZON_DEPENDS} python3-cloudkitty-dashboard"
		fi
	;;
	*)
		HORIZON_DEPENDS="python3-django-horizon"
	;;
	esac
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold ${HORIZON_DEPENDS}"
}

cluster_upgrade_horizon () {
	green_echo "===> Upgrading horizon"
	iterate_on_hosts "${ALL_CTRL}" upgrade_horizon
}


################
### FINALIZE ###
################
distupgrade_everyone () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Dist-upgrading ${HOST_NAME}"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y -o Dpkg::Options::=--force-confold --allow-downgrades"
}

run_puppet_on_node () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Running puppet on ${HOST_NAME}"
	sshi root@${HOST_IP} "oci-puppet || true"
}

restart_all_services_on_node () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	shift
	for i in $@ ; do
		green_echo "-> Restarting ${i} on ${HOST_NAME}"
		sshi root@${HOST_IP} "if [ -e /etc/init.d/${i} ] ; then /etc/init.d/${i} stop ; sleep 1 ; /etc/init.d/${i} start ; fi"
	done
}

cluster_upgrade_finilize () {
# This is a bad idea...
#	green_echo "===> Distupgrading everyone"
#	iterate_on_hosts "${ALL_NODES}" distupgrade_everyone

# We don't switch the initial-cluster-setup, because this triggers the dbsync,
# which restarts the services.
# In Xena, we're moving from puppet.conf to admin-clouds.yaml, though
# it should be fine since this is always re-written, even with initial-cluster-setup
# to no.
#	if [ "${FROM_OS_RELEASE}" != "xena" ] ; then
#		green_echo "===> Activating initial setup"
#		ocicli cluster-set ${CLUSTER_NAME} --initial-cluster-setup yes
#	fi

	green_echo "===> Run puppet on all nodes (but Ceph nodes)"
	iterate_on_hosts -p "${ALL_NODES_BUT_CEPH}" run_puppet_on_node

# See above.
#	if [ "${FROM_OS_RELEASE}" != "xena" ] ; then
#		green_echo "===> Removing initial setup"
#		ocicli cluster-set ${CLUSTER_NAME} --initial-cluster-setup no
#	fi

# Not needed anymore
#	green_echo "===> Restarting all services everywhere"
#	iterate_on_hosts "${ALL_CTRL}" restart_all_services_on_node ${ALL_SERVICES_CTRL}
#	iterate_on_hosts -p "${ALL_COMP}" restart_all_services_on_node ${ALL_SERVICES_COMP}
#	iterate_on_hosts -p "${ALL_VOL}" restart_all_services_on_node ${ALL_SERVICES_VOL}
#	iterate_on_hosts "${ALL_NET}" restart_all_services_on_node ${ALL_SERVICES_NET}
#	iterate_on_hosts "${ALL_MSG}" restart_all_services_on_node ${ALL_SERVICES_MSG}

	green_echo "===> Enabling puppet everywhere (but Ceph nodes)"
	iterate_on_hosts -p "${ALL_NODES_BUT_CEPH}" enable_puppet
}

cluster_check_status () {
	if [ "${USE_EXTERNAL_KEYSTONE}" != "yes" ] ; then
		sshi root@${ONE_CTRL_IP} "keystone-status upgrade check"
	fi
	if [ "${HAS_BARBICAN}" = "yes" ] ; then
		sshi root@${ONE_CTRL_IP} "barbican-status upgrade check"
	fi
	if [ "${HAS_DESIGNATE}" = "yes" ] ; then
		sshi root@${ONE_CTRL_IP} "designate-status upgrade check"
	fi
	if [ "${HAS_HEAT}" = "yes" ] ; then
		sshi root@${ONE_CTRL_IP} "heat-status upgrade check"
	fi
	if [ -n "${ALL_COMP}" ] ; then
		sshi root@${ONE_CTRL_IP} "glance-status upgrade check" 
		sshi root@${ONE_CTRL_IP} "nova-status upgrade check"
		sshi root@${ONE_CTRL_IP} "cinder-status upgrade check || true"
		sshi root@${ONE_CTRL_IP} "neutron-status upgrade check"
		sshi root@${ONE_CTRL_IP} "placement-status upgrade check"
		if [ "${HAS_TELEMETRY}" = "yes" ] ; then
			sshi root@${ONE_CTRL_IP} "aodh-status upgrade check"
			sshi root@${ONE_CTRL_IP} "ceilometer-status upgrade check"
			# This fails if using storage v1
			sshi root@${ONE_CTRL_IP} "cloudkitty-status upgrade check || true"
		fi
	fi
#	sshi root@${ONE_CTRL_IP} "octavia-status upgrade check"
}

############################
### Dependencies upgrade ###
############################
ctrl_upgrade_depends () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Installing depends on ${HOST_NAME}"
	case "${TO_OS_RELEASE}" in
	"wallaby")
		CTRL_DEPENDS="openstack-pkg-tools os-brick-common python3-aodhclient python3-automaton python3-barbicanclient python3-blazarclient python3-castellan python3-cinderclient python3-cliff python3-cloudkittyclient python3-cmd2 python3-designateclient python3-docker python3-dogpile.cache python3-eventlet python3-glance-store python3-glanceclient python3-gnocchiclient python3-heatclient python3-influxdb python3-ironicclient python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-magnumclient python3-manilaclient python3-mistralclient python3-monascaclient python3-msgpack python3-netaddr python3-neutronclient python3-novaclient python3-oauth2client python3-octaviaclient python3-openstackclient python3-openstacksdk python3-os-brick python3-os-ken python3-os-traits python3-os-vif python3-os-win python3-osc-lib python3-osc-placement python3-oslo.cache python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.messaging python3-oslo.middleware python3-oslo.policy python3-oslo.privsep python3-oslo.serialization python3-oslo.service python3-oslo.upgradecheck python3-oslo.utils python3-oslo.versionedobjects python3-oslo.vmware python3-ovsdbapp python3-pbr python3-pymemcache python3-pymysql python3-pyroute2 python3-rtslib-fb python3-saharaclient python3-senlinclient python3-stevedore python3-swiftclient python3-taskflow python3-tenacity python3-tooz python3-troveclient python3-vitrageclient python3-voluptuous python3-zaqarclient python3-zunclient"
	;;
	"xena")
		CTRL_DEPENDS="alembic libjs-sphinxdoc novnc openstack-pkg-tools os-brick-common python3-alembic python3-aodhclient python3-automaton python3-barbicanclient python3-blazarclient python3-calmjs python3-castellan python3-cinderclient python3-cliff python3-cloudkittyclient python3-croniter python3-ddt python3-debtcollector python3-designateclient python3-eventlet python3-falcon python3-flask python3-flask-restful python3-futurist python3-glance-store python3-glanceclient python3-greenlet python3-heatclient python3-idna python3-ironicclient python3-itsdangerous python3-jinja2 python3-jsonpatch python3-jsonschema python3-jwcrypto python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-ldappool python3-magnumclient python3-manilaclient python3-markupsafe python3-migrate python3-mistralclient python3-monascaclient python3-monotonic python3-neutronclient python3-novaclient python3-novnc python3-octavia-lib python3-octaviaclient python3-openstackclient python3-openstacksdk python3-os-brick python3-os-ken python3-os-resource-classes python3-os-traits python3-os-vif python3-os-win python3-osc-lib python3-osc-placement python3-oslo.cache python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.i18n python3-oslo.log python3-oslo.messaging python3-oslo.middleware python3-oslo.policy python3-oslo.privsep python3-oslo.reports python3-oslo.rootwrap python3-oslo.serialization python3-oslo.service python3-oslo.upgradecheck python3-oslo.utils python3-oslo.versionedobjects python3-oslo.vmware python3-osprofiler python3-ovsdbapp python3-pbr python3-pecan python3-pypowervm python3-pyroute2 python3-pyroute2.ethtool python3-pyroute2.ipdb python3-pyroute2.ipset python3-pyroute2.ndb python3-pyroute2.nftables python3-pyroute2.nslink python3-pyroute2.protocols python3-pysaml2 python3-q-text-as-data python3-rfc3986 python3-saharaclient python3-senlinclient python3-sphinx python3-sqlalchemy python3-sqlalchemy-utils python3-stevedore python3-swiftclient python3-taskflow python3-testresources python3-testtools python3-tooz python3-troveclient python3-uhashring python3-vitrageclient python3-websockify python3-werkzeug python3-wrapt python3-wsme python3-yappi python3-yaql python3-zaqarclient python3-zunclient sphinx-common spice-html5 websockify"
	;;
	"yoga")
		CTRL_DEPENDS="alembic os-brick-common python3-alembic python3-amqp python3-aodhclient python3-automaton python3-barbicanclient python3-binary-memcached python3-blazarclient python3-boto3 python3-botocore python3-cachetools python3-castellan python3-cffi python3-cffi-backend python3-cinderclient python3-click python3-cliff python3-cloudkittyclient python3-cmd2 python3-croniter python3-cryptography python3-ddt python3-debtcollector python3-designateclient python3-django python3-django-appconf python3-django-compressor python3-django-debreach python3-django-pyscss python3-dogpile.cache python3-elementpath python3-enmerkar python3-etcd3gw python3-eventlet python3-fasteners python3-glance-store python3-glanceclient python3-gnocchiclient python3-greenlet python3-heatclient python3-idna python3-imagesize python3-importlib-metadata python3-ironicclient python3-iso8601 python3-jinja2 python3-json-pointer python3-jwt python3-kazoo python3-keyring python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-kombu python3-lxml python3-magnumclient python3-manilaclient python3-mistralclient python3-monasca-statsd python3-monascaclient python3-monotonic python3-msgpack python3-neutronclient python3-novaclient python3-oauth2client python3-oauthlib python3-octaviaclient python3-openstackclient python3-openstacksdk python3-os-brick python3-os-client-config python3-os-ken python3-os-traits python3-os-vif python3-os-win python3-osc-lib python3-osc-placement python3-oslo.cache python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.limit python3-oslo.log python3-oslo.messaging python3-oslo.metrics python3-oslo.middleware python3-oslo.policy python3-oslo.privsep python3-oslo.reports python3-oslo.rootwrap python3-oslo.serialization python3-oslo.service python3-oslo.upgradecheck python3-oslo.utils python3-oslo.versionedobjects python3-oslo.vmware python3-ovsdbapp python3-paramiko python3-pbr python3-pecan python3-pkg-resources python3-psutil python3-pure-sasl python3-pyparsing python3-pypowervm python3-pyroute2 python3-pysaml2 python3-pytimeparse python3-rcssmin python3-responses python3-rjsmin python3-rtslib-fb python3-saharaclient python3-semantic-version python3-senlinclient python3-setuptools python3-simplejson   python3-sqlalchemy-utils python3-stevedore python3-swiftclient python3-taskflow python3-tooz python3-troveclient python3-ujson python3-uritemplate python3-vitrageclient python3-voluptuous python3-wcwidth python3-websocket python3-wrapt python3-yaml python3-zake python3-zaqarclient python3-zunclient"
	;;
	"zed")
		CTRL_DEPENDS="alembic novnc python3-alembic python3-aodhclient python3-automaton python3-barbicanclient python3-blazarclient python3-castellan python3-cinderclient python3-cliff python3-cloudkittyclient python3-cursive python3-defusedxml python3-designateclient python3-enmerkar python3-etcd3gw python3-eventlet python3-falcon python3-fixtures python3-flask python3-futurist python3-glance-store python3-glanceclient python3-heatclient python3-ironicclient python3-json-pointer python3-jsonschema python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-magnumclient python3-manilaclient python3-mistralclient python3-monasca-statsd python3-monascaclient python3-msgpack python3-munch python3-mypy python3-neutronclient python3-novaclient python3-novnc python3-oauth2client python3-octavia-lib python3-octaviaclient python3-openstackclient python3-openstacksdk python3-os-brick python3-os-ken python3-os-traits python3-os-vif python3-os-win python3-osc-lib python3-osc-placement python3-oslo.cache python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.i18n python3-oslo.limit python3-oslo.log python3-oslo.messaging python3-oslo.metrics python3-oslo.middleware python3-oslo.policy python3-oslo.privsep python3-oslo.reports python3-oslo.serialization python3-oslo.service python3-oslo.upgradecheck python3-oslo.utils python3-oslo.versionedobjects python3-oslo.vmware python3-osprofiler python3-ovsdbapp python3-pbr python3-pymemcache python3-pyscss python3-q-text-as-data python3-rtslib-fb python3-saharaclient python3-senlinclient python3-sqlalchemy python3-stevedore python3-swiftclient python3-taskflow python3-testresources python3-testtools python3-tooz python3-troveclient python3-uhashring python3-vitrageclient python3-warlock python3-websockify python3-webtest python3-werkzeug python3-wrapt python3-wsme python3-yappi python3-zaqarclient python3-zunclient websockify"
	;;
	*)
		CTRL_DEPENDS="openstack-pkg-tools alembic novnc os-brick-common python3-alembic python3-amqp python3-aodhclient python3-automaton python3-barbicanclient python3-binary-memcached python3-blazarclient python3-boto3 python3-botocore python3-cachetools python3-calmjs python3-castellan python3-cffi python3-cffi-backend python3-cinderclient python3-click python3-cliff python3-cloudkittyclient python3-cmd2 python3-croniter python3-cryptography python3-cursive python3-ddt python3-debtcollector python3-defusedxml python3-designateclient python3-django python3-django-appconf python3-django-compressor python3-django-debreach python3-django-pyscss python3-docker python3-dogpile.cache python3-elementpath python3-enmerkar python3-etcd3gw python3-eventlet python3-falcon python3-fasteners python3-fixtures python3-flask python3-flask-restful python3-futurist python3-glanceclient python3-gnocchiclient python3-greenlet python3-heatclient python3-idna python3-imagesize python3-importlib-metadata python3-influxdb python3-ironicclient python3-iso8601 python3-itsdangerous python3-jinja2 python3-json-pointer python3-jsonpatch python3-jsonschema python3-jwcrypto python3-jwt python3-kazoo python3-keyring python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-kombu python3-ldappool python3-lxml python3-magnumclient python3-manilaclient python3-markupsafe python3-migrate python3-mistralclient python3-monasca-statsd python3-monascaclient python3-monotonic python3-msgpack python3-munch python3-mypy python3-netaddr python3-neutronclient python3-novaclient python3-novnc python3-oauth2client python3-oauthlib python3-octaviaclient python3-openstackclient python3-openstacksdk python3-os-brick python3-os-ken python3-os-resource-classes python3-os-traits python3-os-vif python3-os-win python3-osc-lib python3-osc-placement python3-oslo.cache python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.i18n python3-oslo.log python3-oslo.messaging python3-oslo.middleware python3-oslo.policy python3-oslo.privsep python3-oslo.reports python3-oslo.rootwrap python3-oslo.serialization python3-oslo.service python3-oslo.upgradecheck python3-oslo.utils python3-oslo.versionedobjects python3-oslo.vmware python3-osprofiler python3-ovsdbapp python3-paramiko python3-pbr python3-pecan python3-pkg-resources python3-psutil python3-pure-sasl python3-pymemcache python3-pymysql python3-pyparsing python3-pyroute2 python3-pysaml2 python3-pyscss python3-pytimeparse python3-q-text-as-data python3-rcssmin python3-responses python3-rfc3986 python3-rjsmin python3-rtslib-fb python3-saharaclient python3-semantic-version python3-senlinclient python3-setuptools python3-simplejson python3-sphinx python3-sqlalchemy python3-sqlalchemy-utils python3-stevedore python3-swiftclient python3-taskflow python3-tenacity python3-testresources python3-testtools python3-tooz python3-troveclient python3-uhashring python3-ujson python3-uritemplate python3-vitrageclient python3-voluptuous python3-warlock python3-wcwidth python3-websocket python3-websockify python3-webtest python3-werkzeug python3-wrapt python3-wsme python3-xstatic-angular python3-xstatic-angular-bootstrap python3-xstatic-angular-gettext python3-xstatic-bootstrap-scss python3-xstatic-mdi python3-yaml python3-yappi python3-yaql python3-zake python3-zaqarclient python3-zunclient sphinx-common spice-html5 websockify"
	;;
	esac
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold --allow-downgrades ${CTRL_DEPENDS}"
}

msg_upgrade_depends () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Installing depends on ${HOST_NAME}"
	case "${TO_OS_RELEASE}" in
	"wallaby")
		MSG_DEPENDS="python3-cinderclient python3-cliff python3-cloudkittyclient python3-cmd2 python3-dogpile.cache python3-eventlet python3-glanceclient python3-gnocchiclient python3-influxdb python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-monascaclient python3-msgpack python3-netaddr python3-neutronclient python3-novaclient python3-openstackclient python3-openstacksdk python3-osc-lib python3-oslo.cache python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.messaging python3-oslo.middleware python3-oslo.policy python3-oslo.serialization python3-oslo.service python3-oslo.upgradecheck python3-oslo.utils python3-pbr python3-pymemcache python3-pymysql python3-stevedore python3-swiftclient python3-tenacity python3-tooz python3-voluptuous"
	;;
	"xena")
		MSG_DEPENDS="alembic libjs-sphinxdoc python3-alembic python3-cinderclient python3-cliff python3-cloudkittyclient python3-debtcollector python3-eventlet python3-flask python3-flask-restful python3-futurist python3-glanceclient python3-greenlet python3-idna python3-itsdangerous python3-jinja2 python3-jsonpatch python3-jsonschema python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-markupsafe python3-migrate python3-monascaclient python3-monotonic python3-neutronclient python3-novaclient python3-openstackclient python3-openstacksdk python3-osc-lib python3-oslo.cache python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.i18n python3-oslo.log python3-oslo.messaging python3-oslo.middleware python3-oslo.policy python3-oslo.serialization python3-oslo.service python3-oslo.upgradecheck python3-oslo.utils python3-pbr python3-pecan python3-q-text-as-data python3-rfc3986 python3-sqlalchemy python3-sqlalchemy-utils python3-stevedore python3-swiftclient python3-testresources python3-testtools python3-tooz python3-uhashring python3-werkzeug python3-wrapt python3-wsme python3-yappi"
	;;
	"yoga")
		MSG_DEPENDS="alembic python3-alembic python3-amqp python3-binary-memcached python3-boto3 python3-botocore python3-cachetools python3-cffi python3-cffi-backend python3-cinderclient python3-click python3-cliff python3-cloudkittyclient python3-cmd2 python3-cryptography python3-debtcollector python3-dogpile.cache python3-etcd3gw python3-eventlet python3-fasteners python3-glanceclient python3-gnocchiclient python3-greenlet python3-idna python3-importlib-metadata python3-iso8601 python3-jinja2 python3-json-pointer python3-jwt python3-kazoo python3-keyring python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-kombu python3-lxml python3-monascaclient python3-monotonic python3-msgpack python3-neutronclient python3-novaclient python3-oauthlib python3-openstackclient python3-openstacksdk python3-os-client-config python3-osc-lib python3-oslo.cache python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.log python3-oslo.messaging python3-oslo.metrics python3-oslo.middleware python3-oslo.policy python3-oslo.serialization python3-oslo.service python3-oslo.upgradecheck python3-oslo.utils python3-pbr python3-pecan python3-pkg-resources python3-pure-sasl python3-pyparsing python3-pytimeparse python3-setuptools python3-simplejson python3-sqlalchemy-utils python3-stevedore python3-swiftclient python3-tooz python3-ujson python3-voluptuous python3-wcwidth python3-wrapt python3-yaml python3-zake"
	;;
	"zed")
		MSG_DEPENDS="alembic python3-alembic python3-cinderclient python3-cliff python3-cloudkittyclient python3-etcd3gw python3-eventlet python3-fixtures python3-flask python3-futurist python3-glanceclient python3-json-pointer python3-jsonschema python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-monascaclient python3-msgpack python3-munch python3-neutronclient python3-novaclient python3-openstackclient python3-openstacksdk python3-osc-lib python3-oslo.cache python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.i18n python3-oslo.log python3-oslo.messaging python3-oslo.metrics python3-oslo.middleware python3-oslo.policy python3-oslo.serialization python3-oslo.service python3-oslo.upgradecheck python3-oslo.utils python3-pbr python3-pymemcache python3-q-text-as-data python3-sqlalchemy python3-stevedore python3-swiftclient python3-testresources python3-testtools python3-tooz python3-uhashring python3-warlock python3-webtest python3-werkzeug python3-wrapt python3-wsme python3-yappi"
	;;
	*)
		MSG_DEPENDS="alembic python3-alembic python3-amqp python3-binary-memcached python3-boto3 python3-botocore python3-cachetools python3-cffi python3-cffi-backend python3-cinderclient python3-click python3-cliff python3-cloudkittyclient python3-cmd2 python3-cryptography python3-debtcollector python3-dogpile.cache python3-etcd3gw python3-eventlet python3-fasteners python3-fixtures python3-flask python3-flask-restful python3-futurist python3-glanceclient python3-gnocchiclient python3-greenlet python3-idna python3-importlib-metadata python3-influxdb python3-iso8601 python3-itsdangerous python3-jinja2 python3-json-pointer python3-jsonpatch python3-jsonschema python3-jwt python3-kazoo python3-keyring python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-kombu python3-lxml python3-markupsafe python3-migrate python3-monascaclient python3-monotonic python3-msgpack python3-munch python3-netaddr python3-neutronclient python3-novaclient python3-oauthlib python3-openstackclient python3-openstacksdk python3-os-client-config python3-osc-lib python3-oslo.cache python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.i18n python3-oslo.log python3-oslo.messaging python3-oslo.middleware python3-oslo.policy python3-oslo.serialization python3-oslo.service python3-oslo.upgradecheck python3-oslo.utils python3-pbr python3-pecan python3-pkg-resources python3-pure-sasl python3-pymemcache python3-pymysql python3-pyparsing python3-pytimeparse python3-q-text-as-data python3-rfc3986 python3-setuptools python3-simplejson python3-sqlalchemy python3-sqlalchemy-utils python3-stevedore python3-swiftclient python3-tenacity python3-testresources python3-testtools python3-tooz python3-uhashring python3-ujson python3-voluptuous python3-warlock python3-wcwidth python3-webtest python3-werkzeug python3-wrapt python3-wsme python3-yaml python3-yappi python3-zake"
	;;
	esac
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold --allow-downgrades ${MSG_DEPENDS}"
}

comp_upgrade_depends () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Installing depends on ${HOST_NAME}"
	case "${TO_OS_RELEASE}" in
	"wallaby")
		COMP_DEPENDS="os-brick-common python3-automaton python3-barbicanclient python3-castellan python3-cinderclient python3-cliff python3-cmd2 python3-designateclient python3-dogpile.cache python3-eventlet python3-glanceclient python3-gnocchiclient python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-monascaclient python3-msgpack python3-netaddr python3-neutronclient python3-novaclient python3-oauth2client python3-openstackclient python3-openstacksdk python3-os-brick python3-os-ken python3-os-traits python3-os-vif python3-os-win python3-osc-lib python3-oslo.cache python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.messaging python3-oslo.middleware python3-oslo.policy python3-oslo.privsep python3-oslo.serialization python3-oslo.service python3-oslo.upgradecheck python3-oslo.utils python3-oslo.versionedobjects python3-oslo.vmware python3-ovsdbapp python3-pbr python3-pymemcache python3-pymysql python3-pyroute2 python3-rtslib-fb python3-stevedore python3-swiftclient python3-taskflow python3-tenacity python3-tooz python3-voluptuous python3-zaqarclient python3-colorama python3-pyroute2.core python3-pyroute2.ethtool python3-pyroute2.ipdb python3-pyroute2.ipset python3-pyroute2.ndb python3-pyroute2.nftables python3-pyroute2.nslink python3-pyroute2.protocols"
	;;
	"xena")
		COMP_DEPENDS="alembic libjs-sphinxdoc os-brick-common python3-alembic python3-automaton python3-barbicanclient python3-castellan python3-cinderclient python3-cliff python3-ddt python3-debtcollector python3-designateclient python3-eventlet python3-futurist python3-glanceclient python3-greenlet python3-idna python3-jinja2 python3-jsonpatch python3-jsonschema python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-markupsafe python3-migrate python3-monascaclient python3-monotonic python3-neutronclient python3-novaclient python3-openstackclient python3-openstacksdk python3-os-brick python3-os-ken python3-os-resource-classes python3-os-traits python3-os-vif python3-os-win python3-osc-lib python3-oslo.cache python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.i18n python3-oslo.log python3-oslo.messaging python3-oslo.middleware python3-oslo.policy python3-oslo.privsep python3-oslo.reports python3-oslo.rootwrap python3-oslo.serialization python3-oslo.service python3-oslo.upgradecheck python3-oslo.utils python3-oslo.versionedobjects python3-oslo.vmware python3-osprofiler python3-ovsdbapp python3-pbr python3-pecan python3-pypowervm python3-pyroute2 python3-pyroute2.ethtool python3-pyroute2.ipdb python3-pyroute2.ipset python3-pyroute2.ndb python3-pyroute2.nftables python3-pyroute2.nslink python3-pyroute2.protocols python3-q-text-as-data python3-rfc3986 python3-sqlalchemy python3-stevedore python3-swiftclient python3-taskflow python3-testresources python3-testtools python3-tooz python3-uhashring python3-websockify python3-werkzeug python3-wrapt python3-yappi python3-zaqarclient"
	;;
	"yoga")
		COMP_DEPENDS="alembic os-brick-common python3-alembic python3-amqp python3-automaton python3-barbicanclient python3-binary-memcached python3-boto3 python3-botocore python3-cachetools python3-castellan python3-cffi python3-cffi-backend python3-cinderclient python3-cliff python3-cmd2 python3-cryptography python3-ddt python3-debtcollector python3-designateclient python3-dogpile.cache python3-etcd3gw python3-eventlet python3-fasteners python3-glanceclient python3-gnocchiclient python3-greenlet python3-idna python3-importlib-metadata python3-iso8601 python3-jinja2 python3-json-pointer python3-jwt python3-kazoo python3-keyring python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-kombu python3-lxml python3-monascaclient python3-monotonic python3-msgpack python3-neutronclient python3-novaclient python3-oauth2client python3-oauthlib python3-openstackclient python3-openstacksdk python3-os-brick python3-os-client-config python3-os-ken python3-os-traits python3-os-vif python3-os-win python3-osc-lib python3-oslo.cache python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.log python3-oslo.messaging python3-oslo.metrics python3-oslo.middleware python3-oslo.policy python3-oslo.privsep python3-oslo.reports python3-oslo.rootwrap python3-oslo.serialization python3-oslo.service python3-oslo.upgradecheck python3-oslo.utils python3-oslo.versionedobjects python3-oslo.vmware python3-ovsdbapp python3-paramiko python3-pbr python3-pecan python3-pkg-resources python3-psutil python3-pure-sasl python3-pyparsing python3-pypowervm python3-pyroute2 python3-pytimeparse python3-rtslib-fb python3-setuptools python3-simplejson python3-stevedore python3-swiftclient python3-taskflow python3-tooz python3-ujson python3-uritemplate python3-voluptuous python3-wcwidth python3-wrapt python3-yaml python3-zake python3-zaqarclient"
	;;
	"zed")
		COMP_DEPENDS="alembic os-brick-common python3-alembic python3-automaton python3-barbicanclient python3-castellan python3-ceilometer python3-cinderclient python3-cliff python3-cursive python3-designateclient python3-etcd3gw python3-eventlet python3-fixtures python3-futurist python3-glanceclient python3-json-pointer python3-jsonschema python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-monascaclient python3-msgpack python3-munch python3-mypy python3-networking-l2gw python3-neutronclient python3-novaclient python3-oauth2client python3-openstackclient python3-openstacksdk python3-os-brick python3-os-ken python3-os-traits python3-os-vif python3-os-win python3-osc-lib python3-oslo.cache python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.i18n python3-oslo.limit python3-oslo.log python3-oslo.messaging python3-oslo.metrics python3-oslo.middleware python3-oslo.policy python3-oslo.privsep python3-oslo.reports python3-oslo.serialization python3-oslo.service python3-oslo.upgradecheck python3-oslo.utils python3-oslo.versionedobjects python3-oslo.vmware python3-osprofiler python3-ovsdbapp python3-pbr python3-pymemcache python3-q-text-as-data python3-rtslib-fb python3-sqlalchemy python3-stevedore python3-swiftclient python3-taskflow python3-testresources python3-testtools python3-tooz python3-uhashring python3-warlock python3-websockify python3-webtest python3-werkzeug python3-wrapt python3-yappi python3-zaqarclient"
	;;
	*)
		COMP_DEPENDS="alembic os-brick-common python3-alembic python3-amqp python3-automaton python3-barbicanclient python3-binary-memcached python3-cachetools python3-castellan python3-cffi python3-cffi-backend python3-cinderclient python3-cliff python3-cmd2 python3-cryptography python3-cursive python3-ddt python3-debtcollector python3-designateclient python3-dogpile.cache python3-etcd3gw python3-eventlet python3-fasteners python3-fixtures python3-futurist python3-glanceclient python3-gnocchiclient python3-greenlet python3-idna python3-importlib-metadata python3-iso8601 python3-jinja2 python3-json-pointer python3-jsonpatch python3-jsonschema python3-jwt python3-kazoo python3-keyring python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-kombu python3-lxml python3-markupsafe python3-migrate python3-monascaclient python3-monotonic python3-msgpack python3-munch python3-mypy python3-netaddr python3-neutronclient  python3-novaclient python3-oauth2client python3-oauthlib python3-openstackclient python3-openstacksdk python3-os-brick python3-os-client-config python3-os-ken python3-os-resource-classes python3-os-traits python3-os-vif python3-os-win python3-osc-lib python3-oslo.cache python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.i18n python3-oslo.log python3-oslo.messaging python3-oslo.middleware python3-oslo.policy python3-oslo.privsep python3-oslo.reports python3-oslo.rootwrap python3-oslo.serialization python3-oslo.service python3-oslo.upgradecheck python3-oslo.utils python3-oslo.versionedobjects python3-oslo.vmware python3-osprofiler python3-ovsdbapp python3-paramiko python3-pbr python3-pecan python3-pkg-resources python3-psutil python3-pure-sasl python3-pymemcache python3-pymysql python3-pyparsing python3-pyroute2 python3-pytimeparse python3-rfc3986 python3-rtslib-fb python3-setuptools python3-simplejson python3-sqlalchemy python3-stevedore python3-swiftclient python3-taskflow python3-tenacity python3-testresources python3-testtools python3-tooz python3-uhashring python3-ujson python3-uritemplate python3-voluptuous python3-warlock python3-wcwidth python3-websockify python3-webtest python3-werkzeug python3-wrapt python3-yaml python3-yappi python3-zake"
	;;
	esac
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold --allow-downgrades ${COMP_DEPENDS}"
}

net_upgrade_depends () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Installing depends on ${HOST_NAME}"
	case "${TO_OS_RELEASE}" in
	"wallaby")
		NET_DEPENDS="python3-cinderclient python3-cliff python3-cmd2 python3-designateclient python3-dogpile.cache python3-eventlet python3-glanceclient python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-msgpack python3-netaddr python3-neutron-lib python3-neutronclient python3-novaclient python3-openstackclient python3-openstacksdk python3-os-ken python3-os-traits python3-os-vif python3-osc-lib python3-oslo.cache python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.messaging python3-oslo.middleware python3-oslo.policy python3-oslo.privsep python3-oslo.serialization python3-oslo.service python3-oslo.upgradecheck python3-oslo.utils python3-oslo.versionedobjects python3-ovsdbapp python3-pbr python3-pymemcache python3-pymysql python3-pyroute2 python3-stevedore python3-tenacity python3-tooz python3-voluptuous python3-colorama python3-pyroute2.core python3-pyroute2.ethtool python3-pyroute2.ipdb python3-pyroute2.ipset python3-pyroute2.ndb python3-pyroute2.nftables python3-pyroute2.nslink python3-pyroute2.protocols"
	;;
	"xena")
		NET_DEPENDS="alembic libjs-sphinxdoc python3-alembic python3-cinderclient python3-cliff python3-debtcollector python3-designateclient python3-eventlet python3-futurist python3-glanceclient python3-greenlet python3-idna python3-jinja2 python3-jsonpatch python3-jsonschema python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-markupsafe python3-migrate python3-monotonic python3-neutronclient python3-novaclient python3-openstackclient python3-openstacksdk python3-os-ken python3-os-traits python3-os-vif python3-osc-lib python3-oslo.cache python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.i18n python3-oslo.log python3-oslo.messaging python3-oslo.middleware python3-oslo.policy python3-oslo.privsep python3-oslo.reports python3-oslo.rootwrap python3-oslo.serialization python3-oslo.service python3-oslo.upgradecheck python3-oslo.utils python3-oslo.versionedobjects python3-osprofiler python3-ovsdbapp python3-pbr python3-pecan python3-pyroute2 python3-pyroute2.ethtool python3-pyroute2.ipdb python3-pyroute2.ipset python3-pyroute2.ndb python3-pyroute2.nftables python3-pyroute2.nslink python3-pyroute2.protocols python3-q-text-as-data python3-rfc3986 python3-sqlalchemy python3-stevedore python3-testresources python3-testtools python3-tooz python3-uhashring python3-wrapt python3-yappi"
	;;
	"yoga")
		NET_DEPENDS="alembic python3-alembic python3-amqp python3-binary-memcached python3-cachetools python3-cffi python3-cffi-backend python3-cinderclient python3-cliff python3-cmd2 python3-cryptography python3-debtcollector python3-designateclient python3-dogpile.cache python3-etcd3gw python3-eventlet python3-fasteners python3-glanceclient python3-greenlet python3-idna python3-importlib-metadata python3-iso8601 python3-jinja2 python3-json-pointer python3-jwt python3-kazoo python3-keyring python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-kombu python3-lxml python3-monotonic python3-msgpack python3-neutronclient python3-novaclient python3-oauthlib python3-openstackclient python3-openstacksdk python3-os-client-config python3-os-ken python3-os-traits python3-os-vif python3-osc-lib python3-oslo.cache python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.log python3-oslo.messaging python3-oslo.metrics python3-oslo.middleware python3-oslo.policy python3-oslo.privsep python3-oslo.reports python3-oslo.rootwrap python3-oslo.serialization python3-oslo.service python3-oslo.upgradecheck python3-oslo.utils python3-oslo.versionedobjects python3-ovsdbapp python3-paramiko python3-pbr python3-pecan python3-pkg-resources python3-psutil python3-pure-sasl python3-pyparsing python3-pyroute2 python3-pytimeparse python3-setuptools python3-simplejson python3-stevedore python3-tooz python3-voluptuous python3-wcwidth python3-wrapt python3-yaml python3-zake"
	;;
	"zed")
		NET_DEPENDS="alembic python3-alembic python3-cinderclient python3-cliff python3-designateclient python3-etcd3gw python3-eventlet python3-fixtures python3-futurist python3-glanceclient python3-json-pointer python3-jsonschema python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-msgpack python3-munch python3-neutronclient python3-novaclient python3-openstackclient python3-openstacksdk python3-os-ken python3-os-traits python3-os-vif python3-osc-lib python3-oslo.cache python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.i18n python3-oslo.log python3-oslo.messaging python3-oslo.metrics python3-oslo.middleware python3-oslo.policy python3-oslo.privsep python3-oslo.reports python3-oslo.serialization python3-oslo.service python3-oslo.upgradecheck python3-oslo.utils python3-oslo.versionedobjects python3-osprofiler python3-ovsdbapp python3-pbr python3-pymemcache python3-q-text-as-data python3-sqlalchemy python3-stevedore python3-testresources python3-testtools python3-tooz python3-uhashring python3-warlock python3-webtest python3-wrapt python3-yappi"
	;;
	*)
		NET_DEPENDS="alembic libjs-sphinxdoc python3-alembic python3-amqp python3-binary-memcached python3-cachetools python3-cffi python3-cffi-backend python3-cinderclient python3-cliff python3-cmd2 python3-cryptography python3-debtcollector python3-designateclient python3-dogpile.cache python3-etcd3gw python3-eventlet python3-fasteners python3-fixtures python3-futurist python3-glanceclient python3-greenlet python3-idna python3-importlib-metadata python3-iso8601 python3-jinja2 python3-json-pointer python3-jsonpatch python3-jsonschema python3-jwt python3-kazoo python3-keyring python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-kombu python3-lxml python3-markupsafe python3-migrate python3-monotonic python3-msgpack python3-munch python3-netaddr python3-networking-l2gw python3-neutronclient python3-novaclient python3-oauthlib python3-openstackclient python3-openstacksdk python3-os-client-config python3-os-ken python3-os-traits python3-os-vif python3-osc-lib python3-oslo.cache python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.i18n python3-oslo.log python3-oslo.messaging python3-oslo.middleware python3-oslo.policy python3-oslo.privsep python3-oslo.reports python3-oslo.rootwrap python3-oslo.serialization python3-oslo.service python3-oslo.upgradecheck python3-oslo.utils python3-oslo.versionedobjects python3-osprofiler python3-ovsdbapp python3-paramiko python3-pbr python3-pecan python3-pkg-resources python3-psutil python3-pure-sasl python3-pymemcache python3-pymysql python3-pyparsing python3-pyroute2 python3-pytimeparse python3-rfc3986 python3-setuptools python3-simplejson python3-sqlalchemy python3-stevedore python3-tenacity python3-testresources python3-testtools python3-tooz python3-uhashring python3-voluptuous python3-warlock python3-wcwidth python3-webtest python3-wrapt python3-yaml python3-yappi python3-zake"
	;;
	esac
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold --allow-downgrades ${NET_DEPENDS}"
}

swift_upgrade_depends () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Installing depends on ${HOST_NAME}"
	case "${TO_OS_RELEASE}" in
	"wallaby")
		SWIFT_DEPENDS="openstack-pkg-tools python3-barbicanclient python3-castellan python3-ceilometermiddleware python3-cliff python3-cmd2 python3-dogpile.cache python3-eventlet python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-msgpack python3-netaddr python3-oslo.cache python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.messaging python3-oslo.middleware python3-oslo.serialization python3-oslo.service python3-oslo.utils python3-pbr python3-pymemcache python3-stevedore python3-swiftclient python3-tenacity python3-attr python3-colorama"
	;;
	"xena")
		SWIFT_DEPENDS="libjs-sphinxdoc openstack-pkg-tools python3-barbicanclient python3-castellan python3-ceilometermiddleware python3-cliff python3-debtcollector python3-eventlet python3-futurist python3-greenlet python3-idna python3-jinja2 python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-markupsafe python3-monotonic python3-oslo.cache python3-oslo.config python3-oslo.context python3-oslo.i18n python3-oslo.log python3-oslo.messaging python3-oslo.middleware python3-oslo.serialization python3-oslo.service python3-oslo.utils python3-pbr python3-pyeclib python3-q-text-as-data python3-rfc3986 python3-sqlalchemy python3-stevedore python3-swiftclient python3-testtools python3-uhashring python3-wrapt python3-yappi"
	;;
	"yoga")
		SWIFT_DEPENDS="python3-amqp python3-barbicanclient python3-binary-memcached python3-cachetools python3-castellan python3-ceilometermiddleware python3-cffi-backend python3-cliff python3-cmd2 python3-cryptography python3-debtcollector python3-dogpile.cache python3-eventlet python3-fasteners python3-greenlet python3-idna python3-importlib-metadata python3-iso8601 python3-jinja2 python3-jwt python3-keyring python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-kombu python3-lxml python3-monotonic python3-msgpack python3-oauthlib python3-oslo.cache python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.log python3-oslo.messaging python3-oslo.metrics python3-oslo.middleware python3-oslo.serialization python3-oslo.service python3-oslo.utils python3-pbr python3-pkg-resources python3-pyparsing python3-pytimeparse python3-setuptools python3-stevedore python3-swiftclient python3-wcwidth python3-wrapt python3-yaml"
	;;
	"zed")
		SWIFT_DEPENDS="python3-barbicanclient python3-castellan python3-ceilometermiddleware python3-cliff python3-eventlet python3-fixtures python3-futurist python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-msgpack python3-oslo.cache python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.i18n python3-oslo.log python3-oslo.messaging python3-oslo.metrics python3-oslo.middleware python3-oslo.serialization python3-oslo.service python3-oslo.utils python3-pbr python3-pyeclib python3-pymemcache python3-q-text-as-data python3-sqlalchemy python3-stevedore python3-swiftclient python3-testtools python3-uhashring python3-wrapt python3-yappi"
	;;
	*)
		SWIFT_DEPENDS="python3-amqp python3-barbicanclient python3-binary-memcached python3-cachetools python3-castellan python3-ceilometermiddleware python3-cffi-backend python3-cliff python3-cmd2 python3-cryptography python3-debtcollector python3-dogpile.cache python3-eventlet python3-fasteners python3-fixtures python3-futurist python3-greenlet python3-idna python3-importlib-metadata python3-iso8601 python3-jinja2 python3-jwt python3-keyring python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-kombu python3-lxml python3-markupsafe python3-monotonic python3-msgpack python3-netaddr python3-oauthlib python3-oslo.cache python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.i18n python3-oslo.log python3-oslo.messaging python3-oslo.middleware python3-oslo.serialization python3-oslo.service python3-oslo.utils python3-pbr python3-pkg-resources python3-pyeclib python3-pymemcache python3-pyparsing python3-pytimeparse python3-rfc3986 python3-setuptools python3-sqlalchemy python3-stevedore python3-swiftclient python3-tenacity python3-testtools python3-uhashring python3-wcwidth python3-wrapt python3-yaml python3-yappi"
	;;
	esac
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold --allow-downgrades ${SWIFT_DEPENDS}"
}

vol_upgrade_depends () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Installing depends on ${HOST_NAME}"
	case "${TO_OS_RELEASE}" in
	"wallaby")
		VOL_DEPENDS="os-brick-common python3-automaton python3-barbicanclient python3-castellan python3-cinderclient python3-cliff python3-cmd2 python3-dogpile.cache python3-eventlet python3-glanceclient python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-msgpack python3-netaddr python3-neutronclient python3-novaclient python3-oauth2client python3-openstackclient python3-openstacksdk python3-os-brick python3-os-win python3-osc-lib python3-oslo.cache python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.messaging python3-oslo.middleware python3-oslo.policy python3-oslo.privsep python3-oslo.serialization python3-oslo.service python3-oslo.upgradecheck python3-oslo.utils python3-oslo.versionedobjects python3-oslo.vmware python3-pbr python3-pymemcache python3-pymysql python3-rtslib-fb python3-stevedore python3-swiftclient python3-taskflow python3-tenacity python3-tooz python3-voluptuous python3-colorama"
	;;
	"xena")
		VOL_DEPENDS="alembic libjs-sphinxdoc os-brick-common python3-alembic python3-automaton python3-barbicanclient python3-castellan python3-cinderclient python3-cliff python3-ddt python3-debtcollector python3-eventlet python3-futurist python3-glanceclient python3-greenlet python3-idna python3-jinja2 python3-jsonpatch python3-jsonschema python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-markupsafe python3-migrate python3-monotonic python3-neutronclient python3-novaclient python3-openstackclient python3-openstacksdk python3-os-brick python3-os-win python3-osc-lib python3-oslo.cache python3-oslo.config  python3-oslo.context python3-oslo.db python3-oslo.i18n python3-oslo.log python3-oslo.messaging python3-oslo.middleware python3-oslo.policy python3-oslo.privsep python3-oslo.reports python3-oslo.rootwrap python3-oslo.serialization python3-oslo.service python3-oslo.upgradecheck python3-oslo.utils python3-oslo.versionedobjects python3-oslo.vmware python3-osprofiler python3-pbr python3-pecan python3-q-text-as-data python3-rfc3986 python3-sqlalchemy python3-stevedore python3-swiftclient python3-taskflow python3-testresources python3-testtools python3-tooz python3-uhashring python3-werkzeug python3-wrapt python3-yappi"
	;;
	"yoga")
		VOL_DEPENDS="alembic os-brick-common python3-alembic python3-amqp python3-automaton python3-barbicanclient python3-binary-memcached python3-boto3 python3-botocore python3-cachetools python3-castellan python3-cffi python3-cffi-backend python3-cinderclient python3-cliff python3-cmd2 python3-cryptography python3-ddt python3-debtcollector python3-dogpile.cache python3-etcd3gw python3-eventlet python3-fasteners python3-glanceclient python3-greenlet python3-idna python3-importlib-metadata python3-iso8601 python3-jinja2 python3-json-pointer python3-jwt python3-kazoo python3-keyring python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-kombu python3-lxml python3-monotonic python3-msgpack python3-neutronclient python3-novaclient python3-oauth2client python3-oauthlib python3-openstackclient python3-openstacksdk python3-os-brick python3-os-client-config python3-os-win python3-osc-lib python3-oslo.cache python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.log python3-oslo.messaging python3-oslo.metrics python3-oslo.middleware python3-oslo.policy python3-oslo.privsep python3-oslo.reports python3-oslo.rootwrap python3-oslo.serialization python3-oslo.service python3-oslo.upgradecheck python3-oslo.utils python3-oslo.versionedobjects python3-oslo.vmware python3-paramiko python3-pbr python3-pecan python3-pkg-resources python3-psutil python3-pure-sasl python3-pyparsing python3-pytimeparse python3-rtslib-fb python3-setuptools python3-simplejson python3-stevedore python3-swiftclient python3-taskflow python3-tooz python3-uritemplate python3-voluptuous python3-wcwidth python3-wrapt python3-yaml python3-zake"
	;;
	"zed")
		VOL_DEPENDS="alembic os-brick-common python3-alembic python3-automaton python3-barbicanclient python3-castellan python3-cinderclient python3-cliff python3-cursive python3-etcd3gw python3-eventlet python3-fixtures python3-futurist python3-glanceclient python3-json-pointer python3-jsonschema python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-msgpack python3-munch python3-neutronclient python3-novaclient python3-oauth2client python3-openstackclient python3-openstacksdk python3-os-brick python3-os-win python3-osc-lib python3-oslo.cache python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.i18n python3-oslo.log python3-oslo.messaging python3-oslo.metrics python3-oslo.middleware python3-oslo.policy python3-oslo.privsep python3-oslo.reports python3-oslo.serialization python3-oslo.service python3-oslo.upgradecheck python3-oslo.utils python3-oslo.versionedobjects python3-oslo.vmware python3-osprofiler python3-pbr python3-pymemcache python3-q-text-as-data python3-rtslib-fb python3-sqlalchemy python3-stevedore python3-swiftclient python3-taskflow python3-testresources python3-testtools python3-tooz python3-uhashring python3-warlock python3-webtest python3-werkzeug python3-wrapt python3-yappi"
	;;
	*)
		VOL_DEPENDS="alembic os-brick-common python3-alembic python3-amqp python3-automaton python3-barbicanclient python3-binary-memcached python3-cachetools python3-castellan python3-cffi python3-cffi-backend python3-cinderclient python3-cliff python3-cmd2 python3-cryptography python3-cursive python3-ddt python3-debtcollector python3-dogpile.cache python3-etcd3gw python3-eventlet python3-fasteners python3-fixtures python3-futurist python3-glanceclient python3-greenlet python3-idna python3-importlib-metadata python3-iso8601 python3-jinja2 python3-json-pointer python3-jsonpatch python3-jsonschema python3-jwt python3-kazoo python3-keyring python3-keystoneauth1 python3-keystoneclient python3-keystonemiddleware python3-kombu python3-lxml python3-markupsafe python3-migrate python3-monotonic python3-msgpack python3-munch python3-netaddr python3-neutronclient python3-novaclient python3-oauth2client python3-oauthlib python3-openstackclient python3-openstacksdk python3-os-brick python3-os-client-config python3-os-win python3-osc-lib python3-oslo.cache python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.i18n python3-oslo.log python3-oslo.messaging python3-oslo.middleware python3-oslo.policy python3-oslo.privsep python3-oslo.reports python3-oslo.rootwrap python3-oslo.serialization python3-oslo.service python3-oslo.upgradecheck python3-oslo.utils python3-oslo.versionedobjects python3-oslo.vmware python3-osprofiler python3-paramiko python3-pbr python3-pecan python3-pkg-resources python3-psutil python3-pure-sasl python3-pymemcache python3-pymysql python3-pyparsing python3-pytimeparse python3-q-text-as-data python3-rfc3986 python3-rtslib-fb python3-setuptools python3-simplejson python3-sqlalchemy python3-stevedore python3-swiftclient python3-taskflow python3-tenacity python3-testresources python3-testtools python3-tooz python3-uhashring python3-uritemplate python3-voluptuous python3-warlock python3-wcwidth python3-webtest python3-werkzeug python3-wrapt python3-yaml python3-yappi python3-zake"
	;;
	esac
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold --allow-downgrades ${VOL_DEPENDS}"
}

tempest_upgrade_depends () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Installing depends on ${HOST_NAME}"
	case "${TO_OS_RELEASE}" in
	"wallaby")
		TEMPEST_DEPENDS="barbican-tempest-plugin cinder-tempest-plugin heat-tempest-plugin keystone-tempest-plugin neutron-tempest-plugin octavia-tempest-plugin python3-barbicanclient python3-cinderclient python3-cliff python3-cmd2 python3-dogpile.cache python3-eventlet python3-glanceclient python3-gnocchiclient python3-heatclient python3-keystoneauth1 python3-keystoneclient python3-msgpack python3-netaddr python3-neutron-lib python3-neutronclient python3-novaclient python3-openstackclient python3-openstacksdk python3-os-ken python3-os-traits python3-osc-lib python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.messaging python3-oslo.middleware python3-oslo.policy python3-oslo.serialization python3-oslo.service python3-oslo.utils python3-oslo.versionedobjects python3-pbr python3-pymysql python3-stestr python3-stevedore python3-swiftclient python3-tempest python3-tenacity python3-voluptuous python3-zaqarclient telemetry-tempest-plugin tempest"
	;;
	"xena")
		TEMPEST_DEPENDS="alembic barbican-tempest-plugin cinder-tempest-plugin heat-tempest-plugin keystone-tempest-plugin libjs-sphinxdoc neutron-tempest-plugin octavia-tempest-plugin python3-alembic python3-barbicanclient python3-cinderclient python3-cliff python3-ddt python3-debtcollector python3-eventlet python3-futurist python3-glanceclient python3-greenlet python3-heatclient python3-idna python3-jinja2 python3-jsonpatch python3-jsonschema python3-keystoneauth1 python3-keystoneclient python3-markupsafe python3-migrate python3-monotonic python3-neutron-lib python3-neutronclient python3-novaclient python3-openstackclient python3-openstacksdk python3-os-collect-config python3-os-ken python3-os-testr python3-os-traits python3-osc-lib python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.i18n python3-oslo.log python3-oslo.messaging python3-oslo.middleware python3-oslo.policy python3-oslo.serialization python3-oslo.service python3-oslo.utils python3-oslo.versionedobjects python3-oslotest python3-osprofiler python3-pbr python3-pecan python3-q-text-as-data python3-rally python3-rfc3986 python3-sqlalchemy python3-stestr python3-stevedore python3-swiftclient python3-tempest python3-testresources python3-testtools python3-wrapt python3-yappi python3-zaqarclient rally telemetry-tempest-plugin tempest"
	;;
	"yoga")
		TEMPEST_DEPENDS="alembic barbican-tempest-plugin cinder-tempest-plugin heat-tempest-plugin keystone-tempest-plugin neutron-tempest-plugin octavia-tempest-plugin python3-alembic python3-amqp python3-barbicanclient python3-cachetools python3-cffi-backend python3-cinderclient python3-cliff python3-cmd2 python3-cryptography python3-ddt python3-debtcollector python3-dogpile.cache python3-eventlet python3-fasteners python3-filelock python3-gabbi python3-glanceclient python3-gnocchiclient python3-greenlet python3-heatclient python3-idna python3-importlib-metadata python3-iso8601 python3-jinja2 python3-json-pointer python3-jwt python3-keyring python3-keystoneauth1 python3-keystoneclient python3-kombu python3-lxml python3-monotonic python3-msgpack python3-neutron-lib python3-neutronclient python3-novaclient python3-oauthlib python3-openstackclient python3-openstacksdk python3-os-client-config python3-os-ken python3-os-traits python3-osc-lib python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.log python3-oslo.messaging python3-oslo.metrics python3-oslo.middleware python3-oslo.policy python3-oslo.serialization python3-oslo.service python3-oslo.utils python3-oslo.versionedobjects python3-paramiko python3-pbr python3-pecan python3-pkg-resources python3-pyparsing python3-pytest python3-pytimeparse python3-rally python3-setuptools python3-simplejson python3-stevedore python3-swiftclient python3-tempest python3-ujson python3-voluptuous python3-wcwidth python3-wrapt python3-wsgi-intercept python3-yaml python3-zaqarclient telemetry-tempest-plugin tempest"
	;;
	"zed")
		TEMPEST_DEPENDS="alembic barbican-tempest-plugin cinder-tempest-plugin heat-tempest-plugin keystone-tempest-plugin neutron-tempest-plugin octavia-tempest-plugin python3-alembic python3-barbicanclient python3-cinderclient python3-cliff python3-eventlet python3-fixtures python3-futurist python3-glanceclient python3-heatclient python3-json-pointer python3-jsonschema python3-keystoneauth1 python3-keystoneclient python3-msgpack python3-munch python3-neutron-lib python3-neutronclient python3-novaclient python3-openstackclient python3-openstacksdk python3-os-ken python3-os-traits python3-osc-lib python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.i18n python3-oslo.log python3-oslo.messaging python3-oslo.metrics python3-oslo.middleware python3-oslo.policy python3-oslo.serialization python3-oslo.service python3-oslo.utils python3-oslo.versionedobjects python3-oslotest python3-osprofiler python3-pbr python3-pytest python3-q-text-as-data python3-sqlalchemy python3-stevedore python3-swiftclient python3-tempest python3-testresources python3-testtools python3-warlock python3-webtest python3-wrapt python3-yappi python3-zaqarclient telemetry-tempest-plugin tempest"
	;;
	*)
		TEMPEST_DEPENDS="alembic barbican-tempest-plugin cinder-tempest-plugin heat-tempest-plugin keystone-tempest-plugin neutron-tempest-plugin octavia-tempest-plugin python3-alembic python3-amqp python3-barbicanclient python3-cachetools python3-cffi-backend python3-cinderclient python3-cliff python3-cmd2 python3-cryptography python3-ddt python3-debtcollector python3-dogpile.cache python3-eventlet python3-fasteners python3-filelock python3-gabbi python3-glanceclient python3-gnocchiclient python3-greenlet python3-heatclient python3-idna python3-importlib-metadata python3-iso8601 python3-jinja2 python3-json-pointer python3-jwt python3-keyring python3-keystoneauth1 python3-keystoneclient python3-kombu python3-lxml python3-monotonic python3-msgpack python3-neutron-lib python3-neutronclient python3-novaclient python3-oauthlib python3-openstackclient python3-openstacksdk python3-os-client-config python3-os-ken python3-os-traits python3-osc-lib python3-oslo.concurrency python3-oslo.config python3-oslo.context python3-oslo.db python3-oslo.log python3-oslo.messaging python3-oslo.metrics python3-oslo.middleware python3-oslo.policy python3-oslo.serialization python3-oslo.service python3-oslo.utils python3-oslo.versionedobjects python3-paramiko python3-pbr python3-pecan python3-pkg-resources python3-pyparsing python3-pytest python3-pytimeparse python3-rally python3-setuptools python3-simplejson python3-stevedore python3-swiftclient python3-tempest python3-ujson python3-voluptuous python3-wcwidth python3-wrapt python3-wsgi-intercept python3-yaml python3-zaqarclient telemetry-tempest-plugin tempest"
	;;
	esac
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold --allow-downgrades ${TEMPEST_DEPENDS}"
}

cluster_upgrade_depends () {
	green_echo "===> Upgrading dependencies on the controllers"
	iterate_on_hosts "${ALL_CTRL}" ctrl_upgrade_depends
	green_echo "===> Upgrading dependencies on the messaging nodes"
	iterate_on_hosts "${ALL_MSG}" msg_upgrade_depends
	green_echo "===> Upgrading dependencies on the network nodes"
	iterate_on_hosts "${ALL_NET}" net_upgrade_depends
	green_echo "===> Upgrading dependencies on the compute nodes"
	iterate_on_hosts -p "${ALL_COMP}" comp_upgrade_depends
	green_echo "===> Upgrading dependencies on the swift nodes"
	iterate_on_hosts -p "${ALL_SWIFTPRX}" swift_upgrade_depends
	iterate_on_hosts -p "${ALL_SWIFTSTR}" swift_upgrade_depends
	green_echo "===> Upgrading dependencies on the volume nodes"
	iterate_on_hosts -p "${ALL_VOL}" vol_upgrade_depends
	green_echo "===> Upgrading dependencies on the tempest nodes"
	iterate_on_hosts -p "${ALL_TEMPEST}" tempest_upgrade_depends
}

cluster_upgrade_check_vip_location () {
	WHERE_IS_API_VIP=$(sshi root@${ONE_CTRL_IP} "crm resource locate openstack-api-vip")
	if [ "${WHERE_IS_API_VIP}" != "${ONE_CTRL_HOST}" ] ; then
		red_echo "Error: API VIP not located on controller 1."
		echo "Move it with: crm resource move openstack-api-vip ${ONE_CTRL_HOST}"
		echo "and retry upgrade."
		exit 1
	fi
}
#######################
### START OF SCRIPT ###
#######################

green_echo "===> Check where the VIPs are hosted"
cluster_upgrade_check_vip_location

green_echo "===> Disabling puppet everywhere but on Ceph nodes"
iterate_on_hosts -p "${ALL_NODES_BUT_CEPH}" disable_puppet

green_echo "===> Switching OpenStack release name in APT configuration of OCI"
switch_oci_to_release ${FROM_DEB_RELEASE} ${FROM_OS_RELEASE} ${TO_DEB_RELEASE} ${TO_OS_RELEASE}

green_echo "===> Switch the sources.list everywhere to ${TO_DEB_RELEASE}-${TO_OS_RELEASE}"
iterate_on_hosts -p "${ALL_NODES}" switch_to_release ${FROM_DEB_RELEASE} ${FROM_OS_RELEASE} ${TO_DEB_RELEASE} ${TO_OS_RELEASE}

cluster_upgrade_depends

if [ "${USE_EXTERNAL_KEYSTONE}" != "yes" ] ; then
	cluster_upgrade_keystone
fi
if [ "${HAS_BARBICAN}" = "yes" ] ; then
	cluster_upgrade_barbican
fi

if [ -n "${ALL_COMP}" ] ; then
	if [ "${FROM_OS_RELEASE}" != "rocky" ] ; then
		# As per https://docs.openstack.org/nova/latest/admin/upgrades.html
		# we must upgrade placement before nova.
		cluster_upgrade_placement
	fi
	cluster_upgrade_neutron
	cluster_upgrade_nova
	if [ "${HAS_VOL}" = "yes" ] || [ "${HAS_OSD}" = "yes" ] ; then
		cluster_upgrade_cinder
	fi
	cluster_upgrade_glance

	if [ "${HAS_OCTAVIA}" ] ; then
		cluster_upgrade_octavia
	fi

	if [ "${HAS_TELEMETRY}" = "yes" ] ; then
		cluster_upgrade_gnocchi
		cluster_upgrade_ceilometer
		cluster_upgrade_cloudkitty
		cluster_upgrade_aodh
	fi
fi

if [ "${HAS_HEAT}" = "yes" ] ; then
	cluster_upgrade_heat
fi

if [ "${HAS_DESIGNATE}" = "yes" ] ; then
	cluster_upgrade_designate
fi

cluster_upgrade_swift

cluster_upgrade_horizon

cluster_upgrade_finilize
cluster_check_status
