#!/bin/bash
#############################################################################
#
# Copyright (c) 2022 Nutanix Inc. All rights reserved.
#
# Author: subramanian.ramachan@nutanix.com
#
# A script to check the status of the LCM autoupdate process.
# Execute on the cluster as
# bash -lc "$(curl -fsSL http://subramanian-ramachan.ubvm.nutanix.com:8080/scripts/autoupdate_status.sh.txt)"
# TODO: ^ the above location will be updated after this script is released part of LCM
#############################################################################

# /usr/local/nutanix/profile.d/bashrc defines allssh and it has protections to
# not be sourced for non interactive shells, working around it
# TODO: Check for any security implications here
PS1="x"
# needed to expand aliases defined in /usr/local/nutanix/profile.d/bashrc
shopt -s expand_aliases
source /etc/profile

# -----------------------------------------------------------------------------
# Inference based on zk/static files

echo -e "\n******* LCM framework auto update status *********\n"
nodes=`svmips|wc -w`
echo -e "\nIt is a ${nodes} node cluster\n"
echo -e "\nMapping of svmips to node uuids:\n"
svmips -u; echo

lcm_leader; echo

echo -e "\nLCM update intent: "
zkcat /appliance/logical/lcm/update ; echo

echo -e "\nLCM version as per zknodes /appliance/logical/lcm/update/*"
for n in `zkls /appliance/logical/lcm/update`; do
  echo -n "${n} -> ";
  zkcat /appliance/logical/lcm/update/${n}; echo;
done

echo -e "\nLast framework autoupdate stage as per zknodes /appliance/logical/lcm/update/*"
for n in `zkls /appliance/logical/lcm/last_update_stage`; do
  echo -n "${n} -> ";
  zkcat /appliance/logical/lcm/last_update_stage/${n}; echo;
done

echo -e "\nLCM version as per ~/cluster/config/lcm/version.txt"
allssh "cat ~/cluster/config/lcm/version.txt; echo"

echo -e "\nLCM schema version:"
zkcat /appliance/logical/lcm/schema ; echo

echo -e "\nmd5sum of ~/cluster/config/lcm/idf/* on all nodes ************"
allssh "md5sum ~/cluster/config/lcm/idf/*; echo;"

# Mercury configuration status
echo -e "\n******* LCM Mercury enabled? *********\n"
zkcat /appliance/logical/lcm/mercury_enabled ; echo
# Mercury configuration status
echo -e "\n******* LCM Mercury configuration status *********\n"
zkcat /appliance/logical/lcm/mercury_config ; echo
for n in `zkls /appliance/logical/lcm/mercury_config`; do
  echo -n "${n} -> ";
  zkcat /appliance/logical/lcm/mercury_config/${n}; echo;
done

# -----------------------------------------------------------------------------
# Inference based on genesis.out

echo -e "\n When was LCM framework updated and genesis restarted?"
allssh "grep 'sys.exit(0)' ~/data/logs/genesis.out.*"

echo -e "\n Genesis restarts on all nodes happened at:"
allssh "grep -B 4 'GENESIS START' ~/data/logs/genesis.out.*"

echo -e "\n LCM version from logs:"
allssh "grep 'framework version' ~/data/logs/genesis.out.*"

echo -e "\nBreakdown of framework update stages from genesis logs:"
allssh "grep 'framework_update_subtask.*Updating task' ~/data/logs/genesis.out.* | tail -n 15"

echo -e "\nDid LCM service start on all nodes / errors in starting LCM ?"
allssh "grep -e 'lifecycle_manager.py' ~/data/logs/genesis.out"


# even if all services started, every time genesis starts it tries to start each
# and every service and it should succeed, it might get stuck in the process
# LCM is the last service started in the chain after all services
# and LCM will be started only when it reaches that point
echo -e "Last service started on all nodes "
allssh 'grep -e "node_manager.py.*Starting.*service" ~/data/logs/genesis.out.*|tail -n 2'
# echo -e "Check if all services started: "
# cluster status
