From aad38d49c8a91cafd48add84dc051d04e4ea734b Mon Sep 17 00:00:00 2001 From: Jose Diaz de Grenu Date: Wed, 15 May 2019 11:48:53 +0200 Subject: [PATCH] system-monitor: add system-monitor example This recipe is an example for customer for a system-monitor. Scripts that verify the system status can be added to /etc/system-monitor.d The recovery mechanisms can be implemented on those scripts or in the system-monitor script. The software watchdog systemd support is used to guarantee that the system-monitor is running. When using this it is recommended to enable the systemd hardware watchdog support, refer to the systemd documentation for this. The systemd service provided by this recipe is not enabled by default as it is an example that needs customization. https://jira.digi.com/browse/DEL-6593 Signed-off-by: Jose Diaz de Grenu --- .../packagegroups/packagegroup-dey-core.bb | 1 + .../system-monitor/files/connectivity-check | 3 + .../files/recover-bridge-action | 8 +++ .../files/system-monitor.service | 12 ++++ .../system-monitor/files/system-monitor.sh | 60 +++++++++++++++++++ .../system-monitor/system-monitor_0.1.bb | 36 +++++++++++ 6 files changed, 120 insertions(+) create mode 100644 meta-digi-dey/recipes-core/system-monitor/files/connectivity-check create mode 100644 meta-digi-dey/recipes-core/system-monitor/files/recover-bridge-action create mode 100644 meta-digi-dey/recipes-core/system-monitor/files/system-monitor.service create mode 100644 meta-digi-dey/recipes-core/system-monitor/files/system-monitor.sh create mode 100644 meta-digi-dey/recipes-core/system-monitor/system-monitor_0.1.bb diff --git a/meta-digi-dey/recipes-core/packagegroups/packagegroup-dey-core.bb b/meta-digi-dey/recipes-core/packagegroups/packagegroup-dey-core.bb index bb324daaa..284bcfa0d 100644 --- a/meta-digi-dey/recipes-core/packagegroups/packagegroup-dey-core.bb +++ b/meta-digi-dey/recipes-core/packagegroups/packagegroup-dey-core.bb @@ -45,6 +45,7 @@ RDEPENDS_${PN} = "\ os-release \ recovery-utils \ sysinfo \ + system-monitor \ usbutils \ ${VIRTUAL-RUNTIME_base-utils} \ ${@bb.utils.contains('DISTRO_FEATURES', 'systemd', '', '${VIRTUAL-RUNTIME_base-utils-acpid}', d)} \ diff --git a/meta-digi-dey/recipes-core/system-monitor/files/connectivity-check b/meta-digi-dey/recipes-core/system-monitor/files/connectivity-check new file mode 100644 index 000000000..ec394646e --- /dev/null +++ b/meta-digi-dey/recipes-core/system-monitor/files/connectivity-check @@ -0,0 +1,3 @@ +#!/bin/sh + +[ "$(nmcli n c)" = "full" ] diff --git a/meta-digi-dey/recipes-core/system-monitor/files/recover-bridge-action b/meta-digi-dey/recipes-core/system-monitor/files/recover-bridge-action new file mode 100644 index 000000000..1c85a9c70 --- /dev/null +++ b/meta-digi-dey/recipes-core/system-monitor/files/recover-bridge-action @@ -0,0 +1,8 @@ +#!/bin/sh + +REBOOT_THRESHOLD="9" +checks_failed="${1}" + +if [ "${checks_failed}" -ge "${REBOOT_THRESHOLD}" ]; then + reboot +fi diff --git a/meta-digi-dey/recipes-core/system-monitor/files/system-monitor.service b/meta-digi-dey/recipes-core/system-monitor/files/system-monitor.service new file mode 100644 index 000000000..6cdf411cd --- /dev/null +++ b/meta-digi-dey/recipes-core/system-monitor/files/system-monitor.service @@ -0,0 +1,12 @@ +[Unit] +Description=System monitor + +[Service] +Type=notify +ExecStart=system-monitor.sh +Restart=always +RestartSec=1 +WatchdogSec=200 + +[Install] +WantedBy=multi-user.target diff --git a/meta-digi-dey/recipes-core/system-monitor/files/system-monitor.sh b/meta-digi-dey/recipes-core/system-monitor/files/system-monitor.sh new file mode 100644 index 000000000..267aca492 --- /dev/null +++ b/meta-digi-dey/recipes-core/system-monitor/files/system-monitor.sh @@ -0,0 +1,60 @@ +#!/bin/sh +#=============================================================================== +# +# system-monitor.sh +# +# Copyright (C) 2019 by Digi International Inc. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 2 as published by +# the Free Software Foundation. +# +# !Description: Checks the system status and takes actions if the system is +# not in the desired state. +# +# The system-monitor systemd service calls this script periodically. +# +#=============================================================================== + +# Watchdog time in specified in system-monitor.service +# The script will monitor the system and kick the software watchdod +# every 1/4 of the time watchdog time configured. +WDOG_TIME_SEC=$((WATCHDOG_USEC / 1000000 / 4)) + +log() { + systemd-cat -p "${1}" -t system-monitor printf "%s" "${2}" +} + +checks_failed=0 + +monitor() +{ + # run system check actions + if run-parts -a "${checks_failed}" "/etc/system-monitor/check.d"; then + log info "system check correct" + checks_failed=0 + else + # system check failed, run recover actions + checks_failed=$((checks_failed + 1)) + log warning "system check failed (${checks_failed} consecutive checks failed so far)" + run-parts -a "${checks_failed}" "/etc/system-monitor/recover-action.d" + fi +} + +log info "started" +/bin/systemd-notify --ready + +# system-monitor main loop: +# * Monitor system status +# * Kick software watchdog +# * Wait for remaining time (if any) until next check +while true; do + loop_start=$(date +%s) + + monitor + /bin/systemd-notify WATCHDOG=1 + + time_to_wait=$((WDOG_TIME_SEC - ($(date +%s) - loop_start))) + [ ${time_to_wait} -gt 0 ] && sleep ${time_to_wait} +done diff --git a/meta-digi-dey/recipes-core/system-monitor/system-monitor_0.1.bb b/meta-digi-dey/recipes-core/system-monitor/system-monitor_0.1.bb new file mode 100644 index 000000000..3288f0cdd --- /dev/null +++ b/meta-digi-dey/recipes-core/system-monitor/system-monitor_0.1.bb @@ -0,0 +1,36 @@ +# Copyright (C) 2019 Digi International +SUMMARY = "Install and start a systemd service" +LICENSE = "MPL-2.0" +LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MPL-2.0;md5=815ca599c9df247a0c7f619bab123dad" + +SRC_URI = " \ + file://connectivity-check \ + file://recover-bridge-action \ + file://system-monitor.sh \ + file://system-monitor.service \ +" +S = "${WORKDIR}" + +inherit systemd + +REQUIRED_DISTRO_FEATURES= "systemd" + +SYSTEMD_SERVICE_${PN} = "system-monitor.service" + +# The system-monitor.sh script is an example that needs to be customized. +# This service also needs to be manually enabled. +SYSTEMD_AUTO_ENABLE ?= "disable" + +do_install() { + install -d ${D}${bindir} + install -m 0755 ${WORKDIR}/system-monitor.sh ${D}${bindir} + + install -d ${D}${systemd_unitdir}/system/ + install -m 0644 ${WORKDIR}/system-monitor.service ${D}${systemd_unitdir}/system/ + + install -d ${D}${sysconfdir}/system-monitor/check.d ${D}${sysconfdir}/system-monitor/recover-action.d + install -m 0755 ${WORKDIR}/connectivity-check ${D}${sysconfdir}/system-monitor/check.d + install -m 0755 ${WORKDIR}/recover-bridge-action ${D}${sysconfdir}/system-monitor/recover-action.d +} + +FILES_${PN} += "${systemd_unitdir}/system/system-monitor.service"