system-monitor: add system-monitor example

This recipe is an example for customer for a system-monitor. Scripts that
verify the system status can be added to /etc/system-monitor.d

The recovery mechanisms can be implemented on those scripts or in the
system-monitor script. The software watchdog systemd support is used to
guarantee that the system-monitor is running.

When using this it is recommended to enable the systemd hardware watchdog
support, refer to the systemd documentation for this.

The systemd service provided by this recipe is not enabled by default as it
is an example that needs customization.

https://jira.digi.com/browse/DEL-6593

Signed-off-by: Jose Diaz de Grenu <Jose.DiazdeGrenu@digi.com>

meta-digi-dey/recipes-core/packagegroups/packagegroup-dey-core.bb

@@ -45,6 +45,7 @@ RDEPENDS_${PN} = "\
     os-release \
     recovery-utils \
     sysinfo \
+    system-monitor \
     usbutils \
     ${VIRTUAL-RUNTIME_base-utils} \
     ${@bb.utils.contains('DISTRO_FEATURES', 'systemd', '', '${VIRTUAL-RUNTIME_base-utils-acpid}', d)} \

meta-digi-dey/recipes-core/system-monitor/files/connectivity-check

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+[ "$(nmcli n c)" = "full" ]

meta-digi-dey/recipes-core/system-monitor/files/recover-bridge-action

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+REBOOT_THRESHOLD="9"
+checks_failed="${1}"
+
+if [ "${checks_failed}" -ge "${REBOOT_THRESHOLD}" ]; then
+	reboot
+fi

meta-digi-dey/recipes-core/system-monitor/files/system-monitor.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=System monitor
+
+[Service]
+Type=notify
+ExecStart=system-monitor.sh
+Restart=always
+RestartSec=1
+WatchdogSec=200
+
+[Install]
+WantedBy=multi-user.target

meta-digi-dey/recipes-core/system-monitor/files/system-monitor.sh

@@ -0,0 +1,60 @@
+#!/bin/sh
+#===============================================================================
+#
+#  system-monitor.sh
+#
+#  Copyright (C) 2019 by Digi International Inc.
+#  All rights reserved.
+#
+#  This program is free software; you can redistribute it and/or modify it
+#  under the terms of the GNU General Public License version 2 as published by
+#  the Free Software Foundation.
+#
+#  !Description: Checks the system status and takes actions if the system is
+#                not in the desired state.
+#
+#  The system-monitor systemd service calls this script periodically.
+#
+#===============================================================================
+
+# Watchdog time in specified in system-monitor.service
+# The script will monitor the system and kick the software watchdod
+# every 1/4 of the time watchdog time configured.
+WDOG_TIME_SEC=$((WATCHDOG_USEC / 1000000 / 4))
+
+log() {
+	systemd-cat -p "${1}" -t system-monitor printf "%s" "${2}"
+}
+
+checks_failed=0
+
+monitor()
+{
+	# run system check actions
+	if run-parts -a "${checks_failed}" "/etc/system-monitor/check.d"; then
+		log info "system check correct"
+		checks_failed=0
+	else
+		# system check failed, run recover actions
+		checks_failed=$((checks_failed + 1))
+		log warning "system check failed (${checks_failed} consecutive checks failed so far)"
+		run-parts -a "${checks_failed}" "/etc/system-monitor/recover-action.d"
+	fi
+}
+
+log info "started"
+/bin/systemd-notify --ready
+
+# system-monitor main loop:
+#   * Monitor system status
+#   * Kick software watchdog
+#   * Wait for remaining time (if any) until next check
+while true; do
+	loop_start=$(date +%s)
+
+	monitor
+	/bin/systemd-notify WATCHDOG=1
+
+	time_to_wait=$((WDOG_TIME_SEC - ($(date +%s) - loop_start)))
+	[ ${time_to_wait} -gt 0 ] && sleep ${time_to_wait}
+done

meta-digi-dey/recipes-core/system-monitor/system-monitor_0.1.bb

@@ -0,0 +1,36 @@
+# Copyright (C) 2019 Digi International
+SUMMARY = "Install and start a systemd service"
+LICENSE = "MPL-2.0"
+LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MPL-2.0;md5=815ca599c9df247a0c7f619bab123dad"
+
+SRC_URI = " \
+    file://connectivity-check \
+    file://recover-bridge-action \
+    file://system-monitor.sh \
+    file://system-monitor.service \
+"
+S = "${WORKDIR}"
+
+inherit systemd
+
+REQUIRED_DISTRO_FEATURES= "systemd"
+
+SYSTEMD_SERVICE_${PN} = "system-monitor.service"
+
+# The system-monitor.sh script is an example that needs to be customized.
+# This service also needs to be manually enabled.
+SYSTEMD_AUTO_ENABLE ?= "disable"
+
+do_install() {
+	install -d ${D}${bindir}
+	install -m 0755 ${WORKDIR}/system-monitor.sh ${D}${bindir}
+
+	install -d ${D}${systemd_unitdir}/system/
+	install -m 0644 ${WORKDIR}/system-monitor.service ${D}${systemd_unitdir}/system/
+
+	install -d ${D}${sysconfdir}/system-monitor/check.d ${D}${sysconfdir}/system-monitor/recover-action.d
+	install -m 0755 ${WORKDIR}/connectivity-check ${D}${sysconfdir}/system-monitor/check.d
+	install -m 0755 ${WORKDIR}/recover-bridge-action ${D}${sysconfdir}/system-monitor/recover-action.d
+}
+
+FILES_${PN} += "${systemd_unitdir}/system/system-monitor.service"