Add system status monitors

scripts/start

@@ -78,6 +78,13 @@ echo "Starting karen..."
 echo
 ./karen &>> "${UMBREL_LOGS}/karen.log" &
 
+echo "Starting status monitors..."
+pkill -f ./scripts/status-monitor || true
+./scripts/status-monitor memory 60 &>> "${UMBREL_LOGS}/status-monitor.log" &
+./scripts/status-monitor storage 60 &>> "${UMBREL_LOGS}/status-monitor.log" &
+./scripts/status-monitor temperature 15 &>> "${UMBREL_LOGS}/status-monitor.log" &
+./scripts/status-monitor uptime 15 &>> "${UMBREL_LOGS}/status-monitor.log" &
+
 echo "Starting memory monitor..."
 echo
 ./scripts/memory-monitor &>> "${UMBREL_LOGS}/memory-monitor.log" &

scripts/status

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-# Get Umbrel Status
-
-docker ps -a

scripts/status-monitor

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+UMBREL_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/..)"
+
+resource="${1}"
+interval="${2}"
+
+log () {
+  echo "$(date "+%Y-%m-%d %H:%M:%S") ${@}"
+}
+
+log "Status monitor for \"${resource}\" running every ${interval} seconds!"
+
+status_script="${UMBREL_ROOT}/scripts/status/${resource}"
+json_path="${UMBREL_ROOT}/statuses/${resource}-status.json"
+while true; do
+  result=$("${status_script}")
+  echo "$result" > "${json_path}"
+  sleep "${interval}"
+done

scripts/status/memory

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+UMBREL_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/../..)"
+
+memory_total_bytes() {
+  free --bytes | awk 'NR==2 {print $2}'
+}
+
+memory_used_bytes() {
+  free --bytes | awk 'NR==2 {print $3}'
+}
+
+get_container_memory_use() {
+  local container="${1}"
+
+  local container_memory=0
+
+  local container_pids=$(docker top "${container}" | tail -n +2 | awk '{print $2}')
+  for pid in $container_pids; do
+    local pid_memory=$(ps u "${pid}" | awk '{print $4}' | grep -v 'MEM')
+    if [[ ! -z "${pid_memory}" ]]; then
+      container_memory=$(awk "BEGIN {print ${container_memory}+${pid_memory}}")
+    fi
+  done
+
+  echo "${container_memory}"
+}
+
+get_app_memory_use() {
+  local app="${1}"
+
+  local app_memory=0
+
+  local app_containers=$("${UMBREL_ROOT}/scripts/app" compose "${app}" ps | awk '{print $1}' | grep -v 'Name\|-----')
+  for container in $app_containers; do
+    local container_memory=$(get_container_memory_use "${container}")
+    app_memory=$(awk "BEGIN {print ${app_memory}+${container_memory}}")
+  done
+
+  # Above values return % of total memory used so we need to convert to bytes
+  total=$(memory_total_bytes)
+  awk "BEGIN {print int(${total}*${app_memory}/100)}"
+}
+
+json="{}"
+
+total=$(memory_total_bytes)
+json=$(echo $json | jq --arg total "${total}" '. + {total: $total|tonumber}')
+
+used=$(memory_used_bytes)
+json=$(echo $json | jq --arg used "${used}" '. + {used: $used|tonumber}')
+
+cumulative_app_memory="0"
+for app in $( "${UMBREL_ROOT}/scripts/app" ls-installed)
+do
+  app_memory=$(get_app_memory_use "${app}")
+  cumulative_app_memory=$(($cumulative_app_memory+$app_memory))
+  json=$(echo $json | jq --arg app "${app}" --arg app_memory "${app_memory}" '.breakdown |= .+ [{id: $app, used: $app_memory|tonumber}]')
+done
+
+umbrel=$(($used-$cumulative_app_memory))
+json=$(echo $json | jq --arg umbrel "${umbrel}" '.breakdown |= .+ [{id: "umbrel", used: $umbrel|tonumber}]')
+
+echo $json | jq '.breakdown |= sort_by(-.used)'

scripts/status/storage

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+UMBREL_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/../..)"
+
+filesystem_total_storage_bytes() {
+  local directory="${1}"
+  df --block-size=1 --output=size "${directory}" | tail -n 1
+}
+
+filesystem_used_storage_bytes() {
+  local directory="${1}"
+  df --block-size=1 --output=used "${directory}" | tail -n 1
+}
+
+directory_size_bytes() {
+  local directory="${1}"
+  du --block-size=1 --max-depth=0 "${directory}" | awk '{print $1}'
+}
+
+json="{}"
+
+total=$(filesystem_total_storage_bytes "${UMBREL_ROOT}")
+json=$(echo $json | jq --arg total "${total}" '. + {total: $total|tonumber}')
+
+used=$(filesystem_used_storage_bytes "${UMBREL_ROOT}")
+json=$(echo $json | jq --arg used "${used}" '. + {used: $used|tonumber}')
+
+cumulative_app_size="0"
+for app in $( "${UMBREL_ROOT}/scripts/app" ls-installed)
+do
+  app_size=$(directory_size_bytes "${UMBREL_ROOT}/app-data/${app}")
+  cumulative_app_size=$(($cumulative_app_size+$app_size))
+  json=$(echo $json | jq --arg app "${app}" --arg app_size "${app_size}" '.breakdown |= .+ [{id: $app, used: $app_size|tonumber}]')
+done
+
+umbrel=$(($used-$cumulative_app_size))
+json=$(echo $json | jq --arg umbrel "${umbrel}" '.breakdown |= .+ [{id: "umbrel", used: $umbrel|tonumber}]')
+
+echo $json | jq '.breakdown |= sort_by(-.used)'

scripts/status/temperature

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+# This may not always be the CPU sensor on other hardware.
+# To be sure we need to iterate through /sys/class/thermal/thermal_zone*/type
+# https://www.kernel.org/doc/Documentation/thermal/sysfs-api.txt
+temperature_file="/sys/class/thermal/thermal_zone0/temp"
+
+# Check temperature file exists
+if [[ ! -f "${temperature_file}" ]]
+then
+  echo "null"
+  exit
+fi
+
+# Read temperature file
+temperature=$(cat "${temperature_file}")
+
+# Convert millidegrees celsius to degrees celsius
+echo "${temperature}" | awk '{print int($1/1000)}'

scripts/status/uptime

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+json="{}"
+
+cat /proc/uptime | awk '{print int($1)}'