haccfiles/parsons/monit.nix
stuebinm 8c3d3bf6db monitoring: warn if no deploy for 10 days
this is not entirely accurate — the lastModified attribute of a flake's
self-input gives the date of the last commit, not the last deploy. But I
figure it's close enough and less obscure to check than reading in the
last date via nix-env.

inspired by: we did no server updates for two weeks.
2024-05-02 22:33:47 +02:00

64 lines
2.1 KiB
Nix

{ config, options, lib, pkgs, ... }:
let
checkHash = pkgs.writeScriptBin "check-commit-hash" ''
#!${lib.getExe pkgs.fish}
set wanted (${lib.getExe pkgs.curl} -s https://git.infra4future.de/api/v1/repos/hacc/haccfiles/branches/main \
-H 'accept: application/json' | jq -r .commit.id)
if test $status != 0
echo "could not reach git.infra4future.de"
exit 2
end
set actual (cat /etc/haccfiles-commit)
if test $status != 0
echo "/etc/haccfiles-commit does not exist??"
exit 2
end
if test $actual != $wanted
echo "parsons was built on $actual, but commit on main is $wanted"
exit 1
end
'';
checkDeployAge = pkgs.writeScriptBin "check-deploy-age" ''
#!${lib.getExe pkgs.fish}
set date (date +%s)
# we do this indirection here so monit's config won't change on each deploy
set deploytimestamp (cat /etc/haccfiles-timestamp)
set age (expr $date - $deploytimestamp)
if test $age -ge (expr 3600 \* 24 \* 10)
echo "${config.networking.hostName} has not been deployed since 10 days, perhaps someone should do updates?"
exit 1
end
'';
in
{
mailserver.monitoring = {
enable = true;
alertAddress = "admin@hacc.space";
config = (lib.replaceStrings ["port 22"] ["port ${toString (lib.head config.services.openssh.ports)}"] options.mailserver.monitoring.config.default);
};
services.monit.config = ''
check host onlyoffice with address onlyoffice.infra4future.de
start program "/run/current-system/sw/bin/lxc-start onlyoffice"
stop program "/run/current-system/sw/bin/lxc-stop onlyoffice"
if failed port 443 protocol https status = 302
then restart
check program deployed-commit-on-main path ${lib.getExe checkHash}
if status == 1 for 10 cycles then alert
if status == 2 for 3 cycles then alert
check program is-system-running path ${pkgs.systemd}/bin/systemctl is-system-running
if status != 0 then alert
check program check-deploy-age path ${lib.getExe checkDeployAge}
if status == 1 then alert
'';
}