monit: a couple new checks

move the monit config out of mail.nix, and add two checks:
 - has any systemd unit failed?
 - is the currently deployed commit the tip of the main branch of
   haccfiles?

modules/buildinfo.nix

@@ -27,4 +27,7 @@ in
     Last commit was at ${formatDate self.lastModifiedDate}.
     ${if self ? dirtyRev then "\nPlease remember to commit your changes.\n" else ""}
   '';
+
+  # used by monit
+  environment.etc."haccfiles-commit".text = self.rev or self.dirtyRev;
 }

parsons/configuration.nix

@@ -19,6 +19,7 @@
     ./tracktrain.nix
     ./uffd.nix
     ./lxc.nix
+    ./monit.nix
   ];
 
   hacc.bindToPersist = [ "/var/lib/acme" ];

parsons/mail.nix

@@ -20,13 +20,6 @@
     monitoring = {
       enable = true;
       alertAddress = "admin@hacc.space";
-      config = (lib.replaceStrings ["port 22"] ["port ${toString (lib.head config.services.openssh.ports)}"] options.mailserver.monitoring.config.default) + ''
-        check host onlyoffice with address onlyoffice.infra4future.de
-              start program "/run/current-system/sw/bin/lxc-start onlyoffice"
-              stop program "/run/current-system/sw/bin/lxc-stop onlyoffice"
-              if failed port 443 protocol https status = 302
-              then restart
-      '';
     };
     domains = [
       "hacc.space"

parsons/monit.nix

@@ -0,0 +1,47 @@
+{ config, options, lib, pkgs, ... }:
+
+let
+  checkHash = pkgs.writeScriptBin "check-commit-hash" ''
+    #!${lib.getExe pkgs.fish}
+    set wanted (${lib.getExe pkgs.curl} -s https://git.infra4future.de/api/v1/repos/hacc/haccfiles/branches/main \
+                -H 'accept: application/json' | jq -r .commit.id)
+
+    if test $status != 0
+      echo "could not reach git.infra4future.de"
+      exit 2
+    end
+
+    set actual (cat /etc/haccfiles-commit)
+    if test $status != 0
+      echo "/etc/haccfiles-commit does not exist??"
+      exit 2
+    end
+
+    if test $actual != $wanted
+      echo "parsons was built on $actual, but commit on main is $wanted"
+      exit 1
+    end
+  '';
+in
+{
+  mailserver.monitoring = {
+    enable = true;
+    alertAddress = "admin@hacc.space";
+    config = (lib.replaceStrings ["port 22"] ["port ${toString (lib.head config.services.openssh.ports)}"] options.mailserver.monitoring.config.default);
+  };
+
+  services.monit.config = ''
+      check host onlyoffice with address onlyoffice.infra4future.de
+            start program "/run/current-system/sw/bin/lxc-start onlyoffice"
+            stop program "/run/current-system/sw/bin/lxc-stop onlyoffice"
+            if failed port 443 protocol https status = 302
+            then restart
+
+      check program deployed-commit-on-main path ${lib.getExe checkHash}
+            if status == 1 for 10 cycles then alert
+            if status == 2 for 3 cycles then alert
+
+      check program is-system-running path ${pkgs.systemd}/bin/systemctl is-system-running
+            if status != 0 then alert
+  '';
+}