Add monitor.pl metrics exporters to web servers

This implements everything except the http check, which I believe we are
only using for fulcrum. Everything that can has a 1-second timeout, so
slow services will be considered down.

manifests/profile/prometheus/exporter/webserver/base.pp

@@ -0,0 +1,31 @@
+# Copyright (c) 2024 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+
+class nebula::profile::prometheus::exporter::webserver::base (
+  Hash[String, Hash[String, String]] $mariadb_connect = {},
+  Hash[String, Array[String]] $nfs_mounts = {},
+  Hash[String, Array[String]] $solr_instances = {},
+  Hash[String, Boolean] $check_shibd = {},
+  String $target,
+) {
+  include stdlib
+  $target_mariadb = pick_default($mariadb_connect[$target], {})
+  $target_nfs = pick_default($nfs_mounts[$target], [])
+  $target_solr = pick_default($solr_instances[$target], [])
+  $target_shibd = pick_default($check_shibd[$target], false)
+
+  file { "/usr/local/lib/prom_web_exporter/metrics":
+    mode    => "0755",
+    content => template("nebula/profile/prometheus/exporter/webserver/metrics.sh.erb"),
+    require => File["/usr/local/lib/prom_web_exporter"],
+  }
+
+  file { "/usr/local/lib/prom_web_exporter":
+    ensure => "directory",
+  }
+
+  if $target_mariadb != {} {
+    package { "mariadb-client": }
+  }
+}

manifests/profile/prometheus/exporter/webserver/fulcrum.pp

@@ -0,0 +1,9 @@
+# Copyright (c) 2024 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+
+class nebula::profile::prometheus::exporter::webserver::fulcrum {
+  class { "nebula::profile::prometheus::exporter::webserver::base":
+    target => "fulcrum"
+  }
+}

manifests/profile/prometheus/exporter/webserver/hathitrust.pp

@@ -0,0 +1,9 @@
+# Copyright (c) 2024 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+
+class nebula::profile::prometheus::exporter::webserver::hathitrust {
+  class { "nebula::profile::prometheus::exporter::webserver::base":
+    target => "hathitrust"
+  }
+}

manifests/profile/prometheus/exporter/webserver/quod.pp

@@ -0,0 +1,9 @@
+# Copyright (c) 2024 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+
+class nebula::profile::prometheus::exporter::webserver::quod {
+  class { "nebula::profile::prometheus::exporter::webserver::base":
+    target => "quod"
+  }
+}

manifests/profile/prometheus/exporter/webserver/vhost.pp

@@ -0,0 +1,18 @@
+# Copyright (c) 2024 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+
+class nebula::profile::prometheus::exporter::webserver::vhost (
+  Boolean $testing = false
+) {
+  if $testing {
+    include apache
+  }
+
+  apache::vhost { "prometheus-webserver-exporter":
+    port     => "9180",
+    docroot  => "/usr/local/lib/prom_web_exporter",
+    aliases  => [{ scriptalias => "/", path => "/usr/local/lib/prom_web_exporter/" }],
+    rewrites => [{ rewrite_rule => ["^/$ /metrics [last,redirect=permanent]"]}],
+  }
+}

manifests/profile/prometheus/exporter/webserver/www_lib.pp

@@ -0,0 +1,9 @@
+# Copyright (c) 2024 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+
+class nebula::profile::prometheus::exporter::webserver::www_lib {
+  class { "nebula::profile::prometheus::exporter::webserver::base":
+    target => "www_lib"
+  }
+}

manifests/role/app_host/quod_dev.pp

@@ -19,4 +19,5 @@
   include nebula::profile::quod::dev::perl
   include nebula::profile::prometheus::exporter::mysql
   include nebula::profile::tesseract
+  include nebula::profile::prometheus::exporter::webserver::quod
 }

manifests/role/app_host/quod_prod.pp

@@ -17,4 +17,5 @@
   include nebula::profile::quod::prod::perl
   include nebula::profile::quod::prod::haproxy
   include nebula::profile::networking::firewall::http
+  include nebula::profile::prometheus::exporter::webserver::quod
 }

manifests/role/webhost/fulcrum_www_and_app.pp

@@ -60,4 +60,7 @@
   }
 
   ensure_packages(['pigz'])
+
+  include nebula::profile::prometheus::exporter::webserver::vhost
+  include nebula::profile::prometheus::exporter::webserver::fulcrum
 }

manifests/role/webhost/htvm.pp

@@ -39,4 +39,7 @@
 
   # for HathiTrust deployment scripts
   package { 'rdist': }
+
+  include nebula::profile::prometheus::exporter::webserver::vhost
+  include nebula::profile::prometheus::exporter::webserver::hathitrust
 }

manifests/role/webhost/www_lib_vm.pp

@@ -45,4 +45,7 @@
     config_source    => $shibboleth_config_source,
     watchdog_minutes => '*/30',
   }
+
+  include nebula::profile::prometheus::exporter::webserver::vhost
+  include nebula::profile::prometheus::exporter::webserver::www_lib
 }

spec/classes/profile/prometheus/exporter/webserver/base_spec.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2024 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+require 'spec_helper'
+
+describe 'nebula::profile::prometheus::exporter::webserver::base' do
+  on_supported_os.each do |os, os_facts|
+    context "on #{os}" do
+      let(:facts) { os_facts }
+
+      it { is_expected.not_to compile }
+
+      context "with target set to abcd" do
+        let(:params) { { target: "abcd" } }
+
+        def have_script
+          contain_file("/usr/local/lib/prom_web_exporter/metrics")
+        end
+
+        it { is_expected.to compile }
+        it { is_expected.not_to contain_package("mariadb-client") }
+        it { is_expected.to contain_file("/usr/local/lib/prom_web_exporter").with_ensure("directory") }
+        it { is_expected.to have_script.with_mode("0755") }
+        it { is_expected.to have_script.with_content(/^#!\/usr\/bin\/env bash$/) }
+        it { is_expected.to have_script.with_content(/^ENABLE_MARIADB_CHECK="false"$/) }
+        it { is_expected.to have_script.with_content(/^NFS_MOUNTS=\(\)$/) }
+        it { is_expected.to have_script.with_content(/^SOLR_INSTANCES=\(\)$/) }
+        it { is_expected.to have_script.with_content(/^ENABLE_SHIBD_CHECK="false"$/) }
+
+        context "with mariadb credentials set" do
+          let(:params) do
+            super().merge({ mariadb_connect: { "abcd" => {
+              "hostname" => "abcd-db-host",
+              "database" => "abcd-database",
+              "username" => "abcd-db-monitor",
+              "password" => "random-password",
+            }}})
+          end
+
+          it { is_expected.to contain_package("mariadb-client") }
+          it { is_expected.to have_script.with_content(/^ENABLE_MARIADB_CHECK="true"$/) }
+        end
+
+        context "with nfs mounts set" do
+          let(:params) do
+            super().merge({ nfs_mounts: { "abcd" => %w[/abc /def /ghi] } })
+          end
+
+          it { is_expected.to have_script.with_content(/^NFS_MOUNTS=\("\/abc" "\/def" "\/ghi"\)$/) }
+        end
+
+        context "with solr instances set" do
+          let(:params) do
+            super().merge({ solr_instances: { "abcd" => ["http://solr/solr/core"] } })
+          end
+
+          it { is_expected.to have_script.with_content(/^SOLR_INSTANCES=\("http:\/\/solr\/solr\/core"\)$/) }
+        end
+
+        context "with shibd check enabled" do
+          let(:params) do
+            super().merge({ check_shibd: { "abcd" => true } })
+          end
+
+          it { is_expected.to have_script.with_content(/^ENABLE_SHIBD_CHECK="true"$/) }
+        end
+      end
+    end
+  end
+end

spec/classes/profile/prometheus/exporter/webserver/fulcrum_spec.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2024 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+require 'spec_helper'
+
+describe 'nebula::profile::prometheus::exporter::webserver::fulcrum' do
+  on_supported_os.each do |os, os_facts|
+    context "on #{os}" do
+      let(:facts) { os_facts }
+      let(:base) { "nebula::profile::prometheus::exporter::webserver::base" }
+
+      it { is_expected.to compile }
+      it { is_expected.to contain_class(base).with_target("fulcrum") }
+    end
+  end
+end

spec/classes/profile/prometheus/exporter/webserver/hathitrust_spec.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2024 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+require 'spec_helper'
+
+describe 'nebula::profile::prometheus::exporter::webserver::hathitrust' do
+  on_supported_os.each do |os, os_facts|
+    context "on #{os}" do
+      let(:facts) { os_facts }
+      let(:base) { "nebula::profile::prometheus::exporter::webserver::base" }
+
+      it { is_expected.to compile }
+      it { is_expected.to contain_class(base).with_target("hathitrust") }
+    end
+  end
+end

spec/classes/profile/prometheus/exporter/webserver/quod_spec.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2024 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+require 'spec_helper'
+
+describe 'nebula::profile::prometheus::exporter::webserver::quod' do
+  on_supported_os.each do |os, os_facts|
+    context "on #{os}" do
+      let(:facts) { os_facts }
+      let(:base) { "nebula::profile::prometheus::exporter::webserver::base" }
+
+      it { is_expected.to compile }
+      it { is_expected.to contain_class(base).with_target("quod") }
+    end
+  end
+end

spec/classes/profile/prometheus/exporter/webserver/vhost_spec.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2024 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+require 'spec_helper'
+
+describe 'nebula::profile::prometheus::exporter::webserver::vhost' do
+  on_supported_os.each do |os, os_facts|
+    context "on #{os}" do
+      let(:facts) { os_facts }
+
+      it { is_expected.not_to compile }
+
+      context "with apache configured for testing" do
+        let(:params) { { testing: true } }
+
+        def have_vhost
+          contain_apache__vhost("prometheus-webserver-exporter")
+        end
+
+        it { is_expected.to compile }
+        it { is_expected.to have_vhost.with_port(9180) }
+      end
+    end
+  end
+end

spec/classes/profile/prometheus/exporter/webserver/www_lib_spec.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2024 The Regents of the University of Michigan.
+# All Rights Reserved. Licensed according to the terms of the Revised
+# BSD License. See LICENSE.txt for details.
+require 'spec_helper'
+
+describe 'nebula::profile::prometheus::exporter::webserver::www_lib' do
+  on_supported_os.each do |os, os_facts|
+    context "on #{os}" do
+      let(:facts) { os_facts }
+      let(:base) { "nebula::profile::prometheus::exporter::webserver::base" }
+
+      it { is_expected.to compile }
+      it { is_expected.to contain_class(base).with_target("www_lib") }
+    end
+  end
+end