Add tool to report on local LOCKSS instance.

This is not worthy of production, but I found it useful for monitoring
certain things while I developed this project.

bin/dev-status

+#!/usr/bin/env ruby
+# This is a pile of cobwebs.
+
+# Name of LOCKSS's Docker stack.
+STACK = "lockss-stack"
+HTTP_CHECKS = [
+    [
+        "http://{}/",
+        ->(r) { r.code == "200" && r.body.size.between?(572, 2290) }
+    ],
+    [
+        "http://{}:24621",
+        ->(r) { r.code == "302" && r.body.size == 0 }
+    ],
+    [
+        "http://{}:24621/LoginForm",
+        ->(r) { r.code == "200" && r.body.size.between?(1592, 6364) }
+    ],
+]
+
+require "json"
+require "net/http"
+require "open3"
+require "set"
+require "yaml"
+
+class Posh
+    def initialize
+        @already = 0
+        @queue = {}
+        @separator = "\n"
+    end
+
+    def flush
+        @queue.keys.sort do |a, b|
+            if a > 0 && b == 0
+                -1
+            elsif a == 0 && b > 0
+                1
+            elsif a == 0 && b == 0
+                0
+            else
+                a <=> b
+            end
+        end.each do |level|
+            if level > 0
+                if @already >= level
+                    puts @separator
+                end
+                @already = level
+            end
+
+            @queue.delete(level).each do |i|
+                if level > 0
+                    puts "#{"#" * level} #{i}"
+                else
+                    puts i
+                end
+            end
+        end
+    end
+
+    def dflush
+        flush if @queue[0]&.count.to_i > 0
+    end
+
+    def sec(level, s, force_flush = false)
+        dflush
+        @queue[level] ||= []
+        @queue[level] << s
+        flush if force_flush
+    end
+
+    def pq(s)
+        @queue[0] ||= []
+        @queue[0] << s
+    end
+
+    def p(s)
+        pq s
+        dflush
+    end
+end
+
+def run_cmd(machine, command)
+    Open3.popen2("vagrant",
+                 "ssh", machine.to_s, "--",
+                 "sudo", "--",
+                 *command) do |i, o, t|
+        raise "Command returned nonzero" if t.value.exitstatus > 0
+        return o.read
+    end
+end
+
+def check_http(hostname, checks, &block)
+    checks.each do |url, verifier|
+        begin
+            url = URI.parse url.gsub(/\{\}/, hostname.to_s)
+
+            http = Net::HTTP.new(url.host, url.port)
+            http.open_timeout = 4
+            http.read_timeout = 4
+
+            res = http.get(url)
+            state = if verifier.call(res)
+                        "ok"
+                    else
+                        "fail"
+                    end
+        rescue Errno::ECONNREFUSED
+            state = "fail-conn-refused"
+        rescue Net::OpenTimeout
+            state = "fail-conn"
+        end
+
+        block.call(url, state)
+    end
+end
+
+def get_docker_stack_status(machine, stack)
+    services_spec = run_cmd(
+        machine,
+        [
+            *%w[docker stack ps],
+            stack,
+            *%w[--format "{{json .}}"]
+        ]
+    ).lines.map { |s| JSON.parse s.chomp }
+
+    services_statuses = services_spec.map do |s|
+        name = s["Name"].sub(/^#{stack}_/, "")
+        id = s["ID"]
+        {
+            id => { name: name, state: s["CurrentState"] }
+        }
+    end.inject(:merge)
+
+    return services_statuses.sort_by { |k, v| v[:name] }.to_h
+end
+
+def get_docker_stack_ports(machine, stack)
+    services_spec = run_cmd(
+        machine,
+        [
+            *%w[docker stack services],
+            stack,
+            *%w[--format "{{json .}}"]
+        ]
+    ).lines.map { |s| JSON.parse s.chomp }
+
+    services_info = JSON.parse(
+        run_cmd(
+            machine,
+            [
+                *%[docker service inspect],
+                *services_spec.map { |s| s["ID"] }
+            ]))
+
+    listening_ports = run_cmd(machine, %[ss -ln -A tcp,udp]).lines.drop(1)
+        .map { |l|
+        f = l.split /\s+/
+        proto = f[0]
+        port = f[4].sub(/.*:/, "")
+
+        "#{port}/#{proto}"
+    }.to_set
+
+    services_ports = services_info.map do |s|
+        name = s["Spec"]["Name"].sub(/^#{stack}_/, "")
+        ports = s["Endpoint"]["Ports"].map do |p|
+            pub, tgt, proto = p["PublishedPort"], p["TargetPort"], p["Protocol"]
+
+            s = "#{pub}"
+            s += "/#{proto}" unless proto == "tcp"
+            s += " (closed)" unless listening_ports.include?("#{pub}/#{proto}")
+            s += " -> #{tgt}" unless tgt == pub
+            s
+        end
+        { name => ports }
+    end.inject(:merge)
+
+    return services_ports.sort_by { |k| k }.to_h
+end
+
+machines = YAML.safe_load File.read("vagrant-machines.yml"),
+                          symbolize_names: true
+
+p = Posh.new
+
+p.sec 1, "Dev status report"
+machines.each do |machine, config|
+    p.sec 2, machine
+
+    p.sec 3, "Check some URLs"
+    check_http(config[:hostname], HTTP_CHECKS) do |url, state|
+        p.p "#{url} => #{state}"
+    end
+
+    p.sec 3, "Stack status"
+    stack_status = get_docker_stack_status(machine, STACK)
+    longest = stack_status.map { |k, v| v[:name].length }.max
+    stack_status.each do |service, status|
+        p.p "%-#{longest}s %s" % [status[:name], status[:state]]
+    end
+
+    p.sec 3, "Listening ports"
+    stack_ports = get_docker_stack_ports(machine, STACK)
+    longest = stack_ports.keys.map(&:length).max
+    stack_ports.each do |service, ports|
+        p.p "%-#{longest}s %s" % [service, ports.join(", ")]
+    end
+end