Merge remote-tracking branch 'master' to local master

7 files changed, 591 insertions, 109 deletions

diff --git a/doc/note/maven/maven.txt b/doc/note/maven/maven.txt
index cdfdd9a..309fa63 100644
--- a/doc/note/maven/maven.txt
+++ b/doc/note/maven/maven.txt
@@ -23,6 +23,13 @@ mvn deploy -DaltDeploymentRepository=artifactory-snapshots::default::https://art
 mvn deploy -Dcmake.generate.skip=true -Dcmake.compile.skip=true -DaltDeploymentRepository=artifactory-releases::default::https://artifactory.tools.pnet.ch/artifactory/libs-release-local
 
 
+true `# Deploy custom build 20240206` \
+  && DEPLOPTS= \
+  && mvn clean install -pl '!gateleen-hook-js,!gateleen-playground' \
+  && mvn deploy -DskipTests -pl '!gateleen-hook-js,!gateleen-playground' $DEPLOPTS \
+  && true
+
+
 ## Run e2e locally
 mvn verify -U -DSelBaseUrl=http://localhost:7012/apigateway/services/foo/index.html -Dskip.tests=false -Dserver.host=localhost -Dserver.port=7012 -Ptestsuite
 
diff --git a/doc/note/qemu/qemu.txt b/doc/note/qemu/qemu.txt
index b267698..f7c9498 100644
--- a/doc/note/qemu/qemu.txt
+++ b/doc/note/qemu/qemu.txt
@@ -70,6 +70,17 @@ qemu-system-x86_64 \
     -device usb-ehci,id=usb,bus=pci.0,addr=0x4 -device usb-tablet \
 
 
+## Inspect qcow2 by host mounting it
+
+  $SUDO modprobe nbd
+  $SUDO qemu-nbd -c /dev/nbd0 /path/to/my.qcow2
+  echo 'p' | $SUDO fdisk /dev/nbd0
+  $SUDO mount -o ro /dev/nbd0p2 /mnt/q
+  $SUDO umount /mnt/q  `# cleanup`
+  qemu-nbd -d /dev/nbd0  `# cleanup`
+  $SUDO rmmod nbd  `# cleanup`
+
+
 ### Example manual adapter setup (inside VM) for socket mcast network:
 true \
   && ADDR=192.168.42.101/24 \
@@ -271,4 +282,5 @@ TODO: move this to a better place. Eg: debian/setup.txt or whatever.
 - [qemu monitor via stdio](https://unix.stackexchange.com/a/57835/292722)
 - [qemu raspberry pi TODO](https://blog.agchapman.com/using-qemu-to-emulate-a-raspberry-pi/)
 - [connect VM networks](https://qemu.weilnetz.de/doc/6.0/system/invocation.html#sec-005finvocation)
+- [inspect qcow2 mount host browse](https://www.jamescoyle.net/how-to/1818-access-a-qcow2-virtual-disk-image-from-the-host)
 
diff --git a/doc/note/qemu/setup-windoof.txt b/doc/note/qemu/setup-windoof.txt
index 1bac77f..5df2cac 100644
--- a/doc/note/qemu/setup-windoof.txt
+++ b/doc/note/qemu/setup-windoof.txt
@@ -24,8 +24,11 @@ Install needed software (Maybe: firefox, MsOffice, MsTeams, ..?).
 
 Manually trigger updates, reboot, updates, reboot, (likely some more turns ...)
 
+Configure Performance options. Disable all but screen fonts.
+
 Make sure no more updates are running. Then, I guess best is to reboot without
-internet access once more to cleanup the disk:
+internet access once more to cleanup the disk. Delete unused files like
+trashcan or downloaded installers:
 
   SDelete.exe -nobanner -z C:
 
diff --git a/src/main/firefox/gaga-plugin/main.js b/src/main/firefox/gaga-plugin/main.js
index 2a5bbae..4447719 100644
--- a/src/main/firefox/gaga-plugin/main.js
+++ b/src/main/firefox/gaga-plugin/main.js
@@ -1,15 +1,10 @@
 /*
- * For how to install see:
- *
- * "https://git.hiddenalpha.ch/UnspecifiedGarbage.git/tree/doc/note/firefox/firefox.txt"
+ * [How to install](UnspecifiedGarbage/doc/note/firefox/firefox.txt)
  */
 ;(function(){ try{
 
     var NDEBUG = false;
     var STATUS_INIT = 1;
-    var STATUS_RUNNING = 2;
-    var STATUS_DONE = 3;
-    var STATUS_OBSOLETE = 4;
     var NOOP = function(){};
     var LOGERR = console.error.bind(console);
     var N = null;
@@ -19,11 +14,10 @@
     function main(){
         var app = Object.seal({
             ui: {},
-            status: Object.seal({
-                checklistBtn: STATUS_INIT,
-                developmentBtn: STATUS_INIT,
-            }),
             lastClickEpochMs: 0,
+            wantChecklistExpanded: false,
+            wantDevelopmentExpanaded: false,
+            wantBigTemplateExpanded: false,
         });
         if( NDEBUG ){
             setTimeout = window.setTimeout;
@@ -32,14 +26,16 @@
         }else{ /* fix broken tooling */
             setTimeout = setTimeoutWithCatch.bind(0, app);
             logErrors = logErrorsImpl.bind(N, app);
-            LOGDBG = console.debug.bind(console);
+            LOGDBG = console.debug.bind(console, "[gaga-plugin]");
         }
         document.addEventListener("DOMContentLoaded", logErrors.bind(N, onDOMContentLoaded, app));
+        scheduleNextStateCheck(app);
+        LOGDBG("gaga-plugin initialized");
     }
 
 
     function onDOMContentLoaded( app ){
-        cleanupClutter(app);
+        LOGDBG("onDOMContentLoaded()");
         attachDomObserver(app);
     }
 
@@ -50,83 +46,58 @@
     }
 
 
-    function onDomHasChangedSomehow( app, changes, mutationObserver ){
-        var nowEpochMs = Date.now();
-        if( (app.lastClickEpochMs + 2000) > nowEpochMs ){
-            LOGDBG("ignore, likely triggered by user.");
-            return; }
-        var needsReEval = false;
-        for( var change of changes ){
-            if( change.target.nodeName != "BUTTON" ) continue;
-            var isAriaExpanded = (change.attributeName == "aria-expanded");
-            var isChildAdded = (change.addedNodes.length > 0);
-            var isChildRemoved = (change.removedNodes.length > 0);
-            var isChildAddedOrRemoved = isChildAdded || isChildRemoved;
-            if( !isAriaExpanded && !isChildAddedOrRemoved ) continue;
-            if( isAriaExpanded ){
-                LOGDBG("Suspicious, isExpanded: ", change.target);
-                needsReEval = true; break;
-            }
-            if( !isChildAddedOrRemoved ) continue;
-            var isBloatyChecklistBtnStillThere = document.body.contains(getBloatyChecklistBtn(app));
-            if( !isBloatyChecklistBtnStillThere ){
-                LOGDBG("Suspicious, btn lost");
-                needsReEval = true; break;
-            }
-            var isBloatyDevelopmentBtnStillThere = document.body.contains(getBloatyDevelopmentBtn(app));
-            if( !isBloatyDevelopmentBtnStillThere ){
-                LOGDBG("Suspicious, btn lost");
-                needsReEval = true; break;
-            }
-        }
-        if( needsReEval ){
-            LOGDBG("Change detected! Eval again");
-            app.ui.bloatyChecklistBtn = null;
-            app.ui.bloatyDevelopmentBtn = null;
-            setTimeout(cleanupClutter, 42, app);
+    function scheduleNextStateCheck( app ){
+        //LOGDBG("scheduleNextStateCheck()");
+        if( app.stateCheckTimer ){
+            LOGDBG("Why is stateCheckTimer not zero?", app.stateCheckTimer);
         }
+        app.stateCheckTimer = setTimeout(function(){
+            app.stateCheckTimer = null;
+            scheduleNextStateCheck(app);
+            performStateCheck(app);
+        }, 42);
     }
 
 
-    function cleanupClutter( app ){
-        if( app.bloatyChecklistDone != STATUS_RUNNING ){
-            app.bloatyChecklistDone = STATUS_OBSOLETE
-            setTimeout(hideBloatyButton, 0, app, "checklistBtn");
-        }
-        if( app.bloatyDevelopmentDone != STATUS_RUNNING ){
-            app.bloatyDevelopmentDone = STATUS_OBSOLETE;
-            setTimeout(hideBloatyButton, 0, app, "developmentBtn");
-        }
-        if( app.bloatyDevelopmentDone != STATUS_RUNNING ){
-            app.bloatyDevelopmentDone = STATUS_OBSOLETE;
-            setTimeout(hideBloatyButton, 0, app, "bigTemplateBtn");
+    function performStateCheck( app ){
+        var buttons = [ "checklistBtn", "developmentBtn", "bigTemplateBtn" ];
+        var wantKey = [ "wantChecklistExpanded", "wantDevelopmentExpanaded", "wantBigTemplateExpanded" ];
+        for( var i = 0 ; i < buttons.length ; ++i ){
+            var btnKey = buttons[i];
+            var btnElem = getBloatyButton(app, btnKey);
+            if( !btnElem ) continue;
+            var isExpanded = isAriaBtnExpanded(app, btnElem)
+            var wantExpanded = app[wantKey[i]];
+            //LOGDBG(btnKey +" expanded is", isExpanded);
+            if( isExpanded && !wantExpanded ){
+                collapseAriaBtn(app, btnElem);
+            }
         }
     }
 
 
-    function setLastClickTimeToNow( app ){ app.lastClickEpochMs = Date.now(); }
+    function onDomHasChangedSomehow( app, changes, mutationObserver ){
+        var nowEpochMs = Date.now();
+        LOGDBG("DOM Change detected!");
+        /*refresh dom refs so check will work on correct elems*/
+        Object.keys(app.ui).forEach(function( key ){
+            app.ui[key] = null;
+        });
+    }
 
 
-    function hideBloatyButton( app, btnKey ){
-        if( app.status[btnKey] == STATUS_DONE ){
-            LOGDBG(btnKey +" now hidden");
-            return; }
-        app.status[btnKey] == STATUS_RUNNING;
-        var btn = getBloatyButton(app, btnKey);
-        do{
-            if( !btn ){ LOGDBG(btnKey +" not found. DOM maybe not yet ready?"); break; }
-            var isExpanded = isAriaBtnExpanded(app, btn);
-            if( isExpanded === true ){
-                LOGDBG(btnKey +".click()");
-                btn.click();
-            }else if( isExpanded === false ){
-                app.status[btnKey] = STATUS_DONE;
-            }else{
-                throw Error("Neither true nor false "+ typeof(isExpanded) +" "+ isExpanded);
-            }
-        }while(0);
-        /* try later */
-        setTimeout(hideBloatyButton, 16, app, btnKey);
+    function onBloatyChecklistBtnMousedown( app ){
+        app.wantChecklistExpanded = !app.wantChecklistExpanded;
+    }
+
+
+    function onBloatyDevelopmentBtnMousedown( app ){
+        app.wantDevelopmentExpanaded = !app.wantDevelopmentExpanaded;
+    }
+
+
+    function onBloatyBigTemplateBtnMousedown( app ){
+        app.wantBigTemplateExpanded = !app.wantBigTemplateExpanded;
     }
 
 
@@ -135,19 +106,22 @@
         }else if( btnKey == "checklistBtn" ){
             var selector = "button[aria-label=Checklists]";
             var uiKey = "bloatyChecklistBtn";
+            var onMousedown = onBloatyChecklistBtnMousedown;
         }else if( btnKey == "developmentBtn" ){
             var selector = "button[aria-label=Development]";
             var uiKey = "bloatyDevelopmentBtn";
+            var onMousedown = onBloatyDevelopmentBtnMousedown;
         }else if(  btnKey == "bigTemplateBtn" ){
             var selector = "button[aria-label=BigTemplate]";
             var uiKey = "bloatyBigTemplateBtn";
+            var onMousedown = onBloatyBigTemplateBtnMousedown;
         }else{
             throw Error(btnKey);
         }
         if( !app.ui[uiKey] ){
             var btn = fetchUiRefOrNull(app, document, selector);
             if( btn ){
-                btn.addEventListener("mousedown", logErrors.bind(N, setLastClickTimeToNow, app));
+                btn.addEventListener("mousedown", logErrors.bind(N, onMousedown, app));
                 app.ui[uiKey] = btn;
             }
         }
@@ -155,6 +129,21 @@
     }
 
 
+    function collapseAriaBtn( app, btnElem ){
+        do{
+            var isExpanded = isAriaBtnExpanded(app, btnElem);
+            if( isExpanded === true ){
+                LOGDBG("click()");
+                btnElem.click();
+            }else if( isExpanded === false ){
+                break;
+            }else{
+                throw Error("Neither true nor false "+ typeof(isExpanded) +" "+ isExpanded);
+            }
+        }while(0);
+    }
+
+
     function isAriaBtnExpanded( app, btnElem ){
         var value = btnElem.getAttribute("aria-expanded");
         if( value === "true" ){
diff --git a/src/main/lua/paisa-fleet/FindFullDisks.lua b/src/main/lua/paisa-fleet/FindFullDisks.lua
new file mode 100644
index 0000000..9963838
--- /dev/null
+++ b/src/main/lua/paisa-fleet/FindFullDisks.lua
@@ -0,0 +1,322 @@
+
+local SL = require("scriptlee")
+local newHttpClient = SL.newHttpClient
+local newShellcmd = SL.newShellcmd
+local newSqlite = SL.newSqlite
+local objectSeal = SL.objectSeal
+local parseJSON = SL.parseJSON
+local startOrExecute = SL.reactor.startOrExecute
+SL = nil
+
+local log = io.stdout
+
+
+function printHelp()
+    io.write("\n"
+        .."  WARN: This is experimental.\n"
+        .."  \n"
+        .."  Options:\n"
+        .."      --backendHost <inaddr>  (eg \"localhost\")\n"
+        .."      --backendPort <int>     (eg 80)\n"
+        .."      --sshPort <int>         (eg 22)\n"
+        .."      --sshUser <str>         (eg \"eddieuser\")\n"
+        .."      --state <path>          (eg \"path/to/state\")\n"
+        .."  \n")
+end
+
+
+function parseArgs( app )
+    app.backendPort = 80
+    app.sshPort = 22
+    app.sshUser = os.getenv("USERNAME") or false
+    app.statePath = ":memory:"
+    local iA = 0
+    ::nextArg::
+    iA = iA + 1
+    local arg = _ENV.arg[iA]
+    if not arg then
+        goto verifyResult
+    elseif arg == "--help" then
+        app.isHelp = true return 0
+    elseif arg == "--backendHost" then
+        iA = iA + 1; arg = _ENV.arg[iA]
+        if not arg then log:write("EINVAL: --backendHost needs value\n")return end
+        app.backendHost = arg
+    elseif arg == "--backendPort" then
+        iA = iA + 1; arg = _ENV.arg[iA]
+        if not arg then log:write("EINVAL: --backendPort needs value\n")return end
+        app.backendHost = arg
+    elseif arg == "--sshPort" then
+        iA = iA + 1; arg = _ENV.arg[iA]
+        if not arg then log:write("EINVAL: --sshPort needs value\n")return end
+        app.sshPort = arg
+    elseif arg == "--sshUser" then
+        iA = iA + 1; arg = _ENV.arg[iA]
+        if not arg then log:write("EINVAL: --sshUser needs value\n")return end
+        app.sshUser = arg
+    elseif arg == "--state" then
+        iA = iA + 1; arg = _ENV.arg[iA]
+        if not arg then log:write("EINVAL: --state needs value\n")return end
+        app.statePath = arg
+    else
+        log:write("EINVAL: ".. arg .."\n")return
+    end
+    goto nextArg
+    ::verifyResult::
+    if not app.backendHost then log:write("EINVAL: --backendHost missing\n")return end
+    if not app.sshUser then log:write("EINVAL: --sshUser missing")return end
+    return 0
+end
+
+
+function getStateDb(app)
+    if not app.stateDb then
+        local db = newSqlite{ database = assert(app.statePath) }
+        -- TODO normalize scheme
+        db:prepare("CREATE TABLE IF NOT EXISTS DeviceDfLog(\n"
+            .."  id INTEGER PRIMARY KEY,\n"
+            .."  \"when\" TEXT NOT NULL,\n" -- "https://xkcd.com/1179"
+            .."  hostname TEXT NOT NULL,\n"
+            .."  eddieName TEXT NOT NULL,\n"
+            .."  rootPartitionUsedPercent INT,\n"
+            .."  varLibDockerUsedPercent INT,\n"
+            .."  varLogUsedPercent INT,\n"
+            .."  dataUsedPercent INT,\n"
+            .."  stderr TEXT NOT NULL,\n"
+            .."  stdout TEXT NOT NULL)\n"
+            ..";"):execute()
+        app.stateDb = db
+    end
+    return app.stateDb
+end
+
+
+function storeDiskFullResult( app, hostname, eddieName, stderrBuf, stdoutBuf )
+    assert(app and hostname and eddieName and stderrBuf and stdoutBuf);
+    local rootPartitionUsedPercent = stdoutBuf:match("\n/[^ ]+ +%d+ +%d+ +%d+ +(%d+)%% /\n")
+    local varLibDockerUsedPercent = stdoutBuf:match("\n[^ ]+ +%d+ +%d+ +%d+ +(%d+)%% /var/lib/docker\n")
+    local dataUsedPercent = stdoutBuf:match("\n[^ ]+ +%d+ +%d+ +%d+ +(%d+)%% /data\n")
+    local varLogUsedPercent = stdoutBuf:match("\n[^ ]+ +%d+ +%d+ +%d+ +(%d+)%% /var/log\n")
+    local stmt = getStateDb(app):prepare("INSERT INTO DeviceDfLog("
+        .."  \"when\", hostname, eddieName, stderr, stdout,"
+        .."  rootPartitionUsedPercent, dataUsedPercent, varLibDockerUsedPercent, varLogUsedPercent, dataUsedPercent"
+        ..")VALUES("
+        .."  $when, $hostname, $eddieName, $stderr, $stdout,"
+        .." $rootPartitionUsedPercent, $dataUsedPercent, $varLibDockerUsedPercent, $varLogUsedPercent, $dataUsedPercent);")
+    stmt:bind("$when", os.date("!%Y-%m-%dT%H:%M:%SZ"))
+    stmt:bind("$hostname", hostname)
+    stmt:bind("$eddieName", eddieName)
+    stmt:bind("$stderr", stderrBuf)
+    stmt:bind("$stdout", stdoutBuf)
+    stmt:bind("$rootPartitionUsedPercent", rootPartitionUsedPercent)
+    stmt:bind("$varLibDockerUsedPercent", varLibDockerUsedPercent)
+    stmt:bind("$varLogUsedPercent", varLogUsedPercent)
+    stmt:bind("$dataUsedPercent", dataUsedPercent)
+    stmt:execute()
+end
+
+
+function doWhateverWithDevices( app )
+    for k, dev in pairs(app.devices) do
+        log:write("[INFO ] Inspecting '".. dev.hostname .."' (@ ".. dev.eddieName ..") ...\n")
+        local fookCmd = "true"
+            .." && HOSTNAME=$(hostname|sed 's_.isa.localdomain__')"
+            .." && STAGE=$PAISA_ENV"
+            .." && printf \"remoteHostname=$HOSTNAME, remoteStage=$STAGE\\n\""
+            -- on some machine, df failed with "Stale file handle" But I want to continue
+            -- with next device regardless of such errors.
+            .." && df || true"
+        local eddieCmd = "true"
+            .." && HOSTNAME=$(hostname|sed 's_.pnet.ch__')"
+            .." && STAGE=$PAISA_ENV"
+            .." && printf \"remoteEddieName=$HOSTNAME, remoteStage=$STAGE\\n\""
+            .." && if test \"${HOSTNAME}\" != \"".. dev.eddieName .."\"; then true"
+            .."     && echo wrong host. Want ".. dev.eddieName .." found $HOSTNAME && false"
+            .."     ;fi"
+            .." && ssh -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null"
+            .." -p".. app.sshPort .." ".. app.sshUser .."@".. ((dev.type == "FOOK")and"fook"or dev.hostname)
+            .." \\\n    --"
+            .." sh -c 'true && ".. fookCmd:gsub("'", "'\"'\"'") .."'"
+        local localCmd = assert(os.getenv("SSH_EXE"), "environ.SSH_EXE missing")
+            .." -oRemoteCommand=none -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null"
+            .." -p".. app.sshPort .." ".. app.sshUser .."@".. dev.eddieName ..""
+            .." \\\n    --"
+            .." sh -c 'true && ".. eddieCmd:gsub("'", "'\"'\"'") .."'"
+        -- TODO get rid of this ugly use-tmp-file-as-script workaround
+        local tmpPath = assert(os.getenv("TMP"), "environ.TMP missing"):gsub("\\", "/") .."/b30589uj30oahujotehuj.sh"
+        --log:write("[DEBUG] tmpPath '".. tmpPath .."'\n")
+        local tmpFile = assert(io.open(tmpPath, "wb"), "Failed to open '".. tmpPath .."'")
+        tmpFile:write("#!/bin/sh\n".. localCmd .."\n")
+        tmpFile:close()
+        --log:write("[DEBUG] tmpPath ".. tmpPath .."\n")
+        -- EndOf kludge
+        local cmd = objectSeal{
+            base = false,
+            stdoutBuf = {},
+            stderrBuf = {},
+        }
+        cmd.base = newShellcmd{
+            cls = cmd,
+            cmdLine = "sh \"".. tmpPath .."\"",
+            onStdout = function( buf, cmd ) table.insert(cmd.stdoutBuf, buf or"") end,
+            onStderr = function( buf, cmd ) table.insert(cmd.stderrBuf, buf or"") end,
+        }
+        cmd.base:start()
+        cmd.base:closeSnk()
+        local exit, signal = cmd.base:join(17)
+        cmd.stderrBuf = table.concat(cmd.stderrBuf)
+        cmd.stdoutBuf = table.concat(cmd.stdoutBuf)
+        if exit == 255 and signal ==  nil then
+            log:write("[DEBUG] fd2: ".. cmd.stderrBuf:gsub("\n", "\n[DEBUG] fd2: "):gsub("\n%[DEBUG%] fd2: $", "") .."\n")
+            goto nextDevice
+        end
+        log:write("[DEBUG] fd1: ".. cmd.stdoutBuf:gsub("\n", "\n[DEBUG] fd1: "):gsub("\n%[DEBUG%] fd1: $", "") .."\n")
+        storeDiskFullResult(app, dev.hostname, dev.eddieName, cmd.stderrBuf, cmd.stdoutBuf)
+        if exit ~= 0 or signal ~= nil then
+            error("exit=".. tostring(exit)..", signal="..tostring(signal))
+        end
+        ::nextDevice::
+    end
+end
+
+
+function sortDevicesMostRecentlySeenFirst( app )
+    table.sort(app.devices, function(a, b) return a.lastSeen > b.lastSeen end)
+end
+
+
+-- Don't want to visit just seen devices over and over again. So drop devices
+-- we've recently seen from our devices-to-visit list.
+function dropDevicesRecentlySeen( app )
+    -- Collect recently seen devices.
+    local devicesToRemove = {}
+    local st = getStateDb(app):prepare("SELECT hostname FROM DeviceDfLog WHERE \"when\" > $tresholdDate")
+    st:bind("$tresholdDate", os.date("!%Y-%m-%dT%H:%M:%SZ", os.time()-42*3600))
+    local rs = st:execute()
+    while rs:next() do
+        local hostname = rs:value(1)
+        devicesToRemove[hostname] = true
+    end
+    -- Remove selected devices
+    local numKeep, numDrop = 0, 0
+    local iD = 0 while true do iD = iD + 1
+        local device = app.devices[iD]
+        if not device then break end
+        if devicesToRemove[device.hostname] then
+            --log:write("[DEBUG] Drop '".. device.hostname .."' (".. device.eddieName ..")\n")
+            numDrop = numDrop + 1
+            app.devices[iD] = app.devices[#app.devices]
+            app.devices[#app.devices] = nil
+            iD = iD - 1
+        else
+            --log:write("[DEBUG] Keep '".. device.hostname .."' (".. device.eddieName ..")\n")
+            numKeep = numKeep + 1
+        end
+    end
+    log:write("[INFO ] Of "..(numKeep+numDrop).." devices from state visit ".. numKeep
+        .." and skip ".. numDrop .." (bcause seen recently)\n")
+end
+
+
+function fetchDevices( app )
+    local req = objectSeal{
+        base = false,
+        method = "GET",
+        uri = "/houston/vehicle/inventory/v1/info/devices",
+        rspCode = false,
+        rspBody = false,
+        isDone = false,
+    }
+    req.base = app.http:request{
+        cls = req, connectTimeoutMs = 3000,
+        host = app.backendHost, port = app.backendPort,
+        method = req.method, url = req.uri,
+        onRspHdr = function( rspHdr, req )
+            req.rspCode = rspHdr.status
+            if rspHdr.status ~= 200 then
+                log:write(".-----------------------------------------\n")
+                log:write("| ".. req.method .." ".. req.uri .."\n")
+                log:write("| Host: ".. app.backendHost ..":".. app.backendPort .."\n")
+                log:write("+-----------------------------------------\n")
+                log:write("|  ".. rspHdr.proto .." ".. rspHdr.status .." ".. rspHdr.phrase .."\n")
+                for i,h in ipairs(rspHdr.headers) do log:write("|  ".. h[1] ..": ".. h[2] .."\n") end
+                log:write("|  \n")
+            end
+        end,
+        onRspChunk = function( buf, req )
+            if req.rspCode ~= 200 then log:write("|  ".. buf:gsub("\n", "\n|  ")) return end
+            if buf then
+                if not req.rspBody then req.rspBody = buf
+                else req.rspBody = req.rspBody .. buf end
+            end
+        end,
+        onRspEnd = function( req )
+            if req.rspCode ~= 200 then log:write("\n'-----------------------------------------\n") end
+            req.isDone = true
+        end,
+    }
+    req.base:closeSnk()
+    assert(req.isDone)
+    if req.rspCode ~= 200 then log:write("ERROR: Couldn't fetch devices\n")return end
+    assert(not app.devices)
+    app.devices = {}
+    log:write("[DEBUG] rspBody.len is ".. req.rspBody:len() .."\n")
+    --io.write(req.rspBody)io.write("\n")
+    for iD, device in pairs(parseJSON(req.rspBody).devices) do
+        --print("Wa", iD, device)
+        --for k,v in pairs(device)do print("W",k,v)end
+        -- TODO how to access 'device.type'?
+        local hostname               , eddieName               , lastSeen
+            = device.hostname:value(), device.eddieName:value(), device.lastSeen:value()
+        local typ
+        if false then
+        elseif hostname:find("^eddie%d%d%d%d%d$") then
+            typ = "EDDIE"
+        elseif hostname:find("^fook%-[a-z0-9]+$") then
+            typ = "FOOK"
+        elseif hostname:find("^lunkwill%-[a-z0-9]+$") then
+            typ = "LUNKWILL"
+        elseif hostname:find("^fook$") then
+            log:write("[WARN ] WTF?!? '"..hostname.."'\n")
+            typ = false
+        else error("TODO_359zh8i3wjho "..hostname) end
+        table.insert(app.devices, objectSeal{
+            hostname = hostname,
+            eddieName = eddieName,
+            type = typ,
+            lastSeen = lastSeen,
+        })
+    end
+    log:write("[INFO ] Fetched ".. #app.devices .." devices.\n")
+end
+
+
+function run( app )
+    fetchDevices(app)
+    dropDevicesRecentlySeen(app)
+    --sortDevicesMostRecentlySeenFirst(app)
+    doWhateverWithDevices(app)
+end
+
+
+function main()
+    local app = objectSeal{
+        isHelp = false,
+        backendHost = false,
+        backendPort = false,
+        sshPort = false,
+        sshUser = false,
+        statePath = false,
+        stateDb = false,
+        http = newHttpClient{},
+        devices = false,
+    }
+    if parseArgs(app) ~= 0 then os.exit(1) end
+    if app.isHelp then printHelp() return end
+    run(app)
+end
+
+
+startOrExecute(main)
+
+
diff --git a/src/main/lua/paisa-fleet/RmArtifactBaseDir.lua b/src/main/lua/paisa-fleet/RmArtifactBaseDir.lua
index a30ff6a..949d1fe 100644
--- a/src/main/lua/paisa-fleet/RmArtifactBaseDir.lua
+++ b/src/main/lua/paisa-fleet/RmArtifactBaseDir.lua
@@ -22,6 +22,8 @@ function printHelp()
         .."      --sshPort <int>         (eg 22)\n"
         .."      --sshUser <str>         (eg \"eddieuser\")\n"
         .."      --state <path>          (eg \"path/to/state\")\n"
+        .."      \n"
+        .."      --exportLatestStatus\n"
         .."  \n")
 end
 
@@ -61,19 +63,29 @@ function parseArgs( app )
         iA = iA + 1; arg = _ENV.arg[iA]
         if not arg then log:write("EINVAL: --state needs value\n")return end
         app.statePath = arg
+    elseif arg == "--exportLatestStatus" then
+        app.exportLatestStatus = true
+    else
+        log:write("EINVAL: ".. arg .."\n")return
     end
     goto nextArg
     ::verifyResult::
-    if not app.backendHost then log:write("EINVAL: --backendHost missing\n")return end
-    if not app.backendPath then log:write("EINVAL: --backendPath missing\n")return end
-    if app.backendPath:find("^C:.") then log:write("WARN: Path looks wrong: ".. app.backendPath.."\n") end
+    if app.exportLatestStatus then
+        if not app.statePath then log:write("EINVAL: --state missing\n")return end
+    else
+        if not app.backendHost then log:write("EINVAL: --backendHost missing\n")return end
+        if not app.backendPath then log:write("EINVAL: --backendPath missing\n")return end
+        if app.backendPath:find("^C:.") then log:write("[WARN ] MSYS_NO_PATHCONV=1 likely missing? ".. app.backendPath.."\n") end
+    end
     return 0
 end
 
 
 function removeCompletedEddies( app )
     local db = getStateDb(app)
-    local rs = db:prepare("SELECT eddieName FROM CompletedEddies;"):execute()
+    local rs = db:prepare("SELECT eddieName FROM Eddie"
+        .."  JOIN EddieLog ON Eddie.id = eddieId"
+        .."  WHERE status = \"OK\";"):execute()
     local eddieNamesToRemoveSet = {}
     while rs:next() do
         assert(rs:type(1) == "TEXT", rs:type(1))
@@ -98,15 +110,28 @@ function removeCompletedEddies( app )
 end
 
 
-function markEddieDone( app, eddieName )
+function setEddieStatus( app, statusStr, eddieName, stderrStr, stdoutStr )
     assert(type(app) == "table")
     assert(type(eddieName) == "string")
-    log:write("[DEBUG] markEddieDone(".. eddieName ..")\n")
+    assert(statusStr == "OK" or statusStr == "ERROR")
+    log:write("[DEBUG] setEddieStatus(".. eddieName ..", ".. statusStr ..")\n")
     local db = getStateDb(app)
-    local stmt = db:prepare("INSERT OR IGNORE INTO CompletedEddies(eddieName,doneAt)VALUES($eddieName, $now)")
+    local stmt = db:prepare("INSERT INTO Eddie(eddieName)VALUES($eddieName);")
+    stmt:bind("$eddieName", eddieName)
+    local ok, emsg = xpcall(function()
+        stmt:execute()
+    end, debug.traceback)
+    if not ok and not emsg:find("UNIQUE constraint failed: Eddie.eddieName") then
+        error(emsg)
+    end
+    local stmt = db:prepare("INSERT INTO EddieLog('when',eddieId,status,stderr,stdout)"
+        .."VALUES($when, (SELECT rowid FROM Eddie WHERE eddieName = $eddieName), $status, $stderr, $stdout)")
     stmt:reset()
+    stmt:bind("$when", os.date("!%Y-%m-%dT%H:%M:%S+00:00"))
     stmt:bind("$eddieName", eddieName)
-    stmt:bind("$now", os.date("!%Y-%m-%dT%H:%M:%S+00:00"))
+    stmt:bind("$status", statusStr)
+    stmt:bind("$stderr", stderrStr)
+    stmt:bind("$stdout", stdoutStr)
     stmt:execute()
 end
 
@@ -114,9 +139,18 @@ end
 function getStateDb( app )
     if not app.stateDb then
         app.stateDb = newSqlite{ database = app.statePath }
-        app.stateDb:prepare("CREATE TABLE IF NOT EXISTS CompletedEddies("
-            .."  eddieName TEXT UNIQUE,"
-            .."  doneAt TEXT);"):execute()
+        app.stateDb:prepare("CREATE TABLE IF NOT EXISTS Eddie(\n"
+            .."  id INTEGER PRIMARY KEY,\n"
+            .."  eddieName TEXT UNIQUE NOT NULL)\n"
+            ..";"):execute()
+        app.stateDb:prepare("CREATE TABLE IF NOT EXISTS EddieLog(\n"
+            .."  id INTEGER PRIMARY KEY,\n"
+            .."  'when' TEXT NOT NULL,\n"
+            .."  eddieId INT NOT NULL,\n"
+            .."  status TEXT, -- OneOf OK, ERROR\n"
+            .."  stderr TEXT NOT NULL,\n"
+            .."  stdout TEXT NOT NULL)\n"
+            ..";\n"):execute()
     end
     return app.stateDb
 end
@@ -126,6 +160,8 @@ function loadEddies( app )
     local httpClient = newHttpClient{}
     local req = objectSeal{
         base = false,
+        method = "GET",
+        path = app.backendPath .."/data/preflux/inventory",
         rspCode = false,
         rspBody = false,
         isDone = false,
@@ -133,11 +169,14 @@ function loadEddies( app )
     req.base = httpClient:request{
         cls = req,
         host = app.backendHost, port = app.backendPort,
-        method = "GET", url = app.backendPath .."/data/preflux/inventory",
+        method = req.method, url = req.path,
         onRspHdr = function( rspHdr, req )
             req.rspCode = rspHdr.status
             if rspHdr.status ~= 200 then
                 log:write(".-----------------------------------------\n")
+                log:write("| ".. req.method .." ".. req.path .."\n")
+                log:write("| Host: ".. app.backendHost ..":".. app.backendPort .."\n")
+                log:write("+-----------------------------------------\n")
                 log:write("|  ".. rspHdr.proto .." ".. rspHdr.status .." ".. rspHdr.phrase .."\n")
                 for i,h in ipairs(rspHdr.headers) do
                     log:write("|  ".. h[1] ..": ".. h[2] .."\n")
@@ -159,6 +198,7 @@ function loadEddies( app )
     }
     req.base:closeSnk()
     assert(req.isDone)
+    if req.rspCode ~= 200 then log:write("ERROR: Couldn't load eddies\n")return end
     local prefluxInventory = parseJSON(req.rspBody)
     local eddies = {}
     for eddieName, detail in pairs(prefluxInventory.hosts) do
@@ -172,7 +212,8 @@ end
 
 
 function makeWhateverWithEddies( app )
-    local cmdLinePre = "ssh -oConnectTimeout=5"
+    local ssh = "C:/Users/fankhauseand/.opt/gitPortable-2.27.0-x64/usr/bin/ssh.exe"
+    local cmdLinePre = ssh .." -oConnectTimeout=3 -oRemoteCommand=none"
     if app.sshPort then cmdLinePre = cmdLinePre .." -p".. app.sshPort end
     if app.sshUser then cmdLinePre = cmdLinePre .." \"-oUser=".. app.sshUser .."\"" end
     for k,eddie in pairs(app.eddies) do
@@ -189,34 +230,59 @@ function makeWhateverWithEddies( app )
         assert(isEddie or isTeddie, eddieName or"nil")
         local okMarker = "OK_".. math.random(10000000, 99999999) .."wCAkgQQA2AJAzAIA"
         local cmdLine = cmdLinePre .." ".. eddieName
-            -- report only
-            --.." \"-oRemoteCommand=test -e /data/instances/default && ls -Ahl /data/instances/default\""
-            -- DELETE them
-            .." \"-oRemoteCommand=true"
-            ..    " && if test -e /data/instances/default/\\${ARTIFACT_BASE_DIR}; then true"
-            ..        " && find /data/instances/default/\\${ARTIFACT_BASE_DIR} -type d -mtime +420 -print -delete"
+            .." -- \"true"
+            ..    " && if test \"".. eddieName .."\" != \"$(hostname|sed 's,.pnet.ch$,,'); then true\""
+            ..        " && echo WrongHost expected=".. eddieName .." actual=$(hostname|sed 's,.pnet.ch$,,') && false"
             ..        " ;fi"
-            ..    " && echo ".. okMarker ..""
+            ..    " && echo hostname=$(hostname|sed 's,.pnet.ch,,')"
+            ..    " && echo stage=${PAISA_ENV:?}"
+            ..    " && echo Scan /data/instances/default/??ARTIFACT_BASE_DIR?"
+            --[[report only]]
+            --..    " && test -e /data/instances/default/??ARTIFACT_BASE_DIR? && ls -Ahl /data/instances/default/??ARTIFACT_BASE_DIR?"
+            --[[Find un-/affected eddies]]
+            ..    " && if test -e /data/instances/default/??ARTIFACT_BASE_DIR?; then true"
+            ..        " ;else true"
+            ..        " && echo ".. okMarker
+            ..        " ;fi"
+            --[[DELETE them]]
+            --..    " && if test -e /data/instances/default/??ARTIFACT_BASE_DIR?; then true"
+            --..        " && find /data/instances/default/??ARTIFACT_BASE_DIR? -type d -mtime +420 -print -delete"
+            --..        " ;fi"
+            --..    " && echo ".. okMarker ..""
+            --[[]]
             ..    " \""
-        log:write("\n[DEBUG] ".. cmdLine.."\n")
-        log:write("[DEBUG] sleep ...\n")sleep(3)
-        local isCmdDone, isSuccess = false, false
+        log:write("\n")
+        log:write("[INFO ] Try ".. eddieName .." ...\n")
+        log:write("[DEBUG] ".. cmdLine.."\n")
+        --log:write("[DEBUG] sleep ...\n")sleep(3)
+        local isStdioDone, isSuccess, stderrStr, stdoutStr = false, false, "", ""
         local cmd = newShellcmd{
             cmdLine = cmdLine,
             onStdout = function( buf )
                 if buf then
                     if buf:find("\n"..okMarker.."\n",0,true) then isSuccess = true end
+                    stdoutStr = stdoutStr .. buf
                     io.stdout:write(buf)
-                else isCmdDone = true end
+                else isStdioDone = true end
+            end,
+            onStderr = function( buf )
+                stderrStr = buf and stderrStr .. buf or stderrStr
+                io.stderr:write(buf or"")
             end,
         }
         cmd:start()
         cmd:closeSnk()
         local exitCode, signal = cmd:join(42)
-        log:write("[DEBUG] code="..tostring(exitCode)..", signal="..tostring(signal).."\n")
-        while not isCmdDone do sleep(0.042) end
-        if not isSuccess then log:write("[WARN ] Failed on '"..eddieName.."'\n") goto nextEddie end
-        markEddieDone(app, eddieName)
+        if exitCode ~= 0 and signal ~= nil then
+            log:write("[WARN ] code="..tostring(exitCode)..", signal="..tostring(signal).."\n")
+        end
+        while not isStdioDone do sleep(0.042) end
+        -- Analyze outcome
+        if not isSuccess then
+            setEddieStatus(app, "ERROR", eddieName, stderrStr, stdoutStr)
+            goto nextEddie
+        end
+        setEddieStatus(app, "OK", eddieName, stderrStr, stdoutStr)
         ::nextEddie::
     end
 end
@@ -227,7 +293,63 @@ function sortEddiesMostRecentlySeenFirst( app )
 end
 
 
+function quoteCsvVal( v )
+    local typ = type(v)
+    if false then
+    elseif typ == "string" then
+        if v:find("[\"\r\n]",0,false) then
+            v = '"'.. v:gsub('"', '""') ..'"'
+        end
+    else error("TODO_a928rzuga98oirh "..typ)end
+    return v
+end
+
+
+function exportLatestStatus( app )
+    local snk = io.stdout
+    local db = getStateDb(app)
+    local stmt = db:prepare("SELECT \"when\",eddieName,status,stderr,stdout FROM EddieLog"
+        .." JOIN Eddie ON Eddie.id = eddieId"
+        .." ORDER BY eddieId,[when]"
+        .." ;")
+    rs = stmt:execute()
+    snk:write("c;when;eddieName;status;stderr;stdout\n")
+    local prevWhen, prevEddieName, prevStatus, prevStderr, prevStdout
+    local qt = quoteCsvVal
+    while rs:next() do
+        local when       , eddieName  , status     , stderr     , stdout
+            = rs:value(1), rs:value(2), rs:value(3), rs:value(4), rs:value(5)
+          --log:write("[DEBUG] "..tostring(when).."  "..tostring(eddieName).."  "..tostring(status).."\n")
+        assert(when and eddieName and status and stderr and stdout)
+        if eddieName == prevEddieName then
+            if not prevWhen or when > prevWhen then
+                --log:write("[DEBUG] ".. when .."  ".. eddieName .."  take\n")
+                goto assignPrevThenNextEntry
+            else
+                --log:write("[DEBUG] ".. when .."  ".. eddieName .."  obsolete\n")
+                goto nextEntry
+            end
+        elseif prevEddieName then
+            --log:write("[DEBUG] ".. when .."  ".. eddieName .."  Eddie complete\n")
+            snk:write("r;".. qt(when) ..";".. qt(eddieName) ..";".. qt(status) ..";".. qt(stderr) ..";".. qt(stdout) .."\n")
+        else
+            --log:write("[DEBUG] ".. when .."  ".. eddieName .."  Another eddie\n")
+            goto assignPrevThenNextEntry
+        end
+        ::assignPrevThenNextEntry::
+        --[[]] prevWhen, prevEddieName, prevStatus, prevStderr, prevStdout
+            =  when    , eddieName    , status    , stderr    , stdout
+        ::nextEntry::
+    end
+    snk:write("t;status;OK\n")
+end
+
+
 function run( app )
+    if app.exportLatestStatus then
+        exportLatestStatus(app)
+        return
+    end
     loadEddies(app)
     assert(app.eddies)
     removeCompletedEddies(app)
@@ -246,6 +368,7 @@ function main()
         sshUser = false,
         statePath = false,
         stateDb = false,
+        exportLatestStatus = false,
         eddies = false,
     }
     if parseArgs(app) ~= 0 then os.exit(1) end
diff --git a/src/main/patch/houston/default.patch b/src/main/patch/houston/default.patch
index 52017d2..4169156 100644
--- a/src/main/patch/houston/default.patch
+++ b/src/main/patch/houston/default.patch
@@ -18,6 +18,32 @@ index 0ed4f7f3..b44c5693 100644
  
          <!-- JavaMelody -->
          <jetty.version>9.4.43.v20210629</jetty.version>
+@@ -301,4 +301,25 @@
+             </properties>
+         </profile>
+     </profiles>
++    <build>
++     <plugins>
++         <plugin>
++             <groupId>com.diffplug.spotless</groupId>
++             <artifactId>spotless-maven-plugin</artifactId>
++             <executions>
++                 <execution>
++                     <id>spotless-apply</id>
++                     <phase>none</phase>
++                 </execution>
++                 <execution>
++                     <id>spotless-check</id>
++                     <phase>none</phase>
++                 </execution>
++             </executions>
++             <configuration>
++                 <skip>true</skip>
++             </configuration>
++         </plugin>
++     </plugins>
++    </build>
+ </project>
 diff --git a/houston-process/pom.xml b/houston-process/pom.xml
 index 374dcb97..3c24937c 100644
 --- a/houston-process/pom.xml