Merge remote-tracking branch 'ha-public/master' into wip-LatitudeReSetup-20230620wip-LatitudeReSetup-20230620

1 files changed, 322 insertions, 0 deletions

diff --git a/src/main/lua/paisa-fleet/FindFullDisks.lua b/src/main/lua/paisa-fleet/FindFullDisks.lua
new file mode 100644
index 0000000..9963838
--- /dev/null
+++ b/src/main/lua/paisa-fleet/FindFullDisks.lua
@@ -0,0 +1,322 @@
+
+local SL = require("scriptlee")
+local newHttpClient = SL.newHttpClient
+local newShellcmd = SL.newShellcmd
+local newSqlite = SL.newSqlite
+local objectSeal = SL.objectSeal
+local parseJSON = SL.parseJSON
+local startOrExecute = SL.reactor.startOrExecute
+SL = nil
+
+local log = io.stdout
+
+
+function printHelp()
+    io.write("\n"
+        .."  WARN: This is experimental.\n"
+        .."  \n"
+        .."  Options:\n"
+        .."      --backendHost <inaddr>  (eg \"localhost\")\n"
+        .."      --backendPort <int>     (eg 80)\n"
+        .."      --sshPort <int>         (eg 22)\n"
+        .."      --sshUser <str>         (eg \"eddieuser\")\n"
+        .."      --state <path>          (eg \"path/to/state\")\n"
+        .."  \n")
+end
+
+
+function parseArgs( app )
+    app.backendPort = 80
+    app.sshPort = 22
+    app.sshUser = os.getenv("USERNAME") or false
+    app.statePath = ":memory:"
+    local iA = 0
+    ::nextArg::
+    iA = iA + 1
+    local arg = _ENV.arg[iA]
+    if not arg then
+        goto verifyResult
+    elseif arg == "--help" then
+        app.isHelp = true return 0
+    elseif arg == "--backendHost" then
+        iA = iA + 1; arg = _ENV.arg[iA]
+        if not arg then log:write("EINVAL: --backendHost needs value\n")return end
+        app.backendHost = arg
+    elseif arg == "--backendPort" then
+        iA = iA + 1; arg = _ENV.arg[iA]
+        if not arg then log:write("EINVAL: --backendPort needs value\n")return end
+        app.backendHost = arg
+    elseif arg == "--sshPort" then
+        iA = iA + 1; arg = _ENV.arg[iA]
+        if not arg then log:write("EINVAL: --sshPort needs value\n")return end
+        app.sshPort = arg
+    elseif arg == "--sshUser" then
+        iA = iA + 1; arg = _ENV.arg[iA]
+        if not arg then log:write("EINVAL: --sshUser needs value\n")return end
+        app.sshUser = arg
+    elseif arg == "--state" then
+        iA = iA + 1; arg = _ENV.arg[iA]
+        if not arg then log:write("EINVAL: --state needs value\n")return end
+        app.statePath = arg
+    else
+        log:write("EINVAL: ".. arg .."\n")return
+    end
+    goto nextArg
+    ::verifyResult::
+    if not app.backendHost then log:write("EINVAL: --backendHost missing\n")return end
+    if not app.sshUser then log:write("EINVAL: --sshUser missing")return end
+    return 0
+end
+
+
+function getStateDb(app)
+    if not app.stateDb then
+        local db = newSqlite{ database = assert(app.statePath) }
+        -- TODO normalize scheme
+        db:prepare("CREATE TABLE IF NOT EXISTS DeviceDfLog(\n"
+            .."  id INTEGER PRIMARY KEY,\n"
+            .."  \"when\" TEXT NOT NULL,\n" -- "https://xkcd.com/1179"
+            .."  hostname TEXT NOT NULL,\n"
+            .."  eddieName TEXT NOT NULL,\n"
+            .."  rootPartitionUsedPercent INT,\n"
+            .."  varLibDockerUsedPercent INT,\n"
+            .."  varLogUsedPercent INT,\n"
+            .."  dataUsedPercent INT,\n"
+            .."  stderr TEXT NOT NULL,\n"
+            .."  stdout TEXT NOT NULL)\n"
+            ..";"):execute()
+        app.stateDb = db
+    end
+    return app.stateDb
+end
+
+
+function storeDiskFullResult( app, hostname, eddieName, stderrBuf, stdoutBuf )
+    assert(app and hostname and eddieName and stderrBuf and stdoutBuf);
+    local rootPartitionUsedPercent = stdoutBuf:match("\n/[^ ]+ +%d+ +%d+ +%d+ +(%d+)%% /\n")
+    local varLibDockerUsedPercent = stdoutBuf:match("\n[^ ]+ +%d+ +%d+ +%d+ +(%d+)%% /var/lib/docker\n")
+    local dataUsedPercent = stdoutBuf:match("\n[^ ]+ +%d+ +%d+ +%d+ +(%d+)%% /data\n")
+    local varLogUsedPercent = stdoutBuf:match("\n[^ ]+ +%d+ +%d+ +%d+ +(%d+)%% /var/log\n")
+    local stmt = getStateDb(app):prepare("INSERT INTO DeviceDfLog("
+        .."  \"when\", hostname, eddieName, stderr, stdout,"
+        .."  rootPartitionUsedPercent, dataUsedPercent, varLibDockerUsedPercent, varLogUsedPercent, dataUsedPercent"
+        ..")VALUES("
+        .."  $when, $hostname, $eddieName, $stderr, $stdout,"
+        .." $rootPartitionUsedPercent, $dataUsedPercent, $varLibDockerUsedPercent, $varLogUsedPercent, $dataUsedPercent);")
+    stmt:bind("$when", os.date("!%Y-%m-%dT%H:%M:%SZ"))
+    stmt:bind("$hostname", hostname)
+    stmt:bind("$eddieName", eddieName)
+    stmt:bind("$stderr", stderrBuf)
+    stmt:bind("$stdout", stdoutBuf)
+    stmt:bind("$rootPartitionUsedPercent", rootPartitionUsedPercent)
+    stmt:bind("$varLibDockerUsedPercent", varLibDockerUsedPercent)
+    stmt:bind("$varLogUsedPercent", varLogUsedPercent)
+    stmt:bind("$dataUsedPercent", dataUsedPercent)
+    stmt:execute()
+end
+
+
+function doWhateverWithDevices( app )
+    for k, dev in pairs(app.devices) do
+        log:write("[INFO ] Inspecting '".. dev.hostname .."' (@ ".. dev.eddieName ..") ...\n")
+        local fookCmd = "true"
+            .." && HOSTNAME=$(hostname|sed 's_.isa.localdomain__')"
+            .." && STAGE=$PAISA_ENV"
+            .." && printf \"remoteHostname=$HOSTNAME, remoteStage=$STAGE\\n\""
+            -- on some machine, df failed with "Stale file handle" But I want to continue
+            -- with next device regardless of such errors.
+            .." && df || true"
+        local eddieCmd = "true"
+            .." && HOSTNAME=$(hostname|sed 's_.pnet.ch__')"
+            .." && STAGE=$PAISA_ENV"
+            .." && printf \"remoteEddieName=$HOSTNAME, remoteStage=$STAGE\\n\""
+            .." && if test \"${HOSTNAME}\" != \"".. dev.eddieName .."\"; then true"
+            .."     && echo wrong host. Want ".. dev.eddieName .." found $HOSTNAME && false"
+            .."     ;fi"
+            .." && ssh -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null"
+            .." -p".. app.sshPort .." ".. app.sshUser .."@".. ((dev.type == "FOOK")and"fook"or dev.hostname)
+            .." \\\n    --"
+            .." sh -c 'true && ".. fookCmd:gsub("'", "'\"'\"'") .."'"
+        local localCmd = assert(os.getenv("SSH_EXE"), "environ.SSH_EXE missing")
+            .." -oRemoteCommand=none -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null"
+            .." -p".. app.sshPort .." ".. app.sshUser .."@".. dev.eddieName ..""
+            .." \\\n    --"
+            .." sh -c 'true && ".. eddieCmd:gsub("'", "'\"'\"'") .."'"
+        -- TODO get rid of this ugly use-tmp-file-as-script workaround
+        local tmpPath = assert(os.getenv("TMP"), "environ.TMP missing"):gsub("\\", "/") .."/b30589uj30oahujotehuj.sh"
+        --log:write("[DEBUG] tmpPath '".. tmpPath .."'\n")
+        local tmpFile = assert(io.open(tmpPath, "wb"), "Failed to open '".. tmpPath .."'")
+        tmpFile:write("#!/bin/sh\n".. localCmd .."\n")
+        tmpFile:close()
+        --log:write("[DEBUG] tmpPath ".. tmpPath .."\n")
+        -- EndOf kludge
+        local cmd = objectSeal{
+            base = false,
+            stdoutBuf = {},
+            stderrBuf = {},
+        }
+        cmd.base = newShellcmd{
+            cls = cmd,
+            cmdLine = "sh \"".. tmpPath .."\"",
+            onStdout = function( buf, cmd ) table.insert(cmd.stdoutBuf, buf or"") end,
+            onStderr = function( buf, cmd ) table.insert(cmd.stderrBuf, buf or"") end,
+        }
+        cmd.base:start()
+        cmd.base:closeSnk()
+        local exit, signal = cmd.base:join(17)
+        cmd.stderrBuf = table.concat(cmd.stderrBuf)
+        cmd.stdoutBuf = table.concat(cmd.stdoutBuf)
+        if exit == 255 and signal ==  nil then
+            log:write("[DEBUG] fd2: ".. cmd.stderrBuf:gsub("\n", "\n[DEBUG] fd2: "):gsub("\n%[DEBUG%] fd2: $", "") .."\n")
+            goto nextDevice
+        end
+        log:write("[DEBUG] fd1: ".. cmd.stdoutBuf:gsub("\n", "\n[DEBUG] fd1: "):gsub("\n%[DEBUG%] fd1: $", "") .."\n")
+        storeDiskFullResult(app, dev.hostname, dev.eddieName, cmd.stderrBuf, cmd.stdoutBuf)
+        if exit ~= 0 or signal ~= nil then
+            error("exit=".. tostring(exit)..", signal="..tostring(signal))
+        end
+        ::nextDevice::
+    end
+end
+
+
+function sortDevicesMostRecentlySeenFirst( app )
+    table.sort(app.devices, function(a, b) return a.lastSeen > b.lastSeen end)
+end
+
+
+-- Don't want to visit just seen devices over and over again. So drop devices
+-- we've recently seen from our devices-to-visit list.
+function dropDevicesRecentlySeen( app )
+    -- Collect recently seen devices.
+    local devicesToRemove = {}
+    local st = getStateDb(app):prepare("SELECT hostname FROM DeviceDfLog WHERE \"when\" > $tresholdDate")
+    st:bind("$tresholdDate", os.date("!%Y-%m-%dT%H:%M:%SZ", os.time()-42*3600))
+    local rs = st:execute()
+    while rs:next() do
+        local hostname = rs:value(1)
+        devicesToRemove[hostname] = true
+    end
+    -- Remove selected devices
+    local numKeep, numDrop = 0, 0
+    local iD = 0 while true do iD = iD + 1
+        local device = app.devices[iD]
+        if not device then break end
+        if devicesToRemove[device.hostname] then
+            --log:write("[DEBUG] Drop '".. device.hostname .."' (".. device.eddieName ..")\n")
+            numDrop = numDrop + 1
+            app.devices[iD] = app.devices[#app.devices]
+            app.devices[#app.devices] = nil
+            iD = iD - 1
+        else
+            --log:write("[DEBUG] Keep '".. device.hostname .."' (".. device.eddieName ..")\n")
+            numKeep = numKeep + 1
+        end
+    end
+    log:write("[INFO ] Of "..(numKeep+numDrop).." devices from state visit ".. numKeep
+        .." and skip ".. numDrop .." (bcause seen recently)\n")
+end
+
+
+function fetchDevices( app )
+    local req = objectSeal{
+        base = false,
+        method = "GET",
+        uri = "/houston/vehicle/inventory/v1/info/devices",
+        rspCode = false,
+        rspBody = false,
+        isDone = false,
+    }
+    req.base = app.http:request{
+        cls = req, connectTimeoutMs = 3000,
+        host = app.backendHost, port = app.backendPort,
+        method = req.method, url = req.uri,
+        onRspHdr = function( rspHdr, req )
+            req.rspCode = rspHdr.status
+            if rspHdr.status ~= 200 then
+                log:write(".-----------------------------------------\n")
+                log:write("| ".. req.method .." ".. req.uri .."\n")
+                log:write("| Host: ".. app.backendHost ..":".. app.backendPort .."\n")
+                log:write("+-----------------------------------------\n")
+                log:write("|  ".. rspHdr.proto .." ".. rspHdr.status .." ".. rspHdr.phrase .."\n")
+                for i,h in ipairs(rspHdr.headers) do log:write("|  ".. h[1] ..": ".. h[2] .."\n") end
+                log:write("|  \n")
+            end
+        end,
+        onRspChunk = function( buf, req )
+            if req.rspCode ~= 200 then log:write("|  ".. buf:gsub("\n", "\n|  ")) return end
+            if buf then
+                if not req.rspBody then req.rspBody = buf
+                else req.rspBody = req.rspBody .. buf end
+            end
+        end,
+        onRspEnd = function( req )
+            if req.rspCode ~= 200 then log:write("\n'-----------------------------------------\n") end
+            req.isDone = true
+        end,
+    }
+    req.base:closeSnk()
+    assert(req.isDone)
+    if req.rspCode ~= 200 then log:write("ERROR: Couldn't fetch devices\n")return end
+    assert(not app.devices)
+    app.devices = {}
+    log:write("[DEBUG] rspBody.len is ".. req.rspBody:len() .."\n")
+    --io.write(req.rspBody)io.write("\n")
+    for iD, device in pairs(parseJSON(req.rspBody).devices) do
+        --print("Wa", iD, device)
+        --for k,v in pairs(device)do print("W",k,v)end
+        -- TODO how to access 'device.type'?
+        local hostname               , eddieName               , lastSeen
+            = device.hostname:value(), device.eddieName:value(), device.lastSeen:value()
+        local typ
+        if false then
+        elseif hostname:find("^eddie%d%d%d%d%d$") then
+            typ = "EDDIE"
+        elseif hostname:find("^fook%-[a-z0-9]+$") then
+            typ = "FOOK"
+        elseif hostname:find("^lunkwill%-[a-z0-9]+$") then
+            typ = "LUNKWILL"
+        elseif hostname:find("^fook$") then
+            log:write("[WARN ] WTF?!? '"..hostname.."'\n")
+            typ = false
+        else error("TODO_359zh8i3wjho "..hostname) end
+        table.insert(app.devices, objectSeal{
+            hostname = hostname,
+            eddieName = eddieName,
+            type = typ,
+            lastSeen = lastSeen,
+        })
+    end
+    log:write("[INFO ] Fetched ".. #app.devices .." devices.\n")
+end
+
+
+function run( app )
+    fetchDevices(app)
+    dropDevicesRecentlySeen(app)
+    --sortDevicesMostRecentlySeenFirst(app)
+    doWhateverWithDevices(app)
+end
+
+
+function main()
+    local app = objectSeal{
+        isHelp = false,
+        backendHost = false,
+        backendPort = false,
+        sshPort = false,
+        sshUser = false,
+        statePath = false,
+        stateDb = false,
+        http = newHttpClient{},
+        devices = false,
+    }
+    if parseArgs(app) ~= 0 then os.exit(1) end
+    if app.isHelp then printHelp() return end
+    run(app)
+end
+
+
+startOrExecute(main)
+
+