diff options
Diffstat (limited to 'src/main/lua/paisa-fleet/FindFullDisks.lua')
-rw-r--r-- | src/main/lua/paisa-fleet/FindFullDisks.lua | 322 |
1 files changed, 322 insertions, 0 deletions
diff --git a/src/main/lua/paisa-fleet/FindFullDisks.lua b/src/main/lua/paisa-fleet/FindFullDisks.lua new file mode 100644 index 0000000..9963838 --- /dev/null +++ b/src/main/lua/paisa-fleet/FindFullDisks.lua @@ -0,0 +1,322 @@ + +local SL = require("scriptlee") +local newHttpClient = SL.newHttpClient +local newShellcmd = SL.newShellcmd +local newSqlite = SL.newSqlite +local objectSeal = SL.objectSeal +local parseJSON = SL.parseJSON +local startOrExecute = SL.reactor.startOrExecute +SL = nil + +local log = io.stdout + + +function printHelp() + io.write("\n" + .." WARN: This is experimental.\n" + .." \n" + .." Options:\n" + .." --backendHost <inaddr> (eg \"localhost\")\n" + .." --backendPort <int> (eg 80)\n" + .." --sshPort <int> (eg 22)\n" + .." --sshUser <str> (eg \"eddieuser\")\n" + .." --state <path> (eg \"path/to/state\")\n" + .." \n") +end + + +function parseArgs( app ) + app.backendPort = 80 + app.sshPort = 22 + app.sshUser = os.getenv("USERNAME") or false + app.statePath = ":memory:" + local iA = 0 + ::nextArg:: + iA = iA + 1 + local arg = _ENV.arg[iA] + if not arg then + goto verifyResult + elseif arg == "--help" then + app.isHelp = true return 0 + elseif arg == "--backendHost" then + iA = iA + 1; arg = _ENV.arg[iA] + if not arg then log:write("EINVAL: --backendHost needs value\n")return end + app.backendHost = arg + elseif arg == "--backendPort" then + iA = iA + 1; arg = _ENV.arg[iA] + if not arg then log:write("EINVAL: --backendPort needs value\n")return end + app.backendHost = arg + elseif arg == "--sshPort" then + iA = iA + 1; arg = _ENV.arg[iA] + if not arg then log:write("EINVAL: --sshPort needs value\n")return end + app.sshPort = arg + elseif arg == "--sshUser" then + iA = iA + 1; arg = _ENV.arg[iA] + if not arg then log:write("EINVAL: --sshUser needs value\n")return end + app.sshUser = arg + elseif arg == "--state" then + iA = iA + 1; arg = _ENV.arg[iA] + if not arg then log:write("EINVAL: --state needs value\n")return end + app.statePath = arg + else + log:write("EINVAL: ".. arg .."\n")return + end + goto nextArg + ::verifyResult:: + if not app.backendHost then log:write("EINVAL: --backendHost missing\n")return end + if not app.sshUser then log:write("EINVAL: --sshUser missing")return end + return 0 +end + + +function getStateDb(app) + if not app.stateDb then + local db = newSqlite{ database = assert(app.statePath) } + -- TODO normalize scheme + db:prepare("CREATE TABLE IF NOT EXISTS DeviceDfLog(\n" + .." id INTEGER PRIMARY KEY,\n" + .." \"when\" TEXT NOT NULL,\n" -- "https://xkcd.com/1179" + .." hostname TEXT NOT NULL,\n" + .." eddieName TEXT NOT NULL,\n" + .." rootPartitionUsedPercent INT,\n" + .." varLibDockerUsedPercent INT,\n" + .." varLogUsedPercent INT,\n" + .." dataUsedPercent INT,\n" + .." stderr TEXT NOT NULL,\n" + .." stdout TEXT NOT NULL)\n" + ..";"):execute() + app.stateDb = db + end + return app.stateDb +end + + +function storeDiskFullResult( app, hostname, eddieName, stderrBuf, stdoutBuf ) + assert(app and hostname and eddieName and stderrBuf and stdoutBuf); + local rootPartitionUsedPercent = stdoutBuf:match("\n/[^ ]+ +%d+ +%d+ +%d+ +(%d+)%% /\n") + local varLibDockerUsedPercent = stdoutBuf:match("\n[^ ]+ +%d+ +%d+ +%d+ +(%d+)%% /var/lib/docker\n") + local dataUsedPercent = stdoutBuf:match("\n[^ ]+ +%d+ +%d+ +%d+ +(%d+)%% /data\n") + local varLogUsedPercent = stdoutBuf:match("\n[^ ]+ +%d+ +%d+ +%d+ +(%d+)%% /var/log\n") + local stmt = getStateDb(app):prepare("INSERT INTO DeviceDfLog(" + .." \"when\", hostname, eddieName, stderr, stdout," + .." rootPartitionUsedPercent, dataUsedPercent, varLibDockerUsedPercent, varLogUsedPercent, dataUsedPercent" + ..")VALUES(" + .." $when, $hostname, $eddieName, $stderr, $stdout," + .." $rootPartitionUsedPercent, $dataUsedPercent, $varLibDockerUsedPercent, $varLogUsedPercent, $dataUsedPercent);") + stmt:bind("$when", os.date("!%Y-%m-%dT%H:%M:%SZ")) + stmt:bind("$hostname", hostname) + stmt:bind("$eddieName", eddieName) + stmt:bind("$stderr", stderrBuf) + stmt:bind("$stdout", stdoutBuf) + stmt:bind("$rootPartitionUsedPercent", rootPartitionUsedPercent) + stmt:bind("$varLibDockerUsedPercent", varLibDockerUsedPercent) + stmt:bind("$varLogUsedPercent", varLogUsedPercent) + stmt:bind("$dataUsedPercent", dataUsedPercent) + stmt:execute() +end + + +function doWhateverWithDevices( app ) + for k, dev in pairs(app.devices) do + log:write("[INFO ] Inspecting '".. dev.hostname .."' (@ ".. dev.eddieName ..") ...\n") + local fookCmd = "true" + .." && HOSTNAME=$(hostname|sed 's_.isa.localdomain__')" + .." && STAGE=$PAISA_ENV" + .." && printf \"remoteHostname=$HOSTNAME, remoteStage=$STAGE\\n\"" + -- on some machine, df failed with "Stale file handle" But I want to continue + -- with next device regardless of such errors. + .." && df || true" + local eddieCmd = "true" + .." && HOSTNAME=$(hostname|sed 's_.pnet.ch__')" + .." && STAGE=$PAISA_ENV" + .." && printf \"remoteEddieName=$HOSTNAME, remoteStage=$STAGE\\n\"" + .." && if test \"${HOSTNAME}\" != \"".. dev.eddieName .."\"; then true" + .." && echo wrong host. Want ".. dev.eddieName .." found $HOSTNAME && false" + .." ;fi" + .." && ssh -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null" + .." -p".. app.sshPort .." ".. app.sshUser .."@".. ((dev.type == "FOOK")and"fook"or dev.hostname) + .." \\\n --" + .." sh -c 'true && ".. fookCmd:gsub("'", "'\"'\"'") .."'" + local localCmd = assert(os.getenv("SSH_EXE"), "environ.SSH_EXE missing") + .." -oRemoteCommand=none -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null" + .." -p".. app.sshPort .." ".. app.sshUser .."@".. dev.eddieName .."" + .." \\\n --" + .." sh -c 'true && ".. eddieCmd:gsub("'", "'\"'\"'") .."'" + -- TODO get rid of this ugly use-tmp-file-as-script workaround + local tmpPath = assert(os.getenv("TMP"), "environ.TMP missing"):gsub("\\", "/") .."/b30589uj30oahujotehuj.sh" + --log:write("[DEBUG] tmpPath '".. tmpPath .."'\n") + local tmpFile = assert(io.open(tmpPath, "wb"), "Failed to open '".. tmpPath .."'") + tmpFile:write("#!/bin/sh\n".. localCmd .."\n") + tmpFile:close() + --log:write("[DEBUG] tmpPath ".. tmpPath .."\n") + -- EndOf kludge + local cmd = objectSeal{ + base = false, + stdoutBuf = {}, + stderrBuf = {}, + } + cmd.base = newShellcmd{ + cls = cmd, + cmdLine = "sh \"".. tmpPath .."\"", + onStdout = function( buf, cmd ) table.insert(cmd.stdoutBuf, buf or"") end, + onStderr = function( buf, cmd ) table.insert(cmd.stderrBuf, buf or"") end, + } + cmd.base:start() + cmd.base:closeSnk() + local exit, signal = cmd.base:join(17) + cmd.stderrBuf = table.concat(cmd.stderrBuf) + cmd.stdoutBuf = table.concat(cmd.stdoutBuf) + if exit == 255 and signal == nil then + log:write("[DEBUG] fd2: ".. cmd.stderrBuf:gsub("\n", "\n[DEBUG] fd2: "):gsub("\n%[DEBUG%] fd2: $", "") .."\n") + goto nextDevice + end + log:write("[DEBUG] fd1: ".. cmd.stdoutBuf:gsub("\n", "\n[DEBUG] fd1: "):gsub("\n%[DEBUG%] fd1: $", "") .."\n") + storeDiskFullResult(app, dev.hostname, dev.eddieName, cmd.stderrBuf, cmd.stdoutBuf) + if exit ~= 0 or signal ~= nil then + error("exit=".. tostring(exit)..", signal="..tostring(signal)) + end + ::nextDevice:: + end +end + + +function sortDevicesMostRecentlySeenFirst( app ) + table.sort(app.devices, function(a, b) return a.lastSeen > b.lastSeen end) +end + + +-- Don't want to visit just seen devices over and over again. So drop devices +-- we've recently seen from our devices-to-visit list. +function dropDevicesRecentlySeen( app ) + -- Collect recently seen devices. + local devicesToRemove = {} + local st = getStateDb(app):prepare("SELECT hostname FROM DeviceDfLog WHERE \"when\" > $tresholdDate") + st:bind("$tresholdDate", os.date("!%Y-%m-%dT%H:%M:%SZ", os.time()-42*3600)) + local rs = st:execute() + while rs:next() do + local hostname = rs:value(1) + devicesToRemove[hostname] = true + end + -- Remove selected devices + local numKeep, numDrop = 0, 0 + local iD = 0 while true do iD = iD + 1 + local device = app.devices[iD] + if not device then break end + if devicesToRemove[device.hostname] then + --log:write("[DEBUG] Drop '".. device.hostname .."' (".. device.eddieName ..")\n") + numDrop = numDrop + 1 + app.devices[iD] = app.devices[#app.devices] + app.devices[#app.devices] = nil + iD = iD - 1 + else + --log:write("[DEBUG] Keep '".. device.hostname .."' (".. device.eddieName ..")\n") + numKeep = numKeep + 1 + end + end + log:write("[INFO ] Of "..(numKeep+numDrop).." devices from state visit ".. numKeep + .." and skip ".. numDrop .." (bcause seen recently)\n") +end + + +function fetchDevices( app ) + local req = objectSeal{ + base = false, + method = "GET", + uri = "/houston/vehicle/inventory/v1/info/devices", + rspCode = false, + rspBody = false, + isDone = false, + } + req.base = app.http:request{ + cls = req, connectTimeoutMs = 3000, + host = app.backendHost, port = app.backendPort, + method = req.method, url = req.uri, + onRspHdr = function( rspHdr, req ) + req.rspCode = rspHdr.status + if rspHdr.status ~= 200 then + log:write(".-----------------------------------------\n") + log:write("| ".. req.method .." ".. req.uri .."\n") + log:write("| Host: ".. app.backendHost ..":".. app.backendPort .."\n") + log:write("+-----------------------------------------\n") + log:write("| ".. rspHdr.proto .." ".. rspHdr.status .." ".. rspHdr.phrase .."\n") + for i,h in ipairs(rspHdr.headers) do log:write("| ".. h[1] ..": ".. h[2] .."\n") end + log:write("| \n") + end + end, + onRspChunk = function( buf, req ) + if req.rspCode ~= 200 then log:write("| ".. buf:gsub("\n", "\n| ")) return end + if buf then + if not req.rspBody then req.rspBody = buf + else req.rspBody = req.rspBody .. buf end + end + end, + onRspEnd = function( req ) + if req.rspCode ~= 200 then log:write("\n'-----------------------------------------\n") end + req.isDone = true + end, + } + req.base:closeSnk() + assert(req.isDone) + if req.rspCode ~= 200 then log:write("ERROR: Couldn't fetch devices\n")return end + assert(not app.devices) + app.devices = {} + log:write("[DEBUG] rspBody.len is ".. req.rspBody:len() .."\n") + --io.write(req.rspBody)io.write("\n") + for iD, device in pairs(parseJSON(req.rspBody).devices) do + --print("Wa", iD, device) + --for k,v in pairs(device)do print("W",k,v)end + -- TODO how to access 'device.type'? + local hostname , eddieName , lastSeen + = device.hostname:value(), device.eddieName:value(), device.lastSeen:value() + local typ + if false then + elseif hostname:find("^eddie%d%d%d%d%d$") then + typ = "EDDIE" + elseif hostname:find("^fook%-[a-z0-9]+$") then + typ = "FOOK" + elseif hostname:find("^lunkwill%-[a-z0-9]+$") then + typ = "LUNKWILL" + elseif hostname:find("^fook$") then + log:write("[WARN ] WTF?!? '"..hostname.."'\n") + typ = false + else error("TODO_359zh8i3wjho "..hostname) end + table.insert(app.devices, objectSeal{ + hostname = hostname, + eddieName = eddieName, + type = typ, + lastSeen = lastSeen, + }) + end + log:write("[INFO ] Fetched ".. #app.devices .." devices.\n") +end + + +function run( app ) + fetchDevices(app) + dropDevicesRecentlySeen(app) + --sortDevicesMostRecentlySeenFirst(app) + doWhateverWithDevices(app) +end + + +function main() + local app = objectSeal{ + isHelp = false, + backendHost = false, + backendPort = false, + sshPort = false, + sshUser = false, + statePath = false, + stateDb = false, + http = newHttpClient{}, + devices = false, + } + if parseArgs(app) ~= 0 then os.exit(1) end + if app.isHelp then printHelp() return end + run(app) +end + + +startOrExecute(main) + + |