diff options
Diffstat (limited to 'src/main/lua')
19 files changed, 3113 insertions, 23 deletions
diff --git a/src/main/lua/brgmt-logs/DigBrgmtLogs.lua b/src/main/lua/brgmt-logs/DigBrgmtLogs.lua new file mode 100644 index 0000000..fb1f036 --- /dev/null +++ b/src/main/lua/brgmt-logs/DigBrgmtLogs.lua @@ -0,0 +1,5 @@ +-- +-- NOTHING HERE +-- +-- See "brgmt-beef/scripts/". Instead. +-- diff --git a/src/main/lua/git/GitflowChangelogGen.lua b/src/main/lua/git/GitflowChangelogGen.lua new file mode 100644 index 0000000..3b44ac3 --- /dev/null +++ b/src/main/lua/git/GitflowChangelogGen.lua @@ -0,0 +1,195 @@ + +local log = io.stderr +local main + + +function printHelp() + io.stdout:write(" \n" + .." Helper to extract essential data from a gitflow log which potentially\n" + .." is useful to write a CHANGELOG from.\n" + .." \n" + .." Options:\n" + .." \n" + .." --since <date>\n" + .." Ignore commits with this ISO date and older.\n" + .." \n" + .." --remote <str>\n" + .." Name of the git remote to use. Defaults to 'upstream'.\n" + .." \n" + .." --no-fetch\n" + .." Do NOT update refs from remote. Just use what we have local.\n" + .." \n" + ) +end + + +function parseArgs( app ) + local iA = 0 + while true do iA = iA + 1 + local arg = _ENV.arg[iA] + if not arg then + break + elseif arg == "--since" then + iA = iA + 1; arg = _ENV.arg[iA] + if not arg then log:write("EINVAL: --since needs value\n")return end + app.since = arg + elseif arg == "--remote" then + iA = iA + 1; arg = _ENV.arg[iA] + if not arg then log:write("EINVAL: --remote needs value\n")return end + app.remoteName = arg + elseif arg == "--no-fetch" then + app.isFetch = false + elseif arg == "--help" then + app.isHelp = true; return 0 + else + log:write("EINVAL: ".. arg .."\n")return + end + end + if not app.since then log:write("EINVAL: --since missing\n")return end + if not app.remoteName then app.remoteName = "upstream" end + return 0 +end + + +function readCommitHdr( app ) + --log:write("[DEBUG] parse hdr from '".. app.fullHistory:sub(app.fullHistoryRdBeg, app.fullHistoryRdBeg+256) .."...'\n") + local f, t = app.fullHistory:find("^" + .."commit ........................................[^\n]*\n" + .."Merge: [0-9a-z]+ [0-9a-z]+\n" + .."Author: [^\n]+\n" + .."Date: [^\n]+\n" + .."\n" + , app.fullHistoryRdBeg) + if not f then f, t = app.fullHistory:find("^" + .."commit ........................................[^\n]*\n" + .."Author: [^\n]+\n" + .."Date: [^\n]+\n" + .."\n" + , app.fullHistoryRdBeg) end + if not f then + assert(app.fullHistory:len() == app.fullHistoryRdBeg-1, app.fullHistory:len()..", "..app.fullHistoryRdBeg) + app.parseFn = false + return + end + app.commitHdr = assert(app.fullHistory:sub(f, t-1)) + --log:write("hdrBeginsWith '"..(app.commitHdr:sub(1, 32)).."...'\n") + app.fullHistoryRdBeg = t + 1 + --log:write("hdr parsed. rdCursr now points to '".. app.fullHistory:sub(app.fullHistoryRdBeg, app.fullHistoryRdBeg+16) .."...'\n") + app.parseFn = assert(readCommitMsg) +end + + +function readCommitMsg( app ) + local idxOfC = app.fullHistoryRdBeg + local chrPrev = false + while true do idxOfC = idxOfC + 1 + local chr = app.fullHistory:byte(idxOfC) + --log:write("CHR '"..tostring(app.fullHistory:sub(idxOfC, idxOfC)).."'\n") + if (chr == 0x63) and chrPrev == 0x0A then + idxOfC = idxOfC - 1 + break -- LF followed by 'c' (aka 'commit') found + elseif not chr then + idxOfC = idxOfC - 1 + break + else + chrPrev = assert(chr) + end + end + local mtch = app.fullHistory:sub(app.fullHistoryRdBeg, idxOfC - 1) + assert(mtch) + while mtch:byte(mtch:len()) == 0x0A do mtch = mtch:sub(1, -2) end + mtch = mtch:gsub("\n ", "\n"):gsub("^ ", "") + app.commitMsg = mtch + app.fullHistoryRdBeg = idxOfC + 1 + app.parseFn = readCommitHdr + --log:write("msg parsed. rdCursr now points to '".. app.fullHistory:sub(app.fullHistoryRdBeg, app.fullHistoryRdBeg+16) .."...'\n") + table.insert(app.commits, { + hdr = assert(app.commitHdr), + msg = assert(app.commitMsg), + }) +end + + +function run( app ) + local snk = io.stdout + if app.isFetch then + -- Make sure refs are up-to-date + local gitFetch = "git fetch \"".. app.remoteName .."\"" + log:write("[DEBUG] ".. gitFetch .."\n") + local gitFetch = io.popen(gitFetch) + while true do + local buf = gitFetch:read(1<<16) + if not buf then break end + log:write(buf) + end + end + -- Collect input + local git = "git log --date-order --first-parent --decorate --since \"".. app.since.."\"" + .." \"".. app.remoteName .."/master\"" + .." \"".. app.remoteName .."/develop\"" + log:write("[DEBUG] ".. git .."\n") + local git = io.popen(git) + while true do + local buf = git:read(1<<16) + if not buf then break end + --io.stdout:write(buf) + table.insert(app.fullHistory, buf) + end + -- Parse raw commits + app.fullHistory = table.concat(app.fullHistory) + app.parseFn = assert(readCommitHdr) + while app.parseFn do app.parseFn(app) end + -- Prepare output + local prevDate = "0000-00-00" + local version, prevVersion = "v_._._", false + local dateEntry = false + local entries = {} + for k, v in ipairs(app.commits) do + local date = assert(v.hdr:match("\nDate: +([0-9-]+) ")) + local author = assert(v.hdr:match("\nAuthor: +([^\n]+)\n")) + local prNr, short = v.msg:match("Pull request #(%d+): ([^\n]+)\n") + prevVersion = version + _, version = v.hdr:match("^([^\n]+)\n"):match("tag: ([a-z]+)-([^,]+)[,)]") + if not version then version = prevVersion end + + if version ~= prevVersion or not dateEntry then + if dateEntry then table.insert(entries, dateEntry) end + dateEntry = { + txt = date .." - ".. version .."\n\nResolved issues:\n\n" + } + prevDate = date + end + if prNr then + dateEntry.txt = dateEntry.txt .. short .." (PR ".. prNr ..")\n" + else + dateEntry.txt = dateEntry.txt .. v.msg .."\n" + end + end + if dateEntry then table.insert(entries, dateEntry) end + -- output + for k, v in ipairs(entries) do + snk:write("\n\n") + snk:write(v.txt) + snk:write("\n") + end +end + + +function main() + local app = { + since = false, + remoteName = false, + isFetch = true, + fullHistory = {}, + fullHistoryRdBeg = 1, + commits = {}, + parseFn = false, + } + if parseArgs(app) ~= 0 then os.exit(1) end + if app.isHelp then printHelp() return end + run(app) +end + + +main() + diff --git a/src/main/lua/maven/MvnCentralDepScan.lua b/src/main/lua/maven/MvnCentralDepScan.lua index 5322bc0..7f71afa 100644 --- a/src/main/lua/maven/MvnCentralDepScan.lua +++ b/src/main/lua/maven/MvnCentralDepScan.lua @@ -941,9 +941,6 @@ function mod.exportParentsLatest(app) local stmt = app.stmtCache[stmtStr] if not stmt then stmt = db:prepare(stmtStr) app.stmtCache[stmtStr] = stmt end local rs = stmt:execute() - out:write("h;Title;Parent relations (latest only)\n") - out:write("h;ExportedAt;".. os.date("!%Y-%m-%d_%H:%M:%SZ") .."\n") - out:write("c;GroupId;ArtifactId;Version;ParentGid;ParentAid;ParentVersion\n") -- Need to filter out the older artifacts. local all = {} while rs:next() do @@ -954,18 +951,14 @@ function mod.exportParentsLatest(app) if diff > 0 then -- existing is newer. Keep it and ignore newer one. goto nextRecord else -- Either no entry yet or found a newer one. - local entry = { gid=false, aid=false, ver=false, pgid=false, paid=false, pver=false } - entry.gid = gid - entry.aid = aid - entry.ver = ver - entry.pgid = rs:value(4) - entry.paid = rs:value(5) - entry.pver = rs:value(6) - all[key] = entry + all[key] = { gid=gid, aid=aid, ver=ver, pgid=rs:value(4), paid=rs:value(5), pver=rs:value(6) } end ::nextRecord:: end -- Print + out:write("h;Title;Parent relations (latest only)\n") + out:write("h;ExportedAt;".. os.date("!%Y-%m-%d_%H:%M:%SZ") .."\n") + out:write("c;GroupId;ArtifactId;Version;ParentGid;ParentAid;ParentVersion\n") for _, entry in pairs(all) do out:write("r;".. entry.gid ..";".. entry.aid ..";".. entry.ver ..";".. entry.pgid ..";".. entry.paid ..";".. entry.pver .."\n") @@ -1031,9 +1024,6 @@ function mod.exportDepsLatest(app) local stmt = app.stmtCache[stmtStr] if not stmt then stmt = db:prepare(stmtStr) app.stmtCache[stmtStr] = stmt end local rs = stmt:execute() - out:write("h;Title;Dependencies (of latest only)\n") - out:write("h;ExportedAt;".. os.date("!%Y-%m-%d_%H:%M:%SZ") .."\n") - out:write("c;GroupId;ArtifactId;Version;Dependency GID;Dependency AID;Dependency Version\n") -- Need to filter out the older artifacts. local all = {} local entry, key, gid, aid, ver, diff @@ -1046,18 +1036,14 @@ function mod.exportDepsLatest(app) if diff > 0 then -- existing is newer. Keep it and ignore newer one. goto nextRecord else -- Either no entry yet or found a newer one. - local entry = { gid=false, aid=false, ver=false, dgid=false, daid=false, dver=false } - entry.gid = gid - entry.aid = aid - entry.ver = ver - entry.dgid = rs:value(4) - entry.daid = rs:value(5) - entry.dver = rs:value(6) - all[key] = entry + all[key] = { gid=gid, aid=aid, ver=ver, dgid=rs:value(4), daid=rs:value(5), dver=rs:value(6) } end goto nextRecord ::endFiltering:: -- Print + out:write("h;Title;Dependencies (of latest only)\n") + out:write("h;ExportedAt;".. os.date("!%Y-%m-%d_%H:%M:%SZ") .."\n") + out:write("c;GroupId;ArtifactId;Version;Dependency GID;Dependency AID;Dependency Version\n") for _, entry in pairs(all) do out:write("r;".. entry.gid ..";".. entry.aid ..";".. entry.ver ..";".. entry.dgid ..";".. entry.daid ..";".. entry.dver .."\n") diff --git a/src/main/lua/misc/JavaCallgraph.lua b/src/main/lua/misc/JavaCallgraph.lua new file mode 100644 index 0000000..6d0bd62 --- /dev/null +++ b/src/main/lua/misc/JavaCallgraph.lua @@ -0,0 +1,159 @@ + +local SL = require("scriptlee") +local newJavaClassParser = SL.newJavaClassParser +local objectSeal = SL.objectSeal +SL = nil + +local snk = io.stdout + +local main + + +function initParser( app ) + app.parser = newJavaClassParser{ + cls = app, + onMagic = function(m, app) assert(m == "\xCA\xFE\xBA\xBE") end, + onClassfileVersion = function(maj, min, app) assert(maj == 55 and min == 0) end, + onConstPoolClassRef = function(i, idx, app) + app.constPool[i] = objectSeal{ type = "CLASS_REF", classNameIdx = idx, className = false, } + end, + onConstPoolIfaceMethodRef = function(i, nameIdx, nameAndTypeIdx, app) + app.constPool[i] = objectSeal{ + type = "IFACE_METHOD_REF", nameIdx = nameIdx, nameAndTypeIdx = nameAndTypeIdx, + className = false, methodName = false, methodType = false, + } + end, + onConstPoolMethodRef = function(i, classIdx, nameAndTypeIdx, app) + app.constPool[i] = objectSeal{ + type = "METHOD_REF", classIdx = classIdx, nameAndTypeIdx = nameAndTypeIdx, + className = false, methodName = false, signature = false, + } + end, + onConstPoolMethodType = function(i, descrIdx, app) + app.constPool[i] = objectSeal{ + type = "METHOD_TYPE", descrIdx = descrIdx, descrStr = false, + } + end, + onConstPoolNameAndType = function(i, nameIdx, typeIdx, app) + app.constPool[i] = objectSeal{ + type = "NAME_AND_TYPE", nameIdx = nameIdx, typeIdx = typeIdx, nameStr = false, typeStr = false, + } + end, + onConstPoolUtf8 = function(i, str, app) + app.constPool[i] = objectSeal{ type = "UTF8", str = str, } + end, + + onConstPoolInvokeDynamic = function(i, bootstrapMethodAttrIdx, nameAndTypeIdx, app) + app.constPool[i] = objectSeal{ + type = "INVOKE_DYNAMIC", bootstrapMethodAttrIdx = bootstrapMethodAttrIdx, nameAndTypeIdx = nameAndTypeIdx, + methodName = false, methodType = false, factoryClass = false, factoryMethod = false, factoryType = false, + } + end, + onConstPoolFieldRef = function(i, nameIdx, nameAndTypeIdx, that) + app.constPool[i] = objectSeal{ + type = "FIELD_REF", nameIdx = nameIdx, nameAndTypeIdx = nameAndTypeIdx, + className = false, methodName = false, methodType = false, + } + end, + --onConstPoolMethodHandle = function(i, refKind, refIdx, app) + -- app.constPool[i] = objectSeal{ type = "METHOD_HANDLE", refKind = refKind, refIdx = refIdx, } + --end, + --onConstPoolStrRef = function(i, dstIdx, app) + -- print("ConstPool["..i.."] <StrRef> #"..dstIdx) + --end, + --onThisClass = function(nameIdx, app) + -- -- TODO print("onThisClass(#"..nameIdx..")") + --end, + --onField = function(iField, accessFlags, nameIdx, descrIdx, numAttrs, app) + -- print(string.format("onField(0x%04X, #%d, #%d, %d)",accessFlags,nameIdx,descrIdx,numAttrs)) + --end, + --onMethod = function(accessFlags, nameIdx, descrIdx, app) + -- print(string.format("onMethod(0x%04X, #%d, #%d)",accessFlags,nameIdx,descrIdx)) + --end, + + onConstPoolEnd = function( app ) + -- 1st run + for i, cpe in pairs(app.constPool) do + if false then + elseif cpe.type == "CLASS_REF" then + local tmp + tmp = assert(cpe.classNameIdx) + tmp = assert(app.constPool[cpe.classNameIdx], cpe.classNameIdx) + tmp = assert(tmp.str, tmp) + cpe.className = assert(tmp) + elseif cpe.type == "METHOD_TYPE" then + cpe.descrStr = assert(app.constPool[cpe.descrIdx].str) + elseif cpe.type == "NAME_AND_TYPE" then + cpe.nameStr = assert(app.constPool[cpe.nameIdx].str); + cpe.typeStr = assert(app.constPool[cpe.typeIdx].str); + end + end + -- 2nd run + for i, cpe in pairs(app.constPool) do + if false then + elseif cpe.type == "FIELD_REF" then + local nameAndType = assert(app.constPool[cpe.nameAndTypeIdx]) + cpe.className = assert(app.constPool[cpe.nameIdx].className); + cpe.methodName = assert(app.constPool[nameAndType.nameIdx].str); + cpe.methodType = assert(app.constPool[nameAndType.typeIdx].str); + elseif cpe.type == "METHOD_REF" then + local nameAndType = app.constPool[cpe.nameAndTypeIdx] + cpe.className = assert(app.constPool[cpe.classIdx].className) + cpe.methodName = assert(app.constPool[nameAndType.nameIdx].str) + cpe.signature = assert(app.constPool[nameAndType.typeIdx].str) + elseif cpe.type == "IFACE_METHOD_REF" then + local classRef = assert(app.constPool[cpe.nameIdx]) + local nameAndType = assert(app.constPool[cpe.nameAndTypeIdx]) + cpe.className = assert(classRef.className) + cpe.methodName = assert(app.constPool[nameAndType.nameIdx].str) + cpe.methodType = assert(app.constPool[nameAndType.typeIdx].str) + elseif cpe.type == "INVOKE_DYNAMIC" then + local nameAndType = assert(app.constPool[cpe.nameAndTypeIdx]) + local bootstrapMethod = assert(app.constPool[cpe.bootstrapMethodAttrIdx], cpe.bootstrapMethodAttrIdx); + cpe.methodName = assert(app.constPool[nameAndType.nameIdx].str) + cpe.methodType = assert(app.constPool[nameAndType.typeIdx].str) + --cpe.factoryClass = ; + --cpe.factoryMethod = ; + --cpe.factoryType = ; + end + end + -- debug-print + snk:write("\n") + for _,cpIdx in pairs{ 13, 14, 15, 227, 230, 236, 704, 709, 717 }do + snk:write("CONST_POOL @ ".. cpIdx .."\n") + for k,v in pairs(app.constPool[cpIdx])do print("X",k,v)end + end + for i, cpe in pairs(app.constPool) do + if false then + --elseif cpe.type == "CLASSREF" then + -- snk:write("CLASS \"".. cpe.className .."\"\n") + end + end + end, + } +end + + +function main() + local app = objectSeal{ + parser = false, + constPool = {}, + } + + initParser(app) + + -- Read 1st arg as a classfile and pump it into the parser. + local src = arg[1] and io.open( arg[1], "rb" ) or nil + if not src then + print("ERROR: Failed to open file from 1st arg: "..(arg[1]or"nil")) return + end + while true do + local buf = src:read(8192) + if not buf then break end + app.parser:write(buf) + end + app.parser:closeSnk() +end + + +main() diff --git a/src/main/lua/mshitteams/ListEmlInbox.lua b/src/main/lua/mshitteams/ListEmlInbox.lua new file mode 100644 index 0000000..23b42aa --- /dev/null +++ b/src/main/lua/mshitteams/ListEmlInbox.lua @@ -0,0 +1,322 @@ +-- +-- Sources: +-- - [Authorize](https://learn.microsoft.com/en-us/graph/auth-v2-user?tabs=http) +-- - [Auth witout app register](https://techcommunity.microsoft.com/t5/teams-developer/authenticate-microsoft-graph-api-with-username-and-password/m-p/3940540) +-- +-- TODO: scriptlee 0.0.5-83-gdffa272 seems to SEGFAULT constantly here. No +-- matter if we use socket or newHttpClient. +-- TODO: scriptlee 0.0.5-87-g946ebdc crashes through assertion: +-- Assertion failed: cls->msg.connect.sck->vt->unwrap != NULL, file src/windoof/c/io/AsyncIO.c, line 421 +-- + +local SL = require("scriptlee") +local newHttpClient = SL.newHttpClient +--local AF_INET = SL.posix.AF_INET +--local getaddrinfo = SL.posix.getaddrinfo +--local INADDR_ANY = SL.posix.INADDR_ANY +--local inaddrOfHostname = SL.posix.inaddrOfHostname +--local IPPROTO_TCP = SL.posix.IPPROTO_TCP +local objectSeal = SL.objectSeal +--local SOCK_STREAM = SL.posix.SOCK_STREAM +--local socket = SL.posix.socket +local startOrExecute = SL.reactor.startOrExecute +--for k,v in pairs(SL)do print("SL",k,v)end os.exit(1) +SL = nil + +local authorizeToMsGraphApi, getAccessToken, getAuthHdr, httpUrlEncode, main, parseArgs, printHelp, + run, getMyProfileForDebugging +local inn, out, log = io.stdin, io.stdout, io.stderr + + +function printHelp() + out:write(" \n" + .." Experiments for M$ graph API.\n" + .." \n" + .." WARN: This tool is experimental! Do NOT use it!\n" + .." \n" + .." Options:\n" + .." \n" + .." --user <str>\n" + .." M$ user.\n" + .." \n" + .." --pass <str>\n" + .." M$ password. TODO get rid of this insecure idea.\n" + .." \n" + .." --appId <str>\n" + .." AppId (aka client_id). See M$ doc about it.\n" + .." \n") +end + + +function parseArgs( app ) + if #_ENV.arg == 0 then log:write("EINVAL: Args missing\n")return-1 end + local iA = 0 + --local isYolo = false + while true do iA = iA + 1 + local arg = _ENV.arg[iA] + if not arg then + break + elseif arg == "--help" then + app.isHelp = true; return 0 + elseif arg == "--user" then + iA = iA + 1; arg = _ENV.arg[iA] + if not arg then log:write("EINVAL: --user needs value\n")return-1 end + app.msUser = arg + elseif arg == "--pass" then + iA = iA + 1; arg = _ENV.arg[iA] + if not arg then log:write("EINVAL: --pass needs value\n")return-1 end + app.msPass = arg + elseif arg == "--appId" then + iA = iA + 1; arg = _ENV.arg[iA] + if not arg then log:write("EINVAL: --appId needs value\n")return-1 end + app.msAppId = arg + --elseif arg == "--yolo" then + -- isYolo = true + else + log:write("EINVAL: ".. arg .."\n") return-1 + end + end + if not app.msUser then log:write("EINVAL: --user missing\n") return-1 end + if not app.msPass then log:write("EINVAL: --pass missing\n") return-1 end + if not app.msAppId then log:write("EINVAL: --appId missing\n")return-1 end + --if not isYolo then log:write("EINVAL: --yolo missing\n")return-1 end + return 0 +end + + +function getMyProfileForDebugging( app ) + local http = app.http + local authKey, authVal = getAuthHdr(app) + local req = objectSeal{ + base = false, + method = "GET", + uri = "/v1.0/me", + rspCode = false, + rspBody = {}, + } + req.base = http:request{ + cls = req, + host = app.msGraphHost, + port = app.msGraphPort, + connectTimeoutMs = 3000, + method = req.method, + url = req.uri, + hdrs = { + { authKey, authVal }, + }, + --useHostHdr = , + --useTLS = true, + onRspHdr = function( rsp, cls ) + cls.rspCode = rsp.status + if rsp.status ~= 200 then + log:write("> ".. req.method .." ".. req.uri .."\n> \n") + log:write("< ".. rsp.proto .." ".. rsp.status .." ".. rsp.phrase .."\n") + for _,h in ipairs(rsp.headers)do log:write("< "..h[1]..": "..h[2].."\n")end + log:write("\n") + end + end, + onRspChunk = function(buf, cls) + if cls.rspCode ~= 200 then + log:write("< ") + log:write((buf:gsub("\n", "\n< "))) + log:write("\n") + else + assert(type(buf) == "string") + table.insert(cls.rspBody, buf) + end + end, + onRspEnd = function(cls) + if cls.rspCode ~= 200 then error("Request failed.") end + cls.rspBody = table.concat(cls.rspBody) + log:write("Response was:\n\n") + log:write(cls.rspBody) + log:write("\n\n") + end, + } + req.base:closeSnk() +end + + +function authorizeToMsGraphApi( app ) + local http = app.http + local req = objectSeal{ + base = false, + method = "GET", + host = (app.proxyHost or app.msLoginHost), + port = (app.proxyPort or app.msLoginPort), + uri = false, + hdrs = { + { "Content-Type", "application/x-www-form-urlencoded" }, + }, + reqBody = "" + .. "grant_type=password" + .."&resource=https://graph.microsoft.com" + .."&username=".. httpUrlEncode(app, app.msUser) .."" + .."&password=".. httpUrlEncode(app, app.msPass) .."", + rspProto = false, rspCode = false, rspPhrase = false, + rspHdrs = false, + rspBody = {}, + } + if app.proxyHost then + req.uri = "https://".. app.msLoginHost ..":".. app.msLoginPort + .."/".. app.msTenant .."/oauth2/v2.0/token" + else + req.uri = "/".. app.msTenant .."/oauth2/v2.0/token" + end + local ok, ex = xpcall(function() + req.base = http:request{ + cls = req, + connectTimeoutMs = app.connectTimeoutMs, + host = req.host, + port = req.port, + method = req.method, + url = req.uri, + hdrs = req.hdrs, + onRspHdr = function( rsp, req ) + req.rspProto = rsp.proto + req.rspCode = rsp.status + req.rspPhrase = rsp.phrase + req.rspHdrs = rsp.headers + end, + onRspChunk = function( buf, req ) table.insert(req.rspBody, buf) end, + onRspEnd = function( req ) + local rspBody = table.concat(req.rspBody) req.rspBody = false + if req.rspCode ~= 200 then + log:write("[ERROR] Request failed\n") + log:write("peer ".. req.host ..":".. req.port .."\n") + log:write("> ".. req.method .." ".. req.uri .."\n") + for _, h in ipairs(req.hdrs) do log:write("> ".. h[1] ..": ".. h[2] .."\n") end + log:write("> \n") + log:write("> ".. req.reqBody:gsub("\r?\n", "\n> ") .."\n") + log:write("< ".. req.rspProto .." ".. req.rspCode .." ".. req.rspPhrase .."\n") + for _, h in ipairs(req.rspHdrs) do log:write("< ".. h[1] ..": ".. h[2] .."\n")end + log:write("< \n") + log:write("< ".. rspBody:gsub("\r?\n", "\n< ") .."\n") + error("TODO_10aa11de804e733337e7c244298791c6") + end + log:write("< ".. req.rspProto .." ".. req.rspCode .." ".. req.rspPhrase .."\n") + for _, h in ipairs(req.rspHdrs) do log:write("< ".. h[1] ..": ".. h[2] .."\n")end + log:write("< \n") + log:write("< ".. rspBody:gsub("\r?\n", "\n< ") .."\n") + -- How to continue: + --local token = rsp.bodyJson.access_token + --local authHdr = { "Authorization", "Bearer ".. token, } + end, + } + end, debug.traceback) + if not ok then + log:write("[ERROR] Request failed 2\n") + log:write("peer ".. req.host ..":".. req.port .."\n") + log:write("> ".. req.method .." ".. req.uri .."\n") + for _, h in ipairs(req.hdrs) do log:write("> ".. h[1] ..": ".. h[2] .."\n") end + log:write("> \n") + log:write("> ".. req.reqBody:gsub("\r?\n", "\n> ") .."\n") + error(ex) + end + --req.base:write(req.reqBody) + req.base:closeSnk() +end + + +function httpUrlEncode( app, str ) + local hexDigits, ret, beg, iRd = "0123456789ABCDEF", {}, 1, 0 + ::nextInputChar:: + iRd = iRd + 1 + local byt = str:byte(iRd) + if not byt then + elseif byt == 0x2D -- dash + or byt == 0x2E -- dot + or byt >= 0x30 and byt <= 0x39 -- 0-9 + or byt >= 0x40 and byt <= 0x5A -- A-Z + or byt >= 0x60 and byt <= 0x7A -- a-z + then + goto nextInputChar + end + if beg < iRd then table.insert(ret, str:sub(beg, iRd-1)) end + if not byt then return table.concat(ret) end + table.insert(ret, "%") + local hi = (byt & 0xF0) >> 4 +1 + local lo = (byt & 0x0F) +1 + table.insert(ret, hexDigits:sub(hi, hi) .. hexDigits:sub(lo, lo)) + beg = iRd + 1 + goto nextInputChar +end + + +function getAccessToken( app ) + -- See "https://learn.microsoft.com/en-us/graph/auth-v2-user?tabs=http#3-request-an-access-token" + local method = "POST" + local uri = "/".. app.msTenant .."/oauth2/v2.0/token" + local hdrs = { + { "Host", "https://login.microsoftonline.com" }, + { "Content-Type", "application/x-www-form-urlencoded" }, + } + local body = "" + .."client_id=".. assert(app.appId) + .."&scope=".. scope + .."&code=".. code + .."&redirect_uri=".. redirUri + .."&grant_type=authorization_code" +end + + +-- @return 1 - HTTP header key +-- @return 2 - HTTP header value +function getAuthHdr( app ) + assert(app.msToken) + return "Authorization", ("Bearer ".. app.msToken) +end + + +function run( app ) + app.http = newHttpClient{} + authorizeToMsGraphApi(app) + --getMyProfileForDebugging(app) +end + + +function main() + local loginHost, loginPort, graphHost, graphPort, proxyHost, proxyPort + local choice = 3 + if choice == 1 then + loginHost = "login.microsoftonline.com"; loginPort = 443 + graphHost = "graph.microsoft.com"; graphPort = 443 + proxyHost = "127.0.0.1"; proxyPort = 3128 + elseif choice == 2 then + loginHost = "127.0.0.1"; loginPort = 8081 + graphHost = "127.0.0.1"; graphPort = 8081 + proxyHost = false; proxyPort = false + elseif choice == 3 then + loginHost = "login.microsoftonline.com"; loginPort = 443 + graphHost = "127.0.0.1"; graphPort = 8081 + proxyHost = "127.0.0.1"; proxyPort = 3128 + elseif choice == 4 then + loginHost = "login.microsoftonline.com"; loginPort = 443 + graphHost = "graph.microsoft.com"; graphPort = 443 + proxyHost = false; proxyPort = false + else error("TODO_1700683244") end + local app = objectSeal{ + isHelp = false, + msLoginHost = loginHost, msLoginPort = loginPort, + msGraphHost = graphHost, msGraphPort = graphPort, + proxyHost = proxyHost, proxyPort = proxyPort, + -- TODO take this from a failed api call, which has this in the rsp headers. + msTenant = "common", -- TODO configurable + -- TODO take this from a failed api call, which has this in the rsp headers. + msAppId = false, + msPerms = "offline_access user.read mail.read", + msToken = false, + msUser = false, + msPass = false, + http = false, + connectTimeoutMs = 3000, + --sck = false, + } + if parseArgs(app) ~= 0 then os.exit(1) end + if app.isHelp then printHelp() return end + run(app) +end + + +startOrExecute(main) + diff --git a/src/main/lua/mshitteams/SendRawMsEmail.lua b/src/main/lua/mshitteams/SendRawMsEmail.lua new file mode 100644 index 0000000..2d2940e --- /dev/null +++ b/src/main/lua/mshitteams/SendRawMsEmail.lua @@ -0,0 +1,60 @@ + +local SL = require("scriptlee") +--local newHttpClient = SL.newHttpClient +--local newShellcmd = SL.newShellcmd +--local objectSeal = SL.objectSeal +--local parseJSON = SL.parseJSON +--local sleep = SL.posix.sleep +--local newCond = SL.posix.newCond +--local async = SL.reactor.async +--local startOrExecute = SL.reactor.startOrExecute +--for k,v in pairs(SL)do print("SL",k,v)end os.exit(1) +SL = nil + +local mod = {} +local inn, out, log = io.stdin, io.stdout, io.stderr + + +function mod.printHelp() + out:write(" \n" + .." Options:\n" + .." \n" + .."\n\n") +end + + +function mod.parseArgs( app ) + local isStdinn = false + local iA = 0 + while true do iA = iA + 1 + local arg = _ENV.arg[iA] + if not arg then + break + elseif arg == "--help" then + app.isHelp = true; return 0 + else + log:write("Unknown arg: ".. arg .."\n") return-1 + end + end + if not isStdinn then log:write("Bad args\n")return-1 end + return 0 +end + + +function mod.run( app ) + error("TODO_20230608125925") +end + + +function mod.main() + local app = objectSeal{ + isHelp = false, + } + if mod.parseArgs(app) ~= 0 then os.exit(1) end + if app.isHelp then mod.printHelp() return end + mod.run(app) +end + + +startOrExecute(mod.main) + diff --git a/src/main/lua/paisa-fleet/FindFullDisks.lua b/src/main/lua/paisa-fleet/FindFullDisks.lua new file mode 100644 index 0000000..9963838 --- /dev/null +++ b/src/main/lua/paisa-fleet/FindFullDisks.lua @@ -0,0 +1,322 @@ + +local SL = require("scriptlee") +local newHttpClient = SL.newHttpClient +local newShellcmd = SL.newShellcmd +local newSqlite = SL.newSqlite +local objectSeal = SL.objectSeal +local parseJSON = SL.parseJSON +local startOrExecute = SL.reactor.startOrExecute +SL = nil + +local log = io.stdout + + +function printHelp() + io.write("\n" + .." WARN: This is experimental.\n" + .." \n" + .." Options:\n" + .." --backendHost <inaddr> (eg \"localhost\")\n" + .." --backendPort <int> (eg 80)\n" + .." --sshPort <int> (eg 22)\n" + .." --sshUser <str> (eg \"eddieuser\")\n" + .." --state <path> (eg \"path/to/state\")\n" + .." \n") +end + + +function parseArgs( app ) + app.backendPort = 80 + app.sshPort = 22 + app.sshUser = os.getenv("USERNAME") or false + app.statePath = ":memory:" + local iA = 0 + ::nextArg:: + iA = iA + 1 + local arg = _ENV.arg[iA] + if not arg then + goto verifyResult + elseif arg == "--help" then + app.isHelp = true return 0 + elseif arg == "--backendHost" then + iA = iA + 1; arg = _ENV.arg[iA] + if not arg then log:write("EINVAL: --backendHost needs value\n")return end + app.backendHost = arg + elseif arg == "--backendPort" then + iA = iA + 1; arg = _ENV.arg[iA] + if not arg then log:write("EINVAL: --backendPort needs value\n")return end + app.backendHost = arg + elseif arg == "--sshPort" then + iA = iA + 1; arg = _ENV.arg[iA] + if not arg then log:write("EINVAL: --sshPort needs value\n")return end + app.sshPort = arg + elseif arg == "--sshUser" then + iA = iA + 1; arg = _ENV.arg[iA] + if not arg then log:write("EINVAL: --sshUser needs value\n")return end + app.sshUser = arg + elseif arg == "--state" then + iA = iA + 1; arg = _ENV.arg[iA] + if not arg then log:write("EINVAL: --state needs value\n")return end + app.statePath = arg + else + log:write("EINVAL: ".. arg .."\n")return + end + goto nextArg + ::verifyResult:: + if not app.backendHost then log:write("EINVAL: --backendHost missing\n")return end + if not app.sshUser then log:write("EINVAL: --sshUser missing")return end + return 0 +end + + +function getStateDb(app) + if not app.stateDb then + local db = newSqlite{ database = assert(app.statePath) } + -- TODO normalize scheme + db:prepare("CREATE TABLE IF NOT EXISTS DeviceDfLog(\n" + .." id INTEGER PRIMARY KEY,\n" + .." \"when\" TEXT NOT NULL,\n" -- "https://xkcd.com/1179" + .." hostname TEXT NOT NULL,\n" + .." eddieName TEXT NOT NULL,\n" + .." rootPartitionUsedPercent INT,\n" + .." varLibDockerUsedPercent INT,\n" + .." varLogUsedPercent INT,\n" + .." dataUsedPercent INT,\n" + .." stderr TEXT NOT NULL,\n" + .." stdout TEXT NOT NULL)\n" + ..";"):execute() + app.stateDb = db + end + return app.stateDb +end + + +function storeDiskFullResult( app, hostname, eddieName, stderrBuf, stdoutBuf ) + assert(app and hostname and eddieName and stderrBuf and stdoutBuf); + local rootPartitionUsedPercent = stdoutBuf:match("\n/[^ ]+ +%d+ +%d+ +%d+ +(%d+)%% /\n") + local varLibDockerUsedPercent = stdoutBuf:match("\n[^ ]+ +%d+ +%d+ +%d+ +(%d+)%% /var/lib/docker\n") + local dataUsedPercent = stdoutBuf:match("\n[^ ]+ +%d+ +%d+ +%d+ +(%d+)%% /data\n") + local varLogUsedPercent = stdoutBuf:match("\n[^ ]+ +%d+ +%d+ +%d+ +(%d+)%% /var/log\n") + local stmt = getStateDb(app):prepare("INSERT INTO DeviceDfLog(" + .." \"when\", hostname, eddieName, stderr, stdout," + .." rootPartitionUsedPercent, dataUsedPercent, varLibDockerUsedPercent, varLogUsedPercent, dataUsedPercent" + ..")VALUES(" + .." $when, $hostname, $eddieName, $stderr, $stdout," + .." $rootPartitionUsedPercent, $dataUsedPercent, $varLibDockerUsedPercent, $varLogUsedPercent, $dataUsedPercent);") + stmt:bind("$when", os.date("!%Y-%m-%dT%H:%M:%SZ")) + stmt:bind("$hostname", hostname) + stmt:bind("$eddieName", eddieName) + stmt:bind("$stderr", stderrBuf) + stmt:bind("$stdout", stdoutBuf) + stmt:bind("$rootPartitionUsedPercent", rootPartitionUsedPercent) + stmt:bind("$varLibDockerUsedPercent", varLibDockerUsedPercent) + stmt:bind("$varLogUsedPercent", varLogUsedPercent) + stmt:bind("$dataUsedPercent", dataUsedPercent) + stmt:execute() +end + + +function doWhateverWithDevices( app ) + for k, dev in pairs(app.devices) do + log:write("[INFO ] Inspecting '".. dev.hostname .."' (@ ".. dev.eddieName ..") ...\n") + local fookCmd = "true" + .." && HOSTNAME=$(hostname|sed 's_.isa.localdomain__')" + .." && STAGE=$PAISA_ENV" + .." && printf \"remoteHostname=$HOSTNAME, remoteStage=$STAGE\\n\"" + -- on some machine, df failed with "Stale file handle" But I want to continue + -- with next device regardless of such errors. + .." && df || true" + local eddieCmd = "true" + .." && HOSTNAME=$(hostname|sed 's_.pnet.ch__')" + .." && STAGE=$PAISA_ENV" + .." && printf \"remoteEddieName=$HOSTNAME, remoteStage=$STAGE\\n\"" + .." && if test \"${HOSTNAME}\" != \"".. dev.eddieName .."\"; then true" + .." && echo wrong host. Want ".. dev.eddieName .." found $HOSTNAME && false" + .." ;fi" + .." && ssh -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null" + .." -p".. app.sshPort .." ".. app.sshUser .."@".. ((dev.type == "FOOK")and"fook"or dev.hostname) + .." \\\n --" + .." sh -c 'true && ".. fookCmd:gsub("'", "'\"'\"'") .."'" + local localCmd = assert(os.getenv("SSH_EXE"), "environ.SSH_EXE missing") + .." -oRemoteCommand=none -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null" + .." -p".. app.sshPort .." ".. app.sshUser .."@".. dev.eddieName .."" + .." \\\n --" + .." sh -c 'true && ".. eddieCmd:gsub("'", "'\"'\"'") .."'" + -- TODO get rid of this ugly use-tmp-file-as-script workaround + local tmpPath = assert(os.getenv("TMP"), "environ.TMP missing"):gsub("\\", "/") .."/b30589uj30oahujotehuj.sh" + --log:write("[DEBUG] tmpPath '".. tmpPath .."'\n") + local tmpFile = assert(io.open(tmpPath, "wb"), "Failed to open '".. tmpPath .."'") + tmpFile:write("#!/bin/sh\n".. localCmd .."\n") + tmpFile:close() + --log:write("[DEBUG] tmpPath ".. tmpPath .."\n") + -- EndOf kludge + local cmd = objectSeal{ + base = false, + stdoutBuf = {}, + stderrBuf = {}, + } + cmd.base = newShellcmd{ + cls = cmd, + cmdLine = "sh \"".. tmpPath .."\"", + onStdout = function( buf, cmd ) table.insert(cmd.stdoutBuf, buf or"") end, + onStderr = function( buf, cmd ) table.insert(cmd.stderrBuf, buf or"") end, + } + cmd.base:start() + cmd.base:closeSnk() + local exit, signal = cmd.base:join(17) + cmd.stderrBuf = table.concat(cmd.stderrBuf) + cmd.stdoutBuf = table.concat(cmd.stdoutBuf) + if exit == 255 and signal == nil then + log:write("[DEBUG] fd2: ".. cmd.stderrBuf:gsub("\n", "\n[DEBUG] fd2: "):gsub("\n%[DEBUG%] fd2: $", "") .."\n") + goto nextDevice + end + log:write("[DEBUG] fd1: ".. cmd.stdoutBuf:gsub("\n", "\n[DEBUG] fd1: "):gsub("\n%[DEBUG%] fd1: $", "") .."\n") + storeDiskFullResult(app, dev.hostname, dev.eddieName, cmd.stderrBuf, cmd.stdoutBuf) + if exit ~= 0 or signal ~= nil then + error("exit=".. tostring(exit)..", signal="..tostring(signal)) + end + ::nextDevice:: + end +end + + +function sortDevicesMostRecentlySeenFirst( app ) + table.sort(app.devices, function(a, b) return a.lastSeen > b.lastSeen end) +end + + +-- Don't want to visit just seen devices over and over again. So drop devices +-- we've recently seen from our devices-to-visit list. +function dropDevicesRecentlySeen( app ) + -- Collect recently seen devices. + local devicesToRemove = {} + local st = getStateDb(app):prepare("SELECT hostname FROM DeviceDfLog WHERE \"when\" > $tresholdDate") + st:bind("$tresholdDate", os.date("!%Y-%m-%dT%H:%M:%SZ", os.time()-42*3600)) + local rs = st:execute() + while rs:next() do + local hostname = rs:value(1) + devicesToRemove[hostname] = true + end + -- Remove selected devices + local numKeep, numDrop = 0, 0 + local iD = 0 while true do iD = iD + 1 + local device = app.devices[iD] + if not device then break end + if devicesToRemove[device.hostname] then + --log:write("[DEBUG] Drop '".. device.hostname .."' (".. device.eddieName ..")\n") + numDrop = numDrop + 1 + app.devices[iD] = app.devices[#app.devices] + app.devices[#app.devices] = nil + iD = iD - 1 + else + --log:write("[DEBUG] Keep '".. device.hostname .."' (".. device.eddieName ..")\n") + numKeep = numKeep + 1 + end + end + log:write("[INFO ] Of "..(numKeep+numDrop).." devices from state visit ".. numKeep + .." and skip ".. numDrop .." (bcause seen recently)\n") +end + + +function fetchDevices( app ) + local req = objectSeal{ + base = false, + method = "GET", + uri = "/houston/vehicle/inventory/v1/info/devices", + rspCode = false, + rspBody = false, + isDone = false, + } + req.base = app.http:request{ + cls = req, connectTimeoutMs = 3000, + host = app.backendHost, port = app.backendPort, + method = req.method, url = req.uri, + onRspHdr = function( rspHdr, req ) + req.rspCode = rspHdr.status + if rspHdr.status ~= 200 then + log:write(".-----------------------------------------\n") + log:write("| ".. req.method .." ".. req.uri .."\n") + log:write("| Host: ".. app.backendHost ..":".. app.backendPort .."\n") + log:write("+-----------------------------------------\n") + log:write("| ".. rspHdr.proto .." ".. rspHdr.status .." ".. rspHdr.phrase .."\n") + for i,h in ipairs(rspHdr.headers) do log:write("| ".. h[1] ..": ".. h[2] .."\n") end + log:write("| \n") + end + end, + onRspChunk = function( buf, req ) + if req.rspCode ~= 200 then log:write("| ".. buf:gsub("\n", "\n| ")) return end + if buf then + if not req.rspBody then req.rspBody = buf + else req.rspBody = req.rspBody .. buf end + end + end, + onRspEnd = function( req ) + if req.rspCode ~= 200 then log:write("\n'-----------------------------------------\n") end + req.isDone = true + end, + } + req.base:closeSnk() + assert(req.isDone) + if req.rspCode ~= 200 then log:write("ERROR: Couldn't fetch devices\n")return end + assert(not app.devices) + app.devices = {} + log:write("[DEBUG] rspBody.len is ".. req.rspBody:len() .."\n") + --io.write(req.rspBody)io.write("\n") + for iD, device in pairs(parseJSON(req.rspBody).devices) do + --print("Wa", iD, device) + --for k,v in pairs(device)do print("W",k,v)end + -- TODO how to access 'device.type'? + local hostname , eddieName , lastSeen + = device.hostname:value(), device.eddieName:value(), device.lastSeen:value() + local typ + if false then + elseif hostname:find("^eddie%d%d%d%d%d$") then + typ = "EDDIE" + elseif hostname:find("^fook%-[a-z0-9]+$") then + typ = "FOOK" + elseif hostname:find("^lunkwill%-[a-z0-9]+$") then + typ = "LUNKWILL" + elseif hostname:find("^fook$") then + log:write("[WARN ] WTF?!? '"..hostname.."'\n") + typ = false + else error("TODO_359zh8i3wjho "..hostname) end + table.insert(app.devices, objectSeal{ + hostname = hostname, + eddieName = eddieName, + type = typ, + lastSeen = lastSeen, + }) + end + log:write("[INFO ] Fetched ".. #app.devices .." devices.\n") +end + + +function run( app ) + fetchDevices(app) + dropDevicesRecentlySeen(app) + --sortDevicesMostRecentlySeenFirst(app) + doWhateverWithDevices(app) +end + + +function main() + local app = objectSeal{ + isHelp = false, + backendHost = false, + backendPort = false, + sshPort = false, + sshUser = false, + statePath = false, + stateDb = false, + http = newHttpClient{}, + devices = false, + } + if parseArgs(app) ~= 0 then os.exit(1) end + if app.isHelp then printHelp() return end + run(app) +end + + +startOrExecute(main) + + diff --git a/src/main/lua/paisa-fleet/RmArtifactBaseDir.lua b/src/main/lua/paisa-fleet/RmArtifactBaseDir.lua new file mode 100644 index 0000000..949d1fe --- /dev/null +++ b/src/main/lua/paisa-fleet/RmArtifactBaseDir.lua @@ -0,0 +1,381 @@ + +local SL = require("scriptlee") +local newHttpClient = SL.newHttpClient +local newShellcmd = SL.newShellcmd +local newSqlite = SL.newSqlite +local objectSeal = SL.objectSeal +local parseJSON = SL.parseJSON +local sleep = SL.posix.sleep +local startOrExecute = SL.reactor.startOrExecute +SL = nil +local log = io.stdout + + +function printHelp() + io.write("\n" + .." WARN: This is experimental.\n" + .." \n" + .." Options:\n" + .." --backendHost <inaddr> (eg \"localhost\")\n" + .." --backendPort <int> (eg 80)\n" + .." --backendPath <str> (eg \"/houston\")\n" + .." --sshPort <int> (eg 22)\n" + .." --sshUser <str> (eg \"eddieuser\")\n" + .." --state <path> (eg \"path/to/state\")\n" + .." \n" + .." --exportLatestStatus\n" + .." \n") +end + + +function parseArgs( app ) + app.backendPort = 80 + app.statePath = ":memory:" + local iA = 0 + ::nextArg:: + iA = iA + 1 + local arg = _ENV.arg[iA] + if not arg then + goto verifyResult + elseif arg == "--help" then + app.isHelp = true return 0 + elseif arg == "--backendHost" then + iA = iA + 1; arg = _ENV.arg[iA] + if not arg then log:write("EINVAL: --backendHost needs value\n")return end + app.backendHost = arg + elseif arg == "--backendPort" then + iA = iA + 1; arg = _ENV.arg[iA] + if not arg then log:write("EINVAL: --backendPort needs value\n")return end + app.backendHost = arg + elseif arg == "--backendPath" then + iA = iA + 1; arg = _ENV.arg[iA] + if not arg then log:write("EINVAL: --backendPath needs value\n")return end + app.backendPath = arg + elseif arg == "--sshPort" then + iA = iA + 1; arg = _ENV.arg[iA] + if not arg then log:write("EINVAL: --sshPort needs value\n")return end + app.sshPort = arg + elseif arg == "--sshUser" then + iA = iA + 1; arg = _ENV.arg[iA] + if not arg then log:write("EINVAL: --sshUser needs value\n")return end + app.sshUser = arg + elseif arg == "--state" then + iA = iA + 1; arg = _ENV.arg[iA] + if not arg then log:write("EINVAL: --state needs value\n")return end + app.statePath = arg + elseif arg == "--exportLatestStatus" then + app.exportLatestStatus = true + else + log:write("EINVAL: ".. arg .."\n")return + end + goto nextArg + ::verifyResult:: + if app.exportLatestStatus then + if not app.statePath then log:write("EINVAL: --state missing\n")return end + else + if not app.backendHost then log:write("EINVAL: --backendHost missing\n")return end + if not app.backendPath then log:write("EINVAL: --backendPath missing\n")return end + if app.backendPath:find("^C:.") then log:write("[WARN ] MSYS_NO_PATHCONV=1 likely missing? ".. app.backendPath.."\n") end + end + return 0 +end + + +function removeCompletedEddies( app ) + local db = getStateDb(app) + local rs = db:prepare("SELECT eddieName FROM Eddie" + .." JOIN EddieLog ON Eddie.id = eddieId" + .." WHERE status = \"OK\";"):execute() + local eddieNamesToRemoveSet = {} + while rs:next() do + assert(rs:type(1) == "TEXT", rs:type(1)) + assert(rs:name(1) == "eddieName", rs:name(1)) + local eddieName = rs:value(1) + eddieNamesToRemoveSet[eddieName] = true + end + local oldEddies = app.eddies + app.eddies = {} + local numKeep, numDrop = 0, 0 + for _, eddie in pairs(oldEddies) do + if not eddieNamesToRemoveSet[eddie.eddieName] then + --log:write("[DEBUG] Keep '".. eddie.eddieName .."'\n") + numKeep = numKeep + 1 + table.insert(app.eddies, eddie) + else + numDrop = numDrop + 1 + --log:write("[DEBUG] Drop '".. eddie.eddieName .."': Already done\n") + end + end + log:write("[DEBUG] todo: ".. numKeep ..", done: ".. numDrop .."\n") +end + + +function setEddieStatus( app, statusStr, eddieName, stderrStr, stdoutStr ) + assert(type(app) == "table") + assert(type(eddieName) == "string") + assert(statusStr == "OK" or statusStr == "ERROR") + log:write("[DEBUG] setEddieStatus(".. eddieName ..", ".. statusStr ..")\n") + local db = getStateDb(app) + local stmt = db:prepare("INSERT INTO Eddie(eddieName)VALUES($eddieName);") + stmt:bind("$eddieName", eddieName) + local ok, emsg = xpcall(function() + stmt:execute() + end, debug.traceback) + if not ok and not emsg:find("UNIQUE constraint failed: Eddie.eddieName") then + error(emsg) + end + local stmt = db:prepare("INSERT INTO EddieLog('when',eddieId,status,stderr,stdout)" + .."VALUES($when, (SELECT rowid FROM Eddie WHERE eddieName = $eddieName), $status, $stderr, $stdout)") + stmt:reset() + stmt:bind("$when", os.date("!%Y-%m-%dT%H:%M:%S+00:00")) + stmt:bind("$eddieName", eddieName) + stmt:bind("$status", statusStr) + stmt:bind("$stderr", stderrStr) + stmt:bind("$stdout", stdoutStr) + stmt:execute() +end + + +function getStateDb( app ) + if not app.stateDb then + app.stateDb = newSqlite{ database = app.statePath } + app.stateDb:prepare("CREATE TABLE IF NOT EXISTS Eddie(\n" + .." id INTEGER PRIMARY KEY,\n" + .." eddieName TEXT UNIQUE NOT NULL)\n" + ..";"):execute() + app.stateDb:prepare("CREATE TABLE IF NOT EXISTS EddieLog(\n" + .." id INTEGER PRIMARY KEY,\n" + .." 'when' TEXT NOT NULL,\n" + .." eddieId INT NOT NULL,\n" + .." status TEXT, -- OneOf OK, ERROR\n" + .." stderr TEXT NOT NULL,\n" + .." stdout TEXT NOT NULL)\n" + ..";\n"):execute() + end + return app.stateDb +end + + +function loadEddies( app ) + local httpClient = newHttpClient{} + local req = objectSeal{ + base = false, + method = "GET", + path = app.backendPath .."/data/preflux/inventory", + rspCode = false, + rspBody = false, + isDone = false, + } + req.base = httpClient:request{ + cls = req, + host = app.backendHost, port = app.backendPort, + method = req.method, url = req.path, + onRspHdr = function( rspHdr, req ) + req.rspCode = rspHdr.status + if rspHdr.status ~= 200 then + log:write(".-----------------------------------------\n") + log:write("| ".. req.method .." ".. req.path .."\n") + log:write("| Host: ".. app.backendHost ..":".. app.backendPort .."\n") + log:write("+-----------------------------------------\n") + log:write("| ".. rspHdr.proto .." ".. rspHdr.status .." ".. rspHdr.phrase .."\n") + for i,h in ipairs(rspHdr.headers) do + log:write("| ".. h[1] ..": ".. h[2] .."\n") + end + log:write("| \n") + end + end, + onRspChunk = function( buf, req ) + if req.rspCode ~= 200 then log:write("| ".. buf:gsub("\n", "\n| ")) return end + if buf then + if not req.rspBody then req.rspBody = buf + else req.rspBody = req.rspBody .. buf end + end + end, + onRspEnd = function( req ) + if req.rspCode ~= 200 then log:write("\n'-----------------------------------------\n") end + req.isDone = true + end, + } + req.base:closeSnk() + assert(req.isDone) + if req.rspCode ~= 200 then log:write("ERROR: Couldn't load eddies\n")return end + local prefluxInventory = parseJSON(req.rspBody) + local eddies = {} + for eddieName, detail in pairs(prefluxInventory.hosts) do + table.insert(eddies, objectSeal{ + eddieName = eddieName, + lastSeen = detail.lastSeen:value(), + }) + end + app.eddies = eddies +end + + +function makeWhateverWithEddies( app ) + local ssh = "C:/Users/fankhauseand/.opt/gitPortable-2.27.0-x64/usr/bin/ssh.exe" + local cmdLinePre = ssh .." -oConnectTimeout=3 -oRemoteCommand=none" + if app.sshPort then cmdLinePre = cmdLinePre .." -p".. app.sshPort end + if app.sshUser then cmdLinePre = cmdLinePre .." \"-oUser=".. app.sshUser .."\"" end + for k,eddie in pairs(app.eddies) do + local eddieName = eddie.eddieName + local isEddie = eddieName:find("^eddie%d%d%d%d%d$") + local isTeddie = eddieName:find("^teddie%d%d$") + local isVted = eddieName:find("^vted%d%d$") + local isAws = eddieName:find("^10.117.%d+.%d+$") + local isDevMachine = eddieName:find("^w00[a-z0-9][a-z0-9][a-z0-9]$") + if isAws or isDevMachine or isVted then + log:write("[DEBUG] Skip \"".. eddieName .."\"\n") + goto nextEddie + end + assert(isEddie or isTeddie, eddieName or"nil") + local okMarker = "OK_".. math.random(10000000, 99999999) .."wCAkgQQA2AJAzAIA" + local cmdLine = cmdLinePre .." ".. eddieName + .." -- \"true" + .. " && if test \"".. eddieName .."\" != \"$(hostname|sed 's,.pnet.ch$,,'); then true\"" + .. " && echo WrongHost expected=".. eddieName .." actual=$(hostname|sed 's,.pnet.ch$,,') && false" + .. " ;fi" + .. " && echo hostname=$(hostname|sed 's,.pnet.ch,,')" + .. " && echo stage=${PAISA_ENV:?}" + .. " && echo Scan /data/instances/default/??ARTIFACT_BASE_DIR?" + --[[report only]] + --.. " && test -e /data/instances/default/??ARTIFACT_BASE_DIR? && ls -Ahl /data/instances/default/??ARTIFACT_BASE_DIR?" + --[[Find un-/affected eddies]] + .. " && if test -e /data/instances/default/??ARTIFACT_BASE_DIR?; then true" + .. " ;else true" + .. " && echo ".. okMarker + .. " ;fi" + --[[DELETE them]] + --.. " && if test -e /data/instances/default/??ARTIFACT_BASE_DIR?; then true" + --.. " && find /data/instances/default/??ARTIFACT_BASE_DIR? -type d -mtime +420 -print -delete" + --.. " ;fi" + --.. " && echo ".. okMarker .."" + --[[]] + .. " \"" + log:write("\n") + log:write("[INFO ] Try ".. eddieName .." ...\n") + log:write("[DEBUG] ".. cmdLine.."\n") + --log:write("[DEBUG] sleep ...\n")sleep(3) + local isStdioDone, isSuccess, stderrStr, stdoutStr = false, false, "", "" + local cmd = newShellcmd{ + cmdLine = cmdLine, + onStdout = function( buf ) + if buf then + if buf:find("\n"..okMarker.."\n",0,true) then isSuccess = true end + stdoutStr = stdoutStr .. buf + io.stdout:write(buf) + else isStdioDone = true end + end, + onStderr = function( buf ) + stderrStr = buf and stderrStr .. buf or stderrStr + io.stderr:write(buf or"") + end, + } + cmd:start() + cmd:closeSnk() + local exitCode, signal = cmd:join(42) + if exitCode ~= 0 and signal ~= nil then + log:write("[WARN ] code="..tostring(exitCode)..", signal="..tostring(signal).."\n") + end + while not isStdioDone do sleep(0.042) end + -- Analyze outcome + if not isSuccess then + setEddieStatus(app, "ERROR", eddieName, stderrStr, stdoutStr) + goto nextEddie + end + setEddieStatus(app, "OK", eddieName, stderrStr, stdoutStr) + ::nextEddie:: + end +end + + +function sortEddiesMostRecentlySeenFirst( app ) + table.sort(app.eddies, function(a, b) return a.lastSeen > b.lastSeen end) +end + + +function quoteCsvVal( v ) + local typ = type(v) + if false then + elseif typ == "string" then + if v:find("[\"\r\n]",0,false) then + v = '"'.. v:gsub('"', '""') ..'"' + end + else error("TODO_a928rzuga98oirh "..typ)end + return v +end + + +function exportLatestStatus( app ) + local snk = io.stdout + local db = getStateDb(app) + local stmt = db:prepare("SELECT \"when\",eddieName,status,stderr,stdout FROM EddieLog" + .." JOIN Eddie ON Eddie.id = eddieId" + .." ORDER BY eddieId,[when]" + .." ;") + rs = stmt:execute() + snk:write("c;when;eddieName;status;stderr;stdout\n") + local prevWhen, prevEddieName, prevStatus, prevStderr, prevStdout + local qt = quoteCsvVal + while rs:next() do + local when , eddieName , status , stderr , stdout + = rs:value(1), rs:value(2), rs:value(3), rs:value(4), rs:value(5) + --log:write("[DEBUG] "..tostring(when).." "..tostring(eddieName).." "..tostring(status).."\n") + assert(when and eddieName and status and stderr and stdout) + if eddieName == prevEddieName then + if not prevWhen or when > prevWhen then + --log:write("[DEBUG] ".. when .." ".. eddieName .." take\n") + goto assignPrevThenNextEntry + else + --log:write("[DEBUG] ".. when .." ".. eddieName .." obsolete\n") + goto nextEntry + end + elseif prevEddieName then + --log:write("[DEBUG] ".. when .." ".. eddieName .." Eddie complete\n") + snk:write("r;".. qt(when) ..";".. qt(eddieName) ..";".. qt(status) ..";".. qt(stderr) ..";".. qt(stdout) .."\n") + else + --log:write("[DEBUG] ".. when .." ".. eddieName .." Another eddie\n") + goto assignPrevThenNextEntry + end + ::assignPrevThenNextEntry:: + --[[]] prevWhen, prevEddieName, prevStatus, prevStderr, prevStdout + = when , eddieName , status , stderr , stdout + ::nextEntry:: + end + snk:write("t;status;OK\n") +end + + +function run( app ) + if app.exportLatestStatus then + exportLatestStatus(app) + return + end + loadEddies(app) + assert(app.eddies) + removeCompletedEddies(app) + sortEddiesMostRecentlySeenFirst(app) + makeWhateverWithEddies(app) +end + + +function main() + local app = objectSeal{ + isHelp = false, + backendHost = false, + backendPort = false, + backendPath = false, + sshPort = false, + sshUser = false, + statePath = false, + stateDb = false, + exportLatestStatus = false, + eddies = false, + } + if parseArgs(app) ~= 0 then os.exit(1) end + if app.isHelp then printHelp() return end + run(app) +end + + +startOrExecute(main) + diff --git a/src/main/lua/paisa-jvm-memLeak/LogStatistics.lua b/src/main/lua/paisa-jvm-memLeak/LogStatistics.lua new file mode 100644 index 0000000..cbd84b2 --- /dev/null +++ b/src/main/lua/paisa-jvm-memLeak/LogStatistics.lua @@ -0,0 +1,112 @@ + +local newLogParser = require("PaisaLogParser").newLogParser + +local inn, out, log = io.stdin, io.stdout, io.stderr + +local main, printHelp, parseArgs, run, onLogEntry, printStats + + +function printHelp( app ) + io.stdout:write(" \n" + .." TODO write help page\n" + .." \n") +end + + +function parseArgs( app ) + local arg = _ENV.arg[1] + if arg == "--help" then app.isHelp = true return 0 end + if arg ~= "--yolo" then log:write("EINVAL\n")return end + return 0 +end + + +function onLogEntry( entry, app ) + local isTheEntryWeReSearching = false + -- HOT! + --or (entry.file == "ContextImpl" and entry.msg:find("IllegalStateException: null")) + -- HOT! + or (entry.file == "HttpHeaderUtil" and entry.msg:find("Keep.Alive. values do not match timeout.42 .. timeout.120 for request ")) + -- HOT! + --or (entry.msg:find("timetable")) + -- nope + --or (entry.file == "ContextImpl" and entry.msg:find("IllegalStateException: You must set the Content%-Length header")) + -- nope + --or (entry.file == "LocalHttpServerResponse" and entry.msg:find("non-proper HttpServerResponse occured", 0, true)) + -- TODO + local instantKey = entry.date + local instant = app.instants[instantKey] + if not instant then + instant = { + date = entry.date, + count = 0, + } + app.instants[instantKey] = instant + end + if isTheEntryWeReSearching then + instant.count = instant.count + 1 + end +end + + +function printStats( app ) + -- Arrange data + local numGroups = 0 + local groupSet = {} + local countMax = 1 + for date, instant in pairs(app.instants) do + assert(date == instant.date) + local key = date:sub(1, 15) + local group = groupSet[key] + if not group then + numGroups = numGroups + 1 + group = { key = key, date = date, count = 0, } + groupSet[key] = group + end + group.count = group.count + instant.count + if countMax < group.count then countMax = group.count end + end + local groupArr = {} + for _, group in pairs(groupSet) do + table.insert(groupArr, group) + end + table.sort(groupArr, function( a, b )return a.key < b.key end) + -- Plot + out:write("\n") + out:write(string.format(" Splitted into %9d groups\n", numGroups)) + out:write(string.format(" Peak value %9d num log entries\n", countMax)) + out:write("\n") + local fullBar = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + for _, group in pairs(groupArr) do + out:write(string.format("%s... |", group.key)) + local len = math.floor(group.count / countMax * fullBar:len()) + out:write(fullBar:sub(1, len)) + out:write("\n") + end +end + + +function run( app ) + app.logParser = newLogParser{ + cls = app, + patternV1 = "DATE STAGE SERVICE LEVEL FILE - MSG", + onLogEntry = onLogEntry, + } + app.logParser:tryParseLogs() + printStats(app) +end + + +function main() + local app = { + isHelp = false, + logParser = false, + instants = {}, + } + if parseArgs(app) ~= 0 then os.exit(1) end + if app.isHelp then printHelp() return end + run(app) +end + + +main() diff --git a/src/main/lua/paisa-jvm-memLeak/MemLeakTry1.lua b/src/main/lua/paisa-jvm-memLeak/MemLeakTry1.lua new file mode 100644 index 0000000..b17c00f --- /dev/null +++ b/src/main/lua/paisa-jvm-memLeak/MemLeakTry1.lua @@ -0,0 +1,235 @@ + +local inn, out, log = io.stdin, io.stdout, io.stderr +local main, parseArgs, printHelp, run, runAsPipe, runWithStdinFilelist + + +function printHelp() + io.stdout:write(" \n" + .." Try to get some useful data out of a 'smap' dump.\n" + .." \n" + .." Options:\n" + .." \n" + .." --yolo\n" + .." WARN: Only use if you know what you do.\n" + .." \n" + .." --stdin-filelist\n" + .." Read LF separated file list form stdin.\n" + .." \n") +end + + +function parseArgs( app ) + if #_ENV.arg == 0 then log:write("EINVAL: Try --help\n") return end + app.isHelp = false + local isYolo = false + local iA = 0 + while true do iA = iA + 1 + local arg = _ENV.arg[iA] + if not arg then + break + elseif arg == "--help" then + app.isHelp = true; return 0 + elseif arg == "--yolo" then + isYolo = true + elseif arg == "--date" then + iA = iA + 1 + app.dateStr = _ENV.arg[iA] + if not app.dateStr then log:write("EINVAL: --date needs value\n") return end + elseif arg == "--stdin-filelist" then + app.isStdinFilelist = true + else + log:write("EINVAL: ".. arg .."\n") return + end + end + return 0 +end + + +function runAsPipe( app ) + local iLine = 0 + if #app.whitelist > 0 then + log:write("[INFO ] Filtering enabled\n") + end + local isHdrWritten = false + while true do + iLine = iLine + 1 + local buf = inn:read("l") + if iLine == 1 then goto nextLine end + --log:write("BUF: ".. buf .."\n") + local addr, sz, perm, note = buf:match("^([%w]+) +(%d+[A-Za-z]?) ([^ ]+) +(.*)$") + if not sz and buf:find("^ +total +%d+[KMGTPE]$") then break end + if not sz then log:write("BUF: '"..tostring(buf).."'\n")error("TODO_20231103111415") end + if sz:find("K$") then sz = sz:gsub("K$", "") * 1024 end + if #app.whitelist > 0 then + if not whitelist[addr] then goto nextLine end + end + if not isHdrWritten then + isHdrWritten = true + out:write("c; Addr ; Size ; Perm ; Note ; arg.date\n") + end + out:write(string.format("r; %s ; %12d ; %s ; %-12s ; %s\n", addr, sz, perm, note, (app.dateStr or""))) + ::nextLine:: + end +end + + +function debugPrintRecursive( out, obj, prefix, isSubCall ) + local typ = type(obj) + if false then + elseif typ == "string" then + out:write("\"") out:write((obj:gsub("\n", "\\n"):gsub("\r", "\\r"))) out:write("\"") + elseif typ == "number" then + out:write(obj) + elseif typ == "nil" then + out:write("nil") + elseif typ == "table" then + local subPrefix = (prefix)and(prefix.." ")or(" ") + for k, v in pairs(obj) do + out:write("\n") out:write(prefix or "") + debugPrintRecursive(out, k, prefix, true) out:write(": ") + debugPrintRecursive(out, v, subPrefix, true) + end + else + error(tostring(typ)) + end + if not isSubCall then out:write("\n")end +end + + +function runWithStdinFilelist( app ) + while true do + local srcFilePath = inn:read("l") + if not srcFilePath then break end + --log:write("[DEBUG] src file \"".. srcFilePath .."\"\n") + local srcFile = io.open(srcFilePath, "rb") + if not srcFile then error("fopen(\""..tostring(srcFilePath).."\")") end + collectData(app, srcFile, srcFilePath) + end + removeUnchanged(app) + printResult(app) +end + + +function collectData( app, src, timestamp ) + assert(src) + assert(timestamp) + local iLine = 0 + while true do + iLine = iLine + 1 + local buf = src:read("l") + if iLine == 1 then goto nextLine end + local addr, sz, perm, note = buf:match("^([%w]+) +(%d+[A-Za-z]?) ([^ ]+) +(.*)$") + if not sz and buf:find("^ +total +%d+[A-Za-z]?\r?$") then break end + if not sz then log:write("[ERROR] BUF: '"..tostring(buf).."'\n")error("TODO_20231103111415") end + if sz:find("K$") then sz = sz:gsub("K$", "") * 1024 end + local addrObj = app.addrs[addr] + if not addrObj then + addrObj = { measures = {} } + app.addrs[addr] = addrObj + end + local measure = { ts = timestamp, sz = sz, } + assert(not addrObj.measures[timestamp]) + addrObj.measures[timestamp] = measure + ::nextLine:: + end +end + + +function removeUnchanged( app ) + local addrsWhichHaveChanged = {} + local knownSizes = {} + for addr, addrObj in pairs(app.addrs) do + for ts, measure in pairs(addrObj.measures) do + local knownSizeKey = assert(addr) + local knownSize = knownSizes[knownSizeKey] + if not knownSize then + knownSize = measure.sz; + knownSizes[knownSizeKey] = knownSize + elseif knownSize ~= measure.sz then + addrsWhichHaveChanged[addr] = true + end + end + end + local newAddrs = {} + for addr, addrObj in pairs(app.addrs) do + if addrsWhichHaveChanged[addr] then + newAddrs[addr] = addrObj + end + end + app.addrs = newAddrs +end + + +function printResult( app ) + -- arrange data + local addrSet, tsSet, szByAddrAndTs = {}, {}, {} + for addr, addrObj in pairs(app.addrs) do + local measures = assert(addrObj.measures) + addrSet[addr] = true + for ts, measure in pairs(measures) do + assert(ts == measure.ts) + local sz = measure.sz + tsSet[ts] = true + szByAddrAndTs[addr.."\0"..ts] = sz + end + end + local addrArr, tsArr = {}, {} + for k,v in pairs(addrSet)do table.insert(addrArr, k) end + for k,v in pairs(tsSet)do table.insert(tsArr, k) end + table.sort(addrArr, function( a, b )return a < b end) + table.sort(tsArr, function( a, b )return a < b end) + -- + out:write("c;file") + for _, addr in ipairs(addrArr) do out:write(";".. addr) end + out:write("\n") + for iTs, ts in ipairs(tsArr) do + out:write("r;".. filterTsForOutput(app, ts)) + for iAddr, addr in ipairs(addrArr) do + local sz = szByAddrAndTs[assert(addr).."\0"..assert(ts)] + out:write(";".. sz) + end + out:write("\n") + end +end + + +function filterTsForOutput( app, ts ) + local y, mnth, d, h, min, sec = ts:match("^houston%-prod%-pmap%-(%d%d%d%d)(%d%d)(%d%d)%-(%d%d)(%d%d)(%d%d).txt$") + return "".. os.time{ year=y, month=mnth, day=d, hour=h, min=min, sec=sec, } +end + + +function sortedFromMap( map, smallerPredicate ) + if not smallerPredicate then smallerPredicate = function(a,b)return a.key < b.key end end + local arr = {} + for k, v in pairs(map) do table.insert(arr, {key=k, val=v}) end + table.sort(arr, smallerPredicate) + return arr +end + + +function run( app ) + if app.isStdinFilelist then + runWithStdinFilelist(app) + else + runAsPipe(app) + end +end + + +function main() + local app = { + isHelp = false, + isStdinFilelist = false, + addrs = {}, + whitelist = { + --["00000000DEADBEAF"] = true, + } + } + if parseArgs(app) ~= 0 then os.exit(1) end + if app.isHelp then printHelp() return end + run(app) +end + + +main() diff --git a/src/main/lua/paisa-logs/DigHoustonLogs.lua b/src/main/lua/paisa-logs/DigHoustonLogs.lua new file mode 100644 index 0000000..92ef035 --- /dev/null +++ b/src/main/lua/paisa-logs/DigHoustonLogs.lua @@ -0,0 +1,252 @@ +#!/usr/bin/env lua +--[====================================================================[ + + projDir='/c/path/to/proj/root' + export LUA_PATH="${projDir:?}/src/main/lua/paisa-logs/?.lua" + lua -W "${projDir:?}/src/main/lua/paisa-logs/DigHoustonLogs.lua" + + ]====================================================================] + +local PaisaLogParser = require("PaisaLogParser") +local normalizeIsoDateTime = require("PaisaLogParser").normalizeIsoDateTime +local LOGDBG = function(msg)io.stderr:write(msg)end + +local main, onLogEntry, isWorthToPrint, loadFilters, initFilters + + +function main() + local that = { + logPattern = "DATE STAGE SERVICE LEVEL FILE - MSG", -- Since 2021-09-24 on prod + printRaw = true, + filters = false, + } + loadFilters(that) + initFilters(that) + local parser = PaisaLogParser.newLogParser({ + cls = that, + patternV1 = that.logPattern, + onLogEntry = onLogEntry, + }) + parser:tryParseLogs(); +end + + +function loadFilters( that ) + assert(not that.filters) + that.filters = { + -- General: Append new rules AT END if not closely related to another one. + +-- { action = "drop", beforeDate = "2024-10-18 03:00:00.000", }, +-- { action = "drop", afterDate = "2024-01-31 23:59:59.999", }, + + { action = "drop", level = "TRACE" }, + { action = "drop", level = "DEBUG" }, + { action = "drop", level = "INFO" }, + --{ action = "drop", level = "WARN" }, + + -- FUCK those damn nonsense spam logs!!! + { action = "drop", file = "Forwarder" }, + { action = "drop", level = "ERROR", file = "HttpClientRequestImpl" }, + { action = "drop", level = "ERROR", file = "BisectClient" }, + + -- Seen: 2024-04-10 prod. + -- Reported 20240410 via "https://github.com/swisspost/vertx-redisques/pull/166" + { action = "drop", file = "RedisQues", level = "WARN", + msgPattern = "^Registration for queue .- has changed to .-$", }, + + -- Reported: SDCISA-13717 + -- Seen: 2024-01-05 prod, 2023-10-18 prod + { action = "drop", file = "LocalHttpServerResponse", level = "ERROR", + msgPattern = "^non%-proper HttpServerResponse occured\r?\n" + .."java.lang.IllegalStateException:" + .." You must set the Content%-Length header to be the total size of the message body BEFORE sending any data if you are not using" + .." HTTP chunked encoding.", }, + + -- Reported: <none> + -- Seen: 2024-01-05 prod, 2023-10-18 prod + { action = "drop", file = "ContextImpl", level = "ERROR", + msgPattern = "Unhandled exception\n" + .."java.lang.IllegalStateException: You must set the Content%-Length header to be the total size of the message body BEFORE sending" + .." any data if you are not using HTTP chunked encoding.", }, + + -- Seen: 2023-10-18 + -- Happens all the time as gateleens error reporting is broken-by-desing. + { action = "drop", file = "Forwarder", level = "WARN", + msgPattern = "^..... ................................ Problem to request /from%-houston/[0-9]+/eagle/nsync/v1/push/trillian%-phonebooks" + .."%-affiliated%-planning%-area%-[0-9]+%-vehicles: io.netty.channel.ConnectTimeoutException: connection timed out:" + .." eddie[0-9]+.pnet.ch/[0-9]+:7012", }, + -- Seen: 2023-10-18 + -- Nearly same as above but on ERROR level instead. + { action = "drop", file = "Forwarder", level = "ERROR", + msgPattern = "^%%%w+ %x+ http://eddie%d+:7012/from.houston/%d+/eagle/nsync/v1/push/trillian.phonebooks.affiliated.planning.area.%d+.vehicles" + .." The timeout period of 30000ms has been exceeded while executing POST /from.houston/%d+/eagle/nsync/v1/push/" + .."trillian.phonebooks.affiliated.planning.area.%d+.vehicles for server eddie%d+:7012", }, + -- Seen: 2023-10-18 prod + { action = "drop", file = "Forwarder", level = "ERROR", msgPattern = "^%%%w+ %x+" + .." http://localhost:9089/houston/vehicles/%d+/vehicle/backup/v1/executions/%d+/backup.zip The timeout period of 30000ms has been exceeded" + .." while executing PUT /houston/vehicles/%d+/vehicle/backup/v1/executions/%d+/backup.zip for server localhost:9089", }, + -- Seen: 2023-10-18 prod + { action = "drop", file = "Forwarder", level = "ERROR", msgPattern = "^%%%w+ %x+" + .." http://localhost:9089/houston/vehicles/%d+/vehicle/backup/v1/executions/%d+/backup.zip Timeout$" }, + + -- Seen: 2024-04-10 prod, 2023-10-18 prod + { action = "drop", file = "ConnectionBase", level = "ERROR", msgEquals = "Connection reset by peer", }, + + -- Seen: 2024-04-10 prod, 2023-10-18 prod + { action = "drop", file = "EventBusBridgeImpl", level = "ERROR", msgEquals = "SockJSSocket exception\nio.vertx.core.VertxException: Connection was closed", }, + + -- Seen: 2024-04-10 prod, 2024-01-05 prod, 2023-10-18 prod + -- Reported: TODO link existing issue here + { action = "drop", file = "HttpHeaderUtil", level = "ERROR", + msgPattern = "Keep%-Alive%} values do not match timeout=42 != timeout=120 for request /googleplex/.*", }, + + -- Seen: 2024-01-05 prod + -- Reported: <unknown> + { action = "drop", file = "Utils", level = "ERROR", + msgPattern = "^Exception occurred\njava.lang.Exception: %(TIMEOUT,%-1%) Timed out after waiting 30000%(ms%) for a reply. address: __vertx.reply.%d+, repliedAddress: nsync%-[re]+gister%-sync", + stackPattern = "^" + .."%s-at org.swisspush.nsync.NSyncHandler.lambda.onPutClientSyncBody.%d+" + .."%(NSyncHandler.java:%d+%) ..nsync.-at io.vertx.core.impl.future.FutureImpl.%d+.onFailure%(FutureImpl.java:%d+%)" + ..".-" + .."Caused by: io.vertx.core.eventbus.ReplyException: Timed out after waiting 30000%(ms%) for a reply." + .." address: __vertx.reply.%d+, repliedAddress: nsync%-[re]+gister%-sync" + }, + + -- WELL_KNOWN: I guess happens when vehicle looses connection. Seen 2023-10-18 prod. + { action = "drop", file = "Forwarder", level = "ERROR", msgPattern = "^%%%w+ %x+" + .." http://eddie%d+:7012/from.houston/%d+/eagle/vending/accounting/v1/users/%d+/years/%d+/months/%d%d/account Connection was closed$", }, + -- WELL_KNOWN: I guess happens when vehicle looses connection. Seen 2023-10-18 prod. + { action = "drop", file = "Forwarder", level = "ERROR", msgPattern = "^%%%w+ %x+" + .." http://eddie%d+:7012/from.houston/%d+/eagle/nsync/v1/push/trillian.phonebooks.affiliated.planning.area.%d+.vehicles Connection was closed$", }, + -- Seen 2024-01-10 prod + -- WELL_KNOWN: I guess happens when vehicle looses connection. Seen 2023-10-18 prod. + { action = "drop", file = "Forwarder", level = "ERROR", msgPattern = "^%%%w+ %x+" + .." http://eddie%d+:7012/from.houston/%d+/eagle/nsync/v1/query.index The timeout period of 30000ms has been exceeded while executing" + .." POST /from.houston/%d+/eagle/nsync/v1/query-index for server eddie%d+:7012$", }, + -- WELL_KNOWN: I guess happens when vehicle looses connection. Seen 2023-10-18 prod. + { action = "drop", file = "Forwarder", level = "ERROR", msgPattern = "^%%%w+ %x+" + .." http://eddie%d+:7012/from.houston/%d+/eagle/timetable/notification/v1/planningareas/%d+/notifications/%x+ Connection was closed$", }, + -- WELL_KNOWN: I guess happens when vehicle looses connection. Seen 2023-10-18 prod. + { action = "drop", file = "Forwarder", level = "ERROR", msgPattern = "^%%%w+ %x+ http://eddie%d+:7012/from.houston/%d+/eagle/nsync/v1/push/trillian.phonebooks.affiliated.planning.area.%d+.vehicles Connection reset by peer$", }, + + -- Reported: SDCISA-9574 + -- TODO rm when resolved + -- Seen: 2021-09-17 2022-06-20, 2022-08-30 prod, + { action = "drop", file = "Utils", level = "ERROR", + msgPattern = "%(RECIPIENT_FAILURE,500%) Sync failed.\n{.+}", }, + + -- TODO analyze + -- Seen 2024-03-20 prod + { action = "drop", file = "ContextImpl", level = "ERROR", + msgPattern = "^Unhandled exception\njava.lang.IllegalStateException: Response head already sent", }, + + -- Seen: 2024-04-10 prod. + { action = "drop", level = "ERROR", file = "HttpClientRequestImpl", + msgEquals = "Connection reset by peer\njava.io.IOException: Connection reset by peer", + stackPattern = "^" + .."%s-at sun.nio.ch.FileDispatcherImpl.read0%(.-\n" + .."%s-at sun.nio.ch.SocketDispatcher.read%(.-\n" + .."%s-at sun.nio.ch.IOUtil.readIntoNativeBuffer%(.-\n" + .."%s-at sun.nio.ch.IOUtil.read%(.-\n" + .."%s-at sun.nio.ch.IOUtil.read%(.-\n" + .."%s-at sun.nio.ch.SocketChannelImpl.read%(.-\n" + .."%s-at io.netty.buffer.PooledByteBuf.setBytes%(.-\n" + .."%s-at io.netty.buffer.AbstractByteBuf.writeBytes%(.-\n" + .."%s-at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes%(.-\n" + .."%s-at io.netty.channel.nio.AbstractNioByteChannel.NioByteUnsafe.read%(.-\n" + .."%s-at io.netty.channel.nio.NioEventLoop.processSelectedKey%(.-\n" + .."%s-at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized%(.-\n" + .."%s-at io.netty.channel.nio.NioEventLoop.processSelectedKeys%(.-\n" + .."%s-at io.netty.channel.nio.NioEventLoop.run%(.-\n" + .."%s-at io.netty.util.concurrent.SingleThreadEventExecutor.%d+.run%(.-\n" + .."%s-at io.netty.util.internal.ThreadExecutorMap.%d+.run%(.-\n" + .."%s-at io.netty.util.concurrent.FastThreadLocalRunnable.run%(.-\n" + .."%s-at java.lang.Thread.run%(.-", }, + + -- Seen: 2024-04-10 prod. + { action = "drop", file = "ContextImpl", level = "ERROR", + msgEquals = "Unhandled exception\njava.lang.IllegalStateException: null", + stackPattern = "^" + ..".-io.vertx.-%.HttpClientResponseImpl.checkEnded%(.-\n" + ..".-io.vertx.-%.HttpClientResponseImpl.endHandler%(.-\n" + ..".-gateleen.routing.Forwarder.-\n", }, + + -- Seen: 2024-04-10 prod. + -- TODO get rid of this silly base class. + { action = "drop", file = "ContextImpl", level = "ERROR", + msgEquals = "Unhandled exception\njava.lang.UnsupportedOperationException: Do override this method to mock expected behaviour.", }, + + -- Seen: 2024-04-10 prod. + -- TODO get rid of this silly base class. + { action = "drop", file = "ContextImpl", level = "ERROR", + msgEquals = "Unhandled exception\njava.lang.UnsupportedOperationException: null", }, + + } +end + + +function initFilters( that ) + for iF = 1, #(that.filters) do + local descr = that.filters[iF] + local beforeDate = descr.beforeDate and normalizeIsoDateTime(descr.beforeDate) + local afterDate = descr.afterDate and normalizeIsoDateTime(descr.afterDate) + local file, level, msgPattern, msgEquals = descr.file, descr.level, descr.msgPattern, descr.msgEquals + local rawPattern, stackPattern = descr.rawPattern, descr.stackPattern + local stackStartsWith = descr.stackStartsWith + local filter = { action = descr.action, matches = false, } + local hasAnyCondition = (beforeDate or afterDate or file or level or msgPattern or rawPattern or stackPattern or stackStartsWith); + if not hasAnyCondition then + filter.matches = function( that, log ) --[[LOGDBG("match unconditionally\n")]] return true end + else + filter.matches = function( that, log ) + local match, mismatch = true, false + if not log.date then log:debugPrint() end + if level and level ~= log.level then --[[LOGDBG("level mismatch: \"".. level .."\" != \"".. log.level .."\"\n")]] return mismatch end + if file and file ~= log.file then --[[LOGDBG("file mismatch: \"".. file .."\" != \"".. log.file .."\"\n")]] return mismatch end + local logDate = normalizeIsoDateTime(log.date) + local isBeforeDate = (not beforeDate or logDate < beforeDate); + local isAfterDate = (not afterDate or logDate >= afterDate); + if not isBeforeDate then --[[LOGDBG("not before: \"".. tostring(beforeDate) .."\", \"".. logDate .."\"\n")]] return mismatch end + if not isAfterDate then --[[LOGDBG("not after: \"".. tostring(afterDate) .."\", \"".. logDate .."\"\n")]] return mismatch end + if msgEquals and log.msg ~= msgEquals then return mismatch end + if stackStartsWith and log.stack and log.stack:sub(1, #stackStartsWith) ~= stackStartsWith then return mismatch end + if msgPattern and not log.msg:find(msgPattern) then --[[LOGDBG("match: msgPattern\n")]] return mismatch end + if stackPattern and log.stack and not log.stack:find(stackPattern) then return mismatch end + if rawPattern and not log.raw:find(rawPattern) then return mismatch end + --LOGDBG("DEFAULT match\n") + return match + end + end + that.filters[iF] = filter + end +end + + +function onLogEntry( log, that ) + local isWorthIt = isWorthToPrint(that, log) + if isWorthIt then + if that.printRaw then + print(log.raw) + else + log:debugPrint() + end + end +end + + +function isWorthToPrint( that, log ) + local pass, drop = true, false + for iF = 1, #(that.filters) do + local filter = that.filters[iF] + if filter.matches(that, log) then + if filter.action == "drop" then return drop end + if filter.action == "keep" then return pass end + error("Unknown filter.action: \"".. filter.action .."\""); + end + end + return pass +end + + +main() + diff --git a/src/main/lua/paisa-logs/PaisaLogParser.lua b/src/main/lua/paisa-logs/PaisaLogParser.lua new file mode 100644 index 0000000..f6ac0ce --- /dev/null +++ b/src/main/lua/paisa-logs/PaisaLogParser.lua @@ -0,0 +1,435 @@ + +local exports = {} +local mod = {} +local stderr = io.stderr + + +local LogParse = { -- class + line = nil, + log = nil, +} + + +function exports.newLogParser( config ) + return LogParse:new(nil, config ) +end + + +function LogParse:new(o, config) + if not config or type(config.onLogEntry) ~= "function" then + error( "Arg 'config.onLogEntry' must be a function" ) + end + o = o or {}; + setmetatable(o, self); + self.__index = self; + -- Register callbacks + self.cb_cls = config.cls + self.cb_onLogEntry = config.onLogEntry + self.cb_onEnd = config.onEnd + self.cb_onError = config.onError or function(s) + error(s or "nil") + end + self.cb_onWarn = config.onWarn or function(s) + io.stdout:flush() + warn(s) + end + -- END callbacks + mod.setupParserPattern( o, config ) + return o; +end + + +function mod.setupParserPattern( this, c ) + local inputPat + if c.patternV1 then + inputPat = c.patternV1; -- Use the one from parameter. + else + this.cb_onWarn( "No 'c.patternV1' specified. Fallback to internal obsolete one." ) + inputPat = "DATE POD STAGE SERVICE THREAD LEVEL FILE - MSG" + end + local parts = {} + for part in string.gmatch(inputPat,"[^ ]+") do + table.insert( parts, part ) + end + this.parts = parts +end + + +local function writeStderr(...) + local args = table.pack(...) + for i=1,args.n do + io.stderr:write( args[i] or "nil" ) + end +end + + +function LogParse:tryParseLogs() + while true do + self.line = io.read("l"); + if self.line==nil then -- EOF + self:publishLogEntry(); + break; + end + + --io.write( "\nBUF: ", self.line, "\n\n" ); + --io.flush() + + if self.line:match("%d%d%d%d%-%d%d%-%d%d[ T]%d%d:%d%d:%d%d,%d%d%d ") then + -- Looks like the beginning of a new log entry. + self:initLogEntryFromLine(); + elseif self.line:match("^%s+at [^ ]") then + -- Looks like a line from exception stack + self:appendStacktraceLine(); + elseif self.line:match("^%s*Caused by: ") then + -- Looks like a stacktrace 'Caused by' line + self:appendStacktraceLine(); + elseif self.line:match("^%s+Suppressed: ") then + -- Looks like a stacktrace 'Suppressed: ' line + self:appendStacktraceLine(); + elseif self.line:match("^%\t... (%d+) more$") then + -- Looks like folded stacktrace elements + self:appendStacktraceLine(); + else + -- Probably msg containing newlines. + self:appendLogMsg(); + end + + end +end + + +function LogParse:initLogEntryFromLine() + self:publishLogEntry(); + local log = self:getOrNewLogEntry(); + + -- Try some alternative parsers + mod.parseByPattern( self ) + --if log.date==nil then + -- self:parseOpenshiftServiceLogLine(); + --end + --if log.date==nil then + -- self:parseEagleLogLine(); + --end + --if log.date==nil then + -- self:parseJettyServiceLogLine(); + --end + + if log.date==nil then + self.cb_onWarn("Failed to parse log line:\n\n".. self.line .."\n\n", self.cb_cls) + end +end + + +function mod.parseByPattern( this ) + local date, pod, stage, service, thread, level, file, msg, matchr, match + local line = this.line + local log = this:getOrNewLogEntry(); + + -- We can just return on failure. if log is missing, it will report error + -- on caller side. Just ensure that 'date' is nil. + log.date = nil + + local rdPos = 1 + for i,part in ipairs(this.parts) do + if part=="DATE" then + date = line:gmatch("(%d%d%d%d%-%d%d%-%d%d[ T]%d%d:%d%d:%d%d,%d%d%d) ", rdPos)() + if not date or date=="" then return end + rdPos = rdPos + date:len() + --stderr:write("date: "..tostring(date).." (rdPos="..tostring(rdPos)..")\n") + elseif part=="STAGE" then + match = line:gmatch( " +[^%s]+", rdPos)() + if not match then return end + stage = match:gmatch("[^%s]+")() + rdPos = rdPos + match:len() + --stderr:write("stage: "..tostring(stage).." (rdPos="..tostring(rdPos)..")\n") + elseif part=="SERVICE" then + match = line:gmatch(" +[^%s]+", rdPos)() + if not match then return end + service = match:gmatch("[^%s]+")() + rdPos = rdPos + match:len() + --stderr:write("service: "..tostring(service).." (rdPos="..tostring(rdPos)..")\n"); + elseif part=="LEVEL" then + match = line:gmatch(" +[^%s]+", rdPos)() + if not match then return end + level = match:gmatch("[^%s]+")() + if not level:find("^[ABCDEFGINORTUW]+$") then -- [ABCDEFGINORTUW]+ -> (ERROR|WARN|INFO|DEBUG|TRACE) + this.cb_onWarn( "Does not look like a level: "..(level or"nil"), this.cb_cls ) + end + rdPos = rdPos + match:len() + --stderr:write("level: "..tostring(level).." (rdPos="..tostring(rdPos)..")\n"); + elseif part=="FILE" then + match = line:gmatch(" +[^%s]+", rdPos)() + if not match then return end + file = match:gmatch("[^%s]+")() + if file=="WARN" then stderr:write("\n"..tostring(line).."\n\n")error("Doesn't look like a file: "..tostring(file)) end + rdPos = rdPos + match:len() + --stderr:write("file: "..tostring(file).." (rdPos="..tostring(rdPos)..")\n"); + elseif part=="-" then + match = line:gmatch(" +%-", rdPos)() + rdPos = rdPos + match:len(); + --stderr:write("dash (rdPos="..tostring(rdPos)..")\n"); + elseif part=="MSG" then + match = line:gmatch(" +.*$", rdPos)() + if not match then return end + msg = match:gmatch("[^%s].*$")() + rdPos = rdPos + match:len() + --stderr:write("msg: "..tostring(msg).." (rdPos="..tostring(rdPos)..")\n") + elseif part=="POD" then + match = line:gmatch(" +[^%s]+", rdPos)() + if not match then return end + pod = match:gmatch("[^%s]+")() + rdPos = rdPos + match:len() + --stderr:write("pod: "..tostring(pod).." (rdPos="..tostring(rdPos)..")\n") + elseif part=="THREAD" then + match = line:gmatch(" +[^%s]+", rdPos)() + thread = match:gmatch("[^%s]+")() + rdPos = rdPos + match:len() + --stderr:write("thrd: "..tostring(thread).." (rdPos="..tostring(rdPos)..")\n") + end + end + + log.raw = this.line; + log.date = date; + log.pod = pod; + log.stage = stage; + log.service = service; + log.thread = thread; + log.level = level; + log.file = file; + log.msg = msg; +end + + +function LogParse:parseOpenshiftServiceLogLine() + local date, pod, stage, service, thread, level, file, msg + local this = self + local line = this.line + local log = self:getOrNewLogEntry(); + + -- We can just return on failure. if log is missing, it will report error + -- on caller side. Just ensure that 'date' is nil. + log.date = nil + + -- VERSION 3 (Since 2021-09-24 houstonProd) + local rdPos = 1 + -- Date + date = line:gmatch("(%d%d%d%d%-%d%d%-%d%d[ T]%d%d:%d%d:%d%d,%d%d%d)", rdPos)() + if not date then return end + rdPos = rdPos + date:len() + -- Pod + pod = line:gmatch(" (%a+)", rdPos )() + if not pod then return end + rdPos = rdPos + pod:len() + -- stage + stage = line:gmatch( " (%a+)", rdPos)() + if not stage then return end + rdPos = rdPos + stage:len() + -- service + service = line:gmatch( " (%a+)", rdPos)() + if not service then return end + rdPos = rdPos + service:len() + -- thread (this only maybe exists) + thread = line:gmatch( " ([%a%d%-]+)", rdPos)() + -- [ABCDEFGINORTUW]+ -> (ERROR|WARN|INFO|DEBUG|TRACE) + if thread and thread:find("^[ABCDEFGINORTUW]+$") then + thread = nil; -- Does more look like an error level. So do NOT advance + else + rdPos = rdPos + thread:len() + end + -- level + level = line:gmatch( " ([A-Z]+)", rdPos)() + if not level then return end + rdPos = rdPos + level:len() + -- file + file = line:gmatch(" ([^%s]+)", rdPos)() + if not file then return end + rdPos = rdPos + file:len() + -- msg + msg = line:gmatch(" %- (.*)", rdPos)() + if not msg then return end + rdPos = rdPos + msg:len() + + -- VERSION 2 (Since 2021-09-24 prefluxInt) + --local rdPos = 1 + ---- Date + --date = line:gmatch("(%d%d%d%d%-%d%d%-%d%d[ T]%d%d:%d%d:%d%d,%d%d%d)", rdPos)() + --if not date then return end + --rdPos = rdPos + date:len() + ---- Pod + --pod = line:gmatch(" (%a+)", rdPos )() + --if not pod then return end + --rdPos = rdPos + pod:len() + ---- stage + --stage = line:gmatch( " (%a+)", rdPos)() + --if not stage then return end + --rdPos = rdPos + stage:len() + ---- service + --service = line:gmatch( " (%a+)", rdPos)() + --if not service then return end + --rdPos = rdPos + service:len() + ---- thread (this only maybe exists) + --thread = line:gmatch( " ([%a%d%-]+)", rdPos)() + ---- [ABCDEFGINORTUW]+ -> (ERROR|WARN|INFO|DEBUG|TRACE) + --if thread and thread:find("^[ABCDEFGINORTUW]+$") then + -- thread = nil; -- Does more look like an error level. So do NOT advance + --else + -- rdPos = rdPos + thread:len() + --end + ---- level + --level = line:gmatch( " ([A-Z]+)", rdPos)() + --if not level then return end + --rdPos = rdPos + level:len() + ---- file + --file = line:gmatch(" ([^%s]+)", rdPos)() + --if not file then return end + --rdPos = rdPos + file:len() + ---- msg + --msg = line:gmatch(" %- (.*)", rdPos)() + --if not msg then return end + --rdPos = rdPos + msg:len() + + log.raw = self.line; + log.date = date; + log.pod = pod; + log.stage = stage; + log.service = service; + log.thread = thread; + log.level = level; + log.file = file; + log.msg = msg; +end + + +function LogParse:parseEagleLogLine() + local log = self:getOrNewLogEntry(); + local date, stage, service, level, file, msg = self.line:gmatch("" + .."(%d%d%d%d%-%d%d%-%d%d %d%d:%d%d:%d%d,%d%d%d)" -- datetime + .." (%a+)" -- stage + .." (%a+)" -- service + .." (%a+)" -- level + .." ([^%s]+)" -- file + .." %- (.*)" -- msg + )(); + local pod = service; -- just 'mock' it + log.raw = self.line; + log.date = date; + log.service = service; + log.pod = pod; + log.stage = stage; + log.level = level; + log.file = file; + log.msg = msg; +end + + +function LogParse:parseJettyServiceLogLine() + local log = self:getOrNewLogEntry(); + local date, pod, stage, service, level, file, msg = self.line:gmatch("" + .."(%d%d%d%d%-%d%d%-%d%d %d%d:%d%d:%d%d,%d%d%d)" -- datetime + .." (%S+)" -- pod (aka container) + .." (%a+)" -- stage + .." (%a+)" -- service + .." (%a+)" -- level + .." ([^%s]+)" -- file + .." %- (.*)" -- msg + )(); + log.raw = self.line; + log.date = date; + log.pod = pod; + log.stage = stage; + log.service = service; + log.level = level; + log.file = file; + log.msg = msg; +end + + +function LogParse:appendLogMsg() + local log = self:getOrNewLogEntry() + log.msg = log.msg or ""; + log.raw = log.raw or ""; + + log.msg = log.msg .."\n".. self.line; + -- Also append to raw to have the complete entry there. + log.raw = log.raw .."\n".. self.line; +end + + +function LogParse:appendStacktraceLine() + local log = self:getOrNewLogEntry() + if not log.stack then + log.stack = self.line + else + log.stack = log.stack .."\n".. self.line + end + -- Also append to raw to have the complete entry there. + log.raw = log.raw .."\n".. self.line; +end + + +function LogParse:publishLogEntry() + local log = self.log + if not log then + return -- nothing to do + end + if not log.raw then + -- WhatTheHeck?!? + local msg = "InternalError: Collected log unexpectedly empty" + self.cb_onError(msg, self.cb_cls) + error(msg); return + end + self.log = nil; -- Mark as consumed + -- Make sure log lines do NOT end in 0x0D + local msg = log.msg + if msg:byte(msg:len()) == 0x0D then log.msg = msg:sub(1, -2) end + self.cb_onLogEntry(log, self.cb_cls) +end + + +function LogParse:getOrNewLogEntry() + self.log = self.log or LogEntry:new(nil) + return self.log +end + + +function exports.normalizeIsoDateTime( str ) + if str:find("%d%d%d%d%-%d%d%-%d%dT%d%d:%d%d:%d%d%.%d%d%d") then return str end + local y, mo, d, h, mi, s, ms = str:match("^(%d%d%d%d)-(%d%d)-(%d%d)[ T_-](%d%d):(%d%d):(%d%d)[,.](%d%d%d)$") + return y .."-".. mo .."-".. d .."T".. h ..":".. mi ..":".. s ..".".. ms +end + + +LogEntry = { + raw, + date, + service, + stack, +} + + +function LogEntry:new(o) + o = o or {}; + setmetatable(o, self); + self.__index = self; + return o; +end + + +function LogEntry:debugPrint() + print( "+- PUBLISH ------------------------------------------------------------" ); + print( "| date ---> ", self.date or "nil" ); + print( "| pod ----> ", self.pod or "nil" ); + print( "| service > ", self.service or "nil" ); + print( "| stage --> ", self.stage or "nil" ); + print( "| thread -> ", self.thread or "nil" ); + print( "| level --> ", self.level or "nil" ); + print( "| file ---> ", self.file or "nil" ); + print( "| msg ----> ", self.msg or "nil" ); + print( "| " ) + io.write( "| RAW: ", self.raw or "nil", "\n" ); + print( "`--------------------" ); +end + + +return exports + diff --git a/src/main/lua/pcap/KubeProbeFilter.lua b/src/main/lua/pcap/KubeProbeFilter.lua new file mode 100644 index 0000000..a5967e9 --- /dev/null +++ b/src/main/lua/pcap/KubeProbeFilter.lua @@ -0,0 +1,93 @@ +-- +-- Try to extract kube-probe related requests. +-- + +local newPcapParser = assert(require("pcapit").newPcapParser) +local newPcapDumper = assert(require("pcapit").newPcapDumper) + +local out, log = io.stdout, io.stderr +local main, onPcapFrame, vapourizeUrlVariables + + +function onPcapFrame( app, it ) + local srcPort, dstPort = it:trspSrcPort(), it:trspDstPort() + local userAgent, reqUri + -- + if dstPort ~= 7012 and srcPort ~= 7012 then return end + local trspPayload = it:trspPayload() + local httpReqLinePart1, httpReqLinePart2, httpReqLinePart3 = + trspPayload:match("^([A-Z/1.0]+) ([^ ]+) ([^ \r\n]+)\r?\n") + if httpReqLinePart1 and not httpReqLinePart1:find("^HTTP/1.%d$") then -- assume HTTP request + reqUri = httpReqLinePart2 + userAgent = trspPayload:match("\n[Uu][Ss][Ee][Rr]%-[Aa][Gg][Ee][Nn][Tt]:%s+([^\r\n]+)\r?\n"); + if userAgent then + --if not userAgent:find("^kube%-probe/") then return end -- assume halfrunt + --log:write("User-Agent: ".. userAgent .."\n") + end + elseif httpReqLinePart1 then -- assume HTTP response + --out:write(trspPayload) + end + local srcIp, dstIp = it:netSrcIpStr(), it:netDstIpStr() + local connKey = ((srcPort < dstPort)and(srcPort.."\0"..dstPort)or(dstPort.."\0"..srcPort)) + .."\0"..((srcIp < dstIp)and(srcIp.."\0"..dstIp)or(dstIp.."\0"..srcIp)) + local conn = app.connections[connKey] + if not conn then conn = {isOfInterest=false, pkgs={}} app.connections[connKey] = conn end + conn.isOfInterest = (conn.isOfInterest or reqUri == "/houston/server/info") + if not conn.isOfInterest then + if #conn.pkgs > 3 then -- Throw away all stuff except TCP handshake + conn.pkgs = { conn.pkgs[1], conn.pkgs[2], conn.pkgs[3] } + end + local sec, usec = it:frameArrivalTime() + --for k,v in pairs(getmetatable(it))do print("E",k,v)end + local pkg = { + sec = assert(sec), usec = assert(usec), + caplen = it:frameCaplen(), len = it:frameLen(), + tcpFlags = (conn.isOfInterest)and(it:tcpFlags())or false, + srcPort = srcPort, dstPort = dstPort, + trspPayload = trspPayload, + rawFrame = it:rawFrame(), + } + table.insert(conn.pkgs, pkg) + else + -- Stop memory hogging. Write that stuff to output + if #conn.pkgs > 0 then + for _, pkg in ipairs(conn.pkgs) do + --out:write(string.format("-- PKG 1 %d->%d %d.%09d tcpFlg=0x%04X\n", pkg.srcPort, pkg.dstPort, pkg.sec, pkg.usec, pkg.tcpFlags or 0)) + --out:write(pkg.trspPayload) + --out:write("\n") + app.dumper:dump(pkg.sec, pkg.usec, pkg.caplen, pkg.len, pkg.rawFrame, 1, pkg.rawFrame:len()) + end + conn.pkgs = {} + end + local tcpFlags = it:tcpFlags() + local sec, usec = it:frameArrivalTime() + local rawFrame = it:rawFrame() + --out:write(string.format("-- PKG 2 %d->%d %d.%09d tcpFlg=0x%04X, len=%d\n", srcPort, dstPort, sec, usec, tcpFlags or 0, trspPayload:len())) + --out:write(trspPayload) + --if trspPayload:byte(trspPayload:len()) ~= 0x0A then out:write("\n") end + --out:write("\n") + app.dumper:dump(sec, usec, it:frameCaplen(), it:frameLen(), rawFrame, 1, rawFrame:len()) + end +end + + +function main() + local app = { + parser = false, + dumper = false, + connections = {}, + } + app.parser = newPcapParser{ + dumpFilePath = "-", + onFrame = function(f)onPcapFrame(app, f)end, + } + app.dumper = newPcapDumper{ + dumpFilePath = "C:/work/tmp/KubeProbeFilter.out.pcap", + } + app.parser:resume() +end + + +main() + + diff --git a/src/main/lua/pcap/extractDnsHosts.lua b/src/main/lua/pcap/extractDnsHosts.lua new file mode 100644 index 0000000..655586f --- /dev/null +++ b/src/main/lua/pcap/extractDnsHosts.lua @@ -0,0 +1,147 @@ + +local newPcapParser = assert(require("pcapit").newPcapParser) +local out, log = io.stdout, io.stderr + +local main, onPcapFrame, vapourizeUrlVariables, printResult + + +function main() + local app = { + parser = false, + youngestEpochSec = -math.huge, + oldestEpochSec = math.huge, + dnsResponses = {}, + } + app.parser = newPcapParser{ + dumpFilePath = "-", + onFrame = function(f)onPcapFrame(app, f)end, + } + app.parser:resume() + printResult(app) +end + + +function onPcapFrame( app, it ) + local out = io.stdout + local sec, usec = it:frameArrivalTime() + sec = sec + (usec/1e6) + if sec < app.oldestEpochSec then app.oldestEpochSec = sec end + if sec > app.youngestEpochSec then app.youngestEpochSec = sec end + -- + if it:trspSrcPort() == 53 then + extractHostnameFromDns(app, it) + elseif it:tcpSeqNr() then + extractHostnameFromHttpHeaders(app, it) + end +end + + +function extractHostnameFromDns( app, it ) + local payload = it:trspPayload() + local bug = 8 -- TODO looks as lib has a bug and payload is offset by some bytes. + local dnsFlags = (payload:byte(bug+3) << 8) | (payload:byte(bug+4)) + if (dnsFlags & 0x0004) ~= 0 then return end -- ignore error responses + local numQuestions = payload:byte(bug+5) << 8 | payload:byte(bug+6) + local numAnswers = payload:byte(bug+7) << 8 | payload:byte(bug+8) + if numQuestions ~= 1 then + log:write("[WARN ] numQuestions ".. numQuestions .."?!?\n") + return + end + if numAnswers == 0 then return end -- empty answers are boring + if numAnswers ~= 1 then log:write("[WARN ] dns.count.answers ".. numAnswers .." not supported\n") return end + local questionsOffset = bug+13 + local hostname = payload:match("^([^\0]+)", questionsOffset) + hostname = hostname:gsub("^[\r\n]", "") -- TODO WTF?!? + hostname = hostname:gsub("[\x04\x02]", ".") -- TODO WTF?!? + local answersOffset = bug + 13 + (24 * numQuestions) + local ttl = payload:byte(answersOffset+6) << 24 | payload:byte(answersOffset+7) << 16 + | payload:byte(answersOffset+8) << 8 | payload:byte(answersOffset+9) + local dataLen = payload:byte(answersOffset+10) | payload:byte(answersOffset+11) + if dataLen ~= 4 then log:write("[WARN ] dns.resp.len ".. dataLen .." not impl\n") return end + local ipv4Str = string.format("%d.%d.%d.%d", payload:byte(answersOffset+12), payload:byte(answersOffset+13), + payload:byte(answersOffset+14), payload:byte(answersOffset+15)) + -- + addEntry(app, ipv4Str, hostname, ttl) +end + + +function extractHostnameFromHttpHeaders( app, it ) + local payload = it:trspPayload() + local _, beg = payload:find("^([A-Z]+ [^ \r\n]+ HTTP/1%.%d\r?\n)") + if not beg then return end + beg = beg + 1 + local httpHost + while true do + local line + local f, t = payload:find("^([^\r\n]+)\r?\n", beg) + if not f then return end + if not payload:byte(1) == 0x72 or payload:byte(1) == 0x68 then goto nextHdr end + line = payload:sub(f, t) + httpHost = line:match("^[Hh][Oo][Ss][Tt]:%s*([^\r\n]+)\r?\n$") + if not httpHost then goto nextHdr end + break + ::nextHdr:: + beg = t + end + httpHost = httpHost:gsub("^(.+):%d+$", "%1") + local dstIp = it:netDstIpStr() + if dstIp == httpHost then return end + addEntry(app, dstIp, httpHost, false, "via http host header") +end + + +function addEntry( app, ipv4Str, hostname, ttl, kludge ) + local key + --log:write("addEntry(app, ".. ipv4Str ..", ".. hostname ..")\n") + if kludge == "via http host header" then + key = ipv4Str .."\0".. hostname .."\0".. "via http host header" + else + key = ipv4Str .."\0".. hostname .."\0".. ttl + end + local entry = app.dnsResponses[key] + if not entry then + entry = { ipv4Str = ipv4Str, hostname = hostname, ttl = ttl, } + app.dnsResponses[key] = entry + end +end + + +function printResult( app ) + local sorted = {} + for _, stream in pairs(app.dnsResponses) do + table.insert(sorted, stream) + end + table.sort(sorted, function(a, b) + if a.ipv4Str < b.ipv4Str then return true end + if a.ipv4Str > b.ipv4Str then return false end + return a.hostname < b.hostname + end) + local dumpDurationSec = app.youngestEpochSec - app.oldestEpochSec + local timeFmt = "!%Y-%m-%d_%H:%M:%SZ" + out:write("\n") + out:write(string.format("# Subject Hostname to IP addresses\n")) + out:write(string.format("# Begin %s\n", os.date(timeFmt, math.floor(app.oldestEpochSec)))) + out:write(string.format("# Duration %.3f seconds\n", dumpDurationSec)) + out:write("\n") + --out:write(" .-- KiB per Second\n") + --out:write(" | .-- IP endpoints\n") + --out:write(" | | .-- TCP server port\n") + --out:write(" | | | .-- TCP Payload (less is better)\n") + --out:write(" | | | |\n") + --out:write(".--+----. .----+----------------------. .+--. .-+------------\n") + for i, elem in ipairs(sorted) do + local ipv4Str, hostname, ttl = elem.ipv4Str, elem.hostname, elem.ttl + if ttl then + out:write(string.format("%-14s %-30s # TTL=%ds", ipv4Str, hostname, ttl)) + else + out:write(string.format("%-14s %-30s # ", ipv4Str, hostname)) + end + out:write("\n") + end + out:write("\n") +end + + +main() + + diff --git a/src/main/lua/pcap/httpStats.lua b/src/main/lua/pcap/httpStats.lua new file mode 100644 index 0000000..ff48bd2 --- /dev/null +++ b/src/main/lua/pcap/httpStats.lua @@ -0,0 +1,117 @@ + +local newPcapParser = assert(require("pcapit").newPcapParser) + +local out, log = io.stdout, io.stderr +local main, onPcapFrame, vapourizeUrlVariables, printHttpRequestStats + + +function main() + local app = { + parser = false, + youngestEpochSec = -math.huge, + oldestEpochSec = math.huge, + foundHttpRequests = {}, + } + app.parser = newPcapParser{ + dumpFilePath = "-", + onFrame = function(f)onPcapFrame(app, f)end, + } + app.parser:resume() + printHttpRequestStats(app) +end + + +function onPcapFrame( app, it ) + local sec, usec = it:frameArrivalTime() + local srcPort, dstPort = it:trspSrcPort(), it:trspDstPort() + -- + if sec < app.oldestEpochSec then app.oldestEpochSec = sec end + if sec > app.youngestEpochSec then app.youngestEpochSec = sec end + -- + local portOfInterest = 7012 + if dstPort == portOfInterest then + local httpMethod, httpUri = + it:trspPayload():match("^([A-Z]+) ([^ ]+) [^ \r\n]+\r?\n") + if httpMethod then + --out:write(string.format("%5d->%5d %s %s\n", srcPort, dstPort, httpMethod, httpUri)) + httpUri = vapourizeUrlVariables(app, httpUri) + local key = httpUri -- httpMethod .." ".. httpUri + local obj = app.foundHttpRequests[key] + if not obj then + obj = { count=0, httpMethod=false, httpUri=false, } + app.foundHttpRequests[key] = obj + end + obj.count = obj.count + 1 + obj.httpMethod = httpMethod + obj.httpUri = httpUri + end + elseif srcPort == portOfInterest then + local httpStatus, httpPhrase = + it:trspPayload():match("^HTTP/%d.%d (%d%d%d) ([^\r\n]*)\r?\n") + if httpStatus then + --out:write(string.format("%5d<-%5d %s %s\n", srcPort, dstPort, httpStatus, httpPhrase)) + end + end +end + + +function vapourizeUrlVariables( app, uri ) + -- A very specific case + uri = uri:gsub("^(/houston/users/)%d+(/.*)$", "%1{}%2"); + if uri:find("^/houston/users/[^/]+/user/.*$") then return uri end + -- + -- Try to do some clever guesses to group URIs wich only differ in variable segments + uri = uri:gsub("(/|-)[%dI_-]+/", "%1{}/"):gsub("(/|-)[%dI-]+/", "%1{}/") -- two turns, to also get consecutive number segments + uri = uri:gsub("([/-])[%dI_-]+$", "%1{}") + uri = uri:gsub("/%d+(%.%w+)$", "/{}%1") + uri = uri:gsub("(/|-)[%w%d]+%-[%w%d]+%-[%w%d]+%-[%w%d]+%-[%w%d]+(/?)$", "%1{}%2") + uri = uri:gsub("/v%d/", "/v0/") -- Merge all API versions + -- + -- Generify remaining by trimming URIs from right + uri = uri:gsub("^(/from%-houston/[^/]+/eagle/nsync/).*$", "%1...") + uri = uri:gsub("^(/from%-houston/[^/]+/eagle/fis/information/).*$", "%1...") + uri = uri:gsub("^(/from%-houston/[^/]+/eagle/nsync/v%d/push/trillian%-phonebooks%-).*$", "%1...") + uri = uri:gsub("^(/from%-houston/[^/]+/eagle/timetable/wait/).*$", "%1...") + uri = uri:gsub("^(/houston/service%-instances/).*$", "%1...") + uri = uri:gsub("^(/vortex/stillInterested%?vehicleId%=).*$", "%1...") + uri = uri:gsub("^(/houston/[^/]+/[^/]+/).*$", "%1...") + return uri +end + + +function printHttpRequestStats( app ) + local sorted = {} + local maxOccurValue = 0 + local overallCount = 0 + for _, reqObj in pairs(app.foundHttpRequests) do + if reqObj.count > maxOccurValue then maxOccurValue = reqObj.count end + overallCount = overallCount + reqObj.count + table.insert(sorted, reqObj) + end + table.sort(sorted, function(a, b)return a.count > b.count end) + local dumpDurationSec = app.youngestEpochSec - app.oldestEpochSec + local timeFmt = "!%Y-%m-%d_%H:%M:%SZ" + out:write("\n") + out:write(string.format(" Subject HTTP Request Statistics\n")) + out:write(string.format(" Begin %s\n", os.date(timeFmt,app.oldestEpochSec))) + out:write(string.format(" Duration %d seconds\n", dumpDurationSec)) + out:write(string.format("Throughput %.1f HTTP requests per second\n", overallCount / dumpDurationSec)) + out:write("\n") + out:write(" .-- HTTP Requests per Second\n") + out:write(" | .-- URI\n") + out:write(".--+--. .-+---------\n") + local chartWidth = 60 + local cntPrinted = 0 + for i, elem in ipairs(sorted) do + local count, httpMethod, httpUri = elem.count, elem.httpMethod, elem.httpUri + local cntPerSec = math.floor((count / dumpDurationSec)*10+.5)/10 + out:write(string.format("%7.1f %s\n", cntPerSec, httpUri)) + cntPrinted = cntPrinted + 1 + ::nextPort:: + end + out:write("\n") +end + + +main() + diff --git a/src/main/lua/pcap/tcpDataAmountStats.lua b/src/main/lua/pcap/tcpDataAmountStats.lua new file mode 100644 index 0000000..496687a --- /dev/null +++ b/src/main/lua/pcap/tcpDataAmountStats.lua @@ -0,0 +1,97 @@ + +local newPcapParser = assert(require("pcapit").newPcapParser) + +local main, onPcapFrame, vapourizeUrlVariables, printResult + + +function main() + local app = { + parser = false, + youngestEpochSec = -math.huge, + oldestEpochSec = math.huge, + nextStreamNr = 1, + httpStreams = {}, + } + app.parser = newPcapParser{ + dumpFilePath = "-", + onFrame = function(f)onPcapFrame(app, f)end, + } + app.parser:resume() + printResult(app) +end + + +function onPcapFrame( app, it ) + local out = io.stdout + -- + if not it:tcpSeqNr() then return end + -- + -- + local sec, usec = it:frameArrivalTime() + if sec < app.oldestEpochSec then app.oldestEpochSec = sec end + if sec > app.youngestEpochSec then app.youngestEpochSec = sec end + -- + local srcIp, dstIp = it:netSrcIpStr(), it:netDstIpStr() + local srcPort, dstPort = it:trspSrcPort(), it:trspDstPort() + local lowIp = (srcIp < dstIp)and(srcIp)or(dstIp) + local higIp = (lowIp == dstIp)and(srcIp)or(dstIp) + local lowPort = math.min(srcPort, dstPort) + local streamId = lowIp .."\0".. higIp .."\0".. lowPort + local stream = app.httpStreams[streamId] + if not stream then + stream = { + srcIp = srcIp, dstIp = dstIp, srcPort = srcPort, dstPort = dstPort, + streamNr = app.nextStreamNr, numBytes = 0, + } + app.nextStreamNr = app.nextStreamNr + 1 + app.httpStreams[streamId] = stream + end + local trspPayload = it:trspPayload() + stream.numBytes = stream.numBytes + trspPayload:len() +end + + +function printResult( app ) + local out = io.stdout + local sorted = {} + local overalValue, maxValue = 0, 0 + for _, stream in pairs(app.httpStreams) do + if stream.numBytes > maxValue then maxValue = stream.numBytes end + overalValue = overalValue + stream.numBytes + table.insert(sorted, stream) + end + table.sort(sorted, function(a, b)return a.numBytes > b.numBytes end) + local dumpDurationSec = app.youngestEpochSec - app.oldestEpochSec + local overallBytesPerSec = overalValue / dumpDurationSec + local maxValuePerSec = maxValue / dumpDurationSec + local timeFmt = "!%Y-%m-%d_%H:%M:%SZ" + out:write("\n") + out:write(string.format(" Subject TCP data throughput\n")) + out:write(string.format(" Begin %s\n", os.date(timeFmt,app.oldestEpochSec))) + out:write(string.format(" Duration %d seconds\n", dumpDurationSec)) + out:write(string.format(" Overall %.3f KiB per second (%.3f KiBit per second)\n", + overallBytesPerSec/1024, overallBytesPerSec/1024*8)) + out:write("\n") + out:write(" .-- KiB per Second\n") + out:write(" | .-- IP endpoints\n") + out:write(" | | .-- TCP server port\n") + out:write(" | | | .-- TCP Payload (less is better)\n") + out:write(" | | | |\n") + out:write(".--+----. .----+----------------------. .+--. .-+------------\n") + local bar = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + for i, elem in ipairs(sorted) do + local streamNr, srcIp, dstIp, srcPort, dstPort, numBytes = + elem.streamNr, elem.srcIp, elem.dstIp, elem.srcPort, elem.dstPort, elem.numBytes + local lowPort = math.min(srcPort, dstPort) + local bytesPerSecond = math.floor((numBytes / dumpDurationSec)*10+.5)/10 + out:write(string.format("%9.3f %-14s %-14s %5d ", bytesPerSecond/1024, srcIp, dstIp, lowPort)) + local part = bytesPerSecond / maxValuePerSec; + out:write(bar:sub(0, math.floor(part * bar:len()))) + out:write("\n") + end + out:write("\n") +end + + +main() + diff --git a/src/main/lua/pcap/tcpPortStats.lua b/src/main/lua/pcap/tcpPortStats.lua new file mode 100644 index 0000000..9038db7 --- /dev/null +++ b/src/main/lua/pcap/tcpPortStats.lua @@ -0,0 +1,82 @@ + +local newPcapParser = assert(require("pcapit").newPcapParser) + +local out, log = io.stdout, io.stderr +local main, onPcapFrame, printStats + + +function main() + local app = { + parser = false, + youngestEpochSec = -math.huge, + oldestEpochSec = math.huge, + foundPortNumbers = {}, + } + app.parser = newPcapParser{ + dumpFilePath = "-", + onFrame = function(f)onPcapFrame(app, f)end, + } + app.parser:resume() + printStats(app) +end + + +function onPcapFrame( app, it ) + local sec, usec = it:frameArrivalTime() + local srcPort, dstPort = it:trspSrcPort(), it:trspDstPort() + --local srcIp, dstIp = it:netSrcIpStr(), it:netDstIpStr() + --local isTcp = (it:tcpSeqNr() ~= nil) + -- + if sec < app.oldestEpochSec then app.oldestEpochSec = sec end + if sec > app.youngestEpochSec then app.youngestEpochSec = sec end + -- + if not app.foundPortNumbers[srcPort] then app.foundPortNumbers[srcPort] = 1 + else app.foundPortNumbers[srcPort] = app.foundPortNumbers[srcPort] + 1 end + if not app.foundPortNumbers[dstPort+100000] then app.foundPortNumbers[dstPort+100000] = 1 + else app.foundPortNumbers[dstPort+100000] = app.foundPortNumbers[dstPort+100000] + 1 end +end + + +function printStats( app ) + local sorted = {} + local totalPackets, maxOccurValue = 0, 0 + for port, pkgcnt in pairs(app.foundPortNumbers) do + if pkgcnt > maxOccurValue then maxOccurValue = pkgcnt end + table.insert(sorted, { port=port, pkgcnt=pkgcnt }) + totalPackets = totalPackets + pkgcnt + end + table.sort(sorted, function(a, b)return a.pkgcnt > b.pkgcnt end) + local dumpDurationSec = app.youngestEpochSec - app.oldestEpochSec + local timeFmt = "!%Y-%m-%d_%H:%M:%SZ" + out:write("\n") + out:write(string.format(" Subject TCP/UDP stats\n")) + out:write(string.format(" Begin %s\n", os.date(timeFmt,app.oldestEpochSec))) + out:write(string.format(" Duration %d seconds\n", dumpDurationSec)) + out:write(string.format("Throughput %.1f packets per second\n", totalPackets / dumpDurationSec)) + out:write("\n") + out:write(" .- TCP/UDP Port\n") + out:write(" | .-Direction (Send, Receive)\n") + out:write(" | | .- Packets per second\n") + out:write(".-+-. | .---+-.\n") + local chartWidth = 60 + for i, elem in ipairs(sorted) do + local port, pkgcnt = elem.port, elem.pkgcnt + local dir = (port > 100000)and("R")or("S") + if port > 100000 then port = port - 100000 end + if port > 30000 then goto nextPort end + local pkgsPerSec = math.floor((pkgcnt / dumpDurationSec)*10+.5)/10 + out:write(string.format("%5d %s %7.1f |", port, dir, pkgsPerSec)) + local barLen = pkgcnt / maxOccurValue + --local barLen = (math.log(pkgcnt) / math.log(maxOccurValue)) + for i=1, chartWidth-1 do + out:write((i < (barLen*chartWidth))and("=")or(" ")) + end + out:write("|\n") + ::nextPort:: + end + out:write("\n") +end + + +main() + diff --git a/src/main/lua/pcap/xServiceStats.lua b/src/main/lua/pcap/xServiceStats.lua new file mode 100644 index 0000000..3bc94a4 --- /dev/null +++ b/src/main/lua/pcap/xServiceStats.lua @@ -0,0 +1,90 @@ + +local newPcapParser = assert(require("pcapit").newPcapParser) + +local out, log = io.stdout, io.stderr +local main, onPcapFrame, vapourizeUrlVariables, printStats + + +function main() + local app = { + parser = false, + youngestEpochSec = -math.huge, + oldestEpochSec = math.huge, + services = {}, + } + app.parser = newPcapParser{ + dumpFilePath = "-", + onFrame = function(f)onPcapFrame(app, f)end, + } + app.parser:resume() + printStats(app) +end + + +function onPcapFrame( app, it ) + local sec, usec = it:frameArrivalTime() + local srcPort, dstPort = it:trspSrcPort(), it:trspDstPort() + -- + if sec < app.oldestEpochSec then app.oldestEpochSec = sec end + if sec > app.youngestEpochSec then app.youngestEpochSec = sec end + -- + local portsOfInterest = { + [ 80] = true, + [8080] = true, + [7012] = true, + } + --if not portsOfInterest[dstPort] and not portsOfInterest[srcPort] then return end + local trspPayload = it:trspPayload() + local httpReqLinePart1, httpReqLinePart2, httpReqLinePart3 = + trspPayload:match("^([A-Z/1.0]+) ([^ ]+) [^ \r\n]+\r?\n") + if not httpReqLinePart1 then return end + if httpReqLinePart1:find("^HTTP/1.%d$") then return end + --log:write(string.format("%5d->%5d %s %s %s\n", srcPort, dstPort, httpReqLinePart1, httpReqLinePart2, httpReqLinePart3)) + xService = trspPayload:match("\n[Xx]%-[Ss][Ee][Rr][Vv][Ii][Cc][Ee]:%s+([^\r\n]+)\r?\n"); + if not xService then return end + --log:write("X-Service is '".. xService .."'\n") + local obj = app.services[xService] + if not obj then + app.services[xService] = { + xService = xService, + count=0, + } + else + assert(xService == obj.xService) + obj.count = obj.count + 1 + end +end + + +function printStats( app ) + local sorted = {} + local maxOccurValue = 0 + local overallCount = 0 + for _, reqObj in pairs(app.services) do + if reqObj.count > maxOccurValue then maxOccurValue = reqObj.count end + overallCount = overallCount + reqObj.count + table.insert(sorted, reqObj) + end + table.sort(sorted, function(a, b)return a.count > b.count end) + local dumpDurationSec = app.youngestEpochSec - app.oldestEpochSec + local timeFmt = "!%Y-%m-%d_%H:%M:%SZ" + out:write("\n") + out:write(string.format(" Subject Pressure by Services\n")) + out:write(string.format(" Begin %s\n", os.date(timeFmt,app.oldestEpochSec))) + out:write(string.format(" Duration %d seconds\n", dumpDurationSec)) + out:write(string.format("Matching Requests %.1f (HTTP requests per second)\n", overallCount / dumpDurationSec)) + out:write("\n") + out:write(" .-- HTTP Requests per Second\n") + out:write(" | .-- Service\n") + out:write(".-+---. .-+-----\n") + for i, elem in ipairs(sorted) do + local xService, count = elem.xService, elem.count + local countPerSecond = math.floor((count / dumpDurationSec)*10+.5)/10 + out:write(string.format("%7.1f %s\n", countPerSecond, xService)) + end + out:write("\n") +end + + +main() + diff --git a/src/main/lua/wireshark/HttpTime.lua b/src/main/lua/wireshark/HttpTime.lua index b06c0a7..514c62b 100644 --- a/src/main/lua/wireshark/HttpTime.lua +++ b/src/main/lua/wireshark/HttpTime.lua @@ -10,7 +10,7 @@ local mod = {} function mod.init() local that = mod.seal{ - proto = Proto("__", "Additional Metadata"), + proto = Proto("AdditMeta", "Additional Metadata"), f_andy_httpTime = ProtoField.float("_.httpTime", "HttpTime"), f_andy_synSeen = ProtoField.bool("_.synSeen", "SynSeen"), f_andy_uri = ProtoField.string("_.uri", "Request URI"), |