Tinker around to produce some stats out of pcap files.

3 files changed, 219 insertions, 35 deletions

diff --git a/src/main/lua/pcap/httpStats.lua b/src/main/lua/pcap/httpStats.lua
new file mode 100644
index 0000000..e4a3aaa
--- /dev/null
+++ b/src/main/lua/pcap/httpStats.lua
@@ -0,0 +1,118 @@
+
+local newPcapParser = assert(require("pcapit").newPcapParser)
+
+local main, onPcapFrame, vapourizeUrlVariables, printHttpRequestStats
+
+
+function main()
+    local app = {
+        parser = false,
+        foundHttpRequests = {},
+        youngestEpochSec = -math.huge,
+        oldestEpochSec = math.huge,
+    }
+    app.parser = newPcapParser{
+        dumpFilePath = "-",
+        onFrame = function(f)onPcapFrame(app, f)end,
+    }
+    app.parser:resume()
+    printHttpRequestStats(app)
+end
+
+
+function onPcapFrame( app, it )
+    local out = io.stdout
+    local sec, usec = it:frameArrivalTime()
+    local srcPort, dstPort = it:trspSrcPort(), it:trspDstPort()
+    --
+    if sec < app.oldestEpochSec then app.oldestEpochSec = sec end
+    if sec > app.youngestEpochSec then app.youngestEpochSec = sec end
+    --
+    local portOfInterest = 7012
+    if dstPort == portOfInterest then
+        local httpMethod, httpUri =
+            it:trspPayload():match("^([A-Z]+) ([^ ]+) [^ \r\n]+\r?\n")
+        if httpMethod then
+            --out:write(string.format("%5d->%5d %s %s\n", srcPort, dstPort, httpMethod, httpUri))
+            httpUri = vapourizeUrlVariables(app, httpUri)
+            local key = httpUri -- httpMethod .." ".. httpUri
+            local obj = app.foundHttpRequests[key]
+            if not obj then
+                obj = { count=0, httpMethod=false, httpUri=false, }
+                app.foundHttpRequests[key] = obj
+            end
+            obj.count = obj.count + 1
+            obj.httpMethod = httpMethod
+            obj.httpUri = httpUri
+        end
+    elseif srcPort == portOfInterest then
+        local httpStatus, httpPhrase =
+            it:trspPayload():match("^HTTP/%d.%d (%d%d%d) ([^\r\n]*)\r?\n")
+        if httpStatus then
+            --out:write(string.format("%5d<-%5d %s %s\n", srcPort, dstPort, httpStatus, httpPhrase))
+        end
+    end
+end
+
+
+function vapourizeUrlVariables( app, uri )
+    -- A very specific case
+    uri = uri:gsub("^(/houston/users/)%d+(/.*)$", "%1{}%2");
+    if uri:find("^/houston/users/[^/]+/user/.*$") then return uri end
+    --
+    -- Try to do some clever guesses to group URIs wich only differ in variable segments
+    uri = uri:gsub("(/|-)[%dI_-]+/", "%1{}/"):gsub("(/|-)[%dI-]+/", "%1{}/") -- two turns, to also get consecutive number segments
+    uri = uri:gsub("([/-])[%dI_-]+$", "%1{}")
+    uri = uri:gsub("/%d+(%.%w+)$", "/{}%1")
+    uri = uri:gsub("(/|-)[%w%d]+%-[%w%d]+%-[%w%d]+%-[%w%d]+%-[%w%d]+(/?)$", "%1{}%2")
+    uri = uri:gsub("/v%d/", "/v0/") -- Merge all API versions
+    --
+    -- Generify remaining by trimming URIs from right
+    uri = uri:gsub("^(/from%-houston/[^/]+/eagle/nsync/).*$", "%1...")
+    uri = uri:gsub("^(/from%-houston/[^/]+/eagle/fis/information/).*$", "%1...")
+    uri = uri:gsub("^(/from%-houston/[^/]+/eagle/nsync/v%d/push/trillian%-phonebooks%-).*$", "%1...")
+    uri = uri:gsub("^(/from%-houston/[^/]+/eagle/timetable/wait/).*$", "%1...")
+    uri = uri:gsub("^(/houston/service%-instances/).*$", "%1...")
+    uri = uri:gsub("^(/vortex/stillInterested%?vehicleId%=).*$", "%1...")
+    uri = uri:gsub("^(/houston/[^/]+/[^/]+/).*$", "%1...")
+    return uri
+end
+
+
+function printHttpRequestStats( app )
+    local out = io.stdout
+    local sorted = {}
+    local maxOccurValue = 0
+    local overallCount = 0
+    for _, reqObj in pairs(app.foundHttpRequests) do
+        if reqObj.count > maxOccurValue then maxOccurValue = reqObj.count end
+        overallCount = overallCount + reqObj.count
+        table.insert(sorted, reqObj)
+    end
+    table.sort(sorted, function(a, b)return a.count > b.count end)
+    local dumpDurationSec = app.youngestEpochSec - app.oldestEpochSec
+    local timeFmt = "!%Y-%m-%d_%H:%M:%SZ"
+    out:write("\n")
+    out:write(string.format("   Subject  HTTP Request Statistics\n"))
+    out:write(string.format("     Begin  %s\n", os.date(timeFmt,app.oldestEpochSec)))
+    out:write(string.format("  Duration  %d seconds\n", dumpDurationSec))
+    out:write(string.format("Throughput  %.1f HTTP requests per second\n", overallCount / dumpDurationSec))
+    out:write("\n")
+    out:write("   .-- HTTP Requests per Second\n")
+    out:write("   |       .-- URI\n")
+    out:write(".--+--.  .-+---------\n")
+    local chartWidth = 60
+    local cntPrinted = 0
+    for i, elem in ipairs(sorted) do
+        local count, httpMethod, httpUri = elem.count, elem.httpMethod, elem.httpUri
+        local cntPerSec = math.floor((count / dumpDurationSec)*10+.5)/10
+        out:write(string.format("%7.1f  %s\n", cntPerSec, httpUri))
+        cntPrinted = cntPrinted + 1
+        ::nextPort::
+    end
+    out:write("\n")
+end
+
+
+main()
+
diff --git a/src/main/lua/pcap/makeStats.lua b/src/main/lua/pcap/tcpPortStats.lua
index 2874d9f..9038db7 100644
--- a/src/main/lua/pcap/makeStats.lua
+++ b/src/main/lua/pcap/tcpPortStats.lua
@@ -1,21 +1,17 @@
 
 local newPcapParser = assert(require("pcapit").newPcapParser)
-local newPcapDumper = assert(require("pcapit").newPcapDumper)
 
+local out, log = io.stdout, io.stderr
 local main, onPcapFrame, printStats
 
 
 function main()
     local app = {
-        dumpr = false,
         parser = false,
-        foundPortNumbers = {},
         youngestEpochSec = -math.huge,
         oldestEpochSec = math.huge,
+        foundPortNumbers = {},
     }
-    --app.dumpr = newPcapDumper{
-    --    dumpFilePath = "/tmp/meins/my.out.pcap",
-    --}
     app.parser = newPcapParser{
         dumpFilePath = "-",
         onFrame = function(f)onPcapFrame(app, f)end,
@@ -26,7 +22,6 @@ end
 
 
 function onPcapFrame( app, it )
-    local out = io.stdout
     local sec, usec = it:frameArrivalTime()
     local srcPort, dstPort = it:trspSrcPort(), it:trspDstPort()
     --local srcIp, dstIp = it:netSrcIpStr(), it:netDstIpStr()
@@ -39,48 +34,31 @@ function onPcapFrame( app, it )
     else app.foundPortNumbers[srcPort] = app.foundPortNumbers[srcPort] + 1 end
     if not app.foundPortNumbers[dstPort+100000] then app.foundPortNumbers[dstPort+100000] = 1
     else app.foundPortNumbers[dstPort+100000] = app.foundPortNumbers[dstPort+100000] + 1 end
-    --
-    local portOfInterest = 7012
-    if dstPort == portOfInterest then
-        local httpMethod, httpUri =
-            it:trspPayload():match("^([A-Z]+) ([^ ]+) [^ \r\n]+\r?\n")
-        if httpMethod then
-            out:write(string.format("%5d->%5d %s %s\n", srcPort, dstPort, httpMethod, httpUri))
-        end
-    elseif srcPort == portOfInterest then
-        local httpStatus, httpPhrase =
-            it:trspPayload():match("^HTTP/%d.%d (%d%d%d) ([^\r\n]*)\r?\n")
-        if httpStatus then
-            out:write(string.format("%5d<-%5d %s %s\n", srcPort, dstPort, httpStatus, httpPhrase))
-        end
-    end
-    --if srcPort ~= 53 and dstPort ~= 53 then return end
-    if app.dumpr then it:dumpTo(app.dumpr) end
 end
 
 
 function printStats( app )
-    local out = io.stdout
     local sorted = {}
-    local maxOccurValue = 0
+    local totalPackets, maxOccurValue = 0, 0
     for port, pkgcnt in pairs(app.foundPortNumbers) do
         if pkgcnt > maxOccurValue then maxOccurValue = pkgcnt end
         table.insert(sorted, { port=port, pkgcnt=pkgcnt })
+        totalPackets = totalPackets + pkgcnt
     end
     table.sort(sorted, function(a, b)return a.pkgcnt > b.pkgcnt end)
     local dumpDurationSec = app.youngestEpochSec - app.oldestEpochSec
     local timeFmt = "!%Y-%m-%d_%H:%M:%SZ"
     out:write("\n")
-    out:write("Statistics\n")
-    out:write("From: ")out:write(os.date(timeFmt,app.oldestEpochSec))out:write("\n")
-    out:write("To:   ")out:write(os.date(timeFmt,app.youngestEpochSec))out:write("\n")
+    out:write(string.format("   Subject  TCP/UDP stats\n"))
+    out:write(string.format("     Begin  %s\n", os.date(timeFmt,app.oldestEpochSec)))
+    out:write(string.format("  Duration  %d seconds\n", dumpDurationSec))
+    out:write(string.format("Throughput  %.1f packets per second\n", totalPackets / dumpDurationSec))
     out:write("\n")
-    out:write("  .- Port (TCP/UDP)\n")
+    out:write("  .- TCP/UDP Port\n")
     out:write("  |   .-Direction (Send, Receive)\n")
-    out:write("  |   |     .- Frames per second\n")
-    out:write(".-+-. | .---+-.   Amount of frames compared:\n")
+    out:write("  |   |     .- Packets per second\n")
+    out:write(".-+-. | .---+-.\n")
     local chartWidth = 60
-    local cntPrinted = 0
     for i, elem in ipairs(sorted) do
         local port, pkgcnt = elem.port, elem.pkgcnt
         local dir = (port > 100000)and("R")or("S")
@@ -94,8 +72,6 @@ function printStats( app )
             out:write((i < (barLen*chartWidth))and("=")or(" "))
         end
         out:write("|\n")
-        cntPrinted = cntPrinted + 1
-        if cntPrinted >= 20 then break end
         ::nextPort::
     end
     out:write("\n")
diff --git a/src/main/lua/pcap/xServiceStats.lua b/src/main/lua/pcap/xServiceStats.lua
new file mode 100644
index 0000000..1cc5961
--- /dev/null
+++ b/src/main/lua/pcap/xServiceStats.lua
@@ -0,0 +1,90 @@
+
+local newPcapParser = assert(require("pcapit").newPcapParser)
+
+local out, log = io.stdout, io.stderr
+local main, onPcapFrame, vapourizeUrlVariables, printStats
+
+
+function main()
+    local app = {
+        parser = false,
+        youngestEpochSec = -math.huge,
+        oldestEpochSec = math.huge,
+        services = {},
+    }
+    app.parser = newPcapParser{
+        dumpFilePath = "-",
+        onFrame = function(f)onPcapFrame(app, f)end,
+    }
+    app.parser:resume()
+    printStats(app)
+end
+
+
+function onPcapFrame( app, it )
+    local sec, usec = it:frameArrivalTime()
+    local srcPort, dstPort = it:trspSrcPort(), it:trspDstPort()
+    --
+    if sec < app.oldestEpochSec then app.oldestEpochSec = sec end
+    if sec > app.youngestEpochSec then app.youngestEpochSec = sec end
+    --
+    local portsOfInterest = {
+        [  80] = true,
+        [8080] = true,
+        [7012] = true,
+    }
+    --if not portsOfInterest[dstPort] and not portsOfInterest[srcPort] then return end
+    local trspPayload = it:trspPayload()
+    local httpReqLinePart1, httpReqLinePart2, httpReqLinePart3 =
+        trspPayload:match("^([A-Z/1.0]+) ([^ ]+) [^ \r\n]+\r?\n")
+    if not httpReqLinePart1 then return end
+    if httpReqLinePart1:find("^HTTP/1.%d$") then return end
+    --log:write(string.format("%5d->%5d  %s %s %s\n", srcPort, dstPort, httpReqLinePart1, httpReqLinePart2, httpReqLinePart3))
+    xService = trspPayload:match("\n[Xx]%-[Ss][Ee][Rr][Vv][Ii][Cc][Ee]:%s+([^\r\n]+)\r?\n");
+    if not xService then return end
+    --log:write("X-Service is '".. xService .."'\n")
+    local obj = app.services[xService]
+    if not obj then
+        app.services[xService] = {
+            xService = xService,
+            count=0,
+        }
+    else
+        assert(xService == obj.xService)
+        obj.count = obj.count + 1
+    end
+end
+
+
+function printStats( app )
+    local sorted = {}
+    local maxOccurValue = 0
+    local overallCount = 0
+    for _, reqObj in pairs(app.services) do
+        if reqObj.count > maxOccurValue then maxOccurValue = reqObj.count end
+        overallCount = overallCount + reqObj.count
+        table.insert(sorted, reqObj)
+    end
+    table.sort(sorted, function(a, b)return a.count > b.count end)
+    local dumpDurationSec = app.youngestEpochSec - app.oldestEpochSec
+    local timeFmt = "!%Y-%m-%d_%H:%M:%SZ"
+    out:write("\n")
+    out:write(string.format("   Subject  Pressure by Services\n"))
+    out:write(string.format("     Begin  %s\n", os.date(timeFmt,app.oldestEpochSec)))
+    out:write(string.format("  Duration  %d seconds\n", dumpDurationSec))
+    out:write(string.format("Throughput  %.1f HTTP requests per second\n", overallCount / dumpDurationSec))
+    out:write("\n")
+    out:write("  .-- HTTP Requests per Second\n")
+    out:write("  |        .-- Service\n")
+    out:write(".-+---.  .-+-----\n")
+    for i, elem in ipairs(sorted) do
+        local xService, count = elem.xService, elem.count
+        local countPerSecond = math.floor((count / dumpDurationSec)*10+.5)/10
+        out:write(string.format("%7.1f  %s\n", countPerSecond, xService))
+    end
+    out:write("\n")
+end
+
+
+main()
+