From 0dca5c93398609cd31a3a2284ae742fb6ce34552 Mon Sep 17 00:00:00 2001 From: Andreas Fankhauser hiddenalpha.ch Date: Sun, 11 Dec 2022 03:05:46 +0100 Subject: (MvnDepScan) Kludge around with parallelism. But scriptlee is too buggy in pthread.newCond --- src/main/lua/maven/MvnCentralDepScan.lua | 98 +++++++++++++++++++++----------- 1 file changed, 64 insertions(+), 34 deletions(-) diff --git a/src/main/lua/maven/MvnCentralDepScan.lua b/src/main/lua/maven/MvnCentralDepScan.lua index 7225c58..3484a32 100644 --- a/src/main/lua/maven/MvnCentralDepScan.lua +++ b/src/main/lua/maven/MvnCentralDepScan.lua @@ -8,7 +8,9 @@ local AF_INET = require('scriptlee').posix.AF_INET local AF_INET6 = require('scriptlee').posix.AF_INET6 local IPPROTO_TCP = require('scriptlee').posix.IPPROTO_TCP local SOCK_STREAM = require('scriptlee').posix.SOCK_STREAM +local async = require("scriptlee").reactor.async local inaddrOfHostname = require('scriptlee').posix.inaddrOfHostname +local newCond = require("scriptlee").posix.newCond local newHttpClient = require("scriptlee").newHttpClient local newSqlite = require("scriptlee").newSqlite local newTlsClient = assert(require("scriptlee").newTlsClient) @@ -164,12 +166,12 @@ end function mod.onGetPomRspHdr( msg, req ) - log:write("< "..tostring(msg.proto) .." "..tostring(msg.status).." "..tostring(msg.phrase).."\n") - --for i, h in ipairs(msg.headers) do - -- log:write("< ".. h.key ..": ".. h.val .."\n") - --end - --log:write("< \n") if msg.status ~= 200 then + log:write("< "..tostring(msg.proto) .." "..tostring(msg.status).." "..tostring(msg.phrase).."\n") + for i, h in ipairs(msg.headers) do + log:write("< ".. h.key ..": ".. h.val .."\n") + end + log:write("< \n") error("Unexpected HTTP ".. tostring(msg.status)) end assert(not req.pomParser) @@ -218,7 +220,7 @@ function mod.onGetPomRspHdr( msg, req ) if not mvnArtifact.groupId then mvnArtifact.groupId = mvnArtifact.parentGroupId end if not mvnArtifact.version then mvnArtifact.version = mvnArtifact.parentVersion end local key = mod.getMvnArtifactKey(mvnArtifact) - assert(not app.mvnArtifacts[key]) + assert(not app.mvnArtifacts[key], key) app.mvnArtifacts[key] = mvnArtifact end, } @@ -326,6 +328,9 @@ function mod.getPropValThroughParentChain( app, mvnArtifact, propKey, none ) if propKey == "project.version" then return mvnArtifact.version end + if propKey == "project.groupId" then + return mvnArtifact.groupId + end -- no luck in current artifact. Delegate to parent (if any) if mvnArtifact.parentGroupId and mvnArtifact.parentArtifactId @@ -489,9 +494,7 @@ function mod.storeAsSqliteFile( app ) for _, mvnArtifact in pairs(app.mvnArtifacts) do local mvnDeps = app.mvnDepsByArtifact[mvnArtifact] for _, mvnDep in pairs(mvnDeps or {}) do - if not mvnDep.version then mvnDep.version = "TODO_5bbc0e87011e24d845136c5406302616" end - assert(mvnDep.version, mvnDep.artifactId) - assert(mvnDep.groupId and mvnDep.artifactId and mvnDep.version) + assert(mvnDep.groupId and mvnDep.artifactId) local bucket = mvnArtifactIdsByArtif[mvnDep.artifactId] local depId = nil for _,a in pairs(bucket or {}) do @@ -502,6 +505,11 @@ function mod.storeAsSqliteFile( app ) end end if not depId then -- Artifact not stored yet. Do now. + if not mvnDep.version then + -- TODO mvnDep.version CAN be missing. Eg via depMgnt of + -- unknown parent or similar + mvnDep.version = "TODO_40ba845c5a1bd8" + end depId = insertMvnArtifact({ groupId = mvnDep.groupId, artifactId = mvnDep.artifactId, @@ -524,6 +532,9 @@ function mod.run( app ) assert(not app.mvnDepsByArtifact) app.mvnDepsByArtifact = {} assert(not app.mvnMngdDepsByArtifact) app.mvnMngdDepsByArtifact = {} local pomSrc = mod.newPomUrlSrc(app) + local numInProgress, numInProgressLimit = 0, 1 + local numInProgressCond = newCond() + log:write("numInProgressCond ".. tostring(numInProgressCond).." ("..(debug.getinfo(1).currentline)..")\n"); while true do local pomUrl = pomSrc:nextPomUrl() if not pomUrl then break end @@ -534,23 +545,35 @@ function mod.run( app ) local url = pomUrl:match("^https?://[^/]+(.*)$") if port == 443 then isTLS = true end if not port then port = (isTLS and 443 or 80) end - log:write("> GET ".. proto .."://".. host ..":".. port .. url .."\n") - local req = objectSeal{ - app = app, - base = false, - pomParser = false, - } - req.base = app.http:request{ - cls = req, - host = assert(host), port = assert(port), - method = "GET", url = url, - --hdrs = , - useTLS = isTLS, - onRspHdr = mod.onGetPomRspHdr, - onRspChunk = mod.onGetPomRspChunk, - onRspEnd = mod.onGetPomRspEnd, - } - req.base:closeSnk() + while numInProgress >= numInProgressLimit do + log:write("numInProgress is ".. numInProgress ..". Wait ...\n") + numInProgressCond:waitForever() + end + log:write("numInProgress is ".. numInProgress ..". Go!\n") + numInProgress = numInProgress +1 + async(function() + log:write("> GET ".. proto .."://".. host ..":".. port .. url .."\n") + local req = objectSeal{ + app = app, + base = false, + pomParser = false, + } + req.base = app.http:request{ + cls = req, + host = assert(host), port = assert(port), + method = "GET", url = url, + --hdrs = , + useTLS = isTLS, + onRspHdr = mod.onGetPomRspHdr, + onRspChunk = mod.onGetPomRspChunk, + onRspEnd = mod.onGetPomRspEnd, + } + req.base:closeSnk() + numInProgress = numInProgress -1 + log:write("numInProgress DECR. Is now ".. numInProgress ..". broadcast.\n") + log:write("numInProgressCond ".. tostring(numInProgressCond).." ("..(debug.getinfo(1).currentline)..")\n"); + numInProgressCond:broadcast() + end) end log:write("[INFO ] No more pom URLs\n") mod.resolveDependencyVersionsFromDepsMgmnt(app) @@ -561,13 +584,13 @@ end function mod.newSocketMgr() + --local numConnActive, numConnActiveLimit = 0, 4 + --local numConnActiveCond = newCond() local hosts = {} local openSock = function( t, opts ) for k, v in pairs(opts) do if false then - elseif k=='host' or k=='port' then - elseif k=='useTLS' then - if v then error('TLS not impl') end + elseif k=='host' or k=='port' or k=='useTLS' then else error('Unknown option: '..tostring(k)) end @@ -575,14 +598,17 @@ function mod.newSocketMgr() local inaddr = inaddrOfHostname(opts.host) local af if inaddr:find('^%d+.%d+.%d+.%d+$') then af = AF_INET else af = AF_INET6 end - log:write("opts.useTLS "..tostring(opts.useTLS).." (Override to TRUE ...)\n") - opts.useTLS = true -- TODO why the heck is this needed? (I guess scriptlee bug?) local key = inaddr.."\t"..opts.port.."\t"..tostring(opts.useTLS) - log:write("KEY wr '"..key.."'\n") local existing = hosts[key] + --numConnActive = numConnActive +1 if existing then return table.remove(existing) else + --while numConnActive > numConnActiveLimit do + -- log:write("numConnActive is "..numConnActive..". Waiting ...\n") + -- numConnActiveCond:waitForever() + --end + --log:write("numConnActive is ".. numConnActive ..". Go\n") local sock = socket(af, SOCK_STREAM, IPPROTO_TCP) sock:connect(inaddr, opts.port) if opts.useTLS then @@ -623,7 +649,6 @@ function mod.newSocketMgr() local releaseSock = function( t, sockWrapr ) -- keep-alive (TODO only if header says so) local key = sockWrapr._host.."\t"..sockWrapr._port.."\t"..tostring(sockWrapr._useTLS) - log:write("KEY rd '"..key.."'\n") local host = hosts[key] if not host then host = {} hosts[key] = host end table.insert(host, sockWrapr) @@ -631,7 +656,12 @@ function mod.newSocketMgr() return{ openSock = openSock, releaseSock = releaseSock, - closeSock = function(t, sockWrapr) sockWrapr._sock:release() end, + closeSock = function(t, sockWrapr) + sockWrapr._sock:release() + --numConnActive = numConnActive -1 + --log:write("numConnActive -1. Is now ".. numConnActive ..". Broadcast.\n") + --numConnActiveCond:broadcast() + end, } end -- cgit v1.1