diff options
| -rw-r--r-- | README.txt | 3 | ||||
| -rw-r--r-- | doc/note/maven/base.xml (renamed from doc/note/maven-pom/base.xml) | 0 | ||||
| -rw-r--r-- | doc/note/maven/maven.txt | 5 | ||||
| -rw-r--r-- | src/main/lua/maven/MvnCentralDepScan.lua | 1217 |
4 files changed, 962 insertions, 263 deletions
@@ -4,6 +4,3 @@ Unspecified Garbage Just some random garbage which was handy in some way somewhen. -Not yet migrated scripts see "C:/Users/fankhauseand/OneDrive - POSTCHAG/doc" - - diff --git a/doc/note/maven-pom/base.xml b/doc/note/maven/base.xml index 7218e99..7218e99 100644 --- a/doc/note/maven-pom/base.xml +++ b/doc/note/maven/base.xml diff --git a/doc/note/maven/maven.txt b/doc/note/maven/maven.txt new file mode 100644 index 0000000..fe641e8 --- /dev/null +++ b/doc/note/maven/maven.txt @@ -0,0 +1,5 @@ + +## Print effective-pom + + mvn help:effective-pom -Doutput="the-effective-pom.xml" + diff --git a/src/main/lua/maven/MvnCentralDepScan.lua b/src/main/lua/maven/MvnCentralDepScan.lua index 0ea449f..a1f9418 100644 --- a/src/main/lua/maven/MvnCentralDepScan.lua +++ b/src/main/lua/maven/MvnCentralDepScan.lua @@ -2,20 +2,19 @@ Initially written using scriptlee 0.0.5-46-G . + Begun experimenting with scriptlee 0.0.5-55-G but there's a stack overflow + bug in the XML parser somewhere. + ]====================================================================] -local AF_INET = require('scriptlee').posix.AF_INET -local AF_INET6 = require('scriptlee').posix.AF_INET6 -local IPPROTO_TCP = require('scriptlee').posix.IPPROTO_TCP -local SOCK_STREAM = require('scriptlee').posix.SOCK_STREAM -local inaddrOfHostname = require('scriptlee').posix.inaddrOfHostname +--local newCond = require("scriptlee").posix.newCond -- cannot use. Too buggy :( +local newCsvRecrdInStream = require("scriptlee").newCsvRecrdInStream local newHttpClient = require("scriptlee").newHttpClient local newSqlite = require("scriptlee").newSqlite -local newTlsClient = assert(require("scriptlee").newTlsClient) +local newTlsClient = require("scriptlee").newTlsClient local newXmlParser = require("scriptlee").newXmlParser local objectSeal = require("scriptlee").objectSeal local sleep = require("scriptlee").posix.sleep -local socket = require('scriptlee').posix.socket local startOrExecute = require("scriptlee").reactor.startOrExecute local out, log = io.stdout, io.stderr @@ -28,71 +27,205 @@ function mod.printHelp() .."\n" .." Options:\n" .."\n" - .." --example\n" - .." WARN: only use if you know what you're doing!\n" + .." --state <path>\n" + .." Data file to use for the action. Will be created if it does not\n" + .." yet exist.\n" + .."\n" + .." --asCsv <what>\n" + .." Exports requested data to stdout. <what> can be one of \"parents\"\n" + .." or \"deps\".\n" + .."\n" + .." --nullvalue <str> (default is an empty string)\n" + .." The string to use for NULL values in CSV exports.\n" + .."\n" + .." --uripat <str>\n" + .." URI pattern where the poms can be downloaded from. Use\n" + .." placeholders in curly braces to tell where to put misc parts.\n" + .." Available placeholders are: {aid}, {gid}, {gidWithSlashes} and\n" + .." {version}. Placeholders can be used multiple times. Example:\n" + .." http://example.com/repo/{gid}/{aid}/{aid}-{version}-pom.xml\n" + .."\n" + .." --csvToFetch\n" + .." CSV listing the artifacts which should be merged into the state\n" + .." file. Example:\n" + .."\n" + .." c;groupId;artifactId;version\n" + .." r;com.example;foo-toolz;1.2.3\n" + .."\n" + .."\n" + .." Example \"Export parents\"\n" .."\n" - .." --sqliteOut <path>\n" - .." Path where to export the result.\n" + .." --state theFile --asCsv parents > parents.csv\n" .."\n" + .." Example \"Export dependencies\"\n" + .."\n" + .." --state theFile --asCsv deps > dependencies.csv\n" + .."\n" + .." Example \"Fetch info from listed POMs and merge them into state file\"\n" + .."\n" + .." --state theFile --csvToFetch the.csv --uripat http://example.com/{aid}.xml\n" .."\n") end function mod.parseArgs( app ) local iA = 0 - local isExample = false -::nextArg:: - iA = iA + 1 - local arg = _ENV.arg[iA] - if not arg then - goto endOfArgs - elseif arg == "--help" then - mod.printHelp() return -1 - elseif arg == "--example" then - isExample = true - elseif arg == "--sqliteOut" then + app.statePath = false + app.nullvalue = "" + while true do iA = iA + 1 - arg = _ENV.arg[iA] - if not arg then log:write("Arg --sqliteOut needs value\n")return-1 end - app.sqliteOutFile = arg - else - log:write("Unexpected arg: "..tostring(arg).."\n")return -1 + local arg = _ENV.arg[iA] + if not arg then + break + elseif arg == "--help" then + mod.printHelp() return -1 + elseif arg == "--state" then + iA = iA + 1 + arg = _ENV.arg[iA] + if not arg then log:write("Arg --sqliteOut needs value\n")return-1 end + app.statePath = arg + elseif arg == "--asCsv" then + iA = iA +1 + arg = _ENV.arg[iA] + if arg ~= "parents" and arg ~= "deps" then + log:write("Illegal value for --asCsv: "..tostring(arg).."\n")return-1 end + app.asCsv = arg + elseif arg == "--nullvalue" then + iA = iA +1 + arg = _ENV.arg[iA] + if not arg then log:write("Arg --nullvalue needs value\n")return-1 end + app.nullvalue = arg + elseif arg == "--uripat" then + iA = iA +1 + arg = _ENV.arg[iA] + if not arg then log:write("Arg --uripat needs value\n")return-1 end + app.uripat = arg + elseif arg == "--csvToFetch" then + iA = iA +1 + arg = _ENV.arg[iA] + if not arg then log:write("Arg --csvToFetch needs value\n")return-1 end + app.csvToFetch = arg + else + log:write("Unexpected arg: "..tostring(arg).."\n")return -1 + end end - goto nextArg -::endOfArgs:: - if not isExample then log:write("Bad Args\n") return -1 end + if not app.statePath then log:write("Arg --state missing\n") return -1 end + if not app.csvToFetch and not app.asCsv then log:write("Bad Args\n") return -1 end + if app.csvToFetch and not app.uripat then log:write("Arg --uripat missing\n") return -1 end return 0 end +function mod.newMvnArtifact() + return objectSeal{ + dbId = false, + parentGroupId = false, + parentArtifactId = false, + parentVersion = false, + groupId = false, + artifactId = false, + version = false, + } +end + + +function mod.newMvnDependency() + return objectSeal{ + dbId = false, + groupId = false, + artifactId = false, + version = false, + } +end + + function mod.newPomUrlSrc( app ) - local urls = { - -- TODO insert URLs here! + local t = objectSeal{ + app = app, + remainingArtifacts = false, } local m = { - nextPomUrl = function(t) - return table.remove(urls, 1) + nextPomArtifact = function( t ) + if not t.remainingArtifacts then + local csvParser = newCsvRecrdInStream{ + cls = t, + delimCol = ";", + onRecord = function( recrd, t ) + local recrdType = recrd[1] + if recrdType == "r" then + local artif = mod.newMvnArtifact() + artif.groupId = assert(recrd[2]) + artif.artifactId = assert(recrd[3]) + artif.version = assert(recrd[4]) + table.insert(t.remainingArtifacts, artif) + elseif recrdType == "h" or recrdType == "t" then + log:write("CSV") + for i=1, #recrd do log:write(" ".. recrd[i]) end + log:write("\n") + elseif recrdType == "c" then + assert(recrd[2] == "groupId") + assert(recrd[3] == "artifactId") + assert(recrd[4] == "version") + else + print("Record:") + for iCol, val in ipairs(recrd) do print(" -> ", iCol, val) end + error("TODO_20230127110829") + end + end, + } + local fd = io.open(t.app.csvToFetch, "rb") + if not fd then error("fopen("..tostring(t.app.csvToFetch)..")") end + t.remainingArtifacts = {} + while true do + local buf = fd:read(1<<14) + if buf then + csvParser:write(buf) + else + fd:close() + csvParser:closeSnk() + break + end + end + end + return table.remove(t.remainingArtifacts) end, __index = false, } m.__index = m - return setmetatable({}, m) + return setmetatable(t, m) +end + + +function mod.pomFilepathByArtifact( app, pomArtifact ) + local a = pomArtifact + return assert(os.getenv("HOME")) .."/.m2/repository" + .."/".. a.groupId:gsub("%.", "/") .."/".. a.artifactId .."/".. a.version + .."/".. a.artifactId .."-".. a.version ..".pom" +end + + +function mod.urlByArtifact(app, artifact) + local a = artifact + assert(type(a.artifactId) == "string", tostring(a.artifactId)) + assert(type(a.groupId) == "string", tostring(a.groupId)) + assert(type(a.version) == "string", tostring(a.version)) + local url = assert(app.uripat) + url = url:gsub("{aid}", a.artifactId) + url = url:gsub("{gid}", a.groupId) + url = url:gsub("{gidWithSlashes}", a.groupId:gsub("%.", "/")) + url = url:gsub("{version}", a.version) + return url end function mod.processXmlValue( pomParser ) - local app = pomParser.req.app + local app = pomParser.app local xpath = "" for i, stackElem in ipairs(pomParser.xmlElemStack) do xpath = xpath .."/".. stackElem.tag end --log:write(xpath .."\n") local mvnArtifact = pomParser.mvnArtifact - local newMvnDependency = function()return objectSeal{ - groupId = false, - artifactId = false, - version = false, - }end if false then elseif xpath == "/project/parent/artifactId" then mvnArtifact.parentArtifactId = pomParser.currentValue @@ -107,13 +240,13 @@ function mod.processXmlValue( pomParser ) elseif xpath == "/project/version" then mvnArtifact.version = pomParser.currentValue elseif xpath == "/project/dependencies/dependency/groupId" then - if not pomParser.mvnDependency then pomParser.mvnDependency = newMvnDependency() end + if not pomParser.mvnDependency then pomParser.mvnDependency = mod.newMvnDependency() end pomParser.mvnDependency.groupId = pomParser.currentValue elseif xpath == "/project/dependencies/dependency/artifactId" then - if not pomParser.mvnDependency then pomParser.mvnDependency = newMvnDependency() end + if not pomParser.mvnDependency then pomParser.mvnDependency = mod.newMvnDependency() end pomParser.mvnDependency.artifactId = pomParser.currentValue elseif xpath == "/project/dependencies/dependency/version" then - if not pomParser.mvnDependency then pomParser.mvnDependency = newMvnDependency() end + if not pomParser.mvnDependency then pomParser.mvnDependency = mod.newMvnDependency() end pomParser.mvnDependency.version = pomParser.currentValue elseif xpath == "/project/dependencies/dependency" then assert(pomParser.mvnDependency) @@ -124,13 +257,13 @@ function mod.processXmlValue( pomParser ) if not deps then deps = {} app.mvnDepsByArtifact[mvnArtifact] = deps end table.insert(deps, assert(mvnDependency)) elseif xpath == "/project/dependencyManagement/dependencies/dependency/groupId" then - if not pomParser.mvnMngdDependency then pomParser.mvnMngdDependency = newMvnDependency() end + if not pomParser.mvnMngdDependency then pomParser.mvnMngdDependency = mod.newMvnDependency() end pomParser.mvnMngdDependency.groupId = pomParser.currentValue elseif xpath == "/project/dependencyManagement/dependencies/dependency/artifactId" then - if not pomParser.mvnMngdDependency then pomParser.mvnMngdDependency = newMvnDependency() end + if not pomParser.mvnMngdDependency then pomParser.mvnMngdDependency = mod.newMvnDependency() end pomParser.mvnMngdDependency.artifactId = pomParser.currentValue elseif xpath == "/project/dependencyManagement/dependencies/dependency/version" then - if not pomParser.mvnMngdDependency then pomParser.mvnMngdDependency = newMvnDependency() end + if not pomParser.mvnMngdDependency then pomParser.mvnMngdDependency = mod.newMvnDependency() end pomParser.mvnMngdDependency.version = pomParser.currentValue elseif xpath == "/project/dependencyManagement/dependencies/dependency" then assert(pomParser.mvnMngdDependency) @@ -154,40 +287,39 @@ end function mod.getMvnArtifactKey( mvnArtifact ) - assert(mvnArtifact.artifactId) - assert(mvnArtifact.groupId) - assert(mvnArtifact.version) + if type(mvnArtifact.artifactId) ~= "string" then error(tostring(mvnArtifact.artifactId))end + if type(mvnArtifact.groupId) ~= "string" then error(tostring(mvnArtifact.groupId))end + local version = mvnArtifact.version + local isVersionOk = (type(version) == "string") + if not isVersionOk then warn("Bad version: "..mvnArtifact.groupId.." "..mvnArtifact.artifactId + .." " ..tostring(version)) end return mvnArtifact.groupId .."\t".. mvnArtifact.artifactId - .."\t".. mvnArtifact.version + .."\t".. (isVersionOk and version or "") +end + + +function mod.getMvnArtifactByKey( app, key ) + local gid, aid, version = key:match("^([^\t]+)\t([^\t]+)\t([^\t]+).*$") + local a = mod.newMvnArtifact() + a.artifactId = assert(aid, key) + a.groupId = assert(gid, key) + a.version = version + return a end function mod.onGetPomRspHdr( msg, req ) - log:write("< "..tostring(msg.proto) .." "..tostring(msg.status).." "..tostring(msg.phrase).."\n") - --for i, h in ipairs(msg.headers) do - -- log:write("< ".. h.key ..": ".. h.val .."\n") - --end - --log:write("< \n") if msg.status ~= 200 then + log:write("< "..tostring(msg.proto) .." "..tostring(msg.status).." "..tostring(msg.phrase).."\n") + for i, h in ipairs(msg.headers) do + log:write("< ".. h.key ..": ".. h.val .."\n") + end + log:write("< \n") error("Unexpected HTTP ".. tostring(msg.status)) end assert(not req.pomParser) req.pomParser = objectSeal{ - req = req, - base = false, - xmlElemStack = {}, - currentValue = false, - mvnArtifact = objectSeal{ - parentGroupId = false, - parentArtifactId = false, - parentVersion = false, - groupId = false, - artifactId = false, - version = false, - }, - mvnDependency = false, -- the one we're currently parsing - mvnMngdDependency = false, -- the one we're currently parsing write = function( t, buf ) t.base:write(buf) end, closeSnk = function( t, buf ) t.base:closeSnk() end, } @@ -200,7 +332,7 @@ function mod.onGetPomRspHdr( msg, req ) onElementEnd = function( tag, pomParser ) mod.processXmlValue(pomParser) local elem = table.remove(pomParser.xmlElemStack) - assert(elem.tag == tag); + assert(elem.tag == tag) end, onChunk = function( buf, pomParser ) if pomParser.currentValue then @@ -218,23 +350,27 @@ function mod.onGetPomRspHdr( msg, req ) if not mvnArtifact.groupId then mvnArtifact.groupId = mvnArtifact.parentGroupId end if not mvnArtifact.version then mvnArtifact.version = mvnArtifact.parentVersion end local key = mod.getMvnArtifactKey(mvnArtifact) - assert(not app.mvnArtifacts[key]) - app.mvnArtifacts[key] = mvnArtifact + if app.mvnArtifacts[key] then + local old = app.mvnArtifacts[key] + local oId = mod.getMvnArtifactKey(old) + local nId = mod.getMvnArtifactKey(mvnArtifact) + if oId ~= nId then + print("Already exists BUT DIFFERS:") + for k,v in pairs(old) do print("O",k,v) end + print() + for k,v in pairs(mvnArtifact) do print("N",k,v) end + error("TODO_20221215150040") + else + log:write("Already known. ReUse "..tostring(oId).."\n") + end + else + app.mvnArtifacts[key] = mvnArtifact + end end, } end -function mod.onGetPomRspChunk( buf, req ) - req.pomParser:write(buf) -end - - -function mod.onGetPomRspEnd( req ) - req.pomParser:closeSnk() -end - - function mod.resolveDependencyVersionsFromDepsMgmnt( app ) local mvnArtifacts = app.mvnArtifacts local mvnDepsByArtifact = app.mvnDepsByArtifact @@ -248,7 +384,7 @@ function mod.resolveDependencyVersionsFromDepsMgmnt( app ) if mvnDependency.groupId == mngdDep.groupId and mvnDependency.artifactId == mngdDep.artifactId then - mvnDependency.version = assert(mngdDep.version); + mvnDependency.version = assert(mngdDep.version) break end end @@ -289,9 +425,8 @@ function mod.resolveProperties( app ) return str:match("^%$%{([^}]+)%}$") end for _, mvnArtifact in pairs(mvnArtifacts) do - local set = nil local depsToEnrich = {} - set = mvnDepsByArtifact[mvnArtifact] + local set = mvnDepsByArtifact[mvnArtifact] if set then for _, d in pairs(set) do table.insert(depsToEnrich, d) end end set = mvnMngdDepsByArtifact[mvnArtifact] @@ -300,7 +435,18 @@ function mod.resolveProperties( app ) for _, mvnDependency in pairs(depsToEnrich) do local propKey = getPropKey(mvnDependency.version) if propKey then - local propVal = mod.getPropValThroughParentChain(app, mvnArtifact, propKey) + local propVal + while true do + propVal = mod.getPropValThroughParentChain(app, mvnArtifact, propKey) + if not propVal or not propVal:find("${",0,true) then break end + -- there's a property-in-property. Hangle one further. + propKey = getPropKey(propVal) + if not propKey then + if app.warnIsError then error("Property resolver struggles with "..tostring(propVal)) end + log:write("[WARN ] Cannot resolve property '".. tostring(propVal) .."'\n") -- TODO + break + end + end if propVal then mvnDependency.version = propVal end @@ -326,6 +472,9 @@ function mod.getPropValThroughParentChain( app, mvnArtifact, propKey, none ) if propKey == "project.version" then return mvnArtifact.version end + if propKey == "project.groupId" then + return mvnArtifact.groupId + end -- no luck in current artifact. Delegate to parent (if any) if mvnArtifact.parentGroupId and mvnArtifact.parentArtifactId @@ -378,120 +527,226 @@ function mod.printStuffAtEnd( app ) end +function mod.loadFromSqliteFile( app ) + local db = mod.dbGetInstance(app) + local queryStr = "SELECT id, str FROM String" + local stmt = app.preparedStmts[queryStr] + if not stmt then stmt = db:prepare(queryStr) app.preparedStmts[queryStr] = stmt end + local strings = app.stringIdByStr + -- Load stings + local rs = stmt:execute() + while rs:next() do + local stringKey, stringVal + for iCol=1, rs:numCols() do + local colName = rs:name(iCol) + if colName == "id" then + assert(rs:type(iCol) == "INTEGER") + stringKey = rs:value(iCol) + elseif colName == "str" then + assert(rs:type(iCol) == "TEXT") + stringVal = rs:value(iCol) + else + error("Unexpected col String."..tostring(rs:name(iCol))) + end + end + assert(stringKey) + assert(stringVal) + app.stringIdByStr[stringKey] = stringVal + end + -- Load Artifacts + local stmtMvnArtifacts = db:prepare("" + .." SELECT id, groupId, artifactId, version, parentGroupId, parentArtifactId, parentVersion" + .." FROM MvnArtifact") + local mvnArtifactsByDbId = {} + local rs = stmtMvnArtifacts:execute() + assert(not app.mvnArtifacts) + app.mvnArtifacts = {} + while rs:next() do + local mvnArtif = mod.newMvnArtifact() + for iCol=1, rs:numCols() do + local colName = rs:name(iCol) + if colName == "id" then + mvnArtif.dbId = rs:value(iCol) + else + mvnArtif[colName] = (strings[rs:value(iCol)] or false) + end + end + app.mvnArtifacts[mod.getMvnArtifactKey(mvnArtif)] = mvnArtif; + assert(type(mvnArtif.dbId) == "number", mvnArtif.dbId) + mvnArtifactsByDbId[mvnArtif.dbId] = mvnArtif + end + -- Load Dependencies + local stmtMvnDeps = db:prepare("" + .." SELECT id, mvnArtifactId, needsMvnArtifactId" + .." FROM MvnDependency") + local rs = stmtMvnDeps:execute() + while rs:next() do + local mvnDep = mod.newMvnDependency() + local mvnArtifId, mvnDepId + for iCol=1, rs:numCols() do + local colName = rs:name(iCol) + if colName == "id" then + mvnDep.dbId = assert(rs:value(iCol)) + elseif colName == "mvnArtifactId" then + mvnArtifId = assert(rs:value(iCol)) + elseif colName == "needsMvnArtifactId" then + mvnDepId = assert(rs:value(iCol)) + else + error("TODO_20221215134407 ".. colName) + end + end + local artif = mvnArtifactsByDbId[mvnArtifId] + local dep = mvnArtifactsByDbId[mvnDepId] + local deps = app.mvnDepsByArtifact[artif] + if not deps then deps = {} app.mvnDepsByArtifact[artif] = deps end + table.insert(deps, dep) + end + -- Load properties + local queryStr = "SELECT mvnArtifactId, keyStringId, valStringId FROM MvnProperty" + local stmt = app.preparedStmts[queryStr] + if not stmt then stmt = db:prepare(queryStr); app.preparedStmts[queryStr] = stmt end + local rs = stmt:execute() + while rs:next() do + local prop = { key = false, val = false, } + local mvnArtifactId = false + for iCol=1, rs:numCols() do + local colName = rs:name(iCol) + if false then + elseif colName == "mvnArtifactId" then + mvnArtifactId = rs:value(iCol) + elseif colName == "keyStringId" then + prop.key = assert(strings[rs:value(iCol)]) + elseif colName == "valStringId" then + prop.val = assert(strings[rs:value(iCol)]) + else + error("TODO_20230127134303 ".. tostring(colName)) + end + end + local mvnArtifact = assert(mvnArtifactsByDbId[mvnArtifactId]) + app.mvnPropsByArtifact[mvnArtifact] = assert(prop) + end +end + + +function mod.dbInsertMvnArtifact( app, mvnArtifact ) + if mvnArtifact.dbId then warn("MvnArtifact already has dbId="..tostring(mvnArtifact.dbId)) end + local db = mod.dbGetInstance(app) + local queryStr = "INSERT INTO MvnArtifact" + .." groupId, artifactId, version, parentGroupId, parentArtifactId, parentVersion" + .." VALUES" + .." :groupId, :artifactId, :version, :parentGroupId, :parentArtifactId, :parentVersion" + .." " + local stmt = app.preparedStmts[queryStr] + if not stmt then + stmt = db:prepare(queryStr) + app.preparedStmts[queryStr] = stmt + end + stmt:reset() + mod.bindMvnArtifactAll(app, stmt, mvnArtifact) + stmt:execute() + if db:lastInsertRowid() ~= 0 then + return db:lastInsertRowid() + end + local queryStr = "SELECT id FROM MvnArtifact" + .." WHERE artifactId = :artifactId" + .." AND groupId = :groupId" + .." AND version = :version" + .." AND parentGroupId = :parentGroupId" + .." AND parentArtifactId = :parentArtifactId" + .." AND parentVersion = :parentVersion" + local stmt = app.preparedStmts[queryStr] + stmt:reset() + mod.bindMvnArtifactAll(app, stmt, mvnArtifact) + local rs = stmt:execute() + if not rs:next() then error("TODO_20221215172430") end + mvnArtifact.dbId = assert(rs:value(1)) + if rs:next() then error("TODO_20221215172435") end +end + + +function mod.dbBindMvnArtifactAll( app, stmt, mvnArtifact ) + stmt:bind(":groupId", mvnArtifact.groupId) + stmt:bind(":artifactId", mvnArtifact.artifactId) + stmt:bind(":version", mvnArtifact.version) + stmt:bind(":parentGroupId", mvnArtifact.parentGroupId) + stmt:bind(":parentArtifactId", mvnArtifact.parentArtifactId) + stmt:bind(":parentVersion", mvnArtifact.parentVersion) +end + + function mod.storeAsSqliteFile( app ) - -- TODO could we cache our prepared queries? - local db, stmt - if not app.sqliteOutFile then - log:write("[INFO ] No sqliteOutFile provided. Skip export.\n") - return - end - -- Query to list Artifacts and their parents: - -- SELECT GroupId.str AS 'GID', ArtifactId.str AS 'AID', Version.str AS 'Version', ParentGid.str AS 'ParentGid', ParentAid.str AS 'ParentAid', ParentVersion.str AS 'ParentVersion' - -- FROM MvnArtifact AS A - -- JOIN String GroupId ON GroupId.id = A.groupId - -- JOIN String ArtifactId ON ArtifactId.id = A.artifactId - -- JOIN String Version ON Version.id = A.version - -- JOIN String ParentGid ON ParentGid.id = A.parentGroupId - -- JOIN String ParentAid ON ParentAid.id = A.parentArtifactId - -- JOIN String ParentVersion ON ParentVersion.id = A.parentVersion - -- - -- Query to list dependencies: - -- SELECT GroupId.str AS 'GID', ArtifactId.str AS 'AID', Version.str AS 'Version', DepGid.str AS 'Dependency GID', DepAid.str AS 'Dependnecy AID', DepVersion.str AS 'Dependency Version' - -- FROM MvnArtifact AS A - -- JOIN MvnDependency AS Dep ON Dep.mvnArtifactId = A.id - -- JOIN MvnArtifact AS D ON Dep.needsMvnArtifactId = D.id - -- JOIN String GroupId ON GroupId.id = A.groupId - -- JOIN String ArtifactId ON ArtifactId.id = A.artifactId - -- JOIN String Version ON Version.id = A.version - -- JOIN String DepGid ON DepGid.id = D.groupId - -- JOIN String DepAid ON DepAid.id = D.artifactId - -- JOIN String DepVersion ON DepVersion.id = D.version - -- - db = newSqlite{ - database = app.sqliteOutFile, - } - db:enhancePerf() - db:prepare("CREATE TABLE String (" - .." id INTEGER PRIMARY KEY," - .." str TEXT UNIQUE)" - ):execute() - db:prepare("CREATE TABLE MvnArtifact (" - .." id INTEGER PRIMARY KEY," - .." groupId INT," - .." artifactId INT," - .." version INT," - .." parentGroupId INT," - .." parentArtifactId INT," - .." parentVersion INT)" - ):execute() - db:prepare("CREATE TABLE MvnDependency (" - .." id INTEGER PRIMARY KEY," - .." mvnArtifactId INT," - .." needsMvnArtifactId INT)" - ):execute() - --db:prepare("CREATE TABLE MvnProperty (" - -- .." id INTEGER PRIMARY KEY," - -- .." keyStringId INT," - -- .." valStringId INT)" - --):execute() + local stmt + local db = mod.dbGetInstance(app) local mvnArtifactIds = {} local mvnArtifactIdsByArtif = {} local strings = {} - local getStringId = function( str ) -- create/reUse strings on-demand - if not str then return nil end - local stringId = strings[str] - if not stringId then - local stmt = db:prepare("INSERT INTO String (str)VALUES(:str)") - stmt:reset() - stmt:bind(":str", str) - stmt:execute() - stringId = db:lastInsertRowid() - strings[str] = stringId - end - return stringId - end - local stmtInsMvnArtifact = db:prepare("INSERT INTO MvnArtifact" - .."('groupId', 'artifactId', 'version', 'parentGroupId', 'parentArtifactId', 'parentVersion')" - .."VALUES" - .."(:groupId , :artifactId , :version , :parentGroupId , :parentArtifactId , :parentVersion )") + mod.dbInitTables(app) + local queryStr = "INSERT INTO MvnArtifact" + .." ('groupId', 'artifactId', 'version', 'parentGroupId', 'parentArtifactId', 'parentVersion')" + .." VALUES" + .." (:groupId , :artifactId , :version , :parentGroupId , :parentArtifactId , :parentVersion )" + .." ON CONFLICT DO NOTHING" + local stmt = app.preparedStmts[queryStr] + if not stmt then stmt = db:prepare(queryStr) app.preparedStmts[queryStr] = stmt end local insertMvnArtifact = function(a) - assert(a.groupId and a.artifactId and a.version) + if a.dbId then + -- TODO analyze this case. + --log:write("[WARN ] MvnArtifact "..tostring(a.dbId).." probably already exists. Insert it again\n") + end + assert(a.groupId and a.artifactId) + if not a.version then warn("version missing of "..a.groupId.." "..a.artifactId) end if a.parentGroupId then assert(a.parentArtifactId and a.parentVersion) else assert(not a.parentArtifactId and not a.parentVersion) end - stmtInsMvnArtifact:reset() - stmtInsMvnArtifact:bind(":groupId", getStringId(a.groupId)) - stmtInsMvnArtifact:bind(":artifactId", getStringId(a.artifactId)) - stmtInsMvnArtifact:bind(":version", getStringId(a.version)) - stmtInsMvnArtifact:bind(":parentGroupId", getStringId(a.parentGroupId)) - stmtInsMvnArtifact:bind(":parentArtifactId", getStringId(a.parentArtifactId)) - stmtInsMvnArtifact:bind(":parentVersion", getStringId(a.parentVersion)) - stmtInsMvnArtifact:execute() + local versionDbId = (a.version and mod.dbGetOrNewString(app, a.version) or nil) + stmt:reset() + stmt:bind(":groupId", mod.dbGetOrNewString(app, a.groupId)) + stmt:bind(":artifactId", mod.dbGetOrNewString(app, a.artifactId)) + stmt:bind(":version", versionDbId) + stmt:bind(":parentGroupId", mod.dbGetOrNewString(app, a.parentGroupId)) + stmt:bind(":parentArtifactId", mod.dbGetOrNewString(app, a.parentArtifactId)) + stmt:bind(":parentVersion", mod.dbGetOrNewString(app, a.parentVersion)) + stmt:execute() local dbId = db:lastInsertRowid() + if dbId == 0 then + -- Seems as entry already exists. So need to query its id separately. + local stmt = db:prepare("SELECT id FROM MvnArtifact" + .." WHERE groupId = :groupId AND artifactId = :artifactId AND version = :version" + .." AND parentGroupId = :parentGroupId AND parentArtifactId = :parentArtifactId AND parentVersion = :parentVersion") + stmt:reset() + stmt:bind(":groupId", mod.dbGetOrNewString(app, a.groupId)) + stmt:bind(":artifactId", mod.dbGetOrNewString(app, a.artifactId)) + stmt:bind(":version", versionDbId) + stmt:bind(":parentGroupId", mod.dbGetOrNewString(app, a.parentGroupId)) + stmt:bind(":parentArtifactId", mod.dbGetOrNewString(app, a.parentArtifactId)) + stmt:bind(":parentVersion", mod.dbGetOrNewString(app, a.parentVersion)) + local rs = stmt:execute() + dbId = rs:value(1) + assert(dbId) + end mvnArtifactIds[a] = dbId -- TODO MUST be byString local bucket = mvnArtifactIdsByArtif[assert(a.artifactId)] if not bucket then bucket = {} mvnArtifactIdsByArtif[a.artifactId] = bucket end table.insert(bucket, { dbId = dbId, mvnArtifact = a, }) return dbId end - -- Store artifacts + -- Store new artifacts for _, mvnArtifact in pairs(app.mvnArtifacts) do insertMvnArtifact(mvnArtifact) - local mvnDeps = app.mvnDepsByArtifact[mvnArtifact] or {} - -- dependencies are nothing else than artifacts - for _, mvnDep in pairs(mvnDeps) do + if mvnArtifact.parentArtifactId then + -- TODO? maybe? end end -- Store dependencies - local stmt = db:prepare("INSERT INTO MvnDependency" - .."('mvnArtifactId', 'needsMvnArtifactId')" - .."VALUES" - .."(:mvnArtifactId , :needsMvnArtifactId )") + local queryStr = "INSERT INTO MvnDependency" + .." ( mvnArtifactId, needsMvnArtifactId)" + .." VALUES" + .." ( :mvnArtifactId, :needsMvnArtifactId)" + local stmt = app.preparedStmts[queryStr] + if not stmt then stmt = db:prepare(queryStr) app.preparedStmts[queryStr] = stmt end for _, mvnArtifact in pairs(app.mvnArtifacts) do local mvnDeps = app.mvnDepsByArtifact[mvnArtifact] for _, mvnDep in pairs(mvnDeps or {}) do - if not mvnDep.version then mvnDep.version = "TODO_5bbc0e87011e24d845136c5406302616" end - assert(mvnDep.version, mvnDep.artifactId) - assert(mvnDep.groupId and mvnDep.artifactId and mvnDep.version) + assert(mvnDep.groupId and mvnDep.artifactId) local bucket = mvnArtifactIdsByArtif[mvnDep.artifactId] local depId = nil for _,a in pairs(bucket or {}) do @@ -503,9 +758,11 @@ function mod.storeAsSqliteFile( app ) end if not depId then -- Artifact not stored yet. Do now. depId = insertMvnArtifact({ - groupId = mvnDep.groupId, - artifactId = mvnDep.artifactId, - version = mvnDep.version, + groupId = assert(mvnDep.groupId), + artifactId = assert(mvnDep.artifactId), + -- mvnDep.version MAY be missing. Eg via depMgnt of + -- unknown parent or similar + version = (mvnDep.version), }) end stmt:reset() @@ -514,119 +771,559 @@ function mod.storeAsSqliteFile( app ) stmt:execute() end end - db:close() + -- Store properties + local queryStr = "INSERT INTO MvnProperty" + .." ( mvnArtifactId, keyStringId, valStringId )" + .." VALUES" + .." ( :mvnArtifactId, :keyStringId, :valStringId )" + local stmt = app.preparedStmts[queryStr] + if not stmt then stmt = db:prepare(queryStr); app.preparedStmts[queryStr] = stmt end + for _, mvnArtifact in pairs(app.mvnArtifacts) do + assert(type(mvnArtifact) == "table") + local dbId = mvnArtifactIds[mvnArtifact] + local mvnProps = app.mvnPropsByArtifact[mvnArtifact] + if not mvnProps then goto nextArtifact end + assert(type(mvnProps) == "table") + for _, mvnProp in ipairs(mvnProps) do + stmt:reset() + stmt:bind(":mvnArtifactId", assert(dbId)) + stmt:bind(":keyStringId", mod.dbGetOrNewString(app, assert(mvnProp.key))) + stmt:bind(":valStringId", mod.dbGetOrNewString(app, assert(mvnProp.val))) + stmt:execute() + --local dbId = db:lastInsertRowid() + end + ::nextArtifact:: + end end -function mod.run( app ) - assert(not app.mvnArtifacts) app.mvnArtifacts = {} - assert(not app.mvnPropsByArtifact) app.mvnPropsByArtifact = {} - assert(not app.mvnDepsByArtifact) app.mvnDepsByArtifact = {} - assert(not app.mvnMngdDepsByArtifact) app.mvnMngdDepsByArtifact = {} - local pomSrc = mod.newPomUrlSrc(app) - while true do - local pomUrl = pomSrc:nextPomUrl() - if not pomUrl then break end - local proto = pomUrl:match("^(https?)://") - local isTLS = (proto:upper() == "HTTPS") - local host = pomUrl:match("^https?://([^:/]+)[:/]") - local port = pomUrl:match("^https?://[^:/]+:(%d+)[^%d]") - local url = pomUrl:match("^https?://[^/]+(.*)$") - if port == 443 then isTLS = true end - if not port then port = (isTLS and 443 or 80) end - log:write("> GET ".. proto .."://".. host ..":".. port .. url .."\n") - local req = objectSeal{ - app = app, - base = false, - pomParser = false, - } - req.base = app.http:request{ - cls = req, - host = assert(host), port = assert(port), - method = "GET", url = url, - --hdrs = , - useTLS = isTLS, - onRspHdr = mod.onGetPomRspHdr, - onRspChunk = mod.onGetPomRspChunk, - onRspEnd = mod.onGetPomRspEnd, - } - req.base:closeSnk() +-- returns dbId of the (new or existing) string +function mod.dbGetOrNewString( app, str ) + local db = mod.dbGetInstance(app) + local tryCnt = 0 + --log:write("[DEBUG] Searching String ID for '"..tostring(str).."'\n") + if not str then return nil end +::startOver:: + -- Ask inMemory cache + local stringId = app.stringIdByStr[str] + if stringId then + --log:write("[DEBUG] Using String ".. stringId .." for '"..str.."'\n") + return stringId end - log:write("[INFO ] No more pom URLs\n") - mod.resolveDependencyVersionsFromDepsMgmnt(app) - mod.resolveProperties(app) - mod.storeAsSqliteFile(app) - --mod.printStuffAtEnd(app) + -- Ask DB + local queryStr = "SELECT id FROM String WHERE str = :str" + local stmt = app.preparedStmts[queryStr] + if not stmt then stmt = db:prepare(queryStr) app.preparedStmts[queryStr] = stmt end + stmt:reset() + stmt:bind(":str", str) + local rs = stmt:execute() + if rs:next() then -- DB has an entry :) + stringId = assert(rs:value(1)) + if rs:next() then log:write("[WARN ] DB string duplication: '"..tostring(str).."'\n") end + --log:write("[DEBUG] Using OLD String ".. stringId .."\n") + app.stringIdByStr[str] = stringId + return stringId + end + stmt:close() app.preparedStmts[queryStr] = nil -- TODO WTF?!? + --log:write("[DEBUG] None in DB yet. Make sure it exists\n") + local queryStr = "INSERT INTO String (str)VALUES(:str) ON CONFLICT DO NOTHING" + local stmt = app.preparedStmts[queryStr] + if not stmt then stmt = db:prepare(queryStr) app.preparedStmts[queryStr] = stmt end + stmt:reset() + stmt:bind(":str", str) + stmt:execute() + stmt:close() app.preparedStmts[queryStr] = nil -- TODO WTF?!? + --log:write("[DEBUG] Then try again\n") + if tryCnt > 3 then error("TODO_20221215185428 fixme") end + tryCnt = tryCnt +1 + goto startOver -- recursion stinks end +function mod.dbInitTables( app ) + local db = mod.dbGetInstance(app) + db:prepare("CREATE TABLE IF NOT EXISTS String (" + .." id INTEGER PRIMARY KEY," + .." str TEXT UNIQUE)" + ):execute() + db:prepare("CREATE TABLE IF NOT EXISTS MvnArtifact (" + .." id INTEGER PRIMARY KEY," + .." groupId INT," + .." artifactId INT," + .." version INT," + .." parentGroupId INT," + .." parentArtifactId INT," + .." parentVersion INT)" + ):execute() + db:prepare("CREATE TABLE IF NOT EXISTS MvnDependency (" + .." id INTEGER PRIMARY KEY," + .." mvnArtifactId INT," + .." needsMvnArtifactId INT)" + ):execute() + db:prepare("CREATE TABLE IF NOT EXISTS MvnProperty (" + .." id INTEGER PRIMARY KEY," + .." mvnArtifactId INTEGER," + .." keyStringId INTEGER," + .." valStringId INTEGER)" + ):execute() +end + + +function mod.dbGetInstance( app ) + local db = app.sqlite + if not db then + db = newSqlite{ database = app.statePath, } + db:enhancePerf() + app.sqlite = db + end + return db +end + + +-- Using custom impl because builtin cannot do connection pooling yet function mod.newSocketMgr() - local hosts = {} - local openSock = function( t, opts ) + local AF_INET = require('scriptlee').posix.AF_INET + local AF_INET6 = require('scriptlee').posix.AF_INET6 + local IPPROTO_TCP = require('scriptlee').posix.IPPROTO_TCP + local SOCK_STREAM = require('scriptlee').posix.SOCK_STREAM + local inaddrOfHostname = require('scriptlee').posix.inaddrOfHostname + local newTlsClient = require('scriptlee').newTlsClient + local socket = require('scriptlee').posix.socket + local S = {} + local idleSocketsBySockaddr = {} + + local openSock = function(t, opts) for k, v in pairs(opts) do if false then - elseif k=='host' or k=='port' then - elseif k=='useTLS' then - if v then error('TLS not impl') end + elseif k=='host' or k=='port' or k=='useTLS' then + -- ok else error('Unknown option: '..tostring(k)) end end + local inaddr = inaddrOfHostname(opts.host) local af if inaddr:find('^%d+.%d+.%d+.%d+$') then af = AF_INET else af = AF_INET6 end - local sock = socket(af, SOCK_STREAM, IPPROTO_TCP) - sock:connect(inaddr, opts.port) - log:write("opts.useTLS "..tostring(opts.useTLS).." (Override to TRUE ...)\n") - opts.useTLS = true -- TODO why the heck is this needed? (I guess scriptlee bug?) - if opts.useTLS then - local sockUnderTls = sock - sock = newTlsClient{ - cls = assert(sockUnderTls), - peerHostname = assert(opts.host), - onVerify = function( tlsIssues, sockUnderTls ) - if tlsIssues.CERT_NOT_TRUSTED then - warn("TLS ignore CERT_NOT_TRUSTED"); - tlsIssues.CERT_NOT_TRUSTED = false - end - end, - send = function( buf, sockUnderTls ) - local ret = sockUnderTls:write(buf) - sockUnderTls:flush() -- TODO Why is this flush needed? - return ret - end, - recv = function( sockUnderTls ) return sockUnderTls:read() end, - flush = function( sockUnderTls ) sockUnderTls:flush() end, - closeSnk = function( sockUnderTls ) sockUnderTls:closeSnk() end, - } - assert(not getmetatable(sock).release) - getmetatable(sock).release = function( t ) sockUnderTls:release() end; + local sockaddr = inaddr ..":".. (opts.port or "-1") + local poolForThisHost = idleSocketsBySockaddr[sockaddr] + local sock = poolForThisHost and table.remove(poolForThisHost) or false + if not sock then + -- no sock from pool. Create new one. + sock = socket(af, SOCK_STREAM, IPPROTO_TCP) + sock:connect(inaddr, opts.port) + if opts.useTLS then + local sockUnderTls = sock + sock = newTlsClient{ + cls = sockUnderTls, + peerHostname = opts.host, + onVerify = function( tlsIssues, sockUnderTls ) + if tlsIssues.CERT_NOT_TRUSTED then + warn('TLS ignore CERT_NOT_TRUSTED'); + tlsIssues.CERT_NOT_TRUSTED = false + end + end, + send = function( buf, sockUnderTls ) + local ret = sockUnderTls:write(buf) + sockUnderTls:flush() + return ret + end, + recv = function( sockUnderTls ) return sockUnderTls:read() end, + flush = function( sockUnderTls ) sockUnderTls:flush() end, + closeSnk = function( sockUnderTls ) sockUnderTls:closeSnk() end, + } + assert(not getmetatable(sock).release) + getmetatable(sock).release = function( t ) sockUnderTls:release() end; + end end - return { - _sock = sock, - write = function(t, ...) return sock:write(...)end, - read = function(t, ...) return sock:read(...)end, - flush = function(t, ...) return sock:flush(...)end, + return{ + [S] = sock, + _sockaddr = sockaddr, + write = function(t, ...)return sock:write(...)end, + read = function(t, ...)return sock:read(...)end, + flush = function(t, ...)return sock:flush(...)end, } end + + local releaseSock = function( t, mySock ) + -- TODO just ignroe cleanup for now because we have no connection pooling yet. + local poolForThisHost = idleSocketsBySockaddr[mySock._sockaddr] + if not poolForThisHost then + poolForThisHost = {} + idleSocketsBySockaddr[mySock._sockaddr] = poolForThisHost + end + table.insert(poolForThisHost, assert(mySock[S])) + end + + local closeSock = function( t, mySock ) + mySock[S]:release() + end + return{ openSock = openSock, - releaseSock = function(t, sockWrapr) t:closeSock(sockWrapr) end, - closeSock = function(t, sockWrapr) sockWrapr._sock:release() end, + releaseSock = releaseSock, + closeSock = closeSock, } end +function mod.printCsvParents( app ) + local db = mod.dbGetInstance(app) + local queryStr = "" -- Query + .." SELECT DISTINCT" + .." GroupId.str," + .." ArtifactId.str," + .." Version.str," + .." ParentGid.str," + .." ParentAid.str," + .." ParentVersion.str" + .." FROM MvnArtifact AS A" + .." JOIN String GroupId ON GroupId.id = A.groupId" + .." JOIN String ArtifactId ON ArtifactId.id = A.artifactId" + .." JOIN String Version ON Version.id = A.version" + .." LEFT JOIN String ParentGid ON ParentGid.id = A.parentGroupId" + .." LEFT JOIN String ParentAid ON ParentAid.id = A.parentArtifactId" + .." LEFT JOIN String ParentVersion ON ParentVersion.id = A.parentVersion" + local stmt = app.preparedStmts[queryStr] + if not stmt then stmt = db:prepare(queryStr) app.preparedStmts[queryStr] = stmt end + stmt:reset() + local rs = stmt:execute() + out:write("h;Created;"..mod.escapeCsvValue(os.date("%Y-%m-%d %H:%m:%S")).."\n") + out:write("c;GID;AID;Version;ParentGID;ParentAID;ParentVersion\n") + local nilVal = app.nullvalue + while rs:next() do + out:write("r;") out:write(mod.escapeCsvValue(rs:value(1) or nilVal)) + out:write(";") out:write(mod.escapeCsvValue(rs:value(2) or nilVal)) + out:write(";") out:write(mod.escapeCsvValue(rs:value(3) or nilVal)) + out:write(";") out:write(mod.escapeCsvValue(rs:value(4) or nilVal)) + out:write(";") out:write(mod.escapeCsvValue(rs:value(5) or nilVal)) + out:write(";") out:write(mod.escapeCsvValue(rs:value(6) or nilVal)) + out:write("\n") + end + out:write("t;status;OK\n") +end + + +function mod.printCsvDependencies( app ) + local db = mod.dbGetInstance(app) + local queryStr = "" -- Query + .." SELECT DISTINCT" + .." GroupId.str," + .." ArtifactId.str," + .." Version.str," + .." DepGid.str," + .." DepAid.str," + .." DepVersion.str" + .." FROM MvnArtifact AS A" + .." JOIN MvnDependency AS Dep ON Dep.mvnArtifactId = A.id" + .." LEFT JOIN MvnArtifact AS D ON Dep.needsMvnArtifactId = D.id" + .." LEFT JOIN String GroupId ON GroupId.id = A.groupId" + .." LEFT JOIN String ArtifactId ON ArtifactId.id = A.artifactId" + .." LEFT JOIN String Version ON Version.id = A.version" + .." LEFT JOIN String DepGid ON DepGid.id = D.groupId" + .." LEFT JOIN String DepAid ON DepAid.id = D.artifactId" + .." LEFT JOIN String DepVersion ON DepVersion.id = D.version" + local stmt = app.preparedStmts[queryStr] + if not stmt then stmt = db:prepare(queryStr) app.preparedStmts[queryStr] = stmt end + stmt:reset() + local rs = stmt:execute() + out:write("h;Created;"..mod.escapeCsvValue(os.date("%Y-%m-%d %H:%m:%S")).."\n") + out:write("c;GID;AID;Version;DepGID;DepAID;DepVersion\n") + local nilVal = app.nullvalue + while rs:next() do + out:write("r;") out:write(mod.escapeCsvValue(rs:value(1) or nilVal)) + out:write(";") out:write(mod.escapeCsvValue(rs:value(2) or nilVal)) + out:write(";") out:write(mod.escapeCsvValue(rs:value(3) or nilVal)) + out:write(";") out:write(mod.escapeCsvValue(rs:value(4) or nilVal)) + out:write(";") out:write(mod.escapeCsvValue(rs:value(5) or nilVal)) + out:write(";") out:write(mod.escapeCsvValue(rs:value(6) or nilVal)) + out:write("\n") + end + out:write("t;status;OK\n") +end + + +function mod.escapeCsvValue( str ) + local typ = type(str) + if typ == "string" then + if str:find("[;\r\n\"]") then + str = '"'.. str:gsub('"', '""') ..'"' end + else + error("TODO_20221215181624 "..tostring(typ)) + end + return str +end + + +function mod.enrichFromCbacks( app, opts ) + local writeNextPomTo = assert(opts.writeNextPomTo) + local onParentPomMissing = assert(opts.onParentPomMissing) + opts = nil + while true do -- TODO is this loop reall what we want? + local pomParser = false + local ok = writeNextPomTo(objectSeal{ + write = function( t, buf, beg, len ) + if not pomParser then + pomParser = objectSeal{ + app = app, + base = false, + xmlElemStack = {}, + currentValue = false, + mvnArtifact = mod.newMvnArtifact(), + mvnDependency = false, -- the one we're currently parsing + mvnMngdDependency = false, -- the one we're currently parsing + write = function( pomParser, buf, beg, len ) + assert(beg == 1) + assert(buf:len() == len) + return pomParser.base:write(buf) + end, + closeSnk = function( pomParser ) + return pomParser.base:closeSnk() + end, + } + pomParser.base = newXmlParser{ + cls = pomParser, + onElementBeg = function( tag, pomParser ) + table.insert(pomParser.xmlElemStack, { tag = tag, }) + pomParser.currentValue = false + end, + onElementEnd = function( tag, pomParser ) + mod.processXmlValue(pomParser) + local elem = table.remove(pomParser.xmlElemStack) + assert(elem.tag == tag); + end, + onChunk = function( buf, pomParser ) + if pomParser.currentValue then + pomParser.currentValue = pomParser.currentValue .. buf + else + pomParser.currentValue = buf + end + end, + onEnd = function( pomParser ) + assert(#pomParser.xmlElemStack == 0) + local app = pomParser.app + local mvnArtifact = pomParser.mvnArtifact + pomParser.mvnArtifact = false + if not mvnArtifact.groupId then + mvnArtifact.groupId = mvnArtifact.parentGroupId end + if not mvnArtifact.version then + mvnArtifact.version = mvnArtifact.parentVersion end + local key = mod.getMvnArtifactKey(mvnArtifact) + if app.mvnArtifacts[key] then + local old = app.mvnArtifacts[key] + local oId = mod.getMvnArtifactKey(old) + local nId = mod.getMvnArtifactKey(mvnArtifact) + if oId ~= nId then + print("Already exists BUT DIFFERS:") + for k,v in pairs(old) do print("O",k,v) end + print() + for k,v in pairs(mvnArtifact) do print("N",k,v) end + error("TODO_20221215150040") + else + log:write("Already known. ReUse "..tostring(oId).."\n") + end + else + app.mvnArtifacts[key] = mvnArtifact + end + -- Check for missing poms. + if mvnArtifact.parentArtifactId then + local key = mod.getMvnArtifactKey({ + artifactId = mvnArtifact.parentArtifactId, + groupId = mvnArtifact.parentGroupId, + version = mvnArtifact.parentVersion, + }) + if not app.mvnArtifacts[key] then -- parent pom missing + onParentPomMissing( + mvnArtifact.parentGroupId, + mvnArtifact.parentArtifactId, + mvnArtifact.parentVersion) + end + end + end, + } + end + pomParser:write(buf, beg, len) + end, + closeSnk = function() + if not pomParser then + return -- can happen on 404 because empty body (see also close in http rsp handler) + end + pomParser:closeSnk() + end, + }) + if not ok then break end + end + log:write("[INFO ] No more pom URLs\n") + mod.resolveDependencyVersionsFromDepsMgmnt(app) + mod.resolveProperties(app) + mod.storeAsSqliteFile(app) + log:write("\n\nState DUMP:\n\n") + mod.printStuffAtEnd(app) +end + + +function mod.fileExists( pomFilepath ) + local fd = io.open(pomFilepath) + local exists = not not fd + fd:close() + return exists +end + + +-- Deprecated. Use the callback variant +function mod.enrichFromUrls( app ) + local pomSrc = mod.newPomUrlSrc(app) + local missingPoms, missingDone = {}, {} + mod.enrichFromCbacks(app, objectSeal{ + onParentPomMissing = function( gid, aid, version ) + local a = mod.newMvnArtifact() + a.artifactId = aid + a.groupId = gid + a.version = version + local artifactKey = mod.getMvnArtifactKey(a) + local url = mod.urlByArtifact(app, a) + assert(artifactKey) + if not missingDone[artifactKey] then missingPoms[artifactKey] = true end + end, + writeNextPomTo = function( snk ) + local pomArtifact = pomSrc:nextPomArtifact() + local pomKey = nil + if not pomArtifact then + pomKey, _ = pairs(missingPoms)(missingPoms) + if pomKey then + pomArtifact = mod.getMvnArtifactByKey(app, pomKey) + missingDone[pomKey] = true + missingPoms[pomKey] = nil + log:write("NeedAlso: ".. pomKey .."\n") + end + end + if not pomArtifact then + log:write("No more poms\n") + return false + end + local pomFilepath = mod.pomFilepathByArtifact(app, pomArtifact) + local fd = io.open(pomFilepath, "rb") + -- MUST NOT use local cache for mvn SNAPSHOTs + -- MUST NOT try to read server-placholder '[RELEASE]' from local cache + if fd and not pomFilepath:find("SNAPSHOT",0,true) and not pomFilepath:find("RELEASE",0,true) then + log:write("> fread(".. pomFilepath ..")\n") + local file = io.open(pomFilepath, "rb") + while true do + local buf = file:read(1<<14) + if buf then + snk:write(buf, 1, buf:len()) + else + file:close() + snk:closeSnk() + return true + end + end + --else + -- log:write("[DEBUG] NoSuchFile ".. pomFilepath .."\n") + end + -- No local file. Go the HTTP way. + local pomUrl = mod.urlByArtifact(app, pomArtifact) + local proto = pomUrl:match("^(https?)://") + local isTLS = (proto:upper() == "HTTPS") + local host = pomUrl:match("^https?://([^:/]+)[:/]") + local port = pomUrl:match("^https?://[^:/]+:(%d+)[^%d]") + local url = pomUrl:match("^https?://[^/]+(.*)$") + if port == 443 then isTLS = true end + if not port then port = (isTLS and 443 or 80) end + log:write("> GET ".. proto .."://".. host ..":".. port .. url .."\n") + local req = objectSeal{ + app = app, + base = false, + pomParser = false, + pomArtifact = pomArtifact, + responseIsOk = false, + } + req.base = app.http:request{ + cls = req, + host = assert(host), port = assert(port), + method = "GET", url = url, + useTLS = isTLS, + onRspHdr = function( msg, req ) + if msg.status ~= 200 then + log:write("< "..tostring(msg.proto) .." "..tostring(msg.status).." "..tostring(msg.phrase).."\n") + for i, h in ipairs(msg.headers) do + log:write("< ".. tostring(h[1]) ..": ".. tostring(h[2]) .."\n") + end + log:write("< \n") + local a = req.pomArtifact + local msg = "Unexpected HTTP ".. tostring(msg.status) .." for ".. a.groupId .." ".. a.artifactId .." ".. a.version + if app.warnIsError then error(msg) else log:write("[WARN ] ".. msg .."\n") end + else + req.responseIsOk = true + end + end, + onRspChunk = function( buf, req ) + if req.responseIsOk then snk:write(buf, 1, buf:len()) end + end, + onRspEnd = function( req ) + snk:closeSnk() + end, + } + local ok, emsg = pcall(req.base.closeSnk, req.base) + if not ok then + if tostring(emsg) == "ENOMSG" then + -- This is a bug in scriptlee. It should report 404. + log:write(tostring(emsg).."\n") + snk:closeSnk() + else + error(emsg) + end + end + return true + end, + }) +end + + +function mod.run( app ) + assert(not app.mvnPropsByArtifact) app.mvnPropsByArtifact = {} + assert(not app.mvnDepsByArtifact) app.mvnDepsByArtifact = {} + assert(not app.mvnMngdDepsByArtifact) app.mvnMngdDepsByArtifact = {} + local fileExists = io.open(app.statePath, "rb") + if fileExists then + io.close(fileExists) + mod.loadFromSqliteFile(app) + else + assert(not app.mvnArtifacts) + app.mvnArtifacts = {} + end + if false then + elseif app.asCsv == "parents" then + mod.printCsvParents(app) + elseif app.asCsv == "deps" then + mod.printCsvDependencies(app) + elseif app.csvToFetch then + mod.enrichFromUrls(app) + else + error("TODO_20221215175852") + end + if app.sqlite then app.sqlite:close() app.sqlite = false end +end + + function mod.main() local app = objectSeal{ http = newHttpClient{ socketMgr = assert(mod.newSocketMgr()), }, + warnIsError = false, + csvToFetch = false, + uripat = false, + asCsv = false, + nullvalue = false, mvnArtifacts = false, mvnPropsByArtifact = false, mvnDepsByArtifact = false, mvnMngdDepsByArtifact = false, - sqliteOutFile = false, + sqlite = false, + statePath = false, + preparedStmts = {}, + stringIdByStr = {}, } if mod.parseArgs(app) ~= 0 then os.exit(1) end mod.run(app) |