Fixed issues with scanning all dependencies

src/dorkbox/gradle/DependencyScanner.kt

@@ -19,22 +19,71 @@ import org.gradle.api.Project
 import org.gradle.api.artifacts.ResolvedArtifact
 import org.gradle.api.artifacts.ResolvedDependency
 import java.io.File
+import java.util.*
 
 object DependencyScanner {
 
     /**
      * THIS MUST BE IN "afterEvaluate" or run from a specific task.
      *
-     *  how to resolve dependencies
      *  NOTE: it is possible, when we have a project DEPEND on an older version of that project (ie: bootstrapped from an older version)
-     *    we can have infinite recursion.
+     *    we can have quite deep recursion. A project can never depend on itself, but we check if a project has already been added, and
-     *    This is a problem, so we limit how much a dependency can show up the the tree
+     *    don't parse it more than once
+     *
+     *    This is an actual problem...
      */
-    fun scan(
+    fun scan(project: Project, configurationName: String, includeChildren: Boolean = true): List<DependencyData> {
+
+        val projectDependencies = mutableListOf<DependencyData>()
+        val config = project.configurations.getByName(configurationName)
+        if (!config.isCanBeResolved) {
+            return projectDependencies
+        }
+
+        config.resolve()
+
+        val list = LinkedList<ResolvedDependency>()
+
+        config.resolvedConfiguration.lenientConfiguration.getFirstLevelModuleDependencies(org.gradle.api.specs.Specs.SATISFIES_ALL).forEach { dep ->
+            list.add(dep)
+        }
+
+        var next: ResolvedDependency
+        while (list.isNotEmpty()) {
+            next = list.poll()
+
+            val module = next.module.id
+            val group = module.group
+            val name = module.name
+            val version = module.version
+
+            val artifacts: List<Artifact> = next.moduleArtifacts.map { artifact: ResolvedArtifact ->
+                val artifactModule = artifact.moduleVersion.id
+                Artifact(artifactModule.group, artifactModule.name, artifactModule.version, artifact.file.absoluteFile)
+            }
+
+            projectDependencies.add(DependencyData(group, name, version, artifacts))
+            if (includeChildren) {
+                list.addAll(next.children)
+            }
+        }
+
+        return projectDependencies
+    }
+    /**
+     * THIS MUST BE IN "afterEvaluate" or run from a specific task.
+     *
+     *  NOTE: it is possible, when we have a project DEPEND on an older version of that project (ie: bootstrapped from an older version)
+     *    we can have quite deep recursion. A project can never depend on itself, but we check if a project has already been added, and
+     *    don't parse it more than once
+     *
+     *    This is an actual problem...
+     */
+    fun createTree(
         project: Project,
         configurationName: String,
         projectDependencies: MutableList<Dependency>,
-        existingNames: MutableSet<String>,
+        existingDeps: MutableMap<String, Dependency>,
     ) {
 
         val config = project.configurations.getByName(configurationName)
@@ -44,50 +93,48 @@ object DependencyScanner {
 
         config.resolve()
 
+        // the root parent is tossed out, but not the topmost list of dependencies
+        val rootParent = Dependency("", "", "", listOf(), projectDependencies)
+
+        val parentList = LinkedList<Dependency>()
+        val list = LinkedList<ResolvedDependency>()
+
         config.resolvedConfiguration.lenientConfiguration.getFirstLevelModuleDependencies(org.gradle.api.specs.Specs.SATISFIES_ALL).forEach { dep ->
-            // we know the FIRST series will exist
+            list.add(dep)
-            val makeDepTree = makeDepTree(dep, existingNames)
+            parentList.add(rootParent)
-            if (makeDepTree != null) {
-                // it's only null if we've ALREADY scanned it
-                if (!projectDependencies.contains(makeDepTree)) {
-                    projectDependencies.add(makeDepTree)
-                }
-            }
-        }
         }
 
-    // how to resolve dependencies
+
-    // NOTE: it is possible, when we have a project DEPEND on an older version of that project (ie: bootstrapped from an older version)
+        var next: ResolvedDependency
-    //  we can have infinite recursion.
+        while (list.isNotEmpty()) {
-    //  This is a problem, so we limit how much a dependency can show up the the tree
+            next = list.poll()
-    private fun makeDepTree(dep: ResolvedDependency, existingNames: MutableSet<String>): Dependency? {
+
-        val module = dep.module.id
+            val module = next.module.id
             val group = module.group
             val name = module.name
             val version = module.version
+            val mavenId = "$group:$name:$version"
 
-        if (!existingNames.contains("$group:$name")) {
+            if (!existingDeps.containsKey(mavenId)) {
-            existingNames.add("$group:$name")
+                val artifacts: List<Artifact> = next.moduleArtifacts.map { artifact: ResolvedArtifact ->
-
-            // println("Searching: $group:$name:$version")
-            val artifacts: List<Artifact> = dep.moduleArtifacts.map { artifact: ResolvedArtifact ->
                     val artifactModule = artifact.moduleVersion.id
                     Artifact(artifactModule.group, artifactModule.name, artifactModule.version, artifact.file.absoluteFile)
                 }
 
-            val children = mutableListOf<Dependency>()
+                val dependency = Dependency(group, name, version, artifacts, mutableListOf())
-            dep.children.forEach { child ->
+
-                val makeDep = makeDepTree(child, existingNames)
+                // now add to our parent
-                if (makeDep != null) {
+                val parent = parentList.poll()
-                    children.add(makeDep)
+                (parent.children as MutableList).add(dependency)
-                }
+
+                next.children.forEach { child ->
+                    parentList.add(dependency)
+                    list.add(child)
                 }
 
-            return Dependency(group, name, version, artifacts, children.toList())
+                existingDeps[mavenId] = dependency
+            }
         }
-
-        // we already have this dependency in our chain.
-        return null
     }
 
     /**
@@ -106,6 +153,24 @@ object DependencyScanner {
         }
     }
 
+    data class ProjectDependencies(val tree: List<Dependency>, val dependencies: List<Dependency>)
+
+    data class DependencyData(
+        val group: String,
+        val name: String,
+        val version: String,
+        val artifacts: List<Artifact>
+    ) {
+
+        fun mavenId(): String {
+            return "$group:$name:$version"
+        }
+
+        override fun toString(): String {
+            return mavenId()
+        }
+    }
+
     data class Dependency(
         val group: String,
         val name: String,

src/dorkbox/gradle/StaticMethodsAndTools.kt

@@ -175,14 +175,49 @@ open class StaticMethodsAndTools(private val project: Project) {
         return repositories
     }
 
+    /**
+     * Gets all of the Maven-style Repository URLs for the specified project (or for the root project if not specified).
+     *
+     * @param project which project to get the repository root URLs for
+     * @param onlyRemote true to ONLY get the remote repositories (ie: don't include mavenLocal)
+     */
+    fun getProjectBuildScriptRepositoryUrls(project: Project = this.project, onlyRemote: Boolean = true): List<String> {
+        val repositories = mutableListOf<String>()
+        project.buildscript.repositories.filterIsInstance<org.gradle.api.internal.artifacts.repositories.ResolutionAwareRepository>()
+            .forEach {
+                repo ->
+            val resolver = repo.createResolver()
+            if (resolver is org.gradle.api.internal.artifacts.repositories.resolver.MavenResolver) {
+                // println("searching ${resolver.name}")
+                // println(resolver.root)
+                // all maven patterns are the same!
+                // https://plugins.gradle.org/m2/com/dorkbox/Utilities/maven-metadata.xml
+                // https://repo1.maven.org/maven2/com/dorkbox/Utilities/maven-metadata.xml
+                // https://repo.maven.apache.org/com/dorkbox/Utilities/maven-metadata.xml
+
+                if ((onlyRemote && !resolver.isLocal) || !onlyRemote) {
+                    try {
+                        val toURL = resolver.root.toASCIIString()
+                        if (toURL.endsWith('/')) {
+                            repositories.add(toURL)
+                        } else {
+                            // the root doesn't always end with a '/', and we must guarantee that
+                            repositories.add("$toURL/")
+                        }
+                    } catch (e: Exception) {
+                    }
+                }
+            }
+        }
+        return repositories
+    }
+
     /**
      * Resolves all dependencies of the project buildscript
      *
      * THIS MUST BE IN "afterEvaluate" or run from a specific task.
      */
     fun resolveBuildScriptDependencies(project: Project = this.project): List<DependencyScanner.Maven> {
-        val existingNames = mutableSetOf<String>()
-
         return project.buildscript.configurations.flatMap { config ->
             config.resolvedConfiguration
                 .lenientConfiguration
@@ -192,59 +227,71 @@ open class StaticMethodsAndTools(private val project: Project) {
                     val group = module.group
                     val name = module.name
                     val version = module.version
-                    val moduleName = "$group:$name"
 
-                    if (!existingNames.contains(moduleName)) {
-                        existingNames.add(moduleName)
                     DependencyScanner.Maven(group, name, version)
-                    } else {
-                        null
-                    }
                 }
         }
     }
 
     /**
-     * Resolves all child dependencies of the project
+     * Resolves all *declared* dependencies of the project
      *
      * THIS MUST BE IN "afterEvaluate" or run from a specific task.
      */
-    fun resolveAllDependencies(project: Project = this.project): List<DependencyScanner.Dependency> {
+    fun resolveAllDeclaredDependencies(project: Project = this.project): List<DependencyScanner.DependencyData> {
-        val projectDependencies = mutableListOf<DependencyScanner.Dependency>()
+        // NOTE: we cannot createTree("compile") and createTree("runtime") using the same exitingNames and expect correct results.
-        val existingNames = mutableSetOf<String>()
+        // This is because a dependency might exist for compile and runtime, but have different children, therefore, the list
+        // will be incomplete
 
-        DependencyScanner.scan(project, "compileClasspath", projectDependencies, existingNames)
+        // there will be DUPLICATES! (we don't care about children or hierarchy, so we remove the dupes)
-        DependencyScanner.scan(project, "runtimeClasspath", projectDependencies, existingNames)
+        return (DependencyScanner.scan(project, "compileClasspath", false) +
+                DependencyScanner.scan(project, "runtimeClasspath", false)
+                ).toSet().toList()
+    }
 
-        return projectDependencies
+
+    /**
+     * Recursively resolves all child dependencies of the project
+     *
+     * THIS MUST BE IN "afterEvaluate" or run from a specific task.
+     */
+    fun resolveAllDependencies(project: Project = this.project): List<DependencyScanner.DependencyData> {
+        // NOTE: we cannot createTree("compile") and createTree("runtime") using the same exitingNames and expect correct results.
+        // This is because a dependency might exist for compile and runtime, but have different children, therefore, the list
+        // will be incomplete
+
+        // there will be DUPLICATES! (we don't care about children or hierarchy, so we remove the dupes)
+        return (DependencyScanner.scan(project, "compileClasspath") +
+                DependencyScanner.scan(project, "runtimeClasspath")
+                ).toSet().toList()
     }
 
     /**
-     * Resolves all child compile dependencies of the project
+     * Recursively resolves all child compile dependencies of the project
      *
      * THIS MUST BE IN "afterEvaluate" or run from a specific task.
      */
-    fun resolveCompileDependencies(project: Project = this.project): List<DependencyScanner.Dependency> {
+    fun resolveCompileDependencies(project: Project = this.project): DependencyScanner.ProjectDependencies {
         val projectDependencies = mutableListOf<DependencyScanner.Dependency>()
-        val existingNames = mutableSetOf<String>()
+        val existingNames = mutableMapOf<String, DependencyScanner.Dependency>()
 
-        DependencyScanner.scan(project, "compileClasspath", projectDependencies, existingNames)
+        DependencyScanner.createTree(project, "compileClasspath", projectDependencies, existingNames)
 
-        return projectDependencies
+        return DependencyScanner.ProjectDependencies(projectDependencies, existingNames.map { it.value })
     }
 
     /**
-     * Resolves all child compile dependencies of the project
+     * Recursively resolves all child compile dependencies of the project
      *
      * THIS MUST BE IN "afterEvaluate" or run from a specific task.
      */
-    fun resolveRuntimeDependencies(project: Project = this.project): List<DependencyScanner.Dependency> {
+    fun resolveRuntimeDependencies(project: Project = this.project): DependencyScanner.ProjectDependencies {
         val projectDependencies = mutableListOf<DependencyScanner.Dependency>()
-        val existingNames = mutableSetOf<String>()
+        val existingNames = mutableMapOf<String, DependencyScanner.Dependency>()
 
-        DependencyScanner.scan(project, "runtimeClasspath", projectDependencies, existingNames)
+        DependencyScanner.createTree(project, "runtimeClasspath", projectDependencies, existingNames)
 
-        return projectDependencies
+        return DependencyScanner.ProjectDependencies(projectDependencies, existingNames.map { it.value })
     }
 
     /**