Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support file URLs in downloads #9227

Merged
merged 2 commits into from
Oct 1, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ import org.ossreviewtoolkit.model.utils.DependencyHandler
import org.ossreviewtoolkit.model.utils.parseRepoManifestPath
import org.ossreviewtoolkit.plugins.packagemanagers.gradlemodel.dependencyType
import org.ossreviewtoolkit.plugins.packagemanagers.gradlemodel.isProjectDependency
import org.ossreviewtoolkit.utils.common.collectMessages
import org.ossreviewtoolkit.utils.common.splitOnWhitespace
import org.ossreviewtoolkit.utils.common.withoutPrefix
import org.ossreviewtoolkit.utils.ort.DeclaredLicenseProcessor
Expand Down Expand Up @@ -254,7 +255,9 @@ private fun createRemoteArtifact(
}
}
}.getOrElse {
logger.warn("Unable to get a valid artifact checksum.", it)
logger.warn {
"Unable to get a valid '$algorithm' checksum for the artifact at $artifactUrl: ${it.collectMessages()}"
}

Hash.NONE
}
Expand Down
19 changes: 15 additions & 4 deletions utils/ort/src/main/kotlin/OkHttpClientHelper.kt
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ package org.ossreviewtoolkit.utils.ort
import java.io.File
import java.io.IOException
import java.lang.invoke.MethodHandles
import java.net.URI
import java.time.Duration
import java.util.concurrent.ConcurrentHashMap

Expand Down Expand Up @@ -143,11 +144,17 @@ fun OkHttpClient.Builder.addBasicAuthorization(username: String, password: Strin
* Download from [url] and return a [Result] with a file inside [directory] that holds the response body content on
* success, or a [Result] wrapping an [IOException] (which might be a [HttpDownloadError]) on failure.
*/
fun OkHttpClient.downloadFile(url: String, directory: File): Result<File> =
fun OkHttpClient.downloadFile(url: String, directory: File): Result<File> {
if (url.startsWith("file:/")) {
val source = File(URI.create(url))
val target = directory.resolve(source.name)
return runCatching { source.copyTo(target) }
}

// Disable transparent gzip compression, as otherwise we might end up writing a tar file to disk while
// expecting to find a tar.gz file, and fail to unpack the archive. See
// https://github.com/square/okhttp/blob/parent-3.10.0/okhttp/src/main/java/okhttp3/internal/http/BridgeInterceptor.java#L79
download(url, acceptEncoding = "identity").mapCatching { (response, body) ->
return download(url, acceptEncoding = "identity").mapCatching { (response, body) ->

// Depending on the server, we may only get a useful target file name when looking at the response
// header or at a redirected URL. In case of the Crates registry, for example, we want to resolve
Expand Down Expand Up @@ -186,15 +193,19 @@ fun OkHttpClient.downloadFile(url: String, directory: File): Result<File> =

file
}
}

/**
* Download from [url] and return a [Result] with a string representing the response body content on success, or a
* [Result] wrapping an [IOException] (which might be a [HttpDownloadError]) on failure.
*/
fun OkHttpClient.downloadText(url: String): Result<String> =
download(url).mapCatching { (_, body) ->
fun OkHttpClient.downloadText(url: String): Result<String> {
if (url.startsWith("file:/")) return runCatching { File(URI.create(url)).readText() }

return download(url).mapCatching { (_, body) ->
body.use { it.string() }
}
}

/**
* Download from [url] with optional [acceptEncoding] and return a [Result] with the [Response] and non-nullable
Expand Down
Loading