Sandboxed URL creation to prevent SSRF attacks

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolver.java

@@ -10,6 +10,8 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.pixee.security.HostValidator;
+import io.github.pixee.security.Urls;
 
 import org.apache.commons.compress.utils.Lists;
 import org.gradle.jvm.toolchain.JavaLanguageVersion;
@@ -58,8 +60,7 @@ private Optional<AdoptiumVersionInfo> resolveAvailableVersion(AdoptiumVersionReq
         ObjectMapper mapper = new ObjectMapper();
         try {
             int languageVersion = requestKey.languageVersion.asInt();
-            URL source = new URL(
-                "https://api.adoptium.net/v3/info/release_versions?architecture="
+            URL source = Urls.create("https://api.adoptium.net/v3/info/release_versions?architecture="
                     + requestKey.arch
                     + "&image_type=jdk&os="
                     + requestKey.platform
@@ -68,8 +69,7 @@ private Optional<AdoptiumVersionInfo> resolveAvailableVersion(AdoptiumVersionReq
                     + languageVersion
                     + ","
                     + (languageVersion + 1)
-                    + ")"
-            );
+                    + ")", Urls.HTTP_PROTOCOLS, HostValidator.DENY_COMMON_INFRASTRUCTURE_TARGETS);
             JsonNode jsonNode = mapper.readTree(source);
             JsonNode versionsNode = jsonNode.get("versions");
             return Optional.of(

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/WaitForHttpResource.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.gradle.testclusters;
 
+import io.github.pixee.security.HostValidator;
+import io.github.pixee.security.Urls;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
 
@@ -44,7 +46,7 @@ public class WaitForHttpResource {
     private String password;
 
     public WaitForHttpResource(String protocol, String host, int numberOfNodes) throws MalformedURLException {
-        this(new URL(protocol + "://" + host + "/_cluster/health?wait_for_nodes=>=" + numberOfNodes + "&wait_for_status=yellow"));
+        this(Urls.create(protocol + "://" + host + "/_cluster/health?wait_for_nodes=>=" + numberOfNodes + "&wait_for_status=yellow", Urls.HTTP_PROTOCOLS, HostValidator.DENY_COMMON_INFRASTRUCTURE_TARGETS));
     }
 
     public WaitForHttpResource(URL url) {

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.plugins.cli;
 
+import io.github.pixee.security.HostValidator;
+import io.github.pixee.security.Urls;
 import org.apache.lucene.search.spell.LevenshteinDistance;
 import org.apache.lucene.util.CollectionUtil;
 import org.apache.lucene.util.Constants;
@@ -434,7 +436,7 @@ private String getMavenUrl(String[] coordinates) throws IOException {
     @SuppressForbidden(reason = "Make HEAD request using URLConnection.connect()")
     boolean urlExists(String urlString) throws IOException {
         terminal.println(VERBOSE, "Checking if url exists: " + urlString);
-        URL url = new URL(urlString);
+        URL url = Urls.create(urlString, Urls.HTTP_PROTOCOLS, HostValidator.DENY_COMMON_INFRASTRUCTURE_TARGETS);
         assert "https".equals(url.getProtocol()) : "Only http urls can be checked";
         HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
         urlConnection.addRequestProperty("User-Agent", "elasticsearch-plugin-installer");
@@ -464,7 +466,7 @@ private static List<String> checkMisspelledPlugin(String pluginId) {
     @SuppressForbidden(reason = "We use getInputStream to download plugins")
     Path downloadZip(String urlString, Path tmpDir) throws IOException {
         terminal.println(VERBOSE, "Retrieving zip from " + urlString);
-        URL url = new URL(urlString);
+        URL url = Urls.create(urlString, Urls.HTTP_PROTOCOLS, HostValidator.DENY_COMMON_INFRASTRUCTURE_TARGETS);
         Path zip = Files.createTempFile(tmpDir, null, ".zip");
         URLConnection urlConnection = this.proxy == null ? url.openConnection() : url.openConnection(this.proxy);
         urlConnection.addRequestProperty("User-Agent", "elasticsearch-plugin-installer");
@@ -760,7 +762,7 @@ InputStream getPublicKey() {
      */
     // pkg private for tests
     URL openUrl(String urlString) throws IOException {
-        URL checksumUrl = new URL(urlString);
+        URL checksumUrl = Urls.create(urlString, Urls.HTTP_PROTOCOLS, HostValidator.DENY_COMMON_INFRASTRUCTURE_TARGETS);
         HttpURLConnection connection = this.proxy == null
             ? (HttpURLConnection) checksumUrl.openConnection()
             : (HttpURLConnection) checksumUrl.openConnection(this.proxy);

libs/core/src/main/java/org/elasticsearch/core/internal/provider/EmbeddedImplClassLoader.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.core.internal.provider;
 
+import io.github.pixee.security.HostValidator;
+import io.github.pixee.security.Urls;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
@@ -466,7 +468,7 @@ private static Map<JarMeta, CodeSource> getProviderPrefixes(ClassLoader parent,
     }
 
     private static CodeSource codeSource(URL baseURL, String jarName) throws MalformedURLException {
-        return new CodeSource(new URL(baseURL, jarName), (CodeSigner[]) null /*signers*/);
+        return new CodeSource(Urls.create(baseURL, jarName, Urls.HTTP_PROTOCOLS, HostValidator.DENY_COMMON_INFRASTRUCTURE_TARGETS), (CodeSigner[]) null /*signers*/);
     }
 
     private static boolean isMultiRelease(ClassLoader parent, String jarPrefix) throws IOException {

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.ingest.common;
 
+import io.github.pixee.security.HostValidator;
+import io.github.pixee.security.Urls;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
@@ -94,7 +96,7 @@ public static Map<String, Object> apply(String urlString) {
             uri = new URI(urlString);
         } catch (URISyntaxException e) {
             try {
-                url = new URL(urlString);
+                url = Urls.create(urlString, Urls.HTTP_PROTOCOLS, HostValidator.DENY_COMMON_INFRASTRUCTURE_TARGETS);
             } catch (MalformedURLException e2) {
                 throw new IllegalArgumentException("unable to parse URI [" + urlString + "]");
             }

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/HttpClient.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.ingest.geoip;
 
+import io.github.pixee.security.HostValidator;
+import io.github.pixee.security.Urls;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.SpecialPermission;
@@ -53,8 +55,8 @@ InputStream get(String urlToGet) throws IOException {
                             throw new IllegalStateException("too many redirects connection to [" + urlToGet + "]");
                         }
                         String location = conn.getHeaderField("Location");
-                        URL base = new URL(url);
-                        URL next = new URL(base, location);  // Deal with relative URLs
+                        URL base = Urls.create(url, Urls.HTTP_PROTOCOLS, HostValidator.DENY_COMMON_INFRASTRUCTURE_TARGETS);
+                        URL next = Urls.create(base, location, Urls.HTTP_PROTOCOLS, HostValidator.DENY_COMMON_INFRASTRUCTURE_TARGETS);  // Deal with relative URLs
                         url = next.toExternalForm();
                         conn = createConnection(url);
                         break;
@@ -74,7 +76,7 @@ private static InputStream getInputStream(HttpURLConnection conn) throws IOExcep
     }
 
     private static HttpURLConnection createConnection(String url) throws IOException {
-        HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
+        HttpURLConnection conn = (HttpURLConnection) Urls.create(url, Urls.HTTP_PROTOCOLS, HostValidator.DENY_COMMON_INFRASTRUCTURE_TARGETS).openConnection();
         conn.setConnectTimeout(10000);
         conn.setReadTimeout(10000);
         conn.setDoOutput(false);

modules/lang-painless/src/doc/java/org/elasticsearch/painless/ContextGeneratorCommon.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.painless;
 
+import io.github.pixee.security.HostValidator;
+import io.github.pixee.security.Urls;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.painless.action.PainlessContextClassBindingInfo;
 import org.elasticsearch.painless.action.PainlessContextClassInfo;
@@ -35,7 +37,7 @@
 public class ContextGeneratorCommon {
     @SuppressForbidden(reason = "retrieving data from an internal API not exposed as part of the REST client")
     public static List<PainlessContextInfo> getContextInfos() throws IOException {
-        URLConnection getContextNames = new URL("http://" + System.getProperty("cluster.uri") + "/_scripts/painless/_context")
+        URLConnection getContextNames = Urls.create("http://" + System.getProperty("cluster.uri") + "/_scripts/painless/_context", Urls.HTTP_PROTOCOLS, HostValidator.DENY_COMMON_INFRASTRUCTURE_TARGETS)
             .openConnection();
         XContentParser parser = JsonXContent.jsonXContent.createParser(XContentParserConfiguration.EMPTY, getContextNames.getInputStream());
         parser.nextToken();
@@ -48,9 +50,7 @@ public static List<PainlessContextInfo> getContextInfos() throws IOException {
         List<PainlessContextInfo> contextInfos = new ArrayList<>();
 
         for (String contextName : contextNames) {
-            URLConnection getContextInfo = new URL(
-                "http://" + System.getProperty("cluster.uri") + "/_scripts/painless/_context?context=" + contextName
-            ).openConnection();
+            URLConnection getContextInfo = Urls.create("http://" + System.getProperty("cluster.uri") + "/_scripts/painless/_context?context=" + contextName, Urls.HTTP_PROTOCOLS, HostValidator.DENY_COMMON_INFRASTRUCTURE_TARGETS).openConnection();
             parser = JsonXContent.jsonXContent.createParser(XContentParserConfiguration.EMPTY, getContextInfo.getInputStream());
             contextInfos.add(PainlessContextInfo.fromXContent(parser));
             ((HttpURLConnection) getContextInfo).disconnect();

modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.painless;
 
+import io.github.pixee.security.HostValidator;
+import io.github.pixee.security.Urls;
 import org.elasticsearch.bootstrap.BootstrapInfo;
 import org.elasticsearch.painless.antlr.Walker;
 import org.elasticsearch.painless.ir.ClassNode;
@@ -60,7 +62,7 @@ final class Compiler {
     static {
         try {
             // Setup the code privileges.
-            CODESOURCE = new CodeSource(new URL("file:" + BootstrapInfo.UNTRUSTED_CODEBASE), (Certificate[]) null);
+            CODESOURCE = new CodeSource(Urls.create("file:" + BootstrapInfo.UNTRUSTED_CODEBASE, Urls.HTTP_PROTOCOLS, HostValidator.DENY_COMMON_INFRASTRUCTURE_TARGETS), (Certificate[]) null);
         } catch (MalformedURLException impossible) {
             throw new RuntimeException(impossible);
         }

modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java

@@ -20,6 +20,8 @@
 import com.google.cloud.storage.Storage;
 import com.google.cloud.storage.StorageOptions;
 import com.google.cloud.storage.StorageRetryStrategy;
+import io.github.pixee.security.HostValidator;
+import io.github.pixee.security.Urls;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -244,7 +246,7 @@ static String getDefaultProjectId(@Nullable Proxy proxy) throws IOException {
         if (metaHost == null) {
             metaHost = "metadata.google.internal";
         }
-        URL url = new URL("http://" + metaHost + "/computeMetadata/v1/project/project-id");
+        URL url = Urls.create("http://" + metaHost + "/computeMetadata/v1/project/project-id", Urls.HTTP_PROTOCOLS, HostValidator.DENY_COMMON_INFRASTRUCTURE_TARGETS);
         HttpURLConnection connection = (HttpURLConnection) (proxy != null ? url.openConnection(proxy) : url.openConnection());
         connection.setConnectTimeout(5000);
         connection.setReadTimeout(5000);

modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.common.blobstore.url;
 
+import io.github.pixee.security.HostValidator;
+import io.github.pixee.security.Urls;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobPath;
@@ -110,7 +112,7 @@ public DeleteResult delete(OperationPurpose purpose) {
     @Override
     public InputStream readBlob(OperationPurpose purpose, String name) throws IOException {
         try {
-            return new BufferedInputStream(getInputStream(new URL(path, name)), blobStore.bufferSizeInBytes());
+            return new BufferedInputStream(getInputStream(Urls.create(path, name, Urls.HTTP_PROTOCOLS, HostValidator.DENY_COMMON_INFRASTRUCTURE_TARGETS)), blobStore.bufferSizeInBytes());
         } catch (FileNotFoundException fnfe) {
             throw new NoSuchFileException("blob object [" + name + "] not found");
         }

modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.common.blobstore.url;
 
+import io.github.pixee.security.HostValidator;
+import io.github.pixee.security.Urls;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
@@ -129,9 +131,9 @@ private URL buildPath(BlobPath relativePath) throws MalformedURLException {
         if (paths.isEmpty()) {
             return path();
         }
-        URL blobPath = new URL(this.path, paths.get(0) + "/");
+        URL blobPath = Urls.create(this.path, paths.get(0) + "/", Urls.HTTP_PROTOCOLS, HostValidator.DENY_COMMON_INFRASTRUCTURE_TARGETS);
         for (int i = 1; i < paths.size(); i++) {
-            blobPath = new URL(blobPath, paths.get(i) + "/");
+            blobPath = Urls.create(blobPath, paths.get(i) + "/", Urls.HTTP_PROTOCOLS, HostValidator.DENY_COMMON_INFRASTRUCTURE_TARGETS);
         }
         return blobPath;
     }

modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/HttpURLBlobContainer.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.common.blobstore.url.http;
 
+import io.github.pixee.security.HostValidator;
+import io.github.pixee.security.Urls;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.OperationPurpose;
 import org.elasticsearch.common.blobstore.url.URLBlobContainer;
@@ -58,7 +60,7 @@ public InputStream readBlob(OperationPurpose purpose, String name) throws IOExce
 
     private URI getURIForBlob(String name) throws IOException {
         try {
-            return new URL(path, name).toURI();
+            return Urls.create(path, name, Urls.HTTP_PROTOCOLS, HostValidator.DENY_COMMON_INFRASTRUCTURE_TARGETS).toURI();
         } catch (Exception e) {
             throw new IOException("Unable to get an URI for blob with name [" + name + "]", e);
         }

modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.repositories.url;
 
+import io.github.pixee.security.HostValidator;
+import io.github.pixee.security.Urls;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.cluster.metadata.RepositoryMetadata;
@@ -177,7 +179,7 @@ public boolean isReadOnly() {
 
     private static URL parseURL(String s) {
         try {
-            return new URL(s);
+            return Urls.create(s, Urls.HTTP_PROTOCOLS, HostValidator.DENY_COMMON_INFRASTRUCTURE_TARGETS);
         } catch (MalformedURLException e) {
             throw new IllegalArgumentException("Unable to parse URL repository setting", e);
         }

modules/repository-url/src/yamlRestTest/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java

@@ -10,6 +10,8 @@
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import io.github.pixee.security.HostValidator;
+import io.github.pixee.security.Urls;
 
 import org.apache.http.HttpEntity;
 import org.apache.http.entity.ContentType;
@@ -95,7 +97,7 @@ public void registerRepositories() throws IOException {
         List<String> allowedUrls = (List<String>) XContentMapValues.extractValue("defaults.repositories.url.allowed_urls", clusterSettings);
         for (String allowedUrl : allowedUrls) {
             try {
-                InetAddress inetAddress = InetAddress.getByName(new URL(allowedUrl).getHost());
+                InetAddress inetAddress = InetAddress.getByName(Urls.create(allowedUrl, Urls.HTTP_PROTOCOLS, HostValidator.DENY_COMMON_INFRASTRUCTURE_TARGETS).getHost());
                 if (inetAddress.isAnyLocalAddress() || inetAddress.isLoopbackAddress()) {
                     Request createUrlRepositoryRequest = new Request("PUT", "/_snapshot/repository-url");
                     createUrlRepositoryRequest.setEntity(buildRepositorySettings("url", Settings.builder().put("url", allowedUrl).build()));

plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2Utils.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.discovery.ec2;
 
+import io.github.pixee.security.HostValidator;
+import io.github.pixee.security.Urls;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.Strings;
@@ -37,7 +39,7 @@ static Optional<String> getMetadataToken(String metadataTokenUrl) {
         return SocketAccess.doPrivileged(() -> {
             HttpURLConnection urlConnection;
             try {
-                urlConnection = (HttpURLConnection) new URL(metadataTokenUrl).openConnection();
+                urlConnection = (HttpURLConnection) Urls.create(metadataTokenUrl, Urls.HTTP_PROTOCOLS, HostValidator.DENY_COMMON_INFRASTRUCTURE_TARGETS).openConnection();
                 urlConnection.setRequestMethod("PUT");
                 urlConnection.setConnectTimeout(CONNECT_TIMEOUT);
                 urlConnection.setRequestProperty("X-aws-ec2-metadata-token-ttl-seconds", String.valueOf(METADATA_TOKEN_TTL_SECONDS));

plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java

@@ -10,6 +10,8 @@
 
 import com.amazonaws.util.EC2MetadataUtils;
 import com.amazonaws.util.json.Jackson;
+import io.github.pixee.security.HostValidator;
+import io.github.pixee.security.Urls;
 
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.network.NetworkService;
@@ -142,7 +144,7 @@ static Settings getAvailabilityZoneNodeAttributes(Settings settings, String azMe
         final URL url;
         final URLConnection urlConnection;
         try {
-            url = new URL(azMetadataUrl);
+            url = Urls.create(azMetadataUrl, Urls.HTTP_PROTOCOLS, HostValidator.DENY_COMMON_INFRASTRUCTURE_TARGETS);
             logger.debug("obtaining ec2 [placement/availability-zone] from ec2 meta-data url {}", url);
             urlConnection = SocketAccess.doPrivilegedIOException(url::openConnection);
             urlConnection.setConnectTimeout(2000);

plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2NameResolver.java

@@ -9,6 +9,8 @@
 package org.elasticsearch.discovery.ec2;
 
 import com.amazonaws.util.EC2MetadataUtils;
+import io.github.pixee.security.HostValidator;
+import io.github.pixee.security.Urls;
 
 import org.elasticsearch.common.network.NetworkService.CustomNameResolver;
 import org.elasticsearch.core.IOUtils;
@@ -85,7 +87,7 @@ public static InetAddress[] resolve(Ec2HostnameType type) throws IOException {
         String metadataUrl = EC2MetadataUtils.getHostAddressForEC2MetadataService() + "/latest/meta-data/" + type.ec2Name;
         String metadataTokenUrl = EC2MetadataUtils.getHostAddressForEC2MetadataService() + "/latest/api/token";
         try {
-            URL url = new URL(metadataUrl);
+            URL url = Urls.create(metadataUrl, Urls.HTTP_PROTOCOLS, HostValidator.DENY_COMMON_INFRASTRUCTURE_TARGETS);
             logger.debug("obtaining ec2 hostname from ec2 meta-data url {}", url);
             URLConnection urlConnection = SocketAccess.doPrivilegedIOException(url::openConnection);
             urlConnection.setConnectTimeout(2000);

server/src/main/java/org/elasticsearch/env/Environment.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.env;
 
+import io.github.pixee.security.HostValidator;
+import io.github.pixee.security.Urls;
 import org.apache.lucene.util.Constants;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -214,7 +216,7 @@ public URL resolveRepoURL(URL url) {
                 }
                 String jarTail = file.substring(pos);
                 String filePath = file.substring(0, pos);
-                URL internalUrl = new URL(filePath);
+                URL internalUrl = Urls.create(filePath, Urls.HTTP_PROTOCOLS, HostValidator.DENY_COMMON_INFRASTRUCTURE_TARGETS);
                 URL normalizedUrl = resolveRepoURL(internalUrl);
                 if (normalizedUrl == null) {
                     return null;