Configuration to ignore SSL-certificate issues (#150)

I don't really like adding this, but this is a frequent issue with extractors, and there is no native way to work around it.

It is necessary to do this twice, since the best way to do it is only available on 2.1. Most users will be using a 2.1 compatible version, so this isn't a big problem. On 2.0 they need to call a method explicitly to use this.

ExtractorUtils/Cognite/DestinationUtils.cs

@@ -10,6 +10,8 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
+using System.Linq;
+using System.Net.Security;
 
 namespace Cognite.Extractor.Utils
 {
@@ -27,6 +29,52 @@ private static CogniteDestination GetCogniteDestination(IServiceProvider provide
             return new CogniteDestination(client, logger ?? new NullLogger<CogniteDestination>(), config);
         }
 
+#if NETSTANDARD2_1_OR_GREATER
+        /// <summary>
+        /// Return a http handler configured to ignore certificate errors based on passed CertificateConfig.
+        /// </summary>
+        /// <param name="config">Certificate config to use</param>
+        public static HttpClientHandler GetClientHandler(CertificateConfig? config)
+        {
+            var handler = new HttpClientHandler();
+            if (config == null) return handler;
+
+            handler.ServerCertificateCustomValidationCallback = (message, cert, chain, sslPolicyErrors) => {
+                if (sslPolicyErrors == SslPolicyErrors.None) return true;
+
+                if (config.AcceptAll) return true;
+
+                if (config.AllowList?.Any(acc => acc.ToLower() == cert.GetCertHashString().ToLower()) ?? false) return true;
+
+                return false;
+            };
+
+            return handler;
+        }
+#endif
+
+        private static bool _sslPolicyConfigured;
+        /// <summary>
+        /// Configure global handling of SSL certificates.
+        /// This must be called to ignore certificates if you require the .NET standard 2.0 version of the library,
+        /// since .NET framework lacks local ignoring of SSL errors.
+        /// </summary>
+        /// <param name="config"></param>
+        public static void ConfigureSslPolicy(CertificateConfig config)
+        {
+            if (_sslPolicyConfigured) return;
+            _sslPolicyConfigured = true;
+            ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) =>
+            {
+                if (sslPolicyErrors == SslPolicyErrors.None) return true;
+
+                if (config.AcceptAll) return true;
+
+                if (config.AllowList?.Any(acc => acc.ToLower() == cert.GetCertHashString().ToLower()) ?? false) return true;
+
+                return false;
+            };
+        }
 
         /// <summary>
         /// Adds a configured Cognite client to the <paramref name="services"/> collection as a transient service
@@ -63,7 +111,16 @@ public static void AddCogniteClient(this IServiceCollection services,
                     {
                         var retryConfig = provider.GetService<CogniteConfig>()?.CdfRetries;
                         return CogniteExtensions.GetTimeoutPolicy(retryConfig?.Timeout);
+                    })
+#if NETSTANDARD2_1_OR_GREATER
+                    .ConfigurePrimaryHttpMessageHandler(provider =>
+                    {
+                        var certConfig = provider.GetService<CogniteConfig>()?.Certificates;
+                        return GetClientHandler(certConfig);
                     });
+#else
+                    ;
+#endif
             }
 
             // Configure token based authentication
@@ -75,7 +132,16 @@ public static void AddCogniteClient(this IServiceCollection services,
                     {
                         c.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent);
                     }
+                })
+#if NETSTANDARD2_1_OR_GREATER
+                .ConfigurePrimaryHttpMessageHandler(provider =>
+                {
+                    var certConfig = provider.GetService<CogniteConfig>()?.Certificates;
+                    return GetClientHandler(certConfig);
                 });
+#else
+                ;
+#endif
             services.AddTransient<IAuthenticator>(provider =>
             {
                 var conf = provider.GetService<CogniteConfig>();

ExtractorUtils/Configuration/BaseConfig.cs

@@ -115,6 +115,11 @@ public class CogniteConfig
         /// Configuration for automatically reporting extraction pipeline runs.
         /// </summary>
         public ExtractionRunConfig? ExtractionPipeline { get; set; }
+
+        /// <summary>
+        /// Configuration for handling SSL certificates.
+        /// </summary>
+        public CertificateConfig? Certificates { get; set; }
     }
 
     /// <summary>
@@ -290,5 +295,20 @@ public class RetryConfig
         public int MaxDelay { get; set; } = 5_000;
     }
 
+    /// <summary>
+    /// Configure options relating to SSL certificates.
+    /// </summary>
+    public class CertificateConfig
+    {
+        /// <summary>
+        /// True to accept all certificates. This must be considered a security risk in most circumstances.
+        /// </summary>
+        public bool AcceptAll { get; set; }
+        /// <summary>
+        /// List of certificate thumbprints to manually allow. This is much safer.
+        /// </summary>
+        public IEnumerable<string>? AllowList { get; set; }
+    }
+
     #endregion
 }

ExtractorUtils/config/config.example.yml

@@ -115,6 +115,17 @@ cognite:
         # Frequency to report "Seen", in seconds. Less than or equal to zero will not report automatically.
         # frequency: 600
 
+    # Optional configuration for special handling of SSL certificates.
+    # This should never be considered a permanent solution to certificate problems.
+    certificates:
+        # True to accept all certificates.
+        # This poses a severe security risk.
+        accept-all: false
+        # List of thumbprints of certificates to allow.
+        # This is a smaller risk compared to accepting all certificates.
+        allow-list:
+            # - 99E92D8447AEF30483B1D7527812C9B7B3A915A7
+
 
 # Store state in a local database or in CDF raw to speed up starting, by not having to read state from destinations
 state-store: