Fixes #34205 - External IPAM Integration

Contributors

@@ -10,6 +10,7 @@ Ashley Penney <[email protected]>
 Baptiste Agasse <[email protected]>
 Brandon Weeks <[email protected]>
 Christian Arnold <[email protected]>
+Christopher Smith <[email protected]>
 Corey Osman <[email protected]>
 Daniel Baeurer <[email protected]>
 Daniel Helgenberger <[email protected]>

README.md

@@ -19,6 +19,7 @@ Currently Supported modules:
  * HTTPBoot - endpoint exposing a (TFTP) directory via HTTP(s) for UEFI HTTP booting
  * Logs - log buffer of proxy logs for easier troubleshooting
  * Templates - unattended Foreman endpoint proxy
+ * External IPAM - Integration with External IPAM providers
 
 # Installation
 Read the [Smart Proxy Installation section](https://theforeman.org/manuals/latest/index.html#4.3.1SmartProxyInstallation) of the manual.

config/settings.d/externalipam.yml.example

@@ -0,0 +1,8 @@
+---
+:enabled: false
+
+# Built-in providers:
+#   1. phpIPAM: externalipam_phpipam
+#   2. Netbox:  externalipam_netbox
+
+:use_provider: externalipam_netbox

config/settings.d/externalipam_netbox.yml.example

@@ -0,0 +1,6 @@
+---
+# url is the hostname and path of the Netbox instance
+:url: 'https://netbox.example.com'
+
+# token is the Netbox API token
+:token: 'netbox_token'

config/settings.d/externalipam_phpipam.yml.example

@@ -0,0 +1,12 @@
+---
+# url is the hostname and path of the phpIPAM instance.
+:url: 'https://phpipam.example.com'
+
+# The phpIPAM user name for authentication. Please note that an API Key also needs to be 
+# setup with the exact same name as the user name configured here. When setting up the API 
+# Key in phpIPAM, "User token" must be used for the "App Security" setting.
+:user: 'ipam_user'
+
+# The password for above user account. Note that this is the password of the user, and not 
+# the API Key itself.
+:password: 'ipam_password'

lib/proxy/validations.rb

@@ -9,6 +9,8 @@ class Error < RuntimeError; end
   class InvalidIPAddress < Error; end
   class InvalidMACAddress < Error; end
   class InvalidSubnet < Error; end
+  class InvalidCidr < Error; end
+  class IpNotInCidr < Error; end
 
   private
 
@@ -61,6 +63,23 @@ def validate_subnet(subnet)
     subnet
   end
 
+  def validate_cidr(address, prefix)
+    cidr = "#{address}/#{prefix}"
+    network = IPAddr.new(cidr).to_s
+    if network != IPAddr.new(address).to_s
+      raise InvalidCidr, "Network address #{address} should be #{network} with prefix #{prefix}"
+    end
+    cidr
+  rescue IPAddr::Error => e
+    raise Proxy::Validations::Error, e.to_s
+  end
+
+  def validate_ip_in_cidr(ip, cidr)
+    raise IpNotInCidr, "IP #{ip} is not in #{cidr}" unless IPAddr.new(cidr).include?(IPAddr.new(ip))
+  rescue IPAddr::Error => e
+    raise Proxy::Validations::Error, e.to_s
+  end
+
   def validate_server(server)
     raise Proxy::DHCP::Error, "Invalid Server #{server}" unless server.is_a?(Proxy::DHCP::Server)
     server

lib/smart_proxy_main.rb

@@ -61,6 +61,7 @@ module Proxy
   require 'dhcp_isc/dhcp_isc'
   require 'dhcp_native_ms/dhcp_native_ms'
   require 'dhcp_libvirt/dhcp_libvirt'
+  require 'externalipam/externalipam'
   require 'puppetca/puppetca'
   require 'puppetca_http_api/puppetca_http_api'
   require 'puppetca_hostname_whitelisting/puppetca_hostname_whitelisting'

modules/dhcp_common/free_ips.rb

@@ -33,7 +33,7 @@ def clean_up_allocated_ips
           break if @allocation_timestamps.first.nil? || @allocation_timestamps.first[1] > time_now
           ip, _ = @allocation_timestamps.shift
           @allocated_ips.delete(ip)
-          logger.debug("#{ip} marked as free.")
+          logger.trace("#{ip} marked as free for DHCP.")
         end
       end
     end

modules/externalipam/api_resource.rb

@@ -0,0 +1,55 @@
+require 'yaml'
+require 'json'
+require 'net/http'
+require 'uri'
+require 'externalipam/ipam_helper'
+
+module Proxy::Ipam
+  # Class to handle authentication and HTTP transactions with External IPAM providers
+  class ApiResource
+    include ::Proxy::Log
+    include Proxy::Ipam::IpamHelper
+
+    def initialize(params = {})
+      @api_base = params[:api_base]
+      @token = params[:token]
+      @auth_header = params[:auth_header] || 'Authorization'
+    end
+
+    def get(path, params = nil)
+      url = @api_base + path
+      url += "?#{URI.encode_www_form(params)}" if params
+      uri = URI(url)
+      request = Net::HTTP::Get.new(uri)
+      request[@auth_header] = @token
+      request['Accept'] = 'application/json'
+      request(request, uri)
+    end
+
+    def delete(path)
+      uri = URI(@api_base + path)
+      request = Net::HTTP::Delete.new(uri)
+      request[@auth_header] = @token
+      request['Accept'] = 'application/json'
+      request(request, uri)
+    end
+
+    def post(path, body = nil)
+      uri = URI(@api_base + path)
+      request = Net::HTTP::Post.new(uri)
+      request.body = body
+      request[@auth_header] = @token
+      request['Accept'] = 'application/json'
+      request['Content-Type'] = 'application/json'
+      request(request, uri)
+    end
+
+    private
+
+    def request(request, uri)
+      Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http|
+        http.request(request)
+      end
+    end
+  end
+end

modules/externalipam/configuration_loader.rb

@@ -0,0 +1,8 @@
+module ::Proxy::Ipam
+  class ConfigurationLoader
+    def load_classes
+      require 'externalipam/dependency_injection'
+      require 'externalipam/ipam_api'
+    end
+  end
+end

modules/externalipam/dependency_injection.rb

@@ -0,0 +1,8 @@
+module Proxy::Ipam
+  module DependencyInjection
+    include Proxy::DependencyInjection::Accessors
+    def container_instance
+      @container_instance ||= ::Proxy::Plugins.instance.find { |p| p[:name] == :externalipam }[:di_container]
+    end
+  end
+end

modules/externalipam/externalipam.rb

@@ -0,0 +1,2 @@
+require 'externalipam/externalipam_plugin'
+require 'externalipam/configuration_loader'

modules/externalipam/externalipam_plugin.rb

@@ -0,0 +1,11 @@
+require 'externalipam/phpipam/phpipam_plugin'
+require 'externalipam/netbox/netbox_plugin'
+
+module Proxy::Ipam
+  class Plugin < ::Proxy::Plugin
+    plugin :externalipam, ::Proxy::VERSION
+    uses_provider
+    default_settings use_provider: nil
+    rackup_path File.expand_path('http_config.ru', __dir__)
+  end
+end

modules/externalipam/http_config.ru

@@ -0,0 +1,5 @@
+require 'externalipam/ipam_api'
+
+map '/ipam' do
+  run Proxy::Ipam::Api
+end

modules/externalipam/ip_cache.rb

@@ -0,0 +1,133 @@
+require 'yaml'
+require 'json'
+require 'monitor'
+require 'concurrent'
+require 'time'
+require 'externalipam/ipam_helper'
+require 'singleton'
+
+module Proxy::Ipam
+  # Class for managing temp in-memory cache to prevent same IP's being suggested in race conditions
+  class IpCache
+    include Singleton
+    include Proxy::Log
+    include Proxy::Ipam::IpamHelper
+
+    DEFAULT_CLEANUP_INTERVAL = 180
+
+    def initialize
+      @m = Monitor.new
+      @ip_cache = {'': {}}
+      start_cleanup_task
+    end
+
+    def provider_name(provider)
+      @provider = provider
+    end
+
+    def get_cidr(group_name, cidr)
+      @ip_cache.dig(group_name, cidr)
+    end
+
+    def get_ip(group_name, cidr, mac)
+      @ip_cache.dig(group_name, cidr, mac.to_sym, :ip)
+    end
+
+    def ip_exists?(group_name, cidr, ip)
+      subnet_hash = get_cidr(group_name, cidr)
+      return false if subnet_hash.nil?
+      subnet_hash&.any? { |mac, cached_ip| cached_ip[:ip] == ip }
+    end
+
+    def ip_expired?(group_name, cidr, ip)
+      return true unless ip_exists?(group_name, cidr, ip)
+      subnet_hash = get_cidr(group_name, cidr)
+      subnet_hash&.any? { |mac, cached_ip| cached_ip[:ip] == ip && expired(cached_ip[:timestamp]) }
+    end
+
+    def cleanup_interval
+      DEFAULT_CLEANUP_INTERVAL
+    end
+
+    def add(group_name, cidr, ip, mac = nil)
+      logger.debug("Adding IP '#{ip}' to cache for subnet '#{cidr}' in group '#{group_name}' for IPAM provider #{@provider}")
+      @m.synchronize do
+        mac_addr = mac.nil? || mac.empty? ? SecureRandom.uuid : mac
+        group_hash = @ip_cache[group_name]
+
+        if group_hash&.key?(cidr)
+          @ip_cache[group_name][cidr][mac_addr.to_sym] = { ip: ip.to_s, timestamp: Time.now }
+        else
+          @ip_cache[group_name] = { cidr => { mac_addr.to_sym => { ip: ip.to_s, timestamp: Time.now }}}
+        end
+      end
+    end
+
+    private
+
+    def expired(ip_expiration)
+      Time.now - ip_expiration > DEFAULT_CLEANUP_INTERVAL
+    end
+
+    def start_cleanup_task
+      logger.info("Starting ip cache maintenance for External IPAM provider, used by /next_ip.")
+      @timer_task = Concurrent::TimerTask.new(execution_interval: DEFAULT_CLEANUP_INTERVAL) { clean_cache }
+      @timer_task.execute
+    end
+
+    # @ip_cache structure
+    #
+    # Groups of subnets are cached under the External IPAM Group name. For example,
+    # "IPAM Group Name" would be the section name in phpIPAM. All IP's cached for subnets
+    # that do not have an External IPAM group specified, they are cached under the "" key. IP's
+    # are cached using one of two possible keys:
+    #    1). Mac Address
+    #    2). UUID (Used when Mac Address not specified)
+    #
+    # {
+    #   "": {
+    #     "100.55.55.0/24":{
+    #       "00:0a:95:9d:68:10": {"ip": "100.55.55.1", "timestamp": "2019-09-17 12:03:43 -D400"},
+    #       "906d8bdc-dcc0-4b59-92cb-665935e21662": {"ip": "100.55.55.2", "timestamp": "2019-09-17 11:43:22 -D400"}
+    #     },
+    #   },
+    #   "IPAM Group Name": {
+    #     "123.11.33.0/24":{
+    #       "00:0a:95:9d:68:33": {"ip": "123.11.33.1", "timestamp": "2019-09-17 12:04:43 -0400"},
+    #       "00:0a:95:9d:68:34": {"ip": "123.11.33.2", "timestamp": "2019-09-17 12:05:48 -0400"},
+    #       "00:0a:95:9d:68:35": {"ip": "123.11.33.3", "timestamp:: "2019-09-17 12:06:50 -0400"}
+    #     }
+    #   },
+    #   "Another IPAM Group": {
+    #     "185.45.39.0/24":{
+    #       "00:0a:95:9d:68:55": {"ip": "185.45.39.1", "timestamp": "2019-09-17 12:04:43 -0400"},
+    #       "00:0a:95:9d:68:56": {"ip": "185.45.39.2", "timestamp": "2019-09-17 12:05:48 -0400"}
+    #     }
+    #   }
+    # }
+    def clean_cache
+      @m.synchronize do
+        entries_deleted = 0
+        total_entries = 0
+
+        @ip_cache.each do |group, subnets|
+          subnets.each do |cidr, macs|
+            macs.each do |mac, ip|
+              if expired(ip[:timestamp])
+                @ip_cache[group][cidr].delete(mac)
+                entries_deleted += 1
+              end
+              total_entries += 1
+            end
+            @ip_cache[group].delete(cidr) if @ip_cache[group][cidr].nil? || @ip_cache[group][cidr].empty?
+            @ip_cache.delete(group) if @ip_cache[group].nil? || @ip_cache[group].empty?
+          end
+        end
+
+        cache_count = total_entries - entries_deleted
+        logger.debug("Removing #{entries_deleted} entries from IP cache for IPAM provider #{@provider}") if entries_deleted > 0
+        logger.debug("Current count of IP cache entries for IPAM provider #{@provider}: #{cache_count}") if entries_deleted > 0
+      end
+    end
+  end
+end