diff --git a/.gitignore b/.gitignore index fe21d52..9217d07 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ *.swp .bundle +.idea coverage/ Gemfile.lock vendor/bundle +aws_public_ips-*.gem diff --git a/CHANGELOG.md b/CHANGELOG.md index c630f3e..e7011bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +### 1.0.8 +* Display progress while running by specifying `--progress` +* Include tag data in ec2 records by adding `--include_tags name` +* Multi-region support. Defaults to the region specified by AWS_DEFAULT_REGION or AWS_REGION. Many regions can be specified. `all` will include all regions. + ### 1.0.7 (11/7/2019) * Find all addresses when an EC2 instance has multiple ([@breser](https://github.com/breser)) * Add Docker support @@ -7,11 +12,11 @@ * Update dependencies ### 1.0.5 (05/25/2018) -* Output more details in text formatter when using --verbose +* Output more details in text formatter when using `--verbose` ### 1.0.4 (05/20/2018) * Handle RDS and Redshift instances which are in the process of coming up/down -* Add --help, --version commands to CLI +* Add `--help`, `--version` commands to CLI * Improve CLI error output * Fix issue where using the EC2 check with the JSON formatter could output empty entries * Fix issue with RDS check where the endpoint hostname was not being counted as a public IP diff --git a/aws_public_ips.gemspec b/aws_public_ips.gemspec index f615a97..51cf622 100644 --- a/aws_public_ips.gemspec +++ b/aws_public_ips.gemspec @@ -26,6 +26,7 @@ require 'aws_public_ips/version' gem.add_dependency('aws-sdk-lightsail', '~> 1.10.0') gem.add_dependency('aws-sdk-rds', '~> 1.35.0') gem.add_dependency('aws-sdk-redshift', '~> 1.13.0') + gem.add_dependency('tty-spinner', '~>0.9.0 ') gem.add_development_dependency('bundler-audit', '~> 0.6.0') gem.add_development_dependency('coveralls', '~> 0.8.22') diff --git a/lib/aws_public_ips.rb b/lib/aws_public_ips.rb index b9f492a..7101be2 100644 --- a/lib/aws_public_ips.rb +++ b/lib/aws_public_ips.rb @@ -5,6 +5,7 @@ module AwsPublicIps require 'aws_public_ips/checks' require 'aws_public_ips/cli' +require 'aws_public_ips/cli_options' require 'aws_public_ips/formatters' require 'aws_public_ips/utils' require 'aws_public_ips/version' diff --git a/lib/aws_public_ips/checks/apigateway.rb b/lib/aws_public_ips/checks/apigateway.rb index 40b416c..d5d98f6 100644 --- a/lib/aws_public_ips/checks/apigateway.rb +++ b/lib/aws_public_ips/checks/apigateway.rb @@ -6,22 +6,22 @@ module AwsPublicIps module Checks module Apigateway - def self.run - client = ::Aws::APIGateway::Client.new - return [] unless ::AwsPublicIps::Utils.has_service?(client) + def self.run(options) + ::AwsPublicIps::Utils.probe(::Aws::APIGateway::Client, options[:regions], options[:progress]) do |client| + # TODO(arkadiy) https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-private-integration.html - # TODO(arkadiy) https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-private-integration.html - - # APIGateway doesn't return the full domain in the response, we have to build - # it using the api id and region - client.get_rest_apis.flat_map do |response| - response.items.map do |api| - hostname = "#{api.id}.execute-api.#{client.config.region}.amazonaws.com" - { - id: api.id, - hostname: hostname, - ip_addresses: ::AwsPublicIps::Utils.resolve_hostname(hostname) - } + # APIGateway doesn't return the full domain in the response, we have to build + # it using the api id and region + client.get_rest_apis.flat_map do |response| + response.items.map do |api| + hostname = "#{api.id}.execute-api.#{client.config.region}.amazonaws.com" + { + region: client.config.region, + id: api.id, + hostname: hostname, + ip_addresses: ::AwsPublicIps::Utils.resolve_hostname(hostname) + } + end end end end diff --git a/lib/aws_public_ips/checks/cloudfront.rb b/lib/aws_public_ips/checks/cloudfront.rb index bbf1b2d..0c9aefc 100644 --- a/lib/aws_public_ips/checks/cloudfront.rb +++ b/lib/aws_public_ips/checks/cloudfront.rb @@ -6,20 +6,20 @@ module AwsPublicIps module Checks module Cloudfront - def self.run - client = ::Aws::CloudFront::Client.new - return [] unless ::AwsPublicIps::Utils.has_service?(client) + def self.run(options) + ::AwsPublicIps::Utils.probe(::Aws::CloudFront::Client, options[:regions] & ['us-east-1'], + options[:progress]) do |client| + # Cloudfront distributions are always public, they don't have a concept of VPC + # No "coming up" problem here like with RDS/Redshift - # Cloudfront distrubtions are always public, they don't have a concept of VPC - # No "coming up" problem here like with RDS/Redshift - - client.list_distributions.flat_map do |response| - response.distribution_list.items.flat_map do |distribution| - { - id: distribution.id, - hostname: distribution.domain_name, - ip_addresses: ::AwsPublicIps::Utils.resolve_hostname(distribution.domain_name) - } + client.list_distributions.flat_map do |response| + response.distribution_list.items.flat_map do |distribution| + { + id: distribution.id, + hostname: distribution.domain_name, + ip_addresses: ::AwsPublicIps::Utils.resolve_hostname(distribution.domain_name) + } + end end end end diff --git a/lib/aws_public_ips/checks/ec2.rb b/lib/aws_public_ips/checks/ec2.rb index 5f01525..1067b82 100644 --- a/lib/aws_public_ips/checks/ec2.rb +++ b/lib/aws_public_ips/checks/ec2.rb @@ -6,40 +6,42 @@ module AwsPublicIps module Checks module Ec2 - def self.run - client = ::Aws::EC2::Client.new - return [] unless ::AwsPublicIps::Utils.has_service?(client) - + def self.run(options) # Iterate over all EC2 instances. This will include those from EC2, ECS, EKS, Fargate, Batch, # Beanstalk, and NAT Instances # It will not include NAT Gateways (IPv4) or Egress Only Internet Gateways (IPv6), but they do not allow # ingress traffic so we skip them anyway - client.describe_instances.flat_map do |response| - response.reservations.flat_map do |reservation| - reservation.instances.flat_map do |instance| - # EC2-Classic instances have a `public_ip_address` and no `network_interfaces` - # EC2-VPC instances both set, so we uniq the ip addresses - ip_addresses = [instance.public_ip_address].compact + instance.network_interfaces.flat_map do |interface| - public_ip = [] + ::AwsPublicIps::Utils.probe(::Aws::EC2::Client, options[:regions], options[:progress]) do |client| + client.describe_instances.flat_map do |response| + response.reservations.flat_map do |reservation| + reservation.instances.flat_map do |instance| + # EC2-Classic instances have a `public_ip_address` and no `network_interfaces` + # EC2-VPC instances both set, so we uniq the ip addresses + ip_addresses = [instance.public_ip_address].compact + instance.network_interfaces.flat_map do |interface| + public_ip = [] - interface.private_ip_addresses.flat_map do |private_ip| - if private_ip.association && private_ip.association.public_ip - public_ip << private_ip.association.public_ip + interface.private_ip_addresses.flat_map do |private_ip| + if private_ip.association && private_ip.association.public_ip + public_ip << private_ip.association.public_ip + end end + public_ip + interface.ipv_6_addresses.map(&:ipv_6_address) end - public_ip + interface.ipv_6_addresses.map(&:ipv_6_address) - end - # Don't return an entry if all ips were private - next [] if ip_addresses.empty? + # Don't return an entry if all ips were private + next [] if ip_addresses.empty? - # If hostname is empty string, canonicalize to nil - hostname = instance.public_dns_name.empty? ? nil : instance.public_dns_name - { - id: instance.instance_id, - hostname: hostname, - ip_addresses: ip_addresses.uniq - } + # If hostname is empty string, canonicalize to nil + hostname = instance.public_dns_name.empty? ? nil : instance.public_dns_name + result = { + region: client.config.region, + id: instance.instance_id, + hostname: hostname, + ip_addresses: ip_addresses.uniq + } + ::AwsPublicIps::Utils.add_tags(result, instance, options[:tags]) + result + end end end end diff --git a/lib/aws_public_ips/checks/elasticsearch.rb b/lib/aws_public_ips/checks/elasticsearch.rb index f962ff0..13b5ee2 100644 --- a/lib/aws_public_ips/checks/elasticsearch.rb +++ b/lib/aws_public_ips/checks/elasticsearch.rb @@ -6,28 +6,28 @@ module AwsPublicIps module Checks module Elasticsearch - def self.run - client = ::Aws::ElasticsearchService::Client.new - return [] unless ::AwsPublicIps::Utils.has_service?(client) - + def self.run(options) # ElasticSearch instances can be launched into classic into VPCs. Classic instances are public and have a # `domain_status.endpoint` hostname, and VPC instances have a `domain_status.endpoints['vpc']` hostname. # However VPC ElasticSearch instances create their own Network Interface and AWS will not allow you # to associate an Elastic IP to it. As a result VPC ElasticSearch instances are always private, even with an # internet gateway. + ::AwsPublicIps::Utils.probe(::Aws::ElasticsearchService::Client, + options[:regions], options[:progress]) do |client| + client.list_domain_names.flat_map do |response| + response.domain_names.flat_map do |domain_name| + client.describe_elasticsearch_domain(domain_name: domain_name.domain_name).map do |domain| + hostname = domain.domain_status.endpoint + next unless hostname - client.list_domain_names.flat_map do |response| - response.domain_names.flat_map do |domain_name| - client.describe_elasticsearch_domain(domain_name: domain_name.domain_name).map do |domain| - hostname = domain.domain_status.endpoint - next unless hostname - - { - id: domain.domain_status.domain_id, - hostname: hostname, - ip_addresses: ::AwsPublicIps::Utils.resolve_hostname(hostname) - } - end.compact + { + region: client.config.region, + id: domain.domain_status.domain_id, + hostname: hostname, + ip_addresses: ::AwsPublicIps::Utils.resolve_hostname(hostname) + } + end.compact + end end end end diff --git a/lib/aws_public_ips/checks/elb.rb b/lib/aws_public_ips/checks/elb.rb index 2218c0a..ff3188b 100644 --- a/lib/aws_public_ips/checks/elb.rb +++ b/lib/aws_public_ips/checks/elb.rb @@ -6,23 +6,24 @@ module AwsPublicIps module Checks module Elb - def self.run - client = ::Aws::ElasticLoadBalancing::Client.new - return [] unless ::AwsPublicIps::Utils.has_service?(client) - + def self.run(options) # EC2-Classic load balancers are only returned by the 'elasticloadbalancing' API, and # EC2-VPC ALBs/NLBs are only returned by the 'elasticloadbalancingv2' API - client.describe_load_balancers.flat_map do |response| - response.load_balancer_descriptions.flat_map do |load_balancer| - next [] unless load_balancer.scheme == 'internet-facing' + ::AwsPublicIps::Utils.probe(::Aws::ElasticLoadBalancing::Client, + options[:regions], options[:progress]) do |client| + client.describe_load_balancers.flat_map do |response| + response.load_balancer_descriptions.flat_map do |load_balancer| + next [] unless load_balancer.scheme == 'internet-facing' - # EC2-Classic load balancers get IPv6 DNS records created but they are not returned by the API - hostnames = [load_balancer.dns_name, "ipv6.#{load_balancer.dns_name}"] - { - id: load_balancer.canonical_hosted_zone_name_id, - hostname: load_balancer.dns_name, - ip_addresses: ::AwsPublicIps::Utils.resolve_hostnames(hostnames) - } + # EC2-Classic load balancers get IPv6 DNS records created but they are not returned by the API + hostnames = [load_balancer.dns_name, "ipv6.#{load_balancer.dns_name}"] + { + region: client.config.region, + id: load_balancer.canonical_hosted_zone_name_id, + hostname: load_balancer.dns_name, + ip_addresses: ::AwsPublicIps::Utils.resolve_hostnames(hostnames) + } + end end end end diff --git a/lib/aws_public_ips/checks/elbv2.rb b/lib/aws_public_ips/checks/elbv2.rb index 459ed72..1846f04 100644 --- a/lib/aws_public_ips/checks/elbv2.rb +++ b/lib/aws_public_ips/checks/elbv2.rb @@ -6,25 +6,26 @@ module AwsPublicIps module Checks module Elbv2 - def self.run - client = ::Aws::ElasticLoadBalancingV2::Client.new - return [] unless ::AwsPublicIps::Utils.has_service?(client) - + def self.run(options) # EC2-Classic load balancers are only returned by the 'elasticloadbalancing' API, and # EC2-VPC ALBs/NLBs are only returned by the 'elasticloadbalancingv2' API # NLBs only support IPv4 # ALBs support IPv4 or dualstack. Unlike Classic ELBs which have a separate IPv6 DNS name, # dualstack ALBs only have a single DNS name - client.describe_load_balancers.flat_map do |response| - response.load_balancers.flat_map do |load_balancer| - next [] unless load_balancer.scheme == 'internet-facing' + ::AwsPublicIps::Utils.probe(::Aws::ElasticLoadBalancingV2::Client, + options[:regions], options[:progress]) do |client| + client.describe_load_balancers.flat_map do |response| + response.load_balancers.flat_map do |load_balancer| + next [] unless load_balancer.scheme == 'internet-facing' - { - id: load_balancer.canonical_hosted_zone_id, - hostname: load_balancer.dns_name, - ip_addresses: ::AwsPublicIps::Utils.resolve_hostname(load_balancer.dns_name) - } + { + region: client.config.region, + id: load_balancer.canonical_hosted_zone_id, + hostname: load_balancer.dns_name, + ip_addresses: ::AwsPublicIps::Utils.resolve_hostname(load_balancer.dns_name) + } + end end end end diff --git a/lib/aws_public_ips/checks/lightsail.rb b/lib/aws_public_ips/checks/lightsail.rb index aea8e14..843dce7 100644 --- a/lib/aws_public_ips/checks/lightsail.rb +++ b/lib/aws_public_ips/checks/lightsail.rb @@ -6,35 +6,35 @@ module AwsPublicIps module Checks module Lightsail - def self.run - client = ::Aws::Lightsail::Client.new - return [] unless ::AwsPublicIps::Utils.has_service?(client) - + def self.run(options) # Lightsail instances are always exposed directly, and can also be put behind a load balancer - - instances = client.get_instances.flat_map do |response| - response.instances.map do |instance| - { - # Names are unique - id: instance.name, - hostname: nil, - ip_addresses: [instance.public_ip_address] - } + ::AwsPublicIps::Utils.probe(::Aws::Lightsail::Client, options[:regions], options[:progress]) do |client| + instances = client.get_instances.flat_map do |response| + response.instances.map do |instance| + { + # Names are unique + region: client.config.region, + id: instance.name, + hostname: nil, + ip_addresses: [instance.public_ip_address] + } + end end - end - load_balancers = client.get_load_balancers.flat_map do |response| - response.load_balancers.map do |load_balancer| - { - # Names are unique - id: load_balancer.name, - hostname: load_balancer.dns_name, - ip_addresses: ::AwsPublicIps::Utils.resolve_hostname(load_balancer.dns_name) - } + load_balancers = client.get_load_balancers.flat_map do |response| + response.load_balancers.map do |load_balancer| + { + # Names are unique + region: client.config.region, + id: load_balancer.name, + hostname: load_balancer.dns_name, + ip_addresses: ::AwsPublicIps::Utils.resolve_hostname(load_balancer.dns_name) + } + end end - end - instances + load_balancers + instances + load_balancers + end end end end diff --git a/lib/aws_public_ips/checks/rds.rb b/lib/aws_public_ips/checks/rds.rb index 46f8694..950906e 100644 --- a/lib/aws_public_ips/checks/rds.rb +++ b/lib/aws_public_ips/checks/rds.rb @@ -6,28 +6,28 @@ module AwsPublicIps module Checks module Rds - def self.run - client = ::Aws::RDS::Client.new - return [] unless ::AwsPublicIps::Utils.has_service?(client) - + def self.run(options) # RDS instances can be launched into VPCs or into Classic mode. # In classic mode they are always public. # In VPC mode they can be marked as `publicly_accessible` or not - if they are then its VPC must have # an Internet Gateway attached, and the DNS endpoint will resolve to a public ip address. - client.describe_db_instances.flat_map do |response| - response.db_instances.flat_map do |instance| - next [] unless instance.publicly_accessible + ::AwsPublicIps::Utils.probe(::Aws::RDS::Client, options[:regions], options[:progress]) do |client| + client.describe_db_instances.flat_map do |response| + response.db_instances.flat_map do |instance| + next [] unless instance.publicly_accessible - if instance.endpoint.nil? - raise StandardError, "RDS DB '#{instance.dbi_resource_id}' has a nil endpoint. This likely" \ - ' means the DB is being brought up right now.' - end + if instance.endpoint.nil? + raise StandardError, "RDS DB '#{instance.dbi_resource_id}' has a nil endpoint. This likely" \ + ' means the DB is being brought up right now.' + end - { - id: instance.dbi_resource_id, - hostname: instance.endpoint.address, - ip_addresses: ::AwsPublicIps::Utils.resolve_hostname(instance.endpoint.address) - } + { + region: client.config.region, + id: instance.dbi_resource_id, + hostname: instance.endpoint.address, + ip_addresses: ::AwsPublicIps::Utils.resolve_hostname(instance.endpoint.address) + } + end end end end diff --git a/lib/aws_public_ips/checks/redshift.rb b/lib/aws_public_ips/checks/redshift.rb index b40b358..5080974 100644 --- a/lib/aws_public_ips/checks/redshift.rb +++ b/lib/aws_public_ips/checks/redshift.rb @@ -6,29 +6,29 @@ module AwsPublicIps module Checks module Redshift - def self.run - client = ::Aws::Redshift::Client.new - return [] unless ::AwsPublicIps::Utils.has_service?(client) - + def self.run(options) # Redshift clusters can be launched into VPCs or into Classic mode. # In classic mode they are always public. # In VPC mode they can be marked as `publicly_accessible` or not - if they are then its VPC must have # an Internet Gateway attached, and the DNS endpoint will resolve to a public ip address. - client.describe_clusters.flat_map do |response| - response.clusters.flat_map do |cluster| - next [] unless cluster.publicly_accessible + ::AwsPublicIps::Utils.probe(::Aws::Redshift::Client, options[:regions], options[:progress]) do |client| + client.describe_clusters.flat_map do |response| + response.clusters.flat_map do |cluster| + next [] unless cluster.publicly_accessible - if cluster.endpoint.nil? - raise StandardError, "Redshift cluster '#{cluster.cluster_identifier}' has a nil endpoint. This likely" \ - ' means the cluster is being brought up right now.' - end + if cluster.endpoint.nil? + raise StandardError, "Redshift cluster '#{cluster.cluster_identifier}' has a nil endpoint. This" \ + ' likely means the cluster is being brought up right now.' + end - { - id: cluster.cluster_identifier, - hostname: cluster.endpoint.address, - ip_addresses: cluster.cluster_nodes.map(&:public_ip_address) + - ::AwsPublicIps::Utils.resolve_hostname(cluster.endpoint.address) - } + { + region: client.config.region, + id: cluster.cluster_identifier, + hostname: cluster.endpoint.address, + ip_addresses: cluster.cluster_nodes.map(&:public_ip_address) + + ::AwsPublicIps::Utils.resolve_hostname(cluster.endpoint.address) + } + end end end end diff --git a/lib/aws_public_ips/cli.rb b/lib/aws_public_ips/cli.rb index 74fef76..2d055de 100644 --- a/lib/aws_public_ips/cli.rb +++ b/lib/aws_public_ips/cli.rb @@ -1,101 +1,12 @@ # frozen_string_literal: true -require 'optparse' +require 'aws_public_ips/cli_options' module AwsPublicIps class CLI - # Supported services: - # EC2 (and as a result: ECS, EKS, Beanstalk, Fargate, Batch, & NAT Instances) - # ELB (Classic ELB) - # ELBv2 (ALB/NLB) - # RDS - # Redshift - # APIGateway - # CloudFront - # Lightsail - # ElasticSearch - - # Services that don't need to be supported: - # S3 - all s3 buckets resolve to the same ip addresses - # SQS - there's a single AWS-owned domain per region (i.e. sqs.us-east-1.amazonaws.com//) - # NAT Gateways - these do not allow ingress traffic - # ElastiCache - all elasticache instances are private. You can make one public by using a NAT instance with an EIP: - # https://docs.aws.amazon.com/AmazonElastiCache/latest/red-ug/accessing-elasticache.html#access-from-outside-aws - # but NAT instances are EC2 machines, so this will be caught by the EC2 check. - # DynamoDB - no public endpoints - # SNS - no public endpoints - # Elastic Transcoder - no public endpoints - # Athena - no public endpoints - - # Services that maybe have public endpoints / still need testing: - # fargate - # amazonmq - # directory service (AD) - # emr - # Directconnect - # Kinesis - # SES - # https://aws.amazon.com/products/ - # AWS Neptune (still in preview / not GA yet) - - def all_services - @all_services ||= ::Dir["#{__dir__}/checks/*.rb"].map { |path| ::File.basename(path, '.rb') }.sort - end - - def all_formats - @all_formats ||= ::Dir["#{__dir__}/formatters/*.rb"].map { |path| ::File.basename(path, '.rb') }.sort - end - - def parse(args) - options = { - format: 'text', - services: all_services, - verbose: false - } - - ::OptionParser.new do |parser| - parser.banner = 'Usage: aws_public_ips [options]' - - parser.on('-s', '--services ,,', Array, 'List of AWS services to check. Available services: ' \ - "#{all_services.join(',')}. Defaults to all.") do |services| - services.map(&:downcase!).uniq! - invalid_services = services - all_services - raise ::ArgumentError, "Invalid service(s): #{invalid_services.join(',')}" unless invalid_services.empty? - - options[:services] = services - end - - parser.on('-f', '--format ', String, 'Set output format. Available formats: ' \ - "#{all_formats.join(',')}. Defaults to text.") do |fmt| - unless all_formats.include?(fmt) - raise ::ArgumentError, "Invalid format '#{fmt}'. Valid formats are: #{all_formats.join(',')}" - end - - options[:format] = fmt - end - - parser.on('-v', '--[no-]verbose', 'Enable debug/trace output') do |verbose| - options[:verbose] = verbose - end - - parser.on_tail('--version', 'Print version') do - require 'aws_public_ips/version' - ::STDOUT.puts ::AwsPublicIps::VERSION - return nil # nil to avoid rubocop warning - end - - parser.on_tail('-h', '--help', 'Show this help message') do - ::STDOUT.puts parser - return nil # nil to avoid rubocop warning - end - end.parse(args) - - options - end - - def check_service(service) + def check_service(options, service) require "aws_public_ips/checks/#{service}.rb" - ::AwsPublicIps::Checks.const_get(service.capitalize).run + ::AwsPublicIps::Checks.const_get(service.capitalize).run(options) end def output(formatter, options, results) @@ -106,14 +17,23 @@ def output(formatter, options, results) end def run(args) - options = parse(args) + cli_options = ::AwsPublicIps::CLIOptions.new + parse_and_run(cli_options, args) + end + + def parse_and_run(cli_options, args) + options = cli_options.parse(args) return unless options results = options[:services].map do |service| - [service.to_sym, check_service(service)] + [service.to_sym, check_service(options, service)] end.to_h output(options[:format], options, results) + rescue OptionParser::InvalidOption, ArgumentError => ex + ::STDERR.puts ex + ::STDERR.puts cli_options.usage + ::Process.exit(1) rescue ::StandardError, ::Interrupt => ex ::STDERR.puts ex.inspect ::STDERR.puts ex.backtrace if options && options[:verbose] diff --git a/lib/aws_public_ips/cli_options.rb b/lib/aws_public_ips/cli_options.rb new file mode 100644 index 0000000..239ba02 --- /dev/null +++ b/lib/aws_public_ips/cli_options.rb @@ -0,0 +1,138 @@ +# frozen_string_literal: true + +require 'aws-sdk-ec2' +require 'optparse' + +module AwsPublicIps + class CLIOptions + # Supported services: + # EC2 (and as a result: ECS, EKS, Beanstalk, Fargate, Batch, & NAT Instances) + # ELB (Classic ELB) + # ELBv2 (ALB/NLB) + # RDS + # Redshift + # APIGateway + # CloudFront + # Lightsail + # ElasticSearch + + # Services that don't need to be supported: + # S3 - all s3 buckets resolve to the same ip addresses + # SQS - there's a single AWS-owned domain per region (i.e. sqs.us-east-1.amazonaws.com//) + # NAT Gateways - these do not allow ingress traffic + # ElastiCache - all elasticache instances are private. You can make one public by using a NAT instance with an EIP: + # https://docs.aws.amazon.com/AmazonElastiCache/latest/red-ug/accessing-elasticache.html#access-from-outside-aws + # but NAT instances are EC2 machines, so this will be caught by the EC2 check. + # DynamoDB - no public endpoints + # SNS - no public endpoints + # Elastic Transcoder - no public endpoints + # Athena - no public endpoints + + # Services that maybe have public endpoints / still need testing: + # fargate + # amazonmq + # directory service (AD) + # emr + # Directconnect + # Kinesis + # SES + # https://aws.amazon.com/products/ + # AWS Neptune (still in preview / not GA yet) + def initialize + @options = { + format: 'text', + regions: default_regions, + services: all_services, + tags: nil, + verbose: false, + progress: false + } + @parser = ::OptionParser.new do |p| + p.banner = 'Usage: aws_public_ips [options]' + + p.on('-r', '--regions ,,', Array, 'List of AWS services to check. Defaults to ' \ + '$AWS_DEFAULT_REGION or $AWS_REGION. Specify `all` in include all regions.') do |regions| + regions.map(&:downcase!).uniq! + (regions << all_regions).flatten!.map(&:downcase!).uniq! if regions.reject! { |r| r == 'all' } + invalid_regions = regions - all_regions + raise ::ArgumentError, "Invalid region(s): #{invalid_regions.join(',')}" unless invalid_regions.empty? + + @options[:regions] = regions + end + + p.on('-s', '--services ,,', Array, 'List of AWS services to check. Available services: ' \ + "#{all_services.join(',')}. Defaults to all.") do |services| + services.map(&:downcase!).uniq! + invalid_services = services - all_services + raise ::ArgumentError, "Invalid service(s): #{invalid_services.join(',')}" unless invalid_services.empty? + + @options[:services] = services + end + + p.on('-t', '--include_tags ,,', Array, 'List of tags to include in the results') do |tags| + @options[:tags] = tags.map(&:downcase).uniq + end + + p.on('-f', '--format ', String, 'Set output format. Available formats: ' \ + "#{all_formats.join(',')}. Defaults to text.") do |fmt| + unless all_formats.include?(fmt) + raise ::ArgumentError, "Invalid format '#{fmt}'. Valid formats are: #{all_formats.join(',')}" + end + + @options[:format] = fmt + end + + p.on('-v', '--[no-]verbose', 'Enable debug/trace output') do |verbose| + @options[:verbose] = verbose + end + + p.on('-p', '--[no-]progress', 'Enable progress') do |progress| + @options[:progress] = progress + end + + p.on_tail('--version', 'Print version') do + require 'aws_public_ips/version' + ::STDOUT.puts ::AwsPublicIps::VERSION + exit 0 + end + + p.on_tail('-h', '--help', 'Show this help message') do + ::STDOUT.puts p + exit 0 + end + end + end + + def all_services + @all_services ||= ::Dir["#{__dir__}/checks/*.rb"].map { |path| ::File.basename(path, '.rb') }.sort + end + + def all_regions + @all_regions ||= ::Aws::EC2::Client.new(region: 'us-east-1') + .describe_regions.regions.flat_map(&:region_name).collect.sort + end + + def default_regions + return [::ENV['AWS_DEFAULT_REGION']] unless ::ENV['AWS_DEFAULT_REGION'].nil? || ::ENV['AWS_DEFAULT_REGION'].empty? + + return [::ENV['AWS_REGION']] unless ::ENV['AWS_REGION'].nil? || ::ENV['AWS_REGION'].empty? + + nil + end + + def all_formats + @all_formats ||= ::Dir["#{__dir__}/formatters/*.rb"].map { |path| ::File.basename(path, '.rb') }.sort + end + + def parse(args) + @parser.parse(args) + raise ::ArgumentError, 'missing option: You must specify a region or set AWS_REGION.' unless @options[:regions] + + @options + end + + def usage + @parser.to_s + end + end +end diff --git a/lib/aws_public_ips/utils.rb b/lib/aws_public_ips/utils.rb index 02d59ef..8b43be4 100644 --- a/lib/aws_public_ips/utils.rb +++ b/lib/aws_public_ips/utils.rb @@ -2,6 +2,7 @@ require 'resolv' require 'aws-partitions' +require 'tty-spinner' module AwsPublicIps module Utils @@ -25,10 +26,77 @@ def self.has_service?(client) region_partition = ::Aws::Partitions.partitions.find do |partition| partition.regions.map(&:name).include?(client.config.region) end - service_name = client.class.to_s.split('::')[-2] + service_name = service_name(client.class) aws_service = region_partition.services.find { |service| service.name == service_name } !aws_service.regionalized? || aws_service.regions.include?(client.config.region) end + + def self.add_tags(result, source, tags) + return unless tags && source.tags + + tags.each do |tag| + tag_item = source.tags.find { |t| t.key.downcase == tag } + if tag_item + tag_key = tag_item.key.downcase + result[tag_key.to_sym] = tag_item.value + end + end + end + + def self.probe(client_class, regions, progress, &block) + return [] unless regions && !regions.empty? + + if progress + service_name = service_name(client_class) + spinners = create_spinner(TTY::Spinner::Multi, service_name) + begin + child_spinners = regions.map { |r| [r, spinners.register("[:spinner] #{r}")] }.to_h + result = regions.flat_map do |region| + probe_region_with_progress(child_spinners[region], client_class, region, &block) + end + spinners.success + result + rescue StandardError + spinners.error + [] + end + else + regions.flat_map do |region| + probe_region(client_class, region, &block) + end + end + end + + def self.service_name(client_class) + client_class.to_s.split('::')[-2] + end + + def self.create_spinner(spinner_class, service_name) + spinner_class.new("[:spinner] Probing #{service_name}...", + format: :dots, + success_mark: '+', + errror_mark: 'x') + end + + def self.probe_region_with_progress(spinner, client_class, region, &block) + spinner.run do |s| + begin + result = probe_region(client_class, region, &block) + s.success + result + rescue StandardError + s.error + [] + end + end.value + end + + def self.probe_region(client_class, region, &block) + client = region ? client_class.new(region: region) : client_class.new + return [] unless ::AwsPublicIps::Utils.has_service?(client) + + block.call(client) + end end end diff --git a/spec/aws_public_ips/checks/apigateway_spec.rb b/spec/aws_public_ips/checks/apigateway_spec.rb index 7fe0193..97e11ca 100644 --- a/spec/aws_public_ips/checks/apigateway_spec.rb +++ b/spec/aws_public_ips/checks/apigateway_spec.rb @@ -10,13 +10,15 @@ 'tmtmok31nc.execute-api.us-east-1.amazonaws.com' => %w[54.0.0.3] ) - expect(subject.run).to eq([ + expect(run_check(subject)).to eq([ { + region: 'us-east-1', id: 'e83d6nij2j', hostname: 'e83d6nij2j.execute-api.us-east-1.amazonaws.com', ip_addresses: %w[54.0.0.1 54.0.0.2] }, { + region: 'us-east-1', id: 'tmtmok31nc', hostname: 'tmtmok31nc.execute-api.us-east-1.amazonaws.com', ip_addresses: %w[54.0.0.3] diff --git a/spec/aws_public_ips/checks/cloudfront_spec.rb b/spec/aws_public_ips/checks/cloudfront_spec.rb index 2ab1580..404a9fd 100644 --- a/spec/aws_public_ips/checks/cloudfront_spec.rb +++ b/spec/aws_public_ips/checks/cloudfront_spec.rb @@ -9,7 +9,7 @@ 'd22ycgwdruc4lt.cloudfront.net' => %w[54.0.0.1 54.0.0.2], 'd1k00qwg2uxphp.cloudfront.net' => %w[54.0.0.3] ) - expect(subject.run).to eq([ + expect(run_check(subject)).to eq([ { id: 'E1DABYDY46RHFK', hostname: 'd22ycgwdruc4lt.cloudfront.net', diff --git a/spec/aws_public_ips/checks/ec2_spec.rb b/spec/aws_public_ips/checks/ec2_spec.rb index 12c2176..69c8238 100644 --- a/spec/aws_public_ips/checks/ec2_spec.rb +++ b/spec/aws_public_ips/checks/ec2_spec.rb @@ -5,18 +5,23 @@ stub_request(:post, 'https://ec2.us-east-1.amazonaws.com') .to_return(body: ::IO.read('spec/fixtures/ec2.xml')) - expect(subject.run).to eq([ + expect(run_check(subject)).to eq([ { + region: 'us-east-1', id: 'i-0f22d0af796b3cf3a', hostname: 'ec2-54-234-208-236.compute-1.amazonaws.com', + name: 'foo', ip_addresses: %w[54.234.208.236] }, { + region: 'us-east-1', id: 'i-04ef469b441883eda', hostname: 'ec2-18-206-76-65.compute-1.amazonaws.com', + name: 'bar', ip_addresses: %w[18.206.76.65 2600:1f18:63e0:b400:f50c:59a7:a182:3717] }, { + region: 'us-east-1', id: 'i-03a7b3bc2b4c20742', hostname: nil, ip_addresses: %w[2600:1f18:60f3:eb00:1c6e:5184:8955:170c] @@ -32,6 +37,6 @@ it 'should not return an entry when there are no public ips' do stub_request(:post, 'https://ec2.us-east-1.amazonaws.com') .to_return(body: ::IO.read('spec/fixtures/ec2-private.xml')) - expect(subject.run).to eq([]) + expect(run_check(subject)).to eq([]) end end diff --git a/spec/aws_public_ips/checks/elasticsearch_spec.rb b/spec/aws_public_ips/checks/elasticsearch_spec.rb index 5f831be..bbb7b3e 100644 --- a/spec/aws_public_ips/checks/elasticsearch_spec.rb +++ b/spec/aws_public_ips/checks/elasticsearch_spec.rb @@ -13,8 +13,9 @@ 'search-classic-fd5cbkkjuuiudho2lrwmsjp6rm.us-east-1.es.amazonaws.com' => %w[54.0.0.1] ) - expect(subject.run).to eq([ + expect(run_check(subject)).to eq([ { + region: 'us-east-1', id: '154967844790/classic', hostname: 'search-classic-fd5cbkkjuuiudho2lrwmsjp6rm.us-east-1.es.amazonaws.com', ip_addresses: %w[54.0.0.1] diff --git a/spec/aws_public_ips/checks/elb_spec.rb b/spec/aws_public_ips/checks/elb_spec.rb index 209062d..48785aa 100644 --- a/spec/aws_public_ips/checks/elb_spec.rb +++ b/spec/aws_public_ips/checks/elb_spec.rb @@ -10,8 +10,9 @@ 'ipv6.classic-272004174.us-east-1.elb.amazonaws.com' => %w[2600:1f18:63e0:b401:b952:5715:d981:2776] ) - expect(subject.run).to eq([ + expect(run_check(subject)).to eq([ { + region: 'us-east-1', id: 'Z35SXDOTRQ7X7K', hostname: 'classic-272004174.us-east-1.elb.amazonaws.com', ip_addresses: %w[54.0.0.1 2600:1f18:63e0:b401:b952:5715:d981:2776] diff --git a/spec/aws_public_ips/checks/elbv2_spec.rb b/spec/aws_public_ips/checks/elbv2_spec.rb index bb664db..202e861 100644 --- a/spec/aws_public_ips/checks/elbv2_spec.rb +++ b/spec/aws_public_ips/checks/elbv2_spec.rb @@ -10,13 +10,15 @@ 'alb-vpc-228877692.us-east-1.elb.amazonaws.com' => %w[54.0.0.2 2600:1f18:63e0:b401:b952:5715:d981:2776] ) - expect(subject.run).to eq([ + expect(run_check(subject)).to eq([ { + region: 'us-east-1', id: 'Z26RNL4JYFTOTI', hostname: 'nlb-vpc-d243d0acc9151631.elb.us-east-1.amazonaws.com', ip_addresses: %w[54.0.0.1] }, { + region: 'us-east-1', id: 'Z35SXDOTRQ7X7K', hostname: 'alb-vpc-228877692.us-east-1.elb.amazonaws.com', ip_addresses: %w[54.0.0.2 2600:1f18:63e0:b401:b952:5715:d981:2776] diff --git a/spec/aws_public_ips/checks/lightsail_spec.rb b/spec/aws_public_ips/checks/lightsail_spec.rb index f3131e1..c733a95 100644 --- a/spec/aws_public_ips/checks/lightsail_spec.rb +++ b/spec/aws_public_ips/checks/lightsail_spec.rb @@ -10,13 +10,15 @@ 'ce551c6f952085b4126e4b523a100eda-232829524.us-east-1.elb.amazonaws.com' => %w[54.88.163.90 52.205.146.152] ) - expect(subject.run).to eq([ + expect(run_check(subject)).to eq([ { + region: 'us-east-1', id: 'Amazon_Linux-512MB-Virginia-1', hostname: nil, ip_addresses: %w[18.206.153.10] }, { + region: 'us-east-1', id: 'LoadBalancer-Virginia-1', hostname: 'ce551c6f952085b4126e4b523a100eda-232829524.us-east-1.elb.amazonaws.com', ip_addresses: %w[54.88.163.90 52.205.146.152] diff --git a/spec/aws_public_ips/checks/rds_spec.rb b/spec/aws_public_ips/checks/rds_spec.rb index a98d26c..bc8ec18 100644 --- a/spec/aws_public_ips/checks/rds_spec.rb +++ b/spec/aws_public_ips/checks/rds_spec.rb @@ -10,13 +10,15 @@ 'rds-vpc-public-us-east-1a.cyvjlokb0o75.us-east-1.rds.amazonaws.com' => %w[52.70.34.110] ) - expect(subject.run).to eq([ + expect(run_check(subject)).to eq([ { + region: 'us-east-1', id: 'db-BAKGGXYRB3EBKBDQZTAMDSGCXY', hostname: 'rds-vpc-public.cyvjlokb0o75.us-east-1.rds.amazonaws.com', ip_addresses: %w[35.171.145.174] }, { + region: 'us-east-1', id: 'db-B3N4GULDAFGDKEGXBD7CXKZQV4', hostname: 'rds-vpc-public-us-east-1a.cyvjlokb0o75.us-east-1.rds.amazonaws.com', ip_addresses: %w[52.70.34.110] @@ -28,13 +30,13 @@ stub_request(:post, 'https://rds.us-east-1.amazonaws.com') .to_return(body: ::IO.read('spec/fixtures/rds-vpc-private.xml')) - expect(subject.run).to eq([]) + expect(run_check(subject)).to eq([]) end it 'should handle db instances with a nil endpoint' do stub_request(:post, 'https://rds.us-east-1.amazonaws.com') .to_return(body: ::IO.read('spec/fixtures/rds-empty-endpoint.xml')) - expect { subject.run }.to raise_error(StandardError, /has a nil endpoint/) + expect { run_check(subject) }.to raise_error(StandardError, /has a nil endpoint/) end end diff --git a/spec/aws_public_ips/checks/redshift_spec.rb b/spec/aws_public_ips/checks/redshift_spec.rb index 4a2ec45..125071e 100644 --- a/spec/aws_public_ips/checks/redshift_spec.rb +++ b/spec/aws_public_ips/checks/redshift_spec.rb @@ -5,8 +5,9 @@ stub_request(:post, 'https://redshift.us-east-1.amazonaws.com') .to_return(body: ::IO.read('spec/fixtures/redshift-classic-public.xml')) - expect(subject.run).to eq([ + expect(run_check(subject)).to eq([ { + region: 'us-east-1', id: 'classic', hostname: 'classic.csorkyt5dk7h.us-east-1.redshift.amazonaws.com', ip_addresses: %w[54.167.97.240 54.91.252.196 54.242.227.110] @@ -21,8 +22,9 @@ stub_dns( 'vpc-public-2.csorkyt5dk7h.us-east-1.redshift.amazonaws.com' => %w[54.156.7.130] ) - expect(subject.run).to eq([ + expect(run_check(subject)).to eq([ { + region: 'us-east-1', id: 'vpc-public-2', hostname: 'vpc-public-2.csorkyt5dk7h.us-east-1.redshift.amazonaws.com', ip_addresses: %w[52.44.170.10 54.209.53.206 54.208.75.129 54.156.7.130] @@ -34,12 +36,12 @@ stub_request(:post, 'https://redshift.us-east-1.amazonaws.com') .to_return(body: ::IO.read('spec/fixtures/redshift-vpc-private.xml')) - expect(subject.run).to eq([]) + expect(run_check(subject)).to eq([]) end it 'should handle clusters with a nil endpoint' do stub_request(:post, 'https://redshift.us-east-1.amazonaws.com') .to_return(body: ::IO.read('spec/fixtures/redshift-empty-endpoint.xml')) - expect { subject.run }.to raise_error(StandardError, /has a nil endpoint/) + expect { run_check(subject) }.to raise_error(StandardError, /has a nil endpoint/) end end diff --git a/spec/aws_public_ips/cli_options_spec.rb b/spec/aws_public_ips/cli_options_spec.rb new file mode 100644 index 0000000..55cc2dd --- /dev/null +++ b/spec/aws_public_ips/cli_options_spec.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +describe ::AwsPublicIps::CLIOptions do + it 'should parse the options' do + stub_describe_regions + options = subject.parse(%w[--format json --services ec2,elb,redshift --verbose]) + expect(options).to eq(format: 'json', progress: false, regions: ['us-east-1'], + services: %w[ec2 elb redshift], tags: nil, verbose: true) + end + + it 'should raise on an invalid formatter' do + stub_describe_regions + expect { subject.parse(%w[--format blah]) }.to raise_error(::ArgumentError, /Invalid format/) + end + + it 'should raise on an invalid service' do + stub_describe_regions + expect { subject.parse(%w[--service blah]) }.to raise_error(::ArgumentError, /Invalid service/) + end + + it 'should select the right directory' do + stub_describe_regions + ::Dir.chdir('/') do + options = subject.parse(%w[--service ec2 --format prettyjson]) + expect(options).to include(services: %w[ec2], format: 'prettyjson') + end + end + + it 'should raise on an invalid region' do + stub_describe_regions + expect { subject.parse(%w[--region blah]) }.to raise_error(::ArgumentError, /Invalid region/) + end + + it 'should allow all regions to be specified' do + stub_describe_regions + options = subject.parse(%w[--region all]) + expect(options).to(satisfy { |o| o[:regions].length == 16 }) + end + + it 'should allow tags to be included' do + stub_describe_regions + options = subject.parse(%w[--include_tags name,service,name]) + expect(options[:tags]).to contain_exactly('name', 'service') + end + + it 'should allow progress to be enabled' do + stub_describe_regions + options = subject.parse(%w[--progress]) + expect(options[:progress]).to be_truthy + end + + it 'should be able to print usage' do + expect(subject.usage).to include('Usage: aws_public_ips') + end +end diff --git a/spec/aws_public_ips/cli_spec.rb b/spec/aws_public_ips/cli_spec.rb index e250525..993ebc2 100644 --- a/spec/aws_public_ips/cli_spec.rb +++ b/spec/aws_public_ips/cli_spec.rb @@ -1,27 +1,8 @@ # frozen_string_literal: true describe ::AwsPublicIps::CLI do - it 'should parse the options' do - options = subject.parse(%w[--format json --services ec2,elb,redshift --verbose]) - expect(options).to eq(format: 'json', services: %w[ec2 elb redshift], verbose: true) - end - - it 'should raise on an invalid formatter' do - expect { subject.parse(%w[--format blah]) }.to raise_error(::ArgumentError, /Invalid format/) - end - - it 'should raise on an invalid service' do - expect { subject.parse(%w[--service blah]) }.to raise_error(::ArgumentError, /Invalid service/) - end - - it 'should select the right directory' do - ::Dir.chdir('/') do - options = subject.parse(%w[--service ec2 --format prettyjson]) - expect(options).to include(services: %w[ec2], format: 'prettyjson') - end - end - it 'should run' do + stub_describe_regions expect(::AwsPublicIps::Checks::Ec2).to receive(:run).and_return([{ id: 'i-0f22d0af796b3cf3a', hostname: 'ec2-54-234-208-236.compute-1.amazonaws.com', @@ -31,7 +12,17 @@ subject.run(['-s', 'ec2']) end + it 'should run with progress' do + stub_describe_regions + stub_request(:post, 'https://ec2.us-east-1.amazonaws.com') + .to_return(body: ::IO.read('spec/fixtures/ec2.xml')) + expect(::STDOUT).to receive(:puts) + expect(::STDERR).to receive(:print).at_least(4).times + subject.run(['-p', '-s', 'ec2']) + end + it 'should rescue exceptions' do + stub_describe_regions expect(subject).to receive(:check_service).and_raise(::StandardError) expect(::Process).to receive(:exit).with(1) expect(::STDERR).to receive(:puts) @@ -39,12 +30,25 @@ end it 'should print the version' do + stub_describe_regions expect(::STDOUT).to receive(:puts).with(::AwsPublicIps::VERSION) - subject.run(['--version']) + expect { subject.run(['--version']) }.to raise_error(::SystemExit) end it 'should print help' do + stub_describe_regions expect(::STDOUT).to receive(:puts) - subject.run(['--help']) + expect { subject.run(['--help']) }.to raise_error(::SystemExit) + end + + it 'should print help on option exceptions' do + begin + stub_describe_regions + expectation = expect { subject.run(['--format', 'blah']) } + expectation.to output(/Invalid format/).to_stderr_from_any_process + expectation.to output(/Usage: /).to_stderr_from_any_process + rescue SystemExit => e + expect(e.status).to eq(1) + end end end diff --git a/spec/fixtures/describe-regions.xml b/spec/fixtures/describe-regions.xml new file mode 100644 index 0000000..3cd25ca --- /dev/null +++ b/spec/fixtures/describe-regions.xml @@ -0,0 +1,70 @@ + + + fcc21266-e812-4eb3-978e-c079faa443dc + + + ap-south-1 + ec2.ap-south-1.amazonaws.com + + + eu-west-3 + ec2.eu-west-3.amazonaws.com + + + eu-north-1 + ec2.eu-north-1.amazonaws.com + + + eu-west-2 + ec2.eu-west-2.amazonaws.com + + + eu-west-1 + ec2.eu-west-1.amazonaws.com + + + ap-northeast-2 + ec2.ap-northeast-2.amazonaws.com + + + ap-northeast-1 + ec2.ap-northeast-1.amazonaws.com + + + sa-east-1 + ec2.sa-east-1.amazonaws.com + + + ca-central-1 + ec2.ca-central-1.amazonaws.com + + + ap-southeast-1 + ec2.ap-southeast-1.amazonaws.com + + + ap-southeast-2 + ec2.ap-southeast-2.amazonaws.com + + + eu-central-1 + ec2.eu-central-1.amazonaws.com + + + us-east-1 + ec2.us-east-1.amazonaws.com + + + us-east-2 + ec2.us-east-2.amazonaws.com + + + us-west-1 + ec2.us-west-1.amazonaws.com + + + us-west-2 + ec2.us-west-2.amazonaws.com + + + \ No newline at end of file diff --git a/spec/fixtures/ec2.xml b/spec/fixtures/ec2.xml index 62002ea..c1c5f85 100644 --- a/spec/fixtures/ec2.xml +++ b/spec/fixtures/ec2.xml @@ -58,6 +58,12 @@ hvm + + + Name + foo + + xen false @@ -124,6 +130,12 @@ hvm + + + name + bar + + xen diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 83a4df0..a62e847 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -4,6 +4,7 @@ ::Coveralls.wear! do |config| # Output html report for development & default json report for Travis/Coveralls config.formatter = SimpleCov::Formatter::HTMLFormatter unless ::ENV['TRAVIS'] + config.add_filter 'spec' end ::ENV['COVERALLS_NOISY'] = '1' require 'webmock/rspec' @@ -19,3 +20,13 @@ def stub_dns(mapping) expect(::AwsPublicIps::Utils).to receive(:resolve_hostname).with(hostname).and_return(ips) end end + +def stub_describe_regions + stub_request(:post, 'https://ec2.us-east-1.amazonaws.com/') + .with(body: /Action=DescribeRegions/) + .to_return(status: 200, body: ::IO.read('spec/fixtures/describe-regions.xml')) +end + +def run_check(subject) + subject.run(regions: [::ENV['AWS_REGION']], tags: ['name'], progress: false) +end