From d807ce084187d3ba436b960a61f2dac0f3eacbfb Mon Sep 17 00:00:00 2001 From: Joel Baranick Date: Wed, 30 Jan 2019 19:13:27 -0800 Subject: [PATCH 1/9] Multiregion support with process and ec2 tag data --- .gitignore | 2 + aws_public_ips.gemspec | 1 + lib/aws_public_ips.rb | 1 + lib/aws_public_ips/checks/apigateway.rb | 30 ++-- lib/aws_public_ips/checks/cloudfront.rb | 26 ++-- lib/aws_public_ips/checks/ec2.rb | 47 +++--- lib/aws_public_ips/checks/elasticsearch.rb | 32 ++-- lib/aws_public_ips/checks/elb.rb | 29 ++-- lib/aws_public_ips/checks/elbv2.rb | 25 ++-- lib/aws_public_ips/checks/lightsail.rb | 48 +++--- lib/aws_public_ips/checks/rds.rb | 32 ++-- lib/aws_public_ips/checks/redshift.rb | 34 ++--- lib/aws_public_ips/cli.rb | 108 ++------------ lib/aws_public_ips/cli_options.rb | 138 ++++++++++++++++++ lib/aws_public_ips/utils.rb | 70 ++++++++- lib/aws_public_ips/version.rb | 2 +- spec/aws_public_ips/checks/apigateway_spec.rb | 4 +- spec/aws_public_ips/checks/cloudfront_spec.rb | 2 +- spec/aws_public_ips/checks/ec2_spec.rb | 7 +- .../checks/elasticsearch_spec.rb | 3 +- spec/aws_public_ips/checks/elb_spec.rb | 3 +- spec/aws_public_ips/checks/elbv2_spec.rb | 4 +- spec/aws_public_ips/checks/lightsail_spec.rb | 4 +- spec/aws_public_ips/checks/rds_spec.rb | 8 +- spec/aws_public_ips/checks/redshift_spec.rb | 10 +- spec/aws_public_ips/cli_options_spec.rb | 28 ++++ spec/aws_public_ips/cli_spec.rb | 28 +--- spec/fixtures/regions.xml | 69 +++++++++ spec/spec_helper.rb | 11 ++ 29 files changed, 524 insertions(+), 282 deletions(-) create mode 100644 lib/aws_public_ips/cli_options.rb create mode 100644 spec/aws_public_ips/cli_options_spec.rb create mode 100644 spec/fixtures/regions.xml diff --git a/.gitignore b/.gitignore index fe21d52..9217d07 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ *.swp .bundle +.idea coverage/ Gemfile.lock vendor/bundle +aws_public_ips-*.gem diff --git a/aws_public_ips.gemspec b/aws_public_ips.gemspec index 542956a..5a80de2 100644 --- a/aws_public_ips.gemspec +++ b/aws_public_ips.gemspec @@ -26,6 +26,7 @@ require 'aws_public_ips/version' gem.add_dependency('aws-sdk-lightsail', '~> 1.10.0') gem.add_dependency('aws-sdk-rds', '~> 1.35.0') gem.add_dependency('aws-sdk-redshift', '~> 1.13.0') + gem.add_dependency('tty-spinner', '~>0.9.0 ') gem.add_development_dependency('bundler-audit', '~> 0.6.0') gem.add_development_dependency('coveralls', '~> 0.8.22') diff --git a/lib/aws_public_ips.rb b/lib/aws_public_ips.rb index b9f492a..7101be2 100644 --- a/lib/aws_public_ips.rb +++ b/lib/aws_public_ips.rb @@ -5,6 +5,7 @@ module AwsPublicIps require 'aws_public_ips/checks' require 'aws_public_ips/cli' +require 'aws_public_ips/cli_options' require 'aws_public_ips/formatters' require 'aws_public_ips/utils' require 'aws_public_ips/version' diff --git a/lib/aws_public_ips/checks/apigateway.rb b/lib/aws_public_ips/checks/apigateway.rb index 40b416c..d5d98f6 100644 --- a/lib/aws_public_ips/checks/apigateway.rb +++ b/lib/aws_public_ips/checks/apigateway.rb @@ -6,22 +6,22 @@ module AwsPublicIps module Checks module Apigateway - def self.run - client = ::Aws::APIGateway::Client.new - return [] unless ::AwsPublicIps::Utils.has_service?(client) + def self.run(options) + ::AwsPublicIps::Utils.probe(::Aws::APIGateway::Client, options[:regions], options[:progress]) do |client| + # TODO(arkadiy) https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-private-integration.html - # TODO(arkadiy) https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-private-integration.html - - # APIGateway doesn't return the full domain in the response, we have to build - # it using the api id and region - client.get_rest_apis.flat_map do |response| - response.items.map do |api| - hostname = "#{api.id}.execute-api.#{client.config.region}.amazonaws.com" - { - id: api.id, - hostname: hostname, - ip_addresses: ::AwsPublicIps::Utils.resolve_hostname(hostname) - } + # APIGateway doesn't return the full domain in the response, we have to build + # it using the api id and region + client.get_rest_apis.flat_map do |response| + response.items.map do |api| + hostname = "#{api.id}.execute-api.#{client.config.region}.amazonaws.com" + { + region: client.config.region, + id: api.id, + hostname: hostname, + ip_addresses: ::AwsPublicIps::Utils.resolve_hostname(hostname) + } + end end end end diff --git a/lib/aws_public_ips/checks/cloudfront.rb b/lib/aws_public_ips/checks/cloudfront.rb index bbf1b2d..0c9aefc 100644 --- a/lib/aws_public_ips/checks/cloudfront.rb +++ b/lib/aws_public_ips/checks/cloudfront.rb @@ -6,20 +6,20 @@ module AwsPublicIps module Checks module Cloudfront - def self.run - client = ::Aws::CloudFront::Client.new - return [] unless ::AwsPublicIps::Utils.has_service?(client) + def self.run(options) + ::AwsPublicIps::Utils.probe(::Aws::CloudFront::Client, options[:regions] & ['us-east-1'], + options[:progress]) do |client| + # Cloudfront distributions are always public, they don't have a concept of VPC + # No "coming up" problem here like with RDS/Redshift - # Cloudfront distrubtions are always public, they don't have a concept of VPC - # No "coming up" problem here like with RDS/Redshift - - client.list_distributions.flat_map do |response| - response.distribution_list.items.flat_map do |distribution| - { - id: distribution.id, - hostname: distribution.domain_name, - ip_addresses: ::AwsPublicIps::Utils.resolve_hostname(distribution.domain_name) - } + client.list_distributions.flat_map do |response| + response.distribution_list.items.flat_map do |distribution| + { + id: distribution.id, + hostname: distribution.domain_name, + ip_addresses: ::AwsPublicIps::Utils.resolve_hostname(distribution.domain_name) + } + end end end end diff --git a/lib/aws_public_ips/checks/ec2.rb b/lib/aws_public_ips/checks/ec2.rb index 452254e..8c39d42 100644 --- a/lib/aws_public_ips/checks/ec2.rb +++ b/lib/aws_public_ips/checks/ec2.rb @@ -6,34 +6,37 @@ module AwsPublicIps module Checks module Ec2 - def self.run - client = ::Aws::EC2::Client.new - return [] unless ::AwsPublicIps::Utils.has_service?(client) - + def self.run(options) # Iterate over all EC2 instances. This will include those from EC2, ECS, EKS, Fargate, Batch, # Beanstalk, and NAT Instances # It will not include NAT Gateways (IPv4) or Egress Only Internet Gateways (IPv6), but they do not allow # ingress traffic so we skip them anyway - client.describe_instances.flat_map do |response| - response.reservations.flat_map do |reservation| - reservation.instances.flat_map do |instance| - # EC2-Classic instances have a `public_ip_address` and no `network_interfaces` - # EC2-VPC instances both set, so we uniq the ip addresses - ip_addresses = [instance.public_ip_address].compact + instance.network_interfaces.flat_map do |interface| - public_ip = interface.association ? [interface.association.public_ip].compact : [] - public_ip + interface.ipv_6_addresses.map(&:ipv_6_address) - end + ::AwsPublicIps::Utils.probe(::Aws::EC2::Client, options[:regions], options[:progress]) do |client| + client.describe_instances.flat_map do |response| + response.reservations.flat_map do |reservation| + reservation.instances.flat_map do |instance| + # EC2-Classic instances have a `public_ip_address` and no `network_interfaces` + # EC2-VPC instances both set, so we uniq the ip addresses + ip_addresses = [instance.public_ip_address].compact + instance.network_interfaces + .flat_map do |interface| + public_ip = interface.association ? [interface.association.public_ip].compact : [] + public_ip + interface.ipv_6_addresses.map(&:ipv_6_address) + end - # Don't return an entry if all ips were private - next [] if ip_addresses.empty? + # Don't return an entry if all ips were private + next [] if ip_addresses.empty? - # If hostname is empty string, canonicalize to nil - hostname = instance.public_dns_name.empty? ? nil : instance.public_dns_name - { - id: instance.instance_id, - hostname: hostname, - ip_addresses: ip_addresses.uniq - } + # If hostname is empty string, canonicalize to nil + hostname = instance.public_dns_name.empty? ? nil : instance.public_dns_name + result = { + region: client.config.region, + id: instance.instance_id, + hostname: hostname, + ip_addresses: ip_addresses.uniq + } + ::AwsPublicIps::Utils.add_tags(result, instance, options[:tags]) + result + end end end end diff --git a/lib/aws_public_ips/checks/elasticsearch.rb b/lib/aws_public_ips/checks/elasticsearch.rb index f962ff0..13b5ee2 100644 --- a/lib/aws_public_ips/checks/elasticsearch.rb +++ b/lib/aws_public_ips/checks/elasticsearch.rb @@ -6,28 +6,28 @@ module AwsPublicIps module Checks module Elasticsearch - def self.run - client = ::Aws::ElasticsearchService::Client.new - return [] unless ::AwsPublicIps::Utils.has_service?(client) - + def self.run(options) # ElasticSearch instances can be launched into classic into VPCs. Classic instances are public and have a # `domain_status.endpoint` hostname, and VPC instances have a `domain_status.endpoints['vpc']` hostname. # However VPC ElasticSearch instances create their own Network Interface and AWS will not allow you # to associate an Elastic IP to it. As a result VPC ElasticSearch instances are always private, even with an # internet gateway. + ::AwsPublicIps::Utils.probe(::Aws::ElasticsearchService::Client, + options[:regions], options[:progress]) do |client| + client.list_domain_names.flat_map do |response| + response.domain_names.flat_map do |domain_name| + client.describe_elasticsearch_domain(domain_name: domain_name.domain_name).map do |domain| + hostname = domain.domain_status.endpoint + next unless hostname - client.list_domain_names.flat_map do |response| - response.domain_names.flat_map do |domain_name| - client.describe_elasticsearch_domain(domain_name: domain_name.domain_name).map do |domain| - hostname = domain.domain_status.endpoint - next unless hostname - - { - id: domain.domain_status.domain_id, - hostname: hostname, - ip_addresses: ::AwsPublicIps::Utils.resolve_hostname(hostname) - } - end.compact + { + region: client.config.region, + id: domain.domain_status.domain_id, + hostname: hostname, + ip_addresses: ::AwsPublicIps::Utils.resolve_hostname(hostname) + } + end.compact + end end end end diff --git a/lib/aws_public_ips/checks/elb.rb b/lib/aws_public_ips/checks/elb.rb index 2218c0a..ff3188b 100644 --- a/lib/aws_public_ips/checks/elb.rb +++ b/lib/aws_public_ips/checks/elb.rb @@ -6,23 +6,24 @@ module AwsPublicIps module Checks module Elb - def self.run - client = ::Aws::ElasticLoadBalancing::Client.new - return [] unless ::AwsPublicIps::Utils.has_service?(client) - + def self.run(options) # EC2-Classic load balancers are only returned by the 'elasticloadbalancing' API, and # EC2-VPC ALBs/NLBs are only returned by the 'elasticloadbalancingv2' API - client.describe_load_balancers.flat_map do |response| - response.load_balancer_descriptions.flat_map do |load_balancer| - next [] unless load_balancer.scheme == 'internet-facing' + ::AwsPublicIps::Utils.probe(::Aws::ElasticLoadBalancing::Client, + options[:regions], options[:progress]) do |client| + client.describe_load_balancers.flat_map do |response| + response.load_balancer_descriptions.flat_map do |load_balancer| + next [] unless load_balancer.scheme == 'internet-facing' - # EC2-Classic load balancers get IPv6 DNS records created but they are not returned by the API - hostnames = [load_balancer.dns_name, "ipv6.#{load_balancer.dns_name}"] - { - id: load_balancer.canonical_hosted_zone_name_id, - hostname: load_balancer.dns_name, - ip_addresses: ::AwsPublicIps::Utils.resolve_hostnames(hostnames) - } + # EC2-Classic load balancers get IPv6 DNS records created but they are not returned by the API + hostnames = [load_balancer.dns_name, "ipv6.#{load_balancer.dns_name}"] + { + region: client.config.region, + id: load_balancer.canonical_hosted_zone_name_id, + hostname: load_balancer.dns_name, + ip_addresses: ::AwsPublicIps::Utils.resolve_hostnames(hostnames) + } + end end end end diff --git a/lib/aws_public_ips/checks/elbv2.rb b/lib/aws_public_ips/checks/elbv2.rb index 459ed72..1846f04 100644 --- a/lib/aws_public_ips/checks/elbv2.rb +++ b/lib/aws_public_ips/checks/elbv2.rb @@ -6,25 +6,26 @@ module AwsPublicIps module Checks module Elbv2 - def self.run - client = ::Aws::ElasticLoadBalancingV2::Client.new - return [] unless ::AwsPublicIps::Utils.has_service?(client) - + def self.run(options) # EC2-Classic load balancers are only returned by the 'elasticloadbalancing' API, and # EC2-VPC ALBs/NLBs are only returned by the 'elasticloadbalancingv2' API # NLBs only support IPv4 # ALBs support IPv4 or dualstack. Unlike Classic ELBs which have a separate IPv6 DNS name, # dualstack ALBs only have a single DNS name - client.describe_load_balancers.flat_map do |response| - response.load_balancers.flat_map do |load_balancer| - next [] unless load_balancer.scheme == 'internet-facing' + ::AwsPublicIps::Utils.probe(::Aws::ElasticLoadBalancingV2::Client, + options[:regions], options[:progress]) do |client| + client.describe_load_balancers.flat_map do |response| + response.load_balancers.flat_map do |load_balancer| + next [] unless load_balancer.scheme == 'internet-facing' - { - id: load_balancer.canonical_hosted_zone_id, - hostname: load_balancer.dns_name, - ip_addresses: ::AwsPublicIps::Utils.resolve_hostname(load_balancer.dns_name) - } + { + region: client.config.region, + id: load_balancer.canonical_hosted_zone_id, + hostname: load_balancer.dns_name, + ip_addresses: ::AwsPublicIps::Utils.resolve_hostname(load_balancer.dns_name) + } + end end end end diff --git a/lib/aws_public_ips/checks/lightsail.rb b/lib/aws_public_ips/checks/lightsail.rb index aea8e14..843dce7 100644 --- a/lib/aws_public_ips/checks/lightsail.rb +++ b/lib/aws_public_ips/checks/lightsail.rb @@ -6,35 +6,35 @@ module AwsPublicIps module Checks module Lightsail - def self.run - client = ::Aws::Lightsail::Client.new - return [] unless ::AwsPublicIps::Utils.has_service?(client) - + def self.run(options) # Lightsail instances are always exposed directly, and can also be put behind a load balancer - - instances = client.get_instances.flat_map do |response| - response.instances.map do |instance| - { - # Names are unique - id: instance.name, - hostname: nil, - ip_addresses: [instance.public_ip_address] - } + ::AwsPublicIps::Utils.probe(::Aws::Lightsail::Client, options[:regions], options[:progress]) do |client| + instances = client.get_instances.flat_map do |response| + response.instances.map do |instance| + { + # Names are unique + region: client.config.region, + id: instance.name, + hostname: nil, + ip_addresses: [instance.public_ip_address] + } + end end - end - load_balancers = client.get_load_balancers.flat_map do |response| - response.load_balancers.map do |load_balancer| - { - # Names are unique - id: load_balancer.name, - hostname: load_balancer.dns_name, - ip_addresses: ::AwsPublicIps::Utils.resolve_hostname(load_balancer.dns_name) - } + load_balancers = client.get_load_balancers.flat_map do |response| + response.load_balancers.map do |load_balancer| + { + # Names are unique + region: client.config.region, + id: load_balancer.name, + hostname: load_balancer.dns_name, + ip_addresses: ::AwsPublicIps::Utils.resolve_hostname(load_balancer.dns_name) + } + end end - end - instances + load_balancers + instances + load_balancers + end end end end diff --git a/lib/aws_public_ips/checks/rds.rb b/lib/aws_public_ips/checks/rds.rb index 46f8694..950906e 100644 --- a/lib/aws_public_ips/checks/rds.rb +++ b/lib/aws_public_ips/checks/rds.rb @@ -6,28 +6,28 @@ module AwsPublicIps module Checks module Rds - def self.run - client = ::Aws::RDS::Client.new - return [] unless ::AwsPublicIps::Utils.has_service?(client) - + def self.run(options) # RDS instances can be launched into VPCs or into Classic mode. # In classic mode they are always public. # In VPC mode they can be marked as `publicly_accessible` or not - if they are then its VPC must have # an Internet Gateway attached, and the DNS endpoint will resolve to a public ip address. - client.describe_db_instances.flat_map do |response| - response.db_instances.flat_map do |instance| - next [] unless instance.publicly_accessible + ::AwsPublicIps::Utils.probe(::Aws::RDS::Client, options[:regions], options[:progress]) do |client| + client.describe_db_instances.flat_map do |response| + response.db_instances.flat_map do |instance| + next [] unless instance.publicly_accessible - if instance.endpoint.nil? - raise StandardError, "RDS DB '#{instance.dbi_resource_id}' has a nil endpoint. This likely" \ - ' means the DB is being brought up right now.' - end + if instance.endpoint.nil? + raise StandardError, "RDS DB '#{instance.dbi_resource_id}' has a nil endpoint. This likely" \ + ' means the DB is being brought up right now.' + end - { - id: instance.dbi_resource_id, - hostname: instance.endpoint.address, - ip_addresses: ::AwsPublicIps::Utils.resolve_hostname(instance.endpoint.address) - } + { + region: client.config.region, + id: instance.dbi_resource_id, + hostname: instance.endpoint.address, + ip_addresses: ::AwsPublicIps::Utils.resolve_hostname(instance.endpoint.address) + } + end end end end diff --git a/lib/aws_public_ips/checks/redshift.rb b/lib/aws_public_ips/checks/redshift.rb index b40b358..5080974 100644 --- a/lib/aws_public_ips/checks/redshift.rb +++ b/lib/aws_public_ips/checks/redshift.rb @@ -6,29 +6,29 @@ module AwsPublicIps module Checks module Redshift - def self.run - client = ::Aws::Redshift::Client.new - return [] unless ::AwsPublicIps::Utils.has_service?(client) - + def self.run(options) # Redshift clusters can be launched into VPCs or into Classic mode. # In classic mode they are always public. # In VPC mode they can be marked as `publicly_accessible` or not - if they are then its VPC must have # an Internet Gateway attached, and the DNS endpoint will resolve to a public ip address. - client.describe_clusters.flat_map do |response| - response.clusters.flat_map do |cluster| - next [] unless cluster.publicly_accessible + ::AwsPublicIps::Utils.probe(::Aws::Redshift::Client, options[:regions], options[:progress]) do |client| + client.describe_clusters.flat_map do |response| + response.clusters.flat_map do |cluster| + next [] unless cluster.publicly_accessible - if cluster.endpoint.nil? - raise StandardError, "Redshift cluster '#{cluster.cluster_identifier}' has a nil endpoint. This likely" \ - ' means the cluster is being brought up right now.' - end + if cluster.endpoint.nil? + raise StandardError, "Redshift cluster '#{cluster.cluster_identifier}' has a nil endpoint. This" \ + ' likely means the cluster is being brought up right now.' + end - { - id: cluster.cluster_identifier, - hostname: cluster.endpoint.address, - ip_addresses: cluster.cluster_nodes.map(&:public_ip_address) + - ::AwsPublicIps::Utils.resolve_hostname(cluster.endpoint.address) - } + { + region: client.config.region, + id: cluster.cluster_identifier, + hostname: cluster.endpoint.address, + ip_addresses: cluster.cluster_nodes.map(&:public_ip_address) + + ::AwsPublicIps::Utils.resolve_hostname(cluster.endpoint.address) + } + end end end end diff --git a/lib/aws_public_ips/cli.rb b/lib/aws_public_ips/cli.rb index 74fef76..b403ac5 100644 --- a/lib/aws_public_ips/cli.rb +++ b/lib/aws_public_ips/cli.rb @@ -1,101 +1,12 @@ # frozen_string_literal: true -require 'optparse' +require 'aws_public_ips/cli_options' module AwsPublicIps class CLI - # Supported services: - # EC2 (and as a result: ECS, EKS, Beanstalk, Fargate, Batch, & NAT Instances) - # ELB (Classic ELB) - # ELBv2 (ALB/NLB) - # RDS - # Redshift - # APIGateway - # CloudFront - # Lightsail - # ElasticSearch - - # Services that don't need to be supported: - # S3 - all s3 buckets resolve to the same ip addresses - # SQS - there's a single AWS-owned domain per region (i.e. sqs.us-east-1.amazonaws.com//) - # NAT Gateways - these do not allow ingress traffic - # ElastiCache - all elasticache instances are private. You can make one public by using a NAT instance with an EIP: - # https://docs.aws.amazon.com/AmazonElastiCache/latest/red-ug/accessing-elasticache.html#access-from-outside-aws - # but NAT instances are EC2 machines, so this will be caught by the EC2 check. - # DynamoDB - no public endpoints - # SNS - no public endpoints - # Elastic Transcoder - no public endpoints - # Athena - no public endpoints - - # Services that maybe have public endpoints / still need testing: - # fargate - # amazonmq - # directory service (AD) - # emr - # Directconnect - # Kinesis - # SES - # https://aws.amazon.com/products/ - # AWS Neptune (still in preview / not GA yet) - - def all_services - @all_services ||= ::Dir["#{__dir__}/checks/*.rb"].map { |path| ::File.basename(path, '.rb') }.sort - end - - def all_formats - @all_formats ||= ::Dir["#{__dir__}/formatters/*.rb"].map { |path| ::File.basename(path, '.rb') }.sort - end - - def parse(args) - options = { - format: 'text', - services: all_services, - verbose: false - } - - ::OptionParser.new do |parser| - parser.banner = 'Usage: aws_public_ips [options]' - - parser.on('-s', '--services ,,', Array, 'List of AWS services to check. Available services: ' \ - "#{all_services.join(',')}. Defaults to all.") do |services| - services.map(&:downcase!).uniq! - invalid_services = services - all_services - raise ::ArgumentError, "Invalid service(s): #{invalid_services.join(',')}" unless invalid_services.empty? - - options[:services] = services - end - - parser.on('-f', '--format ', String, 'Set output format. Available formats: ' \ - "#{all_formats.join(',')}. Defaults to text.") do |fmt| - unless all_formats.include?(fmt) - raise ::ArgumentError, "Invalid format '#{fmt}'. Valid formats are: #{all_formats.join(',')}" - end - - options[:format] = fmt - end - - parser.on('-v', '--[no-]verbose', 'Enable debug/trace output') do |verbose| - options[:verbose] = verbose - end - - parser.on_tail('--version', 'Print version') do - require 'aws_public_ips/version' - ::STDOUT.puts ::AwsPublicIps::VERSION - return nil # nil to avoid rubocop warning - end - - parser.on_tail('-h', '--help', 'Show this help message') do - ::STDOUT.puts parser - return nil # nil to avoid rubocop warning - end - end.parse(args) - - options - end - - def check_service(service) + def check_service(options, service) require "aws_public_ips/checks/#{service}.rb" - ::AwsPublicIps::Checks.const_get(service.capitalize).run + ::AwsPublicIps::Checks.const_get(service.capitalize).run(options) end def output(formatter, options, results) @@ -106,14 +17,23 @@ def output(formatter, options, results) end def run(args) - options = parse(args) + cli_options = ::AwsPublicIps::CLIOptions.new + parse_and_run(cli_options, args) + end + + def parse_and_run(cli_options, args) + options = cli_options.parse(args) return unless options results = options[:services].map do |service| - [service.to_sym, check_service(service)] + [service.to_sym, check_service(options, service)] end.to_h output(options[:format], options, results) + rescue OptionParser::InvalidOption, ArgumentError => ex + ::STDOUT.puts ex + ::STDOUT.puts cli_options.usage + ::Process.exit(1) rescue ::StandardError, ::Interrupt => ex ::STDERR.puts ex.inspect ::STDERR.puts ex.backtrace if options && options[:verbose] diff --git a/lib/aws_public_ips/cli_options.rb b/lib/aws_public_ips/cli_options.rb new file mode 100644 index 0000000..54e17c9 --- /dev/null +++ b/lib/aws_public_ips/cli_options.rb @@ -0,0 +1,138 @@ +# frozen_string_literal: true + +require 'aws-sdk-ec2' +require 'optparse' + +module AwsPublicIps + class CLIOptions + # Supported services: + # EC2 (and as a result: ECS, EKS, Beanstalk, Fargate, Batch, & NAT Instances) + # ELB (Classic ELB) + # ELBv2 (ALB/NLB) + # RDS + # Redshift + # APIGateway + # CloudFront + # Lightsail + # ElasticSearch + + # Services that don't need to be supported: + # S3 - all s3 buckets resolve to the same ip addresses + # SQS - there's a single AWS-owned domain per region (i.e. sqs.us-east-1.amazonaws.com//) + # NAT Gateways - these do not allow ingress traffic + # ElastiCache - all elasticache instances are private. You can make one public by using a NAT instance with an EIP: + # https://docs.aws.amazon.com/AmazonElastiCache/latest/red-ug/accessing-elasticache.html#access-from-outside-aws + # but NAT instances are EC2 machines, so this will be caught by the EC2 check. + # DynamoDB - no public endpoints + # SNS - no public endpoints + # Elastic Transcoder - no public endpoints + # Athena - no public endpoints + + # Services that maybe have public endpoints / still need testing: + # fargate + # amazonmq + # directory service (AD) + # emr + # Directconnect + # Kinesis + # SES + # https://aws.amazon.com/products/ + # AWS Neptune (still in preview / not GA yet) + def initialize + @options = { + format: 'text', + regions: default_regions, + services: all_services, + tags: nil, + verbose: false, + progress: false + } + @parser = ::OptionParser.new do |p| + p.banner = 'Usage: aws_public_ips [options]' + + p.on('-r', '--regions ,,', Array, 'List of AWS services to check. Defaults to ' \ + '$AWS_DEFAULT_REGION or $AWS_REGION.') do |regions| + regions.map(&:downcase!).uniq! + (regions << all_regions).flatten!.map(&:downcase!).uniq! if regions.reject! { |r| r == 'all' } + invalid_regions = regions - all_regions + raise ::ArgumentError, "Invalid region(s): #{invalid_regions.join(',')}" unless invalid_regions.empty? + + @options[:regions] = regions + end + + p.on('-s', '--services ,,', Array, 'List of AWS services to check. Available services: ' \ + "#{all_services.join(',')}. Defaults to all.") do |services| + services.map(&:downcase!).uniq! + invalid_services = services - all_services + raise ::ArgumentError, "Invalid service(s): #{invalid_services.join(',')}" unless invalid_services.empty? + + @options[:services] = services + end + + p.on('-t', '--include_tags ,,', Array, 'List of tags to include in the results') do |tags| + @options[:tags] = tags.map(&:downcase).uniq + end + + p.on('-f', '--format ', String, 'Set output format. Available formats: ' \ + "#{all_formats.join(',')}. Defaults to text.") do |fmt| + unless all_formats.include?(fmt) + raise ::ArgumentError, "Invalid format '#{fmt}'. Valid formats are: #{all_formats.join(',')}" + end + + @options[:format] = fmt + end + + p.on('-v', '--[no-]verbose', 'Enable debug/trace output') do |verbose| + @options[:verbose] = verbose + end + + p.on('-p', '--[no-]progress', 'Enable progress') do |progress| + @options[:progress] = progress + end + + p.on_tail('--version', 'Print version') do + require 'aws_public_ips/version' + ::STDOUT.puts ::AwsPublicIps::VERSION + exit 0 + end + + p.on_tail('-h', '--help', 'Show this help message') do + ::STDOUT.puts p + exit 0 + end + end + end + + def all_services + @all_services ||= ::Dir["#{__dir__}/checks/*.rb"].map { |path| ::File.basename(path, '.rb') }.sort + end + + def all_regions + @all_regions ||= ::Aws::EC2::Client.new(region: 'us-east-1') + .describe_regions.regions.flat_map(&:region_name).collect.sort + end + + def default_regions + return [::ENV['AWS_DEFAULT_REGION']] unless ::ENV['AWS_DEFAULT_REGION'].nil? || ::ENV['AWS_DEFAULT_REGION'].empty? + + return [::ENV['AWS_REGION']] unless ::ENV['AWS_REGION'].nil? || ::ENV['AWS_REGION'].empty? + + nil + end + + def all_formats + @all_formats ||= ::Dir["#{__dir__}/formatters/*.rb"].map { |path| ::File.basename(path, '.rb') }.sort + end + + def parse(args) + @parser.parse(args) + raise ::ArgumentError, 'missing option: You must specify a region or set AWS_REGION.' unless @options[:regions] + + @options + end + + def usage + @parser.to_s + end + end +end diff --git a/lib/aws_public_ips/utils.rb b/lib/aws_public_ips/utils.rb index 02d59ef..216dc8b 100644 --- a/lib/aws_public_ips/utils.rb +++ b/lib/aws_public_ips/utils.rb @@ -2,6 +2,7 @@ require 'resolv' require 'aws-partitions' +require 'tty-spinner' module AwsPublicIps module Utils @@ -25,10 +26,77 @@ def self.has_service?(client) region_partition = ::Aws::Partitions.partitions.find do |partition| partition.regions.map(&:name).include?(client.config.region) end - service_name = client.class.to_s.split('::')[-2] + service_name = service_name(client.class) aws_service = region_partition.services.find { |service| service.name == service_name } !aws_service.regionalized? || aws_service.regions.include?(client.config.region) end + + def self.add_tags(result, source, tags) + return unless tags + + tags.each do |tag| + tag_item = source.tags.find { |t| t.key.downcase == tag } + if tag_item + tag_key = tag_item.key.downcase + result[tag_key] = tag_item.value + end + end + end + + def self.probe(client_class, regions, progress, &block) + return [] unless regions && !regions.empty? + + if progress + service_name = service_name(client_class) + spinners = create_spinner(TTY::Spinner::Multi, service_name) + begin + child_spinners = regions.map { |r| [r, spinners.register("[:spinner] #{r}")] }.to_h + result = regions.flat_map do |region| + probe_region_with_progress(child_spinners[region], client_class, region, &block) + end + spinners.success + result + rescue StandardError + spinners.error + [] + end + else + regions.flat_map do |region| + probe_region(client_class, region, &block) + end + end + end + + def self.service_name(client_class) + client_class.to_s.split('::')[-2] + end + + def self.create_spinner(spinner_class, service_name) + spinner_class.new("[:spinner] Probing #{service_name}...", + format: :dots, + success_mark: '+', + errror_mark: 'x') + end + + def self.probe_region_with_progress(spinner, client_class, region, &block) + spinner.run do |s| + begin + result = probe_region(client_class, region, &block) + s.success + result + rescue StandardError + s.error + [] + end + end.value + end + + def self.probe_region(client_class, region, &block) + client = region ? client_class.new(region: region) : client_class.new + return [] unless ::AwsPublicIps::Utils.has_service?(client) + + block.call(client) + end end end diff --git a/lib/aws_public_ips/version.rb b/lib/aws_public_ips/version.rb index 094ec80..ef26d33 100644 --- a/lib/aws_public_ips/version.rb +++ b/lib/aws_public_ips/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module AwsPublicIps - VERSION = '1.0.6'.freeze + VERSION = '1.0.7'.freeze end diff --git a/spec/aws_public_ips/checks/apigateway_spec.rb b/spec/aws_public_ips/checks/apigateway_spec.rb index 7fe0193..97e11ca 100644 --- a/spec/aws_public_ips/checks/apigateway_spec.rb +++ b/spec/aws_public_ips/checks/apigateway_spec.rb @@ -10,13 +10,15 @@ 'tmtmok31nc.execute-api.us-east-1.amazonaws.com' => %w[54.0.0.3] ) - expect(subject.run).to eq([ + expect(run_check(subject)).to eq([ { + region: 'us-east-1', id: 'e83d6nij2j', hostname: 'e83d6nij2j.execute-api.us-east-1.amazonaws.com', ip_addresses: %w[54.0.0.1 54.0.0.2] }, { + region: 'us-east-1', id: 'tmtmok31nc', hostname: 'tmtmok31nc.execute-api.us-east-1.amazonaws.com', ip_addresses: %w[54.0.0.3] diff --git a/spec/aws_public_ips/checks/cloudfront_spec.rb b/spec/aws_public_ips/checks/cloudfront_spec.rb index 2ab1580..404a9fd 100644 --- a/spec/aws_public_ips/checks/cloudfront_spec.rb +++ b/spec/aws_public_ips/checks/cloudfront_spec.rb @@ -9,7 +9,7 @@ 'd22ycgwdruc4lt.cloudfront.net' => %w[54.0.0.1 54.0.0.2], 'd1k00qwg2uxphp.cloudfront.net' => %w[54.0.0.3] ) - expect(subject.run).to eq([ + expect(run_check(subject)).to eq([ { id: 'E1DABYDY46RHFK', hostname: 'd22ycgwdruc4lt.cloudfront.net', diff --git a/spec/aws_public_ips/checks/ec2_spec.rb b/spec/aws_public_ips/checks/ec2_spec.rb index ee73c12..66e8349 100644 --- a/spec/aws_public_ips/checks/ec2_spec.rb +++ b/spec/aws_public_ips/checks/ec2_spec.rb @@ -5,18 +5,21 @@ stub_request(:post, 'https://ec2.us-east-1.amazonaws.com') .to_return(body: ::IO.read('spec/fixtures/ec2.xml')) - expect(subject.run).to eq([ + expect(run_check(subject)).to eq([ { + region: 'us-east-1', id: 'i-0f22d0af796b3cf3a', hostname: 'ec2-54-234-208-236.compute-1.amazonaws.com', ip_addresses: %w[54.234.208.236] }, { + region: 'us-east-1', id: 'i-04ef469b441883eda', hostname: 'ec2-18-206-76-65.compute-1.amazonaws.com', ip_addresses: %w[18.206.76.65 2600:1f18:63e0:b400:f50c:59a7:a182:3717] }, { + region: 'us-east-1', id: 'i-03a7b3bc2b4c20742', hostname: nil, ip_addresses: %w[2600:1f18:60f3:eb00:1c6e:5184:8955:170c] @@ -27,6 +30,6 @@ it 'should not return an entry when there are no public ips' do stub_request(:post, 'https://ec2.us-east-1.amazonaws.com') .to_return(body: ::IO.read('spec/fixtures/ec2-private.xml')) - expect(subject.run).to eq([]) + expect(run_check(subject)).to eq([]) end end diff --git a/spec/aws_public_ips/checks/elasticsearch_spec.rb b/spec/aws_public_ips/checks/elasticsearch_spec.rb index 5f831be..bbb7b3e 100644 --- a/spec/aws_public_ips/checks/elasticsearch_spec.rb +++ b/spec/aws_public_ips/checks/elasticsearch_spec.rb @@ -13,8 +13,9 @@ 'search-classic-fd5cbkkjuuiudho2lrwmsjp6rm.us-east-1.es.amazonaws.com' => %w[54.0.0.1] ) - expect(subject.run).to eq([ + expect(run_check(subject)).to eq([ { + region: 'us-east-1', id: '154967844790/classic', hostname: 'search-classic-fd5cbkkjuuiudho2lrwmsjp6rm.us-east-1.es.amazonaws.com', ip_addresses: %w[54.0.0.1] diff --git a/spec/aws_public_ips/checks/elb_spec.rb b/spec/aws_public_ips/checks/elb_spec.rb index 209062d..48785aa 100644 --- a/spec/aws_public_ips/checks/elb_spec.rb +++ b/spec/aws_public_ips/checks/elb_spec.rb @@ -10,8 +10,9 @@ 'ipv6.classic-272004174.us-east-1.elb.amazonaws.com' => %w[2600:1f18:63e0:b401:b952:5715:d981:2776] ) - expect(subject.run).to eq([ + expect(run_check(subject)).to eq([ { + region: 'us-east-1', id: 'Z35SXDOTRQ7X7K', hostname: 'classic-272004174.us-east-1.elb.amazonaws.com', ip_addresses: %w[54.0.0.1 2600:1f18:63e0:b401:b952:5715:d981:2776] diff --git a/spec/aws_public_ips/checks/elbv2_spec.rb b/spec/aws_public_ips/checks/elbv2_spec.rb index bb664db..202e861 100644 --- a/spec/aws_public_ips/checks/elbv2_spec.rb +++ b/spec/aws_public_ips/checks/elbv2_spec.rb @@ -10,13 +10,15 @@ 'alb-vpc-228877692.us-east-1.elb.amazonaws.com' => %w[54.0.0.2 2600:1f18:63e0:b401:b952:5715:d981:2776] ) - expect(subject.run).to eq([ + expect(run_check(subject)).to eq([ { + region: 'us-east-1', id: 'Z26RNL4JYFTOTI', hostname: 'nlb-vpc-d243d0acc9151631.elb.us-east-1.amazonaws.com', ip_addresses: %w[54.0.0.1] }, { + region: 'us-east-1', id: 'Z35SXDOTRQ7X7K', hostname: 'alb-vpc-228877692.us-east-1.elb.amazonaws.com', ip_addresses: %w[54.0.0.2 2600:1f18:63e0:b401:b952:5715:d981:2776] diff --git a/spec/aws_public_ips/checks/lightsail_spec.rb b/spec/aws_public_ips/checks/lightsail_spec.rb index f3131e1..c733a95 100644 --- a/spec/aws_public_ips/checks/lightsail_spec.rb +++ b/spec/aws_public_ips/checks/lightsail_spec.rb @@ -10,13 +10,15 @@ 'ce551c6f952085b4126e4b523a100eda-232829524.us-east-1.elb.amazonaws.com' => %w[54.88.163.90 52.205.146.152] ) - expect(subject.run).to eq([ + expect(run_check(subject)).to eq([ { + region: 'us-east-1', id: 'Amazon_Linux-512MB-Virginia-1', hostname: nil, ip_addresses: %w[18.206.153.10] }, { + region: 'us-east-1', id: 'LoadBalancer-Virginia-1', hostname: 'ce551c6f952085b4126e4b523a100eda-232829524.us-east-1.elb.amazonaws.com', ip_addresses: %w[54.88.163.90 52.205.146.152] diff --git a/spec/aws_public_ips/checks/rds_spec.rb b/spec/aws_public_ips/checks/rds_spec.rb index a98d26c..bc8ec18 100644 --- a/spec/aws_public_ips/checks/rds_spec.rb +++ b/spec/aws_public_ips/checks/rds_spec.rb @@ -10,13 +10,15 @@ 'rds-vpc-public-us-east-1a.cyvjlokb0o75.us-east-1.rds.amazonaws.com' => %w[52.70.34.110] ) - expect(subject.run).to eq([ + expect(run_check(subject)).to eq([ { + region: 'us-east-1', id: 'db-BAKGGXYRB3EBKBDQZTAMDSGCXY', hostname: 'rds-vpc-public.cyvjlokb0o75.us-east-1.rds.amazonaws.com', ip_addresses: %w[35.171.145.174] }, { + region: 'us-east-1', id: 'db-B3N4GULDAFGDKEGXBD7CXKZQV4', hostname: 'rds-vpc-public-us-east-1a.cyvjlokb0o75.us-east-1.rds.amazonaws.com', ip_addresses: %w[52.70.34.110] @@ -28,13 +30,13 @@ stub_request(:post, 'https://rds.us-east-1.amazonaws.com') .to_return(body: ::IO.read('spec/fixtures/rds-vpc-private.xml')) - expect(subject.run).to eq([]) + expect(run_check(subject)).to eq([]) end it 'should handle db instances with a nil endpoint' do stub_request(:post, 'https://rds.us-east-1.amazonaws.com') .to_return(body: ::IO.read('spec/fixtures/rds-empty-endpoint.xml')) - expect { subject.run }.to raise_error(StandardError, /has a nil endpoint/) + expect { run_check(subject) }.to raise_error(StandardError, /has a nil endpoint/) end end diff --git a/spec/aws_public_ips/checks/redshift_spec.rb b/spec/aws_public_ips/checks/redshift_spec.rb index 4a2ec45..125071e 100644 --- a/spec/aws_public_ips/checks/redshift_spec.rb +++ b/spec/aws_public_ips/checks/redshift_spec.rb @@ -5,8 +5,9 @@ stub_request(:post, 'https://redshift.us-east-1.amazonaws.com') .to_return(body: ::IO.read('spec/fixtures/redshift-classic-public.xml')) - expect(subject.run).to eq([ + expect(run_check(subject)).to eq([ { + region: 'us-east-1', id: 'classic', hostname: 'classic.csorkyt5dk7h.us-east-1.redshift.amazonaws.com', ip_addresses: %w[54.167.97.240 54.91.252.196 54.242.227.110] @@ -21,8 +22,9 @@ stub_dns( 'vpc-public-2.csorkyt5dk7h.us-east-1.redshift.amazonaws.com' => %w[54.156.7.130] ) - expect(subject.run).to eq([ + expect(run_check(subject)).to eq([ { + region: 'us-east-1', id: 'vpc-public-2', hostname: 'vpc-public-2.csorkyt5dk7h.us-east-1.redshift.amazonaws.com', ip_addresses: %w[52.44.170.10 54.209.53.206 54.208.75.129 54.156.7.130] @@ -34,12 +36,12 @@ stub_request(:post, 'https://redshift.us-east-1.amazonaws.com') .to_return(body: ::IO.read('spec/fixtures/redshift-vpc-private.xml')) - expect(subject.run).to eq([]) + expect(run_check(subject)).to eq([]) end it 'should handle clusters with a nil endpoint' do stub_request(:post, 'https://redshift.us-east-1.amazonaws.com') .to_return(body: ::IO.read('spec/fixtures/redshift-empty-endpoint.xml')) - expect { subject.run }.to raise_error(StandardError, /has a nil endpoint/) + expect { run_check(subject) }.to raise_error(StandardError, /has a nil endpoint/) end end diff --git a/spec/aws_public_ips/cli_options_spec.rb b/spec/aws_public_ips/cli_options_spec.rb new file mode 100644 index 0000000..b26f7ec --- /dev/null +++ b/spec/aws_public_ips/cli_options_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +describe ::AwsPublicIps::CLIOptions do + it 'should parse the options' do + stub_describe_regions + options = subject.parse(%w[--format json --services ec2,elb,redshift --verbose]) + expect(options).to eq(format: 'json', progress: false, regions: ['us-east-1'], + services: %w[ec2 elb redshift], tags: nil, verbose: true) + end + + it 'should raise on an invalid formatter' do + stub_describe_regions + expect { subject.parse(%w[--format blah]) }.to raise_error(::ArgumentError, /Invalid format/) + end + + it 'should raise on an invalid service' do + stub_describe_regions + expect { subject.parse(%w[--service blah]) }.to raise_error(::ArgumentError, /Invalid service/) + end + + it 'should select the right directory' do + stub_describe_regions + ::Dir.chdir('/') do + options = subject.parse(%w[--service ec2 --format prettyjson]) + expect(options).to include(services: %w[ec2], format: 'prettyjson') + end + end +end diff --git a/spec/aws_public_ips/cli_spec.rb b/spec/aws_public_ips/cli_spec.rb index e250525..560360a 100644 --- a/spec/aws_public_ips/cli_spec.rb +++ b/spec/aws_public_ips/cli_spec.rb @@ -1,27 +1,8 @@ # frozen_string_literal: true describe ::AwsPublicIps::CLI do - it 'should parse the options' do - options = subject.parse(%w[--format json --services ec2,elb,redshift --verbose]) - expect(options).to eq(format: 'json', services: %w[ec2 elb redshift], verbose: true) - end - - it 'should raise on an invalid formatter' do - expect { subject.parse(%w[--format blah]) }.to raise_error(::ArgumentError, /Invalid format/) - end - - it 'should raise on an invalid service' do - expect { subject.parse(%w[--service blah]) }.to raise_error(::ArgumentError, /Invalid service/) - end - - it 'should select the right directory' do - ::Dir.chdir('/') do - options = subject.parse(%w[--service ec2 --format prettyjson]) - expect(options).to include(services: %w[ec2], format: 'prettyjson') - end - end - it 'should run' do + stub_describe_regions expect(::AwsPublicIps::Checks::Ec2).to receive(:run).and_return([{ id: 'i-0f22d0af796b3cf3a', hostname: 'ec2-54-234-208-236.compute-1.amazonaws.com', @@ -32,6 +13,7 @@ end it 'should rescue exceptions' do + stub_describe_regions expect(subject).to receive(:check_service).and_raise(::StandardError) expect(::Process).to receive(:exit).with(1) expect(::STDERR).to receive(:puts) @@ -39,12 +21,14 @@ end it 'should print the version' do + stub_describe_regions expect(::STDOUT).to receive(:puts).with(::AwsPublicIps::VERSION) - subject.run(['--version']) + expect { subject.run(['--version']) }.to raise_error(::SystemExit) end it 'should print help' do + stub_describe_regions expect(::STDOUT).to receive(:puts) - subject.run(['--help']) + expect { subject.run(['--help']) }.to raise_error(::SystemExit) end end diff --git a/spec/fixtures/regions.xml b/spec/fixtures/regions.xml new file mode 100644 index 0000000..2aba889 --- /dev/null +++ b/spec/fixtures/regions.xml @@ -0,0 +1,69 @@ + + 9b744a10-6c43-487c-9e4c-65a7a22683a4 + + + ap-south-1 + ec2.ap-south-1.amazonaws.com + + + eu-west-3 + ec2.eu-west-3.amazonaws.com + + + eu-north-1 + ec2.eu-north-1.amazonaws.com + + + eu-west-2 + ec2.eu-west-2.amazonaws.com + + + eu-west-1 + ec2.eu-west-1.amazonaws.com + + + ap-northeast-2 + ec2.ap-northeast-2.amazonaws.com + + + ap-northeast-1 + ec2.ap-northeast-1.amazonaws.com + + + sa-east-1 + ec2.sa-east-1.amazonaws.com + + + ca-central-1 + ec2.ca-central-1.amazonaws.com + + + ap-southeast-1 + ec2.ap-southeast-1.amazonaws.com + + + ap-southeast-2 + ec2.ap-southeast-2.amazonaws.com + + + eu-central-1 + ec2.eu-central-1.amazonaws.com + + + us-east-1 + ec2.us-east-1.amazonaws.com + + + us-east-2 + ec2.us-east-2.amazonaws.com + + + us-west-1 + ec2.us-west-1.amazonaws.com + + + us-west-2 + ec2.us-west-2.amazonaws.com + + + \ No newline at end of file diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 83a4df0..cb0b32f 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -4,6 +4,7 @@ ::Coveralls.wear! do |config| # Output html report for development & default json report for Travis/Coveralls config.formatter = SimpleCov::Formatter::HTMLFormatter unless ::ENV['TRAVIS'] + config.add_filter 'spec' end ::ENV['COVERALLS_NOISY'] = '1' require 'webmock/rspec' @@ -19,3 +20,13 @@ def stub_dns(mapping) expect(::AwsPublicIps::Utils).to receive(:resolve_hostname).with(hostname).and_return(ips) end end + +def stub_describe_regions + stub_request(:post, 'https://ec2.us-east-1.amazonaws.com/') + .with(body: /Action=DescribeRegions/) + .to_return(status: 200, body: '', headers: {}) +end + +def run_check(subject) + subject.run(regions: [::ENV['AWS_REGION']], progress: false) +end From dab70ed1f712e11fd4606c3fa0770d95b5c6db1b Mon Sep 17 00:00:00 2001 From: Joel Baranick Date: Wed, 30 Jan 2019 19:23:56 -0800 Subject: [PATCH 2/9] Update changelog --- CHANGELOG.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36b2571..fda655c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,18 @@ +### 1.0.7 +* Display progress while running by specifying `--progress` +* Include tag data in ec2 records by adding `--include_tags name` +* Multi-region support. Defaults to the region specified by AWS_DEFAULT_REGION or AWS_REGION. Many regions can be specified. `all` will include all regions. + ### 1.0.6 (10/26/2018) * Ignore services when they're not available in the requested region * Update dependencies ### 1.0.5 (05/25/2018) -* Output more details in text formatter when using --verbose +* Output more details in text formatter when using `--verbose` ### 1.0.4 (05/20/2018) * Handle RDS and Redshift instances which are in the process of coming up/down -* Add --help, --version commands to CLI +* Add `--help`, `--version` commands to CLI * Improve CLI error output * Fix issue where using the EC2 check with the JSON formatter could output empty entries * Fix issue with RDS check where the endpoint hostname was not being counted as a public IP From 73df73d996b48ed197edfd288ba131cbe7009785 Mon Sep 17 00:00:00 2001 From: Joel Baranick Date: Thu, 31 Jan 2019 10:15:26 -0800 Subject: [PATCH 3/9] Add tests for cli_options --- lib/aws_public_ips/cli_options.rb | 2 +- spec/aws_public_ips/cli_options_spec.rb | 27 ++++++++++ spec/fixtures/describe-regions.xml | 70 +++++++++++++++++++++++++ spec/spec_helper.rb | 2 +- 4 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 spec/fixtures/describe-regions.xml diff --git a/lib/aws_public_ips/cli_options.rb b/lib/aws_public_ips/cli_options.rb index 54e17c9..239ba02 100644 --- a/lib/aws_public_ips/cli_options.rb +++ b/lib/aws_public_ips/cli_options.rb @@ -51,7 +51,7 @@ def initialize p.banner = 'Usage: aws_public_ips [options]' p.on('-r', '--regions ,,', Array, 'List of AWS services to check. Defaults to ' \ - '$AWS_DEFAULT_REGION or $AWS_REGION.') do |regions| + '$AWS_DEFAULT_REGION or $AWS_REGION. Specify `all` in include all regions.') do |regions| regions.map(&:downcase!).uniq! (regions << all_regions).flatten!.map(&:downcase!).uniq! if regions.reject! { |r| r == 'all' } invalid_regions = regions - all_regions diff --git a/spec/aws_public_ips/cli_options_spec.rb b/spec/aws_public_ips/cli_options_spec.rb index b26f7ec..69edcd2 100644 --- a/spec/aws_public_ips/cli_options_spec.rb +++ b/spec/aws_public_ips/cli_options_spec.rb @@ -25,4 +25,31 @@ expect(options).to include(services: %w[ec2], format: 'prettyjson') end end + + it 'should raise on an invalid region' do + stub_describe_regions + expect { subject.parse(%w[--region blah]) }.to raise_error(::ArgumentError, /Invalid region/) + end + + it 'should allow all regions to be specified' do + stub_describe_regions + options = subject.parse(%w[--region all]) + expect(options).to satisfy { |o| o[:regions].length == 16 } + end + + it 'should allow tags to be included' do + stub_describe_regions + options = subject.parse(%w[--include_tags name,service,name]) + expect(options[:tags]).to contain_exactly('name', 'service') + end + + it 'should allow progress to be enabled' do + stub_describe_regions + options = subject.parse(%w[--progress]) + expect(options[:progress]).to be_truthy + end + + it 'should be able to print usage' do + expect(subject.usage).to include('Usage: aws_public_ips') + end end diff --git a/spec/fixtures/describe-regions.xml b/spec/fixtures/describe-regions.xml new file mode 100644 index 0000000..3cd25ca --- /dev/null +++ b/spec/fixtures/describe-regions.xml @@ -0,0 +1,70 @@ + + + fcc21266-e812-4eb3-978e-c079faa443dc + + + ap-south-1 + ec2.ap-south-1.amazonaws.com + + + eu-west-3 + ec2.eu-west-3.amazonaws.com + + + eu-north-1 + ec2.eu-north-1.amazonaws.com + + + eu-west-2 + ec2.eu-west-2.amazonaws.com + + + eu-west-1 + ec2.eu-west-1.amazonaws.com + + + ap-northeast-2 + ec2.ap-northeast-2.amazonaws.com + + + ap-northeast-1 + ec2.ap-northeast-1.amazonaws.com + + + sa-east-1 + ec2.sa-east-1.amazonaws.com + + + ca-central-1 + ec2.ca-central-1.amazonaws.com + + + ap-southeast-1 + ec2.ap-southeast-1.amazonaws.com + + + ap-southeast-2 + ec2.ap-southeast-2.amazonaws.com + + + eu-central-1 + ec2.eu-central-1.amazonaws.com + + + us-east-1 + ec2.us-east-1.amazonaws.com + + + us-east-2 + ec2.us-east-2.amazonaws.com + + + us-west-1 + ec2.us-west-1.amazonaws.com + + + us-west-2 + ec2.us-west-2.amazonaws.com + + + \ No newline at end of file diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index cb0b32f..d694aec 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -24,7 +24,7 @@ def stub_dns(mapping) def stub_describe_regions stub_request(:post, 'https://ec2.us-east-1.amazonaws.com/') .with(body: /Action=DescribeRegions/) - .to_return(status: 200, body: '', headers: {}) + .to_return(status: 200, body: ::IO.read('spec/fixtures/describe-regions.xml')) end def run_check(subject) From ba2303ac175a2a69b6139aecf819b37758a81725 Mon Sep 17 00:00:00 2001 From: Joel Baranick Date: Thu, 31 Jan 2019 11:29:01 -0800 Subject: [PATCH 4/9] Add cli tests --- lib/aws_public_ips/cli.rb | 4 ++-- spec/aws_public_ips/cli_spec.rb | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/aws_public_ips/cli.rb b/lib/aws_public_ips/cli.rb index b403ac5..2d055de 100644 --- a/lib/aws_public_ips/cli.rb +++ b/lib/aws_public_ips/cli.rb @@ -31,8 +31,8 @@ def parse_and_run(cli_options, args) output(options[:format], options, results) rescue OptionParser::InvalidOption, ArgumentError => ex - ::STDOUT.puts ex - ::STDOUT.puts cli_options.usage + ::STDERR.puts ex + ::STDERR.puts cli_options.usage ::Process.exit(1) rescue ::StandardError, ::Interrupt => ex ::STDERR.puts ex.inspect diff --git a/spec/aws_public_ips/cli_spec.rb b/spec/aws_public_ips/cli_spec.rb index 560360a..136f7bd 100644 --- a/spec/aws_public_ips/cli_spec.rb +++ b/spec/aws_public_ips/cli_spec.rb @@ -31,4 +31,15 @@ expect(::STDOUT).to receive(:puts) expect { subject.run(['--help']) }.to raise_error(::SystemExit) end + + it 'should print help on option exceptions' do + begin + stub_describe_regions + expectation = expect { subject.run(['--format', 'blah']) } + expectation.to output(/Invalid format/).to_stderr_from_any_process + expectation.to output(/Usage: /).to_stderr_from_any_process + rescue SystemExit => e + expect(e.status).to eq(1) + end + end end From 2cdc6c98f5d3aa9edd182848a2e17c3760fe5352 Mon Sep 17 00:00:00 2001 From: Joel Baranick Date: Thu, 31 Jan 2019 11:45:23 -0800 Subject: [PATCH 5/9] Add ec2 tags testing --- lib/aws_public_ips/utils.rb | 4 ++-- spec/aws_public_ips/checks/ec2_spec.rb | 2 ++ spec/fixtures/ec2.xml | 12 ++++++++++++ spec/spec_helper.rb | 2 +- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/lib/aws_public_ips/utils.rb b/lib/aws_public_ips/utils.rb index 216dc8b..8b43be4 100644 --- a/lib/aws_public_ips/utils.rb +++ b/lib/aws_public_ips/utils.rb @@ -33,13 +33,13 @@ def self.has_service?(client) end def self.add_tags(result, source, tags) - return unless tags + return unless tags && source.tags tags.each do |tag| tag_item = source.tags.find { |t| t.key.downcase == tag } if tag_item tag_key = tag_item.key.downcase - result[tag_key] = tag_item.value + result[tag_key.to_sym] = tag_item.value end end end diff --git a/spec/aws_public_ips/checks/ec2_spec.rb b/spec/aws_public_ips/checks/ec2_spec.rb index 66e8349..63382a6 100644 --- a/spec/aws_public_ips/checks/ec2_spec.rb +++ b/spec/aws_public_ips/checks/ec2_spec.rb @@ -10,12 +10,14 @@ region: 'us-east-1', id: 'i-0f22d0af796b3cf3a', hostname: 'ec2-54-234-208-236.compute-1.amazonaws.com', + name: 'foo', ip_addresses: %w[54.234.208.236] }, { region: 'us-east-1', id: 'i-04ef469b441883eda', hostname: 'ec2-18-206-76-65.compute-1.amazonaws.com', + name: 'bar', ip_addresses: %w[18.206.76.65 2600:1f18:63e0:b400:f50c:59a7:a182:3717] }, { diff --git a/spec/fixtures/ec2.xml b/spec/fixtures/ec2.xml index 3215224..c3f6d9c 100644 --- a/spec/fixtures/ec2.xml +++ b/spec/fixtures/ec2.xml @@ -58,6 +58,12 @@ hvm + + + Name + foo + + xen false @@ -124,6 +130,12 @@ hvm + + + name + bar + + xen diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index d694aec..a62e847 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -28,5 +28,5 @@ def stub_describe_regions end def run_check(subject) - subject.run(regions: [::ENV['AWS_REGION']], progress: false) + subject.run(regions: [::ENV['AWS_REGION']], tags: ['name'], progress: false) end From c6f5271c750257759f5f35fad0f1ca98a1f2fce4 Mon Sep 17 00:00:00 2001 From: Joel Baranick Date: Thu, 31 Jan 2019 12:24:22 -0800 Subject: [PATCH 6/9] Add basic progress testing --- spec/aws_public_ips/cli_spec.rb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/spec/aws_public_ips/cli_spec.rb b/spec/aws_public_ips/cli_spec.rb index 136f7bd..993ebc2 100644 --- a/spec/aws_public_ips/cli_spec.rb +++ b/spec/aws_public_ips/cli_spec.rb @@ -12,6 +12,15 @@ subject.run(['-s', 'ec2']) end + it 'should run with progress' do + stub_describe_regions + stub_request(:post, 'https://ec2.us-east-1.amazonaws.com') + .to_return(body: ::IO.read('spec/fixtures/ec2.xml')) + expect(::STDOUT).to receive(:puts) + expect(::STDERR).to receive(:print).at_least(4).times + subject.run(['-p', '-s', 'ec2']) + end + it 'should rescue exceptions' do stub_describe_regions expect(subject).to receive(:check_service).and_raise(::StandardError) From fa70624bd7f2999384ec1904a46ab93274e1a89e Mon Sep 17 00:00:00 2001 From: Joel Baranick Date: Thu, 31 Jan 2019 12:28:44 -0800 Subject: [PATCH 7/9] Fix rubocop error --- spec/aws_public_ips/cli_options_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/aws_public_ips/cli_options_spec.rb b/spec/aws_public_ips/cli_options_spec.rb index 69edcd2..55cc2dd 100644 --- a/spec/aws_public_ips/cli_options_spec.rb +++ b/spec/aws_public_ips/cli_options_spec.rb @@ -34,7 +34,7 @@ it 'should allow all regions to be specified' do stub_describe_regions options = subject.parse(%w[--region all]) - expect(options).to satisfy { |o| o[:regions].length == 16 } + expect(options).to(satisfy { |o| o[:regions].length == 16 }) end it 'should allow tags to be included' do From 9a11a84b6307a8278df838fee3425941271308e2 Mon Sep 17 00:00:00 2001 From: Joel Baranick Date: Thu, 31 Jan 2019 12:51:38 -0800 Subject: [PATCH 8/9] Remove unneeded file --- spec/fixtures/regions.xml | 69 --------------------------------------- 1 file changed, 69 deletions(-) delete mode 100644 spec/fixtures/regions.xml diff --git a/spec/fixtures/regions.xml b/spec/fixtures/regions.xml deleted file mode 100644 index 2aba889..0000000 --- a/spec/fixtures/regions.xml +++ /dev/null @@ -1,69 +0,0 @@ - - 9b744a10-6c43-487c-9e4c-65a7a22683a4 - - - ap-south-1 - ec2.ap-south-1.amazonaws.com - - - eu-west-3 - ec2.eu-west-3.amazonaws.com - - - eu-north-1 - ec2.eu-north-1.amazonaws.com - - - eu-west-2 - ec2.eu-west-2.amazonaws.com - - - eu-west-1 - ec2.eu-west-1.amazonaws.com - - - ap-northeast-2 - ec2.ap-northeast-2.amazonaws.com - - - ap-northeast-1 - ec2.ap-northeast-1.amazonaws.com - - - sa-east-1 - ec2.sa-east-1.amazonaws.com - - - ca-central-1 - ec2.ca-central-1.amazonaws.com - - - ap-southeast-1 - ec2.ap-southeast-1.amazonaws.com - - - ap-southeast-2 - ec2.ap-southeast-2.amazonaws.com - - - eu-central-1 - ec2.eu-central-1.amazonaws.com - - - us-east-1 - ec2.us-east-1.amazonaws.com - - - us-east-2 - ec2.us-east-2.amazonaws.com - - - us-west-1 - ec2.us-west-1.amazonaws.com - - - us-west-2 - ec2.us-west-2.amazonaws.com - - - \ No newline at end of file From 8cfebe91ce5cf19a5c3321ea2e1f6625a5659394 Mon Sep 17 00:00:00 2001 From: Joel Baranick Date: Thu, 31 Jan 2019 12:52:25 -0800 Subject: [PATCH 9/9] Upgrade rubocop 0.63.1 because 0.60.0 fails with `unknown keywords: whitelist_classes, whitelist_symbols` on 2.7.0-dev --- aws_public_ips.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_public_ips.gemspec b/aws_public_ips.gemspec index 5a80de2..f8b29ae 100644 --- a/aws_public_ips.gemspec +++ b/aws_public_ips.gemspec @@ -31,6 +31,6 @@ require 'aws_public_ips/version' gem.add_development_dependency('bundler-audit', '~> 0.6.0') gem.add_development_dependency('coveralls', '~> 0.8.22') gem.add_development_dependency('rspec', '~> 3.8.0') - gem.add_development_dependency('rubocop', '~> 0.60.0') + gem.add_development_dependency('rubocop', '~> 0.63.1') gem.add_development_dependency('webmock', '~> 3.4.2') end