Merge pull request #3 from amaabca/handle-many-lambdas

Handle multiple pages of function results

push/loggly/client.rb

@@ -28,12 +28,13 @@ def push!(messages)
 
       def push_single!(messages)
         messages.map do |message|
-          request!(message)
+          body = message.delete("\n")
+          request!(body)
         end
       end
 
       def push_bulk!(messages)
-        body = messages.map(&:chomp).join("\n")
+        body = messages.map { |message| message.delete("\n") }.join("\n")
         request!(body)
       end
 

spec/push/loggly/client_spec.rb

@@ -11,7 +11,7 @@
     let(:messages) do
       [
         "Message one\n",
-        "Message two\n"
+        "Message two\t{ json: \n { key: 'value',\n many: 'lines' } }\n"
       ]
     end
 

spec/subscribe/lambda/function_spec.rb

@@ -7,6 +7,7 @@
   let(:tags_stub) { lambda.stub_data(:list_tags, tags: tags) }
   let(:function_one_arn) { 'arn:aws:lambda:us-west-2:123456789012:function:One' }
   let(:function_two_arn) { 'arn:aws:lambda:us-west-2:123456789012:function:Two' }
+  let(:function_three_arn) { 'arn:aws:lambda:us-west-2:123456789012:function:Three' }
   let(:functions) do
     [
       {
@@ -19,6 +20,14 @@
       }
     ]
   end
+  let(:more_functions) do
+    [
+      {
+        function_name: 'three',
+        function_arn: function_three_arn
+      }
+    ]
+  end
   let(:functions_stub) { lambda.stub_data(:list_functions, functions: functions) }
   subject do
     described_class.new(
@@ -118,15 +127,31 @@
   end
 
   describe '.all' do
-    before(:each) do
-      lambda.stub_responses(:list_functions, functions_stub)
-      lambda.stub_responses(:list_tags, tags_stub)
+    context 'where there is 1 page of results' do
+      before(:each) do
+        lambda.stub_responses(:list_functions, functions_stub)
+      end
+
+      it 'returns an array of Subscribe::Lambda::Function instances' do
+        data = described_class.all(lambda)
+        expect(data.size).to eq(2)
+        expect(data.first).to be_a(described_class)
+      end
     end
 
-    it 'returns an array of Subscribe::Lambda::Function instances' do
-      data = described_class.all(lambda)
-      expect(data.size).to eq(2)
-      expect(data.first).to be_a(described_class)
+    context 'where there are 2 pages of results' do
+      let(:functions_stub) { lambda.stub_data(:list_functions, functions: functions, next_marker: 'abcdefg') }
+      let(:more_functions_stub) { lambda.stub_data(:list_functions, functions: more_functions) }
+
+      before(:each) do
+        lambda.stub_responses(:list_functions, [functions_stub, more_functions_stub])
+      end
+
+      it 'returns an array of Subscribe::Lambda::Function instances' do
+        data = described_class.all(lambda)
+        expect(data.size).to eq(3)
+        expect(data.last).to be_a(described_class)
+      end
     end
   end
 

spec/support/helpers/requests.rb

@@ -7,13 +7,15 @@ def stub_loggly_push_initial
     end
 
     def stub_loggly_push(opts = {})
+      message = opts.fetch(:message)
+      request_body = message.delete("\n")
       stub_loggly_push_initial
         .with(
           headers: {
             'X-LOGGLY-TAG' => opts.fetch(:tags),
             'CONTENT-TYPE' => 'text/plain'
           },
-          body: opts.fetch(:message)
+          body: request_body
         )
         .to_return(
           status: opts.fetch(:status) { 200 },
@@ -23,7 +25,7 @@ def stub_loggly_push(opts = {})
 
     def stub_loggly_push_bulk(opts = {})
       messages = opts.fetch(:messages)
-      request_body = messages.map(&:chomp).join("\n")
+      request_body = messages.map { |message| message.delete("\n") }.join("\n")
       stub_request(:post, loggly_url_for('bulk/1234/'))
         .with(
           headers: { 'CONTENT-TYPE' => 'text/plain' },

subscribe/lambda/function.rb

@@ -7,7 +7,7 @@ class Function
 
       class << self
         def all(lambda = Aws::Lambda::Client.new, cloudwatch = Aws::CloudWatchLogs::Client.new)
-          lambda.list_functions.functions.map do |function|
+          all_functions(lambda).map do |function|
             new(
               name: function.function_name,
               arn: function.function_arn,
@@ -33,6 +33,12 @@ def subscribe_all!(lambda = Aws::Lambda::Client.new, cloudwatch = Aws::CloudWatc
             end
           end
         end
+
+        private
+
+        def all_functions(lambda)
+          lambda.list_functions.flat_map(&:functions)
+        end
       end
 
       def initialize(opts = {})
@@ -59,19 +65,15 @@ def subscribe!
       private
 
       def up_to_date?
-        !stale?
+        filter_pattern == remote_filter_pattern
+      rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
+        true
       end
 
       def suppress?
         !!tags['cloudwatch_loggly_suppress_subscribe']
       end
 
-      def stale?
-        filter_pattern != remote_filter_pattern
-      rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
-        false
-      end
-
       def destination_arn
         ENV.fetch('DESTINATION_ARN')
       end

template.yml

@@ -19,7 +19,7 @@ Metadata:
       - forwarder
       - lambda
     HomePageUrl: https://github.com/amaabca/cloudwatch_loggly
-    SemanticVersion: 0.1.0
+    SemanticVersion: 0.2.0
     SourceCodeUrl: https://github.com/amaabca/cloudwatch_loggly
 
 Parameters: