-
Notifications
You must be signed in to change notification settings - Fork 2
/
kafka.go
90 lines (79 loc) · 1.69 KB
/
kafka.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
package kafka
import (
"errors"
"strings"
"github.com/Shopify/sarama"
"github.com/mozilla-services/heka/message"
. "github.com/mozilla-services/heka/pipeline"
)
type KafkaOutputConfig struct {
Address string
Id string
Topic string
}
type KafkaOutput struct {
config *KafkaOutputConfig
addrs []string
client *sarama.Client
producer *sarama.Producer
}
func (ao *KafkaOutput) ConfigStruct() interface{} {
return &KafkaOutputConfig{}
}
func (ao *KafkaOutput) Init(config interface{}) (err error) {
ao.config = config.(*KafkaOutputConfig)
ao.addrs = strings.Split(ao.config.Address, ",")
if len(ao.addrs) == 1 && len(ao.addrs) == 0 {
err = errors.New("invalid address")
}
err = ao.init()
return
}
func (ao *KafkaOutput) Run(or OutputRunner, h PluginHelper) (err error) {
inChan := or.InChan()
errChan := ao.producer.Errors()
var pack *PipelinePack
var msg *message.Message
ok := true
for ok {
select {
case pack, ok = <-inChan:
if !ok {
break
}
msg = pack.Message
err = ao.producer.QueueMessage(ao.config.Topic, nil, sarama.ByteEncoder(msg.GetPayload()))
if err != nil {
pack.Recycle()
or.LogError(err)
break
}
case err = <-errChan:
break
}
}
return
}
func (ao *KafkaOutput) CleanupForRestart() {
ao.client.Close()
ao.producer.Close()
ao.init()
}
func (ao *KafkaOutput) init() (err error) {
cconf := sarama.NewClientConfig()
ao.client, err = sarama.NewClient(ao.config.Id, ao.addrs, cconf)
if err != nil {
return
}
kconf := sarama.NewProducerConfig()
ao.producer, err = sarama.NewProducer(ao.client, kconf)
if err != nil {
return
}
return
}
func init() {
RegisterPlugin("KafkaOutput", func() interface{} {
return new(KafkaOutput)
})
}