-
Notifications
You must be signed in to change notification settings - Fork 0
/
12-apache-filter.conf
93 lines (71 loc) · 5.16 KB
/
12-apache-filter.conf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
filter {
if "apache" in [tags] or "pykeystone" in [source] {
mutate {
add_tag => [ "in_12" ]
}
### HEADER PROCESSING
grok {
#SYSLOGBASE %{SYSLOGTIMESTAMP:apache_timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
patterns_dir => [ "/etc/logstash/patterns" ]
#match => { "message" => "%{TIMESTAMP_ISO8601:apache_timestamp}%{SPACE}%{HOSTNAME:server}%{SPACE}%{DATA:program}(\[%{POSINT:pid}\]):%{SPACE}\[%{LOGLEVEL:loglevel}\]%{SPACE}%{GREEDYDATA:content}" }
match => { "message" => "%{TIMESTAMP_ISO8601:apache_timestamp} %{HOSTNAME:server} %{DATA:program}(\[%{POSINT:pid}\]): \[%{LOGLEVEL:loglevel}\] %{GREEDYDATA:content}" }
match => { "message" => "%{TIMESTAMP_ISO8601:apache_timestamp} %{HOSTNAME:server} httpd: %{IP:client_ip} - - \[%{HTTPDATE:timestamp}\] \"%{WORD:verb} %{NOTSPACE} %{GREEDYDATA:content}" }
match => { "message" => "%{TIMESTAMP_ISO8601:apache_timestamp} %{HOSTNAME:server} httpd: %{IP:client_ip} \[%{HTTPDATE:timestamp}\] \"%{WORD:verb} %{NOTSPACE} %{GREEDYDATA:content}" }
add_tag => [ "apache_header" ]
#tag_on_failure => ["_gpf_apache_h_error" ]
remove_tag => [ "_grokparsefailure" ]
}
if "apache_header" in [tags] {
mutate {
replace => { message => "%{content}" }
#remove_field => [ "content" ]
}
}
### MAIN PROCESSING
grok {
patterns_dir => [ "/etc/logstash/patterns" ]
match => { "message" => "%{TIMESTAMP_ISO8601}%{SPACE}%{POSINT:pid}%{SPACE}%{AUDITLOGLEVEL:syslog_level}%{SPACE}%{WORD}%{SPACE}%{NOTSPACE}%{SPACE}%{IP:client_ip}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}\[%{HTTPDATE:apache_timestamp}\]%{SPACE}%{NOTSPACE}%{WORD:verb}%{SPACE}%{DATA:url} HTTP/%{NUMBER:httpversion}%{NOTSPACE}%{SPACE}%{POSINT:response}%{SPACE}%{NUMBER:msg_size}" }
match => { "message" => "%{TIMESTAMP_ISO8601:apache_timestamp}%{SPACE}%{POSINT:pid}%{SPACE}%{AUDITLOGLEVEL:loglevel}%{SPACE}%{DATA:module} \[%{DATA:request_id} %{DATA:data2} %{DATA:data3}\] %{WORD:verb} %{URI:url}" }
match => { "message" => "%{TIMESTAMP_ISO8601:apache_timestamp}%{SPACE}%{POSINT:pid}%{SPACE}%{AUDITLOGLEVEL:loglevel}%{SPACE}%{DATA:module} \[%{DATA:request_id} %{DATA:data2} %{DATA:data3}\] %{URI:url} returned with HTTP %{POSINT:return}" }
match => { "message" => "%{TIMESTAMP_ISO8601:apache_timestamp}%{SPACE}%{POSINT:pid}%{SPACE}%{AUDITLOGLEVEL:loglevel}%{SPACE}%{DATA:module} \[%{DATA:request_id} %{DATA:data2} %{DATA:data3}\] %{GREEDYDATA:more_content}" }
match => { "message" => "%{TIMESTAMP_ISO8601:apache_timestamp}%{SPACE}%{POSINT:pid}%{SPACE}%{AUDITLOGLEVEL:loglevel}%{SPACE} %{DATA:module} %{GREEDYDATA:more_content}" }
match => { "message" => "%{TIMESTAMP_ISO8601:apache_timestamp}%{SPACE}%{POSINT:pid}%{SPACE}%{AUDITLOGLEVEL:loglevel}%{SPACE}%{GREEDYDATA:more_content}" }
match => { "message" => "%{TIMESTAMP_ISO8601:apache_timestamp} %{HOSTNAME:server} %{DATA:program}\[%{POSINT:pid}\]: \[%{LOGLEVEL:loglevel}\]%{SPACE}%{GREEDYDATA:more_content}" }
match => { "message" => "%{TIMESTAMP_ISO8601:apache_timestamp} %{HOSTNAME:server} %{DATA:program}(\[%{POSINT:pid}\]):%{SPACE}\[%{LOGLEVEL:loglevel}\]%{SPACE}%{GREEDYDATA:more_content}" }
match => { "message" => "\[%{HTTPDATE:timestamp}\] %{IP:client_ip} %{DATA:user_cn} %{DATA:crypt_type} %{DATA:crypt_type} \"%{WORD:verb} %{DATA:uri} HTTP/%{NUMBER:httpversion}\" %{POSINT:msg_size}" }
match => { "message" => "\[%{HTTPDERROR_DATE:apache_timestamp}\] \[%{LOGLEVEL:loglevel}\] \[client %{IP:client_ip}\] %{GREEDYDATA:content}" }
match => { "message" => "\[%{HTTPDERROR_DATE:apache_timestamp}\] \[%{LOGLEVEL:loglevel}\] %{GREEDYDATA:content}" }
match => { "message" => "HTTP/%{NUMBER:httpversion}\" %{POSINT:response} %{POSINT:msg_size} %{GREEDYDATA:more_content}" }
match => { "message" => "%{IP:client_ip} - %{DATA:user_cn} \[%{HTTPDATE:timestamp}\] \"%{WORD:verb} %{DATA:uri} HTTP/%{NUMBER:httpversion}\" %{POSINT:response} %{POSINT:msg_size}" }
match => { "message" => "%{DATA}HTTP/%{NUMBER:httpversion}\" %{POSINT:response} %{POSINT:msg_size}" }
match => { "message" => "HTTP/%{NUMBER:httpversion}\" %{POSINT:response} %{GREEDYDATA:more_content}" }
match => { "message" => "HTTP/%{NUMBER:httpversion}\" %{POSINT:response} %{POSINT:msg_size} %{GREEDYDATA:more_content}" }
add_tag => [ "apache_processed" ]
tag_on_failure => ["_gpf_apache_error"]
}
}
if "apache_processed" in [tags] {
if [more_content] {
mutate {
replace => { message => "%{more_content}" }
remove_field => [ "more_content", "host_target" ]
}
}
#else {
# mutate {
# replace => { message => "FULLY PROCESSED" }
# #remove_field => [ "content", "host_target" ]
# }
#}
}
date {
match => [ "apache_timestamp", "yyyy-MM-dd'THH:mm:ss:S+ZZ", "yyyy-MM-dd'THH:mm:ss:SSSSSS+ZZ" ]
#2016-08-10T02:04:04.173127+00:00
#2016-08-11T11:03:08.001293+00:00
add_tag => [ "apache_date" ]
tag_on_failure => [ "apache_date_fail" ]
}
mutate {
#remove_field => [ "syslog_hostname", "apache_timestamp" ]
}
}