ELK 7.4.0
pf-09.2019.grok
Analyzing my pfsense logs in Kibana and filtering for grokparsefailures shows most firewall entries cannot be parsed using the latest grok pattern listed above. When the log entries contain IPv6 data see below samples:
NOTE: replaced IPV6&IPV4 IP addresses with random numbers for security concerns.
5,,,1000000003,em0,match,block,in,6,0xe0,0x00000,1,Options,0,32,fe80::413:14ff:fe33:4327,ff421::1,HBH,RTALERT,0x0000,PADN,
5,,,1000000003,em0,match,block,in,6,0x00,0x00000,255,ICMPv6,58,32,2347:f721:454:e1::1,ff44:1:ff2f:9,
9,,,1000000103,em0,match,block,in,4,0x0,,52,5929,1480,none,17,udp,381,208.78.71.14,2.2.2.2,
9,,,1000000103,em0,match,block,in,4,0x0,,1,0,0,none,2,igmp,36,2.2.2.2,224.0.0.1,datalength=12
JSON output for some:
{
"_index": "logstash-pfsense-000001",
"_type": "_doc",
"_id": "dF8tMW4BRPLOAM2DCL-s",
"_version": 1,
"_score": null,
"_source": {
"host": "2.2.2.2",
"prog": "filterlog",
"syslog_severity_code": 5,
"severity_label": "Emergency",
"syslog_facility": "user-level",
"@version": "1",
"evtid": "134",
"tags": [
"_grokparsefailure_sysloginput",
"_grokparsefailure",
"_geoip_lookup_failure"
],
"type": "pfsense",
"@timestamp": "2019-11-03T12:11:14.000Z",
"priority": 0,
"message": "9,,,1000000103,em0,match,block,in,4,0x0,,1,0,0,none,2,igmp,36,2.2.2.2,224.0.0.1,datalength=12 ",
"severity": 0,
"facility": 0,
"facility_label": "kernel",
"syslog_severity": "notice",
"syslog_facility_code": 1
},
"fields": {
"@timestamp": [
"2019-11-03T12:11:14.000Z"
]
},
"sort": [
1572783074000
]
}
{
"_index": "logstash-pfsense-000001",
"_type": "_doc",
"_id": "WV8sMW4BRPLOAM2D9b4c",
"_version": 1,
"_score": null,
"_source": {
"host": "2.2.2.2",
"prog": "filterlog",
"syslog_severity_code": 5,
"severity_label": "Emergency",
"syslog_facility": "user-level",
"@version": "1",
"evtid": "134",
"tags": [
"_grokparsefailure_sysloginput",
"_grokparsefailure",
"_geoip_lookup_failure"
],
"type": "pfsense",
"@timestamp": "2019-11-03T12:11:09.000Z",
"priority": 0,
"message": "5,,,1000000003,em0,match,block,in,6,0x00,0x00000,255,ICMPv6,58,32,2444:f711:111:e1::1,ff11::1:ff14:361a,",
"severity": 0,
"facility": 0,
"facility_label": "kernel",
"syslog_severity": "notice",
"syslog_facility_code": 1
},
"fields": {
"@timestamp": [
"2019-11-03T12:11:09.000Z"
]
},
"sort": [
1572783069000
]
}
{
"_index": "logstash-pfsense-000001",
"_type": "_doc",
"_id": "K2opMW4Bx2SgVlxP1aw1",
"_version": 1,
"_score": null,
"_source": {
"host": "2.2.2.2",
"prog": "filterlog",
"syslog_severity_code": 5,
"severity_label": "Emergency",
"syslog_facility": "user-level",
"@version": "1",
"evtid": "134",
"tags": [
"_grokparsefailure_sysloginput",
"_grokparsefailure",
"_geoip_lookup_failure"
],
"type": "pfsense",
"@timestamp": "2019-11-03T12:07:39.000Z",
"priority": 0,
"message": "5,,,1000000003,em0,match,block,in,6,0xe0,0x00000,1,Options,0,32,fe22::222:10ff:fe22:22,ff02::1,HBH,RTALERT,0x0000,PADN,",
"severity": 0,
"facility": 0,
"facility_label": "kernel",
"syslog_severity": "notice",
"syslog_facility_code": 1
},
"fields": {
"@timestamp": [
"2019-11-03T12:07:39.000Z"
]
},
"sort": [
1572782859000
]
}
{
"_index": "logstash-pfsense-000001",
"_type": "_doc",
"_id": "HWooMW4Bx2SgVlxPLZew",
"_version": 1,
"_score": null,
"_source": {
"host": "2.2.2.2",
"prog": "filterlog",
"syslog_severity_code": 5,
"severity_label": "Emergency",
"syslog_facility": "user-level",
"@version": "1",
"evtid": "134",
"tags": [
"_grokparsefailure_sysloginput",
"_grokparsefailure",
"_geoip_lookup_failure"
],
"type": "pfsense",
"@timestamp": "2019-11-03T12:05:51.000Z",
"priority": 0,
"message": "9,,,1000000103,em0,match,block,in,4,0x0,,50,8301,0,+,17,udp,1500,199.253.60.1,2.2.2.2,53,48193,1631",
"severity": 0,
"facility": 0,
"facility_label": "kernel",
"syslog_severity": "notice",
"syslog_facility_code": 1
},
"fields": {
"@timestamp": [
"2019-11-03T12:05:51.000Z"
]
},
"sort": [
1572782751000
]
}
The filter for logstash:
## 11-pfsense.conf
filter {
if [type] == "pfsense" {
grok {
match => [ "message", "<(?<evtid>.*)>(?<datetime>(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\s+(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) (?:2[0123]|[01]?[0-9]):(?:[0-5][0-9]):(?:[0-5][0-9])) (?<prog>.*?): (?<msg>.*)" ]
}
mutate {
gsub => ["datetime"," "," "]
}
date {
match => [ "datetime", "MMM dd HH:mm:ss" ]
timezone => "America/New York"
}
mutate {
replace => [ "message", "%{msg}" ]
}
mutate {
remove_field => [ "msg", "datetime" ]
}
if [prog] =~ /^filterlog$/ {
mutate {
remove_field => [ "msg", "datetime" ]
}
grok {
add_tag => [ "firewall" ]
patterns_dir => "/opt/elastic/logstash/conf.d/patterns"
match => [ "message", "%{PF_LOG_DATA}%{PF_IP_SPECIFIC_DATA}%{PF_IP_DATA}%{PF_PROTOCOL_DATA}",
message", "%{PF_IPv4_SPECIFIC_DATA}%{PF_IP_DATA}%{PF_PROTOCOL_DATA}",
"message", "%{PF_IPv6_SPECIFIC_DATA}%{PF_IP_DATA}%{PF_PROTOCOL_DATA}"]
}
mutate {
lowercase => [ 'proto' ]
}
if ![geoip] and [src_ip] !~ /^(10\.)/ {
geoip {
add_tag => [ "GeoIP" ]
source => "src_ip"
}
}
if ![geoip] and [dst_ip] !~ /^(10\.)/ {
geoip {
add_tag => [ "GeoIP" ]
source => "src_ip"
}
}
}
}
}
Guide (Update Coming Soon
React
Typescript
Django
Laravel
Facebook
Microsoft
Google
Alibaba
D3
Tencent