I installed a complete new ELK Stack in our test enviroment and tried to use the icingabeat from our productive icinga2 system.
Unfortenetely the elastic search template and the kibana dashboards are not compatible with the new version.
When I start icingabeat in the debugging mode
2019-04-18T17:34:41.781+0200 INFO elasticsearch/client.go:713 Connected to Elasticsearch version 7.0.0
2019-04-18T17:34:41.781+0200 DEBUG [elasticsearch] elasticsearch/client.go:731 HEAD http://IP:9200/_template/icingabeat-6.5.4 <nil>
2019-04-18T17:34:41.783+0200 INFO template/load.go:82 Loading template for Elasticsearch version: 7.0.0
2019-04-18T17:34:41.783+0200 DEBUG [template] template/load.go:116 Load default fields.yml
2019-04-18T17:34:41.817+0200 DEBUG [template] template/load.go:139 Try loading template with name: icingabeat-6.5.4
2019-04-18T17:34:41.820+0200 DEBUG [elasticsearch] elasticsearch/client.go:731 PUT http://IP:9200/_template/icingabeat-6.5.4 map[index_patterns:[icingabeat-6.5.4-*] mappings:{"doc":{"_meta":{"version":"6.5.4"},"date_detection":false,"dynamic_templates":[{"fields":{"mapping":{"type":"keyword"},"match_mapping_type":"string","path_match":"fields.*"}},{"docker.container.labels":{"mapping":{"type":"keyword"},"match_mapping_type":"string","path_match":"docker.container.labels.*"}},{"strings_as_keyword":{"mapping":{"ignore_above":1024,"type":"keyword"},"match_mapping_type":"string"}}],"properties":{"@timestamp":{"type":"date"},"beat":{"properties":{"hostname":{"ignore_above":1024,"type":"keyword"},"name":{"ignore_above":1024,"type":"keyword"},"timezone":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"}}},"docker":{"properties":{"container":{"properties":{"id":{"ignore_above":1024,"type":"keyword"},"image":{"ignore_above":1024,"type":"keyword"},"labels":{"type":"object"},"name":{"ignore_above":1024,"type":"keyword"}}}}},"error":{"properties":{"code":{"type":"long"},"message":{"norms":false,"type":"text"},"type":{"ignore_above":1024,"type":"keyword"}}},"fields":{"type":"object"},"host":{"properties":{"architecture":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"ip":{"type":"ip"},"mac":{"ignore_above":1024,"type":"keyword"},"name":{"ignore_above":1024,"type":"keyword"},"os":{"properties":{"family":{"ignore_above":1024,"type":"keyword"},"platform":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"}}}}},"icinga":{"properties":{"acknowledgement_type":{"type":"long"},"author":{"ignore_above":1024,"type":"keyword"},"check_result":{"properties":{"active":{"type":"boolean"},"check_source":{"ignore_above":1024,"type":"keyword"},"command":{"norms":false,"type":"text"},"execution_end":{"type":"date"},"execution_start":{"type":"date"},"exit_status":{"type":"long"},"output":{"norms":false,"type":"text"},"performance_data":{"norms":false,"type":"text"},"schedule_end":{"type":"date"},"schedule_start":{"type":"date"},"state":{"type":"long"},"ttl":{"type":"long"},"type":{"ignore_above":1024,"type":"keyword"},"vars_after":{"properties":{"attempt":{"type":"long"},"reachable":{"type":"boolean"},"state":{"type":"long"},"state_type":{"type":"long"}}},"vars_before":{"properties":{"attempt":{"type":"long"},"reachable":{"type":"boolean"},"state":{"type":"long"},"state_type":{"type":"long"}}}}},"comment":{"properties":{"__name":{"norms":false,"type":"text"},"author":{"ignore_above":1024,"type":"keyword"},"entry_time":{"type":"date"},"entry_type":{"type":"long"},"expire_time":{"type":"date"},"host_name":{"ignore_above":1024,"type":"keyword"},"legacy_id":{"type":"long"},"name":{"ignore_above":1024,"type":"keyword"},"package":{"ignore_above":1024,"type":"keyword"},"service_name":{"ignore_above":1024,"type":"keyword"},"templates":{"norms":false,"type":"text"},"text":{"norms":false,"type":"text"},"type":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"},"zone":{"ignore_above":1024,"type":"keyword"}}},"downtime":{"properties":{"__name":{"norms":false,"type":"text"},"author":{"ignore_above":1024,"type":"keyword"},"comment":{"norms":false,"type":"text"},"config_owner":{"norms":false,"type":"text"},"duration":{"type":"long"},"end_time":{"type":"date"},"entry_time":{"type":"date"},"fixed":{"type":"boolean"},"host_name":{"ignore_above":1024,"type":"keyword"},"legacy_id":{"type":"long"},"name":{"ignore_above":1024,"type":"keyword"},"package":{"ignore_above":1024,"type":"keyword"},"scheduled_by":{"norms":false,"type":"text"},"service_name":{"ignore_above":1024,"type":"keyword"},"start_time":{"type":"date"},"templates":{"norms":false,"type":"text"},"trigger_time":{"type":"date"},"triggered_by":{"norms":false,"type":"text"},"triggers":{"norms":false,"type":"text"},"type":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"},"was_cancelled":{"type":"boolean"},"zone":{"ignore_above":1024,"type":"keyword"}}},"expiry":{"type":"date"},"host":{"ignore_above":1024,"type":"keyword"},"notification_type":{"ignore_above":1024,"type":"keyword"},"notify":{"ignore_above":1024,"type":"keyword"},"service":{"ignore_above":1024,"type":"keyword"},"state":{"type":"long"},"state_type":{"type":"long"},"status":{"properties":{"active_host_checks":{"type":"long"},"active_host_checks_15min":{"type":"long"},"active_host_checks_1min":{"type":"long"},"active_host_checks_5min":{"type":"long"},"active_service_checks":{"type":"long"},"active_service_checks_15min":{"type":"long"},"active_service_checks_1min":{"type":"long"},"active_service_checks_5min":{"type":"long"},"api":{"properties":{"identity":{"ignore_above":1024,"type":"keyword"},"num_conn_endpoints":{"type":"long"},"num_endpoints":{"type":"long"},"num_not_conn_endpoints":{"type":"long"}}},"avg_execution_time":{"type":"long"},"avg_latency":{"type":"long"},"checkercomponent":{"properties":{"checker":{"properties":{"idle":{"type":"long"},"pending":{"type":"long"}}}}},"filelogger":{"properties":{"main-log":{"type":"long"}}},"icingaapplication":{"properties":{"app":{"properties":{"enable_event_handlers":{"type":"boolean"},"enable_flapping":{"type":"boolean"},"enable_host_checks":{"type":"boolean"},"enable_notifications":{"type":"boolean"},"enable_perfdata":{"type":"boolean"},"enable_service_checks":{"type":"boolean"},"node_name":{"ignore_above":1024,"type":"keyword"},"pid":{"type":"long"},"program_start":{"type":"long"},"version":{"ignore_above":1024,"type":"keyword"}}}}},"idomysqlconnection":{"properties":{"ido-mysql":{"properties":{"connected":{"type":"boolean"},"instance_name":{"ignore_above":1024,"type":"keyword"},"query_queue_items":{"type":"long"},"version":{"ignore_above":1024,"type":"keyword"}}}}},"max_execution_time":{"type":"long"},"max_latency":{"type":"long"},"min_execution_time":{"type":"long"},"min_latency":{"type":"long"},"notificationcomponent":{"properties":{"notification":{"type":"long"}}},"num_hosts_acknowledged":{"type":"long"},"num_hosts_down":{"type":"long"},"num_hosts_flapping":{"type":"long"},"num_hosts_in_downtime":{"type":"long"},"num_hosts_pending":{"type":"long"},"num_hosts_unreachable":{"type":"long"},"num_hosts_up":{"type":"long"},"num_services_acknowledged":{"type":"long"},"num_services_critical":{"type":"long"},"num_services_flapping":{"type":"long"},"num_services_in_downtime":{"type":"long"},"num_services_ok":{"type":"long"},"num_services_pending":{"type":"long"},"num_services_unknown":{"type":"long"},"num_services_unreachable":{"type":"long"},"num_services_warning":{"type":"long"},"passive_host_checks":{"type":"long"},"passive_host_checks_15min":{"type":"long"},"passive_host_checks_1min":{"type":"long"},"passive_host_checks_5min":{"type":"long"},"passive_service_checks":{"type":"long"},"passive_service_checks_15min":{"type":"long"},"passive_service_checks_1min":{"type":"long"},"passive_service_checks_5min":{"type":"long"},"uptime":{"type":"long"}}},"text":{"norms":false,"type":"text"},"timestamp":{"type":"date"},"type":{"ignore_above":1024,"type":"keyword"},"users":{"ignore_above":1024,"type":"keyword"}}},"kubernetes":{"properties":{"annotations":{"type":"object"},"container":{"properties":{"image":{"ignore_above":1024,"type":"keyword"},"name":{"ignore_above":1024,"type":"keyword"}}},"labels":{"type":"object"},"namespace":{"ignore_above":1024,"type":"keyword"},"node":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}},"pod":{"properties":{"name":{"ignore_above":1024,"type":"keyword"},"uid":{"ignore_above":1024,"type":"keyword"}}}}},"meta":{"properties":{"cloud":{"properties":{"availability_zone":{"ignore_above":1024,"type":"keyword"},"instance_id":{"ignore_above":1024,"type":"keyword"},"instance_name":{"ignore_above":1024,"type":"keyword"},"machine_type":{"ignore_above":1024,"type":"keyword"},"project_id":{"ignore_above":1024,"type":"keyword"},"provider":{"ignore_above":1024,"type":"keyword"},"region":{"ignore_above":1024,"type":"keyword"}}}}},"tags":{"ignore_above":1024,"type":"keyword"},"timestamp":{"type":"date"},"type":{"ignore_above":1024,"type":"keyword"}}}} order:1 settings:{"index":{"mapping":{"total_fields":{"limit":10000}},"number_of_routing_shards":30,"query":{"default_field":["type","icinga.type","icinga.host","icinga.service","icinga.author","icinga.notification_type","icinga.text","icinga.users","icinga.notify","icinga.check_result.check_source","icinga.check_result.command","icinga.check_result.output","icinga.check_result.performance_data","icinga.check_result.type","icinga.comment.__name","icinga.comment.author","icinga.comment.host_name","icinga.comment.name","icinga.comment.package","icinga.comment.service_name","icinga.comment.templates","icinga.comment.text","icinga.comment.type","icinga.comment.version","icinga.comment.zone","icinga.downtime.__name","icinga.downtime.author","icinga.downtime.comment","icinga.downtime.config_owner","icinga.downtime.host_name","icinga.downtime.name","icinga.downtime.package","icinga.downtime.scheduled_by","icinga.downtime.service_name","icinga.downtime.templates","icinga.downtime.triggered_by","icinga.downtime.triggers","icinga.downtime.type","icinga.downtime.version","icinga.downtime.zone","icinga.status.api.identity","icinga.status.icingaapplication.app.node_name","icinga.status.icingaapplication.app.version","icinga.status.idomysqlconnection.ido-mysql.instance_name","icinga.status.idomysqlconnection.ido-mysql.version","beat.name","beat.hostname","beat.timezone","beat.version","tags","error.message","error.type","meta.cloud.provider","meta.cloud.instance_id","meta.cloud.instance_name","meta.cloud.machine_type","meta.cloud.availability_zone","meta.cloud.project_id","meta.cloud.region","docker.container.id","docker.container.image","docker.container.name","host.name","host.id","host.architecture","host.os.platform","host.os.version","host.os.family","host.mac","kubernetes.pod.name","kubernetes.pod.uid","kubernetes.namespace","kubernetes.node.name","kubernetes.container.name","kubernetes.container.image","fields.*"]},"refresh_interval":"5s"}}]
So I think the elasticsearch changed somethink to icingabeat needs a new template.
The kibana dashboards are not working too. I can post the debugging log if someone need it at the moment the ELK is down because of easter holiday.