Ansible role for installing and configuring elastic beats clients. Primarily used for shipping logs and metrics to an ELK stack. By default, this role will ship to logstash on the same box using filebeat and metricbeat.
- someplace to ship data to - as of the beats 5.x series this includes shipping to
file
,kafka
,redis
,console
,elasticsearch
, and/orlogstash
You'll need an SSL cert to encrypt logs in transit to the logstash/elasticsearch. This logic is not provided by this role and encryption is not enabled by default. You'll need to reference the official beats documentation output guides to add those options via variable.
---
#### PACKAGING #################################################################
# The libbeat packages to install.
# Options: www.elastic.co/guide/en/beats/libbeat/master/installing-beats.html
beats_client_beats_packages:
- filebeat
- metricbeat
beats_client_version: 5.5.0
# The apt repo URL pegs minor versions (e.g. 5.x) to avoid unexpected upgrades
beats_client_major_version_abbreviated: 5.x
beats_client_beats_prereq:
- apt-transport-https
# Elastic's PGP key for signing their repository
beats_client_elastic_pgp_key: "46095ACC8548582C1A2699A9D27D666CD88E42B4"
beats_client_keyserver: pgp.mit.edu
# Elastic's beats debian repository
beats_client_elastic_repo_url: "deb https://artifacts.elastic.co/packages/{{ beats_client_major_version_abbreviated }}/apt stable main"
#### FILEBEAT ##################################################################
# Sane default of localhost. Override to set to the IP address/DNS of the Logstash server.
beats_client_logserver: "127.0.0.1"
beats_client_port: 5000
# Controls how often Topbeat reports stats (in seconds)
beats_client_topbeat_period: 10
beats_client_logfiles:
- paths:
- /var/log/syslog
- /var/log/auth.log
document_type: syslog
- paths:
- /var/log/dpkg.log
document_type: dpkg
- paths:
- /var/log/apache2/*log
document_type: apache
- paths:
- /var/log/mail.info
- /var/log/mail.warn
- /var/log/mail.err
document_type: postfix
# To send additional logfiles, override the following list.
# Make sure each item has "path" and "type" attributes.
beats_client_extra_logfiles: []
beats_client_filebeat_combined_logfiles: "{{ beats_client_logfiles + beats_client_extra_logfiles }}"
beats_client_filebeat_logging:
level: warning
to_files: true
to_syslog: false
files:
path: /var/log/
name: filebeat.log
keepfiles: 2
beats_client_filebeat_config:
filebeat.prospectors: "{{ beats_client_filebeat_combined_logfiles }}"
output: "{{ beats_client_output }}"
logging: "{{ beats_client_filebeat_logging }}"
#### METRICBEAT ##################################################################
# See: www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-configuration-options.html
# See: www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-modules.html
beats_client_metricbeat_modules:
- module: system
metricsets:
- cpu
- load
- diskio
- filesystem
- fsstat
- memory
- network
- process
- socket
enabled: true
period: "{{ beats_client_topbeat_period }}s"
processes: ['.*']
beats_client_metricbeat_logging:
level: warning
to_files: true
to_syslog: false
files:
path: /var/log/
name: metricbeat.log
keepfiles: 2
beats_client_metricbeat_config:
metricbeat.modules: "{{ beats_client_metricbeat_modules }}"
output: "{{ beats_client_output }}"
logging: "{{ beats_client_metricbeat_logging }}"
#### PACKETBEAT ##################################################################
# See: www.elastic.co/guide/en/beats/packetbeat/master/configuring-packetbeat.html
beats_client_packetbeat_config: {}
#### HEARTBEAT ##################################################################
beats_client_heartbeat_config: {}
#### KIBANA ##################################################################
beats_client_kibana_export: no
beats_client_kibana_index: metrics-logstash-*
beats_client_kibana_url_base: http://localhost:9200
beats_client_kibana_url: "{{ beats_client_kibana_url_base }}/.kibana"
beats_client_kibana_indices:
metricbeat: metrics-logstash-*
filebeat: syslog-*
beats_client_kibana_dash_search:
metricbeat: Metricbeat*
filebeat: Filebeat*
beats_client_kibana_export_parameters: "-only-dashboards -es {{beats_client_kibana_url_base}}"
#### SHARED ##################################################################
# Note that SSL is disabled here by default, you'll need to override this
# variable using attributes from
# www.elastic.co/guide/en/beats/metricbeat/master/logstash-output.html
beats_client_output:
logstash:
enabled: true
hosts:
- "{{ beats_client_logserver }}:{{ beats_client_port }}"
# Master config dictionary variable.
beats_clients_configs:
filebeat: "{{ beats_client_filebeat_config }}"
metricbeat: "{{ beats_client_metricbeat_config }}"
packetbeat: "{{ beats_client_packetbeat_config }}"
heartbeat: "{{ beats_client_heartbeat_config }}"
- name: Configure beats clients.
hosts: clients
roles:
- role: freedomofpress.beats
tags: clients
This role uses Molecule and Testinfra for testing. To use it:
pip install -r requirements.txt
molecule test
You can also run selective commands:
molecule idempotence
molecule verify
To fire up an elasticsearch UI for debugging, run:
make elastic-ui
See the Molecule docs for more info.
The following resources were invaluable in creating this role.
MIT