diff --git a/roles/alpina/collections/services/monitoring/app_config.yml b/roles/alpina/collections/services/monitoring/app_config.yml new file mode 100644 index 0000000..9726b00 --- /dev/null +++ b/roles/alpina/collections/services/monitoring/app_config.yml @@ -0,0 +1,10 @@ +$ANSIBLE_VAULT;1.2;AES256;alpina +39396231313035346438656264623739653031313965353037666664316465356236363065663766 +6462643462653662643166376165643862333562316462650a393339663065653239353263653833 +30316237373738386637633532356333666230626438303332346234376135363165356239666234 +3630633432626635380a396237343633633961653762323431383838366337383966376235663533 +63616530636231393965366461383261623066396662346637373264353265303464323037666435 +62326433646336353236616335616336383035643331613131626266373439623761336337303465 +63613334366164666330613330306531383531616564323965636461626139663137653863663362 +30303162633730363866376562633366326632646466656663396361333132633638333165366532 +3339 \ No newline at end of file diff --git a/roles/alpina/collections/services/monitoring/templates/docker-compose.yml.j2 b/roles/alpina/collections/services/monitoring/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..b0e7b4a --- /dev/null +++ b/roles/alpina/collections/services/monitoring/templates/docker-compose.yml.j2 @@ -0,0 +1,55 @@ +{% import 'contrib/compose_helpers.j2' as helpers with context %} +{##} +version: "3.9" + +networks: + {{ helpers.default_network(251) | indent(2) }} + traefik_traefik: + external: true + +services: + grafana: + image: grafana/grafana:latest + container_name: grafana + labels: + - {{ helpers.traefik_labels('grafana', port='3000') | indent(6) }} + restart: unless-stopped + # Needed to make config files readable + user: "{{ remote_uid }}" + networks: + - default + - traefik_traefik + volumes: + - {{ base_volume_path }}/monitoring/grafana:/var/lib/grafana + - ./grafana_config:/etc/grafana:ro + + loki: + image: grafana/loki:latest + container_name: loki + restart: unless-stopped + # Needed to make config files readable + user: "{{ remote_uid }}" + command: + - -config.file=/etc/loki/loki-config.yaml + # Port forward is needed because not possible to resolve the container name from the host network + ports: + - 3100:3100 + volumes: + - {{ base_volume_path }}/monitoring/loki:/loki + - ./loki_config:/etc/loki:ro + tmpfs: + - /tmp/loki + + promtail: + image: grafana/promtail:latest + container_name: promtail + restart: unless-stopped + command: + - -config.file=/etc/promtail/promtail-config.yaml + ports: + - 514:514 + volumes: + - ./promtail_config:/etc/promtail:ro + - /var/log:/var/log:ro + tmpfs: + - /tmp diff --git a/roles/alpina/collections/services/monitoring/templates/grafana_config/grafana.ini.j2 b/roles/alpina/collections/services/monitoring/templates/grafana_config/grafana.ini.j2 new file mode 100644 index 0000000..1f0b65d --- /dev/null +++ b/roles/alpina/collections/services/monitoring/templates/grafana_config/grafana.ini.j2 @@ -0,0 +1,19 @@ +[server] +domain = grafana.{{ domain }} +root_url = https://%(domain)s/ + +[security] +admin_user = CaZzzer +admin_email = {{ grafana_admin_email }} +admin_password = {{ grafana_admin_password }} + +# TODO: Set up Authentik OAuth +; https://grafana.com/docs/grafana/latest/setup-grafana/configure-security/configure-authentication/generic-oauth/ +;[auth] +;disable_login_form = true +;signout_redirect_url = +; +;[auth.generic_oauth] +;name = Authentik +;enabled = true +;allow_sign_up = true diff --git a/roles/alpina/collections/services/monitoring/templates/grafana_config/provisioning/datasources/alpina.yaml.j2 b/roles/alpina/collections/services/monitoring/templates/grafana_config/provisioning/datasources/alpina.yaml.j2 new file mode 100644 index 0000000..7e6af83 --- /dev/null +++ b/roles/alpina/collections/services/monitoring/templates/grafana_config/provisioning/datasources/alpina.yaml.j2 @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: Loki + type: loki + version: 1 + access: proxy + uid: loki + url: http://loki:3100 + editable: false diff --git a/roles/alpina/collections/services/monitoring/templates/loki_config/loki-config.yaml.j2 b/roles/alpina/collections/services/monitoring/templates/loki_config/loki-config.yaml.j2 new file mode 100644 index 0000000..c1c5d4f --- /dev/null +++ b/roles/alpina/collections/services/monitoring/templates/loki_config/loki-config.yaml.j2 @@ -0,0 +1,30 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + path_prefix: /loki + # TODO: Consider setting up S3 for storage + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2020-10-24 + store: boltdb-shipper + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 24h + +# TODO: Figure this out +ruler: + alertmanager_url: http://localhost:9093 diff --git a/roles/alpina/collections/services/monitoring/templates/promtail_config/promtail-config.yaml b/roles/alpina/collections/services/monitoring/templates/promtail_config/promtail-config.yaml new file mode 100644 index 0000000..7906c88 --- /dev/null +++ b/roles/alpina/collections/services/monitoring/templates/promtail_config/promtail-config.yaml @@ -0,0 +1,48 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + + # local machine logs + - job_name: local + static_configs: + - targets: + - localhost + labels: + job: varlogs + __path__: /var/log/*log + + # syslog target + - job_name: syslog + syslog: + listen_address: "0.0.0.0:514" # make sure you also expose this port on the container + idle_timeout: 60s + label_structured_data: yes + labels: + job: "syslog" + relabel_configs: + - source_labels: ['__syslog_message_hostname'] + target_label: 'host' + - source_labels: ['__syslog_message_facility'] + target_label: 'facility' + - source_labels: ['__syslog_message_severity'] + target_label: 'severity' + - source_labels: ['__syslog_message_app_name'] + target_label: 'app_name' + +## docker logs + +#- job_name: docker +# pipeline_stages: +# - docker: {} +# static_configs: +# - labels: +# job: docker +# __path__: /var/lib/docker/containers/*/*-json.log diff --git a/roles/alpina/tasks/deploy_compose_stack.yml b/roles/alpina/tasks/deploy_compose_stack.yml index 0063d8d..630b504 100644 --- a/roles/alpina/tasks/deploy_compose_stack.yml +++ b/roles/alpina/tasks/deploy_compose_stack.yml @@ -11,7 +11,7 @@ path: "{{ current_stack_dest }}/{{ item.path }}" state: directory mode: "700" - with_community.general.filetree: "{{ current_stack_source }}/templates" + loop: "{{ lookup('community.general.filetree', current_stack_source + '/templates') }}" when: item.state == "directory" # TODO: This is not ideal as it leaks the variables between stacks @@ -25,7 +25,7 @@ src: "{{ item.src }}" dest: "{{ current_stack_dest }}/{{ item.path | regex_replace('\\.j2$', '') }}" mode: "600" - with_community.general.filetree: "{{ current_stack_source }}/templates" + loop: "{{ lookup('community.general.filetree', current_stack_source + '/templates') }}" when: item.state == "file" - name: Deploy docker-compose for {{ current_stack_name }} @@ -35,6 +35,7 @@ # And of course there's an IPv6 bug in docker-compose v1, smh # https://github.com/docker/compose/issues/7670 changed_when: "'created' in docker_compose_output.stderr.lower()" + failed_when: docker_compose_output.rc != 0 - debug: var: docker_compose_output diff --git a/roles/alpina/tasks/main.yml b/roles/alpina/tasks/main.yml index 3074258..ed4c3e8 100644 --- a/roles/alpina/tasks/main.yml +++ b/roles/alpina/tasks/main.yml @@ -1,3 +1,12 @@ +- name: Register uid of remote user + command: id -u + register: remote_uid_command + changed_when: false + +- name: Set fact for uid + set_fact: + remote_uid: "{{ remote_uid_command.stdout }}" + - name: Ensure alpina directory exists file: state: directory @@ -9,6 +18,7 @@ collection: services stacks: - traefik + - monitoring - authentik - minio import_tasks: deploy_collection.yml diff --git a/roles/common/tasks/main.yml b/roles/common/tasks/main.yml index 0fcbac1..986ce7d 100644 --- a/roles/common/tasks/main.yml +++ b/roles/common/tasks/main.yml @@ -49,6 +49,14 @@ state: enabled immediate: yes +- name: Allow 514 tcp for syslog + become: yes + firewalld: + port: 514/tcp + permanent: yes + state: enabled + immediate: yes + - name: Enable Firewall become: yes firewalld: diff --git a/roles/docker_host/tasks/main.yml b/roles/docker_host/tasks/main.yml index 5971ea6..5d2f535 100644 --- a/roles/docker_host/tasks/main.yml +++ b/roles/docker_host/tasks/main.yml @@ -19,6 +19,12 @@ mode: "0644" register: docker_daemon_config +- name: Install Docker loki plugin for logs + community.docker.docker_plugin: + plugin_name: grafana/loki-docker-driver:latest + alias: loki + state: enable + - name: Remove docker0 from firewalld trusted zone become: yes firewalld: @@ -50,6 +56,12 @@ networks: yes when: clean_desired is true +- name: Clean alpina directory + file: + path: "{{ my_svc_path }}" + state: absent + when: clean_desired is true + - name: Restart Docker daemon become: yes service: diff --git a/roles/docker_host/templates/daemon.json.j2 b/roles/docker_host/templates/daemon.json.j2 index 266084d..f9932d2 100644 --- a/roles/docker_host/templates/daemon.json.j2 +++ b/roles/docker_host/templates/daemon.json.j2 @@ -1,4 +1,10 @@ { "ipv6": true, - "fixed-cidr-v6": "{{ docker_ipv6_subnet | ansible.utils.ipsubnet(80, 0) }}" + "fixed-cidr-v6": "{{ docker_ipv6_subnet | ansible.utils.ipsubnet(80, 0) }}", + "log-driver": "loki", + "log-opts": { + "loki-url": "http://localhost:3100/loki/api/v1/push", + "loki-batch-size": "400", + "loki-retries": "5" + } }