monitoring: add initial stack with grafana, loki, and promtail

roles/alpina/collections/services/monitoring/app_config.yml

@@ -0,0 +1,10 @@
+$ANSIBLE_VAULT;1.2;AES256;alpina
+39396231313035346438656264623739653031313965353037666664316465356236363065663766
+6462643462653662643166376165643862333562316462650a393339663065653239353263653833
+30316237373738386637633532356333666230626438303332346234376135363165356239666234
+3630633432626635380a396237343633633961653762323431383838366337383966376235663533
+63616530636231393965366461383261623066396662346637373264353265303464323037666435
+62326433646336353236616335616336383035643331613131626266373439623761336337303465
+63613334366164666330613330306531383531616564323965636461626139663137653863663362
+30303162633730363866376562633366326632646466656663396361333132633638333165366532
+3339

roles/alpina/collections/services/monitoring/templates/docker-compose.yml.j2

@@ -0,0 +1,55 @@
+{% import 'contrib/compose_helpers.j2' as helpers with context %}
+{##}
+version: "3.9"
+
+networks:
+  {{ helpers.default_network(251) | indent(2) }}
+  traefik_traefik:
+    external: true
+
+services:
+  grafana:
+    image: grafana/grafana:latest
+    container_name: grafana
+    labels:
+      - {{ helpers.traefik_labels('grafana', port='3000') | indent(6) }}
+    restart: unless-stopped
+    # Needed to make config files readable
+    user: "{{ remote_uid }}"
+    networks:
+      - default
+      - traefik_traefik
+    volumes:
+      - {{ base_volume_path }}/monitoring/grafana:/var/lib/grafana
+      - ./grafana_config:/etc/grafana:ro
+
+  loki:
+    image: grafana/loki:latest
+    container_name: loki
+    restart: unless-stopped
+    # Needed to make config files readable
+    user: "{{ remote_uid }}"
+    command:
+      - -config.file=/etc/loki/loki-config.yaml
+    # Port forward is needed because not possible to resolve the container name from the host network
+    ports:
+      - 3100:3100
+    volumes:
+      - {{ base_volume_path }}/monitoring/loki:/loki
+      - ./loki_config:/etc/loki:ro
+    tmpfs:
+      - /tmp/loki
+
+  promtail:
+    image: grafana/promtail:latest
+    container_name: promtail
+    restart: unless-stopped
+    command:
+      - -config.file=/etc/promtail/promtail-config.yaml
+    ports:
+      - 514:514
+    volumes:
+        - ./promtail_config:/etc/promtail:ro
+        - /var/log:/var/log:ro
+    tmpfs:
+      - /tmp

roles/alpina/collections/services/monitoring/templates/grafana_config/grafana.ini.j2

@@ -0,0 +1,19 @@
+[server]
+domain = grafana.{{ domain }}
+root_url = https://%(domain)s/
+
+[security]
+admin_user = CaZzzer
+admin_email = {{ grafana_admin_email }}
+admin_password = {{ grafana_admin_password }}
+
+# TODO: Set up Authentik OAuth
+; https://grafana.com/docs/grafana/latest/setup-grafana/configure-security/configure-authentication/generic-oauth/
+;[auth]
+;disable_login_form = true
+;signout_redirect_url =
+;
+;[auth.generic_oauth]
+;name = Authentik
+;enabled = true
+;allow_sign_up = true

roles/alpina/collections/services/monitoring/templates/grafana_config/provisioning/datasources/alpina.yaml.j2

@@ -0,0 +1,10 @@
+apiVersion: 1
+
+datasources:
+  - name: Loki
+    type: loki
+    version: 1
+    access: proxy
+    uid: loki
+    url: http://loki:3100
+    editable: false

roles/alpina/collections/services/monitoring/templates/loki_config/loki-config.yaml.j2

@@ -0,0 +1,30 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+common:
+  path_prefix: /loki
+  # TODO: Consider setting up S3 for storage
+  storage:
+    filesystem:
+      chunks_directory: /loki/chunks
+      rules_directory: /loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: boltdb-shipper
+      object_store: filesystem
+      schema: v11
+      index:
+        prefix: index_
+        period: 24h
+
+# TODO: Figure this out
+ruler:
+  alertmanager_url: http://localhost:9093

roles/alpina/collections/services/monitoring/templates/promtail_config/promtail-config.yaml

@@ -0,0 +1,48 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+
+  # local machine logs
+  - job_name: local
+    static_configs:
+    - targets:
+        - localhost
+      labels:
+        job: varlogs
+        __path__: /var/log/*log
+
+  # syslog target
+  - job_name: syslog
+    syslog:
+      listen_address: "0.0.0.0:514" # make sure you also expose this port on the container
+      idle_timeout: 60s
+      label_structured_data: yes
+      labels:
+        job: "syslog"
+    relabel_configs:
+      - source_labels: ['__syslog_message_hostname']
+        target_label: 'host'
+      - source_labels: ['__syslog_message_facility']
+        target_label: 'facility'
+      - source_labels: ['__syslog_message_severity']
+        target_label: 'severity'
+      - source_labels: ['__syslog_message_app_name']
+        target_label: 'app_name'
+
+## docker logs
+
+#- job_name: docker
+#  pipeline_stages:
+#    - docker: {}
+#  static_configs:
+#    - labels:
+#        job: docker
+#        __path__: /var/lib/docker/containers/*/*-json.log

roles/alpina/tasks/deploy_compose_stack.yml

@@ -11,7 +11,7 @@
     path: "{{ current_stack_dest }}/{{ item.path }}"
     state: directory
     mode: "700"
-  with_community.general.filetree: "{{ current_stack_source }}/templates"
+  loop: "{{ lookup('community.general.filetree', current_stack_source + '/templates') }}"
   when: item.state == "directory"
 
 # TODO: This is not ideal as it leaks the variables between stacks
@@ -25,7 +25,7 @@
     src: "{{ item.src }}"
     dest: "{{ current_stack_dest }}/{{ item.path | regex_replace('\\.j2$', '') }}"
     mode: "600"
-  with_community.general.filetree: "{{ current_stack_source }}/templates"
+  loop: "{{ lookup('community.general.filetree', current_stack_source + '/templates') }}"
   when: item.state == "file"
 
 - name: Deploy docker-compose for {{ current_stack_name }}
@@ -35,6 +35,7 @@
   # And of course there's an IPv6 bug in docker-compose v1, smh
   # https://github.com/docker/compose/issues/7670
   changed_when: "'created' in docker_compose_output.stderr.lower()"
+  failed_when: docker_compose_output.rc != 0
 
 - debug:
     var: docker_compose_output

roles/alpina/tasks/main.yml

@@ -1,3 +1,12 @@
+- name: Register uid of remote user
+  command: id -u
+  register: remote_uid_command
+  changed_when: false
+
+- name: Set fact for uid
+  set_fact:
+    remote_uid: "{{ remote_uid_command.stdout }}"
+
 - name: Ensure alpina directory exists
   file:
     state: directory
@@ -9,6 +18,7 @@
     collection: services
     stacks:
       - traefik
+      - monitoring
       - authentik
       - minio
   import_tasks: deploy_collection.yml

roles/common/tasks/main.yml

@@ -49,6 +49,14 @@
     state: enabled
     immediate: yes
 
+- name: Allow 514 tcp for syslog
+  become: yes
+  firewalld:
+    port: 514/tcp
+    permanent: yes
+    state: enabled
+    immediate: yes
+
 - name: Enable Firewall
   become: yes
   firewalld:

roles/docker_host/tasks/main.yml

@@ -19,6 +19,12 @@
     mode: "0644"
   register: docker_daemon_config
 
+- name: Install Docker loki plugin for logs
+  community.docker.docker_plugin:
+    plugin_name: grafana/loki-docker-driver:latest
+    alias: loki
+    state: enable
+
 - name: Remove docker0 from firewalld trusted zone
   become: yes
   firewalld:
@@ -50,6 +56,12 @@
     networks: yes
   when: clean_desired is true
 
+- name: Clean alpina directory
+  file:
+    path: "{{ my_svc_path }}"
+    state: absent
+  when: clean_desired is true
+
 - name: Restart Docker daemon
   become: yes
   service:

roles/docker_host/templates/daemon.json.j2

@@ -1,4 +1,10 @@
 {
     "ipv6": true,
-    "fixed-cidr-v6": "{{ docker_ipv6_subnet | ansible.utils.ipsubnet(80, 0) }}"
+    "fixed-cidr-v6": "{{ docker_ipv6_subnet | ansible.utils.ipsubnet(80, 0) }}",
+    "log-driver": "loki",
+    "log-opts": {
+        "loki-url": "http://localhost:3100/loki/api/v1/push",
+        "loki-batch-size": "400",
+        "loki-retries": "5"
+    }
 }