WIP: updates: postgres

.idea/jsonSchemas.xml

@@ -39,22 +39,6 @@
             </SchemaInfo>
           </value>
         </entry>
-        <entry key="Loki">
-          <value>
-            <SchemaInfo>
-              <option name="name" value="Loki" />
-              <option name="relativePathToSchema" value="https://json.schemastore.org/loki.json" />
-              <option name="applicationDefined" value="true" />
-              <option name="patterns">
-                <list>
-                  <Item>
-                    <option name="path" value="roles/alpina/templates/services/monitoring/loki_config/loki-config.yaml.j2" />
-                  </Item>
-                </list>
-              </option>
-            </SchemaInfo>
-          </value>
-        </entry>
         <entry key="Traefik v2">
           <value>
             <SchemaInfo>
@@ -90,9 +74,6 @@
                   <Item>
                     <option name="path" value="file:///run/user/1000/kio-fuse-kipURF/sftp/root@debbi.lab.home/mnt/dock/traefik/rules/hello-world.yml" />
                   </Item>
-                  <Item>
-                    <option name="path" value="roles/alpina/templates/services/traefik/rules/traefik-dash.yml.j2" />
-                  </Item>
                 </list>
               </option>
             </SchemaInfo>

README.md

@@ -23,16 +23,15 @@ I'd like to get a /56 or /48, perhaps using Hurricane Electric's tunnel broker.
 ## Upgrading Postgres
 Upgrading the postgres container for a given stack requires a dump and restore.
 
-After making a snapshot or backup of postgres data directory,
-in the compose directory for a given stack, run the following commands:
+In the compose directory for a given stack, run the following commands:
 ```bash
 docker compose down
 docker compose up -d <db_service>
 docker compose exec -it <db_service> pg_dumpall -U <db_user> | tee /tmp/dump.sql
 docker compose down
 
-rm -r <postgres_data_dir>/*  # as root
 # Edit the docker-compose.yml file to use the new postgres image
+sudo rm -r <postgres_data_dir>/*  # TODO: this might not actually work with the sudo wildcard
 docker compose up -d <db_service>
 # For some reason, compose exec doesn't like the input redirection
 docker exec -i <db_container_name> psql -U <db_user> < /tmp/dump.sql
@@ -45,15 +44,3 @@ password hashes. This can be done by running the following command:
 ```bash
 docker compose exec -it <db_service> psql -U <db_user> -c "\password"
 ```
-
-## Nextcloud
-Nextcloud requires some additional work to set up notify_push.
-
-- Initially, comment out the notify_push service in the docker compose.
-- Set up nextcloud and install the Client Push (notify_push) app.
-- Uncomment the notify_push service in the docker compose and `up -d` the stack.
-- ```bash
-  docker compose exec app ./occ notify_push:setup https://nc.<domain>/push
-  ```
-
-I should probably get around to automating this at some point.

roles/alpina/templates/apps/nextcloud/.env.j2

@@ -1 +1 @@
-NEXTCLOUD_VERSION=29-apache
+NEXTCLOUD_VERSION=29-fpm-alpine

roles/alpina/templates/apps/nextcloud/.env.nextcloud.j2

@@ -17,8 +17,7 @@ SMTP_PASSWORD={{ nextcloud_sendgrid_api_key }}
 MAIL_FROM_ADDRESS=nc
 MAIL_DOMAIN=cazzzer.com
 
-# host IPv4 and IPv6 addresses, loopback for notify_push
-TRUSTED_PROXIES={{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] }} {{ hostvars[inventory_hostname]['ansible_default_ipv6']['address'] }} 127.0.0.1 ::1
+TRUSTED_PROXIES={{ traefik_subnet }}
 OVERWRITEHOST=nc.{{ domain }}
 OVERWRITEPROTOCOL=https
 OVERWRITECLIURL=https://nc.{{ domain }}

roles/alpina/templates/apps/nextcloud/.env.notify_push

@@ -1,4 +0,0 @@
-DATABASE_URL=postgres://nextcloud:{{ nextcloud_db_password }}@db/nextcloud
-DATABASE_PREFIX=oc_
-REDIS_URL=redis://:{{ redis_password }}@redis
-NEXTCLOUD_URL=http://localhost

roles/alpina/templates/apps/nextcloud/docker-compose.yml.j2

@@ -10,8 +10,6 @@ services:
   app:
     image: nextcloud:${NEXTCLOUD_VERSION}
     container_name: nextcloud_app
-    labels:
-      - {{ helpers.traefik_labels('nc', port='80') | indent(6) }}
     restart: unless-stopped
     depends_on:
       - db
@@ -42,25 +40,18 @@ services:
   notify_push:
     image: nextcloud:${NEXTCLOUD_VERSION}
     container_name: nextcloud_notify_push
-    {# TODO: Refactor this and minio -#}
-    labels:
-      - traefik.enable=true
-      - traefik.http.routers.nc-notify.rule=Host(`nc.{{ domain }}`) && PathPrefix(`/push`)
-      - traefik.http.routers.nc-notify.entrypoints=websecure
-      - traefik.http.routers.nc-notify.tls=true
-      - traefik.http.routers.nc-notify.tls.certresolver=letsencrypt
-      - traefik.http.routers.nc-notify.tls.domains.0.main={{ domain }}
-      - traefik.http.routers.nc-notify.tls.domains.0.sans=*.{{ domain }}
-      - traefik.http.services.nc-notify.loadbalancer.server.port=7867
     restart: unless-stopped
-    user: www-data
-    env_file:
-      - .env.notify_push
-    network_mode: service:app
+    depends_on:
+      - app
     entrypoint:
       - /var/www/html/custom_apps/notify_push/bin/x86_64/notify_push
+      - /var/www/html/config/config.php
+    networks:
+      - default
     volumes:
       - {{ base_volume_path }}/nextcloud/nextcloud:/var/www/html
+      - {{ base_volume_path }}/nextcloud/nextcloud_config:/var/www/html/config
+      - {{ base_volume_path }}/nextcloud/nextcloud_data:/var/www/html/data
 
   db:
     image: postgres:16-alpine
@@ -74,7 +65,7 @@ services:
       - {{ base_volume_path }}/nextcloud/db:/var/lib/postgresql/data
 
   redis:
-    image: redis:alpine
+    image: redis:7-alpine
     container_name: nextcloud_redis
     restart: unless-stopped
     env_file:
@@ -85,3 +76,20 @@ services:
       - sh
       - -c
       - redis-server --requirepass $$REDIS_PASSWORD
+
+  web:
+    image: nginx:1.23-alpine
+    container_name: nextcloud_web
+    labels:
+      - {{ helpers.traefik_labels('nc') | indent(6) }}
+    restart: unless-stopped
+    links:
+      - app
+    networks:
+      - traefik_traefik
+      - default
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf:ro
+      - {{ base_volume_path }}/nextcloud/nextcloud:/var/www/html
+      - {{ base_volume_path }}/nextcloud/nextcloud_config:/var/www/html/config
+      - {{ base_volume_path }}/nextcloud/nextcloud_data:/var/www/html/data

roles/alpina/templates/apps/nextcloud/nginx.conf.j2

@@ -0,0 +1,182 @@
+# https://github.com/nextcloud/docker/blob/master/.examples/docker-compose/with-nginx-proxy/postgres/fpm/web/nginx.conf
+
+worker_processes auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    # Prevent nginx HTTP Server Detection
+    server_tokens   off;
+
+    keepalive_timeout  65;
+
+    upstream php-handler {
+        server app:9000;
+    }
+
+    server {
+        listen 80;
+
+        # HSTS settings
+        # WARNING: Only add the preload option once you read about
+        # the consequences in https://hstspreload.org/. This option
+        # will add the domain to a hardcoded list that is shipped
+        # in all major browsers and getting removed from this list
+        # could take several months.
+        #add_header Strict-Transport-Security "max-age=15768000; includeSubDomains; preload;" always;
+
+        # set max upload size
+        client_max_body_size 512M;
+        fastcgi_buffers 64 4K;
+
+        # Enable gzip but do not remove ETag headers
+        gzip on;
+        gzip_vary on;
+        gzip_comp_level 4;
+        gzip_min_length 256;
+        gzip_proxied expired no-cache no-store private no_last_modified no_etag auth;
+        gzip_types application/atom+xml application/javascript application/json application/ld+json application/manifest+json application/rss+xml application/vnd.geo+json application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/bmp image/svg+xml image/x-icon text/cache-manifest text/css text/plain text/vcard text/vnd.rim.location.xloc text/vtt text/x-component text/x-cross-domain-policy;
+
+        # Pagespeed is not supported by Nextcloud, so if your server is built
+        # with the `ngx_pagespeed` module, uncomment this line to disable it.
+        #pagespeed off;
+
+        # HTTP response headers borrowed from Nextcloud `.htaccess`
+        add_header Referrer-Policy                      "no-referrer"   always;
+        add_header X-Content-Type-Options               "nosniff"       always;
+        add_header X-Download-Options                   "noopen"        always;
+        add_header X-Frame-Options                      "SAMEORIGIN"    always;
+        add_header X-Permitted-Cross-Domain-Policies    "none"          always;
+        add_header X-Robots-Tag                         "none"          always;
+        add_header X-XSS-Protection                     "1; mode=block" always;
+
+        # Remove X-Powered-By, which is an information leak
+        fastcgi_hide_header X-Powered-By;
+
+        # Path to the root of your installation
+        root /var/www/html;
+
+        # Specify how to handle directories -- specifying `/index.php$request_uri`
+        # here as the fallback means that Nginx always exhibits the desired behaviour
+        # when a client requests a path that corresponds to a directory that exists
+        # on the server. In particular, if that directory contains an index.php file,
+        # that file is correctly served; if it doesn't, then the request is passed to
+        # the front-end controller. This consistent behaviour means that we don't need
+        # to specify custom rules for certain paths (e.g. images and other assets,
+        # `/updater`, `/ocm-provider`, `/ocs-provider`), and thus
+        # `try_files $uri $uri/ /index.php$request_uri`
+        # always provides the desired behaviour.
+        index index.php index.html /index.php$request_uri;
+
+        # Rule borrowed from `.htaccess` to handle Microsoft DAV clients
+        location = / {
+            if ( $http_user_agent ~ ^DavClnt ) {
+                return 302 /remote.php/webdav/$is_args$args;
+            }
+        }
+
+        location = /robots.txt {
+            allow all;
+            log_not_found off;
+            access_log off;
+        }
+
+        # Make a regex exception for `/.well-known` so that clients can still
+        # access it despite the existence of the regex rule
+        # `location ~ /(\.|autotest|...)` which would otherwise handle requests
+        # for `/.well-known`.
+        location ^~ /.well-known {
+            # The rules in this block are an adaptation of the rules
+            # in `.htaccess` that concern `/.well-known`.
+
+            location = /.well-known/carddav { return 301 /remote.php/dav/; }
+            location = /.well-known/caldav  { return 301 /remote.php/dav/; }
+
+            location /.well-known/acme-challenge    { try_files $uri $uri/ =404; }
+            location /.well-known/pki-validation    { try_files $uri $uri/ =404; }
+
+            # Let Nextcloud's API for `/.well-known` URIs handle all other
+            # requests by passing them to the front-end controller.
+            return 301 /index.php$request_uri;
+        }
+
+        # Rules borrowed from `.htaccess` to hide certain paths from clients
+        location ~ ^/(?:build|tests|config|lib|3rdparty|templates|data)(?:$|/)  { return 404; }
+        location ~ ^/(?:\.|autotest|occ|issue|indie|db_|console)                { return 404; }
+
+        # Ensure this block, which passes PHP files to the PHP process, is above the blocks
+        # which handle static assets (as seen below). If this block is not declared first,
+        # then Nginx will encounter an infinite rewriting loop when it prepends `/index.php`
+        # to the URI, resulting in a HTTP 500 error response.
+        location ~ \.php(?:$|/) {
+            # Required for legacy support
+            rewrite ^/(?!index|remote|public|cron|core\/ajax\/update|status|ocs\/v[12]|updater\/.+|oc[ms]-provider\/.+|.+\/richdocumentscode\/proxy) /index.php$request_uri;
+
+            fastcgi_split_path_info ^(.+?\.php)(/.*)$;
+            set $path_info $fastcgi_path_info;
+
+            try_files $fastcgi_script_name =404;
+
+            include fastcgi_params;
+            fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
+            fastcgi_param PATH_INFO $path_info;
+            #fastcgi_param HTTPS on;
+
+            fastcgi_param modHeadersAvailable true;         # Avoid sending the security headers twice
+            fastcgi_param front_controller_active true;     # Enable pretty urls
+            fastcgi_pass php-handler;
+
+            fastcgi_intercept_errors on;
+            fastcgi_request_buffering off;
+        }
+
+        location ~ \.(?:css|js|svg|gif)$ {
+            try_files $uri /index.php$request_uri;
+            expires 6M;         # Cache-Control policy borrowed from `.htaccess`
+            access_log off;     # Optional: Don't log access to assets
+        }
+
+        location ~ \.woff2?$ {
+            try_files $uri /index.php$request_uri;
+            expires 7d;         # Cache-Control policy borrowed from `.htaccess`
+            access_log off;     # Optional: Don't log access to assets
+        }
+
+        # Rule borrowed from `.htaccess`
+        location /remote {
+            return 301 /remote.php$request_uri;
+        }
+
+        location / {
+            try_files $uri $uri/ /index.php$request_uri;
+        }
+
+        location ^~ /push/ {
+            proxy_pass http://notify_push:7867/;
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "Upgrade";
+            proxy_set_header Host $host;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+    }
+}

roles/alpina/templates/services/monitoring/docker-compose.yml.j2

@@ -79,11 +79,6 @@ services:
     image: gcr.io/cadvisor/cadvisor:latest
     container_name: cadvisor
     restart: unless-stopped
-    command:
-      - --docker_only=true
-      - --store_container_labels=false
-      - --whitelisted_container_labels=com.docker.compose.project,com.docker.compose.service
-      - --enable_metrics=cpu,cpuLoad,diskIO,memory,network,oom_event,process
     volumes:
       - /:/rootfs:ro
       - /var/run:/var/run:rw
@@ -104,6 +99,18 @@ services:
     volumes:
       - {{ base_volume_path }}/monitoring/influxdb:/var/lib/influxdb2
 
+  jaeger:
+    image: jaegertracing/all-in-one:1
+    container_name: jaeger
+    restart: unless-stopped
+    environment:
+      - COLLECTOR_OTLP_ENABLED=true
+    ports:
+      - 4317:4317
+      - 4318:4318
+    volumes:
+      - {{ base_volume_path }}/monitoring/jaeger:/jaeger
+
   uptime-kuma:
     image: louislam/uptime-kuma:1
     container_name: uptime-kuma

roles/alpina/templates/services/monitoring/grafana_config/provisioning/datasources/alpina.yaml.j2

@@ -27,3 +27,20 @@ datasources:
     secureJsonData:
       token: {{ influxdb_admin_token }}
     editable: false
+
+  - name: Jaeger
+    type: jaeger
+    access: proxy
+    uid: jaeger
+    url: http://jaeger:16686
+    editable: false
+    jsonData:
+      tracesToLogsV2:
+        datasourceUid: loki
+        spanStartTimeShift: 1h
+        spanEndTimeShift: -1h
+        tags: []
+        filterByTraceID: false
+        filterBySpanID: false
+        customQuery: true
+        query: '{container_name=~".*$${__span.tags["traefik.service.name"]}.*"}'

roles/alpina/templates/services/monitoring/loki_config/loki-config.yaml.j2

@@ -17,13 +17,13 @@ common:
 
 schema_config:
   configs:
-    - from: 2024-10-18
-      index:
-        period: 24h
-        prefix: index_
+    - from: 2020-10-24
+      store: boltdb-shipper
       object_store: filesystem
-      schema: v13
-      store: tsdb
+      schema: v11
+      index:
+        prefix: index_
+        period: 24h
 
 # TODO: Figure this out
 ruler:

roles/alpina/templates/services/traefik/traefik.yml.j2

@@ -22,6 +22,12 @@ metrics:
   prometheus:
     entryPoint: metrics
 
+tracing:
+  otlp:
+    grpc:
+      endpoint: "localhost:4317"
+      insecure: true
+
 certificatesResolvers:
   letsencrypt:
     acme: