WIP: monitoring improvements

monitoring: remove jaeger
Possible to add it back with a proper database, but all-in-one keeps everything in memory.
2024-10-11 18:15:34 -07:00 · 2024-07-25 13:16:52 -07:00 · 2024-06-30 18:57:41 -07:00 · 2024-06-30 13:53:46 -07:00
17 changed files with 276 additions and 337 deletions
--- a/.idea/jsonSchemas.xml
+++ b/.idea/jsonSchemas.xml
@@ -74,6 +74,9 @@
                  <Item>
                    <option name="path" value="file:///run/user/1000/kio-fuse-kipURF/sftp/root@debbi.lab.home/mnt/dock/traefik/rules/hello-world.yml" />
                  </Item>
+                  <Item>
+                    <option name="path" value="roles/alpina/templates/services/traefik/rules/traefik-dash.yml.j2" />
+                  </Item>
                </list>
              </option>
            </SchemaInfo>
--- a/README.md
+++ b/README.md
@@ -23,15 +23,16 @@ I'd like to get a /56 or /48, perhaps using Hurricane Electric's tunnel broker.
 ## Upgrading Postgres
 Upgrading the postgres container for a given stack requires a dump and restore.

-In the compose directory for a given stack, run the following commands:
+After making a snapshot or backup of postgres data directory,
+in the compose directory for a given stack, run the following commands:
 ```bash
 docker compose down
 docker compose up -d <db_service>
 docker compose exec -it <db_service> pg_dumpall -U <db_user> | tee /tmp/dump.sql
 docker compose down

+rm -r <postgres_data_dir>/*  # as root
 # Edit the docker-compose.yml file to use the new postgres image
-sudo rm -r <postgres_data_dir>/*  # TODO: this might not actually work with the sudo wildcard
 docker compose up -d <db_service>
 # For some reason, compose exec doesn't like the input redirection
 docker exec -i <db_container_name> psql -U <db_user> < /tmp/dump.sql
@@ -44,3 +45,15 @@ password hashes. This can be done by running the following command:
 ```bash
 docker compose exec -it <db_service> psql -U <db_user> -c "\password"
 ```
+
+## Nextcloud
+Nextcloud requires some additional work to set up notify_push.
+
+- Initially, comment out the notify_push service in the docker compose.
+- Set up nextcloud and install the Client Push (notify_push) app.
+- Uncomment the notify_push service in the docker compose and `up -d` the stack.
+- ```bash
+  docker compose exec app ./occ notify_push:setup https://nc.<domain>/push
+  ```
+
+I should probably get around to automating this at some point.
--- a/group_vars/alpina/vars.yml
+++ b/group_vars/alpina/vars.yml
@@ -24,6 +24,8 @@ minio_password: "{{ vault_minio_password }}"
 influxdb_admin_password: "{{ vault_influxdb_admin_password }}"
 influxdb_admin_token: "{{ vault_influxdb_admin_token }}"

+alertmanager_discord_webhook: "{{ vault_alertmanager_discord_webhook }}"
+
 # Traefik
 acme_email: "{{ vault_acme_email }}"
 cloudflare_api_token: "{{ vault_cloudflare_api_token }}"
--- a/group_vars/alpina/vault.yml
+++ b/group_vars/alpina/vault.yml
@@ -1,88 +1,96 @@
 $ANSIBLE_VAULT;1.1;AES256
-36636236366435333738633465323539336231393239656538643863643233346563333836623335
-3136393936656261396434316232356338313838373666660a653464613833306133343232623864
-61666561336462376664363463313533353238623031613664353063396236343663643936303730
-6235646336306636360a653238633038306532613436633132363231613862383636313838623461
-32633366326136346435613232396632396365656138643361643139353430663637353565383664
-36623961663030653639316131376535363138343965636437653139646233613765323439393030
-31666137346339663162393836636638636431326232323461353661613062623032306130393965
-38313931313935666633343835303232333961633232623538383138366262663335323764323939
-32373333663834626633363265373632356439633862316562323565646530383534653338353165
-38396434353332623164346137383238343536303130616666643065306431656137303263323135
-34316662353031653932396239623733313037383935383762623136346636323434363231623161
-30393864353466643637316566663366363231373335663331323932663837626239663633663965
-66333531323861663130353531323339386566303630366236636135393439356634393732623033
-31336231363935633436363962316666666336303338313636386163313666636336343464336133
-33313730303961663632323435323963663530623265663664343735643061323332343265343431
-61363039333730623562363233373537633138663239313132336666313237373137353663326538
-32366130326635366433393434653735616132366264386461363063393265623765666461626366
-38636239376534653230663932393930343162333262643130633835343363613061623932363761
-64643164323335376565646137643763316562343565366462376162333633313737303465373362
-63343734633536353661353165346632666230616138396461336332623365366432313734343837
-30613736313961663334326335333834336634373338326631313739363765303036303132346166
-37313030373264383564383936396339623061616134356663333733653838393537306336313135
-32336261356437653863653839373130323035346538343938646265653239376236373932646433
-35373932326535643763396563373138626239393661373231393066323335336264373835336635
-38393732643630336364363834303534663334396363623261383339313939663461303236646237
-36393330373534383836373065373239353836653137306338336638396662363434303839363466
-37303332343464663733653632363239366337656364333532313237633935616637333361383763
-62363063323362323565363837333264346161353032643039323839336666656333336433376231
-36363335626137366135373230613436653232663138343862623562306331336330356630316166
-30613264353165343634663461373630653632366333313837373237613339336638396338376465
-64633638373263376330343561303664666139663237326637663964386133623164626339346635
-66636365366562343636653362656133306164353761346661343430356633613063656466316262
-31633932313532663930303837353863333664393563646566396164666236633832633235653362
-63663931353436623034653733313766393465363466363831643130643939356335643166356436
-38386530333264313263636438376134666235646636316233653330613735323234313036356639
-61316164376434616239646235326661323363333835393430646462323234356138653163616530
-65623233636435396462343437626130353735643530376538633762346332653162353563386366
-32656633633935626238323431643631633434633032303435383037353834653964326336616530
-30363765663133313239373664383830393238303439653531316664636532363135636563356666
-34376636373033353665373261363536393562653638306661663832326139383565613862333831
-38616238616332326532656430393331383161376237393365666639363732363164306332343336
-37366638326464373261386431623731306663616262633837313965633530616265326536323136
-62366365666461383535663637633332626464643062653139623333663038316536353930653266
-37343830613062346533613762663738343138383537396435643765323237623130363564396462
-61663063643135303539313062396338353061346336303938626361343238366366393533363638
-31313437623631626437393761366537636664393863306164373431653133316639623630353336
-65313037636533393362363266366231393334613264343331623531393666336336626265366163
-34663161396633666162326564313735373137303337386538633866653331646635633532336465
-34386166373436386566656135313438363733353139663630613430363332656239356139393532
-35626337666639376664346631323938316538333066353363646562323266353165366632656137
-66366162376165626564363230353062666364646363366637666433636333316536623435623836
-62346566363362363939353038396566653238666138666531396338323262323965383031336362
-34613332363334653531383231363539343133333531666564386133346562323338366139663438
-31613466366438643566333632326239653662636464373337326537313234393038306132343730
-36633136366162643966396362643165313336383862653435343630646431306366656636353230
-64326633346561613662383863356531306563623439363566643733336535303335303164633535
-36356463616162313039386434323637383937613133623131373033373462363365643730666166
-65383166346638313533326366346433656461346439343838306564393336383536633732343965
-39306231386130303433616361366363366163646534316138623362393063663438313165643762
-39393332653564333762663762366633386135353865366338396138666265653662373535666366
-35613937613366323064316561643435353830316239396464393737613835373964626437316464
-39643664656565633966393832643033323130636562383233323636363361353430353062323439
-39396464633336623963633963326461316562333162333766613064336462613235336531623437
-30383063653666633839646533386239366637346230363033306161386537303039376465303535
-34643162323065326264343662303138313063303834353832393663616239383739313133393532
-62393766343037666564326132386139346661383564366366646530346434373366326531356138
-31323531653338653130303733363764636430336563336439666132626434363463306631363334
-39623332376334383338633132653262653735346563626365613336623435396539383630366332
-31316638393562376131363166633163333332633332393062393962613132366538653865663264
-38313237393436353333323431336361653938343034346164353335366535396265633961333138
-65386137356161643732636531613166633464326163303336303439383435376331373935333563
-64633961623761393131333234656530653737346563643963643833383262383434653266343362
-35623832643032346133346363646136646438663761363330666231316434306232623339656535
-34393337666237656262313439386336336466373466663663616139353463316265396135626366
-62313562306334343831616364633933343463386233323637313832316635346235623830333461
-33663530343966383739643261653736363865323438363430653661653964643339633833386438
-36333331366334366461346636636462343335313234663864613864366134356161396662383632
-36663538373761353937313666363262626435623537646665646364353934373638366261333234
-36353439303663656531666637376364313838386130343966316138356338643135316139363630
-30386635376565363931333331336431303562346431323534643238333337386264616161356163
-35663766306635626235373663643064393233346364666663393236353561653362373361666164
-65653566666234626464356338613834323332383939643935323337376162316163333034643062
-63646237646234636561313038383636373936656164333735323461626233633337623764383830
-66383161346336633962643032376662656566396666343662656337306333313836613335643961
-64323961663032373239636430306430383639306333363938303837386139643230353061623937
-36373733636337616264313432643230303935626666633533666135666538626266626266643864
-376430653461346366626432636336653437
+32653863663065353431636364373163613536643238613961666561653663633530646165643766
+3833323937353331313136633965393061616135366534660a333037383066303431623830313464
+65346431633238666534373033663138353438313762326361666233353866663534363536643034
+3636323439316261630a623262336331663431633266336235653034323234383566323963623365
+32626363626164373536663464643632393761346137623866633237643038306265636362626561
+61313634353634373530383061393364613461303132326335316566326436633635633131643433
+31376539396639326464333233643933373737313064363262323639363964643862633035396161
+35643037636535623966626131393538643432396536643365383736636262356135373434376433
+32316361343330303431376234323632323932376635343964383733633761326639393966383039
+35646131343034663962363335373661323065663764396631343461383661663738386163323633
+36303464646532633235663662666663343238633465663334326463383133643239666634653739
+35396130393961303230396236303766336666643930626161333338326137663235323066663032
+33376564373563323635356233616264313663373534333636643236393866613062656338353864
+66386132663362363832366661646462316139353132626662663934336530386534376538633235
+62653131653835323261373435373631396466353738306362616266616532313435323633613933
+61646132346536323632643865326234356535346566346532383162393265613931343962303463
+31636334343736666434353835633734396465653862613234386431306463326134613931646232
+32353535663133623434643866336165616232613662336533383432633338373763643337616637
+38323237646461376433316164646366383438316639633162303739383263656265633364303565
+36643339356136653332666230633939636264306431636562323864373037623138363739616561
+37613364653737353638646564323439646138646536636564303866636233616264383466656439
+33646232653061616437656162353036313834616162313936353533393833313432656534343363
+35636638326236646163323463356634326534623165306461316530353936646162323435633862
+64396464303363323837316162353734626663643962303534336637336632333463393734383532
+66616534666466393333386337363238383432643764373864613461363766333932333862363332
+61313364613031376334326635636432346532613462613265643462636436663963323862353733
+38396261613332396633666130653262313234633132353264363266336231373535306532383661
+65323530653531646339626537653433303332656535346639393466353133363833326236656231
+33336265373463396135653730616266346331376461346433343464326238323034653330393732
+36643432316662333633333036633761653031393433333338663633386264656535623534653463
+36363565303333356361616539376532353066336137336134656465383364636361656664356439
+65326334643631663665376530646433323439653864623964323363396561313663636538356536
+63626336303862333364363166353437353163656238303765636662636137383337623563666264
+66326633343230386638616438393436633431343264343231386563613935626430306337343533
+66656366333332326131343661356236396430303832303834653530623639353036663436373862
+61336437386338343965653563646664643438353232306231316564616462643236646239333062
+38643461346639623964626438396631396139383332666130316635656530653136333662353566
+36313261646330373963663032316662383137366436636534383366636362366435393036373264
+34646537666462363531343335336638343038333633663862666163306662643634326533316561
+61613235366233636530663462353066646530386265623534663336376364323237343936646134
+31616563653864383565306439613932396562613835613562326264326535636630646666366335
+36653631353961353933386236636534393636356334633336313333383238353838336335646630
+63633365666530623562323634303935326362643762616532303531303139333565643835396163
+36353130656365326435343130613234336637346461313639653133623933376163393935366266
+66653337353732363038663164363663623266356366663637343466393836353965343730666362
+38663636336265383331666666616535366334616431306164303738306436333364653765356662
+37316433323563323431623164386337343563663538333435616333343433396236356363333262
+61396664326234343136666331356465333233663135613839616334623033316362336162613731
+38646530326538643337323838326563303130643934623939346635343331356531373235663937
+62396530383365666439373632613633633233376139616138323033613135383330333132643839
+65363833616337656662653462323436303531653635663739633366616532333761323238353764
+39373836303735393165393435323139346661346135636138613731373165386533386333393364
+32336265386334386338653734353565343733393931373436336233333031356531313739636666
+61376234393631343236643137616631373564376132623534333939346162353662306661393438
+32326566373934653463653737383131386431363664333535626361646637613632383132623533
+32343465366562363765353366333330633631353936613930376631336538306230626632303966
+31343936386535663165663066663862656439306363326337313561396132316338363930323632
+33313061623534373338623931663934396339633564353533626639373837323832366132343538
+63373862663137306665383732303863343564343830636233613139666631626532373938386663
+35646331646462356639383964373732393866653963643832633661323430323430613330633364
+35343262366362646165383032333236623863656264353964623136643631326135623538306261
+37393839343331653665356131343063316232303963636462653238333466636334616435666463
+65636662383930353238623130363834616137643830633261646338363435343839633565303562
+37623231396163346464303464333962336261353634396236613132306464643764356265656137
+32373263613964396430646332666235303634373431643939623963633334326135626565656662
+30646166303732643562653166633232666635343665616665653566316632303861613861313333
+38393636663137333231613239353661656338333536656563616237343234623031363535666637
+61343662663965663161666436366630366432363733663537613064386130326466343366383232
+32363662343561666665323565356163383932336361656132373263363239636666613461366339
+31323264393866386239353333386161643330343262366666323533303737373163313262313766
+61303638366263346232353134333431613730386431623235323537323962666133613939353762
+63326361633630323937353163383930626336663365626532613031623532393932316138353335
+32363262393764663135393466616639373965313238323935383531633434633038663437646662
+31633265373937316533373332316132363061386133356231623230393739326464333761336338
+38626234646164616265633061346239363164376532383834356435346232653065326362343363
+39613532356166633133626563643238373661323937353635343464666339323561326136623366
+62633637656462376136633963653263346565366563646533373431613761616231653739613537
+32343332356435393635363837396463613165626337346235303363613764306132343539333836
+63386633626332396339383165303166653334663239313066666632356165643161356262346230
+32636365636364663466343939663538386439343336303537636230306263643534653339313538
+31373165363962373337636138336561336638633762373363646139366339323031313664306534
+30623130663037323839666166323162393065643535663866383062356330633137343239316436
+32303132393739653363376138633430313832383165663366626436653033663637616664346632
+63633439663734393236343265323533633639316133323336373064633138363266316135363335
+31336637666331333139306537333565333064666433653730633430336261656665613263663937
+64313230656333373838346439623061393164393239393934306336373063303934663334353532
+31313637623466313835313566616161376230343532653561343364383133653736646338303631
+36356164303630303433356332343630616465383831623036383833393330663566616333653161
+63393361643266323336393962663263323338633634633033393762656139393665353630633637
+39386462303731396261613961613238616237373332656361303139633763303837653765623464
+64333565666532653864383861333433353731343161613231383836353966353636373762306132
+35333536373939656638356333383135313231306433656536383933623634653263353434393238
+32323037666135316337633465666335376332326633346665643333656139386465353134356636
+36333434303538326135346539313734393939353163316666366438613133333464623732666438
+663934323030303937623038343662646163
--- a/roles/alpina/tasks/deploy_compose_stack.yml
+++ b/roles/alpina/tasks/deploy_compose_stack.yml
@@ -10,7 +10,7 @@
  file:
    path: "{{ current_stack_dest }}/{{ item.path }}"
    state: directory
-    mode: "700"
+    mode: "755"
  loop: "{{ lookup('community.general.filetree', current_stack_source) }}"
  when: item.state == "directory"

@@ -18,7 +18,7 @@
  template:
    src: "{{ item.src }}"
    dest: "{{ current_stack_dest }}/{{ item.path | regex_replace('\\.j2$', '') }}"
-    mode: "600"
+    mode: "644"
  loop: "{{ lookup('community.general.filetree', current_stack_source) }}"
  when: item.state == "file"

@@ -30,5 +30,5 @@
    remove_orphans: yes
  register: docker_compose_output

- debug:
-    var: docker_compose_output
+# - debug:
+#     var: docker_compose_output
--- a/roles/alpina/templates/apps/nextcloud/.env.j2
+++ b/roles/alpina/templates/apps/nextcloud/.env.j2
@@ -1 +1 @@
-NEXTCLOUD_VERSION=29-fpm-alpine
+NEXTCLOUD_VERSION=29-apache
--- a/roles/alpina/templates/apps/nextcloud/.env.nextcloud.j2
+++ b/roles/alpina/templates/apps/nextcloud/.env.nextcloud.j2
@@ -17,7 +17,8 @@ SMTP_PASSWORD={{ nextcloud_sendgrid_api_key }}
 MAIL_FROM_ADDRESS=nc
 MAIL_DOMAIN=cazzzer.com

-TRUSTED_PROXIES={{ traefik_subnet }}
+# host IPv4 and IPv6 addresses, loopback for notify_push
+TRUSTED_PROXIES={{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] }} {{ hostvars[inventory_hostname]['ansible_default_ipv6']['address'] }} 127.0.0.1 ::1
 OVERWRITEHOST=nc.{{ domain }}
 OVERWRITEPROTOCOL=https
 OVERWRITECLIURL=https://nc.{{ domain }}
--- a/roles/alpina/templates/apps/nextcloud/.env.notify_push
+++ b/roles/alpina/templates/apps/nextcloud/.env.notify_push
@@ -0,0 +1,4 @@
+DATABASE_URL=postgres://nextcloud:{{ nextcloud_db_password }}@db/nextcloud
+DATABASE_PREFIX=oc_
+REDIS_URL=redis://:{{ redis_password }}@redis
+NEXTCLOUD_URL=http://localhost
--- a/roles/alpina/templates/apps/nextcloud/docker-compose.yml.j2
+++ b/roles/alpina/templates/apps/nextcloud/docker-compose.yml.j2
@@ -10,6 +10,8 @@ services:
  app:
    image: nextcloud:${NEXTCLOUD_VERSION}
    container_name: nextcloud_app
+    labels:
+      - {{ helpers.traefik_labels('nc', port='80') | indent(6) }}
    restart: unless-stopped
    depends_on:
      - db
@@ -40,18 +42,25 @@ services:
  notify_push:
    image: nextcloud:${NEXTCLOUD_VERSION}
    container_name: nextcloud_notify_push
+    {# TODO: Refactor this and minio -#}
+    labels:
+      - traefik.enable=true
+      - traefik.http.routers.nc-notify.rule=Host(`nc.{{ domain }}`) && PathPrefix(`/push`)
+      - traefik.http.routers.nc-notify.entrypoints=websecure
+      - traefik.http.routers.nc-notify.tls=true
+      - traefik.http.routers.nc-notify.tls.certresolver=letsencrypt
+      - traefik.http.routers.nc-notify.tls.domains.0.main={{ domain }}
+      - traefik.http.routers.nc-notify.tls.domains.0.sans=*.{{ domain }}
+      - traefik.http.services.nc-notify.loadbalancer.server.port=7867
    restart: unless-stopped
-    depends_on:
-      - app
+    user: www-data
+    env_file:
+      - .env.notify_push
+    network_mode: service:app
    entrypoint:
      - /var/www/html/custom_apps/notify_push/bin/x86_64/notify_push
-      - /var/www/html/config/config.php
-    networks:
-      - default
    volumes:
      - {{ base_volume_path }}/nextcloud/nextcloud:/var/www/html
-      - {{ base_volume_path }}/nextcloud/nextcloud_config:/var/www/html/config
-      - {{ base_volume_path }}/nextcloud/nextcloud_data:/var/www/html/data

  db:
    image: postgres:16-alpine
@@ -65,7 +74,7 @@ services:
      - {{ base_volume_path }}/nextcloud/db:/var/lib/postgresql/data

  redis:
-    image: redis:7-alpine
+    image: redis:alpine
    container_name: nextcloud_redis
    restart: unless-stopped
    env_file:
@@ -76,20 +85,3 @@ services:
      - sh
      - -c
      - redis-server --requirepass $$REDIS_PASSWORD
-
-  web:
-    image: nginx:1.23-alpine
-    container_name: nextcloud_web
-    labels:
-      - {{ helpers.traefik_labels('nc') | indent(6) }}
-    restart: unless-stopped
-    links:
-      - app
-    networks:
-      - traefik_traefik
-      - default
-    volumes:
-      - ./nginx.conf:/etc/nginx/nginx.conf:ro
-      - {{ base_volume_path }}/nextcloud/nextcloud:/var/www/html
-      - {{ base_volume_path }}/nextcloud/nextcloud_config:/var/www/html/config
-      - {{ base_volume_path }}/nextcloud/nextcloud_data:/var/www/html/data
--- a/roles/alpina/templates/apps/nextcloud/nginx.conf.j2
+++ b/roles/alpina/templates/apps/nextcloud/nginx.conf.j2
@@ -1,182 +0,0 @@
-# https://github.com/nextcloud/docker/blob/master/.examples/docker-compose/with-nginx-proxy/postgres/fpm/web/nginx.conf
-
-worker_processes auto;
-
-error_log  /var/log/nginx/error.log warn;
-pid        /var/run/nginx.pid;
-
-
-events {
-    worker_connections  1024;
-}
-
-
-http {
-    include       /etc/nginx/mime.types;
-    default_type  application/octet-stream;
-
-    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
-                      '$status $body_bytes_sent "$http_referer" '
-                      '"$http_user_agent" "$http_x_forwarded_for"';
-
-    access_log  /var/log/nginx/access.log  main;
-
-    sendfile        on;
-    #tcp_nopush     on;
-
-    # Prevent nginx HTTP Server Detection
-    server_tokens   off;
-
-    keepalive_timeout  65;
-
-    upstream php-handler {
-        server app:9000;
-    }
-
-    server {
-        listen 80;
-
-        # HSTS settings
-        # WARNING: Only add the preload option once you read about
-        # the consequences in https://hstspreload.org/. This option
-        # will add the domain to a hardcoded list that is shipped
-        # in all major browsers and getting removed from this list
-        # could take several months.
-        #add_header Strict-Transport-Security "max-age=15768000; includeSubDomains; preload;" always;
-
-        # set max upload size
-        client_max_body_size 512M;
-        fastcgi_buffers 64 4K;
-
-        # Enable gzip but do not remove ETag headers
-        gzip on;
-        gzip_vary on;
-        gzip_comp_level 4;
-        gzip_min_length 256;
-        gzip_proxied expired no-cache no-store private no_last_modified no_etag auth;
-        gzip_types application/atom+xml application/javascript application/json application/ld+json application/manifest+json application/rss+xml application/vnd.geo+json application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/bmp image/svg+xml image/x-icon text/cache-manifest text/css text/plain text/vcard text/vnd.rim.location.xloc text/vtt text/x-component text/x-cross-domain-policy;
-
-        # Pagespeed is not supported by Nextcloud, so if your server is built
-        # with the `ngx_pagespeed` module, uncomment this line to disable it.
-        #pagespeed off;
-
-        # HTTP response headers borrowed from Nextcloud `.htaccess`
-        add_header Referrer-Policy                      "no-referrer"   always;
-        add_header X-Content-Type-Options               "nosniff"       always;
-        add_header X-Download-Options                   "noopen"        always;
-        add_header X-Frame-Options                      "SAMEORIGIN"    always;
-        add_header X-Permitted-Cross-Domain-Policies    "none"          always;
-        add_header X-Robots-Tag                         "none"          always;
-        add_header X-XSS-Protection                     "1; mode=block" always;
-
-        # Remove X-Powered-By, which is an information leak
-        fastcgi_hide_header X-Powered-By;
-
-        # Path to the root of your installation
-        root /var/www/html;
-
-        # Specify how to handle directories -- specifying `/index.php$request_uri`
-        # here as the fallback means that Nginx always exhibits the desired behaviour
-        # when a client requests a path that corresponds to a directory that exists
-        # on the server. In particular, if that directory contains an index.php file,
-        # that file is correctly served; if it doesn't, then the request is passed to
-        # the front-end controller. This consistent behaviour means that we don't need
-        # to specify custom rules for certain paths (e.g. images and other assets,
-        # `/updater`, `/ocm-provider`, `/ocs-provider`), and thus
-        # `try_files $uri $uri/ /index.php$request_uri`
-        # always provides the desired behaviour.
-        index index.php index.html /index.php$request_uri;
-
-        # Rule borrowed from `.htaccess` to handle Microsoft DAV clients
-        location = / {
-            if ( $http_user_agent ~ ^DavClnt ) {
-                return 302 /remote.php/webdav/$is_args$args;
-            }
-        }
-
-        location = /robots.txt {
-            allow all;
-            log_not_found off;
-            access_log off;
-        }
-
-        # Make a regex exception for `/.well-known` so that clients can still
-        # access it despite the existence of the regex rule
-        # `location ~ /(\.|autotest|...)` which would otherwise handle requests
-        # for `/.well-known`.
-        location ^~ /.well-known {
-            # The rules in this block are an adaptation of the rules
-            # in `.htaccess` that concern `/.well-known`.
-
-            location = /.well-known/carddav { return 301 /remote.php/dav/; }
-            location = /.well-known/caldav  { return 301 /remote.php/dav/; }
-
-            location /.well-known/acme-challenge    { try_files $uri $uri/ =404; }
-            location /.well-known/pki-validation    { try_files $uri $uri/ =404; }
-
-            # Let Nextcloud's API for `/.well-known` URIs handle all other
-            # requests by passing them to the front-end controller.
-            return 301 /index.php$request_uri;
-        }
-
-        # Rules borrowed from `.htaccess` to hide certain paths from clients
-        location ~ ^/(?:build|tests|config|lib|3rdparty|templates|data)(?:$|/)  { return 404; }
-        location ~ ^/(?:\.|autotest|occ|issue|indie|db_|console)                { return 404; }
-
-        # Ensure this block, which passes PHP files to the PHP process, is above the blocks
-        # which handle static assets (as seen below). If this block is not declared first,
-        # then Nginx will encounter an infinite rewriting loop when it prepends `/index.php`
-        # to the URI, resulting in a HTTP 500 error response.
-        location ~ \.php(?:$|/) {
-            # Required for legacy support
-            rewrite ^/(?!index|remote|public|cron|core\/ajax\/update|status|ocs\/v[12]|updater\/.+|oc[ms]-provider\/.+|.+\/richdocumentscode\/proxy) /index.php$request_uri;
-
-            fastcgi_split_path_info ^(.+?\.php)(/.*)$;
-            set $path_info $fastcgi_path_info;
-
-            try_files $fastcgi_script_name =404;
-
-            include fastcgi_params;
-            fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
-            fastcgi_param PATH_INFO $path_info;
-            #fastcgi_param HTTPS on;
-
-            fastcgi_param modHeadersAvailable true;         # Avoid sending the security headers twice
-            fastcgi_param front_controller_active true;     # Enable pretty urls
-            fastcgi_pass php-handler;
-
-            fastcgi_intercept_errors on;
-            fastcgi_request_buffering off;
-        }
-
-        location ~ \.(?:css|js|svg|gif)$ {
-            try_files $uri /index.php$request_uri;
-            expires 6M;         # Cache-Control policy borrowed from `.htaccess`
-            access_log off;     # Optional: Don't log access to assets
-        }
-
-        location ~ \.woff2?$ {
-            try_files $uri /index.php$request_uri;
-            expires 7d;         # Cache-Control policy borrowed from `.htaccess`
-            access_log off;     # Optional: Don't log access to assets
-        }
-
-        # Rule borrowed from `.htaccess`
-        location /remote {
-            return 301 /remote.php$request_uri;
-        }
-
-        location / {
-            try_files $uri $uri/ /index.php$request_uri;
-        }
-
-        location ^~ /push/ {
-            proxy_pass http://notify_push:7867/;
-            proxy_http_version 1.1;
-            proxy_set_header Upgrade $http_upgrade;
-            proxy_set_header Connection "Upgrade";
-            proxy_set_header Host $host;
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        }
-    }
-}
--- a/roles/alpina/templates/services/monitoring/.env.alertmanager.j2
+++ b/roles/alpina/templates/services/monitoring/.env.alertmanager.j2
@@ -0,0 +1 @@
+DISCORD_WEBHOOK={{ alertmanager_discord_webhook }}
--- a/roles/alpina/templates/services/monitoring/alertmanager_config/alertmanager.yml.j2
+++ b/roles/alpina/templates/services/monitoring/alertmanager_config/alertmanager.yml.j2
@@ -0,0 +1,68 @@
+# The root route on which each incoming alert enters.
+route:
+  group_by: ["alertname", "job"]
+  group_wait: 20s
+  group_interval: 5m
+  repeat_interval: 3h
+  receiver: discord_webhook
+
+receivers:
+  - name: "discord_webhook"
+    discord_configs:
+      - webhook_url: "{{ alertmanager_discord_webhook }}"
+{#        - send_resolved: true#}
+{#            username: 'Alertmanager'#}
+{#            webhook_configs:#}
+{#            - send_resolved: true#}
+{#                url: '{{ alertmanager_discord_webhook }}'#}
+{#                username: 'Alertmanager'#}
+{#                icon_url: 'https://prometheus.io/assets/icon.png'#}
+{#                icon_emoji: ':alert:'#}
+{#                send_resolved: true#}
+{#                text: "{{ .CommonAnnotations.summary }}"#}
+{#                title: "{{ .CommonLabels.alertname }}"#}
+{#                color: '{{ if eq .Status "firing" }}#FF0000{{ else }}#00FF00{{ end }}'#}
+{#                footer: '{{ .CommonLabels.monitor }}'#}
+{#                footer_icon: 'https://prometheus.io/assets/icon.png'#}
+{#                actions:#}
+{#                - type: 'button'#}
+{#                    text: 'Open in Grafana'#}
+{#                    url: '{{ .ExternalURL }}'#}
+{#                    style: 'primary'#}
+{#                    send_resolved: true#}
+{#                    confirm:#}
+{#                    title: 'Are you sure?'#}
+{#                    text: 'This will open Grafana in a new tab.'#}
+{#                    ok_text: 'Yes'#}
+{#                    dismiss_text: 'No'#}
+{#                fields:#}
+{#                - title: 'Description'#}
+{#                    value: "{{ .CommonAnnotations.description }}"#}
+{#                    short: false#}
+{#                - title: 'Details'#}
+{#                    value: "{{ .CommonAnnotations.details }}"#}
+{#                    short: false#}
+{#                - title: 'Severity'#}
+{#                    value: '{{ if eq .Labels.severity "critical" }}Critical{{ else if eq .Labels.severity "warning" }}Warning{{ else }}Info{{ end }}'#}
+{#                    short: true#}
+{#                - title: 'Host'#}
+{#                    value: '{{ .CommonLabels.monitor }}'#}
+{#                    short: true#}
+{#                - title: 'Starts At'#}
+{#                    value: '{{ .StartsAt.Format "2006-01-02 15:04:05" }}'#}
+{#                    short: true#}
+{#                - title: 'Ends At'#}
+{#                    value: '{{ .EndsAt.Format "2006-01-02 15:04:05" }}'#}
+{#                    short: true#}
+{#                - title: 'Runbook'#}
+{#                    value: '{{ .CommonAnnotations.runbook_url }}'#}
+{#                    short: true#}
+{#                - title: 'Dashboard'#}
+{#                    value: '{{ .CommonAnnotations.dashboard_url }}'#}
+{#                    short: true#}
+{#                - title: 'Alerting Rule'#}
+{#                    value: '{{ .CommonLabels.alertname }}'#}
+{#                    short: true#}
+{#                - title: 'Alerting Rule Description'#}
+{#                    value: '{{ .CommonLabels.alertname }}'#}
+{#                    short: true#}
--- a/roles/alpina/templates/services/monitoring/docker-compose.yml.j2
+++ b/roles/alpina/templates/services/monitoring/docker-compose.yml.j2
@@ -66,6 +66,24 @@ services:
      - {{ base_volume_path }}/monitoring/prometheus_configs:/etc/prometheus/extra:ro
      - {{ base_volume_path }}/monitoring/prometheus:/prometheus

+  alertmanager:
+    image: prom/alertmanager:latest
+    container_name: alertmanager
+    restart: unless-stopped
+    # Needed to make config files readable
+    # user: "{{ remote_uid }}"
+    command:
+      - --config.file=/etc/alertmanager/alertmanager.yml
+    volumes:
+      - ./alertmanager_config:/etc/alertmanager:ro
+
+#  alerts-discord:
+#    image: rogerrum/alertmanager-discord:1.0.6
+#    container_name: alerts-discord
+#    restart: unless-stopped
+#    env_file:
+#      - .env.alertmanager
+
  node-exporter:
    image: prom/node-exporter:latest
    container_name: node-exporter
@@ -99,18 +117,6 @@ services:
    volumes:
      - {{ base_volume_path }}/monitoring/influxdb:/var/lib/influxdb2

-  jaeger:
-    image: jaegertracing/all-in-one:1
-    container_name: jaeger
-    restart: unless-stopped
-    environment:
-      - COLLECTOR_OTLP_ENABLED=true
-    ports:
-      - 4317:4317
-      - 4318:4318
-    volumes:
-      - {{ base_volume_path }}/monitoring/jaeger:/jaeger
-
  uptime-kuma:
    image: louislam/uptime-kuma:1
    container_name: uptime-kuma
--- a/roles/alpina/templates/services/monitoring/grafana_config/provisioning/datasources/alpina.yaml.j2
+++ b/roles/alpina/templates/services/monitoring/grafana_config/provisioning/datasources/alpina.yaml.j2
@@ -15,6 +15,19 @@ datasources:
    url: http://prometheus:9090
    editable: false

+  - name: Alertmanager
+    type: alertmanager
+    access: proxy
+    uid: alertmanager
+    url: http://alertmanager:9093
+    jsonData:
+      # Valid options for implementation include mimir, cortex and prometheus
+      implementation: prometheus
+      # Whether Grafana should send alert instances to this Alertmanager
+      ha
+      ndleGrafanaManagedAlerts: false
+    editable: false
+
  - name: InfluxDB
    type: influxdb
    access: proxy
@@ -27,20 +40,3 @@ datasources:
    secureJsonData:
      token: {{ influxdb_admin_token }}
    editable: false
-
-  - name: Jaeger
-    type: jaeger
-    access: proxy
-    uid: jaeger
-    url: http://jaeger:16686
-    editable: false
-    jsonData:
-      tracesToLogsV2:
-        datasourceUid: loki
-        spanStartTimeShift: 1h
-        spanEndTimeShift: -1h
-        tags: []
-        filterByTraceID: false
-        filterBySpanID: false
-        customQuery: true
-        query: '{container_name=~".*$${__span.tags["traefik.service.name"]}.*"}'
--- a/roles/alpina/templates/services/monitoring/prometheus_config/demo-alerts.yml.j2
+++ b/roles/alpina/templates/services/monitoring/prometheus_config/demo-alerts.yml.j2
@@ -0,0 +1,20 @@
+groups:
+  - name: demo-service-alerts
+    rules:
+      - alert: DemoServiceHighErrorRate
+        expr: |
+          (
+            sum without(status, instance) (
+              rate(demo_api_request_duration_seconds_count{status=~"5..",job="demo"}[1m])
+            )
+          /
+            sum without(status, instance) (
+              rate(demo_api_request_duration_seconds_count{job="demo"}[1m])
+            ) * 100 > 0.5
+          )
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          title: 'High 5xx rate for {{'{{ $labels.method }}'}} on {{'{{ $labels.path }}'}}'
+          description: 'The 5xx error rate for path {{'{{ $labels.path }}'}} with method {{'{{ $labels.method }}'}} in {{'{{ $labels.job }}'}} is {{'{{ printf "%.2f" $value }}'}}%.'
--- a/roles/alpina/templates/services/monitoring/prometheus_config/prometheus.yml.j2
+++ b/roles/alpina/templates/services/monitoring/prometheus_config/prometheus.yml.j2
@@ -5,6 +5,11 @@ global:
  external_labels:
    monitor: "{{ ansible_host }}"

+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets: ["alertmanager:9093"]
+
 scrape_configs:
  - job_name: "prometheus"
    static_configs:
@@ -30,7 +35,15 @@ scrape_configs:
    static_configs:
      - targets: ["promtail:9080"]

+  - job_name: 'demo'
+    static_configs:
+      - targets:
+          - 'demo.promlabs.com:10000'
+          - 'demo.promlabs.com:10001'
+          - 'demo.promlabs.com:10002'
+
 rule_files:
+  - "/etc/prometheus/demo-alerts.yml"
  - "/etc/prometheus/extra/rules/*.yml"
  - "/etc/prometheus/extra/rules/*.json"

--- a/roles/alpina/templates/services/traefik/traefik.yml.j2
+++ b/roles/alpina/templates/services/traefik/traefik.yml.j2
@@ -22,12 +22,6 @@ metrics:
  prometheus:
    entryPoint: metrics

-tracing:
-  otlp:
-    grpc:
-      endpoint: "localhost:4317"
-      insecure: true
-
 certificatesResolvers:
  letsencrypt:
    acme:
Author	SHA1	Message	Date
Yuri Tatishchev	6056add4d6	WIP: monitoring improvements	2024-10-11 18:15:34 -07:00
Iurii Tatishchev	aaca0f94f8	monitoring: remove jaeger Possible to add it back with a proper database, but all-in-one keeps everything in memory.	2024-07-25 13:16:52 -07:00
Iurii Tatishchev	97b812eb10	updates: nextcloud refactor to apache image instead of nginx	2024-06-30 18:57:41 -07:00
Iurii Tatishchev	97d1db61d8	updates: postgres upgrade, data migration process	2024-06-30 13:53:46 -07:00
				`@@ -0,0 +1 @@`
				`DISCORD_WEBHOOK={{ alertmanager_discord_webhook }}`