(WIP) monitoring: attempt at container alerts

(WIP) monitoring: containers dashboard, loki storage update
WIP: containers dashboard - loki logs
2024-10-22 23:07:41 -07:00 · 2024-10-22 23:07:08 -07:00 · 2024-10-12 23:32:03 -07:00 · 2024-10-12 23:31:09 -07:00 · 2024-10-12 02:28:52 -07:00
29 changed files with 493 additions and 961 deletions
--- a/.idea/alpina.iml
+++ b/.idea/alpina.iml
@@ -4,7 +4,7 @@
    <content url="file://$MODULE_DIR$">
      <excludeFolder url="file://$MODULE_DIR$/venv" />
    </content>
-    <orderEntry type="inheritedJdk" />
+    <orderEntry type="jdk" jdkName="Poetry (alpina) (4)" jdkType="Python SDK" />
    <orderEntry type="sourceFolder" forTests="false" />
  </component>
  <component name="PyDocumentationSettings">
--- a/.idea/jsonSchemas.xml
+++ b/.idea/jsonSchemas.xml
@@ -31,7 +31,7 @@
                <list>
                  <Item>
                    <option name="directory" value="true" />
-                    <option name="path" value="roles/alpina/templates/services/authentik/blueprints" />
+                    <option name="path" value="roles/alpina/collections/services/authentik/templates/blueprints" />
                    <option name="mappingKind" value="Directory" />
                  </Item>
                </list>
@@ -39,22 +39,6 @@
            </SchemaInfo>
          </value>
        </entry>
-        <entry key="Loki">
-          <value>
-            <SchemaInfo>
-              <option name="name" value="Loki" />
-              <option name="relativePathToSchema" value="https://json.schemastore.org/loki.json" />
-              <option name="applicationDefined" value="true" />
-              <option name="patterns">
-                <list>
-                  <Item>
-                    <option name="path" value="roles/alpina/templates/services/monitoring/loki_config/loki-config.yaml.j2" />
-                  </Item>
-                </list>
-              </option>
-            </SchemaInfo>
-          </value>
-        </entry>
        <entry key="Traefik v2">
          <value>
            <SchemaInfo>
@@ -140,6 +124,25 @@
            </SchemaInfo>
          </value>
        </entry>
+        <entry key="prometheus.rules.json">
+          <value>
+            <SchemaInfo>
+              <option name="name" value="prometheus.rules.json" />
+              <option name="relativePathToSchema" value="https://json.schemastore.org/prometheus.rules.json" />
+              <option name="applicationDefined" value="true" />
+              <option name="patterns">
+                <list>
+                  <Item>
+                    <option name="path" value="roles/alpina/templates/services/monitoring/prometheus_config/container-alerts.yml" />
+                  </Item>
+                  <Item>
+                    <option name="path" value="roles/alpina/templates/services/monitoring/prometheus_config/container.alerts.yml" />
+                  </Item>
+                </list>
+              </option>
+            </SchemaInfo>
+          </value>
+        </entry>
      </map>
    </state>
  </component>
--- a/.idea/misc.xml
+++ b/.idea/misc.xml
@@ -3,5 +3,5 @@
  <component name="Black">
    <option name="sdkName" value="Poetry (alpina) (2)" />
  </component>
-  <component name="ProjectRootManager" version="2" project-jdk-name="Poetry (alpina)" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="Poetry (alpina) (4)" project-jdk-type="Python SDK" />
 </project>
--- a/README.md
+++ b/README.md
@@ -8,22 +8,6 @@ running on top of TrueNAS SCALE, separating all the docker stuff from the applia

 # Notes

-## Monitoring
-The monitoring stack is set up to monitor all the containers and the host.
-
-This is a work in progress, Grafana is set up with grafanalib, a Python library that generates Grafana dashboards.
-The dashboards are generated from Python scripts in 
-[grafana_config/dashboards](roles/alpina/templates/services/monitoring/grafana_config/dashboards).
-
-This requires a custom grafana image, which is built from the 
-[Dockerfile](roles/alpina/templates/services/monitoring/Dockerfile).
-
-This also means it has to be manually rebuilt whenever the dashboards are updated.
-From the services/monitoring directory, run:
-```bash
-docker compose up -d --build --force-recreate grafana
-```
-
 ## IPv6
 The current configuration is designed to work with IPv6. 
 However, because of how (not properly) I'm doing the subnetting 
--- a/group_vars/alpina/vars.yml
+++ b/group_vars/alpina/vars.yml
@@ -14,9 +14,6 @@ authentik_secret_key: "{{ vault_authentik_secret_key }}"
 authentik_sendgrid_api_key: "{{ vault_authentik_sendgrid_api_key }}"

 auth_grafana_client_secret: "{{ vault_auth_grafana_client_secret }}"
-auth_gitea_client_secret: "{{ vault_auth_gitea_client_secret }}"
-auth_nextcloud_client_secret: "{{ vault_auth_nextcloud_client_secret }}"
-auth_minio_client_secret: "{{ vault_auth_minio_client_secret }}"
 arrstack_password: "{{ vault_arrstack_password }}"

 # Minio
@@ -27,6 +24,8 @@ minio_password: "{{ vault_minio_password }}"
 influxdb_admin_password: "{{ vault_influxdb_admin_password }}"
 influxdb_admin_token: "{{ vault_influxdb_admin_token }}"

+alertmanager_discord_webhook: "{{ vault_alertmanager_discord_webhook }}"
+
 # Traefik
 acme_email: "{{ vault_acme_email }}"
 cloudflare_api_token: "{{ vault_cloudflare_api_token }}"
--- a/group_vars/alpina/vault.yml
+++ b/group_vars/alpina/vault.yml
@@ -1,113 +1,96 @@
 $ANSIBLE_VAULT;1.1;AES256
-62376365353162306161343336623464386634383663663165393632366666633530373636633032
-6536633438613664316163613236663334663635363665630a666135396430306536646534616535
-65383432356339643063373232393861333366393038666134346363646130626130633861646536
-3134613738333465300a363031626561376533343730353361646462306434663564336538666565
-38643166326439356138653163323030626539393265613833303661313036336562373938323663
-39336533383636626464343461653836313734393430306238336561323038306238646236393835
-62643638636137646162616239636561666432376561393338336663366438346530346666396662
-63626432326263383561633532613039643862303135643262383636666161663539643465616566
-66303364333133393932643666656263613063373162373265353433616337636337363363353938
-31613638633462383031356433393765353439373434356366336234316361393862343763643333
-61623233633664396564376462336131353061303831316466306632663261666161323137333633
-39623938633861356136636532373139356339636334373137303034646431363438613936636438
-38363463386664643439313564313364613962346631343663633837326532613933336462636265
-32616161663065316661313335373234353161653732303965613731633665646532386139383732
-32363834636532363262646433616563363232643864653365643736353434346130383963393564
-34333861326633393763653639663666333061613161393864323165303638353962333531333661
-36316534303365626562643366393836356337303533313237613534313565643832373438373530
-32393065653538393762333232636235316439653935663437616236326162313464323037336630
-39323262333530363230353334356461343866346438626533633339386162336337623137393366
-32373361393231343134626237323062663634323939613461633866353561636334613234336532
-61306235363037306466656463653836396434313830333031366630373364343637376662346663
-65663132346239343937636261643238623364633062356163323364363466666661346364356239
-32653266303837663237333136316464626161626136336333363964636461616138323962313166
-64643930333964303639393439666432366435386464326561323165353333623765653132383636
-34326633663331376563613766383734613762653834356561616461303361373662653337623863
-37633135393861366137613137633265306137326536363632373962353233373735663065653534
-37333038363330633931353233623236313332336234393333616238353137656363643230633966
-32636336663762636130343933373834386465396536316439386465623130396266393438396262
-63636561623533366166393831383035373935643037326265636634646339336264383937366334
-37373961663330326131343531356238363632663861376362643561643966636364653235303032
-33363861396336666332356130353638373135376336373236383730373665623336373830643137
-35613234343966383264643834353162353533373939346561363438376339656239323364353036
-63623630643930363739326236653435613538393438326331383366666332383763356631356533
-39393363366261393231386239363161313939396431323630323062393962313933633462303439
-35623831356638333431313430343832616438343134613538343064323535613539663431643830
-32623363343733623837366236393136393864353332316538306463346337363264613763326463
-65366536326463303062663262636563306565323861666661376338633334383138626364333039
-34333734656331346334316465333339333535333632383963663633383361383661643235383866
-32326634643633366566306137383066653334323935363066316366313934373663383234316438
-35346139633239323431386536656464666161656434316238356333323665333661623364653865
-33636139333866356630323031323162323834303062363637313430313164326636383436383465
-35333434613632353265633935343164613266383463633631323763633565353039306134656431
-37616430633736326139366438613666346434646363313032366231616436616535393334613264
-34646132303061383034363139613362626235383938393535626339353438626635396561346166
-36666530613634336666653638353734323336366639626465346135323838343565383335313233
-62356631666135666434363061666234396337323838303866343839383164643939323862616632
-34646433333031653939313434613435623036346631643265643663613537323061343733326534
-64626663306338623533333132613333386562306162343438653266356666663535623036616666
-64613866663261386233343236353931353766323833623631373438353664393137613032366461
-63623164353435336564613739353863383037326465363462376536663934626362393132313465
-66353965643763656564366630353131313465656265613434363538343331313666613564313036
-35396436633233623261323432666237303335333339393363636362376536343837346264383935
-30346163353338336661646536643536623262343762303766393438343666623063326463346566
-34663538656133353639333830316562376137643666323832363666623766366131303830626531
-62313832316533663261353365343733636236643333396561333636303065653732646665386136
-31386535663732386165623037373763333731343461393431306339393634346130646462646661
-61646539613964666437623631643333333435353039633531313364366338316365396131346331
-30363963633236653364643061316237326362653462656563656165346134656338383738613932
-65333432393534643331396563643865656435373563613939616234636533383731336561623037
-61373839343132376465343332343165316361383831333538313531333063633632643832633536
-33313464643239323963346338386566313031306233336562616638353365666237346262666134
-33646134393531346637376133393039326638316334626333363162313239393239663865323730
-30343731363031303565643833313135643036666461366666376132663433343662333730373137
-65636236313561613637343262653833666135653832363466613138363332393061653032333933
-66376263663830333937336566333461333431393336333161623233353332396437396664316137
-64363737323036366635613938346261383634353237346337613933303334623434623439616533
-32353465336237396133643039613730646661643039363836333733353033343236373864626634
-37666562653233336464633337353963363361646334373863653032353137363738613561613135
-66356132393630613031316466663837633633383033633064326565303837633062336531373866
-34666537303033323362363163353666383962333536303135363666653930326166323637636266
-34306537343238353833313635306663643737653531313435383064383133366364646331306261
-66363763353534643833316533383364353632343439393032313437633734323031383438633333
-31616362343332373333626135396435366235313465346639326564353265643133313339376639
-63333233653833653333373162633033623035633832333566653536343832373035636664643839
-38393864666430313162366337653836333135333738653763653261343233663666373865383366
-65343038646166343934376633613337306436336130626363396339313236653731653265383661
-34633332343639333533316631643763363664666563353137383639616132313363383137383132
-33343635386139366230363464363731383166393430396533613438366661353439353537346530
-62366461653534333834386637363364346432333964306639376339313531383431323930333530
-37383665373937303732643636383539393039663363623337663938303139663039366536323031
-66613036326263316239646535656163626232626130336465303166336336316435343262373631
-39613536336366366435326230653339356635636432303862306636613935306432323966313234
-65623938316162393931343337326334666235666362313739343564633339653962313062393431
-35373338306332326133333638636137386337343261386663333261333030343635336532373134
-38626136383936393339613534386539663035316335656566656639613837313239626431386362
-62643733326636323635373363333964643132323562633430626666616531656639383231336432
-61653439376663613161396465343638623639653135363863336363343230636336346434326234
-32343962666337646435653035333431333632363239616535333835393761353366386561356366
-37356530333763346137653566643134376136656638386334343038376439643037623338643333
-66626537633931333465383062303766333436346433636434653139333966613865656234346539
-36376239393632653536306363313633636464343366373862343039306235303766623462633932
-32313537306530343032663365626330363838396566356534343766383865653231613538323461
-37303439393733376539613061663937633665663963613236323764653835656563346565636531
-30363239376139343166346664306234363031623031663266643966636265666163353536346132
-65623638323065633361373330386334636332306634636336613365663133373835666135396230
-38373939366534663336376135646237633232646261383964383735353533303862623064313333
-33633533653537376138623635663465336131383838663237653933623634343761623731366335
-64653233366335656365656336303862656663303138643531356661373831633062633734363661
-39306633323337356366383863643034656135393432386638353761323337373631353436383664
-34623631306663636439376464383831323566666266613536613661633266343732646264306162
-36353030343538316330313831626232353165323038363034666161336338316536353832353966
-35336365393563643733363535393763613865663436616130343066303638353431653039356661
-34393936363764393032646133326432656230353232623339646165663932366130363734663762
-34303433376666383639663661356334653939663739643139363237623031666632623239343562
-30656438623236616637643132613666343133393436346635316638633664316363323832393862
-39643831363633643562323664613666393033656132333964643639333230353763383330343835
-64383530373332343838666536303363313033303931646232343037303863343835366139326135
-34336330343365663837396134653566633536643832373433393035366531323035616462363639
-66336133346139336264346636643735383136343336303133313031653230366166396239303335
-64656535326465363563396532376538336434643964336264303061393139656139376635633730
-62326664613766393435383464363538393937313236363630656337356264633134353464393835
-32653133383732656235
+32653863663065353431636364373163613536643238613961666561653663633530646165643766
+3833323937353331313136633965393061616135366534660a333037383066303431623830313464
+65346431633238666534373033663138353438313762326361666233353866663534363536643034
+3636323439316261630a623262336331663431633266336235653034323234383566323963623365
+32626363626164373536663464643632393761346137623866633237643038306265636362626561
+61313634353634373530383061393364613461303132326335316566326436633635633131643433
+31376539396639326464333233643933373737313064363262323639363964643862633035396161
+35643037636535623966626131393538643432396536643365383736636262356135373434376433
+32316361343330303431376234323632323932376635343964383733633761326639393966383039
+35646131343034663962363335373661323065663764396631343461383661663738386163323633
+36303464646532633235663662666663343238633465663334326463383133643239666634653739
+35396130393961303230396236303766336666643930626161333338326137663235323066663032
+33376564373563323635356233616264313663373534333636643236393866613062656338353864
+66386132663362363832366661646462316139353132626662663934336530386534376538633235
+62653131653835323261373435373631396466353738306362616266616532313435323633613933
+61646132346536323632643865326234356535346566346532383162393265613931343962303463
+31636334343736666434353835633734396465653862613234386431306463326134613931646232
+32353535663133623434643866336165616232613662336533383432633338373763643337616637
+38323237646461376433316164646366383438316639633162303739383263656265633364303565
+36643339356136653332666230633939636264306431636562323864373037623138363739616561
+37613364653737353638646564323439646138646536636564303866636233616264383466656439
+33646232653061616437656162353036313834616162313936353533393833313432656534343363
+35636638326236646163323463356634326534623165306461316530353936646162323435633862
+64396464303363323837316162353734626663643962303534336637336632333463393734383532
+66616534666466393333386337363238383432643764373864613461363766333932333862363332
+61313364613031376334326635636432346532613462613265643462636436663963323862353733
+38396261613332396633666130653262313234633132353264363266336231373535306532383661
+65323530653531646339626537653433303332656535346639393466353133363833326236656231
+33336265373463396135653730616266346331376461346433343464326238323034653330393732
+36643432316662333633333036633761653031393433333338663633386264656535623534653463
+36363565303333356361616539376532353066336137336134656465383364636361656664356439
+65326334643631663665376530646433323439653864623964323363396561313663636538356536
+63626336303862333364363166353437353163656238303765636662636137383337623563666264
+66326633343230386638616438393436633431343264343231386563613935626430306337343533
+66656366333332326131343661356236396430303832303834653530623639353036663436373862
+61336437386338343965653563646664643438353232306231316564616462643236646239333062
+38643461346639623964626438396631396139383332666130316635656530653136333662353566
+36313261646330373963663032316662383137366436636534383366636362366435393036373264
+34646537666462363531343335336638343038333633663862666163306662643634326533316561
+61613235366233636530663462353066646530386265623534663336376364323237343936646134
+31616563653864383565306439613932396562613835613562326264326535636630646666366335
+36653631353961353933386236636534393636356334633336313333383238353838336335646630
+63633365666530623562323634303935326362643762616532303531303139333565643835396163
+36353130656365326435343130613234336637346461313639653133623933376163393935366266
+66653337353732363038663164363663623266356366663637343466393836353965343730666362
+38663636336265383331666666616535366334616431306164303738306436333364653765356662
+37316433323563323431623164386337343563663538333435616333343433396236356363333262
+61396664326234343136666331356465333233663135613839616334623033316362336162613731
+38646530326538643337323838326563303130643934623939346635343331356531373235663937
+62396530383365666439373632613633633233376139616138323033613135383330333132643839
+65363833616337656662653462323436303531653635663739633366616532333761323238353764
+39373836303735393165393435323139346661346135636138613731373165386533386333393364
+32336265386334386338653734353565343733393931373436336233333031356531313739636666
+61376234393631343236643137616631373564376132623534333939346162353662306661393438
+32326566373934653463653737383131386431363664333535626361646637613632383132623533
+32343465366562363765353366333330633631353936613930376631336538306230626632303966
+31343936386535663165663066663862656439306363326337313561396132316338363930323632
+33313061623534373338623931663934396339633564353533626639373837323832366132343538
+63373862663137306665383732303863343564343830636233613139666631626532373938386663
+35646331646462356639383964373732393866653963643832633661323430323430613330633364
+35343262366362646165383032333236623863656264353964623136643631326135623538306261
+37393839343331653665356131343063316232303963636462653238333466636334616435666463
+65636662383930353238623130363834616137643830633261646338363435343839633565303562
+37623231396163346464303464333962336261353634396236613132306464643764356265656137
+32373263613964396430646332666235303634373431643939623963633334326135626565656662
+30646166303732643562653166633232666635343665616665653566316632303861613861313333
+38393636663137333231613239353661656338333536656563616237343234623031363535666637
+61343662663965663161666436366630366432363733663537613064386130326466343366383232
+32363662343561666665323565356163383932336361656132373263363239636666613461366339
+31323264393866386239353333386161643330343262366666323533303737373163313262313766
+61303638366263346232353134333431613730386431623235323537323962666133613939353762
+63326361633630323937353163383930626336663365626532613031623532393932316138353335
+32363262393764663135393466616639373965313238323935383531633434633038663437646662
+31633265373937316533373332316132363061386133356231623230393739326464333761336338
+38626234646164616265633061346239363164376532383834356435346232653065326362343363
+39613532356166633133626563643238373661323937353635343464666339323561326136623366
+62633637656462376136633963653263346565366563646533373431613761616231653739613537
+32343332356435393635363837396463613165626337346235303363613764306132343539333836
+63386633626332396339383165303166653334663239313066666632356165643161356262346230
+32636365636364663466343939663538386439343336303537636230306263643534653339313538
+31373165363962373337636138336561336638633762373363646139366339323031313664306534
+30623130663037323839666166323162393065643535663866383062356330633137343239316436
+32303132393739653363376138633430313832383165663366626436653033663637616664346632
+63633439663734393236343265323533633639316133323336373064633138363266316135363335
+31336637666331333139306537333565333064666433653730633430336261656665613263663937
+64313230656333373838346439623061393164393239393934306336373063303934663334353532
+31313637623466313835313566616161376230343532653561343364383133653736646338303631
+36356164303630303433356332343630616465383831623036383833393330663566616333653161
+63393361643266323336393962663263323338633634633033393762656139393665353630633637
+39386462303731396261613961613238616237373332656361303139633763303837653765623464
+64333565666532653864383861333433353731343161613231383836353966353636373762306132
+35333536373939656638356333383135313231306433656536383933623634653263353434393238
+32323037666135316337633465666335376332326633346665643333656139386465353134356636
+36333434303538326135346539313734393939353163316666366438613133333464623732666438
+663934323030303937623038343662646163
--- a/roles/alpina/templates/apps/nextcloud/.env.j2
+++ b/roles/alpina/templates/apps/nextcloud/.env.j2
@@ -1 +1 @@
-NEXTCLOUD_VERSION=30-apache
+NEXTCLOUD_VERSION=29-apache
--- a/roles/alpina/templates/apps/nextcloud/.env.notify_push.j2
+++ b/roles/alpina/templates/apps/nextcloud/.env.notify_push.j2
--- a/roles/alpina/templates/services/authentik/blueprints/apps-oauth2.yaml.j2
+++ b/roles/alpina/templates/services/authentik/blueprints/apps-oauth2.yaml.j2
@@ -5,80 +5,46 @@ metadata:
  name: Alpina - OAuth2 Apps
 entries:
  {% set apps = {
-    "Grafana": {
-      "redirect_uri": "https://grafana."~ domain ~"/login/generic_oauth",
-      "icon": "https://grafana."~ domain ~"/public/img/grafana_icon.svg",
-      "client_secret": auth_grafana_client_secret,
-      "ui_group": "Services",
-      "allowed_for_groups": ["admins"],
-    },
-    "Minio": {
-      "redirect_uri": "https://minio."~ domain ~"/oauth_callback",
-      "icon": "https://minio."~ domain ~"/logo192.png",
-      "client_secret": auth_minio_client_secret,
-      "ui_group": "Services",
-      "allowed_for_groups": ["admins"],
-    },
    "Gitea": {
-      "redirect_uri": "https://gitea."~ domain ~"/user/oauth2/Authentik/callback",
+      "redirect_uris": "https://gitea."~ domain ~"/user/oauth2/Authentik/callback",
      "icon": "https://gitea."~ domain ~"/assets/img/logo.svg",
-      "client_secret": auth_gitea_client_secret,
-      "ui_group": "Apps",
-      "allowed_for_groups": ["admins", "users"],
    },
    "Nextcloud": {
-      "redirect_uri": "https://nc."~ domain ~"/apps/sociallogin/custom_oidc/authentik",
+      "redirect_uris": "https://nc."~ domain ~"/apps/sociallogin/custom_oidc/authentik",
      "icon": "https://nc."~ domain ~"/apps/theming/favicon",
-      "client_secret": auth_nextcloud_client_secret,
-      "ui_group": "Apps",
-      "allowed_for_groups": ["admins", "users"],
    },
  } -%}
  {% for app in apps.keys() -%}
  - identifiers:
      name: {{ app }}
    model: authentik_providers_oauth2.oauth2provider
-    id: {{ app }}
+    id: {{ app | lower }}
    attrs:
+      access_code_validity: minutes=1
+      access_token_validity: minutes=5
      authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]]
-      invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
      client_type: confidential
-      client_id: {{ app | lower }}
-      client_secret: {{ apps[app]["client_secret"] }}
+      issuer_mode: per_provider
+      sub_mode: hashed_user_id
      property_mappings:
        - !Find [authentik_providers_oauth2.scopemapping, [scope_name, openid]]
        - !Find [authentik_providers_oauth2.scopemapping, [scope_name, email]]
        - !Find [authentik_providers_oauth2.scopemapping, [scope_name, profile]]
-        {% if app == "Minio" -%}
-        - !Find [authentik_providers_oauth2.scopemapping, [scope_name, minio]]
-        {%- endif %}
-
-      redirect_uris:
-        - matching_mode: strict
-          url: {{ apps[app]["redirect_uri"] }}
-      # Necessary for JWKS to be generated correctly
+      redirect_uris: {{ apps[app]["redirect_uris"] }}
+      refresh_token_validity: days=30
      signing_key: !Find [authentik_crypto.certificatekeypair, [name, "authentik Self-signed Certificate"]]

  - identifiers:
      slug: {{ app | lower }}
    model: authentik_core.application
-    id: app-{{ app }}
+    id: {{ app | lower }}
    attrs:
      name: {{ app }}
-      group: "{{ apps[app]["ui_group"] }}"
+      group: "Apps"
      meta_description: "Hello, I'm {{ app }}!"
      meta_publisher: Alpina
      icon: "{{ apps[app]["icon"] }}"
      open_in_new_tab: true
-      provider: !KeyOf {{ app }}
-
-  {% for group in apps[app]["allowed_for_groups"] -%}
-  - identifiers:
-      group: !Find [authentik_core.group, [name, {{ group }}]]
-      target: !KeyOf app-{{ app }}
-    model: authentik_policies.policybinding
-    attrs:
-      order: 10
-  {% endfor %}
-
+      policy_engine_mode: any
+      provider: !KeyOf {{ app | lower }}
  {% endfor %}
--- a/roles/alpina/templates/services/authentik/blueprints/apps-proxy.yaml.j2
+++ b/roles/alpina/templates/services/authentik/blueprints/apps-proxy.yaml.j2
@@ -4,47 +4,61 @@ metadata:
    blueprints.goauthentik.io/instantiate: "true"
  name: Alpina - Proxied Apps
 entries:
-  # TODO: Possibly refactor this into a jinja macro (?)
+  - identifiers:
+      name: arrstack
+    model: authentik_core.group
+    id: arrstack
+    attrs:
+      arrstack_username: "arr"
+      arrstack_password: "{{ arrstack_password }}"
+
+  # TODO: Probably refactor this into a jinja macro
  {% set apps = {
-    "Uptime Kuma": {
+    "uptime-kuma": {
      "host": "uptime",
+      "name": "Uptime Kuma",
      "icon": "https://uptime."~ domain ~"/icon.svg",
      "unauthenticated_paths": "^/icon.svg$",
-      "ui_group": "Services",
-      "allowed_for_groups": ["admins"],
+      "group": "Services",
+      "create_admin_group": true,
    },
-    "qBit": {
+    "qbit": {
      "host": "qbit",
+      "name": "qBit",
      "icon": "https://qbit."~ domain ~"/images/qbittorrent-tray.svg",
      "unauthenticated_paths": "^/images/qbittorrent-tray.svg$",
-      "ui_group": "Arrstack",
-      "allowed_for_groups": ["arrstack"],
+      "group": "Arrstack",
+      "create_admin_group": false,
    },
-    "Prowlarr": {
+    "prowlarr": {
      "host": "prowlarr",
+      "name": "Prowlarr",
      "icon": "https://prowlarr."~ domain ~"/Content/Images/logo.svg",
      "unauthenticated_paths": "^/Content/Images/logo.svg$",
-      "ui_group": "Arrstack",
-      "allowed_for_groups": ["arrstack"],
+      "group": "Arrstack",
+      "create_admin_group": false,
    },
-    "Sonarr": {
+    "sonarr": {
      "host": "sonarr",
+      "name": "Sonarr",
      "icon": "https://sonarr."~ domain ~"/Content/Images/logo.svg",
      "unauthenticated_paths": "^/Content/Images/logo.svg$",
-      "ui_group": "Arrstack",
-      "allowed_for_groups": ["arrstack"],
+      "group": "Arrstack",
+      "create_admin_group": false,
    },
-    "Radarr": {
+    "radarr": {
      "host": "radarr",
+      "name": "Radarr",
      "icon": "https://radarr."~ domain ~"/Content/Images/logo.svg",
      "unauthenticated_paths": "^/Content/Images/logo.svg$",
-      "ui_group": "Arrstack",
-      "allowed_for_groups": ["arrstack"],
+      "group": "Arrstack",
+      "create_admin_group": false,
    },
  } -%}
+
  {% for app in apps.keys() -%}
  - identifiers:
-      name: {{ app }}
+      name: {{ apps[app]["name"] }}
    model: authentik_providers_proxy.proxyprovider
    id: {{ app }}
    attrs:
@@ -54,26 +68,39 @@ entries:
      skip_path_regex: "{{ apps[app]["unauthenticated_paths"] }}"

  - identifiers:
-      slug: {{ app | lower | replace(" ", "-") }}
+      slug: {{ app }}
    model: authentik_core.application
-    id: app-{{ app }}
    attrs:
-      name: {{ app }}
-      group: {{ apps[app]["ui_group"] }}
-      meta_description: "Hello, I'm {{ app }}!"
+      name: {{ apps[app]["name"] }}
+      group: {{ apps[app]["group"] }}
+      meta_description: "Hello, I'm {{ apps[app]["name"] }}!"
      meta_publisher: Alpina
      icon: "{{ apps[app]["icon"] }}"
      open_in_new_tab: true
      provider: !KeyOf {{ app }}

-  {% for group in apps[app]["allowed_for_groups"] -%}
+  {% if apps[app]["create_admin_group"] -%}
  - identifiers:
-      group: !Find [authentik_core.group, [name, {{ group }}]]
-      target: !KeyOf app-{{ app }}
+      name: "{{ apps[app]["name"] }} Admins"
+    model: authentik_core.group
+    id: "{{ app }} Admins"
+
+  - identifiers:
+      group: !KeyOf "{{ app }} Admins"
+      target: !Find [authentik_core.application, [ slug, {{ app }}] ]
    model: authentik_policies.policybinding
    attrs:
-      order: 10
-  {% endfor %}
+      order: 0
+  {% endif %}
+
+  {% if apps[app]["group"] == "Arrstack" -%}
+  - identifiers:
+      group: !KeyOf arrstack
+      target: !Find [authentik_core.application, [slug, {{ app }}]]
+    model: authentik_policies.policybinding
+    attrs:
+      order: 0
+  {% endif %}

  {% endfor %}

--- a/roles/alpina/templates/services/authentik/blueprints/default-groups.yaml.j2
+++ b/roles/alpina/templates/services/authentik/blueprints/default-groups.yaml.j2
@@ -1,40 +0,0 @@
-version: 1
-metadata:
-  labels:
-    blueprints.goauthentik.io/instantiate: "true"
-  name: Alpina - Default Groups
-entries:
-  - identifiers:
-      name: "admins"
-    model: authentik_core.group
-    id: "admins"
-    attrs:
-      is_superuser: true
-
-  - identifiers:
-      name: "users"
-    model: authentik_core.group
-    id: "users"
-
-  - identifiers:
-      name: "arrstack"
-    model: authentik_core.group
-    id: "arrstack"
-    attrs:
-      arrstack_username: "arr"
-      arrstack_password: "{{ arrstack_password }}"
-
-  - identifiers:
-      scope_name: "minio"
-    model: authentik_providers_oauth2.scopemapping
-    id: "scope-minio"
-    attrs:
-      name: "Minio Policy"
-      expression: |
-        policy = "default"
-        if ak_is_group_member(request.user, name="admins"):
-            policy = "consoleAdmin"
-
-        return {
-          "policy": policy,
-        }
--- a/roles/alpina/templates/services/authentik/blueprints/services-oauth2.yaml.j2
+++ b/roles/alpina/templates/services/authentik/blueprints/services-oauth2.yaml.j2
@@ -0,0 +1,56 @@
+version: 1
+metadata:
+  labels:
+    blueprints.goauthentik.io/instantiate: "true"
+  name: Alpina - OAuth2 Services
+entries:
+  {% set apps = {
+    "Grafana": {
+      "redirect_uris": "https://grafana."~ domain ~"/login/generic_oauth",
+      "icon": "https://grafana."~ domain ~"/public/img/grafana_icon.svg",
+      "client_secret": auth_grafana_client_secret,
+    },
+  } -%}
+  # TODO: Add Minio
+
+  {% for app in apps.keys() -%}
+  - identifiers:
+      name: {{ app }}
+    model: authentik_providers_oauth2.oauth2provider
+    id: {{ app | lower }}
+    attrs:
+      authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]]
+      client_type: confidential
+      client_id: {{ app | lower }}
+      client_secret: {{ apps[app]["client_secret"] }}
+      property_mappings:
+        - !Find [authentik_providers_oauth2.scopemapping, [scope_name, openid]]
+        - !Find [authentik_providers_oauth2.scopemapping, [scope_name, email]]
+        - !Find [authentik_providers_oauth2.scopemapping, [scope_name, profile]]
+      redirect_uris: {{ apps[app]["redirect_uris"] }}
+
+  - identifiers:
+      slug: {{ app | lower }}
+    model: authentik_core.application
+    attrs:
+      name: {{ app }}
+      group: "Services"
+      meta_description: "Hello, I'm {{ app }}!"
+      meta_publisher: Alpina
+      icon: "{{ apps[app]["icon"] }}"
+      open_in_new_tab: true
+      provider: !KeyOf {{ app | lower }}
+
+  - identifiers:
+      name: "{{ app }} Admins"
+    model: authentik_core.group
+    id: "{{ app }} Admins"
+
+  - identifiers:
+      group: !KeyOf "{{ app }} Admins"
+      target: !Find [authentik_core.application, [slug, {{ app | lower }}]]
+    model: authentik_policies.policybinding
+    attrs:
+      order: 0
+
+  {% endfor %}
--- a/roles/alpina/templates/services/minio/.env.minio.j2
+++ b/roles/alpina/templates/services/minio/.env.minio.j2
@@ -5,16 +5,11 @@ MINIO_DOMAIN=s3.{{ domain }}
 MINIO_SERVER_URL=https://s3.{{ domain }}
 MINIO_BROWSER_REDIRECT_URL=https://minio.{{ domain }}

-# https://min.io/docs/minio/linux/reference/minio-server/settings/iam/openid.html
-MINIO_IDENTITY_OPENID_CONFIG_URL=https://auth.{{ domain }}/application/o/minio/.well-known/openid-configuration
-MINIO_IDENTITY_OPENID_CLIENT_ID=minio
-MINIO_IDENTITY_OPENID_CLIENT_SECRET={{ auth_minio_client_secret }}
-# defaults to "policy"
-#MINIO_IDENTITY_OPENID_CLAIM_NAME=policy
-MINIO_IDENTITY_OPENID_DISPLAY_NAME=Authentik
-# no need to specify scopes,
-# as it defaults to the ones advertised at the discovery url
-#MINIO_IDENTITY_OPENID_SCOPES=openid,profile,email,minio
-#MINIO_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC=off
-#MINIO_IDENTITY_OPENID_CLAIM_USERINFO=on
+#MINIO_IDENTITY_OPENID_CONFIG_URL=https://auth.{{ domain }}/application/o/minio/.well-known/openid-configuration
+#MINIO_IDENTITY_OPENID_CLIENT_ID=
+#MINIO_IDENTITY_OPENID_CLIENT_SECRET=
+#MINIO_IDENTITY_OPENID_CLAIM_NAME=
+#MINIO_IDENTITY_OPENID_CLAIM_PREFIX=
+#MINIO_IDENTITY_OPENID_SCOPES=
+#MINIO_IDENTITY_OPENID_REDIRECT_URI=
 #MINIO_IDENTITY_OPENID_COMMENT=
--- a/roles/alpina/templates/services/monitoring/.env.alertmanager.j2
+++ b/roles/alpina/templates/services/monitoring/.env.alertmanager.j2
@@ -0,0 +1 @@
+DISCORD_WEBHOOK={{ alertmanager_discord_webhook }}
--- a/roles/alpina/templates/services/monitoring/Dockerfile
+++ b/roles/alpina/templates/services/monitoring/Dockerfile
@@ -4,10 +4,6 @@ RUN pip install grafanalib

 COPY ./grafana_config/dashboards /dashboards

-# Required for grafanalib to find the shared python files like common.py
-# https://github.com/weaveworks/grafanalib/issues/58
-ENV PYTHONPATH=/dashboards
-
 RUN generate-dashboards /dashboards/*.dashboard.py

 FROM grafana/grafana:latest
--- a/roles/alpina/templates/services/monitoring/alertmanager_config/alertmanager.yml.j2
+++ b/roles/alpina/templates/services/monitoring/alertmanager_config/alertmanager.yml.j2
@@ -0,0 +1,68 @@
+# The root route on which each incoming alert enters.
+route:
+  group_by: ["alertname", "job"]
+  group_wait: 20s
+  group_interval: 5m
+  repeat_interval: 3h
+  receiver: discord_webhook
+
+receivers:
+  - name: "discord_webhook"
+    discord_configs:
+      - webhook_url: "{{ alertmanager_discord_webhook }}"
+{#        - send_resolved: true#}
+{#            username: 'Alertmanager'#}
+{#            webhook_configs:#}
+{#            - send_resolved: true#}
+{#                url: '{{ alertmanager_discord_webhook }}'#}
+{#                username: 'Alertmanager'#}
+{#                icon_url: 'https://prometheus.io/assets/icon.png'#}
+{#                icon_emoji: ':alert:'#}
+{#                send_resolved: true#}
+{#                text: "{{ .CommonAnnotations.summary }}"#}
+{#                title: "{{ .CommonLabels.alertname }}"#}
+{#                color: '{{ if eq .Status "firing" }}#FF0000{{ else }}#00FF00{{ end }}'#}
+{#                footer: '{{ .CommonLabels.monitor }}'#}
+{#                footer_icon: 'https://prometheus.io/assets/icon.png'#}
+{#                actions:#}
+{#                - type: 'button'#}
+{#                    text: 'Open in Grafana'#}
+{#                    url: '{{ .ExternalURL }}'#}
+{#                    style: 'primary'#}
+{#                    send_resolved: true#}
+{#                    confirm:#}
+{#                    title: 'Are you sure?'#}
+{#                    text: 'This will open Grafana in a new tab.'#}
+{#                    ok_text: 'Yes'#}
+{#                    dismiss_text: 'No'#}
+{#                fields:#}
+{#                - title: 'Description'#}
+{#                    value: "{{ .CommonAnnotations.description }}"#}
+{#                    short: false#}
+{#                - title: 'Details'#}
+{#                    value: "{{ .CommonAnnotations.details }}"#}
+{#                    short: false#}
+{#                - title: 'Severity'#}
+{#                    value: '{{ if eq .Labels.severity "critical" }}Critical{{ else if eq .Labels.severity "warning" }}Warning{{ else }}Info{{ end }}'#}
+{#                    short: true#}
+{#                - title: 'Host'#}
+{#                    value: '{{ .CommonLabels.monitor }}'#}
+{#                    short: true#}
+{#                - title: 'Starts At'#}
+{#                    value: '{{ .StartsAt.Format "2006-01-02 15:04:05" }}'#}
+{#                    short: true#}
+{#                - title: 'Ends At'#}
+{#                    value: '{{ .EndsAt.Format "2006-01-02 15:04:05" }}'#}
+{#                    short: true#}
+{#                - title: 'Runbook'#}
+{#                    value: '{{ .CommonAnnotations.runbook_url }}'#}
+{#                    short: true#}
+{#                - title: 'Dashboard'#}
+{#                    value: '{{ .CommonAnnotations.dashboard_url }}'#}
+{#                    short: true#}
+{#                - title: 'Alerting Rule'#}
+{#                    value: '{{ .CommonLabels.alertname }}'#}
+{#                    short: true#}
+{#                - title: 'Alerting Rule Description'#}
+{#                    value: '{{ .CommonLabels.alertname }}'#}
+{#                    short: true#}
--- a/roles/alpina/templates/services/monitoring/docker-compose.yml.j2
+++ b/roles/alpina/templates/services/monitoring/docker-compose.yml.j2
@@ -60,17 +60,33 @@ services:
  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
+    labels:
+      - {{ helpers.traefik_labels('prom', port='9090') | indent(6) }}
    restart: unless-stopped
    # Needed to make config files readable (not anymore, TODO: remove)
    user: "{{ remote_uid }}"
    command:
      - --config.file=/etc/prometheus/prometheus.yml
      - --storage.tsdb.retention.time=30d
+      - --web.external-url=https://prom.{{ domain }}/
    volumes:
      - ./prometheus_config:/etc/prometheus:ro
      - {{ base_volume_path }}/monitoring/prometheus_configs:/etc/prometheus/extra:ro
      - {{ base_volume_path }}/monitoring/prometheus:/prometheus

+  alertmanager:
+    image: prom/alertmanager:latest
+    container_name: alertmanager
+    labels:
+      - {{ helpers.traefik_labels('alert', port='9093') | indent(6) }}
+    restart: unless-stopped
+    command:
+      - --config.file=/etc/alertmanager/alertmanager.yml
+      - --web.external-url=https://alert.{{ domain }}/
+    volumes:
+      - ./alertmanager_config:/etc/alertmanager:ro
+      - {{ base_volume_path }}/monitoring/alertmanager:/alertmanager
+
  node-exporter:
    image: prom/node-exporter:latest
    container_name: node-exporter
@@ -84,11 +100,6 @@ services:
    image: gcr.io/cadvisor/cadvisor:latest
    container_name: cadvisor
    restart: unless-stopped
-    command:
-      - --docker_only=true
-      - --store_container_labels=false
-      - --whitelisted_container_labels=com.docker.compose.project,com.docker.compose.service
-      - --enable_metrics=cpu,cpuLoad,diskIO,memory,network,oom_event,process
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:rw
--- a/roles/alpina/templates/services/monitoring/grafana_config/dashboards/alpina.yaml
+++ b/roles/alpina/templates/services/monitoring/grafana_config/dashboards/alpina.yaml
@@ -3,7 +3,7 @@ apiVersion: 1
 providers:
  - name: "Grafana"
    org_id: 1
-    folder: "Alpina"
+    folder: "Services"
    type: "file"
    options:
      path: "/etc/grafana/provisioning/dashboards"
--- a/roles/alpina/templates/services/monitoring/grafana_config/dashboards/common.py
+++ b/roles/alpina/templates/services/monitoring/grafana_config/dashboards/common.py
@@ -1,27 +0,0 @@
-from grafanalib.core import Template
-
-# TODO: consider default params for common params like line width, show points, tooltip
-
-PrometheusTemplate = Template(
-    name='datasource',
-    type='datasource',
-    label='Prometheus',
-    query='prometheus',
-)
-
-# TODO: this slightly less (clown emoji), normal Target gave me errors in grafana
-class LokiTarget(object):
-    def __init__(self, loki_datasource, expr, legendFormat, refId):
-        self.loki_datasource = loki_datasource
-        self.expr = expr
-        self.legendFormat = legendFormat
-        self.refId = refId
-
-    def to_json_data(self):
-        return {
-            'datasource': self.loki_datasource,
-            'expr': self.expr,
-            'legendFormat': self.legendFormat,
-            'refId': self.refId,
-            'queryType': 'range',
-        }
--- a/roles/alpina/templates/services/monitoring/grafana_config/dashboards/containers.dashboard.py
+++ b/roles/alpina/templates/services/monitoring/grafana_config/dashboards/containers.dashboard.py
@@ -5,21 +5,28 @@ from grafanalib.core import (
 )
 from grafanalib.formatunits import BYTES_IEC, SECONDS, BYTES_SEC_IEC

-from common import LokiTarget, PrometheusTemplate
-
-prom_datasource='${datasource}'
+prom_datasource='prometheus'
 loki_datasource='loki'

+# TODO: this is (clown emoji), normal Target gave me errors in grafana
+class LokiTarget(object):
+    def to_json_data(self):
+        return {
+            'datasource': loki_datasource,
+            'expr': '{compose_project=~"$compose_project", container_name=~"$container_name"} |= `$logs_query`',
+            'legendFormat': '{{ container_name }}',
+            'refId': 'A',
+            'queryType': 'range',
+        }
+
 dashboard = Dashboard(
    title='Containers',
    uid='containers',
    description='Data for compose projects from default Prometheus datasource collected by Cadvisor',
    tags=[
-        'linux',
-        'docker',
+        'example'
    ],
    templating=Templating(list=[
-        PrometheusTemplate,
        Template(
            name='compose_project',
            label='Compose Project',
@@ -37,6 +44,7 @@ dashboard = Dashboard(
            includeAll=True,
            multi=True,
            refresh=REFRESH_ON_TIME_RANGE_CHANGE,
+
        ),
        Template(
            name='logs_query',
@@ -48,6 +56,7 @@ dashboard = Dashboard(
    timezone='browser',
    panels=[
        TimeSeries(
+            id=1,
            title='Container Memory Usage',
            unit=BYTES_IEC,
            gridPos=GridPos(h=8, w=12, x=0, y=0),
@@ -67,14 +76,13 @@ dashboard = Dashboard(
            ],
        ),
        TimeSeries(
+            id=2,
            title='Container CPU Usage',
            unit=SECONDS,
            gridPos=GridPos(h=8, w=12, x=12, y=0),
            lineWidth=2,
            fillOpacity=10,
            showPoints='never',
-            tooltipMode='all',
-            tooltipSort='desc',
            targets=[
                Target(
                    datasource=prom_datasource,
@@ -85,6 +93,7 @@ dashboard = Dashboard(
            ],
        ),
        TimeSeries(
+            id=3,
            title='Container Network Traffic',
            unit=BYTES_SEC_IEC,
            gridPos=GridPos(h=8, w=12, x=0, y=8),
@@ -109,6 +118,7 @@ dashboard = Dashboard(
            ],
        ),
        Logs(
+            id=4,
            title='',
            gridPos=GridPos(h=8, w=12, x=12, y=8),
            showLabels=True,
@@ -117,12 +127,13 @@ dashboard = Dashboard(
            prettifyLogMessage=True,
            dedupStrategy='numbers',
            targets=[
-                LokiTarget(
-                    loki_datasource=loki_datasource,
-                    expr='{compose_project=~"$compose_project", container_name=~"$container_name"} |= `$logs_query`',
-                    legendFormat='{{ container_name }}',
-                    refId='A',
-                ),
+                LokiTarget(),
+                # Target(
+                #     datasource=loki_datasource,
+                #     expr='{compose_project=~"$compose_project", container_name=~"$container_name"} |= `$logs_query`',
+                #     legendFormat='{{ container_name }}',
+                #     refId='A',
+                # ),
            ],
        ),
    ],
--- a/roles/alpina/templates/services/monitoring/grafana_config/dashboards/example.dashboard.py
+++ b/roles/alpina/templates/services/monitoring/grafana_config/dashboards/example.dashboard.py
@@ -0,0 +1,51 @@
+from grafanalib.core import (
+    Dashboard, TimeSeries, GaugePanel,
+    Target, GridPos,
+    OPS_FORMAT
+)
+
+dashboard = Dashboard(
+    title="Python generated example dashboard",
+    description="Example dashboard using the Random Walk and default Prometheus datasource",
+    tags=[
+        'example'
+    ],
+    timezone="browser",
+    panels=[
+        TimeSeries(
+            title="Random Walk",
+            dataSource='default',
+            targets=[
+                Target(
+                    datasource='grafana',
+                    expr='example',
+                ),
+            ],
+            gridPos=GridPos(h=8, w=16, x=0, y=0),
+        ),
+        GaugePanel(
+            title="Random Walk",
+            dataSource='default',
+            targets=[
+                Target(
+                    datasource='grafana',
+                    expr='example',
+                ),
+            ],
+            gridPos=GridPos(h=4, w=4, x=17, y=0),
+        ),
+        TimeSeries(
+            title="Prometheus http requests",
+            dataSource='prometheus',
+            targets=[
+                Target(
+                    expr='rate(prometheus_http_requests_total[5m])',
+                    legendFormat="{{ handler }}",
+                    refId='A',
+                ),
+            ],
+            unit=OPS_FORMAT,
+            gridPos=GridPos(h=8, w=16, x=0, y=10),
+        ),
+    ],
+).auto_panel_ids()
--- a/roles/alpina/templates/services/monitoring/grafana_config/dashboards/node.dashboard.py
+++ b/roles/alpina/templates/services/monitoring/grafana_config/dashboards/node.dashboard.py
@@ -1,139 +0,0 @@
-from grafanalib.core import Dashboard, Templating, Template, TimeSeries, PERCENT_UNIT_FORMAT, GridPos, Target
-from grafanalib.formatunits import BYTES_IEC
-
-from common import PrometheusTemplate
-from node_consts import CPU_BASIC_COLORS, MEMORY_BASIC_COLORS
-
-dashboard = Dashboard(
-    title='Node Exporter',
-    uid='node',
-    description='Node Exporter (not quite full)',
-    tags=[
-        'linux',
-    ],
-    timezone='browser',
-    templating=Templating(list=[
-        # Datasource
-        PrometheusTemplate,
-        # Job
-        Template(
-            name='job',
-            label='Job',
-            dataSource='${datasource}',
-            query='label_values(node_uname_info, job)',
-        ),
-        # Instance
-        Template(
-            name='instance',
-            label='Instance',
-            dataSource='${datasource}',
-            query='label_values(node_uname_info{job="$job"}, instance)',
-        ),
-    ]),
-    panels=[
-        # CPU Basic
-        TimeSeries(
-            title='CPU Basic',
-            description='Basic CPU usage info',
-            unit=PERCENT_UNIT_FORMAT,
-            gridPos=GridPos(h=8, w=12, x=0, y=0),
-            lineWidth=1,
-            fillOpacity=30,
-            showPoints='never',
-            stacking={'mode': 'percent', 'group': 'A'},
-            tooltipMode='all',
-            tooltipSort='desc',
-            targets=[
-                Target(
-                    datasource='${datasource}',
-                    expr='sum(irate(node_cpu_seconds_total{instance="$instance",job="$job", mode="system"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance="$instance",job="$job"}) by (cpu)))',
-                    legendFormat='Busy System',
-                    refId='A',
-                ),
-                Target(
-                    datasource='${datasource}',
-                    expr='sum(irate(node_cpu_seconds_total{instance="$instance",job="$job", mode="user"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance="$instance",job="$job"}) by (cpu)))',
-                    legendFormat='Busy User',
-                    refId='B',
-                ),
-                Target(
-                    datasource='${datasource}',
-                    expr='sum(irate(node_cpu_seconds_total{instance="$instance",job="$job", mode="iowait"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance="$instance",job="$job"}) by (cpu)))',
-                    legendFormat='Busy Iowait',
-                    refId='C',
-                ),
-                Target(
-                    datasource='${datasource}',
-                    expr='sum(irate(node_cpu_seconds_total{instance="$instance",job="$job", mode=~".*irq"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance="$instance",job="$job"}) by (cpu)))',
-                    legendFormat='Busy IRQs',
-                    refId='D',
-                ),
-                Target(
-                    datasource='${datasource}',
-                    expr='sum(irate(node_cpu_seconds_total{instance="$instance",job="$job",  mode!="idle",mode!="user",mode!="system",mode!="iowait",mode!="irq",mode!="softirq"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance="$instance",job="$job"}) by (cpu)))',
-                    legendFormat='Busy Other',
-                    refId='E',
-                ),
-                Target(
-                    datasource='${datasource}',
-                    expr='sum(irate(node_cpu_seconds_total{instance="$instance",job="$job", mode="idle"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance="$instance",job="$job"}) by (cpu)))',
-                    legendFormat='Idle',
-                    refId='F',
-                ),
-            ],
-            # Extra JSON for the colors
-            extraJson=CPU_BASIC_COLORS,
-        ),
-        # Memory Basic
-        TimeSeries(
-            title='Memory Basic',
-            description='Basic memory usage',
-            unit=BYTES_IEC,
-            gridPos=GridPos(h=8, w=12, x=12, y=0),
-            lineWidth=1,
-            fillOpacity=30,
-            showPoints='never',
-            stacking={'mode': 'normal', 'group': 'A'},
-            tooltipMode='all',
-            tooltipSort='desc',
-            targets=[
-                Target(
-                    datasource='${datasource}',
-                    expr='node_memory_MemTotal_bytes{instance="$instance",job="$job"}',
-                    format='time_series',
-                    legendFormat='RAM Total',
-                    refId='A',
-                ),
-                Target(
-                    datasource='${datasource}',
-                    expr='node_memory_MemTotal_bytes{instance="$instance",job="$job"} - node_memory_MemFree_bytes{instance="$instance",job="$job"} - (node_memory_Cached_bytes{instance="$instance",job="$job"} + node_memory_Buffers_bytes{instance="$instance",job="$job"} + node_memory_SReclaimable_bytes{instance="$instance",job="$job"})',
-                    format='time_series',
-                    legendFormat='RAM Used',
-                    refId='B',
-                ),
-                Target(
-                    datasource='${datasource}',
-                    expr='node_memory_Cached_bytes{instance="$instance",job="$job"} + node_memory_Buffers_bytes{instance="$instance",job="$job"} + node_memory_SReclaimable_bytes{instance="$instance",job="$job"}',
-                    legendFormat='RAM Cache + Buffer',
-                    refId='C',
-                ),
-                Target(
-                    datasource='${datasource}',
-                    expr='node_memory_MemFree_bytes{instance="$instance",job="$job"}',
-                    legendFormat='RAM Free',
-                    refId='D',
-                ),
-                Target(
-                    datasource='${datasource}',
-                    expr='(node_memory_SwapTotal_bytes{instance="$instance",job="$job"} - node_memory_SwapFree_bytes{instance="$instance",job="$job"})',
-                    legendFormat='SWAP Used',
-                    refId='E',
-                ),
-            ],
-            # Extra JSON for the colors
-            extraJson=MEMORY_BASIC_COLORS,
-        ),
-        # TODO: Network Basic
-        # TODO: Disk Basic
-    ],
-).auto_panel_ids()
--- a/roles/alpina/templates/services/monitoring/grafana_config/dashboards/node_consts.py
+++ b/roles/alpina/templates/services/monitoring/grafana_config/dashboards/node_consts.py
@@ -1,487 +0,0 @@
-# TODO: Question life decisions (I'm not sure if this is good)
-
-CPU_BASIC_COLORS = {
-    "fieldConfig": {
-        "overrides": [
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "Busy Iowait"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#890F02",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "Idle"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#052B51",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "Busy Iowait"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#890F02",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "Idle"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#7EB26D",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "Busy System"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#EAB839",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "Busy User"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#0A437C",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "Busy Other"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#6D1F62",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            }
-        ]
-    },
-}
-
-MEMORY_BASIC_COLORS = {
-    "fieldConfig": {
-        "overrides": [
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "Apps"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#629E51",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "Buffers"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#614D93",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "Cache"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#6D1F62",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "Cached"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#511749",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "Committed"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#508642",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "Free"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#0A437C",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#CFFAFF",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "Inactive"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#584477",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "PageTables"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#0A50A1",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "Page_Tables"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#0A50A1",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "RAM_Free"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#E0F9D7",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "SWAP Used"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#BF1B00",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "Slab"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#806EB7",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "Slab_Cache"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#E0752D",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "Swap"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#BF1B00",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "Swap Used"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#BF1B00",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "Swap_Cache"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#C15C17",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "Swap_Free"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#2F575E",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "Unused"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#EAB839",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "RAM Total"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#E0F9D7",
-                            "mode": "fixed"
-                        }
-                    },
-                    {
-                        "id": "custom.fillOpacity",
-                        "value": 0
-                    },
-                    {
-                        "id": "custom.stacking",
-                        "value": {
-                            "group": False,
-                            "mode": "normal"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "RAM Cache + Buffer"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#052B51",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "RAM Free"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#7EB26D",
-                            "mode": "fixed"
-                        }
-                    }
-                ]
-            },
-            {
-                "matcher": {
-                    "id": "byName",
-                    "options": "Available"
-                },
-                "properties": [
-                    {
-                        "id": "color",
-                        "value": {
-                            "fixedColor": "#DEDAF7",
-                            "mode": "fixed"
-                        }
-                    },
-                    {
-                        "id": "custom.fillOpacity",
-                        "value": 0
-                    },
-                    {
-                        "id": "custom.stacking",
-                        "value": {
-                            "group": False,
-                            "mode": "normal"
-                        }
-                    }
-                ]
-            }
-        ]
-    }
-}
--- a/roles/alpina/templates/services/monitoring/grafana_config/datasources/alpina.yaml.j2
+++ b/roles/alpina/templates/services/monitoring/grafana_config/datasources/alpina.yaml.j2
@@ -15,6 +15,18 @@ datasources:
    url: http://prometheus:9090
    editable: false

+  - name: Alertmanager
+    type: alertmanager
+    access: proxy
+    uid: alertmanager
+    url: http://alertmanager:9093
+    jsonData:
+      # Valid options for implementation include mimir, cortex and prometheus
+      implementation: prometheus
+      # Whether Grafana should send alert instances to this Alertmanager
+      handleGrafanaManagedAlerts: true
+    editable: false
+
  - name: InfluxDB
    type: influxdb
    access: proxy
--- a/roles/alpina/templates/services/monitoring/loki_config/loki-config.yaml.j2
+++ b/roles/alpina/templates/services/monitoring/loki_config/loki-config.yaml.j2
@@ -17,6 +17,13 @@ common:

 schema_config:
  configs:
+    - from: 2020-10-24
+      store: boltdb-shipper
+      object_store: filesystem
+      schema: v12
+      index:
+        prefix: index_
+        period: 24h
    - from: 2024-10-18
      index:
        period: 24h
@@ -26,5 +33,5 @@ schema_config:
      store: tsdb

 # TODO: Figure this out
-# ruler:
-#   alertmanager_url: http://localhost:9093
+ruler:
+  alertmanager_url: http://localhost:9093
--- a/roles/alpina/templates/services/monitoring/prometheus_config/container.alerts.yml
+++ b/roles/alpina/templates/services/monitoring/prometheus_config/container.alerts.yml
@@ -0,0 +1,23 @@
+groups:
+  - name: qbit-low-traffic
+    interval: 1m
+    rules:
+      - alert: QbitLowTraffic
+        expr: |
+          rate(container_network_transmit_bytes_total{name=~"gluetun"}[1m]) < 1024
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          title: 'Low traffic on qBit'
+          description: |
+            The traffic on qBittorrent is lower than 1KiB/s for 2 minutes.
+
+            Last value was x bytes/s.
+
+            [Grafana Dashboard](https://grafana.{{ domain }}/d/containers?orgId=1)
+            [View in Grafana](https://grafana.{{ domain }}/d/containers?orgId=1&viewPanel=3)
+
+          __dashboard__uid: 'containers'
+          __orgId__: 1
+          __panelId__: 3
--- a/roles/alpina/templates/services/monitoring/prometheus_config/demo.alerts.yml.j2
+++ b/roles/alpina/templates/services/monitoring/prometheus_config/demo.alerts.yml.j2
@@ -0,0 +1,20 @@
+groups:
+  - name: demo-service-alerts
+    rules:
+      - alert: DemoServiceHighErrorRate
+        expr: |
+          (
+            sum without(status, instance) (
+              rate(demo_api_request_duration_seconds_count{status=~"5..",job="demo"}[1m])
+            )
+          /
+            sum without(status, instance) (
+              rate(demo_api_request_duration_seconds_count{job="demo"}[1m])
+            ) * 100 > 0.5
+          )
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          title: 'High 5xx rate for {{'{{ $labels.method }}'}} on {{'{{ $labels.path }}'}}'
+          description: 'The 5xx error rate for path {{'{{ $labels.path }}'}} with method {{'{{ $labels.method }}'}} in {{'{{ $labels.job }}'}} is {{'{{ printf "%.2f" $value }}'}}%.'
--- a/roles/alpina/templates/services/monitoring/prometheus_config/prometheus.yml.j2
+++ b/roles/alpina/templates/services/monitoring/prometheus_config/prometheus.yml.j2
@@ -5,6 +5,11 @@ global:
  external_labels:
    monitor: "{{ ansible_host }}"

+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets: ["alertmanager:9093"]
+
 scrape_configs:
  - job_name: "prometheus"
    static_configs:
@@ -30,7 +35,15 @@ scrape_configs:
    static_configs:
      - targets: ["promtail:9080"]

+  - job_name: 'demo'
+    static_configs:
+      - targets:
+          - 'demo.promlabs.com:10000'
+          - 'demo.promlabs.com:10001'
+          - 'demo.promlabs.com:10002'
+
 rule_files:
+  - "/etc/prometheus/container.alerts.yml"
  - "/etc/prometheus/extra/rules/*.yml"
  - "/etc/prometheus/extra/rules/*.json"

--- a/services.yml
+++ b/services.yml
@@ -5,11 +5,10 @@
  post_tasks:
    - name: Docker prune objects
      docker_prune:
-        containers: true
-        # Keep images for building grafana
-        images: true
+        containers: yes
+        images: yes
        images_filters:
-          until: "720h"
+          dangling: false
        networks: true
        volumes: true
-        builder_cache: false
+        builder_cache: true
Author	SHA1	Message	Date
Yuri Tatishchev	eb264b73fa	(WIP) monitoring: attempt at container alerts	2024-10-22 23:07:41 -07:00
Yuri Tatishchev	8c6b862495	(WIP) monitoring: containers dashboard, loki storage update	2024-10-22 23:07:08 -07:00
Yuri Tatishchev	30510c6690	WIP: containers dashboard - loki logs	2024-10-12 23:32:03 -07:00
Yuri Tatishchev	c38f94f4ce	WIP: monitoring improvements - containers dashboard	2024-10-12 23:31:09 -07:00
Yuri Tatishchev	002eb40b68	WIP: monitoring improvements	2024-10-12 02:28:52 -07:00
				`@@ -0,0 +1 @@`
				`DISCORD_WEBHOOK={{ alertmanager_discord_webhook }}`