CaZzzer/alpina
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Compare commits

base: CaZzzer:feature/monitoring
Branches Tags
CaZzzer:master CaZzzer:feature/bws CaZzzer:feature/obsidian-livesync CaZzzer:feature/auth CaZzzer:feature/monitoring-1 CaZzzer:feature/monitoring CaZzzer:feature/db CaZzzer:feature/docker-volumes-zfs CaZzzer:feature/zfs-backups
..
compare: CaZzzer:e8fe6adca37777b526d98ef166a4c172182ce572
Branches Tags
CaZzzer:master CaZzzer:feature/bws CaZzzer:feature/obsidian-livesync CaZzzer:feature/auth CaZzzer:feature/monitoring-1 CaZzzer:feature/monitoring CaZzzer:feature/db CaZzzer:feature/docker-volumes-zfs CaZzzer:feature/zfs-backups

9 Commits
feature/mo ... e8fe6adca3

Author SHA1 Message Date
Yuri Tatishchev
e8fe6adca3 (WIP) monitoring: container, node dashboard improvements; separate common.py logic 2024-10-23 22:40:40 -07:00
Yuri Tatishchev
e6cdae8e6b (WIP) monitoring: unbreak prometheus by removing alert shit 2024-10-23 15:56:03 -07:00
Yuri Tatishchev
6a22c3f815 (WIP) monitoring: attempt at container alerts 2024-10-23 15:56:03 -07:00
Yuri Tatishchev
f9b0288038 (WIP) monitoring: containers dashboard 2024-10-23 15:56:02 -07:00
Yuri Tatishchev
d8bef9a23d WIP: containers dashboard - loki logs 2024-10-23 15:53:04 -07:00
Yuri Tatishchev
581346f019 WIP: monitoring improvements - containers dashboard 2024-10-23 15:53:04 -07:00
Yuri Tatishchev
6504100cd5 WIP: monitoring improvements 2024-10-23 15:53:00 -07:00
Yuri Tatishchev
dd0330c85a monitoring: loki, cadvisor config optimizations 2024-10-23 15:35:54 -07:00
Yuri Tatishchev
40fbdc414e monitoring: update loki schema 2024-10-22 23:16:41 -07:00
12 changed files with 696 additions and 35 deletions
Expand all files Collapse all files

2
.idea/alpina.iml generated
View File

@@ -4,7 +4,7 @@
<content url="file://$MODULE_DIR$"> <content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/venv" /> <excludeFolder url="file://$MODULE_DIR$/venv" />
</content> </content>
<orderEntry type="jdk" jdkName="Poetry (alpina) (4)" jdkType="Python SDK" /> <orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" /> <orderEntry type="sourceFolder" forTests="false" />
</component> </component>
<component name="PyDocumentationSettings"> <component name="PyDocumentationSettings">

16
.idea/jsonSchemas.xml generated
View File

@@ -39,6 +39,22 @@
</SchemaInfo> </SchemaInfo>
</value> </value>
</entry> </entry>
<entry key="Loki">
<value>
<SchemaInfo>
<option name="name" value="Loki" />
<option name="relativePathToSchema" value="https://json.schemastore.org/loki.json" />
<option name="applicationDefined" value="true" />
<option name="patterns">
<list>
<Item>
<option name="path" value="roles/alpina/templates/services/monitoring/loki_config/loki-config.yaml.j2" />
</Item>
</list>
</option>
</SchemaInfo>
</value>
</entry>
<entry key="Traefik v2"> <entry key="Traefik v2">
<value> <value>
<SchemaInfo> <SchemaInfo>

2
.idea/misc.xml generated
View File

@@ -3,5 +3,5 @@
<component name="Black"> <component name="Black">
<option name="sdkName" value="Poetry (alpina) (2)" /> <option name="sdkName" value="Poetry (alpina) (2)" />
</component> </component>
<component name="ProjectRootManager" version="2" project-jdk-name="Poetry (alpina) (4)" project-jdk-type="Python SDK" /> <component name="ProjectRootManager" version="2" project-jdk-name="Poetry (alpina)" project-jdk-type="Python SDK" />
</project> </project>

4
roles/alpina/templates/services/monitoring/Dockerfile
View File

@@ -4,6 +4,10 @@ RUN pip install grafanalib
COPY ./grafana_config/dashboards /dashboards COPY ./grafana_config/dashboards /dashboards
# Required for grafanalib to find the shared python files like common.py
# https://github.com/weaveworks/grafanalib/issues/58
ENV PYTHONPATH=/dashboards
RUN generate-dashboards /dashboards/*.dashboard.py RUN generate-dashboards /dashboards/*.dashboard.py
FROM grafana/grafana:latest FROM grafana/grafana:latest

5
roles/alpina/templates/services/monitoring/docker-compose.yml.j2
View File

@@ -100,6 +100,11 @@ services:
image: gcr.io/cadvisor/cadvisor:latest image: gcr.io/cadvisor/cadvisor:latest
container_name: cadvisor container_name: cadvisor
restart: unless-stopped restart: unless-stopped
command:
- --docker_only=true
- --store_container_labels=false
- --whitelisted_container_labels=com.docker.compose.project,com.docker.compose.service
- --enable_metrics=cpu,cpuLoad,diskIO,memory,network,oom_event,process
volumes: volumes:
- /:/rootfs:ro - /:/rootfs:ro
- /var/run:/var/run:rw - /var/run:/var/run:rw

27
roles/alpina/templates/services/monitoring/grafana_config/dashboards/common.py Normal file
View File

@@ -0,0 +1,27 @@
from grafanalib.core import Template
# TODO: consider default params for common params like line width, show points, tooltip
PrometheusTemplate = Template(
name='datasource',
type='datasource',
label='Prometheus',
query='prometheus',
)
# TODO: this slightly less (clown emoji), normal Target gave me errors in grafana
class LokiTarget(object):
def __init__(self, loki_datasource, expr, legendFormat, refId):
self.loki_datasource = loki_datasource
self.expr = expr
self.legendFormat = legendFormat
self.refId = refId
def to_json_data(self):
return {
'datasource': self.loki_datasource,
'expr': self.expr,
'legendFormat': self.legendFormat,
'refId': self.refId,
'queryType': 'range',
}

39
roles/alpina/templates/services/monitoring/grafana_config/dashboards/containers.dashboard.py
View File

@@ -5,28 +5,21 @@ from grafanalib.core import (
) )
from grafanalib.formatunits import BYTES_IEC, SECONDS, BYTES_SEC_IEC from grafanalib.formatunits import BYTES_IEC, SECONDS, BYTES_SEC_IEC
prom_datasource='prometheus' from common import LokiTarget, PrometheusTemplate
loki_datasource='loki'
# TODO: this is (clown emoji), normal Target gave me errors in grafana prom_datasource='${datasource}'
class LokiTarget(object): loki_datasource='loki'
def to_json_data(self):
return {
'datasource': loki_datasource,
'expr': '{compose_project=~"$compose_project", container_name=~"$container_name"} |= `$logs_query`',
'legendFormat': '{{ container_name }}',
'refId': 'A',
'queryType': 'range',
}
dashboard = Dashboard( dashboard = Dashboard(
title='Containers', title='Containers',
uid='containers', uid='containers',
description='Data for compose projects from default Prometheus datasource collected by Cadvisor', description='Data for compose projects from default Prometheus datasource collected by Cadvisor',
tags=[ tags=[
'example' 'linux',
'docker',
], ],
templating=Templating(list=[ templating=Templating(list=[
PrometheusTemplate,
Template( Template(
name='compose_project', name='compose_project',
label='Compose Project', label='Compose Project',
@@ -44,7 +37,6 @@ dashboard = Dashboard(
includeAll=True, includeAll=True,
multi=True, multi=True,
refresh=REFRESH_ON_TIME_RANGE_CHANGE, refresh=REFRESH_ON_TIME_RANGE_CHANGE,
), ),
Template( Template(
name='logs_query', name='logs_query',
@@ -56,7 +48,6 @@ dashboard = Dashboard(
timezone='browser', timezone='browser',
panels=[ panels=[
TimeSeries( TimeSeries(
id=1,
title='Container Memory Usage', title='Container Memory Usage',
unit=BYTES_IEC, unit=BYTES_IEC,
gridPos=GridPos(h=8, w=12, x=0, y=0), gridPos=GridPos(h=8, w=12, x=0, y=0),
@@ -76,13 +67,14 @@ dashboard = Dashboard(
], ],
), ),
TimeSeries( TimeSeries(
id=2,
title='Container CPU Usage', title='Container CPU Usage',
unit=SECONDS, unit=SECONDS,
gridPos=GridPos(h=8, w=12, x=12, y=0), gridPos=GridPos(h=8, w=12, x=12, y=0),
lineWidth=2, lineWidth=2,
fillOpacity=10, fillOpacity=10,
showPoints='never', showPoints='never',
tooltipMode='all',
tooltipSort='desc',
targets=[ targets=[
Target( Target(
datasource=prom_datasource, datasource=prom_datasource,
@@ -93,7 +85,6 @@ dashboard = Dashboard(
], ],
), ),
TimeSeries( TimeSeries(
id=3,
title='Container Network Traffic', title='Container Network Traffic',
unit=BYTES_SEC_IEC, unit=BYTES_SEC_IEC,
gridPos=GridPos(h=8, w=12, x=0, y=8), gridPos=GridPos(h=8, w=12, x=0, y=8),
@@ -118,7 +109,6 @@ dashboard = Dashboard(
], ],
), ),
Logs( Logs(
id=4,
title='', title='',
gridPos=GridPos(h=8, w=12, x=12, y=8), gridPos=GridPos(h=8, w=12, x=12, y=8),
showLabels=True, showLabels=True,
@@ -127,13 +117,12 @@ dashboard = Dashboard(
prettifyLogMessage=True, prettifyLogMessage=True,
dedupStrategy='numbers', dedupStrategy='numbers',
targets=[ targets=[
LokiTarget(), LokiTarget(
# Target( loki_datasource=loki_datasource,
# datasource=loki_datasource, expr='{compose_project=~"$compose_project", container_name=~"$container_name"} |= `$logs_query`',
# expr='{compose_project=~"$compose_project", container_name=~"$container_name"} |= `$logs_query`', legendFormat='{{ container_name }}',
# legendFormat='{{ container_name }}', refId='A',
# refId='A', ),
# ),
], ],
), ),
], ],

139
roles/alpina/templates/services/monitoring/grafana_config/dashboards/node.dashboard.py Normal file
View File

@@ -0,0 +1,139 @@
from grafanalib.core import Dashboard, Templating, Template, TimeSeries, PERCENT_UNIT_FORMAT, GridPos, Target
from grafanalib.formatunits import BYTES_IEC
from common import PrometheusTemplate
from node_consts import CPU_BASIC_COLORS, MEMORY_BASIC_COLORS
dashboard = Dashboard(
title='Node Exporter',
uid='node',
description='Node Exporter (not quite full)',
tags=[
'linux',
],
timezone='browser',
templating=Templating(list=[
# Datasource
PrometheusTemplate,
# Job
Template(
name='job',
label='Job',
dataSource='${datasource}',
query='label_values(node_uname_info, job)',
),
# Instance
Template(
name='instance',
label='Instance',
dataSource='${datasource}',
query='label_values(node_uname_info{job="$job"}, instance)',
),
]),
panels=[
# CPU Basic
TimeSeries(
title='CPU Basic',
description='Basic CPU usage info',
unit=PERCENT_UNIT_FORMAT,
gridPos=GridPos(h=8, w=12, x=0, y=0),
lineWidth=1,
fillOpacity=30,
showPoints='never',
stacking={'mode': 'percent', 'group': 'A'},
tooltipMode='all',
tooltipSort='desc',
targets=[
Target(
datasource='${datasource}',
expr='sum(irate(node_cpu_seconds_total{instance="$instance",job="$job", mode="system"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance="$instance",job="$job"}) by (cpu)))',
legendFormat='Busy System',
refId='A',
),
Target(
datasource='${datasource}',
expr='sum(irate(node_cpu_seconds_total{instance="$instance",job="$job", mode="user"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance="$instance",job="$job"}) by (cpu)))',
legendFormat='Busy User',
refId='B',
),
Target(
datasource='${datasource}',
expr='sum(irate(node_cpu_seconds_total{instance="$instance",job="$job", mode="iowait"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance="$instance",job="$job"}) by (cpu)))',
legendFormat='Busy Iowait',
refId='C',
),
Target(
datasource='${datasource}',
expr='sum(irate(node_cpu_seconds_total{instance="$instance",job="$job", mode=~".*irq"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance="$instance",job="$job"}) by (cpu)))',
legendFormat='Busy IRQs',
refId='D',
),
Target(
datasource='${datasource}',
expr='sum(irate(node_cpu_seconds_total{instance="$instance",job="$job", mode!="idle",mode!="user",mode!="system",mode!="iowait",mode!="irq",mode!="softirq"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance="$instance",job="$job"}) by (cpu)))',
legendFormat='Busy Other',
refId='E',
),
Target(
datasource='${datasource}',
expr='sum(irate(node_cpu_seconds_total{instance="$instance",job="$job", mode="idle"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance="$instance",job="$job"}) by (cpu)))',
legendFormat='Idle',
refId='F',
),
],
# Extra JSON for the colors
extraJson=CPU_BASIC_COLORS,
),
# Memory Basic
TimeSeries(
title='Memory Basic',
description='Basic memory usage',
unit=BYTES_IEC,
gridPos=GridPos(h=8, w=12, x=12, y=0),
lineWidth=1,
fillOpacity=30,
showPoints='never',
stacking={'mode': 'normal', 'group': 'A'},
tooltipMode='all',
tooltipSort='desc',
targets=[
Target(
datasource='${datasource}',
expr='node_memory_MemTotal_bytes{instance="$instance",job="$job"}',
format='time_series',
legendFormat='RAM Total',
refId='A',
),
Target(
datasource='${datasource}',
expr='node_memory_MemTotal_bytes{instance="$instance",job="$job"} - node_memory_MemFree_bytes{instance="$instance",job="$job"} - (node_memory_Cached_bytes{instance="$instance",job="$job"} + node_memory_Buffers_bytes{instance="$instance",job="$job"} + node_memory_SReclaimable_bytes{instance="$instance",job="$job"})',
format='time_series',
legendFormat='RAM Used',
refId='B',
),
Target(
datasource='${datasource}',
expr='node_memory_Cached_bytes{instance="$instance",job="$job"} + node_memory_Buffers_bytes{instance="$instance",job="$job"} + node_memory_SReclaimable_bytes{instance="$instance",job="$job"}',
legendFormat='RAM Cache + Buffer',
refId='C',
),
Target(
datasource='${datasource}',
expr='node_memory_MemFree_bytes{instance="$instance",job="$job"}',
legendFormat='RAM Free',
refId='D',
),
Target(
datasource='${datasource}',
expr='(node_memory_SwapTotal_bytes{instance="$instance",job="$job"} - node_memory_SwapFree_bytes{instance="$instance",job="$job"})',
legendFormat='SWAP Used',
refId='E',
),
],
# Extra JSON for the colors
extraJson=MEMORY_BASIC_COLORS,
),
# TODO: Network Basic
# TODO: Disk Basic
],
).auto_panel_ids()

487
roles/alpina/templates/services/monitoring/grafana_config/dashboards/node_consts.py Normal file
View File

@@ -0,0 +1,487 @@
# TODO: Question life decisions (I'm not sure if this is good)
CPU_BASIC_COLORS = {
"fieldConfig": {
"overrides": [
{
"matcher": {
"id": "byName",
"options": "Busy Iowait"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#890F02",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Idle"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#052B51",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Busy Iowait"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#890F02",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Idle"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#7EB26D",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Busy System"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#EAB839",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Busy User"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#0A437C",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Busy Other"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#6D1F62",
"mode": "fixed"
}
}
]
}
]
},
}
MEMORY_BASIC_COLORS = {
"fieldConfig": {
"overrides": [
{
"matcher": {
"id": "byName",
"options": "Apps"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#629E51",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Buffers"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#614D93",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Cache"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#6D1F62",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Cached"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#511749",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Committed"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#508642",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Free"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#0A437C",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#CFFAFF",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Inactive"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#584477",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "PageTables"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#0A50A1",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Page_Tables"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#0A50A1",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "RAM_Free"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#E0F9D7",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "SWAP Used"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#BF1B00",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Slab"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#806EB7",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Slab_Cache"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#E0752D",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Swap"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#BF1B00",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Swap Used"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#BF1B00",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Swap_Cache"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#C15C17",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Swap_Free"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#2F575E",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Unused"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#EAB839",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "RAM Total"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#E0F9D7",
"mode": "fixed"
}
},
{
"id": "custom.fillOpacity",
"value": 0
},
{
"id": "custom.stacking",
"value": {
"group": False,
"mode": "normal"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "RAM Cache + Buffer"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#052B51",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "RAM Free"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#7EB26D",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Available"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#DEDAF7",
"mode": "fixed"
}
},
{
"id": "custom.fillOpacity",
"value": 0
},
{
"id": "custom.stacking",
"value": {
"group": False,
"mode": "normal"
}
}
]
}
]
}
}

7
roles/alpina/templates/services/monitoring/loki_config/loki-config.yaml.j2
View File

@@ -17,13 +17,6 @@ common:
schema_config: schema_config:
configs: configs:
- from: 2020-10-24
store: boltdb-shipper
object_store: filesystem
schema: v12
index:
prefix: index_
period: 24h
- from: 2024-10-18 - from: 2024-10-18
index: index:
period: 24h period: 24h

2
roles/alpina/templates/services/monitoring/prometheus_config/prometheus.yml.j2
View File

@@ -43,7 +43,7 @@ scrape_configs:
- 'demo.promlabs.com:10002' - 'demo.promlabs.com:10002'
rule_files: rule_files:
- "/etc/prometheus/container.alerts.yml" {# - "/etc/prometheus/container.alerts.yml"#}
- "/etc/prometheus/extra/rules/*.yml" - "/etc/prometheus/extra/rules/*.yml"
- "/etc/prometheus/extra/rules/*.json" - "/etc/prometheus/extra/rules/*.json"

1
services.yml
View File

@@ -12,3 +12,4 @@
networks: true networks: true
volumes: true volumes: true
builder_cache: true builder_cache: true
when: false