Repository navigation
Expand file tree
/
Copy pathbootstrap.sh
More file actions
executable file
·179 lines (163 loc) · 9.3 KB
/
Copy pathbootstrap.sh
File metadata and controls
executable file
·179 lines (163 loc) · 9.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
#!/usr/bin/env bash
# Onboard one project/environment onto this monitoring stack.
#
# ./bootstrap.sh <project> <env>
#
# 1. renders the ProjectTelemetrySilent rule for every covered project/env;
# 2. renders the coverage rule that catches projects never bootstrapped, and
# reloads Grafana;
# 3. creates the project's healthchecks.io watchdog check (if an API key is present);
# 4. prints the copier commands that apply the CML overlay's two layers (copier.yml)
# at a pinned tag: the project layer in the project's repo, and the host layer
# once per machine.
#
# Idempotent.
set -euo pipefail
project="${1:-}"
env_name="${2:-}"
if [[ -z "$project" || -z "$env_name" ]]; then
echo "usage: $0 <project> <env>" >&2
exit 2
fi
# These become label values, a rule uid, and a filename; a quote or brace in a
# label value produces a rule that silently never matches.
valid_pair() { [[ "$1" =~ ^[a-z0-9][a-z0-9-]*$ && "$2" =~ ^[a-z0-9][a-z0-9-]*$ ]]; }
if ! valid_pair "$project" "$env_name"; then
echo "error: project and env must match [a-z0-9][a-z0-9-]* (lowercase, no spaces)" >&2
exit 2
fi
root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
cd "$root"
# A bare ./bootstrap.sh does not load .env, so read single keys out of it.
# Sourcing the whole file would drag the stack's secrets into scope.
env_get() { [[ ! -f .env ]] || sed -n "s/^$1=//p" .env | tail -1; }
if [[ -z "${HEALTHCHECKS_API_KEY:-}" ]]; then
HEALTHCHECKS_API_KEY="$(env_get HEALTHCHECKS_API_KEY)"
export HEALTHCHECKS_API_KEY
fi
# Flat, not a subdirectory: Grafana's alerting provisioner does not recurse, and
# skips a nested directory with only a warning. BOOTSTRAP_OUT_DIR is for `just
# lint`; with it set, nothing past the rendering runs.
out_dir="${BOOTSTRAP_OUT_DIR:-config/grafana/alerting}"
if [[ -z "${BOOTSTRAP_OUT_DIR:-}" ]]; then
# Pinned tag, no fallback to a branch: two projects must never vendor two
# different configs under the same name.
tag="$(git -C "$root" describe --tags --abbrev=0 2>/dev/null)" \
|| { echo "error: no release tag to pin the overlay to; tag a release first" >&2; exit 1; }
git -C "$root" rev-parse -q --verify "${tag}:copier.yml" >/dev/null \
|| { echo "error: tag ${tag} predates the CML overlay (copier.yml); tag a new release before onboarding" >&2; exit 1; }
fi
# ------------------------------------------------------- 1-2. keystone rule and backstop
# Every covered project/env pair, read back from the COVERS marker in the rendered
# file, plus the pair being bootstrapped now.
rendered="${out_dir}/projects.yaml"
pairs="$({ sed -n 's/^# COVERS: //p' "$rendered" 2>/dev/null | tr ' /' '\n ' || true
echo "$project $env_name"; } | sed '/^$/d' | sort -u)"
# The marker comes off disk and lands in a sed replacement and a PromQL label
# value. Refuse the run rather than render a rule that silently never matches.
while read -r p e; do
valid_pair "$p" "$e" \
|| { echo "error: bad '# COVERS:' marker in $rendered: '$p $e'" >&2; exit 2; }
done <<<"$pairs"
covers="$(echo "$pairs" | sed 's| |/|' | paste -sd' ' -)"
selector='s@^\([^ ]*\) \([^ ]*\)$@{__name__=~"telemetry_.+_total", project="\1",env="\2"}@'
silent_expr="$(echo "$pairs" | sed -e "$selector" -e 's/.*/absent(&)/' | paste -sd'@' - | sed 's/@/ or /g')"
covered_expr="$(echo "$pairs" | sed "$selector" | paste -sd'@' - | sed 's/@/ or /g')"
sed -e "s@__COVERS__@${covers}@" \
-e "s@__SILENT_EXPR__@${silent_expr}@" \
-e "s@__COVERED_EXPR__@${covered_expr}@" \
templates/alerting/projects.yaml.tmpl > "$rendered"
echo "rendered $rendered (covering: ${covers})"
[[ -z "${BOOTSTRAP_OUT_DIR:-}" ]] || exit 0
# ------------------------------------------------------------------ 3. reload Grafana
if docker compose ps --status running --services 2>/dev/null | grep -qx grafana; then
# This recreates an exposed Grafana, so the `just up` guards apply here too.
just _guard-if-exposed || exit 1
# A restart, not SIGHUP: Grafana re-reads alert provisioning only at startup.
docker compose up -d --force-recreate grafana >/dev/null 2>&1 \
&& echo "reloaded grafana (restarted)" \
|| echo "WARNING: could not restart grafana; run 'docker compose up -d --force-recreate grafana'" >&2
# Grafana skips a malformed alert group with only a log line, so read the
# rule back.
if [[ -z "${GRAFANA_ADMIN_PASSWORD:-}" ]]; then
GRAFANA_ADMIN_PASSWORD="$(env_get GRAFANA_ADMIN_PASSWORD)"
fi
[[ -n "${GRAFANA_ADMIN_PASSWORD:-}" ]] \
|| { echo "error: GRAFANA_ADMIN_PASSWORD is not set and not readable from .env; cannot verify the rule" >&2; exit 1; }
# curl's config parser unescapes \ and " inside a quoted value, so escape both.
gpw="${GRAFANA_ADMIN_PASSWORD//\\/\\\\}"
gpw="${gpw//\"/\\\"}"
# One rule covers every pair, so read it back and look for this pair's selector.
want="project=\\\"${project}\\\",env=\\\"${env_name}\\\""
verified=""
for _ in $(seq 30); do
sleep 2
# Password on stdin, never argv. A wrong password would otherwise poll
# for a minute and then report the rule as missing.
resp="$(printf 'user = "admin:%s"\n' "$gpw" \
| curl -s -w '\n%{http_code}' -K - "http://localhost:3000/api/v1/provisioning/alert-rules/projects-silent")" || continue
case "${resp##*$'\n'}" in
200)
if grep -qF "$want" <<<"$resp"; then
echo "verified rule projects-silent covers ${project}/${env_name}"
verified=1
break
fi
;;
401 | 403)
echo "error: grafana rejected the admin credentials (HTTP ${resp##*$'\n'}); check GRAFANA_ADMIN_PASSWORD" >&2
exit 1
;;
esac
done
[[ -n "$verified" ]] || { echo "error: grafana restarted but rule projects-silent does not cover ${project}/${env_name}; check 'docker compose logs grafana' for the rejected file" >&2; exit 1; }
else
echo "note grafana is not running; the rules apply next time it starts"
fi
# ------------------------------------------------------- 4. healthchecks.io + printout
if [[ -n "${HEALTHCHECKS_API_KEY:-}" ]]; then
# One watchdog check per project: it reports the project's other scheduled
# jobs, so a check per job would never be pinged. Override with
# HC_JOBS="watchdog nightly-sync" etc. for a job that pings its own check.
for job in ${HC_JOBS:-watchdog}; do
# Key via curl's stdin config, not -H: argv is readable in `ps`.
printf 'header = "X-Api-Key: %s"\n' "$HEALTHCHECKS_API_KEY" \
| curl -fsS -w '\n' -K - -X POST https://healthchecks.io/api/v3/checks/ \
-H "Content-Type: application/json" \
-d "{\"name\":\"${project}-${env_name}-${job}\",\"slug\":\"${project}-${env_name}-${job}\",\"unique\":[\"name\"],\"timeout\":93600,\"grace\":3600,\"channels\":\"*\"}" \
| sed -n 's/.*"ping_url": *"\([^"]*\)".*/ PING_'"$(echo "$job" | tr 'a-z-' 'A-Z_')"'=\1/p'
done
else
echo "note HEALTHCHECKS_API_KEY unset; create these by hand at https://healthchecks.io and note their ping URLs"
fi
cat <<SUMMARY
────────────────────────────────────────────────────────────────────────────
Apply the CML overlay (pinned at ${tag})
────────────────────────────────────────────────────────────────────────────
In the project's repo, the project layer (run_scheduled.sh, telemetry.env.example):
copier copy --vcs-ref=${tag} --data project=${project} --data env=${env_name} \\
gh:CMLPlatform/monitoring .
Once per machine, in the repo that owns it, the host layer (the Alloy agent).
STACKS names every Compose project on the machine as compose-project=project/env:
copier copy --vcs-ref=${tag} --data layer=host \\
--answers-file .copier-answers.cml-host.yml \\
--data project=<owner> --data env=<owner env> --data stacks='<STACKS>' \\
gh:CMLPlatform/monitoring .
On a machine that already has the host layer, add this project to its stacks:
copier update --answers-file .copier-answers.cml-host.yml \\
--data stacks='<STACKS> <compose-project>=${project}/${env_name}'
Start from a clean checkout. Review and commit each layer before updating another.
A later tag reaches either layer as
copier update --answers-file <its answers file>
Add the lines of telemetry.env.example (project) and telemetry-host.env.example
(host) to that repo's .env, with:
OTLP_AUTH_TOKEN=<the stack's shared ingest token>
Then, where the host layer lives, include the overlay and bring it up:
docker compose -f compose.yml -f compose.telemetry.yml up -d
For an NVIDIA host, answer gpu=true when copying the host layer, then run:
docker compose -f compose.yml -f compose.telemetry.yml -f compose.telemetry.gpu.yml up -d
GPU_METRICS=1 alone does not include the GPU file.
Verify from the monitoring host, within ~2 minutes:
count(telemetry_datapoints_total{project="${project}",env="${env_name}"}) -> non-zero
count(container_start_time_seconds{project="${project}",name!=""}) -> one per container
SUMMARY