ghūl coverage report

Appearance

← Back

src/analysis/watchdog.ghul

1
namespace Analysis is
2
use IO.Std;
3
4
// Decides when the long-lived analyser should exit so the IDE can spawn a
5
// fresh process. Two independent triggers:
6
//
7
// - heap growth: the managed heap is sampled after every full compile.
8
// The first few readings are warm-up — the JIT, interned symbols and
9
// reflection-metadata caches are still filling — so the baseline is not
10
// fixed until that has settled: it is the high-water reading across the
11
// first warm_up_compiles compiles. Past that, the compiler retains state
12
// across compiles (ASTs, caches) that clear_symbols does not all
13
// reclaim, so a steady climb is the expected leak signature, and a
14
// recycle returns that memory to a fresh process.
15
//
16
// - instability: a burst of exceptions escaping command handlers. Isolated
17
// failures decay away; a sustained run of them means the analyser's
18
// state is corrupt and a fresh process is the only recovery.
19
class WATCHDOG is
20
// Recycle once the heap has grown past both thresholds relative to the
21
// baseline: a multiple of it, and an absolute floor so a small baseline
22
// can't trip a recycle on ordinary working-set noise.
23
heap_growth_factor: double static => 2.0D;
24
heap_growth_floor_bytes: long static => 256L * 1024L * 1024L;
25
26
// Consecutive failing operations needed to trip an instability recycle.
27
// request_restart() adds 2 per failure, a clean operation decays 1, so
28
// this is reached by roughly this many failures in an unbroken run.
29
instability_limit: int static => 20;
30
31
// How often to echo a heap reading to the log (every Nth full compile).
32
heap_log_interval: int static => 32;
33
34
// Once the baseline is fixed, the heap is sampled only every Nth full
35
// compile, not after every one — check_heap forces a full GC, too
36
// costly to run that often. Warm-up compiles (before the baseline is
37
// fixed) are always sampled: the baseline is their high-water reading.
38
heap_check_interval: int static => 32;
39
40
// Full compiles to let pass before the baseline is fixed. The analyser's
41
// working set climbs over the first compiles as the JIT and caches fill
42
// — growth that is not a leak — so anchoring the baseline to the cold
43
// first reading would let ordinary warm-up trip a recycle.
44
warm_up_compiles: int static => 5;
45
46
// When set, check_heap is skipped — the analyser does not sample the
47
// heap or force a GC after compiles. Driven by --no-analysis-heap-
48
// watchdog; intended for profiling, where the forced GC dominates.
49
heap_check_disabled: bool public;
50
51
_restart_score: int;
52
53
_have_baseline: bool;
54
_baseline_heap_bytes: long;
55
_warm_up_peak_heap_bytes: long;
56
_full_compile_pending: bool;
57
_compile_count: int;
58
_compiles_since_heap_check: int;
59
60
want_restart: bool;
61
62
init() is si
63
64
// An exception escaped a command handler. Nudge the instability score;
65
// a sustained run of failures trips want_restart, isolated ones decay.
66
request_restart() is
67
_restart_score = _restart_score + 2;
68
69
if _restart_score > instability_limit then
70
want_restart = true;
71
fi
72
si
73
74
// A full project compile completed successfully. The heap reading is
75
// taken at the next operation boundary (on_operation_complete), once
76
// the in-flight response frame has been flushed.
77
note_full_compile() is
78
_full_compile_pending = true;
79
_compile_count = _compile_count + 1;
80
_compiles_since_heap_check = _compiles_since_heap_check + 1;
81
si
82
83
// Called by the despatcher after every command, once the handler has
84
// returned and its response is on the wire. This is the only place a
85
// recycle is issued, so a recycle never truncates a response frame.
86
on_operation_complete(writer: IO.TextWriter) is
87
if want_restart then
88
recycle(writer, "instability");
89
fi
90
91
if _restart_score > 0 then
92
_restart_score = _restart_score - 1;
93
fi
94
95
if _full_compile_pending then
96
_full_compile_pending = false;
97
98
if !heap_check_disabled /\ should_check_heap() then
99
check_heap(writer);
100
fi
101
fi
102
si
103
104
// Whether the every-Nth-compile fallback should sample the heap now.
105
// Always during warm-up (the baseline needs every reading); afterwards
106
// once heap_check_interval compiles have passed since the last sample.
107
// An explicit #HEAPCHECK# resets that count (note_heap_sampled), so a
108
// recent IDE-driven check suppresses a redundant fallback sample.
109
should_check_heap() -> bool =>
110
!_have_baseline \/ _compiles_since_heap_check >= heap_check_interval;
111
112
// Record that the heap was just sampled — restarts the fallback
113
// interval. Called whenever check_heap runs, so both an explicit
114
// #HEAPCHECK# and a fallback sample reset it.
115
note_heap_sampled() is
116
_compiles_since_heap_check = 0;
117
si
118
119
// Sample the heap in response to an explicit IDE request — the VS Code
120
// extension sends one during a lull in editing, so the forced GC lands
121
// outside the latency path of an interactive request. The every-Nth-
122
// compile path above stays as a fallback for clients that never send
123
// it. Respects --no-analysis-heap-watchdog.
124
check_heap_on_request(writer: IO.TextWriter) is
125
if !heap_check_disabled then
126
check_heap(writer);
127
fi
128
si
129
130
check_heap(writer: IO.TextWriter) is
131
note_heap_sampled();
132
133
let heap = System.GC.get_total_memory(true);
134
135
note_heap(heap);
136
137
if is_heap_over_limit(heap) then
138
log_heap("heap", heap);
139
recycle(writer, "heap growth");
140
fi
141
si
142
143
// Fold one full-compile heap reading into the watchdog. Pure with
144
// respect to the system — the caller supplies the reading — so unit
145
// tests drive it directly. During warm-up it tracks the high-water
146
// reading and fixes the baseline to it once warm_up_compiles full
147
// compiles have completed.
148
note_heap(heap: long) is
149
if !_have_baseline then
150
if heap > _warm_up_peak_heap_bytes then
151
_warm_up_peak_heap_bytes = heap;
152
fi
153
154
if _compile_count >= warm_up_compiles then
155
set_baseline(_warm_up_peak_heap_bytes);
156
fi
157
158
return;
159
fi
160
161
if _compile_count % heap_log_interval == 0 then
162
log_heap("heap", heap);
163
fi
164
si
165
166
set_baseline(heap: long) is
167
_have_baseline = true;
168
_baseline_heap_bytes = heap;
169
170
log_heap("baseline", heap);
171
si
172
173
// The recycle decision for a heap reading. Pure — no sampling, no exit —
174
// so it can be exercised directly by unit tests. Both thresholds must be
175
// crossed: a multiple of the baseline and an absolute byte floor.
176
is_heap_over_limit(heap: long) -> bool =>
177
_have_baseline /\
178
heap - _baseline_heap_bytes > heap_growth_floor_bytes /\
179
cast double(heap) > cast double(_baseline_heap_bytes) * heap_growth_factor;
180
181
log_heap(label: string, heap: long) is
182
Std.error.write_line(
183
"watchdog: {label} {heap / 1024L / 1024L} MB (baseline {_baseline_heap_bytes / 1024L / 1024L} MB, {_compile_count} compiles)"
184
);
185
Std.error.flush();
186
si
187
188
recycle(writer: IO.TextWriter, reason: string) is
189
Std.error.write_line("watchdog: recycling analyser ({reason})");
190
Std.error.flush();
191
192
Protocol.JSON_PROTOCOL.write_response(writer, Protocol.Response.RESTART());
193
194
System.Environment.exit(1);
195
si
196
si
197
si