]>
Commit | Line | Data |
---|---|---|
0d8dd5f2 JR |
1 | #! /usr/bin/perl |
2 | # -*- mode: perl; perl-indent-level: 4 -*- | |
3 | ||
4 | =head1 NAME | |
5 | ||
6 | nvme - Munin plugin to monitor the use of NVMe devices | |
7 | ||
8 | =head1 APPLICABLE SYSTEMS | |
9 | ||
10 | Linux systems with NVMe (Non-Volatile Memory storage attached via PCIe | |
11 | bus). | |
12 | ||
13 | =head1 CONFIGURATION | |
14 | ||
15 | The plugin uses nvme(1) from the nvme-cli project to read status from | |
16 | the NVMe devices. This requires root access. | |
17 | ||
18 | [nvme] | |
19 | user root | |
20 | ||
21 | When setting alert levels per device, use graph and basename of device | |
22 | name, e.g., 'nvme0n1', to make environment variable: | |
23 | ||
24 | env.nvme_usage_nvme0n1_warning 5: | |
25 | env.nvme_usage_warning 8: | |
26 | ||
27 | If your device names change on reboot you can also use the labels | |
28 | (based on serial numbers) to set the warning and critical labels | |
29 | ||
30 | env.nvme_usage_SN_1234567_warning 8:101 | |
31 | env.nvme_usage_SN_1234567_critical 5:101 | |
32 | ||
33 | =head1 INTERPRETATION | |
34 | ||
35 | This is a multigraph plugin which makes three graphs. | |
36 | ||
37 | =head2 nvme_usage | |
38 | ||
39 | This reports how much of capacity is allocated in each NVMe | |
40 | "namespace". The report is in percent. This number may not have much | |
41 | relation to actual use, e.g., if deleted data areas have not been | |
42 | trimmed/discarded. | |
43 | ||
44 | Default warning and critical: '95', '98' | |
45 | ||
46 | =head2 nvme_bytes | |
47 | ||
48 | This reports read and write activity on each NVMe device, in bytes per | |
49 | second. Ideally there should be much more read than write. If they | |
50 | are symmetrical, you are using your NVMe as a very expensive FIFO, and | |
51 | if you write more than you read, you should probably look for archival | |
52 | storage instead. | |
53 | ||
54 | It is a good idea to compare these numbers to I/O counters from | |
55 | diskstats. If they are much higher, look into whether the write | |
56 | amplification can be due to suboptimal I/O request sizes. | |
57 | ||
58 | This graph does not support alerting. | |
59 | ||
60 | =head2 nvme_writecycles | |
61 | ||
62 | This graphs is intended to give an indication of how much life there | |
63 | is left in your NVMe. It calculates the number of bytes written | |
64 | during each device's lifetime against the capacity of the device, | |
65 | thereby getting an average number of write cycle each cell has | |
66 | experienced. | |
67 | ||
68 | A prosumer NVMe will handle a few thousand writes to each cell before | |
69 | the error rate gets out of hand. | |
70 | ||
71 | No default values for warning and critical. | |
72 | ||
73 | =head2 nvme_spare | |
74 | ||
75 | All NVMe has set a side reserve space to remap media errors. This | |
76 | graphs how much is left in percent, taken directly from smart-log | |
77 | output. | |
78 | ||
79 | Default warning and critical: '10:', '3:' | |
80 | ||
81 | =head1 MAGIC MARKERS | |
82 | ||
83 | #%# family=auto | |
84 | #%# capabilities=autoconf | |
85 | ||
86 | =head1 BUGS | |
87 | ||
88 | None known. | |
89 | ||
90 | =head1 VERSION | |
91 | ||
92 | 1.1 | |
93 | ||
94 | =head1 AUTHOR | |
95 | ||
96 | Kjetil Torgrim Homme <kjetil.homme@redpill-linpro.com> | |
97 | ||
98 | =head1 LICENSE | |
99 | ||
100 | GPLv2 | |
101 | ||
102 | =cut | |
103 | ||
104 | use strict; | |
105 | use Munin::Plugin; | |
106 | use IPC::Cmd qw(can_run); | |
107 | use File::Basename; | |
108 | ||
109 | # Check that multigraph is supported | |
110 | need_multigraph(); | |
111 | ||
112 | # Return undef if no problem, otherwise explanation | |
113 | sub autoconf_problem { | |
114 | return if can_run('nvme'); | |
115 | if (open(my $mods, '/proc/modules')) { | |
116 | while (<$mods>) { | |
117 | return "missing nvme(1)" if /^nvme[^a-z]/; | |
118 | } | |
119 | close($mods); | |
120 | } | |
121 | return "missing nvme"; # vague message for non-Linux | |
122 | } | |
123 | ||
124 | sub run_nvme { | |
125 | my (@cmd) = @_; | |
126 | my @lines; | |
127 | if (can_run('nvme') && open(my $nvme, '-|', 'nvme', @cmd)) { | |
128 | @lines = <$nvme>; | |
129 | close($nvme); | |
130 | warn "nvme: probably needs to run as user root\n" if $? && $> != 0; | |
131 | } | |
132 | @lines; | |
133 | } | |
134 | ||
135 | sub human_to_bytes { | |
136 | my ($str) = @_; | |
137 | my %units = ( | |
138 | kB => 1000, | |
139 | MB => 1000_000, | |
140 | GB => 1000_000_000, | |
141 | TB => 1000_000_000_000, | |
142 | PB => 1000_000_000_000_000, # I wish I had need for this | |
143 | ); | |
144 | $str =~ /(\d+(\.\d+)?)\s+(.B)/; | |
145 | int($1 * $units{$3}); | |
146 | } | |
147 | ||
148 | sub nvme_list { | |
149 | # Node SN Model Namespace Usage Format FW Rev | |
150 | # ---------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- -------- | |
151 | # /dev/nvme1n1 S464NB0K601188N Samsung SSD 970 EVO 2TB 1 695.50 GB / 2.00 TB 512 B + 0 B 1B2QEXE7 | |
152 | my %devices; | |
153 | ||
154 | my $recognised_output; | |
155 | my $lineno = 0; | |
156 | for (run_nvme('list')) { | |
157 | ++$lineno; | |
158 | if (m:^Node\s+SN\s+Model\s+Namespace Usage:) { | |
159 | ++$recognised_output; | |
160 | } elsif (m:^(/\S+)\s+(\S+)\s+(\S.*\S)\s{3,}(\d+)\s+(\S+\s+.B)\s+/\s+(\S+\s+.B):) { | |
161 | $devices{'SN_'.$2} = { | |
162 | device => $1, | |
163 | sn => $2, | |
164 | model => $3, | |
165 | namespace => $4, | |
166 | usage => human_to_bytes($5), | |
167 | capacity => human_to_bytes($6), | |
168 | }; | |
169 | } elsif ($lineno > 2) { | |
170 | # could not parse device information | |
171 | $recognised_output = 0; | |
172 | } | |
173 | } | |
174 | if ($lineno && !$recognised_output) { | |
175 | warn "Could not recognise output from 'nvme list', please report\n"; | |
176 | } | |
177 | \%devices; | |
178 | } | |
179 | ||
180 | sub smart_log { | |
181 | my ($dev) = @_; | |
182 | my %info; | |
183 | for (run_nvme('smart-log', $dev)) { | |
184 | next if /^Smart Log/; | |
185 | if (/(.*?)\s+:\s+(.*)/) { | |
186 | my ($var, $value) = ($1, $2); | |
187 | $var =~ s/\s/_/g; | |
188 | if ($value =~ /^\d+(,\d\d\d)+$/) { | |
189 | $value =~ s/,//g; | |
190 | } | |
191 | $info{lc $var} = $value; | |
192 | } | |
193 | } | |
194 | return \%info; | |
195 | } | |
196 | ||
197 | sub my_print_thresholds { | |
198 | my ($label, $graph, $device, $warn_default, $crit_default) = @_; | |
199 | my $dev = basename($device); | |
200 | my ($warn_label, $crit_label) = get_thresholds($graph, "${graph}_${label}_warning", "${graph}_${label}_critical", | |
201 | $warn_default, $crit_default); | |
202 | my ($warn, $crit) = get_thresholds($graph, "${graph}_${dev}_warning", "${graph}_${dev}_critical", | |
203 | $warn_label, $crit_label); | |
204 | print "${label}.warning $warn\n" if defined $warn; | |
205 | print "${label}.critical $crit\n" if defined $crit; | |
206 | } | |
207 | ||
208 | use Data::Dumper; | |
209 | ||
210 | my $mode = ($ARGV[0] or "print"); | |
211 | ||
212 | my $problem = autoconf_problem(); | |
213 | my $list = nvme_list(); | |
214 | ||
215 | if ($mode eq 'autoconf') { | |
216 | if (keys %{$list}) { | |
217 | print "yes\n"; | |
218 | } else { | |
219 | printf("no (%s)\n", $problem || "no devices to monitor"); | |
220 | } | |
221 | exit 0; | |
222 | } | |
223 | ||
224 | my @sn = sort keys %{$list}; | |
225 | ||
226 | if ($mode eq 'config') { | |
227 | my $sn_list = join(' ', @sn); | |
228 | ||
229 | print <<'EOF'; | |
230 | multigraph nvme_usage | |
231 | graph_title NVME Namespace Usage | |
232 | graph_order $sn_list | |
233 | graph_vlabel Percent used | |
234 | graph_scale no | |
235 | graph_category disk | |
236 | graph_info How much space is used | |
237 | EOF | |
238 | for (@sn) { | |
239 | my $device = $list->{$_}->{device}; | |
240 | print <<"EOF"; | |
241 | $_.label $device used | |
242 | $_.type GAUGE | |
243 | $_.max 100 | |
244 | $_.min 0 | |
245 | EOF | |
246 | my_print_thresholds($_, 'nvme_usage', $device, '95', '98'); | |
247 | } | |
248 | print <<'EOF'; | |
249 | multigraph nvme_bytes | |
250 | graph_title NVME Bytes Read / Written | |
251 | graph_order $sn_list | |
252 | graph_vlabel bytes read (-) / written (+) per ${graph_period}' | |
253 | graph_category disk | |
254 | graph_info How much data is read and written | |
255 | graph_period second | |
256 | EOF | |
257 | for (@sn) { | |
258 | print <<"EOF"; | |
259 | ${_}_r.label $list->{$_}->{device} | |
260 | ${_}_r.type DERIVE | |
261 | ${_}_r.min 0 | |
262 | ${_}_r.graph no | |
263 | ${_}_w.label $list->{$_}->{device} | |
264 | ${_}_w.type DERIVE | |
265 | ${_}_w.min 0 | |
266 | ${_}_w.negative ${_}_r | |
267 | EOF | |
268 | } | |
269 | print <<'EOF'; | |
270 | multigraph nvme_writecycles | |
271 | graph_title NVME Write Cycles | |
272 | graph_order $sn_list | |
273 | graph_vlabel Cycles | |
274 | graph_args --logarithmic | |
275 | graph_category disk | |
276 | graph_info How much data has been written in lifetime divided by capacity | |
277 | EOF | |
278 | for (@sn) { | |
279 | my $device = $list->{$_}->{device}; | |
280 | print <<"EOF"; | |
281 | $_.label $device write cycles | |
282 | $_.type GAUGE | |
283 | $_.min 0 | |
284 | EOF | |
285 | my_print_thresholds($_, 'nvme_writecycles', $device); | |
286 | } | |
287 | print <<'EOF'; | |
288 | multigraph nvme_spare | |
289 | graph_title Available spare blocks | |
290 | graph_order $sn_list | |
291 | graph_vlabel Percent | |
292 | graph_category disk | |
293 | graph_info Spare capacity for replacing bad blocks | |
294 | EOF | |
295 | for (@sn) { | |
296 | my $device = $list->{$_}->{device}; | |
297 | print <<"EOF"; | |
298 | $_.label $device spare capacity | |
299 | $_.type GAUGE | |
300 | $_.min 0 | |
301 | $_.max 100 | |
302 | EOF | |
303 | my_print_thresholds($_, 'nvme_spare', $device, '10:', '3:'); | |
304 | } | |
305 | } else { | |
306 | for (@sn) { | |
307 | $list->{$_}->{smart} = smart_log($list->{$_}->{device}); | |
308 | } | |
309 | print "multigraph nvme_usage\n"; | |
310 | for (@sn) { | |
311 | my $info = $list->{$_}; | |
312 | my $used = 100 * $info->{usage} / $info->{capacity}; | |
313 | print "$_.value $used\n"; | |
314 | } | |
315 | print "multigraph nvme_bytes\n"; | |
316 | for (@sn) { | |
317 | my $info = $list->{$_}; | |
318 | my $rbytes = $info->{smart}->{data_units_read}; | |
319 | my $wbytes = $info->{smart}->{data_units_written}; | |
320 | print "${_}_r.value $rbytes\n"; | |
321 | print "${_}_w.value $wbytes\n"; | |
322 | } | |
323 | print "multigraph nvme_writecycles\n"; | |
324 | for (@sn) { | |
325 | my $info = $list->{$_}; | |
326 | ||
327 | # The unit size reported is 1000 blocks. | |
328 | my $cycles = $info->{smart}->{data_units_written} * 512_000 / $info->{capacity}; | |
329 | print "$_.value $cycles\n"; | |
330 | } | |
331 | print "multigraph nvme_spare\n"; | |
332 | for (@sn) { | |
333 | my $info = $list->{$_}; | |
334 | ||
335 | # The unit size reported is 1000 blocks. | |
336 | my $spare = $info->{smart}->{available_spare}; | |
337 | $spare =~ s/%//; | |
338 | print "$_.value $spare\n"; | |
339 | } | |
340 | } |