1 // Copyright 2018 The Prometheus Authors
2 // Licensed under the Apache License, Version 2.0 (the "License");
3 // you may not use this file except in compliance with the License.
4 // You may obtain a copy of the License at
6 // http://www.apache.org/licenses/LICENSE-2.0
8 // Unless required by applicable law or agreed to in writing, software
9 // distributed under the License is distributed on an "AS IS" BASIS,
10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 // See the License for the specific language governing permissions and
12 // limitations under the License.
16 // While implementing parsing of /proc/[pid]/mountstats, this blog was used
17 // heavily as a reference:
18 // https://utcc.utoronto.ca/~cks/space/blog/linux/NFSMountstatsIndex
20 // Special thanks to Chris Siebenmann for all of his posts explaining the
21 // various statistics available for NFS.
32 // Constants shared between multiple functions.
42 fieldTransport10TCPLen = 10
43 fieldTransport10UDPLen = 7
45 fieldTransport11TCPLen = 13
46 fieldTransport11UDPLen = 10
49 // A Mount is a device mount parsed from /proc/[pid]/mountstats.
51 // Name of the device.
53 // The mount point of the device.
55 // The filesystem type used by the device.
57 // If available additional statistics related to this Mount.
58 // Use a type assertion to determine if additional statistics are available.
62 // A MountStats is a type which contains detailed statistics for a specific
64 type MountStats interface {
68 // A MountStatsNFS is a MountStats implementation for NFSv3 and v4 mounts.
69 type MountStatsNFS struct {
70 // The version of statistics provided.
72 // The optional mountaddr of the NFS mount.
74 // The age of the NFS mount.
76 // Statistics related to byte counters for various operations.
78 // Statistics related to various NFS event occurrences.
80 // Statistics broken down by filesystem operation.
81 Operations []NFSOperationStats
82 // Statistics about the NFS RPC transport.
83 Transport NFSTransportStats
86 // mountStats implements MountStats.
87 func (m MountStatsNFS) mountStats() {}
89 // A NFSBytesStats contains statistics about the number of bytes read and written
90 // by an NFS client to and from an NFS server.
91 type NFSBytesStats struct {
92 // Number of bytes read using the read() syscall.
94 // Number of bytes written using the write() syscall.
96 // Number of bytes read using the read() syscall in O_DIRECT mode.
98 // Number of bytes written using the write() syscall in O_DIRECT mode.
100 // Number of bytes read from the NFS server, in total.
102 // Number of bytes written to the NFS server, in total.
104 // Number of pages read directly via mmap()'d files.
106 // Number of pages written directly via mmap()'d files.
110 // A NFSEventsStats contains statistics about NFS event occurrences.
111 type NFSEventsStats struct {
112 // Number of times cached inode attributes are re-validated from the server.
113 InodeRevalidate uint64
114 // Number of times cached dentry nodes are re-validated from the server.
115 DnodeRevalidate uint64
116 // Number of times an inode cache is cleared.
117 DataInvalidate uint64
118 // Number of times cached inode attributes are invalidated.
119 AttributeInvalidate uint64
120 // Number of times files or directories have been open()'d.
122 // Number of times a directory lookup has occurred.
124 // Number of times permissions have been checked.
126 // Number of updates (and potential writes) to pages.
128 // Number of pages read directly via mmap()'d files.
130 // Number of times a group of pages have been read.
132 // Number of pages written directly via mmap()'d files.
134 // Number of times a group of pages have been written.
136 // Number of times directory entries have been read with getdents().
138 // Number of times attributes have been set on inodes.
140 // Number of pending writes that have been forcefully flushed to the server.
142 // Number of times fsync() has been called on directories and files.
144 // Number of times locking has been attempted on a file.
146 // Number of times files have been closed and released.
147 VFSFileRelease uint64
148 // Unknown. Possibly unused.
149 CongestionWait uint64
150 // Number of times files have been truncated.
152 // Number of times a file has been grown due to writes beyond its existing end.
153 WriteExtension uint64
154 // Number of times a file was removed while still open by another process.
156 // Number of times the NFS server gave less data than expected while reading.
158 // Number of times the NFS server wrote less data than expected while writing.
160 // Number of times the NFS server indicated EJUKEBOX; retrieving data from
163 // Number of NFS v4.1+ pNFS reads.
165 // Number of NFS v4.1+ pNFS writes.
169 // A NFSOperationStats contains statistics for a single operation.
170 type NFSOperationStats struct {
171 // The name of the operation.
173 // Number of requests performed for this operation.
175 // Number of times an actual RPC request has been transmitted for this operation.
177 // Number of times a request has had a major timeout.
179 // Number of bytes sent for this operation, including RPC headers and payload.
181 // Number of bytes received for this operation, including RPC headers and payload.
183 // Duration all requests spent queued for transmission before they were sent.
184 CumulativeQueueTime time.Duration
185 // Duration it took to get a reply back after the request was transmitted.
186 CumulativeTotalResponseTime time.Duration
187 // Duration from when a request was enqueued to when it was completely handled.
188 CumulativeTotalRequestTime time.Duration
191 // A NFSTransportStats contains statistics for the NFS mount RPC requests and
193 type NFSTransportStats struct {
194 // The transport protocol used for the NFS mount.
196 // The local port used for the NFS mount.
198 // Number of times the client has had to establish a connection from scratch
199 // to the NFS server.
201 // Number of times the client has made a TCP connection to the NFS server.
203 // Duration (in jiffies, a kernel internal unit of time) the NFS mount has
204 // spent waiting for connections to the server to be established.
205 ConnectIdleTime uint64
206 // Duration since the NFS mount last saw any RPC traffic.
207 IdleTime time.Duration
208 // Number of RPC requests for this mount sent to the NFS server.
210 // Number of RPC responses for this mount received from the NFS server.
212 // Number of times the NFS server sent a response with a transaction ID
213 // unknown to this client.
214 BadTransactionIDs uint64
215 // A running counter, incremented on each request as the current difference
216 // ebetween sends and receives.
217 CumulativeActiveRequests uint64
218 // A running counter, incremented on each request by the current backlog
220 CumulativeBacklog uint64
222 // Stats below only available with stat version 1.1.
224 // Maximum number of simultaneously active RPC requests ever used.
225 MaximumRPCSlotsUsed uint64
226 // A running counter, incremented on each request as the current size of the
228 CumulativeSendingQueue uint64
229 // A running counter, incremented on each request as the current size of the
231 CumulativePendingQueue uint64
234 // parseMountStats parses a /proc/[pid]/mountstats file and returns a slice
235 // of Mount structures containing detailed information about each mount.
236 // If available, statistics for each mount are parsed as well.
237 func parseMountStats(r io.Reader) ([]*Mount, error) {
240 statVersionPrefix = "statvers="
248 s := bufio.NewScanner(r)
250 // Only look for device entries in this function
251 ss := strings.Fields(string(s.Bytes()))
252 if len(ss) == 0 || ss[0] != device {
256 m, err := parseMount(ss)
261 // Does this mount also possess statistics information?
262 if len(ss) > deviceEntryLen {
263 // Only NFSv3 and v4 are supported for parsing statistics
264 if m.Type != nfs3Type && m.Type != nfs4Type {
265 return nil, fmt.Errorf("cannot parse MountStats for fstype %q", m.Type)
268 statVersion := strings.TrimPrefix(ss[8], statVersionPrefix)
270 stats, err := parseMountStatsNFS(s, statVersion)
278 mounts = append(mounts, m)
281 return mounts, s.Err()
284 // parseMount parses an entry in /proc/[pid]/mountstats in the format:
285 // device [device] mounted on [mount] with fstype [type]
286 func parseMount(ss []string) (*Mount, error) {
287 if len(ss) < deviceEntryLen {
288 return nil, fmt.Errorf("invalid device entry: %v", ss)
291 // Check for specific words appearing at specific indices to ensure
292 // the format is consistent with what we expect
298 {i: 2, s: "mounted"},
304 for _, f := range format {
306 return nil, fmt.Errorf("invalid device entry: %v", ss)
317 // parseMountStatsNFS parses a MountStatsNFS by scanning additional information
318 // related to NFS statistics.
319 func parseMountStatsNFS(s *bufio.Scanner, statVersion string) (*MountStatsNFS, error) {
320 // Field indicators for parsing specific types of data
324 fieldBytes = "bytes:"
325 fieldEvents = "events:"
326 fieldPerOpStats = "per-op"
327 fieldTransport = "xprt:"
330 stats := &MountStatsNFS{
331 StatVersion: statVersion,
335 ss := strings.Fields(string(s.Bytes()))
340 return nil, fmt.Errorf("not enough information for NFS stats: %v", ss)
345 for _, opt := range strings.Split(ss[1], ",") {
346 split := strings.Split(opt, "=")
347 if len(split) == 2 && split[0] == "mountaddr" {
348 stats.MountAddress = split[1]
352 // Age integer is in seconds
353 d, err := time.ParseDuration(ss[1] + "s")
360 bstats, err := parseNFSBytesStats(ss[1:])
365 stats.Bytes = *bstats
367 estats, err := parseNFSEventsStats(ss[1:])
372 stats.Events = *estats
375 return nil, fmt.Errorf("not enough information for NFS transport stats: %v", ss)
378 tstats, err := parseNFSTransportStats(ss[1:], statVersion)
383 stats.Transport = *tstats
386 // When encountering "per-operation statistics", we must break this
387 // loop and parse them separately to ensure we can terminate parsing
388 // before reaching another device entry; hence why this 'if' statement
389 // is not just another switch case
390 if ss[0] == fieldPerOpStats {
395 if err := s.Err(); err != nil {
399 // NFS per-operation stats appear last before the next device entry
400 perOpStats, err := parseNFSOperationStats(s)
405 stats.Operations = perOpStats
410 // parseNFSBytesStats parses a NFSBytesStats line using an input set of
412 func parseNFSBytesStats(ss []string) (*NFSBytesStats, error) {
413 if len(ss) != fieldBytesLen {
414 return nil, fmt.Errorf("invalid NFS bytes stats: %v", ss)
417 ns := make([]uint64, 0, fieldBytesLen)
418 for _, s := range ss {
419 n, err := strconv.ParseUint(s, 10, 64)
427 return &NFSBytesStats{
439 // parseNFSEventsStats parses a NFSEventsStats line using an input set of
441 func parseNFSEventsStats(ss []string) (*NFSEventsStats, error) {
442 if len(ss) != fieldEventsLen {
443 return nil, fmt.Errorf("invalid NFS events stats: %v", ss)
446 ns := make([]uint64, 0, fieldEventsLen)
447 for _, s := range ss {
448 n, err := strconv.ParseUint(s, 10, 64)
456 return &NFSEventsStats{
457 InodeRevalidate: ns[0],
458 DnodeRevalidate: ns[1],
459 DataInvalidate: ns[2],
460 AttributeInvalidate: ns[3],
464 VFSUpdatePage: ns[7],
467 VFSWritePage: ns[10],
468 VFSWritePages: ns[11],
474 VFSFileRelease: ns[17],
475 CongestionWait: ns[18],
477 WriteExtension: ns[20],
481 JukeboxDelay: ns[24],
487 // parseNFSOperationStats parses a slice of NFSOperationStats by scanning
488 // additional information about per-operation statistics until an empty
490 func parseNFSOperationStats(s *bufio.Scanner) ([]NFSOperationStats, error) {
492 // Number of expected fields in each per-operation statistics set
496 var ops []NFSOperationStats
499 ss := strings.Fields(string(s.Bytes()))
501 // Must break when reading a blank line after per-operation stats to
502 // enable top-level function to parse the next device entry
506 if len(ss) != numFields {
507 return nil, fmt.Errorf("invalid NFS per-operations stats: %v", ss)
510 // Skip string operation name for integers
511 ns := make([]uint64, 0, numFields-1)
512 for _, st := range ss[1:] {
513 n, err := strconv.ParseUint(st, 10, 64)
521 ops = append(ops, NFSOperationStats{
522 Operation: strings.TrimSuffix(ss[0], ":"),
524 Transmissions: ns[1],
525 MajorTimeouts: ns[2],
527 BytesReceived: ns[4],
528 CumulativeQueueTime: time.Duration(ns[5]) * time.Millisecond,
529 CumulativeTotalResponseTime: time.Duration(ns[6]) * time.Millisecond,
530 CumulativeTotalRequestTime: time.Duration(ns[7]) * time.Millisecond,
537 // parseNFSTransportStats parses a NFSTransportStats line using an input set of
538 // integer fields matched to a specific stats version.
539 func parseNFSTransportStats(ss []string, statVersion string) (*NFSTransportStats, error) {
540 // Extract the protocol field. It is the only string value in the line
546 var expectedLength int
547 if protocol == "tcp" {
548 expectedLength = fieldTransport10TCPLen
549 } else if protocol == "udp" {
550 expectedLength = fieldTransport10UDPLen
552 return nil, fmt.Errorf("invalid NFS protocol \"%s\" in stats 1.0 statement: %v", protocol, ss)
554 if len(ss) != expectedLength {
555 return nil, fmt.Errorf("invalid NFS transport stats 1.0 statement: %v", ss)
558 var expectedLength int
559 if protocol == "tcp" {
560 expectedLength = fieldTransport11TCPLen
561 } else if protocol == "udp" {
562 expectedLength = fieldTransport11UDPLen
564 return nil, fmt.Errorf("invalid NFS protocol \"%s\" in stats 1.1 statement: %v", protocol, ss)
566 if len(ss) != expectedLength {
567 return nil, fmt.Errorf("invalid NFS transport stats 1.1 statement: %v", ss)
570 return nil, fmt.Errorf("unrecognized NFS transport stats version: %q", statVersion)
573 // Allocate enough for v1.1 stats since zero value for v1.1 stats will be okay
574 // in a v1.0 response. Since the stat length is bigger for TCP stats, we use
575 // the TCP length here.
577 // Note: slice length must be set to length of v1.1 stats to avoid a panic when
578 // only v1.0 stats are present.
579 // See: https://github.com/prometheus/node_exporter/issues/571.
580 ns := make([]uint64, fieldTransport11TCPLen)
581 for i, s := range ss {
582 n, err := strconv.ParseUint(s, 10, 64)
590 // The fields differ depending on the transport protocol (TCP or UDP)
591 // From https://utcc.utoronto.ca/%7Ecks/space/blog/linux/NFSMountstatsXprt
593 // For the udp RPC transport there is no connection count, connect idle time,
594 // or idle time (fields #3, #4, and #5); all other fields are the same. So
595 // we set them to 0 here.
596 if protocol == "udp" {
597 ns = append(ns[:2], append(make([]uint64, 3), ns[2:]...)...)
600 return &NFSTransportStats{
605 ConnectIdleTime: ns[3],
606 IdleTime: time.Duration(ns[4]) * time.Second,
609 BadTransactionIDs: ns[7],
610 CumulativeActiveRequests: ns[8],
611 CumulativeBacklog: ns[9],
612 MaximumRPCSlotsUsed: ns[10],
613 CumulativeSendingQueue: ns[11],
614 CumulativePendingQueue: ns[12],