2 Copyright 2018 The Kubernetes Authors.
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
8 http://www.apache.org/licenses/LICENSE-2.0
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
23 "github.com/prometheus/client_golang/prometheus"
24 reflectormetrics "k8s.io/client-go/tools/cache"
25 clientmetrics "k8s.io/client-go/tools/metrics"
26 workqueuemetrics "k8s.io/client-go/util/workqueue"
29 // this file contains setup logic to initialize the myriad of places
30 // that client-go registers metrics. We copy the names and formats
31 // from Kubernetes so that we match the core controllers.
36 requestLatency = prometheus.NewHistogramVec(
37 prometheus.HistogramOpts{
38 Name: "rest_client_request_latency_seconds",
39 Help: "Request latency in seconds. Broken down by verb and URL.",
40 Buckets: prometheus.ExponentialBuckets(0.001, 2, 10),
42 []string{"verb", "url"},
45 requestResult = prometheus.NewCounterVec(
46 prometheus.CounterOpts{
47 Name: "rest_client_requests_total",
48 Help: "Number of HTTP requests, partitioned by status code, method, and host.",
50 []string{"code", "method", "host"},
55 // TODO(directxman12): update these to be histograms once the metrics overhaul KEP
58 reflectorSubsystem = "reflector"
60 listsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
61 Subsystem: reflectorSubsystem,
63 Help: "Total number of API lists done by the reflectors",
66 listsDuration = prometheus.NewSummaryVec(prometheus.SummaryOpts{
67 Subsystem: reflectorSubsystem,
68 Name: "list_duration_seconds",
69 Help: "How long an API list takes to return and decode for the reflectors",
72 itemsPerList = prometheus.NewSummaryVec(prometheus.SummaryOpts{
73 Subsystem: reflectorSubsystem,
74 Name: "items_per_list",
75 Help: "How many items an API list returns to the reflectors",
78 watchesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
79 Subsystem: reflectorSubsystem,
80 Name: "watches_total",
81 Help: "Total number of API watches done by the reflectors",
84 shortWatchesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
85 Subsystem: reflectorSubsystem,
86 Name: "short_watches_total",
87 Help: "Total number of short API watches done by the reflectors",
90 watchDuration = prometheus.NewSummaryVec(prometheus.SummaryOpts{
91 Subsystem: reflectorSubsystem,
92 Name: "watch_duration_seconds",
93 Help: "How long an API watch takes to return and decode for the reflectors",
96 itemsPerWatch = prometheus.NewSummaryVec(prometheus.SummaryOpts{
97 Subsystem: reflectorSubsystem,
98 Name: "items_per_watch",
99 Help: "How many items an API watch returns to the reflectors",
102 lastResourceVersion = prometheus.NewGaugeVec(prometheus.GaugeOpts{
103 Subsystem: reflectorSubsystem,
104 Name: "last_resource_version",
105 Help: "Last resource version seen for the reflectors",
110 workQueueSubsystem = "workqueue"
112 depth = prometheus.NewGaugeVec(prometheus.GaugeOpts{
113 Subsystem: workQueueSubsystem,
115 Help: "Current depth of workqueue",
118 adds = prometheus.NewCounterVec(prometheus.CounterOpts{
119 Subsystem: workQueueSubsystem,
121 Help: "Total number of adds handled by workqueue",
124 latency = prometheus.NewHistogramVec(prometheus.HistogramOpts{
125 Subsystem: workQueueSubsystem,
126 Name: "queue_latency_seconds",
127 Help: "How long in seconds an item stays in workqueue before being requested.",
128 Buckets: prometheus.ExponentialBuckets(10e-9, 10, 10),
131 workDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
132 Subsystem: workQueueSubsystem,
133 Name: "work_duration_seconds",
134 Help: "How long in seconds processing an item from workqueue takes.",
135 Buckets: prometheus.ExponentialBuckets(10e-9, 10, 10),
138 retries = prometheus.NewCounterVec(prometheus.CounterOpts{
139 Subsystem: workQueueSubsystem,
140 Name: "retries_total",
141 Help: "Total number of retries handled by workqueue",
144 longestRunning = prometheus.NewGaugeVec(prometheus.GaugeOpts{
145 Subsystem: workQueueSubsystem,
146 Name: "longest_running_processor_microseconds",
147 Help: "How many microseconds has the longest running " +
148 "processor for workqueue been running.",
151 unfinishedWork = prometheus.NewGaugeVec(prometheus.GaugeOpts{
152 Subsystem: workQueueSubsystem,
153 Name: "unfinished_work_seconds",
154 Help: "How many seconds of work has done that " +
155 "is in progress and hasn't been observed by work_duration. Large " +
156 "values indicate stuck threads. One can deduce the number of stuck " +
157 "threads by observing the rate at which this increases.",
162 registerClientMetrics()
163 registerReflectorMetrics()
164 registerWorkqueueMetrics()
167 // registerClientMetrics sets up the client latency metrics from client-go
168 func registerClientMetrics() {
169 // register the metrics with our registry
170 Registry.MustRegister(requestLatency)
171 Registry.MustRegister(requestResult)
173 // register the metrics with client-go
174 clientmetrics.Register(&latencyAdapter{metric: requestLatency}, &resultAdapter{metric: requestResult})
177 // registerReflectorMetrics sets up reflector (reconile) loop metrics
178 func registerReflectorMetrics() {
179 Registry.MustRegister(listsTotal)
180 Registry.MustRegister(listsDuration)
181 Registry.MustRegister(itemsPerList)
182 Registry.MustRegister(watchesTotal)
183 Registry.MustRegister(shortWatchesTotal)
184 Registry.MustRegister(watchDuration)
185 Registry.MustRegister(itemsPerWatch)
186 Registry.MustRegister(lastResourceVersion)
188 reflectormetrics.SetReflectorMetricsProvider(reflectorMetricsProvider{})
191 // registerWorkQueueMetrics sets up workqueue (other reconcile) metrics
192 func registerWorkqueueMetrics() {
193 Registry.MustRegister(depth)
194 Registry.MustRegister(adds)
195 Registry.MustRegister(latency)
196 Registry.MustRegister(workDuration)
197 Registry.MustRegister(retries)
198 Registry.MustRegister(longestRunning)
199 Registry.MustRegister(unfinishedWork)
201 workqueuemetrics.SetProvider(workqueueMetricsProvider{})
204 // this section contains adapters, implementations, and other sundry organic, artisinally
205 // hand-crafted syntax trees required to convince client-go that it actually wants to let
206 // someone use its metrics.
208 // Client metrics adapters (method #1 for client-go metrics),
209 // copied (more-or-less directly) from k8s.io/kubernetes setup code
210 // (which isn't anywhere in an easily-importable place).
212 type latencyAdapter struct {
213 metric *prometheus.HistogramVec
216 func (l *latencyAdapter) Observe(verb string, u url.URL, latency time.Duration) {
217 l.metric.WithLabelValues(verb, u.String()).Observe(latency.Seconds())
220 type resultAdapter struct {
221 metric *prometheus.CounterVec
224 func (r *resultAdapter) Increment(code, method, host string) {
225 r.metric.WithLabelValues(code, method, host).Inc()
228 // Reflector metrics provider (method #2 for client-go metrics),
229 // copied (more-or-less directly) from k8s.io/kubernetes setup code
230 // (which isn't anywhere in an easily-importable place).
232 type reflectorMetricsProvider struct{}
234 func (reflectorMetricsProvider) NewListsMetric(name string) reflectormetrics.CounterMetric {
235 return listsTotal.WithLabelValues(name)
238 func (reflectorMetricsProvider) NewListDurationMetric(name string) reflectormetrics.SummaryMetric {
239 return listsDuration.WithLabelValues(name)
242 func (reflectorMetricsProvider) NewItemsInListMetric(name string) reflectormetrics.SummaryMetric {
243 return itemsPerList.WithLabelValues(name)
246 func (reflectorMetricsProvider) NewWatchesMetric(name string) reflectormetrics.CounterMetric {
247 return watchesTotal.WithLabelValues(name)
250 func (reflectorMetricsProvider) NewShortWatchesMetric(name string) reflectormetrics.CounterMetric {
251 return shortWatchesTotal.WithLabelValues(name)
254 func (reflectorMetricsProvider) NewWatchDurationMetric(name string) reflectormetrics.SummaryMetric {
255 return watchDuration.WithLabelValues(name)
258 func (reflectorMetricsProvider) NewItemsInWatchMetric(name string) reflectormetrics.SummaryMetric {
259 return itemsPerWatch.WithLabelValues(name)
262 func (reflectorMetricsProvider) NewLastResourceVersionMetric(name string) reflectormetrics.GaugeMetric {
263 return lastResourceVersion.WithLabelValues(name)
266 // Workqueue metrics (method #3 for client-go metrics),
267 // copied (more-or-less directly) from k8s.io/kubernetes setup code
268 // (which isn't anywhere in an easily-importable place).
269 // TODO(directxman12): stop "cheating" and calling histograms summaries when we pull in the latest deps
271 type workqueueMetricsProvider struct{}
273 func (workqueueMetricsProvider) NewDepthMetric(name string) workqueuemetrics.GaugeMetric {
274 return depth.WithLabelValues(name)
277 func (workqueueMetricsProvider) NewAddsMetric(name string) workqueuemetrics.CounterMetric {
278 return adds.WithLabelValues(name)
281 func (workqueueMetricsProvider) NewLatencyMetric(name string) workqueuemetrics.SummaryMetric {
282 return latency.WithLabelValues(name)
285 func (workqueueMetricsProvider) NewWorkDurationMetric(name string) workqueuemetrics.SummaryMetric {
286 return workDuration.WithLabelValues(name)
289 func (workqueueMetricsProvider) NewRetriesMetric(name string) workqueuemetrics.CounterMetric {
290 return retries.WithLabelValues(name)
293 func (workqueueMetricsProvider) NewLongestRunningProcessorMicrosecondsMetric(name string) workqueuemetrics.SettableGaugeMetric {
294 return longestRunning.WithLabelValues(name)
297 func (workqueueMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) workqueuemetrics.SettableGaugeMetric {
298 return unfinishedWork.WithLabelValues(name)