feat(pvc): support kube_persistentvolumeclaim_deletion_timestamp

Signed-off-by: Maxime Leroy <[email protected]>

Author: maxime1907

docs/persistentvolumeclaim-metrics.md

@@ -9,8 +9,36 @@
 | kube_persistentvolumeclaim_resource_requests_storage_bytes | Gauge | | | `namespace`=&lt;persistentvolumeclaim-namespace&gt; <br> `persistentvolumeclaim`=&lt;persistentvolumeclaim-name&gt; | STABLE |
 | kube_persistentvolumeclaim_status_condition | Gauge | | | `namespace` =&lt;persistentvolumeclaim-namespace&gt; <br> `persistentvolumeclaim`=&lt;persistentvolumeclaim-name&gt; <br> `type`=&lt;persistentvolumeclaim-condition-type&gt; <br> `status`=&lt;true\false\unknown&gt; | EXPERIMENTAL |
 | kube_persistentvolumeclaim_status_phase | Gauge | | | `namespace`=&lt;persistentvolumeclaim-namespace&gt; <br> `persistentvolumeclaim`=&lt;persistentvolumeclaim-name&gt; <br> `phase`=&lt;Pending\Bound\Lost&gt; | STABLE |
-| kube_persistentvolumeclaim_created | Gauge | Unix Creation Timestamp | seconds | `namespace`=&lt;persistentvolumeclaim-namespace&gt; <br> `persistentvolumeclaim`=&lt;persistentvolumeclaim-name&gt; | EXPERIMENTAL |
+| kube_persistentvolumeclaim_created | Gauge | Unix creation timestamp | seconds | `namespace`=&lt;persistentvolumeclaim-namespace&gt; <br> `persistentvolumeclaim`=&lt;persistentvolumeclaim-name&gt; | EXPERIMENTAL |
+| kube_persistentvolumeclaim_deletion_timestamp | Gauge | Unix deletion timestamp | seconds | `namespace`=&lt;persistentvolumeclaim-namespace&gt; <br> `persistentvolumeclaim`=&lt;persistentvolumeclaim-name&gt; | EXPERIMENTAL |
 
 Note:
 
 - An empty string will be used if PVC has no storage class.
+
+## Useful metrics queries
+
+### How to retrieve non-standard PVC state
+
+It is not straightforward to get the PVC states for certain cases like "Terminating" since it is not stored behind a field in the `PersistentVolumeClaim.Status`.
+
+So to mimic the [logic](https://github.com/kubernetes/kubernetes/blob/v1.17.3/pkg/printers/internalversion/printers.go#L1402) used by the `kubectl` command line, you will need to compose multiple metrics.
+
+For example:
+
+* For PVCs in `Terminating` state: `count(kube_persistentvolumeclaim_deletion_timestamp) by (namespace, persistentvolumeclaim) * count(kube_persistentvolumeclaim_status_phase{phase="Bound"} == 1) by (namespace, persistentvolumeclaim)`
+
+Here is an example of a Prometheus rule that can be used to alert on a PVC that has been in the `Terminating` state for more than `5m`.
+
+```yaml
+groups:
+- name: PVC state
+  rules:
+  - alert: PVCBlockedInTerminatingState
+    expr: count(kube_persistentvolumeclaim_deletion_timestamp) by (namespace, persistentvolumeclaim) * count(kube_persistentvolumeclaim_status_phase{phase="Bound"} == 1) by (namespace, persistentvolumeclaim) > 0
+    for: 5m
+    labels:
+      severity: page
+    annotations:
+      summary: PVC {{$labels.namespace}}/{{$labels.persistentvolumeclaim}} blocked in Terminating state.
+```

internal/store/persistentvolumeclaim.go

@@ -226,6 +226,28 @@ func persistentVolumeClaimMetricFamilies(allowAnnotationsList, allowLabelsList [
 					})
 				}
 
+				return &metric.Family{
+					Metrics: ms,
+				}
+			}),
+		),
+		*generator.NewFamilyGeneratorWithStability(
+			"kube_persistentvolumeclaim_deletion_timestamp",
+			"Unix deletion timestamp",
+			metric.Gauge,
+			basemetrics.ALPHA,
+			"",
+			wrapPersistentVolumeClaimFunc(func(p *v1.PersistentVolumeClaim) *metric.Family {
+				ms := []*metric.Metric{}
+
+				if p.DeletionTimestamp != nil && !p.DeletionTimestamp.IsZero() {
+					ms = append(ms, &metric.Metric{
+						LabelKeys:   []string{},
+						LabelValues: []string{},
+						Value:       float64(p.DeletionTimestamp.Unix()),
+					})
+				}
+
 				return &metric.Family{
 					Metrics: ms,
 				}

internal/store/persistentvolumeclaim_test.go

@@ -276,6 +276,40 @@ func TestPersistentVolumeClaimStore(t *testing.T) {
 `,
 			MetricNames: []string{"kube_persistentvolumeclaim_created", "kube_persistentvolumeclaim_info", "kube_persistentvolumeclaim_status_phase", "kube_persistentvolumeclaim_resource_requests_storage_bytes", "kube_persistentvolumeclaim_annotations", "kube_persistentvolumeclaim_labels", "kube_persistentvolumeclaim_access_mode", "kube_persistentvolumeclaim_status_condition"},
 		},
+		{
+			Obj: &v1.PersistentVolumeClaim{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:              "terminating-data",
+					CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)},
+					DeletionTimestamp: &metav1.Time{Time: time.Unix(1800000000, 0)},
+				},
+				Spec: v1.PersistentVolumeClaimSpec{
+					AccessModes: []v1.PersistentVolumeAccessMode{
+						v1.ReadWriteOnce,
+					},
+					Resources: v1.ResourceRequirements{
+						Requests: v1.ResourceList{
+							v1.ResourceStorage: resource.MustParse("1Gi"),
+						},
+					},
+					VolumeName: "pvc-postgresql-data",
+				},
+				Status: v1.PersistentVolumeClaimStatus{
+					Phase: v1.ClaimBound,
+				},
+			},
+			Want: `
+				# HELP kube_persistentvolumeclaim_deletion_timestamp Unix deletion timestamp
+				# HELP kube_persistentvolumeclaim_status_phase [STABLE] The phase the persistent volume claim is currently in.
+				# TYPE kube_persistentvolumeclaim_deletion_timestamp gauge
+				# TYPE kube_persistentvolumeclaim_status_phase gauge
+				kube_persistentvolumeclaim_deletion_timestamp{namespace="",persistentvolumeclaim="terminating-data"} 1.8e+09
+				kube_persistentvolumeclaim_status_phase{namespace="",persistentvolumeclaim="terminating-data",phase="Bound"} 1
+				kube_persistentvolumeclaim_status_phase{namespace="",persistentvolumeclaim="terminating-data",phase="Lost"} 0
+				kube_persistentvolumeclaim_status_phase{namespace="",persistentvolumeclaim="terminating-data",phase="Pending"} 0
+`,
+			MetricNames: []string{"kube_persistentvolumeclaim_deletion_timestamp", "kube_persistentvolumeclaim_status_phase"},
+		},
 	}
 	for i, c := range cases {
 		c.Func = generator.ComposeMetricGenFuncs(persistentVolumeClaimMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList))