Container Service for Kubernetes:Use a Prometheus client to monitor applications

Step 1: Instrument an application

Prometheus clients support most programming languages. For more information, see CLIENT LIBRARIES. The following example shows how to instrument a Go application to expose monitoring data:

package main
import (
    "flag"
    "fmt"
    "log"
    "math"
    "math/rand"
    "net/http"
    "time"
    "github.com/prometheus/client_golang/prometheus"
    "github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
    addr              = flag.String("listen-address", ":8080", "The address to listen on for HTTP requests.")
    uniformDomain     = flag.Float64("uniform.domain", 0.0002, "The domain for the uniform distribution.")
    normDomain        = flag.Float64("normal.domain", 0.0002, "The domain for the normal distribution.")
    normMean          = flag.Float64("normal.mean", 0.00001, "The mean for the normal distribution.")
    oscillationPeriod = flag.Duration("oscillation-period", 10*time.Minute, "The duration of the rate oscillation period.")
)

var (
    // Create a summary to track fictional interservice RPC latencies for three distinct services with different latency distributions. 
    // These services are differentiated via a "service" label.
    rpcDurations = prometheus.NewSummaryVec(
        prometheus.SummaryOpts{
            Name:       "rpc_durations_seconds",
            Help:       "RPC latency distributions.",
            Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
        },
        []string{"service"},
    )
    // The same as above, but now as a histogram, and only for the normal
    // distribution. The buckets are targeted to the parameters of the
    // normal distribution, with 20 buckets centered on the mean, each
    // half-sigma wide.
    rpcDurationsHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{
        Name:    "rpc_durations_histogram_seconds",
        Help:    "RPC latency distributions.",
        Buckets: prometheus.LinearBuckets(*normMean-5**normDomain, .5**normDomain, 20),
    })
)

func init() {
    // Register the summary and the histogram with Prometheus's default registry.
    prometheus.MustRegister(rpcDurations)
    prometheus.MustRegister(rpcDurationsHistogram)
    // Add Go module build info.
    prometheus.MustRegister(prometheus.NewBuildInfoCollector())
}

func main() {
    flag.Parse()
    start := time.Now()
    oscillationFactor := func() float64 {
        return 2 + math.Sin(math.Sin(2*math.Pi*float64(time.Since(start))/float64(*oscillationPeriod)))
    }
    // Periodically record some sample latencies for the three services.
    go func() {
        for {
            v := rand.Float64() * *uniformDomain
            rpcDurations.WithLabelValues("uniform").Observe(v)
            time.Sleep(time.Duration(100*oscillationFactor()) * time.Millisecond)
        }
    }()
    go func() {
        for {
            v := (rand.NormFloat64() * *normDomain) + *normMean
            rpcDurations.WithLabelValues("normal").Observe(v)
            // Demonstrate exemplar support with a dummy ID. This
            // would be something like a trace ID in a real
            // application.  Note the necessary type assertion. We
            // already know that rpcDurationsHistogram implements
            // the ExemplarObserver interface and thus don't need to
            // check the outcome of the type assertion.
            rpcDurationsHistogram.(prometheus.ExemplarObserver).ObserveWithExemplar(
                v, prometheus.Labels{"dummyID": fmt.Sprint(rand.Intn(100000))},
            )
            time.Sleep(time.Duration(75*oscillationFactor()) * time.Millisecond)
        }
    }()
    go func() {
        for {
            v := rand.ExpFloat64() / 1e6
            rpcDurations.WithLabelValues("exponential").Observe(v)
            time.Sleep(time.Duration(50*oscillationFactor()) * time.Millisecond)
        }
    }()
    // Expose the registered metrics via HTTP.
    http.Handle("/metrics", promhttp.HandlerFor(
        prometheus.DefaultGatherer,
        promhttp.HandlerOpts{
            // Opt into OpenMetrics to support exemplars.
            EnableOpenMetrics: true,
        },
    ))
    log.Fatal(http.ListenAndServe(*addr, nil))
}

Step 2: Package the application into a container image and upload the image to a repository

Package the instrumented application into a container image and upload the image to a repository in Container Registry.

1. Run the following command to build an image:

   docker build -t <Name of the local temporary Docker image>:<Tag of the local temporary Docker image> . --no-cache

   Example:

   docker build -t prometheus-demo:v1 . --no-cache

2. Run the following command to tag the image:

   sudo docker tag <Name of the local temporary Docker image>:<Tag of the local temporary Docker image> <Registry domain name>/<Namespace>/<Image name>:<Image tag>

   Example:

   sudo docker tag prometheus-demo:v1 registry.cn-hangzhou.aliyuncs.com/ringtail/prometheus-demo:v1

3. Run the following command to push the image to the repository:

   sudo docker push <Registry domain name>/<Namespace>/<Image name>:<Image tag>

   Example:

   sudo docker push registry.cn-hangzhou.aliyuncs.com/ringtail/prometheus-demo:v1

4. View the image in the Container Registry console.

   1. Log on to the Container Registry console.

   2. In the top navigation bar, select a region.

   3. In the left-side navigation pane, click Instances.

   4. On the Instances page, click the Personal Edition instance that you want to manage.

   5. In the left-side navigation pane of the management page of the Personal Edition instance, choose Repository > Repositories.

   6. On the Repositories page, find the repository that you want to manage and click Manage in the Actions column.

   7. In the left-side navigation pane, click Tags.

      You can find the uploaded image on the Tags page.

Step 3: Deploy the application to an ACK cluster

1. Log on to the ACK console.

2. In the left-side navigation pane of the ACK console, click Clusters.

3. On the Clusters page, find the cluster that you want to manage and click the name of the cluster or click Details in the Actions column. The details page of the cluster appears.

4. Create pods.

   1. In the left-side navigation pane of the details page, choose Workloads > Pods.

   2. On the Pods page, click Create from YAML in the upper-right corner.

   3. On the Create page, enter the following code in the Template code editor and click Create:

      apiVersion: apps/v1 # for versions before 1.8.0 use apps/v1beta1
      kind: Deployment
      metadata:
        name: demo-app
        labels:
          app: demo-app
      spec:
        replicas: 2
        selector:
          matchLabels:
            app: demo-app
        template:
          metadata:
            labels:
              app: demo-app
          spec:
            containers:
            - name: demo-app
              image: <Repository domain name>/<Namespace>/<Image name>:<Image tag>
              command:
              - /random 
              ports:
              - containerPort: 8080

      Sample code:

      apiVersion: apps/v1 # for versions before 1.8.0 use apps/v1beta1
      kind: Deployment
      metadata:
        name: demo-app
        labels:
          app: demo-app
      spec:
        replicas: 2
        selector:
          matchLabels:
            app: demo-app
        template:
          metadata:
            labels:
              app: demo-app
          spec:
            containers:
            - name: demo-app
              image: registry.cn-hangzhou.aliyuncs.com/ringtail/prometheus-demo:v1
              command:
              - /random 
              ports:
              - containerPort: 8080

   On the Pods page, you can find the pods that you created.

5. Create a Service.

   1. In the left-side navigation pane of the details page, choose Network > Services

   2. On the Services page, click Create Resources in YAML.

   3. On the Create page, enter the following code in the Template code editor and click Create:

      apiVersion: v1
      kind: Service
      metadata:
        labels:
          app: demo-app
        name: demo-app
        namespace: default
      spec:
        ports:
        - name: http-metrics
          port: 8080
          protocol: TCP
          targetPort: 8080
        selector:
          app: demo-app
        type: ClusterIP 

   On the Services page, you can find the Service that you created.

Step 4: Configure service discovery

Configure service discovery in ARMS Prometheus to capture data from the Go application. The following example shows how to configure service discovery in the ARMS console:

1. Log on to the ARMS console.

2. In the left-side navigation pane, choose Prometheus Service > Prometheus Instances.

3. In the top navigation bar, select the region where the cluster is deployed.

4. On the Prometheus Monitoring page, click the name of a cluster.

5. In the left-side navigation pane, click Service Discovery. On the page that appears, click the Configure tab.

6. Click the ServiceMonitor tab. On the page that appears, click Add ServiceMonitor.

7. In the Add ServiceMonitor dialog box, enter the following code and click OK.

   apiVersion: monitoring.coreos.com/v1
   kind: ServiceMonitor
   metadata:
     labels:
       app: demo-app
     name: demo-app
     namespace: default
   spec:
     endpoints:
     - interval: 30s
       port: http-metrics
     jobLabel: app
     namespaceSelector:
       matchNames:
       - default
     selector:
       matchLabels:
         app: demo-app

   On the ServiceMonitor tab, you can find the ServiceMonitor that you configured.

Step 5: Check whether the Prometheus client can monitor the application based on the metric

1. Click the Metrics tab on the Service Discovery page.

2. In the search box, enter rpc_durations_seconds and click . rpc_durations_seconds is the name of the metric that you registered in Step 1: Instrument an application.

   If data about the rpc_durations_seconds metric is returned, it indicates that the Prometheus client can monitor the application based on the metric.