dependencies {
    implementation("io.ktor:ktor-server-call-logging-jvm:$ktor_version")
    implementation("io.ktor:ktor-server-call-id-jvm:$ktor_version")
    implementation("io.ktor:ktor-server-core-jvm:$ktor_version")

fun Application.configureMonitoring() {
    install(CallLogging) {
        level = Level.INFO
        filter { call -> call.request.path().startsWith("/") } // ログを出力する条件を指定できる
        callIdMdc("call-id")　// これを設定しておくことで、logback.xmlの　%X{call-id}　の部分に値を埋め込む事が可能
    }
    install(CallId) {
        header(HttpHeaders.XRequestId) //どのヘッダーにIDの値を格納するか
        verify { callId: String ->
            callId.isNotEmpty() //値が存在するか検証する
        }
+      generate {
+          UUID.randomUUID().toString() // なかったら値を生成して埋め込む
+      }
    }

dependencies {
...
+    implementation("io.ktor:ktor-client-core:$ktor_version")
+    implementation("io.ktor:ktor-client-cio:$ktor_version")
...
}

routing {
+    get("/call") {
+        application.log.info("Application is called")
+        val client = HttpClient(CIO) {
+        　   defaultRequest {
+                header(HttpHeaders.XRequestId, MDC.get("call-id"))
+            }
+        }
+        val response: HttpResponse = client.get("http://backend:8000/")
+        call.respond(HttpStatusCode.OK, response.bodyAsText())
+    }

curl -v localhost:8000/
curl -v -H "X-Request-Id: $(uuidgen)" localhost:8000/call

routing {
...
  get("/livez") {
      call.respond("OK") // Web serverが起動しているかどうかだけ伝えられればいいので、200をかえせばOK
  }

  get("/readyz") {
      // DBやその他の依存サービスに応じてpingを送る実装をアプリケーションの用途に応じて記述
    　// SQL ClientやHTTP Clientにはリクエストタイムアウトが設定できるので、期待した時間内に接続できるか記述する
      call.respond("OK")
  }
}

...
livenessProbe:
  httpGet:
    path: /livez
    port: 8080
readinessProbe:
  httpGet:
    path: /readyz
    port: 8080
  initialDelaySeconds: 15 # コンテナが起動して15秒後にreadiness probeに聞きに来る、defaultは0
  periodSeconds: 20 # 20秒に一回
  timeoutSeconds: 5 # 5秒以内に結果が帰ってくることを期待
  successThreshold: 1 # 1回成功すれば起動成功と判定
  failureThreshold: 3 # 3回連続失敗すればpodが再起動される
...

    implementation("io.ktor:ktor-server-metrics-micrometer-jvm:$ktor_version")
    implementation("io.micrometer:micrometer-registry-prometheus:$prometeus_version")

    val appMicrometerRegistry = PrometheusMeterRegistry(PrometheusConfig.DEFAULT)

    install(MicrometerMetrics) {
        registry = appMicrometerRegistry
    }
    routing {
        get("/metrics-micrometer") {
            call.respond(appMicrometerRegistry.scrape())
        }
    }

kind: Service
metadata:
  name: backend
  namespace: sample
+  annotations:
+    prometheus.io/scrape: 'true'
+    prometheus.io/path: '/metrics-micrometer'
+    prometheus.io/port: '8080'

resources:
  requests:
    memory: "512Mi"
    cpu: "500m"
  limits:
    memory: "512Mi"
    cpu: "750m"

helm repo add grafana https://grafana.github.io/helm-charts
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update
helm install prometheus prometheus-community/prometheus -n prometheus --create-namespace
helm install loki grafana/loki-stack -n grafana --create-namespace
helm install grafana grafana/grafana -n grafana
export POD_NAME=$(kubectl get pods --namespace grafana -l "app.kubernetes.io/name=grafana,app.kubernetes.io/instance=grafana" -o jsonpath="{.items[0].metadata.name}")
kubectl --namespace grafana port-forward $POD_NAME 3000
# 上記コマンドは実行後閉じないように別のターミナルを開いて
kubectl get secret --namespace grafana grafana -o jsonpath="{.data.admin-password}" | base64 --decode #| pbcopy # Macをお使いの方はこちらをコメントアウトするとクリップボードにパスワードがコピーされます

plugins {
    application
    kotlin("jvm") version "1.8.21"
    id("io.ktor.plugin") version "2.3.1"
+   id("com.google.cloud.tools.jib") version "3.3.1"
}
...


+ jib {
+    from {
+        platforms {
+            platform {
+                architecture = "amd64"
+                os = "linux"
+            }
+        }
+    }
+    to {
+        image = "sample-jib-image"
+        tags = setOf("alpha")
+    }
+    container {
+        jvmFlags = listOf("-Xms512m", "-Xmx512m")
+        mainClass = "com.example.ApplicationKt"
+        ports = listOf("80", "8080")
+    }
+}

-    <root level="trace">
+    <root level="info">

    install(CallLogging) {
        level = Level.INFO
-       filter { call -> call.request.path().startsWith("/") }
+       filter { call -> !arrayOf("/livez", "/readyz", "/metrics-micrometer")
+            .any { it.equals(call.request.path(), ignoreCase = true) }}
        callIdMdc("call-id")
    }

./gradlew jibDockerBuild && kubectl apply -f services.yaml # Buildするたびにdocker tagを修正する
# Skaffoldをインストールしている方は以下のコマンドで
skaffold init # yamlファイルが生成される
skaffold run # 一回だけビルドデプロイ作業を実行
skaffold dev # ソースコード修正するたびに継続的にビルド、デプロイ作業が走る

apiVersion: skaffold/v4beta5
kind: Config
metadata:
  name: observability
build:
  artifacts:
    - image: sample-jib-image
-    buildpacks: # ビルドが遅いので消す
-        builder: gcr.io/buildpacks/builder:v1
+      jib: {} # JAVA_HOMEに正しいPATHが入っていないと実行エラーになる可能性あり
manifests:
  rawYaml:
    - service.yaml
+portForward:
+  - resourceType: service
+    resourceName: frontend
+    namespace: sample
+    port: 8000
+    localPort: 8000

apiVersion: v1
kind: Namespace
metadata:
  name: sample
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: backend-deployment
  namespace: sample
spec:
  replicas: 2
  selector:
    matchLabels:
      app: backend
  template:
    metadata:
      labels:
        app: backend
    spec:
      containers:
        - name: backend
          image: sample-jib-image:alpha
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 8080
          # Liveness probe, readiness probeを実装するまでコメントアウトしておいてください
          # livenessProbe:
          #   httpGet:
          #     path: /livez
          #     port: 8080
          #   initialDelaySeconds: 15
          #   periodSeconds: 20
          #   timeoutSeconds: 5
          #   successThreshold: 1
          #   failureThreshold: 3
          # readinessProbe:
          #   httpGet:
          #     path: /readyz
          #     port: 8080
          resources:
            requests:
              memory: "512Mi"
              cpu: "500m"
            limits:
              memory: "512Mi"
              cpu: "750m"
---
apiVersion: v1
kind: Service
metadata:
  name: backend
  namespace: sample
  annotations:
    prometheus.io/scrape: 'true'
    prometheus.io/path: '/metrics-micrometer'
    prometheus.io/port: '8080'
spec:
  selector:
    app: backend
  ports:
    - protocol: TCP
      port: 8000
      targetPort: 8080
  type: ClusterIP
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: frontend-deployment
  namespace: sample
spec:
  replicas: 2
  selector:
    matchLabels:
      app: frontend
  template:
    metadata:
      labels:
        app: frontend
    spec:
      containers:
        - name: frontend
          image: sample-jib-image:alpha
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 8080
          # Liveness probe, readiness probeを実装するまでコメントアウトしておいてください
          # livenessProbe:
          #   httpGet:
          #     path: /livez
          #     port: 8080
          #   initialDelaySeconds: 15
          #   periodSeconds: 20
          #   timeoutSeconds: 5
          #   successThreshold: 1
          #   failureThreshold: 3
          # readinessProbe:
          #   httpGet:
          #     path: /readyz
          #     port: 8080
          resources:
            requests:
              memory: "512Mi"
              cpu: "500m"
            limits:
              memory: "512Mi"
              cpu: "750m"
---
apiVersion: v1
kind: Service
metadata:
  name: frontend
  namespace: sample
  annotations:
    prometheus.io/scrape: 'true'
    prometheus.io/path: '/metrics-micrometer'
    prometheus.io/port: '8080'
spec:
  selector:
    app: frontend
  ports:
    - protocol: TCP
      port: 8000
      targetPort: 8080
  type: LoadBalancer

skaffold delete
docker rmi $(docker images | grep 'sample-jib-image')
# kubectl delete all --all -n sample # skaffoldを実行していない場合
helm uninstall grafana -n grafana
helm uninstall loki -n grafana
helm uninstall prometheus -n prometheus