KEELããŒã ã®çžåã§ãã ååã®ãšã³ããªã¯ãæ¯èŒçå®å
šã«MCPãµãŒããåãããã§ããã www.lifull.blog ä»åã¯ä¿¡é Œã§ããå¯èŠ³æž¬æ§åºç€ãæäŸããã¹ããKubernetesã¯ã©ã¹ã¿ã«ãããPullåã¢ãããŒãç±æ¥ã®ãã°ã»ã¡ããªã¯ã¹æ¬ æãšè²ã
åãåã£ãè©±ãæžããŸãã Pullåã¢ãããŒããšãã®ãã¬ãŒããªã kubeletã«ãã°ãã¡ã€ã«ãåé€ãããåã«Fluentdã«èªãŸããã eBPFã§unlinkat(2)ãé
å»¶å®è¡ããã PrometheusãCounterã®ã€ã³ã¯ãªã¡ã³ããScrapeãããŸã§PodãåŸ
æ©ãããã SIGTERMãåãåã£ããæ¬¡åã®ScrapeãŸã§åŸ
æ©ãããããã·ãæã ãŸãšã Pullåã¢ãããŒããšãã®ãã¬ãŒããªã ãŸãPullåã¢ãããŒãã«ã€ããŠè»œãè§ŠããŠãããŸãã Pullåã¢ãããŒããšã¯ããã°ãã¡ããªã¯ã¹ããã®æã¡äž»ãã©ããã«å
¬éããŠãããŠãFluentdãPrometheusãªã©ã®ä»ã®èª°ãã«ååŸãã«æ¥ãŠãããã¢ãããŒããæããŸãã 察ãšãªãPushåã¢ãããŒãã¯éã«æã¡äž»ããã°ãã¡ããªã¯ã¹ãçŽæ¥å¯Ÿè±¡ã«éãã€ãããšãããã®ã§ãã Pushåã¢ãããŒãã¯ãµãŒãåŽã§æµéã®ã³ã³ãããŒã«ãé£ããã£ãããRate Limitæãªã©ã®ãªãã©ã€ã®è²¬ä»»ãã¯ã©ã€ã¢ã³ãåŽã«èŠæ±ãããäžæ¹ã§ãPullåã¢ãããŒãã¯ãã®ç¹ãåçŽãšããé¢ä¿æ§ã«ãããŸãã Kubernetesã¯ã©ã¹ã¿ã§çŽ çŽã«ãã°ãšã¡ããªã¯ã¹ãåéããããšæããšã倧äœPullåã®ã¢ãããŒããæ¡çšããããšãå€ãã®ã§ã¯ãªãããšæããŸãã ã³ã³ããã©ã³ã¿ã€ã ã¯ã³ã³ããã®æšæºåºåããã¡ã€ã«ãšããŠåºåããŠããã®ã§ãããFluentdãPromtailã§åéããã¡ããªã¯ã¹ã¯Prometheus Exporterã§å
¬éããŠPrometheusãScrapeãããšãã£ãæãã§ãã ããã§èããããªã¹ã¯ããã Podãåé€ãããæãŸã§ã«æ¬åœã«ãã®ãã°ã»ã¡ããªã¯ã¹ã¯åéãããŠããã®ã ããšãããã®ã§ãã Kubernetesäžã§ã³ã³ããã©ã³ã¿ã€ã ãåžãkubeletã¯ãPodã®å逿ã«ãã°ã®ãã¡ã€ã«ãåé€ããŸãã Podãåé€ãããã°åœç¶Scrapeããããã®ãšã³ããã€ã³ããç¡å¹ã«ãªãããã¡ããªã¯ã¹ãåéã§ããŸããã åé€çŽåã«åºåããããã°ãã€ã³ã¯ãªã¡ã³ãããã Counter ã¯ã©ããªãã®ã§ããããã æãããããã¯æ¬ æããŠããå¯èœæ§ãé«ãã§ãã Fluentdã¯æ¯èŒçããã«ãã°ãèªã¿ãŸããããã§ãèªã¿èŸŒã¿ãéã«åããæ¬ æããããšã¯ãããŸãããPrometheusã®Scrapeã®ééã¯äžè¬ã«çããšãç§åäœãªã®ã§æã¿ã¯èãã§ãã kubeletã«ãã°ãã¡ã€ã«ãåé€ãããåã«Fluentdã«èªãŸããã ãŸãã¯ãã°ã®æ¬ æããèããŠãããŸãããã æåã«æãã€ã察åŠã¯kubeletããã°ãåé€ãããŸã§ã«é
å»¶ãå
¥ããããåé€ãç¡å¹åã«ãããšãã£ãããšã ãšæããŸãã ããã¯ãã®èŸºã®Issuesã§è°è«ãããŠããŸãããä»ã®ãšãããããšãã£ãæ¹æ³ã¯ãããŸããã(Grafana Alloyãšãã¯Kubernetes APIçµç±ã§ã®ãã°ååŸã«ã察å¿ããŠããŸããããã¡ãã®åé€ã¿ã€ãã³ã°ãåæ§ã®ã¯ãã§ã) github.com github.com ããšãã£ãŠå
šãŠã®Podã«ãã°è»¢éçšã®ãµã€ãã«ãŒããããã€ããŠéä¿¡ãšããã®ãèŸãã®ã§ã©ãã«ããã©ãããã©ãŒã åŽã§å¯ŸåŠããããšããã§ãã ãããªæãeBPFã¯ãã€ã§ãç§éã®éã®åŒŸäžžã«ãªã£ãŠãããŸãã (eBPFãšã¯ããšãã話ã¯ä»¥åã«æžããã®ã§ãã¡ããã芧ãã ãã) www.lifull.blog eBPFã§unlinkat(2)ãé
å»¶å®è¡ããã ã¢ãããŒããšããŠã¯ãeBPFã§kubeletãçºè¡ãããã¡ã€ã«åé€ã®unlinkat(2)ãhookããã bpf_override_return ã§ãã¡ã€ã«ãåé€ããããšãªãçµäºã³ãŒããåœè£
ããŠè¿ããè£ã§Fluentdã察象ã®ãã¡ã€ã«ãèªã¿åã£ãããšã確èªåºæ¥ããå®éã®åé€ãè¡ããšãããã®ã§ãã Fluentd㯠*.pos ãã¡ã€ã«ã§èªã¿èŸŒãã ãã¡ã€ã«ã®ãã€ãæ°ãèšé²ããŠããããããããèŠãã°ãã¡ã€ã«ãèªã¿åã£ãããšã確èªã§ããŸãã ãŸãã¯ãããªæãã«unlinkat(2)ãhookããŸãããã bpf_override_return 㯠kprobeã®äžéšã®é¢æ° ããããå©çšã§ããªãããšã«æ³šæããŠãã ããã // /sys/kernel/debug/tracing/events/syscalls/sys_enter_unlinkat/format SEC ( "tracepoint/syscalls/sys_enter_unlinkat" ) int sys_enter_unlinkat ( struct trace_event_raw_sys_enter *ctx) { u64 __pid_tgid = bpf_get_current_pid_tgid (); gid_t tgid = __pid_tgid >> 32 ; pid_t pid = __pid_tgid; struct task_struct *task = ( struct task_struct *) bpf_get_current_task (); u32 ppid = BPF_CORE_READ (task, real_parent, tgid); if (tool_config.this == tgid || tool_config.this == ppid) { return 0 ; } const char *pathname = ( const char *)ctx->args[ 1 ]; int flag = ( int )ctx->args[ 2 ]; // AT_REMOVEDIR // https://elixir.bootlin.com/linux/v6.10.6/source/include/uapi/linux/fcntl.h#L104 if (flag & 0x200 ) { return 0 ; } struct arg arg = { .pathname = pathname, }; bpf_map_update_elem (&args, &pid, &arg, BPF_ANY); return 0 ; } SEC ( "kprobe/" SYS_PREFIX "sys_unlinkat" ) int BPF_KPROBE (sys_unlinkat) { u64 __pid_tgid = bpf_get_current_pid_tgid (); gid_t tgid = __pid_tgid >> 32 ; pid_t pid = __pid_tgid; struct arg *argp = bpf_map_lookup_elem (&args, &pid); if (!argp) { return 0 ; } int zero = 0 ; struct event *eventp = bpf_map_lookup_elem (&unlinkat_heap, &zero); if (!eventp) { goto end; } eventp->tgid = tgid; eventp->pid = pid; eventp->uid = bpf_get_current_uid_gid (); eventp->pathname[ 0 ] = '\0' ; if ( bpf_probe_read_user (eventp->pathname, sizeof (eventp->pathname), argp->pathname) < 0 ) { goto end; } u8 directory[DIRECTORY_MAX]; if ( bpf_probe_read_kernel (directory, sizeof (directory), tool_config.directory) < 0 ) { goto end; } if (! filter_directory (directory, eventp->pathname)) { goto end; } bpf_override_return (ctx, 0 ); bpf_perf_event_output (ctx, &events, BPF_F_CURRENT_CPU, eventp, sizeof (*eventp)); end : bpf_map_delete_elem (&args, &pid); return 0 ; } bpf_override_return(ctx, 0); ã«ãã£ãŠå³åº§ã«kubeletã«è¿ãã®ã§ãããã§ãã¡ã€ã«ã®åé€åŠçãã¹ãããããããšãã§ããŸãã unlinkat_heap 㯠PERCPU_ARRAY ã«ããŠããã®ã§null byteãåºåãæåã«ããŠããŸãã Kubernetesã®ããŒãäžã§å®è¡ããããšã«ãªãäœèšãªunlinkat(2)ãæµããŠããããã filter_directory ã§containerdã®ãã°ãã£ã¬ã¯ããªä»¥å€ã®ã€ãã³ãã¯ç¡èŠããªããã°ãªããŸããã if (tool_config.this == tgid || tool_config.this == ppid) ã¯ããã¡ã€ã«åé€ãé
å»¶å®è¡ããé¢ä¿äžunlinkat(2)ãèªèº«ãå®è¡ãããããç¡éã«ãŒã鲿¢ã®ããã« hostPID: true ã«ãããèªèº«ã®pidã this ãšããŠåãåã£ãŠã¹ãããããŠããŸãã ããšã¯ std::ffi::CStr::from_bytes_until_nul ã§ãŠãŒã¶ç©ºéãã pathname ãååŸããŠã *.pos ãã¡ã€ã«ãšçªåããªããFluentdãèªã¿åããŸã§åŸ
æ©ããŠãã¡ã€ã«ãåé€ããã ãã§ãã ããããæµéã¯å€ããªãã®ã§ãmtimeãèŠãŠé©åœã« *.pos ãã¡ã€ã«ã¯ãã£ãã·ã¥ããŠãããŸãã while let Some (pathname) = unlink_rx. recv ().await { let current_mtime = std :: fs :: metadata ( & args.pos_file) . and_then ( | m | m. modified ()) . unwrap_or ( std :: time :: SystemTime :: UNIX_EPOCH); let needs_refresh = pos_cache. read ().await.mtime != current_mtime; if needs_refresh { let new_pos = parse_pos_file ( & args.pos_file); let mut cache = pos_cache. write ().await; cache.mtime = current_mtime; cache.entries = new_pos; } if let Some (offset) = pos_cache . read () .await .entries . get ( & pathname) . map ( | (offset, _) | * offset) { let path = std :: path :: Path :: new ( & pathname); if let Ok (metadata) = path. metadata () && metadata. len () != offset { let unlink_tx = unlink_tx. clone (); tokio :: spawn (async move { tokio :: time :: sleep ( std :: time :: Duration :: from_secs ( args.delayed_seconds, )) .await; if let Err (e) = unlink_tx. send (pathname. clone ()) { eprintln! ( "Failed to re-queue {}: {}" , pathname, e); } }); continue ; } } let result = std :: fs :: remove_file ( & pathname). or_else ( | e | { if e. kind () == std :: io :: ErrorKind :: IsADirectory { std :: fs :: remove_dir ( & pathname) } else { Err (e) } }); if let Err (e) = result { if e. kind () == std :: io :: ErrorKind :: DirectoryNotEmpty { if let Ok (entries) = std :: fs :: read_dir ( & pathname) { for entry in entries. flatten () { let entry_path = entry. path (). to_string_lossy (). to_string (); if let Err (e) = unlink_tx. send (entry_path. clone ()) { eprintln! ( "Failed to queue {}: {}" , entry_path, e); } } } let unlink_tx = unlink_tx. clone (); tokio :: spawn (async move { tokio :: time :: sleep ( std :: time :: Duration :: from_secs ( args.delayed_seconds, )) .await; if let Err (e) = unlink_tx. send (pathname. clone ()) { eprintln! ( "Failed to re-queue {}: {}" , pathname, e); } }); } else { eprintln! ( "Failed to remove {}: {}" , pathname, e); } } } kubeletãå®è¡ããGoã® os.RemoveAll ã¯ãã¡ã€ã«ãšãã£ã¬ã¯ããªãåºå¥ããã«ãšããããunlinkat(2)ããŠããã®ã§ããŠãŒã¶ç©ºéåŽã§ã¯äžèº«ãåé€ããªãããã£ã¬ã¯ããªã空ã«ãªããŸã§åŸ
æ©ããããã«ããŠããŸãã æ¬æ¥ã¯ EISDIR ãåãåã£ãŠããAT_REMOVEDIRä»ãã®unlinkat(2)ã«ãã©ãŒã«ããã¯ãããã ãšæããŸãããä»åã¯bpf_override_returnã§EISDIRãæ¡ãæœ°ããŠããŸã£ãŠããã®ã§ã os.RemoveAll åæã®æ°æã¡æªãã¯ããã€ã€ããã¡ãã§åé€ãŸã§è²¬ä»»ãæã¡ãŸãã (ãã¡ã€ã«ã®ã¿ãæž¡ã£ãŠãã vfs_unlink ã䜿ãããšãããã§ããã ALLOW_ERROR_INJECTION ãããŠããããã¡ã㯠bpf_override_return ã䜿ããŸãã) ãŸãäžã€ã ãæ³šæç¹ããã£ãŠãcontainerd㯠/var/log/containers 以äžã«ãã°ãã¡ã€ã«ãäœæããŸãããå®éã«ã¯ãã㯠/var/log/pods ãžã®ã·ã³ããªãã¯ãªã³ã¯ãšãªã£ãŠãããããunlinkat(2)ãåŒã°ãã pathname ãšFluentdã® *.pos ãã¡ã€ã«ãæããã¡ã€ã«ãç°ãªãå¯èœæ§ããããŸãã ç§ã¯FluentdåŽãä¿®æ£ãã *.pos ãã¡ã€ã«ãèªãæã«ã€ãã§ã«ã·ã³ããªãã¯ãªã³ã¯ã解決ããŠããŸããŸããã ãã®èŸºããã£ãŠ *.pos ãã¡ã€ã«ã¯ãã£ãã·ã¥ããŠããŸãã fn parse_pos_file (path: & std :: path :: Path) -> std :: collections :: HashMap < String , ( u64 , u64 ) > { let mut pos = std :: collections :: HashMap :: new (); if let Ok (file) = std :: fs :: read_to_string (path) { for line in file. lines () { let mut parts = line. split_whitespace (); if let ( Some (path), Some (offset), Some (inode)) = (parts. next (), parts. next (), parts. next ()) { let resolved_path = std :: fs :: canonicalize (path) . map ( | p | p. to_string_lossy (). to_string ()) . unwrap_or_else ( | _ | path. to_string ()); pos. insert ( resolved_path, ( u64 :: from_str_radix (offset, 16 ). unwrap_or_default (), u64 :: from_str_radix (inode, 16 ). unwrap_or_default (), ), ); } } } pos } ããã§æŽããŠkubeletã®å®è¡ããunlinkat(2)ãæ¡ãã€ã¶ããããŠãŒã¶ç©ºéã§å®å
šã«Fluentdã®in_tailãåŸ
ã£ãŠããé
å»¶åé€ãããããã«ãªããŸããã åããããªã¢ãããŒãã¯Promtailãªã©ã§ãæå¹ãªã¯ãã§ãã PrometheusãCounterã®ã€ã³ã¯ãªã¡ã³ããScrapeãããŸã§PodãåŸ
æ©ãããã æ¬¡ã¯ã¡ããªã¯ã¹ã®æ¬ æã§ãã Gauge(UpDownCounterã®å®è£
ãGaugeãªããšããããŸãããçšéãšããŠã®Gauge)ãªã©ã®Metric typeã¯å²ãšã©ãã§ããããã§ãããCounterãªã©ã¯æå€ãšéèŠãªã±ãŒã¹ããã£ãŠã€ã³ã¯ãªã¡ã³ããæ¬ æããŠããŸããšå°ãããšããããŸãã LIFULLã§ã¯å€§ããã®Kubernetesã¯ã©ã¹ã¿ããã«ãããã³ãã§éçšããŠãããšããããšããããPrometheusãScrapeããééã¯30ç§çšåºŠãéçã§ãPodãåé€ããããŸã§ã®30ç§éã®ã€ã³ã¯ãªã¡ã³ãã倱ãããŠããŸããšããåé¡ããããŸããã ãã£ãšçãééã§éçšã§ããã®ã§ããã°ãKubernetes 1.29ããå©çšã§ããããã«ãªã£ã KEP-3960: Introducing Sleep Action for PreStop Hook ã§éã«sleepããŠãããã§ãããåçç¡çšã§30ç§ãåŸ
ã£ãŠããŸããšPodã®ããŒã«ã¢ãŠããé
ããªã£ãŠããŸãããããããããŸããã ãªãã¹ãããŒã«ã¢ãŠãã«åœ±é¿ãäžããªãããããã£ã¡ãScrapeããããŸã§ãåŸ
ãŠããšçæ³ã§ãã SIGTERMãåãåã£ããæ¬¡åã®ScrapeãŸã§åŸ
æ©ãããããã·ãæã çŽ çŽã«ã¯Scrapeãããããšãæ€ç¥ã§ããªããšæãã®ã§ãéã«ãããã·ãæããšã«ããŸãããã ã¢ã€ãã¢ã¯ã·ã³ãã«ã§ã以äžã®ãããªPrometheusãšPrometheus Exporterã®éã«æãã§Scrapeæå»ãèšé²ãããããã·ãäœããŸãã Prometheus Exporterã¯Podå逿ã«éãããŠããSIGTERMã§ãã®ãŸãŸGraceful ShutdownãããŠããŸãã®ã§ãçµäºçŽåã®ã¡ããªã¯ã¹ã cachedMetrics ã«å
¥ããŠãããŠSingleHostReverseProxyã®ErrorHandlerã§Prometheus Exporterãã·ã£ããããŠã³ããŠçéããªããªã£ãããããè¿ãããã«ããŠããŸãã type CachedResponse struct { Body [] byte ContentType string } var ( lastScrape atomic.Value terminating atomic.Bool scrapeChan chan struct {} cachedMetrics atomic.Pointer[CachedResponse] ) targetURL, err := url.Parse(a.TargetURL) if err != nil { return xerrors.Errorf( "failed to parse target URL: %w" , err) } proxy := httputil.NewSingleHostReverseProxy(targetURL) proxy.ErrorHandler = func (w http.ResponseWriter, r *http.Request, err error ) { if cached := cachedMetrics.Load(); cached != nil { w.Header().Set( "Content-Type" , cached.ContentType) w.WriteHeader(http.StatusOK) _, _ = w.Write(cached.Body) return } http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable) } server := &http.Server{ Handler: http.HandlerFunc( func (w http.ResponseWriter, r *http.Request) { lastScrape.Store(time.Now()) if terminating.Load() { select { case scrapeChan <- struct {}{}: default : } } proxy.ServeHTTP(w, r) }), } ãã®ããã«ãã®ãããã·ã¯SIGTERMãåãåã£ãæã«Prometheus Exporterããææ°ã®ã¡ããªã¯ã¹ãååŸããŠãããã£ãã·ã¥ãã scrapeChan ã§åŸ
ã¡åããŠæ¬¡åã®ScrapeãŸã§åŸ
æ©ããŸãã SIGTERMã®éä¿¡ãšEndpoint ControllerãEndpointãåé€ããåŠçã¯åæã«è¡ããããšããããšã¯åºãç¥ãããŠããŠãLIFULLã§ã¯ãããžã®å¯ŸåŠã®ããã«å
šãŠã®Podã¯çµäºæã«æ°ç§sleepããããã«ããŠããŸãã kubernetes.io ææ°ã®ã¡ããªã¯ã¹ãSIGTERMæã«ååŸããåŠçã¯ããã®èšå®ãšGraceful Shutdownã®å®è£
ã«äŸåããŠããéšåãããããšã«ã¯æ³šæãããã§ãã signal.Notify(quit, syscall.SIGTERM) <-quit ctx, cancel := context.WithTimeout(context.Background(), a.TerminationGracePeriod) defer cancel() t, ok := lastScrape.Load().(time.Time) if ok && !t.IsZero() { timeSinceLastScrape := time.Since(t) if timeSinceLastScrape > a.ScrapeWaitThreshold { if request, err := http.NewRequestWithContext(ctx, http.MethodGet, a.TargetURL, nil ); err == nil { if response, err := http.DefaultClient.Do(request); err == nil { defer func () { _ = response.Body.Close() }() if response.StatusCode < 400 { if body, err := io.ReadAll(response.Body); err == nil { cachedMetrics.Store(&CachedResponse{ Body: body, ContentType: response.Header.Get( "Content-Type" ), }) } } } } terminating.Store( true ) select { case <-scrapeChan: case <-ctx.Done(): } } } ãŸãšãããšãã®ãããã·ã¯ä»¥äžã®ããã«åããŸãã åžžã«PrometheusãšPrometheus Exporterã®éã«å
¥ã£ãŠæçµScrapeæå»ãèšé²ããŠãã SIGTERMãåãåã£ãæã«ScrapeWaitThreshold以å
ã«ScrapeãããŠããªããã°ãPrometheus Exporterããææ°ã®ã¡ããªã¯ã¹ãååŸããŠãã£ãã·ã¥ãã æ¬¡åScrapeæã«ãã®ãã£ãã·ã¥ããã¡ããªã¯ã¹ãè¿ããŠçµäºãã ããã§PodçµäºçŽåã«ã€ã³ã¯ãªã¡ã³ããããCounterãªã©ã®å€ãPrometheusã«Scrapeãããæ¬ æããåé¡ã解決ã§ããŸããã ããšã¯Prometheus Operatorã䜿ã£ãŠãããªã PodMonitor ããPodã®ã¢ãããŒã·ã§ã³ã«ããService Discoveryãå©çšããŠããå Žå㯠metadata.annotations ããã®ãããã·ã«åããã ãã§ãã LIFULLã§ã¯Podã®ã¢ãããŒã·ã§ã³ã«ããService Discoveryãå©çšããŠããããã prometheus.io/wait: true ãšä»äžãããšãã®ãããã·ã泚å
¥ããMutating Admission WebhookãéçºããŠããã®ã¿ã€ãã³ã°ã§ prometheus.io/port ããããã·ã®ããŒãã«æžãæããããã«ããŸããã ããã«ãããå©çšè
ã¯Podã¢ãããŒã·ã§ã³ã®ä»äžãããã ãã§æå°éã®ããŒã«ã¢ãŠãé
å»¶ãšåŒãæãã«ä¿¡é Œæ§ã®é«ãã¡ããªã¯ã¹ãåŸãããšãã§ããŸãã ãªããKubernetesãSIGTERMéä¿¡åŸã«è«ŠããŠSIGKILLãéããŸã§ã® spec.terminationGracePeriodSeconds ã®ããã©ã«ãå€ã¯30ç§ã§ãããããScrapeééãé·ããªã©ã§ãããè¶
ããŠãããã·ãåŸ
æ©ããªããã°ãªããªãå Žåã¯Podã® spec.terminationGracePeriodSeconds ã䌞ã°ãå¿
èŠããããŸãã ãŸãšã ãã®ãšã³ããªã§ã¯Kubernetesã¯ã©ã¹ã¿ã®å¯èŠ³æž¬æ§åºç€ã§ããæ¡çšãããPullåã¢ãããŒãã®ãã¬ãŒããªããšããã®ãã¬ãŒããªããžã®å¯ŸåŠã玹ä»ããŸããã äžéšã³ãã¥ããã£ã§ã¯åé¡èŠãããŠãããã®ã®ãçŽ çŽã«å©çšããŠãããšæå€ãšèŠèœãšããã¡ãªåé¡ã ãšæã£ãŠããŠããæå
ã®Kubernetesã¯ã©ã¹ã¿ã®ç£èŠãèŠçŽããã£ããã«ãªãã°å¹žãã§ãã ã¡ããªã¯ã¹ã¯ãšããããã°ã¯æ¬ æããŠããŸããšãããªãã«åé¡ãããã¯ãã§ãç¹ã«CronJobãšãã¯å®è¡ãã°ãåºåããŠããæ¯èŒçããPodãçµäºããããšãå€ãã successfulJobsHistoryLimit , failedJobsHistoryLimit ã0ã«ããŠãããšçŽåã®ãã°ã¯ã»ãŒæ¬ æããŠãããšèããŠããã§ãããã æè¿ã¯ãã¬ãŒã¹ã®Spanã«ãã°ãæãããŠããŸãããšãå¢ããŸããããäŸç¶ãã°ã¯éèŠãªå¯èŠ³æž¬æ§ã®ã·ã°ãã«ã ãšæãã®ã§æ¬ æããªãããã«éçšããŠãããããšããã§ãã Fluentdã®ãããã¡ã¯ã¡ãããšèšèšããŸããããšããPrometheusã®åé·åãšããGrafana Loki㯠max_chunk_age ã®å€ã«å¿ããŠéä¿¡ãé
å»¶ããŠããŸã£ãåäžã¹ããªãŒã å
ã®å€ããã°ãæšãŠãŠããŸãã®ã§ãfluent-plugin-grafana-lokiã«ãããåœãŠãŠè©²åœãšã©ãŒã®å Žåã«UnrecoverableErrorãè¿ãããšã§Fluentdã®secondaryã«æµããŠãå¥ã¹ããªãŒã ãšããŠéä¿¡ãçŽãããšã§åãããŒããªãããã«ããŸããããšããPullåã¢ãããŒãç±æ¥ä»¥å€ã®æ¬ æãé²ã話ã¯ãŸãå¥ã®æ©äŒã«æžããããšæããŸãã ãèå³ããæã¡ããã ããŸãããããã²ä»¥äžã®ããŒãžãã芧ãã ããã hrmos.co hrmos.co