KEELããŒã ã®çžåã§ãã ååã®ãšã³ããªã¯ãLLMãå©çšããPlatform Engineeringãã§ããã www.lifull.blog ä»åã¯ãå°ããçµè·¯æé©åããã«ãŠã§ã¢ãå®è£
ããŠAZééä¿¡ãåæžããè©±ãæžããããšæããŸãã èæ¯ æã
KEELããŒã ã¯KubernetsããŒã¹ã®å
補PaaSã§ããKEELãéçºããŠãããLIFULLã®ã»ãšãã©ã®ãµãŒãã¹ããã®KEELäžã§åããŠããŸãã www.lifull.blog ãããŠãKEELã¯å·šå€§ãªãã«ãããã³ãã®Kubernetesã¯ã©ã¹ã¿ãšããŠAWSã®è€æ°ã®Availability Zone(以äžAZ)ã«å±éãããŠããŠãå€ãã®microservicesãäºãã«éä¿¡ããã£ãŠããŸãã ãã®ããAZééä¿¡ã¯ãã©ãããã©ãŒã ãšããŠéèŠãªé¢å¿äºã®äžã€ã§ãã ã¬ã€ãã³ã·ãAWSã®AZééä¿¡ã«å¯Ÿãã課éãæå°éã«æããããããªãã¹ãAZééä¿¡ãæžãããããšããèŠæ±ããããŸãã æã
ã®ãã©ãããã©ãŒã ã§ã¯çŸåšLLMã®ãµããŒãã匷ããŠãããããã«äŒŽãæ±ããã€ããŒãã®ãµã€ãºãå¢å åŸåã«ããããšããã察å¿ã®åªå
床ãäžãã£ãŠããŠããŸãã www.lifull.blog å®éã«Kubernetesã® Topology Aware Routing ããã©ãããã©ãŒã åŽã§é
åžããŠããã©ã«ãã§æå¹ã«ããããIstioã® Locality Load Balancing ãã¯ã©ã¹ã¿å
šäœã§æå¹ã«ããŠãã¯ã©ã¹ã¿å
ã®éä¿¡ã®çµè·¯æé©åã«åªããŠããŸããã â»Istioãæå¹ã«ãªã£ãŠããPodã§ã¯Topology Aware Routingã¯æ©èœããªããããå
šãŠã®Podã«Istioãå°å
¥ã§ããŠããªãå Žåã¯ã¯ã©ã¹ã¿å
šäœãšããŠäž¡æ¹ãæå¹ã«ããå¿
èŠããããŸã äžæ¹ã§ãKEELã¯ã¹ããŒãã¬ã¹ãªã¯ã©ã¹ã¿ãšããŠæ°žç¶æ§ãå¿
èŠãšãããããŒã¿ã¹ãã¢ã¯åºæ¬çã«ã¯ã©ã¹ã¿å€ã«äŸåããŠãããæªã æé©åã§ããŠããªãçµè·¯ãååšããŠããŸãã ããã§ãã®æ®ãããAZééä¿¡ãåæžãã¹ããå°ããçµè·¯æé©åããã«ãŠã§ã¢ãå®è£
ãã倿ã«è³ããŸããã èæ¯ å®è£
ã·ã³ãã«ãªConnection Pool äœè«: Connection Storm Topology Aware Routing 泚æç¹ æåŸã« å®è£
ããŠãKubernetesãIstioãå®çŸããã¯ã©ã¹ã¿å
éä¿¡ã®çµè·¯æé©åã¯Kubernetesã®Service Discoveryã«äŸåããŠãããããã¯ã©ã¹ã¿å€ãžã®éä¿¡ã§ç䌌ããããšã¯ã§ããŸããã ã€ãŸãã¯ã©ã€ã¢ã³ãã¯ãµãŒããã©ã®ããããžã«é
眮ãããŠããããç¥ãå
±éã®ææ®µãæã£ãŠããªããšããããšã§ãã ãµãŒãåŽãDNSã®SRVã¬ã³ãŒããªã©ã«å¯Ÿå¿ããŠããã°ããããã®ãŸãŸå©çšã§ããŸãããããã§ããªãå Žåå
šãŠã®ãµãŒãã«æ°èŠã«Service Discoveryãå
¥ãããšããããšã¯ããŸãã«å€§å€ã§ãã ããã§æã
ã¯TCPãµãŒããšããŠæ¯ãèãConnection Poolã®ããã«ãŠã§ã¢ãšããŠçµè·¯æé©åãå®çŸããããšã«æ±ºããŸããã LIFULLã«ã¯Rubyã®Unicornãªã©1ãªã¯ãšã¹ã1ããã»ã¹ãªããã»ã¹ã¢ãã«ã®ã¢ããªã±ãŒã·ã§ã³ãµãŒããããã€ããããå
ã
ããã»ã¹ããŸããã Connection Poolãå®çŸããããšããèŠæ±ããããŸãã ãããããã©ãããã©ãŒã ãšããŠProxySQLãæäŸãããªã©ç¹å®ã®ãŠãŒã¹ã±ãŒã¹ã§ã¯å¯Ÿå¿ã§ããŠãããã®ã®ã ããããAZééä¿¡ ã«ã¯å¯Ÿå¿ã§ããŠããŸããã§ããã Connection Poolã§ã®çµè·¯æé©åã®ã¢ã€ãã¢ã¯ã·ã³ãã«ã§ãç¶æããŠããæ¥ç¶ãããããžããšã«ç®¡çããŠããªãã¹ãã¯ã©ã€ã¢ã³ããšåãããããžã®æ¥ç¶ãåãåºããŠäœ¿ãã ãã§ãã ã€ãŸãConnection Poolãç¶æããŠããæ¥ç¶ãç䌌çã«Service DiscoveryãšããŠå©çšãããšããããšã§ãã ãããå®çŸã§ããã°è²ã
ãšéœåãè¯ããããªã®ã§ãã£ãŠãããŸãããã ã·ã³ãã«ãªConnection Pool ãŸãã¯æ®éã®TCPãããã·ã«ã·ã³ãã«ãªConnection Poolãå®è£
ããŠãããŸããæ¥ç¶ãåãåãã ãã®ãœãããŠã§ã¢ã«ãªãã®ã§Goãç¡é£ã§ãããã MinIdleConnections , MaxIdleConnections ã§ã¢ã€ãã«ãªæ¥ç¶ãã³ã³ãããŒã«ã§ããããã«ã㊠MaxIdleTime , MaxLifetime ã§çåæéãèšå®ã§ãããããããã€ã§ãã èšå®ã¯å€§äœããããã€ã³ã¿ãã§ãŒã¹ã§ããããã Connection 㯠MaxIdleTime ã®å€å®ã§å©çšãã returnedAt ãæãŠãããã«ããã ãã® net.Conn ã®èãã©ãããŒã§ãã type ConnectionPoolStrategy int const ( FIFO ConnectionPoolStrategy = iota LIFO ) type ConnectionPoolOption struct { MaxConnections uint MaxIdleConnections uint MinIdleConnections uint MaxIdleTime time.Duration MaxLifetime time.Duration Dialer func (context.Context) (net.Conn, error ) ConnectionPoolStrategy ConnectionPoolStrategy } type ConnectionPool struct { option *ConnectionPoolOption connectionMutex sync.Mutex connections []*Connection idleConnections []*Connection } ã³ã³ã¹ãã©ã¯ã¿ã¯çç¥ãããšããŠãæ¥ç¶ãåãåºãéšåã¯ãããªæãã«ãªããŸãã func (p *ConnectionPool) Get(ctx context.Context) (*Connection, error ) { p.connectionMutex.Lock() defer p.connectionMutex.Unlock() defer func () { go p.prepareIdleConnection(ctx) }() for { connection, err := p.getIdleConnection(ctx) if err != nil { return nil , xerrors.Errorf( "failed to get idle connection: %w" , err) } if connection == nil { break } now := nowFunc() if (p.option.MaxIdleTime > 0 && now.Sub(connection.returnedAt) > p.option.MaxIdleTime) || (p.option.MaxLifetime > 0 && now.Sub(connection.createdAt) > p.option.MaxLifetime) || !connection.isHealthy() { _ = connection.Close() p.removeConnection(ctx, connection) continue } return connection, nil } connection, err := p.newConnection(ctx) if err != nil { return nil , xerrors.Errorf( "failed to create connection: %w" , err) } return connection, nil } ããã¯ãåããªããã¢ã€ãã«ãªæ¥ç¶ãåãåºããçåæéãæºãããŠããã°æ»æŽ»ç£èŠãããŠããæ¥ç¶ãè¿ããŸãã ã¢ã€ãã«ãªæ¥ç¶ããªããã°æ°èŠã«æ¥ç¶ãäœæããŠè¿ããŸãã prepareIdleConnection 㯠MinIdleConnections ãæºããããã«ã¢ã€ãã«ãªæ¥ç¶ãçšæããŠããŠã isHealthy ã¯ãœã±ããã« read(2) ã㊠EAGAIN or EWOULDBLOCK ã§ããããšã®ç¢ºèªã§ãã func (c *Connection) isHealthy() bool { // A zero time value disables the deadline. _ = c.conn.SetReadDeadline(time.Time{}) syscallConnection, ok := c.conn.(syscall.Conn) if !ok { return false } rawConnection, err := syscallConnection.SyscallConn() if err != nil { return false } healthy := false if err := rawConnection.Read( func (fd uintptr ) bool { b := make ([] byte , 1 ) _, err := syscall.Read( int (fd), b) if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EWOULDBLOCK) { healthy = true } return true }); err != nil { return false } return healthy } ãããŠã䜿ãçµãã£ãæ¥ç¶ãç¶æããããã®Putãå®è£
ããŸãã MaxIdleTime ã§å€æããããã® returnedAt ã®æŽæ°ãå¿ããªãããã«ããŸãããã func (p *ConnectionPool) Put(ctx context.Context, connection *Connection) error { p.connectionMutex.Lock() defer p.connectionMutex.Unlock() if p.IdleConnections() < int (p.option.MaxIdleConnections) { connection.returnedAt = nowFunc() p.idleConnections = append (p.idleConnections, connection) } else { _ = connection.Close() p.removeConnection(ctx, connection) } return nil } ã¡ãªã¿ã«ããããŸã§ã®å
容ã®TCPãããã·ã§ã®å©çšã€ã¡ãŒãžã¯ãããªæãã«ãªããŸãã func main() { ... semaphore := make ( chan struct {}, maxConnections) shutdown := make ( chan struct {}, 1 ) wg := sync.WaitGroup{} go func () { ctx := context.Background() for { local, err := listener.AcceptTCP() if err != nil { select { case <-shutdown: return default : continue } } semaphore <- struct {}{} wg.Add( 1 ) go func () { defer func () { <-semaphore wg.Done() }() defer local.Close() remote, err := connectionPool.Get(ctx) if err != nil { return } defer connectionPool.Put(ctx, remote) defer remote.Cancel() c := make ( chan struct {}, 2 ) f := func (c chan struct {}, dst io.Writer , src io.Reader ) { _, _ = io.Copy(dst, src) c <- struct {}{} } go f(c, remote, local) go f(c, local, remote) select { case <-c: case <-shutdown: local.CloseWrite() } }() } }() // Graceful shutdown quit := make ( chan os.Signal, 1 ) signal.Notify(quit, syscall.SIGTERM) <-quit time.Sleep(time.Duration(lameduck) * time.Second) close (shutdown) listener.Close() wg.Wait() } äœè«: Connection Storm ãã®å®è£
ã§ã¯ãç¶æããŠããæ¥ç¶ã MaxLifetime ã«éããæã«åŒ·å¶çã«ãã®æ¥ç¶ãç Žæ£ããŸãã å€ãæ¥ç¶ã䜿ããç¶ããããšãé²ãããã®ãã®ã§ãããããã«ã¯æ¯èŒç倧ããªãã¬ãŒããªãããããŸãã ãã㯠Connection Storm ãšåŒã°ããçŸè±¡ã§ãç¶æãããŠããæ¥ç¶ãäžæã« MaxLifetime ã«ãã£ãŠåæãããããšã§ãæ°èŠã®æ¥ç¶ã倧éã«ç¢ºç«ããŠæ¥ç¶å
ã«å€§ããªè² è·ãããããšãããã®ã§ãã ãããé²ãããã«ã¯ ãªãã©ã€ãªã©ã§ããç¥ãããJitter ãå
¥ãããšããã§ãã Jitterã¯ã©ã³ãã æ§ã泚å
¥ããæŠå¿µã§ãçåæéã®å€å®ç®æã«Jitterãå
¥ããŠäžæã«åæãããããšãé²ããŸãã ãªãã©ã€ã«ãããJitterã¯ãæ¥ç¶å
ã®é害çºçæãªã©ã«è€æ°ã®ãªãã©ã€ãåæã«è¡ããããšã«ã¹ã±ãŒãé害ãèµ·ãããŠããŸãããããªãã©ã€ã«ã©ã³ãã æ§ãæãããããã«å©çšãããŠããŸãã type ConnectionPoolOption struct { MinIdleConnections uint MaxIdleTime time.Duration MaxLifetime time.Duration + Jitter func (time.Duration) time.Duration Dialer func (context.Context) (net.Conn, error ) ConnectionPoolStrategy ConnectionPoolStrategy } now := nowFunc() - if (p.option.MaxIdleTime > 0 && now.Sub(connection.returnedAt) > p.option.MaxIdleTime) || - (p.option.MaxLifetime > 0 && now.Sub(connection.createdAt) > p.option.MaxLifetime) || + if (p.option.MaxIdleTime > 0 && now.Sub(connection.returnedAt) > p.option.Jitter(p.option.MaxIdleTime)) || + (p.option.MaxLifetime > 0 && now.Sub(connection.createdAt) > p.option.Jitter(p.option.MaxLifetime)) || !connection.isHealthy() { _ = connection.Close() p.removeConnection(ctx, connection) å®éã®Jitterã®å®è£
ã¯ãããªæãã«ãªããšæããŸãã jitterPercentage ã§ã©ã®çšåºŠã©ã³ãã æ§ã«éããæããããã調æŽã§ããããã«ããã€ã¡ãŒãžã§ãã ãã¹ãã®æã«ã¯åŒæ°ã® duration ããã®ãŸãŸè¿ãã°ããã§ãããã func (duration time.Duration) time.Duration { jitter := time.Duration( float64 (duration) * jitterPercentage * (rand.Float64()* 2 - 1 )) return duration + jitter } é話äŒé¡ã Topology Aware Routing ãããŸã§ã§ã·ã³ãã«ãªConnection Poolã®å®è£
ã宿ããã®ã§ãããããããããæ¬é¡ã®çµè·¯æé©åã®å®è£
ã«å
¥ãããšæããŸãã ã¢ã€ãã¢ã¯å
ã«æžããéãã·ã³ãã«ã§ãç¶æããŠããæ¥ç¶ãããããžããšã«ç®¡çããŠããªãã¹ãã¯ã©ã€ã¢ã³ããšåãããããžã®æ¥ç¶ãåãåºããŠäœ¿ãã ãã®ãã®ãç®æããŸãã ããããžã¯IPãããŒã¹ã«å€æãããšããŠãä»åã¯ãµã€ãã«ãŒãšããŠã¯ã©ã€ã¢ã³ãã®ããã°ã©ã ãšåããµãŒãã§åããããšãæ³å®ããŠãããããã¯ã©ã€ã¢ã³ãã®ããããžã¯èªèº«ã®IPãã倿ããããšãã§ãããã§ãã ããããžã®äžèЧã --topologies=a=192.168.0.0/24,b=192.168.1.0/24,c=192.168.2.0/24 ã®ããã«åãåããããã°ã©ã å
ã§ã¯ä»¥äžã®ããã«åãæ±ãããšã«ããŸãã type Topology struct { Name string CIDR net.IPNet } var topologyList []Topology var topologyName string if topologyAwareRouting { for _, t := range strings.Split(topologies, "," ) { if t == "" { continue } parts := strings.Split(t, "=" ) if len (parts) != 2 { log.Fatalf( "invalid topologies: %s" , topologies) } _, cidr, err := net.ParseCIDR(parts[ 1 ]) if err != nil { log.Fatalf( "failed to parse CIDR %s: %+v" , parts[ 1 ], err) } topology := Topology{ Name: parts[ 0 ], CIDR: *cidr, } topologyList = append (topologyList, topology) if topology.CIDR.Contains(net.ParseIP(ownIP)) { topologyName = topology.Name } } } CIDRãæã£ãããããžã®äžèЧã topologyList ãšããŠç®¡çãããŠããŠãèªèº«ã®IPãã倿ããã¯ã©ã€ã¢ã³ãã®ããããžãäºåã« topologyName ãšããŠå®çŸ©ããŠãããŸãã ( topologyName ãäºåã«å®çŸ©ããã®ã¯èšç®ã³ã¹ããå«ã£ãŠããã ãã§ãåœç¶ã¯ã©ã€ã¢ã³ãã®æ¥ç¶ããéœåºŠå€æããããšãå¯èœãªã®ã§ãµã€ãã«ãŒã§ã¯ãªãç¬ç«ããŠãããã€ããããšãå¯èœã§ã) Kubernetesã§ã¯èªèº«ã®IPã Downward API ããååŸå¯èœã§ãAWSã§ã ã€ã³ã¹ã¿ã³ã¹ã¡ã¿ããŒã¿ ããååŸã§ããã®ã§ããã䜿ããŸãã ããã§ã¯å
ã»ã©ã®Connection Poolã®å®è£
ã«ããããäžããŠTopology Aware Routingã«å¯Ÿå¿ããŠãããŸãããã ãŸã㯠topologyList ãåãåãããããã«ãã€ã€ãç¶æããŠããæ¥ç¶ãHashMapã§ããããžããšã«ç®¡çã§ããããã«æ§é äœã倿ŽããŸãã type ConnectionPoolOption struct { MinIdleConnections uint MaxIdleTime time.Duration MaxLifetime time.Duration Jitter func (time.Duration) time.Duration + TopologyList []Topology Dialer func (context.Context) (net.Conn, error ) ConnectionPoolStrategy ConnectionPoolStrategy } type ConnectionPool struct { connectionMutex sync.Mutex connections []*Connection - idleConnections []*Connection + idleConnections map [ string ][]*Connection } 次ã«ãæ¥ç¶ãåãåºã Get ã§ã¯ã©ã€ã¢ã³ãã®ããããž topologyName ãåãåã£ãŠãåãããããžã®æ¥ç¶ãåãåºãéšåã§ãã - func (p *ConnectionPool) Get(ctx context.Context) (*Connection, error ) { + func (p *ConnectionPool) Get(ctx context.Context, topologyName string ) (*Connection, error ) { p.connectionMutex.Lock() defer p.connectionMutex.Unlock() defer func () { go p.prepareIdleConnection(ctx) }() for { - connection, err := p.getIdleConnection(ctx) + connection, err := p.getIdleConnection(ctx, topologyName) if err != nil { return nil , xerrors.Errorf( "failed to get idle connection: %w" , err) } if connection == nil { + if n := p.pickRandomIdleTopologyName(); n != nil { + topologyName = *n + continue + } break } now := nowFunc() if (p.option.MaxIdleTime > 0 && now.Sub(connection.returnedAt) > p.option.Jitter(p.option.MaxIdleTime)) || (p.option.MaxLifetime > 0 && now.Sub(connection.createdAt) > p.option.Jitter(p.option.MaxLifetime)) || !connection.isHealthy() { _ = connection.Close() - p.removeConnection(ctx, connection) + p.removeConnection(ctx, topologyName, connection) continue } return connection, nil } connection, err := p.newConnection(ctx) if err != nil { return nil , xerrors.Errorf( "failed to create connection: %w" , err) } return connection, nil } pickRandomIdleTopologyName ã¯ãã©ãŒã«ããã¯ã®åŠçã§ãåãããããžã®æ¥ç¶ããªãã£ãéã«é©åœãªããããžã®æ¥ç¶ã代ããã«äœ¿ããŸãã ãã©ãŒã«ããã¯ã®æŠç¥ãå€ããäžããããããã«ããŠãããã§ããã ä»ã¯ããããå
ã®HashMapãæ±ã£ãŠæ¥ç¶ãåãåºãããåé€ãããããŠãã ãã§ãã 现ãã話ã§ããã idleConnections ãHashMapã«ãªãããšã§ç·æ¥ç¶æ°ãåŸãããã®èšç®ã³ã¹ãã O(1) ã§ãªããªã£ãŠããŸããããå¥ã§Atomicãªã«ãŠã³ã¿ãŒãçšæããŠãããæ¹ãããã§ãã æåŸã« Put ã®å®è£
ã§ãã func (p *ConnectionPool) Put(ctx context.Context, connection *Connection) error p.connectionMutex.Lock() defer p.connectionMutex.Unlock() + topologyName := p.topologyName(connection.RemoteAddr()) + if p.IdleConnections() < int (p.option.MaxIdleConnections) { connection.returnedAt = nowFunc() - p.idleConnections = append (p.idleConnections, connection) + p.idleConnections[topologyName] = append (p.idleConnections[topologyName], connection) } else { _ = connection.Close() - p.removeConnection(ctx, connection) + p.removeConnection(ctx, topologyName, connection) } return nil } net.Conn ã® RemoteAddr() ã§æ¥ç¶å
ã®IPãååŸãã以äžã®é¢æ°ã§ topologyList ãšããŠåãåã£ãCIDRããé©åãªããããžãååŸããŸãã func (p *ConnectionPool) topologyName(addr net.Addr) string { for _, t := range p.option.TopologyList { switch addr := addr.( type ) { case *net.TCPAddr: if t.CIDR.Contains(addr.IP) { return t.Name } case *net.UDPAddr: if t.CIDR.Contains(addr.IP) { return t.Name } } } return "" } ããã§çµè·¯æé©åãè¡ãTopology Aware Routingã宿ã§ãã ç°¡æçãªå®è£
ã«ããæ¥œèгçãªçµè·¯æé©åã§ãããæ¥ç¶å
ã®Service Discoveryã«äŸåããªã ããããAZééä¿¡ ã«å¯Ÿå¿ãããã®ãã§ããŸããã äŸãã°AWSã®Elastic Load Balancerã¯AZããšã«IPãæã£ãŠããããããDNSã©ãŠã³ãããã³ã§è¿åŽãããŸãã ãã®ããã«ãŠã§ã¢ãã¯ã©ã€ã¢ã³ãã«å°å
¥ããããšã§ãAWSã®ããŒããã©ã³ãµã«å¯Ÿããæ¥ç¶ã§ãã£ãŠãAZééä¿¡ãåæžããããšã«æåããŸããã ãªããAWS Application Load Balancerã¯ããã©ã«ãã§ ã¯ãã¹ãŸãŒã³è² è·åæ£ ãæå¹ã«ãªã£ãŠããŠããããå°å
¥ããŠãããŒããã©ã³ãµãšUpstreamã®éã§ã¯AZééä¿¡ãçºçããå¯èœæ§ããããŸãã Upstreamãååã«AZã«åæ£ãããŠããã°ã¯ãã¹ãŸãŒã³è² è·åæ£ãç¡å¹ã«ãã倿ãã§ãããããããŸããã æ³šæç¹ ãã®ããã«ãŠã§ã¢ã«éãããããã£ãçµè·¯æé©åã§èæ
®ããªããã°ãªããªãããšããããŸãã ããã¯ããããžéã®åæ£ã§ãã ã¯ã©ã€ã¢ã³ããšãµãŒãã§ããããžéã®åæ£ãäžååãªå ŽåãäŸãã°ã¯ã©ã€ã¢ã³ããäžã€ã®ããããžã«éäžããŠãããã€ãããŠããŸã£ãŠããå Žåã«ããã®ããããžã®ãµãŒãã«è² è·ãéäžããŠããŸããŸãã ãããé²ãããã«ã¯æ
éã«ããããžéã§åæ£ãããå¿
èŠããããããã«ã¯Kubernetesã® Pod Topology Spread Constraints ãå©çšããããšãã§ããŸãã ããã¯ãããã€æã«ããããžéã®ã°ãã€ãã«å¯Ÿããå¶çŽããããããæ©èœã§ãLIFULLã§ã¯Kubernetes Manifestãçæããããã®ã³ãŒããžã§ãã¬ãŒã¿ã§Kubernetesã®Topology Aware Routingã®èšå®ãšãšãã«ããã©ã«ãã®èšå®ãšããŠé
åžããŠããŸãã www.lifull.blog ãã ãããã¯ãããŸã§ãããã€æã®å¶çŽã§ãPodãã¹ã±ãŒã«ã€ã³ããåŸã®ãªãã©ã³ã¹ãŸã§ã¯é¢åãèŠãŠãããŸããã ããã§ãããã㊠kubernetes-sigs/descheduler ãå°å
¥ã㊠RemovePodsViolatingTopologySpreadConstraint ãæå¹ã«ããããšããå§ãããŸãã ããã«ãã Pod Topology Spread Constraints ã«éåããŠããPodãåã¹ã±ãžã¥ãŒã«ããŠãããããããã®åŸã®åãããã€ã§å床å¶çŽããããããšãã§ããŸãã ãŸããã®å¶çŽãæ©èœãããããã«ã¯ããŒãã®ããããžéã®ãã©ã³ã¹ãéèŠã§ã cluster-autoscaler ãªã©ã§é©åã«èšå®ããå¿
èŠãããããšã«ã泚æããŠãã ããã (AWSã®Auto Scaling Groupã¯ã¹ãããã€ã³ã¹ã¿ã³ã¹ã䜿ã£ãŠããªããã° ããæãã«ãã£ãŠããã ã®ã§ç¹ã«æ°ã«ããå¿
èŠã¯ãªãæ°ãããŸã) æåŸã« Connection Poolã®ããã«ãŠã§ã¢ã«ç°¡æçãªTopology Aware Routingãå®è£
ããããšã§ ããããAZééä¿¡ ãåæžããããšã«æåããŸããã ããã«ããã¬ã€ãã³ã·ã®æ¹åãšAZééä¿¡ã«å¯Ÿãã課éãæããããšãã§ããŸãã AZéã®ã¬ã€ãã³ã·ã¯LIFULLã®ç°å¢ã§ã¯ææªã±ãŒã¹ã§5msã«ãåã¶ããšãããããããªãã®ã€ã³ãã¯ããæåŸ
ããŠããŸãã ãã«ãããã»ã¹ãªã¢ããªã±ãŒã·ã§ã³ãµãŒããéçšããŠãããšããã£ãœãConnection Poolãæ¬²ãããªãããšã¯ãã£ãŠãæåå³åã§ãããã®ã¢ãããŒãã¯ã¡ããã©ããå¡©æ¢
ãªã®ã§ã¯ãªãã§ããããã æè¿ã¯èª¿åã«ä¹ã£ãŠããã«Redisã®ãããã³ã«ããŒãµãèŒããŠãRedisãžã®ã¯ãšãªã®Read/Writeã®å颿©èœãå®è£
ãããªã©ãããŠããŸã£ãŠããŸãã KEELã§ã¯Read Heavyãªãã£ãã·ã¥ãªã©ã®çšéã«åããŠéçšãç°¡åãªRedis Sentinelã¯ã©ã¹ã¿ãæäŸããŠããŸããŠãã¯ã©ã€ã¢ã³ãå®è£
ã«æãå
¥ãã¥ããéã«ãããå
¥ããã ãã§ããæãã«åããšããã¡ããã©ããTCPãããã·ãšããŠã®ç«ã¡äœçœ®ã確ç«ãã€ã€ãããŸãã KEELããŒã ã§ã¯ãããã€ãã®æ©èœãProxy-Wasmã§å®è£
ããŠIstioçµç±ã§é
åžããŠããããããŸãããããããå®è£
ã¯Proxy-Wasmã§ã¯å®çŸãã¥ãã第äºã®Service MeshãšããŠä»åŸããã®ã¡ããã©ããTCPãããã·ãæ¹åããŠããäºå®ã§ãã (ãªãŒããŒãšã³ãžãã¢ãªã³ã°ãšã®å¢ç®ãæ°ã«ããªããã)ãã©ãããã©ãŒã ãšããŠå¿
èŠãªãã®ãããã°äœã§ãå®è£
ããKEELããŒã ã«ããèå³ãæã£ãŠããã ããå Žåã¯ãæ¯éã«ãžã¥ã¢ã«é¢è«ããããŠãã ããïŒ hrmos.co hrmos.co 次ã®ãšã³ããªã§ã¯ãKubernetesã®Podã®çµäºéã®ãã°ãæ¬ æããŠããŸãåé¡ãeBPFã§ unlink(2) ã bpf_override_return ããŠå¯ŸåŠããåé¡ã«ã€ããŠæžãããšæãã®ã§è³ŒèªããŠãåŸ
ã¡ãã ãã :D