Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit b9ffb7e

Browse filesBrowse files
pacoxupacoxu
committed
add seperate health check/probe for multi etcd override servers
- grouping health checks for exclusion purposes & add exclude integration test Signed-off-by: Paco Xu <paco.xu@daocloud.io>
1 parent b82fd6c commit b9ffb7e
Copy full SHA for b9ffb7e

File tree

Expand file treeCollapse file tree

5 files changed

+172
-83
lines changed
Filter options
Expand file treeCollapse file tree

5 files changed

+172
-83
lines changed

‎staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go

Copy file name to clipboardExpand all lines: staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go
+35-4Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,11 @@ type HealthChecker interface {
4343
Check(req *http.Request) error
4444
}
4545

46+
type GroupedHealthChecker interface {
47+
HealthChecker
48+
GroupName() string
49+
}
50+
4651
// PingHealthz returns true automatically when checked
4752
var PingHealthz HealthChecker = ping{}
4853

@@ -151,6 +156,14 @@ func NamedCheck(name string, check func(r *http.Request) error) HealthChecker {
151156
return &healthzCheck{name, check}
152157
}
153158

159+
// NamedGroupedCheck returns a healthz checker for the given name and function.
160+
func NamedGroupedCheck(name string, groupName string, check func(r *http.Request) error) HealthChecker {
161+
return &groupedHealthzCheck{
162+
groupName: groupName,
163+
HealthChecker: &healthzCheck{name, check},
164+
}
165+
}
166+
154167
// InstallHandler registers handlers for health checking on the path
155168
// "/healthz" to mux. *All handlers* for mux must be specified in
156169
// exactly one call to InstallHandler. Calling InstallHandler more
@@ -232,6 +245,17 @@ func (c *healthzCheck) Check(r *http.Request) error {
232245
return c.check(r)
233246
}
234247

248+
type groupedHealthzCheck struct {
249+
groupName string
250+
HealthChecker
251+
}
252+
253+
var _ GroupedHealthChecker = &groupedHealthzCheck{}
254+
255+
func (c *groupedHealthzCheck) GroupName() string {
256+
return c.groupName
257+
}
258+
235259
// getExcludedChecks extracts the health check names to be excluded from the query param
236260
func getExcludedChecks(r *http.Request) sets.String {
237261
checks, found := r.URL.Query()["exclude"]
@@ -246,17 +270,24 @@ func handleRootHealth(name string, firstTimeHealthy func(), checks ...HealthChec
246270
var notifyOnce sync.Once
247271
return func(w http.ResponseWriter, r *http.Request) {
248272
excluded := getExcludedChecks(r)
273+
unknownExcluded := excluded.Clone()
249274
// failedVerboseLogOutput is for output to the log. It indicates detailed failed output information for the log.
250275
var failedVerboseLogOutput bytes.Buffer
251276
var failedChecks []string
252277
var individualCheckOutput bytes.Buffer
253278
for _, check := range checks {
254279
// no-op the check if we've specified we want to exclude the check
255280
if excluded.Has(check.Name()) {
256-
excluded.Delete(check.Name())
281+
unknownExcluded.Delete(check.Name())
257282
fmt.Fprintf(&individualCheckOutput, "[+]%s excluded: ok\n", check.Name())
258283
continue
259284
}
285+
// no-op the check if it is a grouped check and we want to exclude the group
286+
if check, ok := check.(GroupedHealthChecker); ok && excluded.Has(check.GroupName()) {
287+
unknownExcluded.Delete(check.GroupName())
288+
fmt.Fprintf(&individualCheckOutput, "[+]%s excluded with %s: ok\n", check.Name(), check.GroupName())
289+
continue
290+
}
260291
if err := check.Check(r); err != nil {
261292
slis.ObserveHealthcheck(context.Background(), check.Name(), name, slis.Error)
262293
// don't include the error since this endpoint is public. If someone wants more detail
@@ -270,10 +301,10 @@ func handleRootHealth(name string, firstTimeHealthy func(), checks ...HealthChec
270301
fmt.Fprintf(&individualCheckOutput, "[+]%s ok\n", check.Name())
271302
}
272303
}
273-
if excluded.Len() > 0 {
274-
fmt.Fprintf(&individualCheckOutput, "warn: some health checks cannot be excluded: no matches for %s\n", formatQuoted(excluded.List()...))
304+
if unknownExcluded.Len() > 0 {
305+
fmt.Fprintf(&individualCheckOutput, "warn: some health checks cannot be excluded: no matches for %s\n", formatQuoted(unknownExcluded.List()...))
275306
klog.V(6).Infof("cannot exclude some health checks, no health checks are installed matching %s",
276-
formatQuoted(excluded.List()...))
307+
formatQuoted(unknownExcluded.List()...))
277308
}
278309
// always be verbose on failure
279310
if len(failedChecks) > 0 {

‎staging/src/k8s.io/apiserver/pkg/server/options/etcd.go

Copy file name to clipboardExpand all lines: staging/src/k8s.io/apiserver/pkg/server/options/etcd.go
+65-33Lines changed: 65 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"context"
2121
"fmt"
2222
"net/http"
23+
"sort"
2324
"strconv"
2425
"strings"
2526
"time"
@@ -106,19 +107,8 @@ func (s *EtcdOptions) Validate() []error {
106107
allErrors = append(allErrors, fmt.Errorf("--storage-backend invalid, allowed values: %s. If not specified, it will default to 'etcd3'", strings.Join(storageTypes.List(), ", ")))
107108
}
108109

109-
for _, override := range s.EtcdServersOverrides {
110-
tokens := strings.Split(override, "#")
111-
if len(tokens) != 2 {
112-
allErrors = append(allErrors, fmt.Errorf("--etcd-servers-overrides invalid, must be of format: group/resource#servers, where servers are URLs, semicolon separated"))
113-
continue
114-
}
115-
116-
apiresource := strings.Split(tokens[0], "/")
117-
if len(apiresource) != 2 {
118-
allErrors = append(allErrors, fmt.Errorf("--etcd-servers-overrides invalid, must be of format: group/resource#servers, where servers are URLs, semicolon separated"))
119-
continue
120-
}
121-
110+
if _, err := ParseEtcdServersOverrides(s.EtcdServersOverrides); err != nil {
111+
allErrors = append(allErrors, err)
122112
}
123113

124114
if len(s.EncryptionProviderConfigFilepath) == 0 && s.EncryptionProviderConfigAutomaticReload {
@@ -363,58 +353,68 @@ func (s *EtcdOptions) addEtcdHealthEndpoint(c *server.Config) error {
363353
if err != nil {
364354
return err
365355
}
366-
c.AddHealthChecks(healthz.NamedCheck("etcd", func(r *http.Request) error {
356+
c.AddHealthChecks(healthz.NamedGroupedCheck("etcd", "etcd", func(r *http.Request) error {
367357
return healthCheck()
368358
}))
369359

370360
readyCheck, err := storagefactory.CreateReadyCheck(s.StorageConfig, c.DrainedNotify())
371361
if err != nil {
372362
return err
373363
}
374-
c.AddReadyzChecks(healthz.NamedCheck("etcd-readiness", func(r *http.Request) error {
364+
c.AddReadyzChecks(healthz.NamedGroupedCheck("etcd-readiness", "etcd-readiness", func(r *http.Request) error {
375365
return readyCheck()
376366
}))
377367

378368
if len(s.EtcdServersOverrides) != 0 {
379-
if err := s.addOverrideEtcdHealthEndpoint(c); err != nil {
369+
// multi overrides servers may in different order
370+
// example: ["apps/deployments#s2.example.com;s1.example.com","apps/replicasets#s1.example.com;s2.example.com"]
371+
// ParseEtcdServersOverrides will sort the servers
372+
overrides, err := ParseEtcdServersOverrides(s.EtcdServersOverrides)
373+
if err != nil {
380374
return err
381375
}
376+
// multi overrides may point to the same servers
377+
// example: ["apps/deployments#s1.example.com;s2.example.com","apps/replicasets#s1.example.com;s2.example.com"]
378+
serversSets := sets.NewString()
379+
for _, override := range overrides {
380+
sortedServers := make([]string, len(override.Servers))
381+
// use a copied slice to avoid modifying the original slice for client in SetEtcdLocation
382+
copy(sortedServers, override.Servers)
383+
sort.Strings(sortedServers)
384+
serversKeyStr := strings.Join(sortedServers, ";")
385+
if serversSets.Has(serversKeyStr) {
386+
continue
387+
}
388+
serversSets.Insert(serversKeyStr)
389+
err := s.SetEtcdLocation(c, override.GroupResource, override.Servers, len(serversSets)-1)
390+
if err != nil {
391+
return err
392+
}
393+
}
382394
}
383-
384395
return nil
385396
}
386397

387-
func (s *EtcdOptions) addOverrideEtcdHealthEndpoint(c *server.Config) error {
398+
// SetEtcdLocation sets the etcd location for the given resource
399+
func (s *EtcdOptions) SetEtcdLocation(c *server.Config, group schema.GroupResource, servers []string, index int) error {
388400
sc := s.StorageConfig
389-
serverSet := sets.Set[string]{}
390-
391-
for _, override := range s.EtcdServersOverrides {
392-
tokens := strings.Split(override, "#")
393-
servers := strings.Split(tokens[1], ";")
394-
for _, server := range servers {
395-
serverSet.Insert(server)
396-
}
397-
}
398-
if serverSet.Len() != 0 {
399-
sc.Transport.ServerList = serverSet.UnsortedList()
400-
}
401+
sc.Transport.ServerList = servers
401402

402403
healthCheck, err := storagefactory.CreateHealthCheck(sc, c.DrainedNotify())
403404
if err != nil {
404405
return err
405406
}
406-
c.AddHealthChecks(healthz.NamedCheck("etcd-override", func(r *http.Request) error {
407+
c.AddHealthChecks(healthz.NamedGroupedCheck(fmt.Sprintf("etcd-override-%d", index), "etcd", func(r *http.Request) error {
407408
return healthCheck()
408409
}))
409410

410411
readyCheck, err := storagefactory.CreateReadyCheck(sc, c.DrainedNotify())
411412
if err != nil {
412413
return err
413414
}
414-
c.AddReadyzChecks(healthz.NamedCheck("etcd-override-readiness", func(r *http.Request) error {
415+
c.AddReadyzChecks(healthz.NamedGroupedCheck(fmt.Sprintf("etcd-override-readiness-%d", index), "etcd-readiness", func(r *http.Request) error {
415416
return readyCheck()
416417
}))
417-
418418
return nil
419419
}
420420

@@ -558,3 +558,35 @@ func (t *transformerStorageFactory) Configs() []storagebackend.Config {
558558
func (t *transformerStorageFactory) Backends() []serverstorage.Backend {
559559
return t.delegate.Backends()
560560
}
561+
562+
type EtcdServerOverride struct {
563+
GroupResource schema.GroupResource
564+
Servers []string
565+
}
566+
567+
var errOverridesInvalid = fmt.Errorf("--etcd-servers-overrides invalid, must be of format: group/resource#servers, where servers are URLs, semicolon separated")
568+
569+
func ParseEtcdServersOverrides(etcdServersOverrides []string) ([]EtcdServerOverride, error) {
570+
var overrides []EtcdServerOverride
571+
for _, override := range etcdServersOverrides {
572+
tokens := strings.Split(override, "#")
573+
if len(tokens) != 2 {
574+
return nil, errOverridesInvalid
575+
}
576+
apiresource := strings.Split(tokens[0], "/")
577+
if len(apiresource) != 2 {
578+
return nil, errOverridesInvalid
579+
}
580+
servers := strings.Split(tokens[1], ";")
581+
for _, server := range servers {
582+
if len(server) == 0 {
583+
return nil, errOverridesInvalid
584+
}
585+
}
586+
overrides = append(overrides, EtcdServerOverride{
587+
GroupResource: schema.GroupResource{Group: apiresource[0], Resource: apiresource[1]},
588+
Servers: servers,
589+
})
590+
}
591+
return overrides, nil
592+
}

‎staging/src/k8s.io/apiserver/pkg/server/options/etcd_test.go

Copy file name to clipboardExpand all lines: staging/src/k8s.io/apiserver/pkg/server/options/etcd_test.go
+38-6Lines changed: 38 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -375,11 +375,12 @@ func TestKMSHealthzEndpoint(t *testing.T) {
375375

376376
func TestReadinessCheck(t *testing.T) {
377377
testCases := []struct {
378-
name string
379-
wantReadyzChecks []string
380-
wantHealthzChecks []string
381-
wantLivezChecks []string
382-
skipHealth bool
378+
name string
379+
wantReadyzChecks []string
380+
wantHealthzChecks []string
381+
wantLivezChecks []string
382+
skipHealth bool
383+
etcdServersOverrides []string
383384
}{
384385
{
385386
name: "Readyz should have etcd-readiness check",
@@ -394,6 +395,37 @@ func TestReadinessCheck(t *testing.T) {
394395
wantLivezChecks: nil,
395396
skipHealth: true,
396397
},
398+
{
399+
name: "Health checks should not have duplicated servers from etcd-servers-overrides",
400+
wantReadyzChecks: []string{"etcd", "etcd-readiness", "etcd-override-0", "etcd-override-readiness-0"},
401+
wantHealthzChecks: []string{"etcd", "etcd-override-0"},
402+
wantLivezChecks: []string{"etcd", "etcd-override-0"},
403+
etcdServersOverrides: []string{"/r1#s1.com;s2.com", "/r2#s1.com;s2.com"},
404+
},
405+
{
406+
name: "Health checks should not have duplicated servers from etcd-servers-overrides " +
407+
"if servers are provided in different orders",
408+
wantReadyzChecks: []string{"etcd", "etcd-readiness", "etcd-override-0", "etcd-override-readiness-0"},
409+
wantHealthzChecks: []string{"etcd", "etcd-override-0"},
410+
wantLivezChecks: []string{"etcd", "etcd-override-0"},
411+
etcdServersOverrides: []string{"/r1#s1.com;s2.com", "/r2#s2.com;s1.com"},
412+
},
413+
{
414+
name: "Health checks should allow multiple overrides in etcd-servers-overrides",
415+
wantReadyzChecks: []string{"etcd", "etcd-readiness", "etcd-override-0", "etcd-override-readiness-0",
416+
"etcd-override-1", "etcd-override-readiness-1"},
417+
wantHealthzChecks: []string{"etcd", "etcd-override-0", "etcd-override-1"},
418+
wantLivezChecks: []string{"etcd", "etcd-override-0", "etcd-override-1"},
419+
etcdServersOverrides: []string{"/r1#s1.com;s2.com", "/r2#s3.com;s4.com"},
420+
},
421+
{
422+
name: "Health checks should allow multiple overrides in etcd-servers-overrides if servers overlap between overrides",
423+
wantReadyzChecks: []string{"etcd", "etcd-readiness", "etcd-override-0", "etcd-override-readiness-0",
424+
"etcd-override-1", "etcd-override-readiness-1"},
425+
wantHealthzChecks: []string{"etcd", "etcd-override-0", "etcd-override-1"},
426+
wantLivezChecks: []string{"etcd", "etcd-override-0", "etcd-override-1"},
427+
etcdServersOverrides: []string{"/r1#s1.com;s2.com", "/r2#s2.com;s3.com"},
428+
},
397429
}
398430

399431
scheme := runtime.NewScheme()
@@ -402,7 +434,7 @@ func TestReadinessCheck(t *testing.T) {
402434
for _, tc := range testCases {
403435
t.Run(tc.name, func(t *testing.T) {
404436
serverConfig := server.NewConfig(codecs)
405-
etcdOptions := &EtcdOptions{SkipHealthEndpoints: tc.skipHealth}
437+
etcdOptions := &EtcdOptions{SkipHealthEndpoints: tc.skipHealth, EtcdServersOverrides: tc.etcdServersOverrides}
406438
if err := etcdOptions.ApplyTo(serverConfig); err != nil {
407439
t.Fatalf("Failed to add healthz error: %v", err)
408440
}

‎staging/src/k8s.io/apiserver/pkg/storage/storagebackend/config.go

Copy file name to clipboardExpand all lines: staging/src/k8s.io/apiserver/pkg/storage/storagebackend/config.go
-35Lines changed: 0 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -125,38 +125,3 @@ func NewDefaultConfig(prefix string, codec runtime.Codec) *Config {
125125
Transport: TransportConfig{TracerProvider: noopoteltrace.NewTracerProvider()},
126126
}
127127
}
128-
129-
// DeepCopy returns a deep copy of the Config.
130-
func (config *Config) DeepCopy() *Config {
131-
if config == nil {
132-
return nil
133-
}
134-
// Deep copy the transport config
135-
transport := config.Transport
136-
transportCopy := TransportConfig{
137-
ServerList: make([]string, len(transport.ServerList)),
138-
KeyFile: transport.KeyFile,
139-
CertFile: transport.CertFile,
140-
TrustedCAFile: transport.TrustedCAFile,
141-
EgressLookup: transport.EgressLookup,
142-
TracerProvider: transport.TracerProvider,
143-
}
144-
copy(transportCopy.ServerList, transport.ServerList)
145-
146-
return &Config{
147-
Type: config.Type,
148-
Prefix: config.Prefix,
149-
Transport: transportCopy,
150-
Codec: config.Codec,
151-
EncodeVersioner: config.EncodeVersioner,
152-
Transformer: config.Transformer,
153-
CompactionInterval: config.CompactionInterval,
154-
CountMetricPollPeriod: config.CountMetricPollPeriod,
155-
DBMetricPollInterval: config.DBMetricPollInterval,
156-
EventsHistoryWindow: config.EventsHistoryWindow,
157-
HealthcheckTimeout: config.HealthcheckTimeout,
158-
ReadycheckTimeout: config.ReadycheckTimeout,
159-
LeaseManagerConfig: config.LeaseManagerConfig,
160-
StorageObjectCountTracker: config.StorageObjectCountTracker,
161-
}
162-
}

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.