pet_set_utils.go 5.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168
  1. /*
  2. Copyright 2016 The Kubernetes Authors.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. package petset
  14. import (
  15. "fmt"
  16. "sync"
  17. "k8s.io/kubernetes/pkg/api"
  18. "k8s.io/kubernetes/pkg/apis/apps"
  19. "k8s.io/kubernetes/pkg/client/cache"
  20. client "k8s.io/kubernetes/pkg/client/unversioned"
  21. "k8s.io/kubernetes/pkg/controller"
  22. "github.com/golang/glog"
  23. )
  24. // overlappingPetSets sorts a list of PetSets by creation timestamp, using their names as a tie breaker.
  25. // Generally used to tie break between PetSets that have overlapping selectors.
  26. type overlappingPetSets []apps.PetSet
  27. func (o overlappingPetSets) Len() int { return len(o) }
  28. func (o overlappingPetSets) Swap(i, j int) { o[i], o[j] = o[j], o[i] }
  29. func (o overlappingPetSets) Less(i, j int) bool {
  30. if o[i].CreationTimestamp.Equal(o[j].CreationTimestamp) {
  31. return o[i].Name < o[j].Name
  32. }
  33. return o[i].CreationTimestamp.Before(o[j].CreationTimestamp)
  34. }
  35. // updatePetCount attempts to update the Status.Replicas of the given PetSet, with a single GET/PUT retry.
  36. func updatePetCount(kubeClient *client.Client, ps apps.PetSet, numPets int) (updateErr error) {
  37. if ps.Status.Replicas == numPets || kubeClient == nil {
  38. return nil
  39. }
  40. psClient := kubeClient.Apps().PetSets(ps.Namespace)
  41. var getErr error
  42. for i, ps := 0, &ps; ; i++ {
  43. glog.V(4).Infof(fmt.Sprintf("Updating replica count for PetSet: %s/%s, ", ps.Namespace, ps.Name) +
  44. fmt.Sprintf("replicas %d->%d (need %d), ", ps.Status.Replicas, numPets, ps.Spec.Replicas))
  45. ps.Status = apps.PetSetStatus{Replicas: numPets}
  46. _, updateErr = psClient.UpdateStatus(ps)
  47. if updateErr == nil || i >= statusUpdateRetries {
  48. return updateErr
  49. }
  50. if ps, getErr = psClient.Get(ps.Name); getErr != nil {
  51. return getErr
  52. }
  53. }
  54. }
  55. // claimClient returns the pvcClient for the given kubeClient/ns.
  56. func claimClient(kubeClient *client.Client, ns string) client.PersistentVolumeClaimInterface {
  57. return kubeClient.PersistentVolumeClaims(ns)
  58. }
  59. // podClient returns the given podClient for the given kubeClient/ns.
  60. func podClient(kubeClient *client.Client, ns string) client.PodInterface {
  61. return kubeClient.Pods(ns)
  62. }
  63. // unhealthyPetTracker tracks unhealthy pets for petsets.
  64. type unhealthyPetTracker struct {
  65. pc petClient
  66. store cache.Store
  67. storeLock sync.Mutex
  68. }
  69. // Get returns a previously recorded blocking pet for the given petset.
  70. func (u *unhealthyPetTracker) Get(ps *apps.PetSet, knownPets []*api.Pod) (*pcb, error) {
  71. u.storeLock.Lock()
  72. defer u.storeLock.Unlock()
  73. // We "Get" by key but "Add" by object because the store interface doesn't
  74. // allow us to Get/Add a related obj (eg petset: blocking pet).
  75. key, err := controller.KeyFunc(ps)
  76. if err != nil {
  77. return nil, err
  78. }
  79. obj, exists, err := u.store.GetByKey(key)
  80. if err != nil {
  81. return nil, err
  82. }
  83. hc := defaultPetHealthChecker{}
  84. // There's no unhealthy pet blocking a scale event, but this might be
  85. // a controller manager restart. If it is, knownPets can be trusted.
  86. if !exists {
  87. for _, p := range knownPets {
  88. if hc.isHealthy(p) && !hc.isDying(p) {
  89. glog.V(4).Infof("Ignoring healthy pet %v for PetSet %v", p.Name, ps.Name)
  90. continue
  91. }
  92. glog.Infof("No recorded blocking pet, but found unhealthy pet %v for PetSet %v", p.Name, ps.Name)
  93. return &pcb{pod: p, parent: ps}, nil
  94. }
  95. return nil, nil
  96. }
  97. // This is a pet that's blocking further creates/deletes of a petset. If it
  98. // disappears, it's no longer blocking. If it exists, it continues to block
  99. // till it turns healthy or disappears.
  100. bp := obj.(*pcb)
  101. blockingPet, exists, err := u.pc.Get(bp)
  102. if err != nil {
  103. return nil, err
  104. }
  105. if !exists {
  106. glog.V(4).Infof("Clearing blocking pet %v for PetSet %v because it's been deleted", bp.pod.Name, ps.Name)
  107. return nil, nil
  108. }
  109. blockingPetPod := blockingPet.pod
  110. if hc.isHealthy(blockingPetPod) && !hc.isDying(blockingPetPod) {
  111. glog.V(4).Infof("Clearing blocking pet %v for PetSet %v because it's healthy", bp.pod.Name, ps.Name)
  112. u.store.Delete(blockingPet)
  113. blockingPet = nil
  114. }
  115. return blockingPet, nil
  116. }
  117. // Add records the given pet as a blocking pet.
  118. func (u *unhealthyPetTracker) Add(blockingPet *pcb) error {
  119. u.storeLock.Lock()
  120. defer u.storeLock.Unlock()
  121. if blockingPet == nil {
  122. return nil
  123. }
  124. glog.V(4).Infof("Adding blocking pet %v for PetSet %v", blockingPet.pod.Name, blockingPet.parent.Name)
  125. return u.store.Add(blockingPet)
  126. }
  127. // newUnHealthyPetTracker tracks unhealthy pets that block progress of petsets.
  128. func newUnHealthyPetTracker(pc petClient) *unhealthyPetTracker {
  129. return &unhealthyPetTracker{pc: pc, store: cache.NewStore(pcbKeyFunc)}
  130. }
  131. // pcbKeyFunc computes the key for a given pcb.
  132. // If it's given a key, it simply returns it.
  133. func pcbKeyFunc(obj interface{}) (string, error) {
  134. if key, ok := obj.(string); ok {
  135. return key, nil
  136. }
  137. p, ok := obj.(*pcb)
  138. if !ok {
  139. return "", fmt.Errorf("not a valid pet control block %#v", p)
  140. }
  141. if p.parent == nil {
  142. return "", fmt.Errorf("cannot compute pet control block key without parent pointer %#v", p)
  143. }
  144. return controller.KeyFunc(p.parent)
  145. }