deadlock-detector.go 2.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120
  1. /*
  2. Copyright 2015 The Kubernetes Authors.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. package threading
  14. import (
  15. "os"
  16. "sync"
  17. "time"
  18. "github.com/golang/glog"
  19. )
  20. type rwMutexToLockableAdapter struct {
  21. rw *sync.RWMutex
  22. }
  23. func (r *rwMutexToLockableAdapter) Lock() {
  24. r.rw.RLock()
  25. }
  26. func (r *rwMutexToLockableAdapter) Unlock() {
  27. r.rw.RUnlock()
  28. }
  29. type deadlockDetector struct {
  30. name string
  31. lock sync.Locker
  32. maxLockPeriod time.Duration
  33. exiter exiter
  34. exitChannelFn func() <-chan time.Time
  35. // Really only useful for testing
  36. stopChannel <-chan bool
  37. }
  38. // DeadlockWatchdogReadLock creates a watchdog on read/write mutex. If the mutex can not be acquired
  39. // for read access within 'maxLockPeriod', the program exits via glog.Exitf() or os.Exit() if that fails
  40. // 'name' is a semantic name that is useful for the user and is printed on exit.
  41. func DeadlockWatchdogReadLock(lock *sync.RWMutex, name string, maxLockPeriod time.Duration) {
  42. DeadlockWatchdog(&rwMutexToLockableAdapter{lock}, name, maxLockPeriod)
  43. }
  44. func DeadlockWatchdog(lock sync.Locker, name string, maxLockPeriod time.Duration) {
  45. if maxLockPeriod <= 0 {
  46. panic("maxLockPeriod is <= 0, that can't be what you wanted")
  47. }
  48. detector := &deadlockDetector{
  49. lock: lock,
  50. name: name,
  51. maxLockPeriod: maxLockPeriod,
  52. exitChannelFn: func() <-chan time.Time { return time.After(maxLockPeriod) },
  53. stopChannel: make(chan bool),
  54. }
  55. go detector.run()
  56. }
  57. // Useful for injecting tests
  58. type exiter interface {
  59. Exitf(format string, args ...interface{})
  60. }
  61. type realExiter struct{}
  62. func (realExiter) Exitf(format string, args ...interface{}) {
  63. func() {
  64. defer func() {
  65. // Let's just be extra sure we die, even if Exitf panics
  66. if r := recover(); r != nil {
  67. glog.Errorf(format, args...)
  68. os.Exit(2)
  69. }
  70. }()
  71. glog.Exitf(format, args...)
  72. }()
  73. }
  74. func (d *deadlockDetector) run() {
  75. for {
  76. if !d.runOnce() {
  77. return
  78. }
  79. time.Sleep(d.maxLockPeriod / 2)
  80. }
  81. }
  82. func (d *deadlockDetector) runOnce() bool {
  83. ch := make(chan bool, 1)
  84. go func() {
  85. d.lock.Lock()
  86. d.lock.Unlock()
  87. ch <- true
  88. }()
  89. exitCh := d.exitChannelFn()
  90. select {
  91. case <-exitCh:
  92. d.exiter.Exitf("Deadlock on %s, exiting", d.name)
  93. // return is here for when we use a fake exiter in testing
  94. return false
  95. case <-ch:
  96. glog.V(6).Infof("%s is not deadlocked", d.name)
  97. case <-d.stopChannel:
  98. glog.V(4).Infof("Stopping deadlock detector for %s", d.name)
  99. return false
  100. }
  101. return true
  102. }