trace.go 9.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282
  1. /*
  2. Copyright 2015 The Kubernetes Authors.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. package trace
  14. import (
  15. "bytes"
  16. "context"
  17. "fmt"
  18. "math/rand"
  19. "time"
  20. "k8s.io/klog/v2"
  21. )
  22. var klogV = func(lvl klog.Level) bool {
  23. return klog.V(lvl).Enabled()
  24. }
  25. // Field is a key value pair that provides additional details about the trace.
  26. type Field struct {
  27. Key string
  28. Value interface{}
  29. }
  30. func (f Field) format() string {
  31. return fmt.Sprintf("%s:%v", f.Key, f.Value)
  32. }
  33. func writeFields(b *bytes.Buffer, l []Field) {
  34. for i, f := range l {
  35. b.WriteString(f.format())
  36. if i < len(l)-1 {
  37. b.WriteString(",")
  38. }
  39. }
  40. }
  41. func writeTraceItemSummary(b *bytes.Buffer, msg string, totalTime time.Duration, startTime time.Time, fields []Field) {
  42. b.WriteString(fmt.Sprintf("%q ", msg))
  43. if len(fields) > 0 {
  44. writeFields(b, fields)
  45. b.WriteString(" ")
  46. }
  47. b.WriteString(fmt.Sprintf("%vms (%v)", durationToMilliseconds(totalTime), startTime.Format("15:04:00.000")))
  48. }
  49. func durationToMilliseconds(timeDuration time.Duration) int64 {
  50. return timeDuration.Nanoseconds() / 1e6
  51. }
  52. type traceItem interface {
  53. // time returns when the trace was recorded as completed.
  54. time() time.Time
  55. // writeItem outputs the traceItem to the buffer. If stepThreshold is non-nil, only output the
  56. // traceItem if its the duration exceeds the stepThreshold.
  57. // Each line of output is prefixed by formatter to visually indent nested items.
  58. writeItem(b *bytes.Buffer, formatter string, startTime time.Time, stepThreshold *time.Duration)
  59. }
  60. type traceStep struct {
  61. stepTime time.Time
  62. msg string
  63. fields []Field
  64. }
  65. func (s traceStep) time() time.Time {
  66. return s.stepTime
  67. }
  68. func (s traceStep) writeItem(b *bytes.Buffer, formatter string, startTime time.Time, stepThreshold *time.Duration) {
  69. stepDuration := s.stepTime.Sub(startTime)
  70. if stepThreshold == nil || *stepThreshold == 0 || stepDuration >= *stepThreshold || klogV(4) {
  71. b.WriteString(fmt.Sprintf("%s---", formatter))
  72. writeTraceItemSummary(b, s.msg, stepDuration, s.stepTime, s.fields)
  73. }
  74. }
  75. // Trace keeps track of a set of "steps" and allows us to log a specific
  76. // step if it took longer than its share of the total allowed time
  77. type Trace struct {
  78. name string
  79. fields []Field
  80. threshold *time.Duration
  81. startTime time.Time
  82. endTime *time.Time
  83. traceItems []traceItem
  84. parentTrace *Trace
  85. }
  86. func (t *Trace) time() time.Time {
  87. if t.endTime != nil {
  88. return *t.endTime
  89. }
  90. return t.startTime // if the trace is incomplete, don't assume an end time
  91. }
  92. func (t *Trace) writeItem(b *bytes.Buffer, formatter string, startTime time.Time, stepThreshold *time.Duration) {
  93. if t.durationIsWithinThreshold() || klogV(4) {
  94. b.WriteString(fmt.Sprintf("%v[", formatter))
  95. writeTraceItemSummary(b, t.name, t.TotalTime(), t.startTime, t.fields)
  96. if st := t.calculateStepThreshold(); st != nil {
  97. stepThreshold = st
  98. }
  99. t.writeTraceSteps(b, formatter+" ", stepThreshold)
  100. b.WriteString("]")
  101. return
  102. }
  103. // If the trace should not be written, still check for nested traces that should be written
  104. for _, s := range t.traceItems {
  105. if nestedTrace, ok := s.(*Trace); ok {
  106. nestedTrace.writeItem(b, formatter, startTime, stepThreshold)
  107. }
  108. }
  109. }
  110. // New creates a Trace with the specified name. The name identifies the operation to be traced. The
  111. // Fields add key value pairs to provide additional details about the trace, such as operation inputs.
  112. func New(name string, fields ...Field) *Trace {
  113. return &Trace{name: name, startTime: time.Now(), fields: fields}
  114. }
  115. // Step adds a new step with a specific message. Call this at the end of an execution step to record
  116. // how long it took. The Fields add key value pairs to provide additional details about the trace
  117. // step.
  118. func (t *Trace) Step(msg string, fields ...Field) {
  119. if t.traceItems == nil {
  120. // traces almost always have less than 6 steps, do this to avoid more than a single allocation
  121. t.traceItems = make([]traceItem, 0, 6)
  122. }
  123. t.traceItems = append(t.traceItems, traceStep{stepTime: time.Now(), msg: msg, fields: fields})
  124. }
  125. // Nest adds a nested trace with the given message and fields and returns it.
  126. // As a convenience, if the receiver is nil, returns a top level trace. This allows
  127. // one to call FromContext(ctx).Nest without having to check if the trace
  128. // in the context is nil.
  129. func (t *Trace) Nest(msg string, fields ...Field) *Trace {
  130. newTrace := New(msg, fields...)
  131. if t != nil {
  132. newTrace.parentTrace = t
  133. t.traceItems = append(t.traceItems, newTrace)
  134. }
  135. return newTrace
  136. }
  137. // Log is used to dump all the steps in the Trace. It also logs the nested trace messages using indentation.
  138. // If the Trace is nested it is not immediately logged. Instead, it is logged when the trace it is nested within
  139. // is logged.
  140. func (t *Trace) Log() {
  141. endTime := time.Now()
  142. t.endTime = &endTime
  143. // an explicit logging request should dump all the steps out at the higher level
  144. if t.parentTrace == nil { // We don't start logging until Log or LogIfLong is called on the root trace
  145. t.logTrace()
  146. }
  147. }
  148. // LogIfLong only logs the trace if the duration of the trace exceeds the threshold.
  149. // Only steps that took longer than their share or the given threshold are logged.
  150. // If klog is at verbosity level 4 or higher and the trace took longer than the threshold,
  151. // all substeps and subtraces are logged. Otherwise, only those which took longer than
  152. // their own threshold.
  153. // If the Trace is nested it is not immediately logged. Instead, it is logged when the trace it
  154. // is nested within is logged.
  155. func (t *Trace) LogIfLong(threshold time.Duration) {
  156. t.threshold = &threshold
  157. t.Log()
  158. }
  159. // logTopLevelTraces finds all traces in a hierarchy of nested traces that should be logged but do not have any
  160. // parents that will be logged, due to threshold limits, and logs them as top level traces.
  161. func (t *Trace) logTrace() {
  162. if t.durationIsWithinThreshold() {
  163. var buffer bytes.Buffer
  164. traceNum := rand.Int31()
  165. totalTime := t.endTime.Sub(t.startTime)
  166. buffer.WriteString(fmt.Sprintf("Trace[%d]: %q ", traceNum, t.name))
  167. if len(t.fields) > 0 {
  168. writeFields(&buffer, t.fields)
  169. buffer.WriteString(" ")
  170. }
  171. // if any step took more than it's share of the total allowed time, it deserves a higher log level
  172. buffer.WriteString(fmt.Sprintf("(%v) (total time: %vms):", t.startTime.Format("02-Jan-2006 15:04:05.000"), totalTime.Milliseconds()))
  173. stepThreshold := t.calculateStepThreshold()
  174. t.writeTraceSteps(&buffer, fmt.Sprintf("\nTrace[%d]: ", traceNum), stepThreshold)
  175. buffer.WriteString(fmt.Sprintf("\nTrace[%d]: [%v] [%v] END\n", traceNum, t.endTime.Sub(t.startTime), totalTime))
  176. klog.Info(buffer.String())
  177. return
  178. }
  179. // If the trace should not be logged, still check if nested traces should be logged
  180. for _, s := range t.traceItems {
  181. if nestedTrace, ok := s.(*Trace); ok {
  182. nestedTrace.logTrace()
  183. }
  184. }
  185. }
  186. func (t *Trace) writeTraceSteps(b *bytes.Buffer, formatter string, stepThreshold *time.Duration) {
  187. lastStepTime := t.startTime
  188. for _, stepOrTrace := range t.traceItems {
  189. stepOrTrace.writeItem(b, formatter, lastStepTime, stepThreshold)
  190. lastStepTime = stepOrTrace.time()
  191. }
  192. }
  193. func (t *Trace) durationIsWithinThreshold() bool {
  194. if t.endTime == nil { // we don't assume incomplete traces meet the threshold
  195. return false
  196. }
  197. return t.threshold == nil || *t.threshold == 0 || t.endTime.Sub(t.startTime) >= *t.threshold
  198. }
  199. // TotalTime can be used to figure out how long it took since the Trace was created
  200. func (t *Trace) TotalTime() time.Duration {
  201. return time.Since(t.startTime)
  202. }
  203. // calculateStepThreshold returns a threshold for the individual steps of a trace, or nil if there is no threshold and
  204. // all steps should be written.
  205. func (t *Trace) calculateStepThreshold() *time.Duration {
  206. if t.threshold == nil {
  207. return nil
  208. }
  209. lenTrace := len(t.traceItems) + 1
  210. traceThreshold := *t.threshold
  211. for _, s := range t.traceItems {
  212. nestedTrace, ok := s.(*Trace)
  213. if ok && nestedTrace.threshold != nil {
  214. traceThreshold = traceThreshold - *nestedTrace.threshold
  215. lenTrace--
  216. }
  217. }
  218. // the limit threshold is used when the threshold(
  219. //remaining after subtracting that of the child trace) is getting very close to zero to prevent unnecessary logging
  220. limitThreshold := *t.threshold / 4
  221. if traceThreshold < limitThreshold {
  222. traceThreshold = limitThreshold
  223. lenTrace = len(t.traceItems) + 1
  224. }
  225. stepThreshold := traceThreshold / time.Duration(lenTrace)
  226. return &stepThreshold
  227. }
  228. // ContextTraceKey provides a common key for traces in context.Context values.
  229. type ContextTraceKey struct{}
  230. // FromContext returns the trace keyed by ContextTraceKey in the context values, if one
  231. // is present, or nil If there is no trace in the Context.
  232. // It is safe to call Nest() on the returned value even if it is nil because ((*Trace)nil).Nest returns a top level
  233. // trace.
  234. func FromContext(ctx context.Context) *Trace {
  235. if v, ok := ctx.Value(ContextTraceKey{}).(*Trace); ok {
  236. return v
  237. }
  238. return nil
  239. }
  240. // ContextWithTrace returns a context with trace included in the context values, keyed by ContextTraceKey.
  241. func ContextWithTrace(ctx context.Context, trace *Trace) context.Context {
  242. return context.WithValue(ctx, ContextTraceKey{}, trace)
  243. }