Module: TestRunner::RetryClassifier

Defined in:
lib/test-runner/retry_classifier.rb

Constant Summary collapse

ANSI_ESCAPE =
%r{\e(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])}
APT_GENERIC_FAILURES =
[
  /^E: Some index files failed to download\./,
  /^E: Unable to fetch some archives/
].freeze
APK_GENERIC_FAILURES =
[
  %r{^ERROR: Not continuing due to stale/unavailable repositories\.}
].freeze
APK_FAILURE_SUMMARIES =
[
  /^\d+ unavailable, \d+ stale;/,
  /^(?:\d+ errors?|ERRORS);/
].freeze
SWH_PROGRESS =
/SWH vault: (?:requested bundle cooking|Processing)/

Class Method Summary collapse

Class Method Details

.apk(error) ⇒ Object



56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
# File 'lib/test-runner/retry_classifier.rb', line 56

def apk(error)
  return unless error.is_a?(OsVm::CommandFailed)

  output_lines = command_output_lines(error)
  terminal_line = output_lines.last
  return unless terminal_line
  return unless APK_FAILURE_SUMMARIES.any? { |pattern| pattern.match?(terminal_line) } ||
                APK_GENERIC_FAILURES.any? { |pattern| pattern.match?(terminal_line) }

  diagnostic_lines = output_lines.select do |line|
    line.start_with?('WARNING:', 'ERROR:')
  end
  return if diagnostic_lines.empty?

  reasons = diagnostic_lines.filter_map { |line| apk_failure_reason(line) }
  return if reasons.empty?

  # A failed index refresh can leave cached package URLs stale. Only
  # tolerate package 404s when this same operation reported a transient
  # APKINDEX fetch failure; a standalone 404 is not retryable.
  transient_index_failure = diagnostic_lines.any? do |line|
    line.start_with?('WARNING:') && line.include?('APKINDEX.tar.gz:') && apk_failure_reason(line)
  end

  return unless diagnostic_lines.all? do |line|
    apk_failure_reason(line) ||
    APK_GENERIC_FAILURES.any? { |pattern| pattern.match?(line) } ||
    (transient_index_failure && /\AERROR: [^\s:]+: HTTP 404: Not Found\z/.match?(line))
  end

  reasons.last
end

.apt(error) ⇒ Object



23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
# File 'lib/test-runner/retry_classifier.rb', line 23

def apt(error)
  return unless error.is_a?(OsVm::CommandFailed)

  output_lines = command_output_lines(error)
  return unless output_lines.last&.start_with?('E: ')

  error_lines = output_lines.select { |line| line.start_with?('E: ') }
  return unless error_lines.all? do |line|
    generic_failure = APT_GENERIC_FAILURES.any? { |pattern| pattern.match?(line) }
    fetch_failure = line.start_with?('E: Failed to fetch ') && apt_failure_reason(line)

    generic_failure || fetch_failure
  end

  reasons = error_lines.filter_map { |line| apt_failure_reason(line) }
  reasons.last
end

.guix_operation(error) ⇒ Object



103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
# File 'lib/test-runner/retry_classifier.rb', line 103

def guix_operation(error)
  return unless error.is_a?(OsVm::CommandFailed)

  terminal_error = command_output_lines(error).reverse.find do |line|
    line.match?(/(?:^|:) error:/i) || line.match?(/Git error:/i)
  end
  return unless terminal_error

  case terminal_error
  when /Git error:.*(?:SSL error|Resource temporarily unavailable|Temporary failure in name resolution|Could not resolve host|Name or service not known|Connection timed out|Connection reset by peer|502 Bad Gateway|503 Service Unavailable|504 Gateway Time-out)/i
    'transient Guix Git failure'
  when /some substitutes .* failed \(usually happens due to networking issues\)/i
    'transient Guix substitute failure'
  end
end

.guix_preparation(error) ⇒ Object



119
120
121
122
123
124
125
126
127
128
129
130
131
132
# File 'lib/test-runner/retry_classifier.rb', line 119

def guix_preparation(error)
  return unless error.is_a?(OsVm::CommandFailed)

  if error.message.match?(/failed with status 124\./)
    output_lines = command_output_lines(error)
    progress_index = output_lines.rindex { |line| SWH_PROGRESS.match?(line) }

    if progress_index && output_lines[(progress_index + 1)..].all? { |line| line == 'Terminated' }
      return 'Software Heritage fallback stalled'
    end
  end

  guix_operation(error)
end