Class: TestRunner::Executor

Inherits:
Object
  • Object
show all
Defined in:
lib/test-runner/executor.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(tests, **opts) ⇒ Executor

Returns a new instance of Executor.

Parameters:

  • tests (Array<Test>)
  • opts (Hash)

Options Hash (**opts):

  • :state_dir (String)
  • :jobs (Integer)
  • :default_timeout (Integer)
  • :stop_on_failure (Boolean)
  • :destructive (Boolean) 


21
22
23
24
25
26
27
28
29
30
 
# File 'lib/test-runner/executor.rb', line 21

def initialize(tests, **opts)
  @tests = tests
  @opts = opts
  @workers = []
  @queue = Queue.new
  tests.each_with_index { |t, i| @queue << [i, t] }
  @results = []
  @stop_work = false
  @mutex = Mutex.new
end

Instance Attribute Details

#mutex ⇒ Object (readonly, protected)

Returns the value of attribute mutex.



93
94
95
 
# File 'lib/test-runner/executor.rb', line 93

def mutex
  @mutex
end

#opts ⇒ Hash (readonly)

Returns:

  • (Hash) 


9
10
11
 
# File 'lib/test-runner/executor.rb', line 9

def opts
  @opts
end

#queue ⇒ Object (readonly, protected)

Returns the value of attribute queue.



93
94
95
 
# File 'lib/test-runner/executor.rb', line 93

def queue
  @queue
end

#results ⇒ Array<TestResult> (readonly)

Returns:



12
13
14
 
# File 'lib/test-runner/executor.rb', line 12

def results
  @results
end

#tests ⇒ Array<Test> (readonly)

Returns:



6
7
8
 
# File 'lib/test-runner/executor.rb', line 6

def tests
  @tests
end

#workers ⇒ Object (readonly, protected)

Returns the value of attribute workers.



93
94
95
 
# File 'lib/test-runner/executor.rb', line 93

def workers
  @workers
end

Instance Method Details

#log(msg) ⇒ Object (protected) 



210
211
212
 
# File 'lib/test-runner/executor.rb', line 210

def log(msg)
  mutex.synchronize { puts "[#{Time.now}] #{msg}" }
end

#run ⇒ Array<TestResult>

Returns:



33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
 
# File 'lib/test-runner/executor.rb', line 33

def run
  log("Running #{tests.length} tests, #{opts[:jobs]} at a time")
  log("State directory is #{state_dir}")
  t1 = Time.now

  opts[:jobs].times do |i|
    start_worker(i)
  end

  wait_for_workers

  log("Run #{results.length} tests in #{(Time.now - t1).round(2)} seconds")

  expected_successful = results.select do |r|
    r.expected_to_succeed? && r.successful?
  end

  expected_failed = results.select do |r|
    r.expected_to_fail? && r.failed?
  end

  unexpected_failed = results.select do |r|
    r.expected_to_succeed? && r.failed?
  end

  unexpected_successful = results.select do |r|
    r.expected_to_fail? && r.successful?
  end

  if expected_successful.any?
    log("#{expected_successful.length} tests successful")
  end

  if expected_failed.any?
    log("#{expected_failed.length} tests failed as expected")
  end

  if unexpected_failed.any?
    log("#{unexpected_failed.length} tests should have succeeded, but failed")
  end

  if unexpected_successful.any?
    log("#{unexpected_successful.length} tests should have failed, but succeeded")
  end

  if unexpected_failed.any?
    log("Unexpectedly failed tests:\n#{unexpected_failed.map { |r| "  #{r.test.path}" }.join("\n")}")
    puts
  end

  if unexpected_successful.any?
    log("Unexpectedly successful tests:\n#{unexpected_successful.map { |r| "  #{r.test.path}" }.join("\n")}")
    puts
  end

  results
end

#run_test(test) ⇒ Object (protected) 



139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
 
# File 'lib/test-runner/executor.rb', line 139

def run_test(test)
  t1 = Time.now
  dir = test_state_dir(test)

  pid = Process.fork do
    FileUtils.mkdir_p(dir)

    out = File.open(File.join(dir, 'test-runner.log'), 'w')
    $stdout.reopen(out)
    $stderr.reopen(out)
    $stdin.close

    ev = TestRunner::TestEvaluator.new(
      test,
      state_dir: dir,
      sock_dir: test_sock_dir,
      default_timeout: opts[:default_timeout],
      destructive: opts[:destructive]
    )
    ev.run
  end

  Process.wait(pid)

  result = TestResult.new(
    test,
    $?.exitstatus == 0,
    Time.now - t1,
    dir
  )

  File.open(File.join(dir, 'test-result.txt'), 'w') do |f|
    str =
      if result.expected_result?
        if result.successful?
          'expected_success'
        else
          'expected_failure'
        end
      elsif result.successful?
        'unexpected_success'
      else
        'unexpected_failure'
      end

    f.puts(str)
  end

  result
end

#run_worker(_w_i) ⇒ Object (protected) 



103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
 
# File 'lib/test-runner/executor.rb', line 103

def run_worker(_w_i)
  loop do
    return if stop_work?

    begin
      i, t = queue.pop(true)
    rescue ThreadError
      return
    end

    prefix = "[#{i + 1}/#{tests.length}]"
    log("#{prefix} Running test '#{t.path}'")
    result = run_test(t)

    secs = result.elapsed_time.round(2)

    if result.expected_result?
      if result.successful?
        log("#{prefix} Test '#{t.path}' successful in #{secs} seconds")
      else
        log("#{prefix} Test '#{t.path}' failed as expected in #{secs} seconds")
      end
    else # unexpected result
      if result.successful?
        log("#{prefix} Test '#{t.path}' unexpectedly succeeded in #{secs} seconds, see #{result.state_dir}")
      else
        log("#{prefix} Test '#{t.path}' failed after #{secs} seconds, see #{result.state_dir}")
      end

      stop_work! if opts[:stop_on_failure]
    end

    mutex.synchronize { results << result }
  end
end

#start_worker(i) ⇒ Object (protected) 



95
96
97
 
# File 'lib/test-runner/executor.rb', line 95

def start_worker(i)
  workers << Thread.new { run_worker(i) }
end

#state_dir ⇒ Object (protected) 



206
207
208
 
# File 'lib/test-runner/executor.rb', line 206

def state_dir
  opts[:state_dir]
end

#stop_work! ⇒ Object (protected) 



190
191
192
 
# File 'lib/test-runner/executor.rb', line 190

def stop_work!
  @stop_work = true
end

#stop_work? ⇒ Boolean (protected)

Returns:

  • (Boolean) 


194
195
196
 
# File 'lib/test-runner/executor.rb', line 194

def stop_work?
  @stop_work
end

#test_sock_dir ⇒ Object (protected) 



202
203
204
 
# File 'lib/test-runner/executor.rb', line 202

def test_sock_dir
  File.join(state_dir, 'socks')
end

#test_state_dir(test) ⇒ Object (protected) 



198
199
200
 
# File 'lib/test-runner/executor.rb', line 198

def test_state_dir(test)
  File.join(state_dir, "os-test-#{test.name}")
end

#wait_for_workers ⇒ Object (protected) 



99
100
101
 
# File 'lib/test-runner/executor.rb', line 99

def wait_for_workers
  workers.each(&:join)
end