Class: TestRunner::TestEvaluator
- Inherits:
-
Object
- Object
- TestRunner::TestEvaluator
- Includes:
- RSpec::Matchers
- Defined in:
- lib/test-runner/test_evaluator.rb
Instance Attribute Summary collapse
Instance Method Summary collapse
-
#after(type, &block) ⇒ Object
Code block executed after suite, context or example.
-
#before(type, &block) ⇒ Object
Code block executed before suite, context or example.
-
#breakpoint ⇒ Object
Invoke interactive shell from within a test.
-
#configure_examples {|| ... } ⇒ Object
Configure default settings for example groups.
-
#describe(obj, order: nil) ⇒ Object
(also: #context)
Create an example group.
- #do_run ⇒ Object protected
-
#example(message, pending: false, skip: false, &block) ⇒ Object
(also: #it)
Create a test example.
-
#get_container_id(base = 'testct') ⇒ String
Generate container id that is unique to the test run.
- #get_example_count(groups: nil) ⇒ Object protected
-
#initialize(test, scripts, **opts) ⇒ TestEvaluator
constructor
A new instance of TestEvaluator.
-
#interactive ⇒ Object
Run interactive shell.
- #log(msg) ⇒ Object protected
-
#pending(message = '', skip: false, &block) ⇒ Object
Create a pending example or mark the currently evaluated example as pending.
-
#run {|one| ... } ⇒ Hash<String, Boolean>
Run the test scripts.
- #run_examples ⇒ Object protected
-
#skip(message = '', &block) ⇒ Object
Create a skipped example or mark the currently evaluated example as pending.
-
#start_all ⇒ Object
Start all machines.
- #test_script(name) ⇒ Object protected
-
#wait_for_block(name:, timeout: @default_timeout) ⇒ any
Wait for block to succeed.
Constructor Details
#initialize(test, scripts, **opts) ⇒ TestEvaluator
Returns a new instance of TestEvaluator.
19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 |
# File 'lib/test-runner/test_evaluator.rb', line 19 def initialize(test, scripts, **opts) scripts.each do |s| next if s.test == test raise ArgumentError, "script #{s.name} is not of test #{test.path}" end @test = test @scripts = scripts @config = TestConfig.build(test) @opts = opts @machines = {} @default_timeout = opts.fetch(:default_timeout) @used_container_ids = [] @config['machines'].each do |name, cfg| var = :"@#{name}" m = OsVm::Machine.new( name, OsVm::MachineConfig.new(cfg), opts[:state_dir], opts[:sock_dir], default_timeout: opts[:default_timeout], hash_base: test.path ) instance_variable_set(var, m) define_singleton_method(name) do instance_variable_get(var) end machines[name] = m end end |
Instance Attribute Details
#machines ⇒ Hash<String, OsVm::Machine> (readonly)
10 11 12 |
# File 'lib/test-runner/test_evaluator.rb', line 10 def machines @machines end |
Instance Method Details
#after(type, &block) ⇒ Object
Code block executed after suite, context or example
174 175 176 177 178 179 180 181 182 183 |
# File 'lib/test-runner/test_evaluator.rb', line 174 def after(type, &block) if type == :suite @after << block return end raise 'Called outside of an example group, use from #describe block' if @group_stack.empty? @group_stack.last.add_after(type, block) end |
#before(type, &block) ⇒ Object
Code block executed before suite, context or example
161 162 163 164 165 166 167 168 169 170 |
# File 'lib/test-runner/test_evaluator.rb', line 161 def before(type, &block) if type == :suite @before << block return end raise 'Called outside of an example group, use from #describe block' if @group_stack.empty? @group_stack.last.add_before(type, block) end |
#breakpoint ⇒ Object
Invoke interactive shell from within a test
122 123 124 |
# File 'lib/test-runner/test_evaluator.rb', line 122 def breakpoint binding.pry # rubocop:disable Lint/Debugger end |
#configure_examples {|| ... } ⇒ Object
Configure default settings for example groups
128 129 130 |
# File 'lib/test-runner/test_evaluator.rb', line 128 def configure_examples yield(@example_config) end |
#describe(obj, order: nil) ⇒ Object Also known as: context
Create an example group
Example groups can be nested. Groups are evaluated in random order. Groups contain examples which are defined within the yielded block using #it.
140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 |
# File 'lib/test-runner/test_evaluator.rb', line 140 def describe(obj, order: nil, &) grp = ExampleGroup.new(obj, parent: @group_stack.last, order:, config: @example_config, &) if @group_stack.any? @group_stack.last.add_group(grp) else @example_groups << grp end @group_stack << grp grp.load @group_stack = @group_stack[0..-2] nil end |
#do_run ⇒ Object (protected)
357 358 359 360 361 362 363 364 365 366 367 368 369 370 |
# File 'lib/test-runner/test_evaluator.rb', line 357 def do_run yield machines.each_value do |m| m.stop if m.running? end ensure machines.each_value do |m| m.kill m.destroy if @opts[:destructive] m.finalize m.cleanup end end |
#example(message, pending: false, skip: false, &block) ⇒ Object Also known as: it
193 194 195 196 197 198 199 |
# File 'lib/test-runner/test_evaluator.rb', line 193 def example(, pending: false, skip: false, &block) raise 'Called outside of an example group, use from #describe block' if @group_stack.empty? grp = @group_stack.last grp.add_example(Example.new(grp, , pending:, skip: skip || block.nil?, &block)) nil end |
#get_container_id(base = 'testct') ⇒ String
Generate container id that is unique to the test run
227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 |
# File 'lib/test-runner/test_evaluator.rb', line 227 def get_container_id(base = 'testct') 10.times do new_id = "#{base}-#{SecureRandom.hex(2)}" if @used_container_ids.include?(new_id) sleep(0.05) next end @used_container_ids << new_id return new_id end raise 'Unable to generate unique container id' end |
#get_example_count(groups: nil) ⇒ Object (protected)
342 343 344 345 346 347 348 349 350 351 |
# File 'lib/test-runner/test_evaluator.rb', line 342 def get_example_count(groups: nil) cnt = 0 (groups || @example_groups).each do |grp| cnt += grp.examples.count(&:evaluate?) cnt += get_example_count(groups: grp.groups) end cnt end |
#interactive ⇒ Object
Run interactive shell
110 111 112 113 114 |
# File 'lib/test-runner/test_evaluator.rb', line 110 def interactive do_run do binding.pry # rubocop:disable Lint/Debugger end end |
#log(msg) ⇒ Object (protected)
353 354 355 |
# File 'lib/test-runner/test_evaluator.rb', line 353 def log(msg) warn "[#{Time.now}] #{msg}" end |
#pending(message = '', skip: false, &block) ⇒ Object
Create a pending example or mark the currently evaluated example as pending
206 207 208 209 210 211 212 |
# File 'lib/test-runner/test_evaluator.rb', line 206 def pending( = '', skip: false, &block) if block || @current_example.nil? it(, pending: true, skip:, &block) else @current_example.send(:set_pending) end end |
#run {|one| ... } ⇒ Hash<String, Boolean>
Run the test scripts
57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 |
# File 'lib/test-runner/test_evaluator.rb', line 57 def run ret = {} do_run do @scripts.each do |script| success = false t1 = Time.now begin log "Running script #{script.name}" test_script(script.name) do |progress| if progress[:type] == :example yield({ type: :example, script: script.name, example: progress[:result].example., progress: progress[:progress], total: progress[:total], success: progress[:result].success?, pending: progress[:result].pending?, skip: progress[:result].skip?, elapsed_time: progress[:result].elapsed_time }) end end t2 = Time.now log "Script #{script.name} finished in #{(t2 - t1).round(2)}s" rescue Exception => e # rubocop:disable Lint/RescueException t2 = Time.now log "Exception occurred while running script #{script.name} in #{(t2 - t1).round(2)}s" log e. else success = true end result = { type: :script, script: script.name, success:, elapsed_time: t2 - t1 } ret[script.name] = result yield(result) if block_given? end end ret end |
#run_examples ⇒ Object (protected)
285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 |
# File 'lib/test-runner/test_evaluator.rb', line 285 def run_examples example_count = get_example_count i = 1 log 'Evaluating examples' @before.each(&:call) results = @example_groups.shuffle.map do |grp| grp.evaluate do |type, example_or_result| if type == :before log "[#{i}/#{example_count}] Evaluating '#{example_or_result.}'" @current_example = example_or_result else result = example_or_result status = if result.success? if result.pending? 'pending' elsif result.skip? 'skipped' else 'succeeded' end elsif result.pending? 'unexpectedly succeeded' else 'failed' end log "[#{i}/#{example_count}] '#{result.title}' #{status} in #{result.elapsed_time.round(2)}s" yield({ type: :example, progress: i, total: example_count, result: result }) @current_example = nil i += 1 end end end.flatten @after.each(&:call) failed = results.select(&:failure?) return if failed.empty? warn "\n#{failed.count} examples failed:" failed.each do |result| warn result.title warn result.error warn "\n\n" end raise 'One or more examples failed' end |
#skip(message = '', &block) ⇒ Object
Create a skipped example or mark the currently evaluated example as pending
216 217 218 219 220 221 222 223 |
# File 'lib/test-runner/test_evaluator.rb', line 216 def skip( = '', &block) if block || @current_example.nil? it(, skip: true, &block) else @current_example.send(:set_skip) throw :skip end end |
#start_all ⇒ Object
Start all machines
117 118 119 |
# File 'lib/test-runner/test_evaluator.rb', line 117 def start_all machines.each(&:start) end |
#test_script(name) ⇒ Object (protected)
271 272 273 274 275 276 277 278 279 280 281 282 283 |
# File 'lib/test-runner/test_evaluator.rb', line 271 def test_script(name, &) @example_config = ExampleConfiguration.new @before = [] @after = [] @example_groups = [] @group_stack = [] binding.eval(@config['testScripts'][name]['script']) # rubocop:disable Security/Eval return if @example_groups.empty? run_examples(&) end |
#wait_for_block(name:, timeout: @default_timeout) ⇒ any
Wait for block to succeed
Yield until the code block returns a truthy value or a timeout is reached.
251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 |
# File 'lib/test-runner/test_evaluator.rb', line 251 def wait_for_block(name:, timeout: @default_timeout) t1 = Time.now cur_timeout = timeout loop do ret = yield return ret if ret cur_timeout = timeout - (Time.now - t1) if cur_timeout <= 0 raise OsVm::TimeoutError, "Timeout occurred while waiting for #{name}" end sleep(1) end end |