Class: TestRunner::TestEvaluator

Inherits:
Object
  • Object
show all
Includes:
RSpec::Matchers
Defined in:
lib/test-runner/test_evaluator.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(test, scripts, **opts) ⇒ TestEvaluator

Returns a new instance of TestEvaluator.

Parameters:

Options Hash (**opts):

  • :default_timeout (Integer)
  • :destructive (Boolean)
  • :recreate_disks (Boolean)
  • :state_dir (String)
  • :sock_dir (String) 


28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
 
# File 'lib/test-runner/test_evaluator.rb', line 28

def initialize(test, scripts, **opts)
  scripts.each do |s|
    next if s.test == test

    raise ArgumentError, "script #{s.name} is not of test #{test.path}"
  end

  @test = test
  @scripts = scripts
  @config = TestConfig.build(test)
  @opts = opts
  @machines = {}
  @default_timeout = opts.fetch(:default_timeout)
  @used_container_ids = []

  @config['machines'].each do |name, cfg|
    var = :"@#{name}"

    machine_config = OsVm::MachineConfig.from_config(cfg)

    m = machine_class_for(machine_config).new(
      name,
      machine_config,
      opts[:state_dir],
      opts[:sock_dir],
      default_timeout: opts[:default_timeout],
      hash_base: test.path
    )

    m.destroy_disks if opts[:recreate_disks]

    instance_variable_set(var, m)

    define_singleton_method(name) do
      instance_variable_get(var)
    end

    machines[name] = m
  end
end

Instance Attribute Details

#machines ⇒ Hash<String, OsVm::Machine> (readonly)

Returns:

  • (Hash<String, OsVm::Machine>) 


18
19
20
 
# File 'lib/test-runner/test_evaluator.rb', line 18

def machines
  @machines
end

Instance Method Details

#after(type, &block) ⇒ Object

Code block executed after suite, context or example

Parameters:

  • type (:suite, :context, :example) 


188
189
190
191
192
193
194
195
196
197
 
# File 'lib/test-runner/test_evaluator.rb', line 188

def after(type, &block)
  if type == :suite
    @after << block
    return
  end

  raise 'Called outside of an example group, use from #describe block' if @group_stack.empty?

  @group_stack.last.add_after(type, block)
end

#before(type, &block) ⇒ Object

Code block executed before suite, context or example

Parameters:

  • type (:suite, :context, :example) 


175
176
177
178
179
180
181
182
183
184
 
# File 'lib/test-runner/test_evaluator.rb', line 175

def before(type, &block)
  if type == :suite
    @before << block
    return
  end

  raise 'Called outside of an example group, use from #describe block' if @group_stack.empty?

  @group_stack.last.add_before(type, block)
end

#breakpoint ⇒ Object

Invoke interactive shell from within a test



135
136
137
 
# File 'lib/test-runner/test_evaluator.rb', line 135

def breakpoint
  binding.pry # rubocop:disable Lint/Debugger
end

#build_test_result(script_results, elapsed_time) ⇒ Object (protected) 



395
396
397
398
399
400
401
402
403
404
405
 
# File 'lib/test-runner/test_evaluator.rb', line 395

def build_test_result(script_results, elapsed_time)
  test_success = script_results.all?(&:expected_result?)

  TestResult.new(
    @test,
    script_results,
    test_success,
    elapsed_time,
    @opts[:state_dir]
  )
end

#call_after_test_run_hook(test_result) ⇒ Object (protected) 



369
370
371
372
373
374
375
376
377
378
379
380
 
# File 'lib/test-runner/test_evaluator.rb', line 369

def call_after_test_run_hook(test_result)
  Hook.call(
    :after_test_run,
    kwargs: {
      test: @test,
      test_result:,
      scripts: @scripts,
      machines:,
      state_dir: @opts[:state_dir]
    }
  )
end

#call_after_test_script_hook(script_result) ⇒ Object (protected) 



382
383
384
385
386
387
388
389
390
391
392
393
 
# File 'lib/test-runner/test_evaluator.rb', line 382

def call_after_test_script_hook(script_result)
  Hook.call(
    :after_test_script_run,
    kwargs: {
      test: @test,
      script: script_result.test_script,
      script_result:,
      machines:,
      state_dir: @opts[:state_dir]
    }
  )
end

#configure_examples {|| ... } ⇒ Object

Configure default settings for example groups

Yield Parameters:



141
142
143
 
# File 'lib/test-runner/test_evaluator.rb', line 141

def configure_examples
  yield(@example_config)
end

#describe(obj, order: nil) ⇒ Object Also known as: context

Create an example group

Example groups can be nested. Groups are evaluated in the configured order, which defaults to random. Groups contain examples which are defined within the yielded block using #it.

Parameters:

  • obj (#to_s)
  • order (nil, :defined, :rand, Random, Integer) (defaults to: nil) —

    order in which examples and subgroups are evaluated



154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
 
# File 'lib/test-runner/test_evaluator.rb', line 154

def describe(obj, order: nil, &)
  grp = ExampleGroup.new(obj, parent: @group_stack.last, order:, config: @example_config, &)

  if @group_stack.any?
    @group_stack.last.add_group(grp)
  else
    @example_groups << grp
  end

  @group_stack << grp

  grp.load

  @group_stack = @group_stack[0..-2]
  nil
end

#do_run ⇒ Object (protected) 



497
498
499
500
501
502
503
504
505
506
507
508
509
510
 
# File 'lib/test-runner/test_evaluator.rb', line 497

def do_run
  yield

  machines.each_value do |m|
    m.stop if m.running? && m.can_execute?
  end
ensure
  machines.each_value do |m|
    m.kill
    m.destroy if @opts[:destructive]
    m.finalize
    m.cleanup
  end
end

#example(message, pending: false, skip: false, &block) ⇒ Object Also known as: it

Create a test example

#it must be called from within a #describe block. Examples within a group are evaluated in the group’s configured order.

Parameters:

  • message (String)
  • pending (Boolean) (defaults to: false)
  • skip (Boolean) (defaults to: false) 


207
208
209
210
211
212
213
 
# File 'lib/test-runner/test_evaluator.rb', line 207

def example(message, pending: false, skip: false, &block)
  raise 'Called outside of an example group, use from #describe block' if @group_stack.empty?

  grp = @group_stack.last
  grp.add_example(Example.new(grp, message, pending:, skip: skip || block.nil?, &block))
  nil
end

#get_container_id(base = 'testct') ⇒ String

Generate container id that is unique to the test run

Returns:

  • (String) 


241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
 
# File 'lib/test-runner/test_evaluator.rb', line 241

def get_container_id(base = 'testct')
  10.times do
    new_id = "#{base}-#{SecureRandom.hex(2)}"

    if @used_container_ids.include?(new_id)
      sleep(0.05)
      next
    end

    @used_container_ids << new_id
    return new_id
  end

  raise 'Unable to generate unique container id'
end

#get_example_count(groups: nil) ⇒ Object (protected) 



482
483
484
485
486
487
488
489
490
491
 
# File 'lib/test-runner/test_evaluator.rb', line 482

def get_example_count(groups: nil)
  cnt = 0

  (groups || @example_groups).each do |grp|
    cnt += grp.examples.count(&:evaluate?)
    cnt += get_example_count(groups: grp.groups)
  end

  cnt
end

#interactive ⇒ Object

Run interactive shell



123
124
125
126
127
 
# File 'lib/test-runner/test_evaluator.rb', line 123

def interactive
  do_run do
    binding.pry # rubocop:disable Lint/Debugger
  end
end

#log(msg) ⇒ Object (protected) 



493
494
495
 
# File 'lib/test-runner/test_evaluator.rb', line 493

def log(msg)
  warn "[#{Time.now}] #{msg}"
end

#machine_class_for(config) ⇒ Object (protected) 



512
513
514
515
516
517
518
519
520
521
522
523
524
 
# File 'lib/test-runner/test_evaluator.rb', line 512

def machine_class_for(config)
  klass = Hook.call(:machine_class_for, args: [config])
  return klass if klass

  case config.spin
  when 'vpsadminos'
    OsVm::VpsadminosMachine
  when 'nixos'
    OsVm::NixosMachine
  else
    raise ArgumentError, "Unknown machine spin #{config.spin.inspect}"
  end
end

#pending(message = '', skip: false, &block) ⇒ Object

Create a pending example or mark the currently evaluated example as pending

Parameters:

  • message (String) (defaults to: '')
  • skip (Boolean) (defaults to: false) 


220
221
222
223
224
225
226
 
# File 'lib/test-runner/test_evaluator.rb', line 220

def pending(message = '', skip: false, &block)
  if block || @current_example.nil?
    it(message, pending: true, skip:, &block)
  else
    @current_example.send(:set_pending)
  end
end

#run {|one| ... } ⇒ Hash<String, TestScriptResult>

Run the test scripts

Yield Parameters:

Returns:



72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
 
# File 'lib/test-runner/test_evaluator.rb', line 72

def run
  ret = {}
  script_results = []
  test_started_at = Time.now

  do_run do
    @scripts.each do |script|
      success = false
      t1 = Time.now

      begin
        log "Running script #{script.name}"

        test_script(script.name) do |progress|
          if progress[:type] == :example
            example_result = progress[:result].to_h(
              script: script.name,
              progress: progress[:progress],
              total: progress[:total]
            )

            yield(example_result) if block_given?
          end
        end

        t2 = Time.now
        log "Script #{script.name} finished in #{(t2 - t1).round(2)}s"
      rescue Exception => e # rubocop:disable Lint/RescueException
        t2 = Time.now
        log "Exception occurred while running script #{script.name} in #{(t2 - t1).round(2)}s"
        log e.full_message
      else
        success = true
      end

      script_result = TestScriptResult.new(script, success, t2 - t1)

      ret[script.name] = script_result
      script_results << script_result
      call_after_test_script_hook(script_result)
      yield(script_result) if block_given?
    end

    test_result = build_test_result(script_results, Time.now - test_started_at)
    call_after_test_run_hook(test_result)
  end

  ret
end

#run_examples ⇒ Object (protected) 



421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
 
# File 'lib/test-runner/test_evaluator.rb', line 421

def run_examples
  example_count = get_example_count
  i = 1

  log 'Evaluating examples'

  @before.each(&:call)

  results = ExampleOrdering.sort_by_order(@example_groups, @example_config.default_order).map do |grp|
    grp.evaluate do |type, example_or_result|
      if type == :before
        log "[#{i}/#{example_count}] Evaluating '#{example_or_result.full_message}'"
        @current_example = example_or_result
      else
        result = example_or_result

        status =
          if result.success?
            if result.pending?
              'pending'
            elsif result.skip?
              'skipped'
            else
              'succeeded'
            end
          elsif result.pending?
            'unexpectedly succeeded'
          else
            'failed'
          end

        log "[#{i}/#{example_count}] '#{result.title}' #{status} in #{result.elapsed_time.round(2)}s"

        # No block is given in debug mode
        if block_given?
          yield({ type: :example, progress: i, total: example_count, result: result })
        end

        @current_example = nil

        i += 1
      end
    end
  end.flatten

  @after.each(&:call)

  failed = results.select(&:failure?)
  return if failed.empty?

  warn "\n#{failed.count} examples failed:"

  failed.each do |result|
    warn result.title
    warn result.error
    warn "\n\n"
  end

  raise 'One or more examples failed'
end

#skip(message = '', &block) ⇒ Object

Create a skipped example or mark the currently evaluated example as pending

Parameters:

  • message (String) (defaults to: '') 


230
231
232
233
234
235
236
237
 
# File 'lib/test-runner/test_evaluator.rb', line 230

def skip(message = '', &block)
  if block || @current_example.nil?
    it(message, skip: true, &block)
  else
    @current_example.send(:set_skip)
    throw :skip
  end
end

#start_all ⇒ Object

Start all machines



130
131
132
 
# File 'lib/test-runner/test_evaluator.rb', line 130

def start_all
  machines.each(&:start)
end

#test_script(name) ⇒ Object (protected) 



407
408
409
410
411
412
413
414
415
416
417
418
419
 
# File 'lib/test-runner/test_evaluator.rb', line 407

def test_script(name, &)
  @example_config = ExampleConfiguration.new
  @before = []
  @after = []
  @example_groups = []
  @group_stack = []

  binding.eval(@config['testScripts'][name]['script']) # rubocop:disable Security/Eval

  return if @example_groups.empty?

  run_examples(&)
end

#wait_for_block(name:, timeout: @default_timeout) ⇒ any

Wait for block to succeed

Yield until the code block returns a truthy value or a timeout is reached. RSpec expectation failures inside the block are treated as a falsey return value and retried until the timeout.

Parameters:

  • name (String) —

    block name for error reporting

  • timeout (Integer) (defaults to: @default_timeout)

Returns:

  • (any) —

    yielded value

Raises:

  • (OsVm::TimeoutError) 


267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
 
# File 'lib/test-runner/test_evaluator.rb', line 267

def wait_for_block(name:, timeout: @default_timeout)
  t1 = Time.now
  cur_timeout = timeout

  loop do
    ret =
      begin
        yield
      rescue RSpec::Expectations::ExpectationNotMetError
        false
      end
    return ret if ret

    cur_timeout = timeout - (Time.now - t1)

    if cur_timeout <= 0
      raise OsVm::TimeoutError, "Timeout occurred while waiting for #{name}"
    end

    sleep(1)
  end
end

#wait_until_block_fails(name:, timeout: @default_timeout) ⇒ any

Wait for a block that eventually fails

Intended for polling code that is expected to start failing with OsVm::CommandFailed. OsVm::CommandSucceeded is treated as a retryable result, because the block may succeed before it starts failing. If no exception is raised, the method returns once the block yields a falsey value. RSpec expectations are not handled here because they cannot be reliably distinguished.

Parameters:

  • name (String) —

    block name for error reporting

  • timeout (Integer) (defaults to: @default_timeout)

Returns:

  • (any) —

    block return value or true when failure is signaled with OsVm::CommandFailed

Raises:

  • (OsVm::TimeoutError) —

    when the block does not fail in time



342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
 
# File 'lib/test-runner/test_evaluator.rb', line 342

def wait_until_block_fails(name:, timeout: @default_timeout)
  t1 = Time.now
  cur_timeout = timeout

  loop do
    begin
      ret = yield
    rescue OsVm::CommandFailed
      return true
    rescue OsVm::CommandSucceeded
      ret = true
    end

    return ret unless ret

    cur_timeout = timeout - (Time.now - t1)

    if cur_timeout <= 0
      raise OsVm::TimeoutError, "Timeout occurred while waiting for #{name} to fail"
    end

    sleep(1)
  end
end

#wait_until_block_succeeds(name:, timeout: @default_timeout) ⇒ any

Wait for a block that eventually succeeds

Intended for polling code that can temporarily fail with OsVm::CommandFailed (treated as a retry) or signal success via OsVm::CommandSucceeded. RSpec expectation failures are also treated as retryable. The method returns as soon as the block yields a truthy value or raises OsVm::CommandSucceeded.

Parameters:

  • name (String) —

    block name for error reporting

  • timeout (Integer) (defaults to: @default_timeout)

Returns:

  • (any) —

    block return value or true when success is signaled with OsVm::CommandSucceeded

Raises:

  • (OsVm::TimeoutError) —

    when the block does not succeed in time



303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
 
# File 'lib/test-runner/test_evaluator.rb', line 303

def wait_until_block_succeeds(name:, timeout: @default_timeout)
  t1 = Time.now
  cur_timeout = timeout

  loop do
    ret =
      begin
        yield
      rescue OsVm::CommandFailed, RSpec::Expectations::ExpectationNotMetError
        false
      rescue OsVm::CommandSucceeded
        true
      end
    return ret if ret

    cur_timeout = timeout - (Time.now - t1)

    if cur_timeout <= 0
      raise OsVm::TimeoutError, "Timeout occurred while waiting for #{name} to succeed"
    end

    sleep(1)
  end
end