Class: TestRunner::TestEvaluator

Inherits:
Object
  • Object
show all
Includes:
RSpec::Matchers
Defined in:
lib/test-runner/test_evaluator.rb

Defined Under Namespace

Classes: ScriptState

Constant Summary collapse

SCRIPT_STATE_KEY = 
:test_runner_script_state

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(test, scripts, system: NixCli::DEFAULT_SYSTEM, test_config_path: nil, repo_root: nil, **opts) ⇒ TestEvaluator

Returns a new instance of TestEvaluator.

Parameters:

Options Hash (**opts):

  • :default_timeout (Integer)
  • :destructive (Boolean)
  • :recreate_disks (Boolean)
  • :state_dir (String)
  • :sock_dir (String) 


39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
 
# File 'lib/test-runner/test_evaluator.rb', line 39

def initialize(test, scripts, system: NixCli::DEFAULT_SYSTEM, test_config_path: nil, repo_root: nil, **opts)
  scripts.each do |s|
    next if s.test == test

    raise ArgumentError, "script #{s.name} is not of test #{test.path}"
  end

  @test = test
  @scripts = scripts
  @opts = opts
  @config = TestConfig.build(
    test,
    system:,
    test_config_path:,
    repo_root:,
    config_path: File.join(@opts.fetch(:state_dir), 'config.json')
  )
  @machines = {}
  @default_timeout = opts.fetch(:default_timeout)
  @used_container_ids = []
  @used_container_ids_mutex = Mutex.new
  @log_mutex = Mutex.new

  @config['machines'].each do |name, cfg|
    var = :"@#{name}"

    machine_config = OsVm::MachineConfig.from_config(cfg)

    m = machine_class_for(machine_config).new(
      name,
      machine_config,
      opts[:state_dir],
      opts[:sock_dir],
      default_timeout: opts[:default_timeout],
      hash_base: test.path
    )

    m.destroy_disks if opts[:recreate_disks]

    instance_variable_set(var, m)

    define_singleton_method(name) do
      instance_variable_get(var)
    end

    machines[name] = m
  end
end

Instance Attribute Details

#machines ⇒ Hash<String, OsVm::Machine> (readonly)

Returns:

  • (Hash<String, OsVm::Machine>) 


29
30
31
 
# File 'lib/test-runner/test_evaluator.rb', line 29

def machines
  @machines
end

Instance Method Details

#after(type, &block) ⇒ Object

Code block executed after suite, context or example

Parameters:

  • type (:suite, :context, :example) 


199
200
201
202
203
204
205
206
207
208
209
210
 
# File 'lib/test-runner/test_evaluator.rb', line 199

def after(type, &block)
  state = current_script_state

  if type == :suite
    state.after << block
    return
  end

  raise 'Called outside of an example group, use from #describe block' if state.group_stack.empty?

  state.group_stack.last.add_after(type, block)
end

#before(type, &block) ⇒ Object

Code block executed before suite, context or example

Parameters:

  • type (:suite, :context, :example) 


184
185
186
187
188
189
190
191
192
193
194
195
 
# File 'lib/test-runner/test_evaluator.rb', line 184

def before(type, &block)
  state = current_script_state

  if type == :suite
    state.before << block
    return
  end

  raise 'Called outside of an example group, use from #describe block' if state.group_stack.empty?

  state.group_stack.last.add_before(type, block)
end

#breakpoint ⇒ Object

Invoke interactive shell from within a test



143
144
145
 
# File 'lib/test-runner/test_evaluator.rb', line 143

def breakpoint
  binding.pry # rubocop:disable Lint/Debugger
end

#build_test_result(script_results, elapsed_time) ⇒ Object (protected) 



425
426
427
428
429
430
431
432
433
434
435
 
# File 'lib/test-runner/test_evaluator.rb', line 425

def build_test_result(script_results, elapsed_time)
  test_success = script_results.all?(&:expected_result?)

  TestResult.new(
    @test,
    script_results,
    test_success,
    elapsed_time,
    @opts[:state_dir]
  )
end

#call_after_test_run_hook(test_result) ⇒ Object (protected) 



399
400
401
402
403
404
405
406
407
408
409
410
 
# File 'lib/test-runner/test_evaluator.rb', line 399

def call_after_test_run_hook(test_result)
  Hook.call(
    :after_test_run,
    kwargs: {
      test: @test,
      test_result:,
      scripts: @scripts,
      machines:,
      state_dir: @opts[:state_dir]
    }
  )
end

#call_after_test_script_hook(script_result) ⇒ Object (protected) 



412
413
414
415
416
417
418
419
420
421
422
423
 
# File 'lib/test-runner/test_evaluator.rb', line 412

def call_after_test_script_hook(script_result)
  Hook.call(
    :after_test_script_run,
    kwargs: {
      test: @test,
      script: script_result.test_script,
      script_result:,
      machines:,
      state_dir: @opts[:state_dir]
    }
  )
end

#configure_examples {|| ... } ⇒ Object

Configure default settings for example groups

Yield Parameters:



149
150
151
 
# File 'lib/test-runner/test_evaluator.rb', line 149

def configure_examples
  yield(current_script_state.example_config)
end

#current_script_state ⇒ Object (protected) 



600
601
602
 
# File 'lib/test-runner/test_evaluator.rb', line 600

def current_script_state
  Thread.current[SCRIPT_STATE_KEY] || (raise 'No test script is running')
end

#describe(obj, order: nil) ⇒ Object Also known as: context

Create an example group

Example groups can be nested. Groups are evaluated in the configured order, which defaults to random. Groups contain examples which are defined within the yielded block using #it.

Parameters:

  • obj (#to_s)
  • order (nil, :defined, :rand, Random, Integer) (defaults to: nil) —

    order in which examples and subgroups are evaluated



162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
 
# File 'lib/test-runner/test_evaluator.rb', line 162

def describe(obj, order: nil, &)
  state = current_script_state
  grp = ExampleGroup.new(obj, parent: state.group_stack.last, order:, config: state.example_config, &)

  if state.group_stack.any?
    state.group_stack.last.add_group(grp)
  else
    state.example_groups << grp
  end

  state.group_stack << grp

  grp.load

  state.group_stack = state.group_stack[0..-2]
  nil
end

#do_run ⇒ Object (protected) 



612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
 
# File 'lib/test-runner/test_evaluator.rb', line 612

def do_run
  yield

  machines.each_value do |m|
    m.stop if !m.kernel_failed? && m.running? && m.can_execute?
  end
ensure
  begin
    machines.each_value do |m|
      if m.kernel_failed?
        m.kill_after_kernel_failure
      else
        m.kill
      end
      m.destroy if @opts[:destructive]
      m.finalize
      m.cleanup
    end
  ensure
    raise_if_kernel_failed!
  end
end

#example(message, pending: false, skip: false, &block) ⇒ Object Also known as: it

Create a test example

#it must be called from within a #describe block. Examples within a group are evaluated in the group's configured order.

Parameters:

  • message (String)
  • pending (Boolean) (defaults to: false)
  • skip (Boolean) (defaults to: false) 


220
221
222
223
224
225
226
227
 
# File 'lib/test-runner/test_evaluator.rb', line 220

def example(message, pending: false, skip: false, &block)
  state = current_script_state
  raise 'Called outside of an example group, use from #describe block' if state.group_stack.empty?

  grp = state.group_stack.last
  grp.add_example(Example.new(grp, message, pending:, skip: skip || block.nil?, &block))
  nil
end

#get_container_id(base = 'testct') ⇒ String

Generate container id that is unique to the test run

Returns:

  • (String) 


259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
 
# File 'lib/test-runner/test_evaluator.rb', line 259

def get_container_id(base = 'testct')
  @used_container_ids_mutex.synchronize do
    10.times do
      new_id = "#{base}-#{SecureRandom.hex(2)}"

      if @used_container_ids.include?(new_id)
        sleep(0.05)
        next
      end

      @used_container_ids << new_id
      return new_id
    end
  end

  raise 'Unable to generate unique container id'
end

#get_example_count(groups: nil) ⇒ Object (protected) 



583
584
585
586
587
588
589
590
591
592
 
# File 'lib/test-runner/test_evaluator.rb', line 583

def get_example_count(groups: nil)
  cnt = 0

  (groups || current_script_state.example_groups).each do |grp|
    cnt += grp.examples.count(&:evaluate?)
    cnt += get_example_count(groups: grp.groups)
  end

  cnt
end

#interactive ⇒ Object

Run interactive shell



130
131
132
133
134
 
# File 'lib/test-runner/test_evaluator.rb', line 130

def interactive
  do_run do
    binding.pry # rubocop:disable Lint/Debugger
  end
end

#kernel_failed? ⇒ Boolean (protected)

Returns:

  • (Boolean) 


635
636
637
 
# File 'lib/test-runner/test_evaluator.rb', line 635

def kernel_failed?
  machines.each_value.any?(&:kernel_failed?)
end

#log(msg) ⇒ Object (protected) 



594
595
596
597
598
 
# File 'lib/test-runner/test_evaluator.rb', line 594

def log(msg)
  @log_mutex.synchronize do
    warn "[#{Time.now}] #{msg}"
  end
end

#machine_class_for(config) ⇒ Object (protected) 



643
644
645
646
647
648
649
650
651
652
653
654
655
 
# File 'lib/test-runner/test_evaluator.rb', line 643

def machine_class_for(config)
  klass = Hook.call(:machine_class_for, args: [config])
  return klass if klass

  case config.spin
  when 'vpsadminos'
    OsVm::VpsadminosMachine
  when 'nixos'
    OsVm::NixosMachine
  else
    raise ArgumentError, "Unknown machine spin #{config.spin.inspect}"
  end
end

#pending(message = '', skip: false, &block) ⇒ Object

Create a pending example or mark the currently evaluated example as pending

Parameters:

  • message (String) (defaults to: '')
  • skip (Boolean) (defaults to: false) 


234
235
236
237
238
239
240
241
242
 
# File 'lib/test-runner/test_evaluator.rb', line 234

def pending(message = '', skip: false, &block)
  state = current_script_state

  if block || state.current_example.nil?
    it(message, pending: true, skip:, &block)
  else
    state.current_example.send(:set_pending)
  end
end

#raise_if_kernel_failed! ⇒ Object (protected) 



639
640
641
 
# File 'lib/test-runner/test_evaluator.rb', line 639

def raise_if_kernel_failed!
  machines.each_value(&:raise_if_kernel_failed!)
end

#run {|one| ... } ⇒ Hash<String, TestScriptResult>

Run the test scripts

Yield Parameters:

Returns:



95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
 
# File 'lib/test-runner/test_evaluator.rb', line 95

def run
  ret = {}
  script_results_by_name = {}
  test_started_at = Time.now
  event_mutex = Mutex.new
  result_mutex = Mutex.new

  do_run do
    run_script_workers do |script, worker_i|
      OsVm::Machine.with_shell(worker_i) do
        script_result = run_script(script) do |event|
          event_mutex.synchronize { yield(event) } if block_given?
        end

        call_after_test_script_hook(script_result)

        result_mutex.synchronize do
          ret[script.name] = script_result
          script_results_by_name[script.name] = script_result
        end

        event_mutex.synchronize { yield(script_result) } if block_given?
      end
    end
    raise_if_kernel_failed!

    script_results = @scripts.map { |script| script_results_by_name.fetch(script.name) }
    test_result = build_test_result(script_results, Time.now - test_started_at)
    call_after_test_run_hook(test_result)
  end

  ret
end

#run_examples ⇒ Object (protected) 



516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
 
# File 'lib/test-runner/test_evaluator.rb', line 516

def run_examples
  state = current_script_state
  example_count = get_example_count
  i = 1

  log 'Evaluating examples'

  raise_if_kernel_failed!
  state.before.each(&:call)
  raise_if_kernel_failed!

  results = ExampleOrdering.sort_by_order(state.example_groups, state.example_config.default_order).map do |grp|
    grp.evaluate do |type, example_or_result|
      if type == :before
        raise_if_kernel_failed!
        log "[#{i}/#{example_count}] Evaluating '#{example_or_result.full_message}'"
        state.current_example = example_or_result
      else
        result = example_or_result
        raise_if_kernel_failed!

        status =
          if result.success?
            if result.pending?
              'pending'
            elsif result.skip?
              'skipped'
            else
              'succeeded'
            end
          elsif result.pending?
            'unexpectedly succeeded'
          else
            'failed'
          end

        log "[#{i}/#{example_count}] '#{result.title}' #{status} in #{result.elapsed_time.round(2)}s"

        # No block is given in debug mode
        if block_given?
          yield({ type: :example, progress: i, total: example_count, result: result })
        end

        state.current_example = nil

        i += 1
      end
    end
  end.flatten

  state.after.each(&:call)
  raise_if_kernel_failed!

  failed = results.select(&:failure?)
  return if failed.empty?

  warn "\n#{failed.count} examples failed:"

  failed.each do |result|
    warn result.title
    warn result.error
    warn "\n\n"
  end

  raise 'One or more examples failed'
end

#run_script(script) ⇒ Object (protected) 



464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
 
# File 'lib/test-runner/test_evaluator.rb', line 464

def run_script(script)
  success = false
  t1 = Time.now

  begin
    log "Running script #{script.name}"

    script_context = clone
    script_context.send(:test_script, script.name) do |progress|
      next unless progress[:type] == :example

      example_result = progress[:result].to_h(
        script: script.name,
        progress: progress[:progress],
        total: progress[:total]
      )

      yield(example_result) if block_given?
    end
    raise_if_kernel_failed!

    t2 = Time.now
    log "Script #{script.name} finished in #{(t2 - t1).round(2)}s"
  rescue Exception => e # rubocop:disable Lint/RescueException
    t2 = Time.now
    log "Exception occurred while running script #{script.name} in #{(t2 - t1).round(2)}s"
    log e.full_message
  else
    success = true
  end

  TestScriptResult.new(script, success, t2 - t1)
end

#run_script_workers ⇒ Object (protected) 



437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
 
# File 'lib/test-runner/test_evaluator.rb', line 437

def run_script_workers
  worker_count = [@test.test_script_jobs, @scripts.length].min
  return if worker_count <= 0

  queue = Queue.new
  @scripts.each { |script| queue << script }

  workers = worker_count.times.map do |worker_i|
    Thread.new do
      loop do
        break if kernel_failed?

        script =
          begin
            queue.pop(true)
          rescue ThreadError
            break
          end

        yield(script, worker_i)
      end
    end
  end

  workers.each(&:join)
end

#skip(message = '', &block) ⇒ Object

Create a skipped example or mark the currently evaluated example as pending

Parameters:

  • message (String) (defaults to: '') 


246
247
248
249
250
251
252
253
254
255
 
# File 'lib/test-runner/test_evaluator.rb', line 246

def skip(message = '', &block)
  state = current_script_state

  if block || state.current_example.nil?
    it(message, skip: true, &block)
  else
    state.current_example.send(:set_skip)
    throw :skip
  end
end

#start_all ⇒ Object

Start all machines



137
138
139
140
 
# File 'lib/test-runner/test_evaluator.rb', line 137

def start_all
  machines.each_value { |machine| machine.start(wait_for_boot: false) }
  machines.each_value(&:wait_for_boot)
end

#test_config ⇒ Object 



88
89
90
 
# File 'lib/test-runner/test_evaluator.rb', line 88

def test_config
  @config.dig('framework', 'testConfig') || {}
end

#test_script(name) ⇒ Object (protected) 



498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
 
# File 'lib/test-runner/test_evaluator.rb', line 498

def test_script(name, &)
  state = ScriptState.new(
    example_config: ExampleConfiguration.new,
    before: [],
    after: [],
    example_groups: [],
    group_stack: []
  )

  with_script_state(state) do
    binding.eval(@config['testScripts'][name]['script']) # rubocop:disable Security/Eval

    return if state.example_groups.empty?

    run_examples(&)
  end
end

#wait_for_block(name:, timeout: @default_timeout) ⇒ any

Wait for block to succeed

Yield until the code block returns a truthy value or a timeout is reached. RSpec expectation failures inside the block are treated as a falsey return value and retried until the timeout.

Parameters:

  • name (String) —

    block name for error reporting

  • timeout (Integer) (defaults to: @default_timeout)

Returns:

  • (any) —

    yielded value

Raises:

  • (OsVm::TimeoutError) 


287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
 
# File 'lib/test-runner/test_evaluator.rb', line 287

def wait_for_block(name:, timeout: @default_timeout)
  t1 = Time.now
  cur_timeout = timeout

  loop do
    raise_if_kernel_failed!

    ret =
      begin
        yield
      rescue RSpec::Expectations::ExpectationNotMetError
        false
      end
    raise_if_kernel_failed!
    return ret if ret

    cur_timeout = timeout - (Time.now - t1)

    if cur_timeout <= 0
      raise OsVm::TimeoutError, "Timeout occurred while waiting for #{name}"
    end

    sleep(1)
  end
end

#wait_until_block_fails(name:, timeout: @default_timeout) ⇒ any

Wait for a block that eventually fails

Intended for polling code that is expected to start failing with OsVm::CommandFailed. OsVm::CommandSucceeded is treated as a retryable result, because the block may succeed before it starts failing. If no exception is raised, the method returns once the block yields a falsey value. RSpec expectations are not handled here because they cannot be reliably distinguished.

Parameters:

  • name (String) —

    block name for error reporting

  • timeout (Integer) (defaults to: @default_timeout)

Returns:

  • (any) —

    block return value or true when failure is signaled with OsVm::CommandFailed

Raises:

  • (OsVm::TimeoutError) —

    when the block does not fail in time



368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
 
# File 'lib/test-runner/test_evaluator.rb', line 368

def wait_until_block_fails(name:, timeout: @default_timeout)
  t1 = Time.now
  cur_timeout = timeout

  loop do
    raise_if_kernel_failed!

    begin
      ret = yield
    rescue OsVm::CommandFailed
      return true
    rescue OsVm::CommandSucceeded
      ret = true
    end

    raise_if_kernel_failed!

    return ret unless ret

    cur_timeout = timeout - (Time.now - t1)

    if cur_timeout <= 0
      raise OsVm::TimeoutError, "Timeout occurred while waiting for #{name} to fail"
    end

    sleep(1)
  end
end

#wait_until_block_succeeds(name:, timeout: @default_timeout) ⇒ any

Wait for a block that eventually succeeds

Intended for polling code that can temporarily fail with OsVm::CommandFailed (treated as a retry) or signal success via OsVm::CommandSucceeded. RSpec expectation failures are also treated as retryable. The method returns as soon as the block yields a truthy value or raises OsVm::CommandSucceeded.

Parameters:

  • name (String) —

    block name for error reporting

  • timeout (Integer) (defaults to: @default_timeout)

Returns:

  • (any) —

    block return value or true when success is signaled with OsVm::CommandSucceeded

Raises:

  • (OsVm::TimeoutError) —

    when the block does not succeed in time



326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
 
# File 'lib/test-runner/test_evaluator.rb', line 326

def wait_until_block_succeeds(name:, timeout: @default_timeout)
  t1 = Time.now
  cur_timeout = timeout

  loop do
    raise_if_kernel_failed!

    ret =
      begin
        yield
      rescue OsVm::CommandFailed, RSpec::Expectations::ExpectationNotMetError
        false
      rescue OsVm::CommandSucceeded
        true
      end
    raise_if_kernel_failed!
    return ret if ret

    cur_timeout = timeout - (Time.now - t1)

    if cur_timeout <= 0
      raise OsVm::TimeoutError, "Timeout occurred while waiting for #{name} to succeed"
    end

    sleep(1)
  end
end

#with_script_state(state) ⇒ Object (protected) 



604
605
606
607
608
609
610
 
# File 'lib/test-runner/test_evaluator.rb', line 604

def with_script_state(state)
  previous_state = Thread.current[SCRIPT_STATE_KEY]
  Thread.current[SCRIPT_STATE_KEY] = state
  yield
ensure
  Thread.current[SCRIPT_STATE_KEY] = previous_state
end