Class: OsVm::Machine

Inherits:
Object
  • Object
show all
Defined in:
lib/osvm/machine.rb

Direct Known Subclasses

NixosMachine, VpsadminosMachine

Defined Under Namespace

Classes: QemuReaper

Constant Summary collapse

SHELL_INDEX_KEY = 
:osvm_machine_shell_index
KERNEL_FAILURE_PATTERN = 
Regexp.union(
  /BUG: unable to handle/,
  /BUG: kernel NULL pointer dereference/,
  /\bBUG: Bad rss-counter state\b/,
  /kernel BUG at/,
  /Oops:/,
  /general protection fault/,
  /Kernel panic - not syncing:/
).freeze
SOCKET_HASH_LENGTH = 
16
SOCKET_HASH_IDENTITIES =

Process-wide collision registry; all access is serialized by the mutex.

{}
SOCKET_HASH_MUTEX =

rubocop:disable Style/MutableConstant

Mutex.new

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(name, config, tmpdir, sockdir, default_timeout: 900, hash_base: '', interactive_console: false) ⇒ Machine

Returns a new instance of Machine.

Parameters:

  • name (String)
  • config (MachineConfig)
  • tmpdir (String)
  • sockdir (String)
  • default_timeout (Integer) (defaults to: 900)
  • hash_base (String) (defaults to: '')
  • interactive_console (Boolean) (defaults to: false) 


167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
 
# File 'lib/osvm/machine.rb', line 167

def initialize(name, config, tmpdir, sockdir, default_timeout: 900, hash_base: '', interactive_console: false)
  @name = name
  @config = config
  @tmpdir = tmpdir
  @sockdir = sockdir
  @default_timeout = default_timeout || 900
  @hash_base = hash_base
  @interactive_console = interactive_console
  @start_kernel_params = []
  @running = false
  @shared_dir = SharedDir.new(self)
  @shared_filesystems = {
    shared_dir.fs_name => shared_dir.host_path
  }.merge(config.shared_filesystems)
  @virtiofsd_pids = []
  @mutex = Mutex.new
  @console_cv = ConditionVariable.new
  @start_mutex = Mutex.new
  @shared_dir_mutex = Mutex.new
  @shared_dir_mounted = false
  @kernel_failure = nil
  @kernel_failure_detected_at = nil
  @allowed_kernel_failure_patterns = []
  @console_output = ''
  @console_scan_buffer = ''
  @console_last_output_at = nil
  @console_eof = false

  FileUtils.mkdir_p(tmpdir)
  FileUtils.mkdir_p(sockdir)
  @log = MachineLog.new(File.join(tmpdir, "#{name}-log.log"))
  named_shells = {}
  worker_shell_count = config.test_shells - config.shell_names.length
  @shell_instances = Array.new(config.test_shells) do |i|
    shell_name = i >= worker_shell_count ? config.shell_names[i - worker_shell_count] : nil
    shell = Shell.new(
      self, i, shell_socket_path(i), shell_log_path(i),
      default_timeout:, name: shell_name, transport: config.test_shell_transport
    )
    named_shells[shell_name] = shell if shell_name
    shell
  end
  @shell_collection = ShellCollection.new(self, named_shells)
end

Instance Attribute Details

#config ⇒ Object (readonly, protected)

Returns the value of attribute config.



741
742
743
 
# File 'lib/osvm/machine.rb', line 741

def config
  @config
end

#console_thread ⇒ Object (readonly, protected)

Returns the value of attribute console_thread.



741
742
743
 
# File 'lib/osvm/machine.rb', line 741

def console_thread
  @console_thread
end

#hash_base ⇒ Object (readonly, protected)

Returns the value of attribute hash_base.



741
742
743
 
# File 'lib/osvm/machine.rb', line 741

def hash_base
  @hash_base
end

#log ⇒ Object (readonly, protected)

Returns the value of attribute log.



741
742
743
 
# File 'lib/osvm/machine.rb', line 741

def log
  @log
end

#name ⇒ String (readonly)

Returns:

  • (String) 


154
155
156
 
# File 'lib/osvm/machine.rb', line 154

def name
  @name
end

#qemu_pid ⇒ Object (readonly, protected)

Returns the value of attribute qemu_pid.



741
742
743
 
# File 'lib/osvm/machine.rb', line 741

def qemu_pid
  @qemu_pid
end

#qemu_read ⇒ Object (readonly, protected)

Returns the value of attribute qemu_read.



741
742
743
 
# File 'lib/osvm/machine.rb', line 741

def qemu_read
  @qemu_read
end

#qemu_reaper ⇒ Object (readonly, protected)

Returns the value of attribute qemu_reaper.



741
742
743
 
# File 'lib/osvm/machine.rb', line 741

def qemu_reaper
  @qemu_reaper
end

#shared_dir ⇒ Object (readonly, protected)

Returns the value of attribute shared_dir.



741
742
743
 
# File 'lib/osvm/machine.rb', line 741

def shared_dir
  @shared_dir
end

#shared_filesystems ⇒ Object (readonly, protected)

Returns the value of attribute shared_filesystems.



741
742
743
 
# File 'lib/osvm/machine.rb', line 741

def shared_filesystems
  @shared_filesystems
end

#shell_instances ⇒ Object (readonly, protected)

Returns the value of attribute shell_instances.



741
742
743
 
# File 'lib/osvm/machine.rb', line 741

def shell_instances
  @shell_instances
end

#sockdir ⇒ Object (readonly, protected)

Returns the value of attribute sockdir.



741
742
743
 
# File 'lib/osvm/machine.rb', line 741

def sockdir
  @sockdir
end

#start_kernel_params ⇒ Array<String> (readonly)

Kernel parameters passed to #start

Returns:

  • (Array<String>) 


158
159
160
 
# File 'lib/osvm/machine.rb', line 158

def start_kernel_params
  @start_kernel_params
end

#tmpdir ⇒ Object (readonly, protected)

Returns the value of attribute tmpdir.



741
742
743
 
# File 'lib/osvm/machine.rb', line 741

def tmpdir
  @tmpdir
end

#virtiofsd_pids ⇒ Object (readonly, protected)

Returns the value of attribute virtiofsd_pids.



741
742
743
 
# File 'lib/osvm/machine.rb', line 741

def virtiofsd_pids
  @virtiofsd_pids
end

Class Method Details

.with_shell(index) ⇒ Object 



145
146
147
148
149
150
151
 
# File 'lib/osvm/machine.rb', line 145

def self.with_shell(index)
  original_index = Thread.current[SHELL_INDEX_KEY]
  Thread.current[SHELL_INDEX_KEY] = index
  yield
ensure
  Thread.current[SHELL_INDEX_KEY] = original_index
end

Instance Method Details

#all_fail(*cmds, shell: nil) ⇒ Array<Array<[Integer, String]>>

Execute all commands and check that they all fail

Parameters:

  • cmds (String)

Returns:

  • (Array<Array<[Integer, String]>>) 


522
523
524
 
# File 'lib/osvm/machine.rb', line 522

def all_fail(*cmds, shell: nil)
  command_shell(shell).all_fail(*cmds)
end

#all_succeed(*cmds, shell: nil) ⇒ Array<Array<[Integer, String]>>

Execute all commands and check that they all succeed

Parameters:

  • cmds (String)

Returns:

  • (Array<Array<[Integer, String]>>) 


515
516
517
 
# File 'lib/osvm/machine.rb', line 515

def all_succeed(*cmds, shell: nil)
  command_shell(shell).all_succeed(*cmds)
end

#allow_kernel_failure(pattern) ⇒ Object

Allow matching fatal kernel console output only while the block runs. Other kernel failure signatures remain fatal.

Parameters:

  • pattern (Regexp) 


599
600
601
602
603
604
605
606
607
608
609
610
611
 
# File 'lib/osvm/machine.rb', line 599

def allow_kernel_failure(pattern)
  raise ArgumentError, 'pattern must be a Regexp' unless pattern.is_a?(Regexp)

  @mutex.synchronize { @allowed_kernel_failure_patterns << pattern }
  yield
ensure
  if pattern.is_a?(Regexp)
    @mutex.synchronize do
      i = @allowed_kernel_failure_patterns.rindex(pattern)
      @allowed_kernel_failure_patterns.delete_at(i) if i
    end
  end
end

#append_console_output(data, flush: false) ⇒ Object (protected) 



1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
 
# File 'lib/osvm/machine.rb', line 1060

def append_console_output(data, flush: false)
  events = @mutex.synchronize do
    @console_output << data
    @console_scan_buffer << data
    @console_last_output_at = Process.clock_gettime(Process::CLOCK_MONOTONIC) unless data.empty?
    @console_eof = true if flush
    @console_cv.broadcast if flush || !data.empty?

    lines = @console_scan_buffer.split("\n", -1)
    @console_scan_buffer = lines.pop || ''

    if flush && !@console_scan_buffer.empty?
      lines << @console_scan_buffer
      @console_scan_buffer = ''
    end

    lines.filter_map do |raw_line|
      line = raw_line.delete_suffix("\r")
      next unless KERNEL_FAILURE_PATTERN.match?(line)

      expected = @allowed_kernel_failure_patterns.any? { |pattern| pattern.match?(line) }

      unless expected || @kernel_failure
        @kernel_failure = { line: }
        @kernel_failure_detected_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
      end

      { line:, expected: }
    end
  end

  events.each { |event| log.kernel_failure(event.fetch(:line), expected: event.fetch(:expected)) }
end

#base_kernel_params(kernel_params) ⇒ Object (protected) 



790
791
792
793
794
795
 
# File 'lib/osvm/machine.rb', line 790

def base_kernel_params(kernel_params)
  [
    'console=ttyS0',
    "init=#{config.toplevel}/init"
  ] + config.kernel_params + kernel_params
end

#booted? ⇒ Boolean

Returns:

  • (Boolean) 


446
447
448
 
# File 'lib/osvm/machine.rb', line 446

def booted?
  current_shell.up?
end

#can_execute? ⇒ Boolean

Returns:

  • (Boolean) 


451
452
453
 
# File 'lib/osvm/machine.rb', line 451

def can_execute?
  shell_instances.any?(&:up?)
end

#cleanup ⇒ Machine

Cleanup machine state

Returns:



428
429
430
431
432
433
434
435
436
437
438
 
# File 'lib/osvm/machine.rb', line 428

def cleanup
  shell_instances.each(&:cleanup)

  shared_filesystems.each_key do |fs_name|
    File.unlink(virtiofs_socket_path(fs_name))
  rescue Errno::ENOENT
    # ignore
  end

  self
end

#cleanup_runtime_resources ⇒ Object (protected) 



970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
 
# File 'lib/osvm/machine.rb', line 970

def cleanup_runtime_resources
  cleanup_error = nil
  cleanup_steps = [
    lambda do
      @qemu_read&.close
    rescue IOError
      # ignore
    ensure
      @qemu_read = nil
    end,
    lambda do
      if @console_thread
        console_thread.join
        @console_thread = nil
      end
    end,
    -> { shell_instances.each(&:close) },
    -> { stop_virtiofs },
    -> { cleanup }
  ]

  cleanup_steps.each do |step|
    step.call
  rescue StandardError => e
    cleanup_error ||= e
  end

  raise cleanup_error if cleanup_error
end

#command_shell(name) ⇒ Object (protected) 



1196
1197
1198
 
# File 'lib/osvm/machine.rb', line 1196

def command_shell(name)
  name.nil? ? current_shell : shells.fetch(name)
end

#console_log_path ⇒ Object (protected) 



1129
1130
1131
 
# File 'lib/osvm/machine.rb', line 1129

def console_log_path
  File.join(tmpdir, "#{name}-console.log")
end

#console_output ⇒ String

Return text captured from the machine's console so far

Returns:

  • (String) 


658
659
660
 
# File 'lib/osvm/machine.rb', line 658

def console_output
  @mutex.synchronize { (@console_output || '').dup }
end

#current_shell ⇒ Object (protected) 



1192
1193
1194
 
# File 'lib/osvm/machine.rb', line 1192

def current_shell
  shell_for(current_shell_index)
end

#current_shell_index ⇒ Object (protected) 



1186
1187
1188
1189
1190
 
# File 'lib/osvm/machine.rb', line 1186

def current_shell_index
  index = Thread.current[SHELL_INDEX_KEY] || 0
  validate_shell_index(index)
  index
end

#custom_qemu_numa_memory_backend? ⇒ Boolean (protected)

Returns:

  • (Boolean) 


855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
 
# File 'lib/osvm/machine.rb', line 855

def custom_qemu_numa_memory_backend?
  opts = config.extra_qemu_options
  has_memory_backend = false
  has_numa_memdev = false

  opts.each_cons(2) do |arg, value|
    if arg == '-object' && value.start_with?('memory-backend-')
      has_memory_backend = true
    elsif arg == '-numa' && value.start_with?('node,') && value.include?('memdev=')
      has_numa_memdev = true
    end
  end

  has_memory_backend && has_numa_memdev
end

#destroy ⇒ Machine

Destroy the machine

Returns:



403
404
405
406
407
408
 
# File 'lib/osvm/machine.rb', line 403

def destroy
  log.destroy
  shared_dir.destroy
  destroy_disks
  self
end

#destroy_disks ⇒ Machine

Destroy file-backed disks

Disks are destroyed automatically or when #destroy is called. #destroy_disks can be used to reset storage between machine runs.

Returns:



416
417
418
419
420
421
422
423
424
 
# File 'lib/osvm/machine.rb', line 416

def destroy_disks
  config.all_disks.each do |disk|
    next unless disk.managed?

    FileUtils.rm_f(disk_path(disk.device))
  end

  self
end

#disk_path(path) ⇒ Object (protected) 



1133
1134
1135
1136
1137
1138
1139
1140
1141
 
# File 'lib/osvm/machine.rb', line 1133

def disk_path(path)
  resolved = path.gsub('{machine}', name)

  if resolved.start_with?('/')
    resolved
  else
    File.join(tmpdir, resolved)
  end
end

#execute(cmd, timeout: @default_timeout, shell: nil, deadline: nil, timeout_message: nil) ⇒ Array<Integer, String>

Execute a command

Parameters:

  • cmd (String)
  • timeout (Integer) (defaults to: @default_timeout)

Returns:

  • (Array<Integer, String>) —

    exit status and output

Raises:



469
470
471
472
473
474
 
# File 'lib/osvm/machine.rb', line 469

def execute(cmd, timeout: @default_timeout, shell: nil, deadline: nil, timeout_message: nil)
  options = { timeout: }
  options[:deadline] = deadline if deadline
  options[:timeout_message] = timeout_message if timeout_message
  command_shell(shell).execute(cmd, **options)
end

#expanded_extra_qemu_options ⇒ Object (protected) 



871
872
873
874
875
 
# File 'lib/osvm/machine.rb', line 871

def expanded_extra_qemu_options
  config.extra_qemu_options.map do |arg|
    arg.gsub('@OSVM_TMPDIR@', tmpdir)
  end
end

#fails(cmd, timeout: @default_timeout, shell: nil) ⇒ Array<Integer, String>

Execute command and check that it fails

Parameters:

  • cmd (String)
  • timeout (Integer) (defaults to: @default_timeout)

Returns:

  • (Array<Integer, String>) 


498
499
500
 
# File 'lib/osvm/machine.rb', line 498

def fails(cmd, timeout: @default_timeout, shell: nil)
  command_shell(shell).fails(cmd, timeout:)
end

#fails_with_retries(cmd, attempts:, retry_delay: 1, timeout: @default_timeout, shell: nil) ⇒ Array<Integer, String>

Execute a command repeatedly until it fails or all attempts are used

Parameters:

  • cmd (String)
  • attempts (Integer)
  • retry_delay (Numeric) (defaults to: 1)
  • timeout (Integer) (defaults to: @default_timeout) —

    timeout for each attempt

Returns:

  • (Array<Integer, String>) 


508
509
510
 
# File 'lib/osvm/machine.rb', line 508

def fails_with_retries(cmd, attempts:, retry_delay: 1, timeout: @default_timeout, shell: nil)
  command_shell(shell).fails_with_retries(cmd, attempts:, retry_delay:, timeout:)
end

#finalize ⇒ Object 



212
213
214
215
 
# File 'lib/osvm/machine.rb', line 212

def finalize
  log.close
  shell_instances.each(&:finalize)
end

#inspect ⇒ Object 



735
736
737
 
# File 'lib/osvm/machine.rb', line 735

def inspect
  "#<#{self.class.name}:#{object_id} name=#{name}>"
end

#join(timeout: @default_timeout) ⇒ Object

Block until the machine stops



327
328
329
330
331
332
 
# File 'lib/osvm/machine.rb', line 327

def join(timeout: @default_timeout)
  reaper = qemu_reaper
  wait_for_reaper(reaper, timeout:) if reaper
  raise_if_kernel_failed!
  nil
end

#kernel_failed? ⇒ Boolean

Returns:

  • (Boolean) 


613
614
615
 
# File 'lib/osvm/machine.rb', line 613

def kernel_failed?
  @mutex.synchronize { !@kernel_failure.nil? }
end

#kill(signal: 'TERM') ⇒ Machine

Kill the machine

Parameters:

  • signal ('TERM', 'KILL') (defaults to: 'TERM')

Returns:



367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
 
# File 'lib/osvm/machine.rb', line 367

def kill(signal: 'TERM')
  reaper = qemu_reaper
  unless reaper && running?
    log.kill('NONE')
    return self
  end

  log.kill(signal)

  begin
    reaper.signal(signal)
  rescue Errno::ESRCH
    warn "Unable to kill machine #{name} using SIG#{signal}"
  end

  if signal == 'KILL'
    reaper.join
    return self
  elsif reaper.join(60)
    return self
  end

  log.kill('KILL')

  begin
    reaper.signal('KILL')
  rescue Errno::ESRCH
    warn "Unable to kill machine #{name} using SIGKILL"
  end

  reaper.join
  self
end

#kill_after_kernel_failure(drain_timeout: 600, quiet_timeout: 30) ⇒ Machine

A panic can pause before dumping per-CPU traces, then stream for minutes over the serial console. Keep collecting without delaying fatal detection or allowing a continuously noisy guest to prevent teardown indefinitely.

Parameters:

  • drain_timeout (Numeric) (defaults to: 600) —

    maximum seconds from failure detection

  • quiet_timeout (Numeric) (defaults to: 30) —

    seconds without new console bytes

Returns:



634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
 
# File 'lib/osvm/machine.rb', line 634

def kill_after_kernel_failure(drain_timeout: 600, quiet_timeout: 30)
  @mutex.synchronize do
    detected_at = @kernel_failure_detected_at

    if detected_at
      deadline = detected_at + drain_timeout

      loop do
        break if @console_eof

        quiet_deadline = (@console_last_output_at || detected_at) + quiet_timeout
        remaining = [deadline, quiet_deadline].min - Process.clock_gettime(Process::CLOCK_MONOTONIC)
        break unless remaining > 0

        @console_cv.wait(@mutex, remaining)
      end
    end
  end

  kill(signal: 'KILL')
end

#mark_runtime_stopped ⇒ Object (protected) 



1000
1001
1002
1003
1004
1005
 
# File 'lib/osvm/machine.rb', line 1000

def mark_runtime_stopped
  @qemu_pid = nil
  @qemu_reaper = nil
  @running = false
  @stopped_at = Time.now
end

#mkdir(path) ⇒ Machine

Create a directory inside the machine

Parameters:

  • path (String) —

    path within the machine

Returns:



698
699
700
701
 
# File 'lib/osvm/machine.rb', line 698

def mkdir(path)
  succeeds("mkdir \"#{path}\"")
  self
end

#mkdir_p(path) ⇒ Machine

Create a directory inside the machine

Parameters:

  • path (String) —

    path within the machine

Returns:



706
707
708
709
 
# File 'lib/osvm/machine.rb', line 706

def mkdir_p(path)
  succeeds("mkdir -p \"#{path}\"")
  self
end

#monotonic_deadline(timeout) ⇒ Object (protected) 



745
746
747
 
# File 'lib/osvm/machine.rb', line 745

def monotonic_deadline(timeout)
  monotonic_now + timeout
end

#monotonic_now ⇒ Object (protected) 



756
757
758
 
# File 'lib/osvm/machine.rb', line 756

def monotonic_now
  Process.clock_gettime(Process::CLOCK_MONOTONIC)
end

#mount_shared_dir_once ⇒ Object (protected) 



1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
 
# File 'lib/osvm/machine.rb', line 1211

def mount_shared_dir_once
  return if @shared_dir_mounted

  @shared_dir_mutex.synchronize do
    return if @shared_dir_mounted

    shared_dir.mount
    @shared_dir_mounted = true
  end
end

#poweroff_command ⇒ Object (protected) 



786
787
788
 
# File 'lib/osvm/machine.rb', line 786

def poweroff_command
  'poweroff -f'
end

#prepare_disks ⇒ Object (protected) 



1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
 
# File 'lib/osvm/machine.rb', line 1094

def prepare_disks
  config.all_disks.each do |disk|
    next unless disk.managed?

    path = disk_path(disk.device)
    next if disk.preserve && File.exist?(path)

    # An interrupted preparation must not replace a usable disk or leave
    # a partial image that a later start would reuse.
    Tempfile.create(['osvm-disk-', '.img'], File.dirname(path)) do |image|
      if disk.image
        FileUtils.cp(disk.image, image.path)
      else
        system('truncate', '-s', disk.size, '--', image.path, exception: true)
      end
      File.chmod(0o644, image.path)
      File.rename(image.path, path)
    end
  end
end

#pull_file(src, preserve: false) ⇒ String

Pull file from the machine to the host

Parameters:

  • src (String) —

    file within the machine

Returns:

  • (String) —

    path to the file on the host



726
727
728
 
# File 'lib/osvm/machine.rb', line 726

def pull_file(src, preserve: false)
  shared_dir.pull_file(src, preserve:)
end

#push_file(src, dst, preserve: false, mkpath: false) ⇒ Machine

Push file from the host to the machine

Parameters:

  • src (String) —

    file on the host

  • dst (String) —

    file within the machine

  • preserve (Boolean) (defaults to: false)
  • mkpath (Boolean) (defaults to: false)

Returns:



717
718
719
720
721
 
# File 'lib/osvm/machine.rb', line 717

def push_file(src, dst, preserve: false, mkpath: false)
  mkdir_p(File.dirname(dst)) if mkpath
  shared_dir.push_file(src, dst, preserve:)
  self
end

#qemu_boot_media_options ⇒ Object (protected) 



811
812
813
814
815
 
# File 'lib/osvm/machine.rb', line 811

def qemu_boot_media_options
  return [] if config.iso.nil?

  ['-cdrom', config.iso]
end

#qemu_boot_options(kernel_params) ⇒ Object (protected) 



797
798
799
800
801
802
803
804
805
806
807
808
809
 
# File 'lib/osvm/machine.rb', line 797

def qemu_boot_options(kernel_params)
  if config.boot_mode == 'direct'
    [
      '-kernel', config.kernel,
      '-initrd', config.initrd,
      '-append', base_kernel_params(kernel_params).join(' ')
    ]
  else
    ret = []
    ret += ['-boot', "order=#{config.boot_order}"] if config.boot_order
    ret
  end
end

#qemu_command(kernel_params: []) ⇒ Object (protected)

Raises:

  • (NotImplementedError) 


778
779
780
 
# File 'lib/osvm/machine.rb', line 778

def qemu_command(kernel_params: [])
  raise NotImplementedError, "#{self.class} must implement #qemu_command"
end

#qemu_disk_options ⇒ Object (protected) 



827
828
829
830
831
832
833
834
835
836
 
# File 'lib/osvm/machine.rb', line 827

def qemu_disk_options
  ret = []

  config.disks.each_with_index do |disk, i|
    ret << '-drive' << "id=disk#{i},file=#{disk_path(disk.device)},if=none,format=raw"
    ret << '-device' << "ide-hd,drive=disk#{i},bus=ahci.#{i}"
  end

  ret
end

#qemu_shell_options ⇒ Object (protected) 



817
818
819
820
821
822
823
824
825
 
# File 'lib/osvm/machine.rb', line 817

def qemu_shell_options
  ret = ['-device', 'virtio-serial']

  shell_instances.each do |shell|
    ret += shell.qemu_options
  end

  ret
end

#qemu_virtiofs_options ⇒ Object (protected) 



838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
 
# File 'lib/osvm/machine.rb', line 838

def qemu_virtiofs_options
  ret = []

  shared_filesystems.each_with_index do |fs, i|
    name, = fs
    ret << '-chardev' << "socket,id=char#{i},path=#{virtiofs_socket_path(name)}"
    ret << '-device' << "vhost-user-fs-pci,queue-size=1024,chardev=char#{i},tag=#{name}"
  end

  if ret.any? && !custom_qemu_numa_memory_backend?
    ret << '-object' << "memory-backend-file,id=m0,size=#{config.memory}M,mem-path=/dev/shm,share=on"
    ret << '-numa' << 'node,memdev=m0'
  end

  ret
end

#raise_if_kernel_failed! ⇒ Object

Raises:



617
618
619
620
621
622
623
624
625
626
 
# File 'lib/osvm/machine.rb', line 617

def raise_if_kernel_failed!
  failure = @mutex.synchronize { @kernel_failure&.dup }
  return self unless failure

  raise KernelFailure.new(
    machine_name: name,
    console_line: failure.fetch(:line),
    console_log_path:
  )
end

#read_nonblock(io) ⇒ Object (protected) 



1222
1223
1224
1225
1226
 
# File 'lib/osvm/machine.rb', line 1222

def read_nonblock(io)
  io.read_nonblock(4096)
rescue IO::WaitReadable
  ''
end

#remaining_time!(deadline, timeout_message) ⇒ Object (protected)

Raises:



749
750
751
752
753
754
 
# File 'lib/osvm/machine.rb', line 749

def remaining_time!(deadline, timeout_message)
  remaining = deadline - monotonic_now
  raise TimeoutError, timeout_message if remaining <= 0

  remaining
end

#reset_kernel_failure ⇒ Object (protected) 



1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
 
# File 'lib/osvm/machine.rb', line 1049

def reset_kernel_failure
  @mutex.synchronize do
    @kernel_failure = nil
    @kernel_failure_detected_at = nil
    @console_output = ''
    @console_scan_buffer = ''
    @console_last_output_at = nil
    @console_eof = false
  end
end

#rollback_start(qemu_pid:, qemu_reaper:, qemu_write:) ⇒ Object (protected) 



901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
 
# File 'lib/osvm/machine.rb', line 901

def rollback_start(qemu_pid:, qemu_reaper:, qemu_write:)
  begin
    qemu_write&.close
  rescue IOError
    # The spawn handoff may already have closed this exact descriptor.
  end

  if qemu_pid
    if qemu_reaper
      qemu_reaper.signal('KILL')
      qemu_reaper.join
    else
      terminate_unreaped_child(qemu_pid, 'KILL')
    end
  end
ensure
  begin
    # A completed reaper has already run these operations. Every step is
    # deliberately idempotent so rollback can repeat them after join.
    cleanup_runtime_resources
  ensure
    mark_runtime_stopped
  end
end

#run_console_thread ⇒ Object (protected) 



1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
 
# File 'lib/osvm/machine.rb', line 1022

def run_console_thread
  console_read = qemu_read

  @console_thread = Thread.new do
    console_log = File.open(console_log_path, 'w')

    begin
      loop do
        rs = console_read.wait_readable
        next unless rs

        data = read_nonblock(console_read)
        append_console_output(data)

        console_log.write(data)
        console_log.flush
      end
    rescue EOFError
    rescue IOError
      # pass
    ensure
      append_console_output('', flush: true)
      console_log.close unless console_log.closed?
    end
  end
end

#run_qemu_reaper(pid) ⇒ Object (protected) 



940
941
942
943
944
945
946
947
948
949
950
 
# File 'lib/osvm/machine.rb', line 940

def run_qemu_reaper(pid)
  QemuReaper.new(pid) do |status|
    log.exit(status) if status
  ensure
    begin
      cleanup_runtime_resources
    ensure
      mark_runtime_stopped
    end
  end
end

#running? ⇒ Boolean

Returns:

  • (Boolean) 


441
442
443
 
# File 'lib/osvm/machine.rb', line 441

def running?
  @running
end

#service_check_command(_name) ⇒ Object (protected) 



782
783
784
 
# File 'lib/osvm/machine.rb', line 782

def service_check_command(_name)
  nil
end

#shell_chardev_id(index) ⇒ Object (protected) 



1115
1116
1117
 
# File 'lib/osvm/machine.rb', line 1115

def shell_chardev_id(index)
  shell_for(index).chardev_id
end

#shell_for(index) ⇒ Object (protected) 



1200
1201
1202
1203
 
# File 'lib/osvm/machine.rb', line 1200

def shell_for(index)
  validate_shell_index(index)
  @shell_instances[index]
end

#shell_log_path(index = 0) ⇒ Object (protected) 



1124
1125
1126
1127
 
# File 'lib/osvm/machine.rb', line 1124

def shell_log_path(index = 0)
  suffix = index == 0 ? 'shell' : "shell#{index}"
  File.join(tmpdir, "#{name}-#{suffix}.log")
end

#shell_socket_path(index = 0) ⇒ Object (protected) 



1119
1120
1121
1122
 
# File 'lib/osvm/machine.rb', line 1119

def shell_socket_path(index = 0)
  suffix = index == 0 ? 'shell' : "shell#{index}"
  socket_path("#{suffix}.sock")
end

#shells ⇒ ShellCollection

Returns:



731
732
733
 
# File 'lib/osvm/machine.rb', line 731

def shells
  @shell_collection
end

#sleep_with_deadline(duration, deadline, timeout_message) ⇒ Object (protected) 



771
772
773
774
775
776
 
# File 'lib/osvm/machine.rb', line 771

def sleep_with_deadline(duration, deadline, timeout_message)
  return sleep(duration) unless deadline

  sleep([duration, remaining_time!(deadline, timeout_message)].min)
  remaining_time!(deadline, timeout_message)
end

#socket_hash ⇒ Object (protected) 



1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
 
# File 'lib/osvm/machine.rb', line 1155

def socket_hash
  @socket_hash ||= begin
    identity = [hash_base.to_s.b, name.to_s.b].map(&:freeze).freeze
    digest = socket_identity_digest(identity)[0, SOCKET_HASH_LENGTH]
    key = [File.expand_path(sockdir), digest].freeze

    SOCKET_HASH_MUTEX.synchronize do
      registered_identity = SOCKET_HASH_IDENTITIES[key]

      if registered_identity && registered_identity != identity
        raise Error, "Socket hash collision for #{digest} in #{sockdir}"
      end

      SOCKET_HASH_IDENTITIES[key] ||= identity
    end

    digest
  end
end

#socket_identity_digest(identity) ⇒ Object (protected) 



1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
 
# File 'lib/osvm/machine.rb', line 1175

def socket_identity_digest(identity)
  digest = Digest::SHA256.new

  identity.each do |field|
    digest << [field.bytesize].pack('Q>')
    digest << field
  end

  digest.hexdigest
end

#socket_path(socket) ⇒ Object (protected) 



1151
1152
1153
 
# File 'lib/osvm/machine.rb', line 1151

def socket_path(socket)
  File.join(sockdir, "#{socket_hash}-#{socket}")
end

#start(kernel_params: [], wait_for_boot: false, deadline: nil, timeout_message: nil) ⇒ Machine

Start the machine

Parameters:

  • kernel_params (Array<String>) (defaults to: [])
  • wait_for_boot (Boolean) (defaults to: false)

Returns:



221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
 
# File 'lib/osvm/machine.rb', line 221

def start(kernel_params: [], wait_for_boot: false, deadline: nil, timeout_message: nil)
  timeout_message ||= "Timeout while starting machine #{name}"

  with_deadline(deadline, timeout_message) do
    @start_mutex.synchronize do
      if running?
        unless start_kernel_params == kernel_params
          raise 'Machine already started with different kernel parameters'
        end

        self.wait_for_boot(deadline:, timeout_message:) if wait_for_boot
        return self
      end

      qemu_write = nil
      started_qemu_pid = nil
      started_qemu_reaper = nil

      startup = proc do
        # virtiofsd cannot be relaunched right away, it needs some time settle
        # for unknown reasons, so we ensure there's a 5 second gap between stop
        # and start of this machine
        if @stopped_at
          diff = Time.now - @stopped_at
          delay = 5
          if diff <= delay
            sleep_with_deadline(
              [delay - diff, delay].min,
              deadline,
              timeout_message
            )
          end
        end

        log.start
        prepare_disks

        shell_instances.each(&:prepare)

        shared_dir.setup
        @shared_dir_mounted = false
        start_virtiofs
        sleep_with_deadline(1, deadline, timeout_message)

        qemu_kwargs = {}

        unless @interactive_console
          # Rollback must always own both ends of a prepared pipe. Defer the
          # asynchronous deadline until the descriptors are recorded.
          Thread.handle_interrupt(TimeoutError => :never) do
            @qemu_read, qemu_write = IO.pipe
          end

          qemu_kwargs = {
            in: :close,
            out: qemu_write,
            err: qemu_write
          }
        end

        @start_kernel_params = kernel_params
        reset_kernel_failure

        # Keep QEMU creation, PID publication, pipe/console handoff and
        # reaper ownership indivisible with respect to the asynchronous
        # deadline. Publish running before starting the reaper: an
        # immediately exiting QEMU may otherwise clear the state before
        # this thread incorrectly republishes it as live.
        Thread.handle_interrupt(TimeoutError => :never) do
          started_qemu_pid = Process.spawn(
            *qemu_command(kernel_params:),
            **qemu_kwargs
          )
          @qemu_pid = started_qemu_pid
          started_qemu_reaper = run_qemu_reaper(started_qemu_pid)
          @qemu_reaper = started_qemu_reaper
          qemu_write&.close
          qemu_write = nil
          @running = true
          run_console_thread unless @interactive_console
          started_qemu_reaper&.start
        end

        self.wait_for_boot(deadline:, timeout_message:) if wait_for_boot
        self
      end

      # Mask asynchronous startup timeouts across the exception-to-rollback
      # handoff. The startup body explicitly enables them, while rollback
      # keeps every exact process, thread and descriptor under ownership
      # until cleanup and stopped-state publication are complete.
      Thread.handle_interrupt(TimeoutError => :never) do
        Thread.handle_interrupt(TimeoutError => :immediate, &startup)
      rescue StandardError
        rollback_start(
          qemu_pid: started_qemu_pid,
          qemu_reaper: started_qemu_reaper,
          qemu_write:
        )
        raise
      end
    end
  end
end

#start_virtiofs ⇒ Object (protected) 



877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
 
# File 'lib/osvm/machine.rb', line 877

def start_virtiofs
  shared_filesystems.each do |name, path|
    f = File.open(virtiofs_log_path(name), 'w')

    begin
      # If the deadline fires as spawn returns, publish the child PID before
      # delivering it so startup rollback cannot lose ownership.
      Thread.handle_interrupt(TimeoutError => :never) do
        virtiofsd_pids << Process.spawn(
          File.join(config.virtiofsd, 'bin/virtiofsd'),
          '--socket-path', virtiofs_socket_path(name),
          '--shared-dir', path,
          '--cache', 'auto',
          in: :close,
          out: f,
          err: f
        )
      end
    ensure
      f.close
    end
  end
end

#stop(timeout: @default_timeout) ⇒ Machine

Stop the machine

Parameters:

  • timeout (Integer) (defaults to: @default_timeout)

Returns:



337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
 
# File 'lib/osvm/machine.rb', line 337

def stop(timeout: @default_timeout)
  reaper = qemu_reaper
  return self unless reaper

  log.stop
  begin
    shutdown_shell = current_shell
    shutdown_shell = shell_instances.find(&:up?) unless shutdown_shell.up?
    shutdown_shell ||= current_shell

    if shutdown_shell == current_shell
      execute(poweroff_command)
    else
      shutdown_shell.execute(poweroff_command)
    end
  rescue MachineShellClosed
    # The shell logs the failed command.
  end

  if reaper && !wait_for_reaper(reaper, timeout:)
    raise UnrecoverableTimeoutError, "Timeout while stopping machine #{name}"
  end

  raise_if_kernel_failed!
  self
end

#stop_virtiofs ⇒ Object (protected) 



926
927
928
929
930
931
932
933
934
935
936
937
938
 
# File 'lib/osvm/machine.rb', line 926

def stop_virtiofs
  virtiofsd_pids.delete_if do |pid|
    Process.kill('TERM', pid)
    false
  rescue Errno::ESRCH
    true
  end

  virtiofsd_pids.delete_if do |pid|
    Process.wait(pid)
    true
  end
end

#succeeds(cmd, timeout: @default_timeout, shell: nil) ⇒ Array<Integer, String>

Execute command and check that it succeeds

Parameters:

  • cmd (String)
  • timeout (Integer) (defaults to: @default_timeout)

Returns:

  • (Array<Integer, String>) 


480
481
482
 
# File 'lib/osvm/machine.rb', line 480

def succeeds(cmd, timeout: @default_timeout, shell: nil)
  command_shell(shell).succeeds(cmd, timeout:)
end

#succeeds_with_retries(cmd, attempts:, retry_delay: 1, timeout: @default_timeout, shell: nil) ⇒ Array<Integer, String>

Execute a command repeatedly until it succeeds or all attempts are used

Parameters:

  • cmd (String)
  • attempts (Integer)
  • retry_delay (Numeric) (defaults to: 1)
  • timeout (Integer) (defaults to: @default_timeout) —

    timeout for each attempt

Returns:

  • (Array<Integer, String>) 


490
491
492
 
# File 'lib/osvm/machine.rb', line 490

def succeeds_with_retries(cmd, attempts:, retry_delay: 1, timeout: @default_timeout, shell: nil)
  command_shell(shell).succeeds_with_retries(cmd, attempts:, retry_delay:, timeout:)
end

#terminate_unreaped_child(pid, signal) ⇒ Object (protected)

Before a reaper exists, this thread is the sole child owner. Poll before signaling; after a live poll an exit remains an unreaped zombie until the final wait, which prevents PID reuse across the signal operation.



955
956
957
958
959
960
961
962
963
964
965
966
967
968
 
# File 'lib/osvm/machine.rb', line 955

def terminate_unreaped_child(pid, signal)
  waited = Process.waitpid2(pid, Process::WNOHANG)
  return waited.last if waited

  begin
    Process.kill(signal, pid)
  rescue Errno::ESRCH
    # The child may have exited after the nonblocking wait.
  end

  Process.waitpid2(pid).last
rescue Errno::ECHILD
  nil
end

#validate_shell_index(index) ⇒ Object (protected)

Raises:

  • (ArgumentError) 


1205
1206
1207
1208
1209
 
# File 'lib/osvm/machine.rb', line 1205

def validate_shell_index(index)
  return if index.is_a?(Integer) && index >= 0 && index < config.test_shells

  raise ArgumentError, "invalid shell index #{index.inspect}"
end

#virtiofs_log_path(mount_name) ⇒ Object (protected) 



1147
1148
1149
 
# File 'lib/osvm/machine.rb', line 1147

def virtiofs_log_path(mount_name)
  File.join(tmpdir, "#{name}-fs-#{mount_name}.log")
end

#virtiofs_socket_path(mount_name) ⇒ Object (protected) 



1143
1144
1145
 
# File 'lib/osvm/machine.rb', line 1143

def virtiofs_socket_path(mount_name)
  socket_path("fs-#{mount_name}.sock")
end

#wait_for_boot(timeout: @default_timeout, deadline: nil, timeout_message: nil) ⇒ Object

Wait until the system has booted

Parameters:

  • timeout (Integer) (defaults to: @default_timeout) 


457
458
459
460
461
462
 
# File 'lib/osvm/machine.rb', line 457

def wait_for_boot(timeout: @default_timeout, deadline: nil, timeout_message: nil)
  options = { timeout: }
  options[:deadline] = deadline if deadline
  options[:timeout_message] = timeout_message if timeout_message
  current_shell.wait(**options)
end

#wait_for_console_text(regex, timeout: @default_timeout) ⇒ Machine

Wait for text to appear in console output

Parameters:

  • regex (Regexp)

Returns:



665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
 
# File 'lib/osvm/machine.rb', line 665

def wait_for_console_text(regex, timeout: @default_timeout)
  t1 = Time.now
  cur_timeout = timeout

  log_started_at = log.console_wait_begin(regex)

  loop do
    raise_if_kernel_failed!

    if regex =~ console_output
      log.console_wait_end(true, nil, log_started_at)
      return self
    end

    cur_timeout = timeout - (Time.now - t1)

    if cur_timeout <= 0
      log.console_wait_end(false, 'timeout', log_started_at)
      raise TimeoutError, "Timeout occurred while waiting for #{regex.inspect} on the console"
    elsif !running?
      log.console_wait_end(false, 'machine not running', log_started_at)
      raise Error, 'Machine is not running'
    end

    sleep(1)
  end

  self
end

#wait_for_reaper(reaper, timeout:) ⇒ Object (protected) 



1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
 
# File 'lib/osvm/machine.rb', line 1007

def wait_for_reaper(reaper, timeout:)
  deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout if timeout

  loop do
    raise_if_kernel_failed!
    remaining = deadline ? deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC) : 1
    return false if remaining <= 0

    if reaper.join([remaining, 1].min)
      raise_if_kernel_failed!
      return true
    end
  end
end

#wait_for_service(name) ⇒ Machine

Wait for a system service to start

Parameters:

  • name (String)

Returns:

Raises:

  • (NotImplementedError) 


588
589
590
591
592
593
594
 
# File 'lib/osvm/machine.rb', line 588

def wait_for_service(name)
  cmd = service_check_command(name)
  raise NotImplementedError, "#{self.class} does not implement service checks" if cmd.nil?

  wait_until_succeeds(cmd)
  self
end

#wait_for_shutdown(timeout: @default_timeout) ⇒ Machine

Wait until the machine shuts down

Parameters:

  • timeout (Integer) (defaults to: @default_timeout)

Returns:



570
571
572
573
574
575
576
577
578
579
580
581
582
583
 
# File 'lib/osvm/machine.rb', line 570

def wait_for_shutdown(timeout: @default_timeout)
  t1 = Time.now

  loop do
    raise_if_kernel_failed!
    return self unless running?

    if t1 + timeout < Time.now
      raise TimeoutError, 'Timeout occurred while waiting for shutdown'
    end

    sleep(1)
  end
end

#wait_until_fails(cmd, timeout: @default_timeout, shell: nil) ⇒ Array<Integer, String>

Wait until command fails

Returns:

  • (Array<Integer, String>) 


534
535
536
 
# File 'lib/osvm/machine.rb', line 534

def wait_until_fails(cmd, timeout: @default_timeout, shell: nil)
  command_shell(shell).wait_until_fails(cmd, timeout:)
end

#wait_until_online(timeout: 20 * 60) ⇒ Machine

Wait until network is operational, including DNS

Returns:



540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
 
# File 'lib/osvm/machine.rb', line 540

def wait_until_online(timeout: 20 * 60)
  timeout_message = 'Timeout occurred while waiting for network to become online'
  deadline = monotonic_deadline(timeout)

  remaining_time!(deadline, timeout_message)
  start(deadline:, timeout_message:) unless running?
  wait_for_boot(
    timeout: remaining_time!(deadline, timeout_message),
    deadline:,
    timeout_message:
  )

  loop do
    status, = execute(
      'curl --head --max-time 10 https://check-online.vpsadminos.org',
      timeout: remaining_time!(deadline, timeout_message),
      deadline:,
      timeout_message:
    )
    remaining_time!(deadline, timeout_message)

    return self if status == 0

    sleep_with_deadline(1, deadline, timeout_message)
  end
end

#wait_until_succeeds(cmd, timeout: @default_timeout, shell: nil) ⇒ Array<Integer, String>

Wait until command succeeds

Returns:

  • (Array<Integer, String>) 


528
529
530
 
# File 'lib/osvm/machine.rb', line 528

def wait_until_succeeds(cmd, timeout: @default_timeout, shell: nil)
  command_shell(shell).wait_until_succeeds(cmd, timeout:)
end

#with_deadline(deadline, timeout_message, &block) ⇒ Object (protected) 



760
761
762
763
764
765
766
767
768
769
 
# File 'lib/osvm/machine.rb', line 760

def with_deadline(deadline, timeout_message, &block)
  return block.call unless deadline

  ::Timeout.timeout(
    remaining_time!(deadline, timeout_message),
    TimeoutError,
    timeout_message,
    &block
  )
end