Class: OsVm::Shell

Inherits:
Object
  • Object
show all
Defined in:
lib/osvm/shell.rb

Constant Summary collapse

OUTPUT_END_MARKER =
'__OSVM_OUTPUT_END__'.freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(machine, index, socket_path, log_path, default_timeout:, name: nil, transport: 'virtconsole') ⇒ Shell

Returns a new instance of Shell.

Parameters:

  • machine (Machine)
  • index (Integer)
  • name (String, nil) (defaults to: nil)
  • socket_path (String)
  • log_path (String)
  • default_timeout (Integer)
  • transport ('virtconsole', 'virtserialport') (defaults to: 'virtconsole')


30
31
32
33
34
35
36
37
38
39
40
41
42
# File 'lib/osvm/shell.rb', line 30

def initialize(machine, index, socket_path, log_path, default_timeout:, name: nil, transport: 'virtconsole')
  @machine = machine
  @index = index
  @name = name
  @transport = transport
  @socket_path = socket_path
  @default_timeout = default_timeout
  @log = ShellLog.new(log_path, shell_index: index, shell_name: name)
  @mutex = Mutex.new
  @up = false
  @server = nil
  @io = nil
end

Instance Attribute Details

#index ⇒ Integer (readonly)

Returns:

  • (Integer)


15
16
17
# File 'lib/osvm/shell.rb', line 15

def index
  @index
end

#io ⇒ Object (readonly, protected)

Returns the value of attribute io.



294
295
296
# File 'lib/osvm/shell.rb', line 294

def io
  @io
end

#log ⇒ Object (readonly, protected)

Returns the value of attribute log.



294
295
296
# File 'lib/osvm/shell.rb', line 294

def log
  @log
end

#machine ⇒ Machine (readonly)

Returns:



12
13
14
# File 'lib/osvm/shell.rb', line 12

def machine
  @machine
end

#mutex ⇒ Object (readonly, protected)

Returns the value of attribute mutex.



294
295
296
# File 'lib/osvm/shell.rb', line 294

def mutex
  @mutex
end

#name ⇒ String? (readonly)

Returns:

  • (String, nil)


18
19
20
# File 'lib/osvm/shell.rb', line 18

def name
  @name
end

#server ⇒ Object (readonly, protected)

Returns the value of attribute server.



294
295
296
# File 'lib/osvm/shell.rb', line 294

def server
  @server
end

#socket_path ⇒ String (readonly)

Returns:

  • (String)


21
22
23
# File 'lib/osvm/shell.rb', line 21

def socket_path
  @socket_path
end

Instance Method Details

#accept(timeout: @default_timeout, deadline: nil, timeout_message: nil) ⇒ Object (protected)



318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
# File 'lib/osvm/shell.rb', line 318

def accept(timeout: @default_timeout, deadline: nil, timeout_message: nil)
  machine.raise_if_kernel_failed!
  raise "machine #{machine.name} is not running" unless machine.running?
  return io unless io.nil?

  timeout_message ||= 'Timeout occurred while waiting for shell connection'
  deadline = effective_deadline(timeout, deadline)

  with_deadline(deadline, timeout_message) do
    loop do
      machine.raise_if_kernel_failed!
      remaining = remaining_time!(deadline, timeout_message)
      raise Error, 'Machine is not running' unless machine.running?

      rs = server.wait_readable([1, remaining].min)
      next unless rs

      begin
        @io = server.accept_nonblock
        return io
      rescue IO::WaitReadable, Errno::EINTR
        next
      end
    end
  end
end

#all_fail(*cmds) ⇒ Array<Array<[Integer, String]>>

Execute all commands and check that they all fail

Parameters:

  • cmds (String)

Returns:

  • (Array<Array<[Integer, String]>>)


254
255
256
# File 'lib/osvm/shell.rb', line 254

def all_fail(*cmds)
  cmds.map { |cmd| fails(cmd) }
end

#all_succeed(*cmds) ⇒ Array<Array<[Integer, String]>>

Execute all commands and check that they all succeed

Parameters:

  • cmds (String)

Returns:

  • (Array<Array<[Integer, String]>>)


247
248
249
# File 'lib/osvm/shell.rb', line 247

def all_succeed(*cmds)
  cmds.map { |cmd| succeeds(cmd) }
end

#chardev_id ⇒ Object



68
69
70
# File 'lib/osvm/shell.rb', line 68

def chardev_id
  index == 0 ? 'shell' : "shell#{index}"
end

#cleanup ⇒ Object



96
97
98
99
100
# File 'lib/osvm/shell.rb', line 96

def cleanup
  File.unlink(socket_path)
rescue Errno::ENOENT
  # ignore
end

#close ⇒ Object



76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
# File 'lib/osvm/shell.rb', line 76

def close
  begin
    server&.close
  rescue IOError
    # ignore
  ensure
    @server = nil
  end

  begin
    io&.close
  rescue IOError
    # ignore
  ensure
    @io = nil
  end

  @up = false
end

#effective_deadline(timeout, outer_deadline) ⇒ Object (protected)



501
502
503
504
# File 'lib/osvm/shell.rb', line 501

def effective_deadline(timeout, outer_deadline)
  local_deadline = monotonic_now + timeout
  outer_deadline ? [local_deadline, outer_deadline].min : local_deadline
end

#execute(cmd, timeout: @default_timeout, deadline: nil, timeout_message: nil) ⇒ Array<Integer, String>

Returns exit status and output.

Parameters:

  • cmd (String)
  • timeout (Integer) (defaults to: @default_timeout)

Returns:

  • (Array<Integer, String>) —

    exit status and output



156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
# File 'lib/osvm/shell.rb', line 156

def execute(cmd, timeout: @default_timeout, deadline: nil, timeout_message: nil)
  timeout_message ||= "Timeout occurred while running command '#{cmd}'"
  deadline = effective_deadline(timeout, deadline)
  trace = { phase: 'before_protocol' }

  with_deadline(deadline, timeout_message) do
    machine.raise_if_kernel_failed!
    unless machine.running?
      machine.start(deadline:, timeout_message:)
    end
    machine.raise_if_kernel_failed!

    wait(
      timeout: remaining_time!(deadline, timeout_message),
      deadline:,
      timeout_message:
    )

    mutex.synchronize do
      wait(
        timeout: remaining_time!(deadline, timeout_message),
        deadline:,
        timeout_message:
      )
      execute_command(
        cmd,
        timeout: remaining_time!(deadline, timeout_message),
        deadline:,
        timeout_message:,
        trace:
      )
    end
  end
rescue TimeoutError => e
  # The deadline has been cancelled before logging. In particular, a
  # second timeout during reset must not erase the first protocol error.
  log.execute_timeout(e, trace)
  raise
end

#execute_command(cmd, timeout:, deadline: nil, timeout_message: nil, trace: nil) ⇒ Object (protected)



345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
# File 'lib/osvm/shell.rb', line 345

def execute_command(cmd, timeout:, deadline: nil, timeout_message: nil, trace: nil)
  timeout_message ||= "Timeout occurred while running command '#{cmd}'"
  deadline = effective_deadline(timeout, deadline)
  remaining = remaining_time!(deadline, timeout_message)
  protocol_reserve = [5.0, remaining * 0.2].min
  kill_after = [2.0, remaining * 0.1].min
  guest_timeout = remaining - protocol_reserve - kill_after
  vm_command = "set -euo pipefail; #{cmd}"
  timeout_command = "timeout --kill-after=#{format_timeout(kill_after)} #{format_timeout(guest_timeout)}"

  marker = "osvm-#{SecureRandom.hex(16)}"
  completion_marker = "#{marker}-completed"
  # The command may legitimately exit 124 (including its own timeout).
  # Reserve the outer timeout's status for the driver deadline, and carry
  # the completed command's status in its command-specific payload.
  completed_command = "bash -c #{Shellwords.escape(vm_command)}; " \
                      "printf ':#{completion_marker}:%s' \"$?\""

  # Prefix commands to tolerate the guest console dropping initial bytes.
  workaround = ' ' * 10
  log_started_at = log.execute_begin(cmd)
  protocol_started = true
  replies_consumed = false
  output = nil

  # Keep wrapped output for old console transports. A matching begin marker
  # separates this payload from incomplete or late replies on the stream.
  trace[:phase] = 'command_write' if trace
  io.write(
    "#{workaround}printf '\\n:#{marker}:begin\\n'; " \
    "#{timeout_command} bash -c #{Shellwords.escape(completed_command)} 2>&1 | base64 -w 76; " \
    "printf '\\n:#{marker}:%s\\n' \"${PIPESTATUS[0]}\"\n"
  )

  begin
    trace[:phase] = 'output_read' if trace
    status, output = read_command_result(
      marker,
      timeout: remaining_time!(deadline, timeout_message),
      deadline:,
      timeout_message:,
      command: vm_command,
      trace:
    )
    replies_consumed = true
    trace[:phase] = 'replies_consumed' if trace
  rescue MachineShellClosed
    log.execute_end(-1, '[machine shell closed]', log_started_at)
    raise
  end

  if [124, 137].include?(status)
    log.execute_end(-1, output, log_started_at)
    raise TimeoutError, "#{timeout_message}, output: #{output.inspect}"
  end

  if status == 0
    completion = /:#{Regexp.escape(completion_marker)}:(\d+)\z/.match(output)
    raise Error, 'Missing command completion status' unless completion

    status = completion[1].to_i
    output = output.byteslice(0, completion.begin(0))
  end

  log.execute_end(status, output, log_started_at)
  [status, output]
rescue TimeoutError => e
  if trace
    trace[:failed_phase] = trace[:phase]
    trace[:protocol_error] = "#{e.class}: #{e.message}"
  end
  # A command and its output/status replies form one stream protocol
  # transaction. Once any part of the command may have reached the guest,
  # a deadline before its matching result is consumed leaves that stream
  # desynchronized. Close it and make the timeout unrecoverable so polling
  # callers cannot send a second command over ambiguous bytes.
  if protocol_started && !replies_consumed
    reset
    log.execute_end(-1, output || '[shell protocol timeout]', log_started_at) if log_started_at
    raise if e.is_a?(UnrecoverableTimeoutError)

    raise UnrecoverableTimeoutError, e.message
  end

  raise
end

#expect_with_retries(cmd, attempts:, retry_delay:, timeout:, success:) ⇒ Object (protected)

Raises:



296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
# File 'lib/osvm/shell.rb', line 296

def expect_with_retries(cmd, attempts:, retry_delay:, timeout:, success:)
  unless attempts.is_a?(Integer) && attempts > 0
    raise ArgumentError, 'attempts must be a positive integer'
  end

  status = output = nil

  attempts.times do |attempt|
    status, output = execute(cmd, timeout:)
    matches = success ? (status == 0) : (status != 0)
    return [status, output] if matches

    sleep(retry_delay) unless attempt + 1 == attempts
  end

  if success
    raise CommandFailed, "Command '#{cmd}' failed with status #{status}. Output:\n #{output}"
  end

  raise CommandSucceeded, "Command '#{cmd}' succeeds with status #{status}. Output:\n #{output}"
end

#fails(cmd, timeout: @default_timeout) ⇒ Array<Integer, String>

Execute command and check that it fails

Parameters:

  • cmd (String)
  • timeout (Integer) (defaults to: @default_timeout)

Returns:

  • (Array<Integer, String>)


224
225
226
227
228
229
230
231
232
# File 'lib/osvm/shell.rb', line 224

def fails(cmd, timeout: @default_timeout)
  status, output = execute(cmd, timeout:)

  if status == 0
    raise CommandSucceeded, "Command '#{cmd}' succeeds with status #{status}. Output:\n #{output}"
  end

  [status, output]
end

#fails_with_retries(cmd, attempts:, retry_delay: 1, timeout: @default_timeout) ⇒ Array<Integer, String>

Execute a command repeatedly until it fails or all attempts are used

Parameters:

  • cmd (String)
  • attempts (Integer)
  • retry_delay (Numeric) (defaults to: 1)
  • timeout (Integer) (defaults to: @default_timeout) —

    timeout for each attempt

Returns:

  • (Array<Integer, String>)


240
241
242
# File 'lib/osvm/shell.rb', line 240

def fails_with_retries(cmd, attempts:, retry_delay: 1, timeout: @default_timeout)
  expect_with_retries(cmd, attempts:, retry_delay:, timeout:, success: false)
end

#finalize ⇒ Object



102
103
104
# File 'lib/osvm/shell.rb', line 102

def finalize
  log.close
end

#format_timeout(timeout) ⇒ Object (protected)



526
527
528
# File 'lib/osvm/shell.rb', line 526

def format_timeout(timeout)
  format('%.9f', timeout).sub(/\.?0+\z/, '')
end

#monotonic_now ⇒ Object (protected)



513
514
515
# File 'lib/osvm/shell.rb', line 513

def monotonic_now
  Process.clock_gettime(Process::CLOCK_MONOTONIC)
end

#prepare ⇒ Object



44
45
46
47
48
49
50
51
52
53
54
# File 'lib/osvm/shell.rb', line 44

def prepare
  File.unlink(socket_path)
rescue Errno::ENOENT
  # ignore
ensure
  # Do not let an asynchronous startup deadline split UNIXServer creation
  # from recording the descriptor which rollback has to close.
  Thread.handle_interrupt(TimeoutError => :never) do
    @server = UNIXServer.new(socket_path)
  end
end

#qemu_options ⇒ Object



56
57
58
59
60
61
62
63
64
65
66
# File 'lib/osvm/shell.rb', line 56

def qemu_options
  device = "#{@transport},chardev=#{chardev_id}"
  # Console ports discard output under backpressure; new guests use a
  # flow-controlled serial port instead. Keep old guest JSON compatible.
  device << ",name=org.osvm.shell#{index}" if @transport == 'virtserialport'

  [
    '-chardev', "socket,id=#{chardev_id},path=#{socket_path}",
    '-device', device
  ]
end

#read_command_result(marker, timeout:, command:, deadline: nil, timeout_message: nil, trace: nil) ⇒ Object (protected)



432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
# File 'lib/osvm/shell.rb', line 432

def read_command_result(marker, timeout:, command:, deadline: nil, timeout_message: nil, trace: nil)
  timeout_message ||= "Timeout occurred while running command '#{command}'"
  deadline = effective_deadline(timeout, deadline)
  start_marker = ":#{marker}:begin"
  pattern = /\A:#{Regexp.escape(marker)}:(\d+)\z/
  encoded = nil

  loop do
    # Discard stale records without renewing the absolute command deadline.
    raw_output = read_output(
      timeout: remaining_time!(deadline, timeout_message),
      deadline:,
      timeout_message:,
      command:,
      trace:
    )

    raw_output.each_line do |line|
      line = line.chomp
      if line == start_marker
        encoded = +''
      elsif encoded
        match = pattern.match(line)
        return [match[1].to_i, Base64.strict_decode64(encoded)] if match

        if %r{\A[A-Za-z0-9+/=]*\z}.match?(line)
          encoded << line
        else
          encoded = nil
        end
      end
    end
  end
rescue ArgumentError
  reset
  raise
end

#read_nonblock(io) ⇒ Object (protected)



542
543
544
545
546
# File 'lib/osvm/shell.rb', line 542

def read_nonblock(io)
  io.read_nonblock(4096)
rescue IO::WaitReadable
  ''
end

#read_output(timeout:, command:, deadline: nil, timeout_message: nil, terminator: "\n", trace: nil) ⇒ Object (protected)



470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
# File 'lib/osvm/shell.rb', line 470

def read_output(timeout:, command:, deadline: nil, timeout_message: nil, terminator: "\n", trace: nil)
  timeout_message ||= "Timeout occurred while running command '#{command}'"
  deadline = effective_deadline(timeout, deadline)
  buffer = ''
  trace[:reply_buffer] = buffer if trace

  loop do
    machine.raise_if_kernel_failed!
    remaining = remaining_time!(
      deadline,
      "#{timeout_message}, buffer contents: #{buffer.inspect}",
      error_class: UnrecoverableTimeoutError
    )

    rs = io.wait_readable([1, remaining].min)
    next unless rs

    begin
      buffer << read_nonblock(io)
    rescue EOFError
      reset
      machine.raise_if_kernel_failed!
      raise MachineShellClosed
    end

    break if buffer.end_with?(terminator)
  end

  buffer
end

#remaining_time!(deadline, timeout_message, error_class: TimeoutError) ⇒ Object (protected)

Raises:

  • (error_class)


506
507
508
509
510
511
# File 'lib/osvm/shell.rb', line 506

def remaining_time!(deadline, timeout_message, error_class: TimeoutError)
  remaining = deadline - monotonic_now
  raise error_class, timeout_message if remaining <= 0

  remaining
end

#reset ⇒ Object (protected)



530
531
532
533
534
535
536
537
538
539
540
# File 'lib/osvm/shell.rb', line 530

def reset
  @up = false

  return if io.nil?

  io.close unless io.closed?
rescue IOError
  # ignore
ensure
  @io = nil
end

#succeeds(cmd, timeout: @default_timeout) ⇒ Array<Integer, String>

Execute command and check that it succeeds

Parameters:

  • cmd (String)
  • timeout (Integer) (defaults to: @default_timeout)

Returns:

  • (Array<Integer, String>)


200
201
202
203
204
205
206
207
208
# File 'lib/osvm/shell.rb', line 200

def succeeds(cmd, timeout: @default_timeout)
  status, output = execute(cmd, timeout:)

  if status != 0
    raise CommandFailed, "Command '#{cmd}' failed with status #{status}. Output:\n #{output}"
  end

  [status, output]
end

#succeeds_with_retries(cmd, attempts:, retry_delay: 1, timeout: @default_timeout) ⇒ Array<Integer, String>

Execute a command repeatedly until it succeeds or all attempts are used

Parameters:

  • cmd (String)
  • attempts (Integer)
  • retry_delay (Numeric) (defaults to: 1)
  • timeout (Integer) (defaults to: @default_timeout) —

    timeout for each attempt

Returns:

  • (Array<Integer, String>)


216
217
218
# File 'lib/osvm/shell.rb', line 216

def succeeds_with_retries(cmd, attempts:, retry_delay: 1, timeout: @default_timeout)
  expect_with_retries(cmd, attempts:, retry_delay:, timeout:, success: true)
end

#up? ⇒ Boolean

Returns:

  • (Boolean)


72
73
74
# File 'lib/osvm/shell.rb', line 72

def up?
  @up
end

#wait(timeout: @default_timeout, deadline: nil, timeout_message: nil) ⇒ void

This method returns an undefined value.

Parameters:

  • timeout (Integer) (defaults to: @default_timeout)


108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
# File 'lib/osvm/shell.rb', line 108

def wait(timeout: @default_timeout, deadline: nil, timeout_message: nil)
  machine.raise_if_kernel_failed!
  raise "machine #{machine.name} is not running" unless machine.running?
  return if up?

  timeout_message ||= 'Timeout occurred while waiting for shell'
  deadline = effective_deadline(timeout, deadline)

  with_deadline(deadline, timeout_message) do
    buffer = ''

    loop do
      machine.raise_if_kernel_failed!
      remaining = remaining_time!(deadline, timeout_message)

      reset if io&.closed?
      if io.nil?
        accept(
          timeout: remaining,
          deadline:,
          timeout_message:
        )
      end

      remaining = remaining_time!(deadline, timeout_message)
      rs = io.wait_readable([1, remaining].min)
      next unless rs

      begin
        buffer << read_nonblock(io)
      rescue EOFError
        reset
        buffer = ''
        next
      end

      next unless buffer.include?("test-shell-ready\n")

      @up = true
      machine.__send__(:mount_shared_dir_once)
      return
    end
  end
end

#wait_until_fails(cmd, timeout: @default_timeout) ⇒ Array<Integer, String>

Wait until command fails

Returns:

  • (Array<Integer, String>)


277
278
279
280
281
282
283
284
285
286
287
288
289
290
# File 'lib/osvm/shell.rb', line 277

def wait_until_fails(cmd, timeout: @default_timeout)
  t1 = Time.now
  cur_timeout = timeout

  loop do
    status, output = execute(cmd, timeout: cur_timeout)
    return [status, output] if status != 0

    cur_timeout = timeout - (Time.now - t1)
    raise TimeoutError, "Timeout occurred while running command '#{cmd}'" if cur_timeout <= 0

    sleep(1)
  end
end

#wait_until_succeeds(cmd, timeout: @default_timeout) ⇒ Array<Integer, String>

Wait until command succeeds

Returns:

  • (Array<Integer, String>)


260
261
262
263
264
265
266
267
268
269
270
271
272
273
# File 'lib/osvm/shell.rb', line 260

def wait_until_succeeds(cmd, timeout: @default_timeout)
  t1 = Time.now
  cur_timeout = timeout

  loop do
    status, output = execute(cmd, timeout: cur_timeout)
    return [status, output] if status == 0

    cur_timeout = timeout - (Time.now - t1)
    raise TimeoutError, "Timeout occurred while running command '#{cmd}'" if cur_timeout <= 0

    sleep(1)
  end
end

#with_deadline(deadline, timeout_message) ⇒ Object (protected)



517
518
519
520
521
522
523
524
# File 'lib/osvm/shell.rb', line 517

def with_deadline(deadline, timeout_message, &)
  ::Timeout.timeout(
    remaining_time!(deadline, timeout_message),
    TimeoutError,
    timeout_message,
    &
  )
end