Class: OsCtld::Container::NfsCancellation

Inherits:
Object
  • Object
show all
Includes:
OsCtl::Lib::Utils::Log
Defined in:
lib/osctld/container/nfs_cancellation.rb

Overview

Namespace handles belong to one container run, not a reusable PID. Never follow tenant mount paths or use the tenant's potentially replaced /sys.

Defined Under Namespace

Classes: WorkerTimeout

Constant Summary collapse

NS_GET_USERNS = 
0xb701
NS_GET_PARENT = 
0xb702
PF_EXITING = 
0x00000004
WORKER_TIMEOUT = 
30
MAX_WORKER_OUTPUT = 
65_536

Instance Method Summary collapse

Constructor Details

#initialize(ct, proc_root: '/proc') ⇒ NfsCancellation

Returns a new instance of NfsCancellation.



21
22
23
24
25
26
27
28
29
30
31
32
33
34
 
# File 'lib/osctld/container/nfs_cancellation.rb', line 21

def initialize(ct, proc_root: '/proc')
  # Run identity is immutable. Never acquire the container lock while
  # holding @mutex: Container#stopped closes this object under that lock.
  @ident = ct.ident.dup.freeze
  @payload = File.join('/', ct.cgroup_path, "lxc.payload.#{ct.id}").freeze
  @payload_prefix = "#{@payload}/".freeze
  @proc_root = proc_root
  @mutex = Mutex.new
  @owner = nil
  @netns = {}
  @init_proc = nil
  @exit_cancelled = false
  @closed = false
end

Instance Method Details

#abort ⇒ Object

The caller must quiesce tenant work before requesting terminal abort. Retained handles remain usable even after init's /proc entry disappears.



62
63
64
 
# File 'lib/osctld/container/nfs_cancellation.rb', line 62

def abort
  @mutex.synchronize { abort_locked }
end

#abort_if_exiting ⇒ Object

PID1 can block in exit_files(), before LXC emits STOPPING. A retained proc directory identifies the original task even if its PID is reused.



68
69
70
71
72
73
74
75
76
77
78
79
 
# File 'lib/osctld/container/nfs_cancellation.rb', line 68

def abort_if_exiting
  @mutex.synchronize do
    return 0 if @exit_cancelled || !@init_proc

    path = File.join(@proc_root, 'self/fd', @init_proc.fileno.to_s)
    return 0 unless init_exiting?(path)

    count = abort_locked
    @exit_cancelled = true
    count
  end
end

#abort_locked ⇒ Object (protected) 



105
106
107
108
109
 
# File 'lib/osctld/container/nfs_cancellation.rb', line 105

def abort_locked
  return 0 unless @owner

  cancel_in_worker
end

#cancel_filesystems(path) ⇒ Object (protected) 



338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
 
# File 'lib/osctld/container/nfs_cancellation.rb', line 338

def cancel_filesystems(path)
  return 0 unless Dir.exist?(path)

  count = 0
  Dir.children(path).each do |name|
    next unless /\A(?:[0-9]+:[0-9]+|server-[0-9]+)\z/.match?(name)

    begin
      File.open(File.join(path, name, 'shutdown'), File::WRONLY) { |f| f.write("1\n") }
      count += 1
    rescue Errno::ENOENT
      next
    end
  end
  count
end

#cancel_in_worker ⇒ Object (protected) 



200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
 
# File 'lib/osctld/container/nfs_cancellation.rb', line 200

def cancel_in_worker
  deadline = monotonic_time + WORKER_TIMEOUT
  reader, writer = IO.pipe
  pid = SwitchUser.fork(keep_fds: [writer, @owner, *@netns.values].compact) do
    Process.setproctitle("osctld: #{@ident} NFS cancellation")
    # Bound the process/thread scan with the same deadline as sysfs work.
    # Newly discovered handles need live only until this worker exits.
    capture_descendants
    writer.puts(JSON.generate(count: cancel_namespaces(deadline)))
    exit!(true)
  rescue StandardError => e
    writer.puts(JSON.generate(error: "#{e.class}: #{e.message}"))
    exit!
  end
  writer.close
  output = read_worker_output(reader, deadline)
  status = wait_for_worker(pid, deadline)
  pid = nil
  result = JSON.parse(output, symbolize_names: true)
  raise "NFS cancellation failed: #{result[:error]}" unless status.success?

  result.fetch(:count)
ensure
  reader&.close
  writer&.close unless writer&.closed?
  terminate_worker(pid) if pid
end

#cancel_namespace(path, subtree: false, deadline: monotonic_time + WORKER_TIMEOUT) ⇒ Object (protected) 



304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
 
# File 'lib/osctld/container/nfs_cancellation.rb', line 304

def cancel_namespace(path, subtree: false, deadline: monotonic_time + WORKER_TIMEOUT)
  return 0 unless Dir.exist?(path)

  tree_control = File.join(path, 'net/nfs_client/shutdown_tree')
  if subtree && File.exist?(tree_control)
    # Only select the authenticated run owner, never an ancestor reached
    # through an inherited host namespace or a tenant-controlled path.
    write_shutdown_control(tree_control, deadline)
    return 1
  end

  control = File.join(path, 'net/nfs_client/shutdown')
  if File.exist?(control)
    # Sticky shutdown covers initializing mounts and future RPC clients.
    write_shutdown_control(control, deadline)
    return 1
  end

  # Older kernels still enforce soft mounts and lack the admission barrier.
  cancel_filesystems(path)
end

#cancel_namespaces(deadline) ⇒ Object (protected) 



277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
 
# File 'lib/osctld/container/nfs_cancellation.rb', line 277

def cancel_namespaces(deadline)
  sys = OsCtl::Lib::Sys.new
  sys.unshare_ns(OsCtl::Lib::Sys::CLONE_NEWNS)
  sys.make_rslave('/')
  count = 0

  Dir.mktmpdir('nfs-cancel-', '/run/osctl') do |mountpoint|
    @netns.each_value do |netns|
      sys.setns_io(netns, OsCtl::Lib::Sys::CLONE_NEWNET)
      sys.mount_sysfs(mountpoint)
      begin
        count += cancel_namespace(File.join(mountpoint, 'fs/nfs'), subtree: root_network_namespace?(netns), deadline: deadline)
      ensure
        sys.unmount(mountpoint)
      end
    end
  end
  count
end

#capture(pid) ⇒ Object 



36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
 
# File 'lib/osctld/container/nfs_cancellation.rb', line 36

def capture(pid)
  return if pid.nil?

  @mutex.synchronize do
    return if @closed

    with_process(pid) do |path, dir|
      next unless member?(path)

      File.open(File.join(path, 'ns/user')) do |userns|
        next if same_namespace?(userns, File.stat(File.join(@proc_root, 'self/ns/user')))
        next if @owner && !same_namespace?(userns, @owner.stat)

        @owner ||= userns.dup
        capture_netns(path)
        retain_init(path, dir)
      end
    end
  end
  nil
rescue Errno::ENOENT, Errno::ESRCH
  nil
end

#capture_descendants ⇒ Object (protected) 



177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
 
# File 'lib/osctld/container/nfs_cancellation.rb', line 177

def capture_descendants
  Dir.foreach(@proc_root) do |entry|
    next unless /\A[0-9]+\z/.match?(entry)

    begin
      with_process(entry) do |path|
        # setns() is per-thread. The group leader need not use the
        # network namespace containing another thread's NFS client.
        Dir.children(File.join(path, 'task')).each do |tid|
          thread_path = File.join(path, 'task', tid)
          begin
            capture_netns(thread_path) if member?(thread_path)
          rescue Errno::ENOENT, Errno::ESRCH
            next
          end
        end
      end
    rescue Errno::ENOENT, Errno::ESRCH
      next
    end
  end
end

#capture_netns(path) ⇒ Object (protected) 



168
169
170
171
172
173
174
175
 
# File 'lib/osctld/container/nfs_cancellation.rb', line 168

def capture_netns(path)
  File.open(File.join(path, 'ns/net')) do |netns|
    next unless owned_network_namespace?(netns)

    stat = netns.stat
    @netns[[stat.dev, stat.ino]] ||= netns.dup
  end
end

#close ⇒ Object 



81
82
83
84
85
86
87
88
89
90
91
 
# File 'lib/osctld/container/nfs_cancellation.rb', line 81

def close
  @mutex.synchronize do
    @closed = true
    @netns.each_value(&:close)
    @netns.clear
    @owner&.close
    @owner = nil
    @init_proc&.close
    @init_proc = nil
  end
end

#init_exiting?(path) ⇒ Boolean (protected)

Returns:

  • (Boolean) 


120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
 
# File 'lib/osctld/container/nfs_cancellation.rb', line 120

def init_exiting?(path)
  # The group leader can call pthread_exit while another init thread
  # keeps the namespace alive. Require every remaining thread to exit.
  Dir.children(File.join(path, 'task')).all? do |tid|
    stat = File.read(File.join(path, 'task', tid, 'stat'))
    fields = stat[(stat.rindex(')') + 1)..].split
    # stat field 9 is flags; the fields array starts at field 3 (state).
    Integer(fields.fetch(6)).anybits?(PF_EXITING)
  rescue Errno::ENOENT, Errno::ESRCH
    true
  end
rescue Errno::ENOENT, Errno::ESRCH
  # Init vanished before we sampled PF_EXITING; its retained namespaces
  # may still be held by the LXC monitor or detached mounts.
  true
end

#log_type ⇒ Object 



93
94
95
 
# File 'lib/osctld/container/nfs_cancellation.rb', line 93

def log_type
  "nfs-cancel=#{@ident}"
end

#member?(path) ⇒ Boolean (protected)

Returns:

  • (Boolean) 


137
138
139
140
141
142
143
144
 
# File 'lib/osctld/container/nfs_cancellation.rb', line 137

def member?(path)
  File.foreach(File.join(path, 'cgroup')).any? do |line|
    _id, controllers, cgroup = line.strip.split(':', 3)
    next false unless controllers == '' || controllers&.split(',')&.include?('freezer')

    cgroup == @payload || cgroup&.start_with?(@payload_prefix)
  end
end

#monotonic_time ⇒ Object (protected) 



228
229
230
 
# File 'lib/osctld/container/nfs_cancellation.rb', line 228

def monotonic_time
  Process.clock_gettime(Process::CLOCK_MONOTONIC)
end

#owned_network_namespace?(netns) ⇒ Boolean (protected)

Returns:

  • (Boolean) 


151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
 
# File 'lib/osctld/container/nfs_cancellation.rb', line 151

def owned_network_namespace?(netns)
  owner = IO.for_fd(netns.ioctl(NS_GET_USERNS))
  begin
    loop do
      return true if same_namespace?(owner, @owner.stat)

      parent = IO.for_fd(owner.ioctl(NS_GET_PARENT))
      owner.close
      owner = parent
    end
  rescue Errno::EPERM
    false
  ensure
    owner.close
  end
end

#read_worker_output(reader, deadline) ⇒ Object (protected) 



239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
 
# File 'lib/osctld/container/nfs_cancellation.rb', line 239

def read_worker_output(reader, deadline)
  output = +''
  loop do
    left = worker_time_left(deadline)
    chunk = reader.read_nonblock(4096, exception: false)
    case chunk
    when nil
      return output
    when :wait_readable
      reader.wait_readable(left)
    else
      output << chunk
      raise 'NFS cancellation worker output is too large' if output.bytesize > MAX_WORKER_OUTPUT
    end
  end
end

#retain_init(path, dir) ⇒ Object (protected) 



111
112
113
114
115
116
117
118
 
# File 'lib/osctld/container/nfs_cancellation.rb', line 111

def retain_init(path, dir)
  return if @init_proc

  nspid = File.foreach(File.join(path, 'status')).find { |line| line.start_with?('NSpid:') }
  return unless nspid&.split&.last == '1'

  @init_proc = dir.dup
end

#root_network_namespace?(netns) ⇒ Boolean (protected)

Returns:

  • (Boolean) 


297
298
299
300
301
302
 
# File 'lib/osctld/container/nfs_cancellation.rb', line 297

def root_network_namespace?(netns)
  owner = IO.for_fd(netns.ioctl(NS_GET_USERNS))
  same_namespace?(owner, @owner.stat)
ensure
  owner&.close
end

#same_namespace?(io, stat) ⇒ Boolean (protected)

Returns:

  • (Boolean) 


146
147
148
149
 
# File 'lib/osctld/container/nfs_cancellation.rb', line 146

def same_namespace?(io, stat)
  actual = io.stat
  actual.dev == stat.dev && actual.ino == stat.ino
end

#terminate_worker(pid) ⇒ Object (protected) 



265
266
267
268
269
270
271
272
273
274
275
 
# File 'lib/osctld/container/nfs_cancellation.rb', line 265

def terminate_worker(pid)
  # An unreaped child keeps its PID reserved. Never signal a reaped PID.
  return if Process.wait2(pid, Process::WNOHANG)

  Process.kill('KILL', pid)
  # A broken kernel path might remain uninterruptible. Reap asynchronously
  # rather than turn the error/timeout path into another unbounded wait.
  Process.detach(pid)
rescue Errno::ECHILD
  nil
end

#wait_for_worker(pid, deadline) ⇒ Object (protected) 



256
257
258
259
260
261
262
263
 
# File 'lib/osctld/container/nfs_cancellation.rb', line 256

def wait_for_worker(pid, deadline)
  loop do
    result = Process.wait2(pid, Process::WNOHANG)
    return result.last if result

    sleep([worker_time_left(deadline), 0.05].min)
  end
end

#with_process(pid) ⇒ Object (protected) 



99
100
101
102
103
 
# File 'lib/osctld/container/nfs_cancellation.rb', line 99

def with_process(pid)
  File.open(File.join(@proc_root, pid.to_s)) do |dir|
    yield File.join(@proc_root, 'self/fd', dir.fileno.to_s), dir
  end
end

#worker_time_left(deadline) ⇒ Object (protected)

Raises:



232
233
234
235
236
237
 
# File 'lib/osctld/container/nfs_cancellation.rb', line 232

def worker_time_left(deadline)
  left = deadline - monotonic_time
  raise WorkerTimeout, 'NFS cancellation worker timed out' unless left > 0

  left
end

#write_shutdown_control(control, deadline) ⇒ Object (protected) 



326
327
328
329
330
331
332
333
334
335
336
 
# File 'lib/osctld/container/nfs_cancellation.rb', line 326

def write_shutdown_control(control, deadline)
  loop do
    File.open(control, File::WRONLY) { |f| f.write("1\n") }
    return
  rescue Errno::EBUSY
    # Kernel-side subtree admission uses a trylock to avoid blocking a
    # concurrent netns teardown. Retry here, outside the sysfs callback,
    # within the existing bounded cancellation-worker deadline.
    sleep([worker_time_left(deadline), 0.05].min)
  end
end