Class: OsCtld::Container::Recovery

Inherits:
Object
  • Object
show all
Includes:
OsCtl::Lib::Utils::Log, OsCtl::Lib::Utils::System, Utils::Ip
Defined in:
lib/osctld/container/recovery.rb

Overview

Contains method to work with an unresponsive or dead containers

Defined Under Namespace

Classes: InvalidNetifIdentity, RouteList

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from Utils::Ip

#delete_link_by_ifindex, #ip, #link_ifindex_by_name, #tc

Constructor Details

#initialize(ct) ⇒ Recovery

Returns a new instance of Recovery.

Parameters:



14
15
16
 
# File 'lib/osctld/container/recovery.rb', line 14

def initialize(ct)
  @ct = ct
end

Instance Attribute Details

#ct ⇒ Object (readonly, protected)

Returns the value of attribute ct.



262
263
264
 
# File 'lib/osctld/container/recovery.rb', line 262

def ct
  @ct
end

Instance Method Details

#cleanup_cgroups ⇒ Object

Remove left-over cgroups in container path



161
162
163
164
165
 
# File 'lib/osctld/container/recovery.rb', line 161

def cleanup_cgroups
  CGroup.rmpath_all(File.join(ct.cgroup_path, "lxc.payload.#{ct.id}"))
  CGroup.rmpath_all(File.join(ct.cgroup_path, "lxc.monitor.#{ct.id}"))
  CGroup.rmpath_all(File.join(ct.cgroup_path, "lxc.pivot.#{ct.id}"))
end

#cleanup_netifs {|veth, routes| ... } ⇒ Object

Find and remove left-over network interfaces used by the container

Yield Parameters:



170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
 
# File 'lib/osctld/container/recovery.rb', line 170

def cleanup_netifs
  # discovered name => exact configured netif and routes
  veths = {}

  [4, 6].each do |ip_v|
    routes = RouteList.new(ip_v)

    ct.netifs.each do |netif|
      next if netif.type != :routed

      netif.routes.each_version(ip_v) do |route|
        veth = routes.veth_of(route)

        next unless veth

        log(:info, "Found route #{route.addr.to_string} on #{veth}")
        entry = veths[veth]
        if entry && !entry[:netif].equal?(netif)
          raise InvalidNetifIdentity,
                "host veth #{veth.inspect} is associated with multiple routed netifs"
        end

        entry ||= (veths[veth] = { netif:, routes: [] })
        entry[:routes] << route
      end
    end
  end

  NetInterface.sync_host_link_registry do
    veths.each do |discovered_name, entry|
      _recorded_name, veth_ifindex, ifb_ifindex = validate_recovery_identity(
        entry[:netif],
        discovered_name
      )
      validate_unshared_recovery_identity(
        discovered_name,
        veth_ifindex,
        ifb_ifindex,
        owner_netif: entry[:netif]
      )

      yield(discovered_name, entry[:routes]) if block_given?
      log(:info, "Removing #{discovered_name} (ifindex #{veth_ifindex})")
      # The netif owns both the kernel mutation and its durable identity.
      # Delegating cleanup ensures an IFB deletion is persisted before the
      # veth attempt and a completed deletion clears the record atomically
      # from recovery's point of view.
      entry[:netif].down(discovered_name)
    end
  end
end

#cleanup_or_taint ⇒ Boolean

Cleanup after the container or retain a separate recovery taint

Returns:

  • (Boolean) 


96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
 
# File 'lib/osctld/container/recovery.rb', line 96

def cleanup_or_taint
  taint = false

  begin
    cleanup_cgroups
  rescue StandardError => e
    log(:warn, "Failed to cleanup cgroups: #{e.class}: #{e.message}")
    taint = true
  end

  begin
    take_down_netifs
  rescue StandardError => e
    log(:warn, "Failed to take down netifs: #{e.class}: #{e.message}")
    taint = true
  end

  begin
    cleanup_netifs
  rescue StandardError => e
    log(:warn, "Failed to cleanup netifs: #{e.class}: #{e.message}")
    taint = true
  end

  begin
    NetInterface.sync_host_link_registry do
      NetInterface.validate_container_host_links_released!(ct)
    end
  rescue StandardError => e
    log(:warn, "Failed to verify released host links: #{e.class}: #{e.message}")
    taint = true
  end

  if taint
    ct.taint_recovery!
  elsif ct.recovery_tainted?
    ct.clear_recovery_taint!
  end
  !taint
end

#ensure_stopped! ⇒ Object

Caller holds the container manipulation lock. This is an explicit privileged acknowledgment, never part of normal callback cleanup.

Raises:



139
140
141
142
143
144
 
# File 'lib/osctld/container/recovery.rb', line 139

def ensure_stopped!
  state = ContainerControl::Commands::State.run!(ct, force: true)
  return if state.state == :stopped && state.init_pid.to_i <= 0

  raise InvalidNetifIdentity, 'the container has to be stopped'
end

#forget_host_link(name) ⇒ Object 



146
147
148
149
150
151
152
153
154
155
156
157
158
 
# File 'lib/osctld/container/recovery.rb', line 146

def forget_host_link(name)
  ensure_stopped!

  NetInterface.sync_host_link_registry do
    netif = ct.netifs[name]
    unless netif && netif.respond_to?(:forget_absent_host_link!)
      raise InvalidNetifIdentity, 'network interface not found or not a veth'
    end

    NetInterface.validate_host_link_owner!(owner_ct: ct, owner_netif: netif)
    netif.forget_absent_host_link!
  end
end

#kill_all(signal: 'KILL') ⇒ Object

Kill all processes found in the container's cgroup with signal

Parameters:

  • signal (String) (defaults to: 'KILL') 


68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
 
# File 'lib/osctld/container/recovery.rb', line 68

def kill_all(signal: 'KILL')
  pl = OsCtl::Lib::ProcessList.new do |p|
    ctid = p.ct_id
    next(false) if ctid.nil?

    ctid[0] == ct.pool.name && ctid[1] == ct.id
  end

  log(:info, "#{pl.length} processes to kill")
  pl.each do |p|
    # Double check
    ctid = p.ct_id
    next if ctid.nil?

    next unless ctid[0] == ct.pool.name && ctid[1] == ct.id

    log(:info, "kill -SIG#{signal} #{p.pid} #{p.name}")

    begin
      Process.kill(signal, p.pid)
    rescue Errno::ESRCH
      # ignore
    end
  end
end

#log_type ⇒ Object 



222
223
224
 
# File 'lib/osctld/container/recovery.rb', line 222

def log_type
  "recover=#{ct.pool.name}:#{ct.id}"
end

#preflight_netif_cleanup ⇒ Object (protected)

Validate every recorded link owned by this container against all daemon records before direct teardown can mutate any of them.



277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
 
# File 'lib/osctld/container/recovery.rb', line 277

def preflight_netif_cleanup
  ct.netifs.each do |netif|
    next unless netif.respond_to?(:host_link_identity)

    recorded_identity = netif.host_link_identity
    recorded_name, veth_ifindex, ifb_ifindex = recorded_identity
    next if recorded_name.nil? && veth_ifindex.nil? && ifb_ifindex.nil?

    recorded_name, veth_ifindex, ifb_ifindex = validate_recovery_identity(
      netif,
      recorded_name,
      recorded_identity:
    )
    validate_unshared_recovery_identity(
      recorded_name,
      veth_ifindex,
      ifb_ifindex,
      owner_netif: netif
    )
  end
end

#recover_state ⇒ Object

Rediscover container state

If the container is found dead, appropriate actions and hooks for container stop are run.



22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
 
# File 'lib/osctld/container/recovery.rb', line 22

def recover_state
  orig_state = ct.state
  current_state = ct.current_state
  netif_cleanup_failed = false

  if orig_state == current_state
    # do nothing

  elsif current_state == :stopped
    # Put all network interfaces down
    begin
      take_down_netifs
    rescue StandardError => e
      log(:warn, "Failed to take down netifs: #{e.class}: #{e.message}")
      netif_cleanup_failed = true
    end

    # Unload AppArmor profile and destroy namespace
    if AppArmor.enabled?
      ct.apparmor.destroy_namespace
      ct.apparmor.unload_profile
    end

    ct.stopped

    # User-defined hook
    Hook.run(ct, :post_stop)

    # Announce the change first as :aborting, that will cause a waiting
    # osctl ct start to give it up
    Eventd.report(:state, pool: ct.pool.name, id: ct.id, state: :aborting)
    Eventd.report(:state, pool: ct.pool.name, id: ct.id, state: :stopped)

  else
    # Announce the change
    Eventd.report(:state, pool: ct.pool.name, id: ct.id, state: current_state)
  end

  Eventd.report(:state_recovery, pool: ct.pool.name, id: ct.id, state: current_state)
  nil
ensure
  cleanup_or_taint if netif_cleanup_failed
end

#take_down_netifs ⇒ Object (protected) 



264
265
266
267
268
269
270
271
272
273
 
# File 'lib/osctld/container/recovery.rb', line 264

def take_down_netifs
  # Freeze both the daemon container set and all runtime host-link record
  # mutations from the first all-record check through the final delegated
  # deletion. A clone from DB::Containers.get alone is not a stable
  # authority snapshot.
  NetInterface.sync_host_link_registry do
    preflight_netif_cleanup
    ct.netifs.take_down
  end
end

#validate_recovery_identity(netif, discovered_name, recorded_identity: nil) ⇒ Object (protected) 



299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
 
# File 'lib/osctld/container/recovery.rb', line 299

def validate_recovery_identity(netif, discovered_name, recorded_identity: nil)
  unless netif.respond_to?(:host_link_identity)
    raise InvalidNetifIdentity,
          "netif for #{discovered_name.inspect} has no recorded host-link identity"
  end

  recorded_name, veth_ifindex, ifb_ifindex =
    recorded_identity || netif.host_link_identity
  unless recorded_name.is_a?(String) && !recorded_name.empty?
    raise InvalidNetifIdentity,
          "netif for #{discovered_name.inspect} has no recorded host-veth name"
  end
  if recorded_name != discovered_name
    raise InvalidNetifIdentity,
          "discovered host veth #{discovered_name.inspect} does not match " \
          "recorded #{recorded_name.inspect}"
  end
  unless veth_ifindex.is_a?(Integer) && veth_ifindex > 0
    raise InvalidNetifIdentity,
          "host veth #{recorded_name.inspect} has no valid recorded ifindex"
  end
  if ifb_ifindex && (!ifb_ifindex.is_a?(Integer) || ifb_ifindex <= 0)
    raise InvalidNetifIdentity,
          "host veth #{recorded_name.inspect} has invalid recorded IFB ifindex " \
          "#{ifb_ifindex.inspect}"
  end
  if ifb_ifindex == veth_ifindex
    raise InvalidNetifIdentity,
          "host veth #{recorded_name.inspect} reuses ifindex #{veth_ifindex} for its IFB"
  end

  [recorded_name, veth_ifindex, ifb_ifindex]
end

#validate_unshared_recovery_identity(name, veth_ifindex, ifb_ifindex, owner_netif:) ⇒ Object (protected) 



333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
 
# File 'lib/osctld/container/recovery.rb', line 333

def validate_unshared_recovery_identity(
  name,
  veth_ifindex,
  ifb_ifindex,
  owner_netif:
)
  NetInterface.validate_host_link_claim!(
    owner_ct: ct,
    owner_netif:,
    name:,
    veth_ifindex:,
    ifb_ifindex:,
    ifb_name: ifb_ifindex && "ifb#{name}"
  )
rescue NetInterface::HostLinkClaimError => e
  raise InvalidNetifIdentity, e.message
end