Class: TestRunner::ResourcePool

Inherits:
Object
  • Object
show all
Defined in:
lib/test-runner/resource_pool.rb

Defined Under Namespace

Classes: CapacitySource, HostResourceDetector

Constant Summary collapse

DEFAULT_MEMORY_RESERVE_MIB =
8192
DEFAULT_SHM_RESERVE_MIB =
8192
DEFAULT_CPU_RESERVE =
0
DEFAULT_MEMORY_OVERCOMMIT =
1.0
DEFAULT_SHM_OVERCOMMIT =
1.0
DEFAULT_CPU_OVERCOMMIT =
1.5

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(memory_mib:, shm_mib:, cpus:, memory_capacity: nil, shm_capacity: nil, cpu_capacity: nil) ⇒ ResourcePool

Returns a new instance of ResourcePool.



399
400
401
402
403
404
405
406
407
408
409
410
# File 'lib/test-runner/resource_pool.rb', line 399

def initialize(memory_mib:, shm_mib:, cpus:, memory_capacity: nil, shm_capacity: nil, cpu_capacity: nil)
  @memory_capacity = memory_capacity
  @shm_capacity = shm_capacity
  @cpu_capacity = cpu_capacity
  @memory_mib = memory_mib
  @shm_mib = shm_mib
  @cpus = cpus
  @used = TestResources.new
  @running = 0

  refresh_capacity
end

Instance Attribute Details

#cpus ⇒ Integer? (readonly)

Returns:

  • (Integer, nil)


116
117
118
# File 'lib/test-runner/resource_pool.rb', line 116

def cpus
  @cpus
end

#memory_mib ⇒ Integer? (readonly)

Returns:

  • (Integer, nil)


110
111
112
# File 'lib/test-runner/resource_pool.rb', line 110

def memory_mib
  @memory_mib
end

#running ⇒ Integer (readonly)

Returns:

  • (Integer)


122
123
124
# File 'lib/test-runner/resource_pool.rb', line 122

def running
  @running
end

#shm_mib ⇒ Integer? (readonly)

Returns:

  • (Integer, nil)


113
114
115
# File 'lib/test-runner/resource_pool.rb', line 113

def shm_mib
  @shm_mib
end

#used ⇒ TestResources (readonly)

Returns:



119
120
121
# File 'lib/test-runner/resource_pool.rb', line 119

def used
  @used
end

Class Method Details

.capacity(value, reserve) ⇒ Object



203
204
205
206
207
# File 'lib/test-runner/resource_pool.rb', line 203

def self.capacity(value, reserve)
  return nil if value.nil?

  [value - reserve, 0].max
end

.cgroup_path_ancestors(path) ⇒ Object



330
331
332
333
334
335
336
337
338
339
340
341
342
343
# File 'lib/test-runner/resource_pool.rb', line 330

def self.cgroup_path_ancestors(path)
  ret = []
  relative_path = path.sub(%r{\A/}, '')

  loop do
    ret << relative_path
    break if relative_path == ''

    relative_path = File.dirname(relative_path)
    relative_path = '' if relative_path == '.'
  end

  ret
end

.current_cgroup_path(controller) ⇒ Object



311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
# File 'lib/test-runner/resource_pool.rb', line 311

def self.current_cgroup_path(controller)
  return nil unless File.file?('/proc/self/cgroup')

  File.readlines('/proc/self/cgroup').each do |line|
    _id, controllers, path = line.strip.split(':', 3)
    next if path.nil?

    if controller.nil?
      return path if controllers == ''
    elsif controllers.split(',').include?(controller)
      return path
    end
  end

  nil
rescue Errno::ENOENT, Errno::EACCES
  nil
end

.detect_cgroup_memory_available_mib ⇒ Object



249
250
251
# File 'lib/test-runner/resource_pool.rb', line 249

def self.detect_cgroup_memory_available_mib
  detect_cgroup_v2_memory_available_mib || detect_cgroup_v1_memory_available_mib
end

.detect_cgroup_memory_limit_file(path) ⇒ Object



345
346
347
348
349
350
351
352
353
354
355
356
357
# File 'lib/test-runner/resource_pool.rb', line 345

def self.detect_cgroup_memory_limit_file(path)
  return nil unless File.file?(path)

  value = File.read(path).strip
  return nil if ['', 'max'].include?(value)

  bytes = Integer(value)
  return nil if bytes <= 0

  bytes / 1024 / 1024
rescue Errno::ENOENT, Errno::EACCES, ArgumentError
  nil
end

.detect_cgroup_memory_limit_mib ⇒ Object



245
246
247
# File 'lib/test-runner/resource_pool.rb', line 245

def self.detect_cgroup_memory_limit_mib
  detect_cgroup_v2_memory_limit_mib || detect_cgroup_v1_memory_limit_mib
end

.detect_cgroup_memory_usage_file(path) ⇒ Object



359
360
361
362
363
364
365
366
367
368
# File 'lib/test-runner/resource_pool.rb', line 359

def self.detect_cgroup_memory_usage_file(path)
  return nil unless File.file?(path)

  bytes = Integer(File.read(path).strip)
  return nil if bytes < 0

  bytes / 1024 / 1024
rescue Errno::ENOENT, Errno::EACCES, ArgumentError
  nil
end

.detect_cgroup_v1_memory_available_mib ⇒ Object



293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
# File 'lib/test-runner/resource_pool.rb', line 293

def self.detect_cgroup_v1_memory_available_mib
  path = current_cgroup_path('memory')
  return nil if path.nil?

  headrooms = cgroup_path_ancestors(path).filter_map do |ancestor|
    base = File.join('/sys/fs/cgroup/memory', ancestor)
    limit = detect_cgroup_memory_limit_file(File.join(base, 'memory.limit_in_bytes'))
    next if limit.nil?

    usage = detect_cgroup_memory_usage_file(File.join(base, 'memory.usage_in_bytes'))
    next if usage.nil?

    [limit - usage, 0].max
  end

  headrooms.min
end

.detect_cgroup_v1_memory_limit_mib ⇒ Object



284
285
286
287
288
289
290
291
# File 'lib/test-runner/resource_pool.rb', line 284

def self.detect_cgroup_v1_memory_limit_mib
  path = current_cgroup_path('memory')
  return nil if path.nil?

  detect_cgroup_memory_limit_file(
    File.join('/sys/fs/cgroup/memory', path.sub(%r{\A/}, ''), 'memory.limit_in_bytes')
  )
end

.detect_cgroup_v2_memory_available_mib ⇒ Object



266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
# File 'lib/test-runner/resource_pool.rb', line 266

def self.detect_cgroup_v2_memory_available_mib
  path = current_cgroup_path(nil)
  return nil if path.nil?

  headrooms = cgroup_path_ancestors(path).filter_map do |ancestor|
    base = File.join('/sys/fs/cgroup', ancestor)
    limit = detect_cgroup_memory_limit_file(File.join(base, 'memory.max'))
    next if limit.nil?

    usage = detect_cgroup_memory_usage_file(File.join(base, 'memory.current'))
    next if usage.nil?

    [limit - usage, 0].max
  end

  headrooms.min
end

.detect_cgroup_v2_memory_limit_mib ⇒ Object



253
254
255
256
257
258
259
260
261
262
263
264
# File 'lib/test-runner/resource_pool.rb', line 253

def self.detect_cgroup_v2_memory_limit_mib
  path = current_cgroup_path(nil)
  return nil if path.nil?

  limits = cgroup_path_ancestors(path).filter_map do |ancestor|
    detect_cgroup_memory_limit_file(
      File.join('/sys/fs/cgroup', ancestor, 'memory.max')
    )
  end

  limits.min
end

.detect_cpus ⇒ Object



390
391
392
393
394
395
396
397
# File 'lib/test-runner/resource_pool.rb', line 390

def self.detect_cpus
  out, status = Open3.capture2('nproc')
  return nil unless status.success?

  Integer(out.strip)
rescue Errno::ENOENT, ArgumentError
  nil
end

.detect_df_mib(path, column) ⇒ Object



378
379
380
381
382
383
384
385
386
387
388
# File 'lib/test-runner/resource_pool.rb', line 378

def self.detect_df_mib(path, column)
  out, status = Open3.capture2('df', '-Pk', path)
  return nil unless status.success?

  line = out.lines[1]
  return nil if line.nil?

  line.split[column].to_i / 1024
rescue Errno::ENOENT
  nil
end

.detect_meminfo_mib(key) ⇒ Object



234
235
236
237
238
239
240
241
242
243
# File 'lib/test-runner/resource_pool.rb', line 234

def self.detect_meminfo_mib(key)
  return nil unless File.file?('/proc/meminfo')

  line = File.readlines('/proc/meminfo').detect { |v| v.start_with?(key) }
  return nil if line.nil?

  line.split[1].to_i / 1024
rescue Errno::ENOENT, Errno::EACCES
  nil
end

.detect_memory_available_mib ⇒ Object



227
228
229
230
231
232
# File 'lib/test-runner/resource_pool.rb', line 227

def self.detect_memory_available_mib
  host_available = detect_meminfo_mib('MemAvailable:')
  cgroup_available = detect_cgroup_memory_available_mib

  [host_available, cgroup_available].compact.min
end

.detect_memory_mib ⇒ Object



216
217
218
219
220
221
222
223
224
225
# File 'lib/test-runner/resource_pool.rb', line 216

def self.detect_memory_mib
  mem_total = detect_meminfo_mib('MemTotal:')
  cgroup_limit = detect_cgroup_memory_limit_mib

  if mem_total && cgroup_limit
    [mem_total, cgroup_limit].min
  else
    cgroup_limit || mem_total
  end
end

.detect_shm_available_mib ⇒ Object



374
375
376
# File 'lib/test-runner/resource_pool.rb', line 374

def self.detect_shm_available_mib
  detect_df_mib('/dev/shm', 3)
end

.detect_shm_mib ⇒ Object



370
371
372
# File 'lib/test-runner/resource_pool.rb', line 370

def self.detect_shm_mib
  detect_df_mib('/dev/shm', 1)
end

.detector_value(detector, *methods) ⇒ Object



209
210
211
212
213
214
# File 'lib/test-runner/resource_pool.rb', line 209

def self.detector_value(detector, *methods)
  method = methods.detect { |m| detector.respond_to?(m) }
  return nil if method.nil?

  detector.public_send(method)
end

.factor_option(*values) ⇒ Object

Raises:

  • (ArgumentError)


193
194
195
196
197
198
199
200
201
# File 'lib/test-runner/resource_pool.rb', line 193

def self.factor_option(*values)
  value = values.detect { |v| !v.nil? && v.to_s != '' }
  return nil if value.nil?

  ret = Float(value)
  raise ArgumentError, 'overcommit factors must be positive' if ret <= 0

  ret
end

.from_options(opts, env: ENV) ⇒ Object



124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
# File 'lib/test-runner/resource_pool.rb', line 124

def self.from_options(opts, env: ENV)
  memory_reserve_mib = integer_option(
    opts[:memory_reserve_mib],
    env['TEST_RUNNER_MEMORY_RESERVE_MIB'],
    DEFAULT_MEMORY_RESERVE_MIB
  )
  shm_reserve_mib = integer_option(
    opts[:shm_reserve_mib],
    env['TEST_RUNNER_SHM_RESERVE_MIB'],
    DEFAULT_SHM_RESERVE_MIB
  )
  cpu_reserve = integer_option(
    opts[:cpu_reserve],
    env['TEST_RUNNER_CPU_RESERVE'],
    DEFAULT_CPU_RESERVE
  )
  memory_overcommit = factor_option(
    opts[:memory_overcommit],
    env['TEST_RUNNER_MEMORY_OVERCOMMIT'],
    DEFAULT_MEMORY_OVERCOMMIT
  )
  shm_overcommit = factor_option(
    opts[:shm_overcommit],
    env['TEST_RUNNER_SHM_OVERCOMMIT'],
    DEFAULT_SHM_OVERCOMMIT
  )
  cpu_overcommit = factor_option(
    opts[:cpu_overcommit],
    env['TEST_RUNNER_CPU_OVERCOMMIT'],
    DEFAULT_CPU_OVERCOMMIT
  )
  detector = opts[:resource_detector] || HostResourceDetector.new

  new(
    memory_mib: nil,
    shm_mib: nil,
    cpus: nil,
    memory_capacity: CapacitySource.new(
      max_value: integer_option(opts[:max_memory_mib], env['TEST_RUNNER_MAX_MEMORY_MIB'], nil),
      reserve: memory_reserve_mib,
      detector: -> { detector_value(detector, :memory_mib) },
      available_detector: -> { detector_value(detector, :memory_available_mib) },
      overcommit: memory_overcommit,
      monotonic: true
    ),
    shm_capacity: CapacitySource.new(
      max_value: integer_option(opts[:max_shm_mib], env['TEST_RUNNER_MAX_SHM_MIB'], nil),
      reserve: shm_reserve_mib,
      detector: -> { detector_value(detector, :shm_mib) },
      available_detector: -> { detector_value(detector, :shm_available_mib) },
      overcommit: shm_overcommit,
      monotonic: true
    ),
    cpu_capacity: CapacitySource.new(
      max_value: integer_option(opts[:max_cpus], env['TEST_RUNNER_MAX_CPUS'], nil),
      reserve: cpu_reserve,
      detector: -> { detector.cpus },
      overcommit: cpu_overcommit
    )
  )
end

.integer_option(*values) ⇒ Object



186
187
188
189
190
191
# File 'lib/test-runner/resource_pool.rb', line 186

def self.integer_option(*values)
  value = values.detect { |v| !v.nil? && v.to_s != '' }
  return nil if value.nil?

  Integer(value)
end

Instance Method Details

#can_reserve?(resources) ⇒ Boolean

Returns:

  • (Boolean)


422
423
424
425
426
# File 'lib/test-runner/resource_pool.rb', line 422

def can_reserve?(resources)
  fits?(memory_mib, used.memory_mib + resources.memory_mib) &&
    fits?(shm_mib, used.shm_mib + resources.shm_mib) &&
    fits?(cpus, used.cpus + resources.cpus)
end

#capacities ⇒ Object (protected)



466
467
468
# File 'lib/test-runner/resource_pool.rb', line 466

def capacities
  [memory_mib, shm_mib, cpus]
end

#capacity_status ⇒ Object



457
458
459
460
461
462
# File 'lib/test-runner/resource_pool.rb', line 457

def capacity_status
  [
    format_capacity_source('memory', @memory_capacity),
    format_capacity_source('shm', @shm_capacity)
  ].join('; ')
end

#fits?(capacity, value) ⇒ Boolean (protected)

Returns:

  • (Boolean)


470
471
472
# File 'lib/test-runner/resource_pool.rb', line 470

def fits?(capacity, value)
  capacity.nil? || value <= capacity
end

#format_capacity_source(name, source) ⇒ Object (protected)



493
494
495
496
497
498
# File 'lib/test-runner/resource_pool.rb', line 493

def format_capacity_source(name, source)
  "#{name} assigned=#{format_optional_mib(source.detected)}, " \
    "available-at-start=#{format_optional_mib(source.initial_available)}, " \
    "reserve=#{format_mib(source.reserve)}, " \
    "limit=#{format_optional_mib(source.current_value)}"
end

#format_mib(value) ⇒ Object (protected)



487
488
489
490
491
# File 'lib/test-runner/resource_pool.rb', line 487

def format_mib(value)
  return "#{value} MiB" if value < 1024

  format('%.1f GiB', value / 1024.0)
end

#format_optional_mib(value) ⇒ Object (protected)



500
501
502
# File 'lib/test-runner/resource_pool.rb', line 500

def format_optional_mib(value)
  value.nil? ? 'unknown' : format_mib(value)
end

#format_used(capacity, value, unit: true) ⇒ Object (protected)



474
475
476
477
478
479
480
481
482
483
484
485
# File 'lib/test-runner/resource_pool.rb', line 474

def format_used(capacity, value, unit: true)
  if capacity.nil?
    formatted = unit ? format_mib(value) : value
    return "#{formatted}/unlimited"
  end

  if unit
    "#{format_mib(value)}/#{format_mib(capacity)}"
  else
    "#{value}/#{capacity}"
  end
end

#prebuild_memory_mib ⇒ Object



428
429
430
# File 'lib/test-runner/resource_pool.rb', line 428

def prebuild_memory_mib
  @memory_capacity ? @memory_capacity.preparation_capacity : memory_mib
end

#prebuild_memory_status ⇒ Object



432
433
434
435
436
437
438
439
# File 'lib/test-runner/resource_pool.rb', line 432

def prebuild_memory_status
  return "explicit pool capacity=#{memory_mib.inspect} MiB" unless @memory_capacity

  "detected assigned=#{@memory_capacity.detected.inspect} MiB, " \
    "available-at-start=#{@memory_capacity.initial_available.inspect} MiB, " \
    "configured maximum=#{@memory_capacity.max_value.inspect} MiB, " \
    "reserve=#{@memory_capacity.reserve} MiB"
end

#refresh_capacity ⇒ Object



412
413
414
415
416
417
418
419
420
# File 'lib/test-runner/resource_pool.rb', line 412

def refresh_capacity
  previous = capacities

  @memory_mib = @memory_capacity.current if @memory_capacity
  @shm_mib = @shm_capacity.current if @shm_capacity
  @cpus = @cpu_capacity.current if @cpu_capacity

  previous != capacities
end

#release(resources) ⇒ Object



446
447
448
449
# File 'lib/test-runner/resource_pool.rb', line 446

def release(resources)
  @used -= resources
  @running -= 1
end

#reserve(resources) ⇒ Object



441
442
443
444
# File 'lib/test-runner/resource_pool.rb', line 441

def reserve(resources)
  @used += resources
  @running += 1
end

#status ⇒ Object



451
452
453
454
455
# File 'lib/test-runner/resource_pool.rb', line 451

def status
  "memory=#{format_used(memory_mib, used.memory_mib)}, " \
    "shm=#{format_used(shm_mib, used.shm_mib)}, " \
    "cpus=#{format_used(cpus, used.cpus, unit: false)}, running=#{running}"
end