diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..5ace4600a --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8755f249..bb118f400 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,7 @@ concurrency: jobs: build: runs-on: ubuntu-latest + timeout-minutes: 10 strategy: fail-fast: false @@ -21,7 +22,7 @@ jobs: name: "Tests: Ruby ${{ matrix.ruby }}" steps: - name: Clone Repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Ruby ${{ matrix.ruby }} uses: ruby/setup-ruby@v1 with: @@ -33,14 +34,15 @@ jobs: isolated: name: "Test isolated" runs-on: ubuntu-latest + timeout-minutes: 10 strategy: fail-fast: false matrix: - ruby: [ 2.3, 3.2 ] # oldest and latest CRuby + ruby: [ 2.3, ruby ] # oldest and latest CRuby env: RUBYOPT: '-w' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: ruby/setup-ruby@v1 with: ruby-version: ${{ matrix.ruby }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 34dab01fd..780f0741c 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -18,7 +18,7 @@ jobs: env: BUNDLE_WITH: "documentation" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - uses: ruby/setup-ruby@v1 @@ -29,7 +29,7 @@ jobs: - run: ruby support/generate_docs.rb - name: Upload artifact - uses: actions/upload-pages-artifact@v1 + uses: actions/upload-pages-artifact@v3 with: path: docs @@ -45,4 +45,4 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v1 + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/experimental.yml b/.github/workflows/experimental.yml index 6e469a4a1..28048747c 100644 --- a/.github/workflows/experimental.yml +++ b/.github/workflows/experimental.yml @@ -3,12 +3,11 @@ on: schedule: - cron: '0 0 * * *' # Runs every day at midnight workflow_dispatch: - branches: [ master ] jobs: build: runs-on: ubuntu-latest - continue-on-error: true + timeout-minutes: 10 strategy: matrix: @@ -22,7 +21,7 @@ jobs: name: "Tests: Experimental Ruby ${{ matrix.ruby }}" steps: - name: Clone Repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Ruby ${{ matrix.ruby }} uses: ruby/setup-ruby@v1 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index a9426af24..b9cc412e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ ## Current +## Release v1.3.1 (29 May 2024) + +* Release 1.3.0 was broken when pushed to RubyGems. 1.3.1 is a packaging fix. + +## Release v1.3.0 (28 May 2024) + +* (#1042) Align Java Executor Service behavior for `shuttingdown?`, `shutdown?` +* (#1038) Add `Concurrent.usable_processor_count` that is cgroups aware. + ## Release v1.2.3 (16 Jan 2024) * See [the GitHub release](https://github.com/ruby-concurrency/concurrent-ruby/releases/tag/v1.2.3) for details. diff --git a/docs-source/signpost.md b/docs-source/signpost.md index 8f9bb7b1a..161812a8f 100644 --- a/docs-source/signpost.md +++ b/docs-source/signpost.md @@ -3,7 +3,7 @@ Pick a `concurrent-ruby` version: * [master](./master/index.html) -* [1.2.0 with edge 0.7.0](./1.2.3/index.html) +* [1.3.1 with edge 0.7.0](./1.3.1/index.html) * [1.1.10 with edge 0.6.0](./1.1.10/index.html) * [1.1.9 with edge 0.6.0](./1.1.9/index.html) * [1.1.8 with edge 0.6.0](./1.1.8/index.html) diff --git a/lib/concurrent-ruby/concurrent/executor/java_executor_service.rb b/lib/concurrent-ruby/concurrent/executor/java_executor_service.rb index 7c9ab178e..b2bc69a6e 100644 --- a/lib/concurrent-ruby/concurrent/executor/java_executor_service.rb +++ b/lib/concurrent-ruby/concurrent/executor/java_executor_service.rb @@ -57,15 +57,11 @@ def ns_running? end def ns_shuttingdown? - if @executor.respond_to? :isTerminating - @executor.isTerminating - else - false - end + @executor.isShutdown && !@executor.isTerminated end def ns_shutdown? - @executor.isShutdown || @executor.isTerminated + @executor.isTerminated end class Job diff --git a/lib/concurrent-ruby/concurrent/utility/processor_counter.rb b/lib/concurrent-ruby/concurrent/utility/processor_counter.rb index 986e2d523..e31808722 100644 --- a/lib/concurrent-ruby/concurrent/utility/processor_counter.rb +++ b/lib/concurrent-ruby/concurrent/utility/processor_counter.rb @@ -11,6 +11,7 @@ class ProcessorCounter def initialize @processor_count = Delay.new { compute_processor_count } @physical_processor_count = Delay.new { compute_physical_processor_count } + @cpu_quota = Delay.new { compute_cpu_quota } end def processor_count @@ -21,6 +22,25 @@ def physical_processor_count @physical_processor_count.value end + def available_processor_count + cpu_count = processor_count.to_f + quota = cpu_quota + + return cpu_count if quota.nil? + + # cgroup cpus quotas have no limits, so they can be set to higher than the + # real count of cores. + if quota > cpu_count + cpu_count + else + quota + end + end + + def cpu_quota + @cpu_quota.value + end + private def compute_processor_count @@ -60,6 +80,24 @@ def compute_physical_processor_count rescue return 1 end + + def compute_cpu_quota + if RbConfig::CONFIG["target_os"].include?("linux") + if File.exist?("/sys/fs/cgroup/cpu.max") + # cgroups v2: https://docs.kernel.org/admin-guide/cgroup-v2.html#cpu-interface-files + cpu_max = File.read("/sys/fs/cgroup/cpu.max") + return nil if cpu_max.start_with?("max ") # no limit + max, period = cpu_max.split.map(&:to_f) + max / period + elsif File.exist?("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us") + # cgroups v1: https://kernel.googlesource.com/pub/scm/linux/kernel/git/glommer/memcg/+/cpu_stat/Documentation/cgroups/cpu.txt + max = File.read("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us").to_i + return nil if max == 0 + period = File.read("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_period_us").to_f + max / period + end + end + end end end @@ -107,4 +145,31 @@ def self.processor_count def self.physical_processor_count processor_counter.physical_processor_count end + + # Number of processors cores available for process scheduling. + # Returns `nil` if there is no #cpu_quota, or a `Float` if the + # process is inside a cgroup with a dedicated CPU quota (typically Docker). + # + # For performance reasons the calculated value will be memoized on the first + # call. + # + # @return [nil, Float] number of available processors + def self.available_processor_count + processor_counter.available_processor_count + end + + # The maximum number of processors cores available for process scheduling. + # Returns `nil` if there is no enforced limit, or a `Float` if the + # process is inside a cgroup with a dedicated CPU quota (typically Docker). + # + # Note that nothing prevents setting a CPU quota higher than the actual number of + # cores on the system. + # + # For performance reasons the calculated value will be memoized on the first + # call. + # + # @return [nil, Float] Maximum number of available processors as set by a cgroup CPU quota, or nil if none set + def self.cpu_quota + processor_counter.cpu_quota + end end diff --git a/lib/concurrent-ruby/concurrent/version.rb b/lib/concurrent-ruby/concurrent/version.rb index 9a1c29223..afa6a4b04 100644 --- a/lib/concurrent-ruby/concurrent/version.rb +++ b/lib/concurrent-ruby/concurrent/version.rb @@ -1,3 +1,3 @@ module Concurrent - VERSION = '1.2.3' + VERSION = '1.3.1' end diff --git a/spec/concurrent/executor/executor_service_shared.rb b/spec/concurrent/executor/executor_service_shared.rb index e5ffa367c..7b87922d5 100644 --- a/spec/concurrent/executor/executor_service_shared.rb +++ b/spec/concurrent/executor/executor_service_shared.rb @@ -3,7 +3,7 @@ require 'concurrent/atomic/atomic_fixnum' require 'timeout' -RSpec.shared_examples :executor_service do +RSpec.shared_examples :executor_service do |immediate_type: false| after(:each) do subject.shutdown @@ -84,6 +84,48 @@ end end + context '#shuttingdown?' do + it 'returns false when the thread pool is running' do + expect(subject).not_to be_shuttingdown + end + + it 'returns true when the thread pool is shutting down' do + skip "will never be in shuttingdown? state" if immediate_type + + subject.post{ sleep(0.5) } + subject.shutdown + expect(subject).to be_shuttingdown + expect(subject.wait_for_termination(pool_termination_timeout)).to eq true + end + + it 'returns false when the thread pool is shutdown' do + subject.shutdown + expect(subject.wait_for_termination(pool_termination_timeout)).to eq true + expect(subject).not_to be_shuttingdown + end + end + + context '#shutdown?' do + it 'returns false when the thread pool is running' do + expect(subject).not_to be_shutdown + end + + it 'returns false when the thread pool is shutting down' do + skip "will never be in shuttingdown? state" if immediate_type + + subject.post{ sleep(0.5) } + subject.shutdown + expect(subject).not_to be_shutdown + expect(subject.wait_for_termination(pool_termination_timeout)).to eq true + end + + it 'returns true when the thread pool is shutdown' do + subject.shutdown + expect(subject.wait_for_termination(pool_termination_timeout)).to eq true + expect(subject).to be_shutdown + end + end + context '#shutdown' do it 'stops accepting new tasks' do diff --git a/spec/concurrent/executor/immediate_executor_spec.rb b/spec/concurrent/executor/immediate_executor_spec.rb index c53efd72e..53f2748ab 100644 --- a/spec/concurrent/executor/immediate_executor_spec.rb +++ b/spec/concurrent/executor/immediate_executor_spec.rb @@ -7,6 +7,6 @@ module Concurrent subject { ImmediateExecutor.new } - it_should_behave_like :executor_service + it_should_behave_like :executor_service, immediate_type: true end end diff --git a/spec/concurrent/executor/indirect_immediate_executor_spec.rb b/spec/concurrent/executor/indirect_immediate_executor_spec.rb index f40364f28..c3c043d73 100644 --- a/spec/concurrent/executor/indirect_immediate_executor_spec.rb +++ b/spec/concurrent/executor/indirect_immediate_executor_spec.rb @@ -7,7 +7,7 @@ module Concurrent subject { IndirectImmediateExecutor.new } - it_should_behave_like :executor_service + it_should_behave_like :executor_service, immediate_type: true it "runs its tasks synchronously" do start = Time.now diff --git a/spec/concurrent/executor/serialized_execution_spec.rb b/spec/concurrent/executor/serialized_execution_spec.rb index 5f0cb4172..11b9bc72c 100644 --- a/spec/concurrent/executor/serialized_execution_spec.rb +++ b/spec/concurrent/executor/serialized_execution_spec.rb @@ -8,6 +8,6 @@ module Concurrent subject { SerializedExecutionDelegator.new(ImmediateExecutor.new) } - it_should_behave_like :executor_service + it_should_behave_like :executor_service, immediate_type: true end end diff --git a/spec/concurrent/utility/processor_count_spec.rb b/spec/concurrent/utility/processor_count_spec.rb index 229125feb..fdc44b0ae 100644 --- a/spec/concurrent/utility/processor_count_spec.rb +++ b/spec/concurrent/utility/processor_count_spec.rb @@ -17,4 +17,79 @@ module Concurrent expect(Concurrent::physical_processor_count).to be >= 1 end end + + RSpec.describe '#cpu_quota' do + + let(:counter) { Concurrent::Utility::ProcessorCounter.new } + + it 'returns #compute_cpu_quota' do + expect(Concurrent::cpu_quota).to be == counter.cpu_quota + end + + it 'returns nil if no quota is detected' do + if RbConfig::CONFIG["target_os"].include?("linux") + expect(File).to receive(:exist?).twice.and_return(nil) # Checks for cgroups V1 and V2 + end + expect(counter.cpu_quota).to be_nil + end + + it 'returns nil if cgroups v2 sets no limit' do + expect(RbConfig::CONFIG).to receive(:[]).with("target_os").and_return("linux") + expect(File).to receive(:exist?).with("/sys/fs/cgroup/cpu.max").and_return(true) + expect(File).to receive(:read).with("/sys/fs/cgroup/cpu.max").and_return("max 100000\n") + expect(counter.cpu_quota).to be_nil + end + + it 'returns a float if cgroups v2 sets a limit' do + expect(RbConfig::CONFIG).to receive(:[]).with("target_os").and_return("linux") + expect(File).to receive(:exist?).with("/sys/fs/cgroup/cpu.max").and_return(true) + expect(File).to receive(:read).with("/sys/fs/cgroup/cpu.max").and_return("150000 100000\n") + expect(counter.cpu_quota).to be == 1.5 + end + + it 'returns nil if cgroups v1 sets no limit' do + expect(RbConfig::CONFIG).to receive(:[]).with("target_os").and_return("linux") + expect(File).to receive(:exist?).with("/sys/fs/cgroup/cpu.max").and_return(false) + expect(File).to receive(:exist?).with("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us").and_return(true) + + expect(File).to receive(:read).with("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us").and_return("max\n") + expect(counter.cpu_quota).to be_nil + end + + it 'returns a float if cgroups v1 sets a limit' do + expect(RbConfig::CONFIG).to receive(:[]).with("target_os").and_return("linux") + expect(File).to receive(:exist?).with("/sys/fs/cgroup/cpu.max").and_return(false) + expect(File).to receive(:exist?).with("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us").and_return(true) + + expect(File).to receive(:read).with("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us").and_return("150000\n") + expect(File).to receive(:read).with("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_period_us").and_return("100000\n") + expect(counter.cpu_quota).to be == 1.5 + end + + end + + RSpec.describe '#available_processor_count' do + + it 'returns #processor_count if #cpu_quota is nil' do + expect(Concurrent::processor_counter).to receive(:cpu_quota).and_return(nil) + available_processor_count = Concurrent.available_processor_count + expect(available_processor_count).to be == Concurrent::processor_count + expect(available_processor_count).to be_a Float + end + + it 'returns #processor_count if #cpu_quota is higher' do + expect(Concurrent::processor_counter).to receive(:cpu_quota).and_return(Concurrent::processor_count.to_f * 2) + available_processor_count = Concurrent.available_processor_count + expect(available_processor_count).to be == Concurrent::processor_count + expect(available_processor_count).to be_a Float + end + + it 'returns #cpu_quota if #cpu_quota is lower than #processor_count' do + expect(Concurrent::processor_counter).to receive(:cpu_quota).and_return(Concurrent::processor_count.to_f / 2) + available_processor_count = Concurrent.available_processor_count + expect(available_processor_count).to be == Concurrent::processor_count.to_f / 2 + expect(available_processor_count).to be_a Float + end + + end end