From 7f9b2705ff552ae2427ac04fc8784d8ce656a2e1 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Wed, 3 Jul 2024 16:14:35 +0000 Subject: [PATCH 01/21] Use `bundle install --target-rbconfig` to install gems for Wasm target --- Rakefile | 6 +- bin/setup | 4 +- ext/ruby_wasm/src/lib.rs | 3 + lib/ruby_wasm/build/product/crossruby.rb | 6 +- lib/ruby_wasm/cli.rb | 2 +- lib/ruby_wasm/packager.rb | 2 +- lib/ruby_wasm/packager/core.rb | 58 +++++++++++-------- lib/ruby_wasm/packager/file_system.rb | 1 - .../ruby-3.2-wasm-wasi/.gitignore | 1 + .../Gemfile | 0 .../Gemfile.lock | 2 +- .../ruby-3.3-wasm-wasi/.gitignore | 1 + .../npm-packages/ruby-3.3-wasm-wasi/Gemfile | 8 +++ .../ruby-3.3-wasm-wasi/Gemfile.lock | 29 ++++++++++ .../ruby-head-wasm-wasi/.gitignore | 1 + .../npm-packages/ruby-head-wasm-wasi/Gemfile | 8 +++ .../ruby-head-wasm-wasi/Gemfile.lock | 29 ++++++++++ sig/ruby_wasm/build.rbs | 1 + 18 files changed, 129 insertions(+), 33 deletions(-) rename packages/npm-packages/{ruby-wasm-wasi => ruby-3.2-wasm-wasi}/Gemfile (100%) rename packages/npm-packages/{ruby-wasm-wasi => ruby-3.2-wasm-wasi}/Gemfile.lock (97%) create mode 100644 packages/npm-packages/ruby-3.3-wasm-wasi/Gemfile create mode 100644 packages/npm-packages/ruby-3.3-wasm-wasi/Gemfile.lock create mode 100644 packages/npm-packages/ruby-head-wasm-wasi/Gemfile create mode 100644 packages/npm-packages/ruby-head-wasm-wasi/Gemfile.lock diff --git a/Rakefile b/Rakefile index d10148a861..05ff059282 100644 --- a/Rakefile +++ b/Rakefile @@ -28,20 +28,20 @@ NPM_PACKAGES = [ { name: "ruby-head-wasm-wasi", ruby_version: "head", - gemfile: "packages/npm-packages/ruby-wasm-wasi/Gemfile", + gemfile: "packages/npm-packages/ruby-head-wasm-wasi/Gemfile", target: "wasm32-unknown-wasip1", enable_component_model: true, }, { name: "ruby-3.3-wasm-wasi", ruby_version: "3.3", - gemfile: "packages/npm-packages/ruby-wasm-wasi/Gemfile", + gemfile: "packages/npm-packages/ruby-3.3-wasm-wasi/Gemfile", target: "wasm32-unknown-wasip1" }, { name: "ruby-3.2-wasm-wasi", ruby_version: "3.2", - gemfile: "packages/npm-packages/ruby-wasm-wasi/Gemfile", + gemfile: "packages/npm-packages/ruby-3.2-wasm-wasi/Gemfile", target: "wasm32-unknown-wasip1" }, { name: "ruby-wasm-wasi", target: "wasm32-unknown-wasip1" } diff --git a/bin/setup b/bin/setup index 4e02eebb45..8bb60140ce 100755 --- a/bin/setup +++ b/bin/setup @@ -6,7 +6,9 @@ set -vx root="$(cd "$(dirname "$0")/.." && pwd)" env BUNDLE_GEMFILE="$root/Gemfile" bundle install -env BUNDLE_GEMFILE="$root/packages/npm-packages/ruby-wasm-wasi/Gemfile" bundle install +for gemfile in $root/packages/npm-packages/*/Gemfile; do + env BUNDLE_GEMFILE="$gemfile" bundle install +done # Build vendored jco if Rust toolchain is available and submodule is checked out if command -v rustc && [ -f vendor/jco/package.json ]; then diff --git a/ext/ruby_wasm/src/lib.rs b/ext/ruby_wasm/src/lib.rs index c282ed321d..4ccdfa8836 100644 --- a/ext/ruby_wasm/src/lib.rs +++ b/ext/ruby_wasm/src/lib.rs @@ -252,6 +252,9 @@ impl WasiVirt { // Disable sockets for now since `sockets/ip-name-lookup` is not // supported by @bytecodealliance/preview2-shim yet virt.sockets(false); + // Disable http for now since `http` is not supported by + // wasmtime yet + virt.http(false); Ok(()) }) } diff --git a/lib/ruby_wasm/build/product/crossruby.rb b/lib/ruby_wasm/build/product/crossruby.rb index cfd5786d7a..4f705cdfd2 100644 --- a/lib/ruby_wasm/build/product/crossruby.rb +++ b/lib/ruby_wasm/build/product/crossruby.rb @@ -71,7 +71,7 @@ def do_extconf(executor, crossruby) return end objdir = product_build_dir crossruby - rbconfig_rb = Dir.glob(File.join(crossruby.dest_dir, "usr/local/lib/ruby/*/wasm32-wasi/rbconfig.rb")).first + rbconfig_rb = crossruby.rbconfig_rb raise "rbconfig.rb not found" unless rbconfig_rb extconf_args = [ "-C", objdir, @@ -301,6 +301,10 @@ def extinit_c_erb File.expand_path("../crossruby/extinit.c.erb", __FILE__) end + def rbconfig_rb + Dir.glob(File.join(dest_dir, "usr/local/lib/ruby/*/wasm32-wasi/rbconfig.rb")).first + end + def baseruby_path File.join(@baseruby.install_dir, "bin/ruby") end diff --git a/lib/ruby_wasm/cli.rb b/lib/ruby_wasm/cli.rb index b0575494ac..bc59b59e3c 100644 --- a/lib/ruby_wasm/cli.rb +++ b/lib/ruby_wasm/cli.rb @@ -321,7 +321,7 @@ def derive_packager(options) Bundler.ui.level = old_level end end - RubyWasm.logger.info "Using Gemfile: #{definition.gemfiles}" if definition + RubyWasm.logger.info "Using Gemfile: #{definition.gemfiles.map(&:to_s).join(", ")}" if definition RubyWasm::Packager.new( root, build_config(options), definition, features: RubyWasm::FeatureSet.derive_from_env diff --git a/lib/ruby_wasm/packager.rb b/lib/ruby_wasm/packager.rb index b54e70ba19..f1a395303c 100644 --- a/lib/ruby_wasm/packager.rb +++ b/lib/ruby_wasm/packager.rb @@ -34,7 +34,7 @@ def package(executor, dest_dir, options) ruby_core.build_gem_exts(executor, fs.bundle_dir) - fs.package_gems + fs.package_gems unless features.support_component_model? fs.remove_non_runtime_files(executor) if options[:stdlib] options[:without_stdlib_components].each do |component| diff --git a/lib/ruby_wasm/packager/core.rb b/lib/ruby_wasm/packager/core.rb index 6f36eec495..1cb1deaa65 100644 --- a/lib/ruby_wasm/packager/core.rb +++ b/lib/ruby_wasm/packager/core.rb @@ -187,31 +187,41 @@ def _build_gem_exts(executor, build, gem_home) baseruby.build(executor) end - exts = specs_with_extensions.flat_map do |spec, exts| - exts.map do |ext| - ext_feature = File.dirname(ext) # e.g. "ext/cgi/escape" - ext_srcdir = File.join(spec.full_gem_path, ext_feature) - ext_relative_path = File.join(spec.full_name, ext_feature) - prod = RubyWasm::CrossRubyExtProduct.new( - ext_srcdir, - build.toolchain, - features: @packager.features, - ext_relative_path: ext_relative_path - ) - [prod, spec] - end - end + crossruby = build.crossruby + rbconfig_rb = crossruby.rbconfig_rb - exts.each do |prod, spec| - libdir = File.join(gem_home, "gems", spec.full_name, spec.raw_require_paths.first) - extra_mkargs = [ - "sitearchdir=#{libdir}", - "sitelibdir=#{libdir}", - ] - executor.begin_section prod.class, prod.name, "Building" - prod.build(executor, build.crossruby, extra_mkargs) - executor.end_section prod.class, prod.name - end + options = @packager.full_build_options + target_triplet = options[:target] + + local_path = File.join("bundle", target_triplet) + env = { + "BUNDLE_APP_CONFIG" => File.join(".bundle", target_triplet), + "BUNDLE_PATH" => local_path, + "BUNDLE_WITHOUT" => "build", + # FIXME: BUNDLE_PATH is set as a installation destination here, but + # it is also used as a source of gems to be loaded by RubyGems itself. + # RubyGems loads "psych" gem and if Gemfile includes "psych" gem, + # RubyGems tries to load "psych" gem from BUNDLE_PATH at the second + # time of "bundle install" command. But the extension of "psych" gem + # under BUNDLE_PATH is built for Wasm target, not for host platform, + # so it fails to load the extension. + # + # Thus we preload psych from the default LOAD_PATH here to avoid + # loading Wasm version of psych.so via `Kernel#require` patched by + # RubyGems. + "RUBYOPT" => "-rpsych", + } + + args = [ + File.join(baseruby.install_dir, "bin", "bundle"), + "install", + "--standalone", + "--target-rbconfig", + rbconfig_rb, + ] + + executor.system(*args, env: env) + executor.cp_r(local_path, gem_home) end def cache_key(digest) diff --git a/lib/ruby_wasm/packager/file_system.rb b/lib/ruby_wasm/packager/file_system.rb index bb71d1cb6c..6d11902021 100644 --- a/lib/ruby_wasm/packager/file_system.rb +++ b/lib/ruby_wasm/packager/file_system.rb @@ -44,7 +44,6 @@ def remove_stdlib_component(executor, component) when "enc" # Remove all encodings except for encdb.so and transdb.so enc_dir = File.join(@ruby_root, "lib", "ruby", ruby_version, "wasm32-wasi", "enc") - puts File.join(enc_dir, "**/*.so") Dir.glob(File.join(enc_dir, "**/*.so")).each do |entry| next if entry.end_with?("encdb.so", "transdb.so") RubyWasm.logger.debug "Removing stdlib encoding: #{entry}" diff --git a/packages/npm-packages/ruby-3.2-wasm-wasi/.gitignore b/packages/npm-packages/ruby-3.2-wasm-wasi/.gitignore index aa1ec1ea06..fad2dcd07e 100644 --- a/packages/npm-packages/ruby-3.2-wasm-wasi/.gitignore +++ b/packages/npm-packages/ruby-3.2-wasm-wasi/.gitignore @@ -1 +1,2 @@ *.tgz +/bundle diff --git a/packages/npm-packages/ruby-wasm-wasi/Gemfile b/packages/npm-packages/ruby-3.2-wasm-wasi/Gemfile similarity index 100% rename from packages/npm-packages/ruby-wasm-wasi/Gemfile rename to packages/npm-packages/ruby-3.2-wasm-wasi/Gemfile diff --git a/packages/npm-packages/ruby-wasm-wasi/Gemfile.lock b/packages/npm-packages/ruby-3.2-wasm-wasi/Gemfile.lock similarity index 97% rename from packages/npm-packages/ruby-wasm-wasi/Gemfile.lock rename to packages/npm-packages/ruby-3.2-wasm-wasi/Gemfile.lock index bd0a025962..b9d2b12b4e 100644 --- a/packages/npm-packages/ruby-wasm-wasi/Gemfile.lock +++ b/packages/npm-packages/ruby-3.2-wasm-wasi/Gemfile.lock @@ -26,4 +26,4 @@ DEPENDENCIES test-unit BUNDLED WITH - 2.5.3 + 2.5.9 diff --git a/packages/npm-packages/ruby-3.3-wasm-wasi/.gitignore b/packages/npm-packages/ruby-3.3-wasm-wasi/.gitignore index aa1ec1ea06..fad2dcd07e 100644 --- a/packages/npm-packages/ruby-3.3-wasm-wasi/.gitignore +++ b/packages/npm-packages/ruby-3.3-wasm-wasi/.gitignore @@ -1 +1,2 @@ *.tgz +/bundle diff --git a/packages/npm-packages/ruby-3.3-wasm-wasi/Gemfile b/packages/npm-packages/ruby-3.3-wasm-wasi/Gemfile new file mode 100644 index 0000000000..4132b8115c --- /dev/null +++ b/packages/npm-packages/ruby-3.3-wasm-wasi/Gemfile @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "js", path: "../../gems/js" +gem "ruby_wasm", path: "../../../" +gem "power_assert" +gem "test-unit" diff --git a/packages/npm-packages/ruby-3.3-wasm-wasi/Gemfile.lock b/packages/npm-packages/ruby-3.3-wasm-wasi/Gemfile.lock new file mode 100644 index 0000000000..b9d2b12b4e --- /dev/null +++ b/packages/npm-packages/ruby-3.3-wasm-wasi/Gemfile.lock @@ -0,0 +1,29 @@ +PATH + remote: ../../.. + specs: + ruby_wasm (2.6.2) + +PATH + remote: ../../gems/js + specs: + js (2.6.2) + +GEM + remote: https://rubygems.org/ + specs: + power_assert (2.0.3) + test-unit (3.6.2) + power_assert + +PLATFORMS + ruby + x86_64-linux + +DEPENDENCIES + js! + power_assert + ruby_wasm! + test-unit + +BUNDLED WITH + 2.5.9 diff --git a/packages/npm-packages/ruby-head-wasm-wasi/.gitignore b/packages/npm-packages/ruby-head-wasm-wasi/.gitignore index 5c5e4b4641..a2ddcfabd1 100644 --- a/packages/npm-packages/ruby-head-wasm-wasi/.gitignore +++ b/packages/npm-packages/ruby-head-wasm-wasi/.gitignore @@ -1,2 +1,3 @@ *.tgz /tmp +/bundle diff --git a/packages/npm-packages/ruby-head-wasm-wasi/Gemfile b/packages/npm-packages/ruby-head-wasm-wasi/Gemfile new file mode 100644 index 0000000000..4132b8115c --- /dev/null +++ b/packages/npm-packages/ruby-head-wasm-wasi/Gemfile @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "js", path: "../../gems/js" +gem "ruby_wasm", path: "../../../" +gem "power_assert" +gem "test-unit" diff --git a/packages/npm-packages/ruby-head-wasm-wasi/Gemfile.lock b/packages/npm-packages/ruby-head-wasm-wasi/Gemfile.lock new file mode 100644 index 0000000000..d77f2106cd --- /dev/null +++ b/packages/npm-packages/ruby-head-wasm-wasi/Gemfile.lock @@ -0,0 +1,29 @@ +PATH + remote: ../../.. + specs: + ruby_wasm (2.6.2) + +PATH + remote: ../../gems/js + specs: + js (2.6.2) + +GEM + remote: https://rubygems.org/ + specs: + power_assert (2.0.3) + test-unit (3.6.2) + power_assert + +PLATFORMS + ruby + wasm32-wasi + +DEPENDENCIES + js! + power_assert + ruby_wasm! + test-unit + +BUNDLED WITH + 2.6.0.dev diff --git a/sig/ruby_wasm/build.rbs b/sig/ruby_wasm/build.rbs index 4c6376471e..47a381c678 100644 --- a/sig/ruby_wasm/build.rbs +++ b/sig/ruby_wasm/build.rbs @@ -219,6 +219,7 @@ module RubyWasm def extinit_c_erb: -> String def baseruby_path: -> String def configure_args: (String build_triple, Toolchain toolchain) -> Array[String] + def rbconfig_rb: -> String? end class WitBindgen From 2c29240451bfbe8f178592469058fe1997303320 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Thu, 4 Jul 2024 03:25:47 +0000 Subject: [PATCH 02/21] Fix Gemfile.lock issue --- lib/ruby_wasm/cli.rb | 7 +++++-- lib/ruby_wasm/packager/core.rb | 27 ++++++++++++++++----------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/lib/ruby_wasm/cli.rb b/lib/ruby_wasm/cli.rb index bc59b59e3c..7df2ae58fe 100644 --- a/lib/ruby_wasm/cli.rb +++ b/lib/ruby_wasm/cli.rb @@ -310,7 +310,10 @@ def self.bundled_patches_path def derive_packager(options) __skip__ = definition = nil - __skip__ = if defined?(Bundler) && !options[:disable_gems] + features = RubyWasm::FeatureSet.derive_from_env + # The head ruby & dynamic linking uses "bundle" command to build gems instead of in-process integration. + use_in_process_gem_building = !(options[:ruby_version] == "head" && features.support_dynamic_linking?) + __skip__ = if defined?(Bundler) && !options[:disable_gems] && use_in_process_gem_building begin # Silence Bundler UI if --print-ruby-cache-key is specified not to bother the JSON output. level = options[:print_ruby_cache_key] ? :silent : Bundler.ui.level @@ -324,7 +327,7 @@ def derive_packager(options) RubyWasm.logger.info "Using Gemfile: #{definition.gemfiles.map(&:to_s).join(", ")}" if definition RubyWasm::Packager.new( root, build_config(options), definition, - features: RubyWasm::FeatureSet.derive_from_env + features: features, ) end diff --git a/lib/ruby_wasm/packager/core.rb b/lib/ruby_wasm/packager/core.rb index 1cb1deaa65..1b7a08a265 100644 --- a/lib/ruby_wasm/packager/core.rb +++ b/lib/ruby_wasm/packager/core.rb @@ -19,7 +19,6 @@ def build(executor, options) def build_strategy @build_strategy ||= begin - has_exts = @packager.specs.any? { |spec| spec.extensions.any? } if @packager.features.support_dynamic_linking? DynamicLinking.new(@packager) else @@ -59,14 +58,6 @@ def specs_with_extensions end end - def wasi_exec_model - # TODO: Detect WASI exec-model from binary exports (_start or _initialize) - use_js_gem = @packager.specs.any? do |spec| - spec.name == "js" - end - use_js_gem ? "reactor" : "command" - end - def with_unbundled_env(&block) __skip__ = if defined?(Bundler) Bundler.with_unbundled_env(&block) @@ -138,12 +129,16 @@ def _link_gem_exts(executor, build, ruby_root, gem_home, module_bytes) wasi_sdk_path = toolchain.wasi_sdk_path libraries << File.join(wasi_sdk_path, "share/wasi-sysroot/lib/wasm32-wasi", lib) end - wasi_adapter = RubyWasm::Packager::ComponentAdapter.wasi_snapshot_preview1(wasi_exec_model) - adapters = [wasi_adapter] dl_openable_libs = [] dl_openable_libs << [File.dirname(ruby_root), Dir.glob(File.join(ruby_root, "lib", "ruby", "**", "*.so"))] dl_openable_libs << [gem_home, Dir.glob(File.join(gem_home, "**", "*.so"))] + has_js_so = dl_openable_libs.any? do |root, libs| + libs.any? { |lib| lib.end_with?("/js.so") } + end + wasi_adapter = RubyWasm::Packager::ComponentAdapter.wasi_snapshot_preview1(has_js_so ? "reactor" : "command") + adapters = [wasi_adapter] + linker = RubyWasmExt::ComponentLink.new linker.use_built_in_libdl(true) linker.stub_missing_functions(false) @@ -198,6 +193,8 @@ def _build_gem_exts(executor, build, gem_home) "BUNDLE_APP_CONFIG" => File.join(".bundle", target_triplet), "BUNDLE_PATH" => local_path, "BUNDLE_WITHOUT" => "build", + # Do not auto-switch bundler version by Gemfile.lock + "BUNDLE_VERSION" => "system", # FIXME: BUNDLE_PATH is set as a installation destination here, but # it is also used as a source of gems to be loaded by RubyGems itself. # RubyGems loads "psych" gem and if Gemfile includes "psych" gem, @@ -347,6 +344,14 @@ def build_gem_exts(executor, gem_home) # No-op because we already built extensions as part of the Ruby build end + def wasi_exec_model + # TODO: Detect WASI exec-model from binary exports (_start or _initialize) + use_js_gem = @packager.specs.any? do |spec| + spec.name == "js" + end + use_js_gem ? "reactor" : "command" + end + def link_gem_exts(executor, ruby_root, gem_home, module_bytes) return module_bytes unless @packager.features.support_component_model? From 899fb60ac7bfc5fa0ac6cf04752a09511258dee2 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Wed, 17 Jul 2024 02:32:18 +0000 Subject: [PATCH 03/21] Try to fix Gemfile.lock auto-self-downgrade issue --- bin/setup | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/bin/setup b/bin/setup index 8bb60140ce..8fe847b529 100755 --- a/bin/setup +++ b/bin/setup @@ -7,7 +7,21 @@ root="$(cd "$(dirname "$0")/.." && pwd)" env BUNDLE_GEMFILE="$root/Gemfile" bundle install for gemfile in $root/packages/npm-packages/*/Gemfile; do + # FIXME: This is a workaround for the following issue: + # 1. `bundle install` does not support auto-self-upgrade for pre-release bundler versions suffixed with ".dev" + # 2. ruby-head-wasm-wasi's component build depends on 2.6.0.dev, which added --target-rbconfig support + # 3. If the "bundle" command here is earlier than 2.6.0.dev, "bundle install" does *not* self-upgrade to + # the specified pre-release version, and it overwrites the Gemfile.lock with the earlier command version. + # 4. Overwritten Gemfile.lock with the earlier bundler version causes auto-self-downgrade when running + # "bundle install --target-rbconfig" inside "rbwasm build" command, then it fails because the earlier + # bundler version does not support --target-rbconfig. + # + # Thus, we temporarily discard the Gemfile.lock changes here to prevent the auto-self-downgrade issue. + # This workaround should be removed once we drop static Ruby builds from ruby-head-wasm-wasi or + # a stable bundler version is released with --target-rbconfig support. + cp "$gemfile.lock" "$gemfile.lock.orig" env BUNDLE_GEMFILE="$gemfile" bundle install + mv "$gemfile.lock.orig" "$gemfile.lock" done # Build vendored jco if Rust toolchain is available and submodule is checked out From c4c9585aebe02b7a468a03ab3aac0823c76c0295 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Wed, 17 Jul 2024 03:42:26 +0000 Subject: [PATCH 04/21] Upload artifact before running tests This helps investigate test failures --- .github/workflows/build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9575aed6e7..12668e4641 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -142,6 +142,11 @@ jobs: if: ${{ inputs.prerel_name != '' && matrix.entry.prerelease != '' }} - name: rake ${{ matrix.entry.task }} run: ./build-exec rake --verbose ${{ matrix.entry.task }} + - uses: actions/upload-artifact@v4 + if: ${{ matrix.entry.artifact }} + with: + name: ${{ matrix.entry.artifact_name }} + path: ${{ matrix.entry.artifact }} - uses: ruby/setup-ruby@v1 if: ${{ matrix.entry.test != '' }} with: @@ -152,11 +157,6 @@ jobs: bundle install --with=check --without=development rake ${{ matrix.entry.test }} if: ${{ matrix.entry.test != '' }} - - uses: actions/upload-artifact@v4 - if: ${{ matrix.entry.artifact }} - with: - name: ${{ matrix.entry.artifact_name }} - path: ${{ matrix.entry.artifact }} release-artifacts: needs: [rake-tasks] From d4ea1cf6e21f6ee61a0fd03a3f60c7d3b0320352 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Wed, 17 Jul 2024 03:52:23 +0000 Subject: [PATCH 05/21] Make `--enable-component-model` the default for `js` gem In this way, we can avoid the need to specify `BUNDLE_BUILD__JS=--enable-component-model` when running `bundle install` for the `js` gem. When running `bundle install`, it's usually dynamic-linking enabled, so it makes sense to enable the component model by default. --- lib/ruby_wasm/build/product/crossruby.rb | 4 ++-- packages/gems/js/ext/js/extconf.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/ruby_wasm/build/product/crossruby.rb b/lib/ruby_wasm/build/product/crossruby.rb index 4f705cdfd2..8444599a12 100644 --- a/lib/ruby_wasm/build/product/crossruby.rb +++ b/lib/ruby_wasm/build/product/crossruby.rb @@ -78,7 +78,7 @@ def do_extconf(executor, crossruby) "#{@srcdir}/extconf.rb", "--target-rbconfig=#{rbconfig_rb}", ] - extconf_args << "--enable-component-model" if @features.support_component_model? + extconf_args << "--disable-component-model" unless @features.support_component_model? executor.system crossruby.baseruby_path, *extconf_args end @@ -111,7 +111,7 @@ def do_legacy_extconf(executor, crossruby) "-I#{crossruby.build_dir}", "--", ] - extconf_args << "--enable-component-model" if @features.support_component_model? + extconf_args << "--disable-component-model" unless @features.support_component_model? # Clear RUBYOPT to avoid loading unrelated bundle setup executor.system crossruby.baseruby_path, *extconf_args, diff --git a/packages/gems/js/ext/js/extconf.rb b/packages/gems/js/ext/js/extconf.rb index 86ff76a22e..380073c572 100644 --- a/packages/gems/js/ext/js/extconf.rb +++ b/packages/gems/js/ext/js/extconf.rb @@ -11,7 +11,7 @@ $objs = %w[js-core.o witapi-core.o] -use_component_model = enable_config("component-model", false) +use_component_model = enable_config("component-model", true) $stderr.print "Building with component model: " $stderr.puts use_component_model ? "\e[1;32myes\e[0m" : "\e[1;31mno\e[0m" if use_component_model From 805b6c63e055a308bee4debb89bb1135f231d07b Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Wed, 17 Jul 2024 05:41:42 +0000 Subject: [PATCH 06/21] Use vendor/cache for js gem to build extension as a part of "bundle install" --- packages/gems/js/lib/js/version.rb | 2 +- packages/npm-packages/ruby-head-wasm-wasi/.gitignore | 1 + packages/npm-packages/ruby-head-wasm-wasi/Gemfile | 11 +++++++++-- rakelib/packaging.rake | 6 ++++++ 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/packages/gems/js/lib/js/version.rb b/packages/gems/js/lib/js/version.rb index 8affa242c9..80cda8b965 100644 --- a/packages/gems/js/lib/js/version.rb +++ b/packages/gems/js/lib/js/version.rb @@ -1,3 +1,3 @@ module JS - VERSION = "2.6.2" + VERSION = "2.6.2.dev" end diff --git a/packages/npm-packages/ruby-head-wasm-wasi/.gitignore b/packages/npm-packages/ruby-head-wasm-wasi/.gitignore index a2ddcfabd1..cbb80a5cd4 100644 --- a/packages/npm-packages/ruby-head-wasm-wasi/.gitignore +++ b/packages/npm-packages/ruby-head-wasm-wasi/.gitignore @@ -1,3 +1,4 @@ *.tgz /tmp /bundle +/vendor diff --git a/packages/npm-packages/ruby-head-wasm-wasi/Gemfile b/packages/npm-packages/ruby-head-wasm-wasi/Gemfile index 4132b8115c..5e83ff0b18 100644 --- a/packages/npm-packages/ruby-head-wasm-wasi/Gemfile +++ b/packages/npm-packages/ruby-head-wasm-wasi/Gemfile @@ -2,7 +2,14 @@ source "https://rubygems.org" -gem "js", path: "../../gems/js" -gem "ruby_wasm", path: "../../../" +# We build ./vendor/cache/js-local.gem just before evaluating this Gemfile +# so that Bundler builds extensions even from the local gem. (gem extensions +# from "path:" gems are not built by Bundler.) +# Thus even we specify version of "js" gem here, it should always installed +# from the ./vendor/cache/js-local.gem, not from rubygems.org. To achieve this, +# we always use non-exist version during development. +require_relative "../../gems/js/lib/js/version.rb" +gem "js", JS::VERSION +gem "ruby_wasm", path: "../../../", group: [:build] gem "power_assert" gem "test-unit" diff --git a/rakelib/packaging.rake b/rakelib/packaging.rake index a43b87fc38..aed794650f 100644 --- a/rakelib/packaging.rake +++ b/rakelib/packaging.rake @@ -67,6 +67,12 @@ namespace :npm do mkdir_p dist_dir if pkg[:target].start_with?("wasm32-unknown-wasi") Dir.chdir(cwd || base_dir) do + vendor_cache_dir = File.join(pkg_dir, "vendor", "cache") + mkdir_p vendor_cache_dir + require_relative "../packages/gems/js/lib/js/version" + sh "gem", "-C", File.join(base_dir, "packages/gems/js"), "build", "-o", + File.join(pkg_dir, "vendor", "cache", "js-local.gem") + sh env, *build_command, "--no-stdlib", From d2847892b9c900e4d1936a1169351e55263a9e0e Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Wed, 17 Jul 2024 05:42:20 +0000 Subject: [PATCH 07/21] Update release instructions to include bump_dev_version task --- CONTRIBUTING.md | 1 + lib/ruby_wasm/version.rb | 2 +- rakelib/version.rake | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2ff94daa0a..d40a63e847 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -117,6 +117,7 @@ $ gh run download $ for pkg in cross-gem/pkg/ruby_wasm-*; do gem push $pkg; done $ gem build && gem push ruby_wasm-*.gem && rm ruby_wasm-*.gem $ (cd packages/gems/js/ && gem build && gem push js-*.gem && rm js-*.gem) +$ rake bump_dev_version ``` ## Release Channels diff --git a/lib/ruby_wasm/version.rb b/lib/ruby_wasm/version.rb index 1b211ff827..2ca5dd70f3 100644 --- a/lib/ruby_wasm/version.rb +++ b/lib/ruby_wasm/version.rb @@ -1,3 +1,3 @@ module RubyWasm - VERSION = "2.6.2" + VERSION = "2.6.2.dev" end diff --git a/rakelib/version.rake b/rakelib/version.rake index 546241f11f..a742b547d8 100644 --- a/rakelib/version.rake +++ b/rakelib/version.rake @@ -38,3 +38,17 @@ task :bump_version, %i[version] do |t, args| # Update Gemfile.lock sh "BUNDLE_GEMFILE=packages/npm-packages/ruby-wasm-wasi/Gemfile bundle install" end + +def bump_dev_version_rb(version_rb) + version_rb_content = File.read(version_rb) + version_rb_content.sub!(/VERSION = "(.+)"$/) do + dev_version = $1.end_with?(".dev") ? $1 : $1 + ".dev" + "VERSION = \"#{dev_version}\"" + end + File.write(version_rb, version_rb_content) +end + +task :bump_dev_version do + bump_dev_version_rb("lib/ruby_wasm/version.rb") + bump_dev_version_rb("packages/gems/js/lib/js/version.rb") +end From 449c3d6dfc5bbd1d8d35a9e235d4b048e605d98c Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Wed, 17 Jul 2024 05:49:48 +0000 Subject: [PATCH 08/21] Fix bin/setup not to install ruby-head-wasm-wasi's Gemfile The ruby-head-wasm-wasi's Gemfile uses vendor/cache to install js gem but the cache gem is not yet available at setup-time. And also with cache gem, the Gemfile is not needed to be installed at setup-time. So just skip the Gemfile installation here --- bin/setup | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/setup b/bin/setup index 8fe847b529..8fdd6f73ab 100755 --- a/bin/setup +++ b/bin/setup @@ -7,6 +7,11 @@ root="$(cd "$(dirname "$0")/.." && pwd)" env BUNDLE_GEMFILE="$root/Gemfile" bundle install for gemfile in $root/packages/npm-packages/*/Gemfile; do + # Skip ruby-head-wasm-wasi's Gemfile because it does not need to be installed here + if [[ "$gemfile" == *"/ruby-head-wasm-wasi/Gemfile" ]]; then + continue + fi + # FIXME: This is a workaround for the following issue: # 1. `bundle install` does not support auto-self-upgrade for pre-release bundler versions suffixed with ".dev" # 2. ruby-head-wasm-wasi's component build depends on 2.6.0.dev, which added --target-rbconfig support From eddd567b97aefa5c6e53dfe694c54045bccf48fe Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Wed, 17 Jul 2024 08:39:57 +0000 Subject: [PATCH 09/21] Create vendor gem cache before computing cache key for npm package --- rakelib/packaging.rake | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/rakelib/packaging.rake b/rakelib/packaging.rake index aed794650f..fa15f6f10f 100644 --- a/rakelib/packaging.rake +++ b/rakelib/packaging.rake @@ -25,6 +25,8 @@ def npm_pkg_build_command(pkg) end def npm_pkg_rubies_cache_key(pkg) + vendor_gem_cache(pkg) + build_command = npm_pkg_build_command(pkg) return nil unless build_command require "open3" @@ -39,6 +41,17 @@ def npm_pkg_rubies_cache_key(pkg) JSON.parse(stdout)["hexdigest"] end +def vendor_gem_cache(pkg) + return unless pkg[:gemfile] + pkg_dir = File.dirname(pkg[:gemfile]) + pkg_dir = File.expand_path(pkg_dir) + vendor_cache_dir = File.join(pkg_dir, "vendor", "cache") + mkdir_p vendor_cache_dir + require_relative "../packages/gems/js/lib/js/version" + sh "gem", "-C", "packages/gems/js", "build", "-o", + File.join(vendor_cache_dir, "js-local.gem") +end + namespace :npm do NPM_PACKAGES.each do |pkg| base_dir = Dir.pwd @@ -51,6 +64,8 @@ namespace :npm do # Skip if the package does not require building ruby next unless build_command + vendor_gem_cache(pkg) + env = { # Share ./build and ./rubies in the same workspace "RUBY_WASM_ROOT" => base_dir @@ -67,11 +82,6 @@ namespace :npm do mkdir_p dist_dir if pkg[:target].start_with?("wasm32-unknown-wasi") Dir.chdir(cwd || base_dir) do - vendor_cache_dir = File.join(pkg_dir, "vendor", "cache") - mkdir_p vendor_cache_dir - require_relative "../packages/gems/js/lib/js/version" - sh "gem", "-C", File.join(base_dir, "packages/gems/js"), "build", "-o", - File.join(pkg_dir, "vendor", "cache", "js-local.gem") sh env, *build_command, From a2f893d60abffa543b2a9698961970e5238492e4 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Fri, 19 Jul 2024 03:19:32 +0000 Subject: [PATCH 10/21] Update Gemfile.lock --- packages/npm-packages/ruby-3.2-wasm-wasi/Gemfile.lock | 4 ++-- packages/npm-packages/ruby-3.3-wasm-wasi/Gemfile.lock | 4 ++-- packages/npm-packages/ruby-head-wasm-wasi/Gemfile.lock | 10 +++------- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/packages/npm-packages/ruby-3.2-wasm-wasi/Gemfile.lock b/packages/npm-packages/ruby-3.2-wasm-wasi/Gemfile.lock index b9d2b12b4e..5addb36ca0 100644 --- a/packages/npm-packages/ruby-3.2-wasm-wasi/Gemfile.lock +++ b/packages/npm-packages/ruby-3.2-wasm-wasi/Gemfile.lock @@ -1,12 +1,12 @@ PATH remote: ../../.. specs: - ruby_wasm (2.6.2) + ruby_wasm (2.6.2.dev) PATH remote: ../../gems/js specs: - js (2.6.2) + js (2.6.2.dev) GEM remote: https://rubygems.org/ diff --git a/packages/npm-packages/ruby-3.3-wasm-wasi/Gemfile.lock b/packages/npm-packages/ruby-3.3-wasm-wasi/Gemfile.lock index b9d2b12b4e..5addb36ca0 100644 --- a/packages/npm-packages/ruby-3.3-wasm-wasi/Gemfile.lock +++ b/packages/npm-packages/ruby-3.3-wasm-wasi/Gemfile.lock @@ -1,12 +1,12 @@ PATH remote: ../../.. specs: - ruby_wasm (2.6.2) + ruby_wasm (2.6.2.dev) PATH remote: ../../gems/js specs: - js (2.6.2) + js (2.6.2.dev) GEM remote: https://rubygems.org/ diff --git a/packages/npm-packages/ruby-head-wasm-wasi/Gemfile.lock b/packages/npm-packages/ruby-head-wasm-wasi/Gemfile.lock index d77f2106cd..97198a4c02 100644 --- a/packages/npm-packages/ruby-head-wasm-wasi/Gemfile.lock +++ b/packages/npm-packages/ruby-head-wasm-wasi/Gemfile.lock @@ -1,16 +1,12 @@ PATH remote: ../../.. specs: - ruby_wasm (2.6.2) - -PATH - remote: ../../gems/js - specs: - js (2.6.2) + ruby_wasm (2.6.2.dev) GEM remote: https://rubygems.org/ specs: + js (2.6.2.dev) power_assert (2.0.3) test-unit (3.6.2) power_assert @@ -20,7 +16,7 @@ PLATFORMS wasm32-wasi DEPENDENCIES - js! + js (= 2.6.2.dev) power_assert ruby_wasm! test-unit From 19fdf2c1c0b6a0e7676007521c7e3b256e57d6da Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Fri, 19 Jul 2024 03:35:21 +0000 Subject: [PATCH 11/21] Seems like bundler respects the .gem file name --- packages/npm-packages/ruby-head-wasm-wasi/Gemfile | 4 ++-- rakelib/packaging.rake | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/npm-packages/ruby-head-wasm-wasi/Gemfile b/packages/npm-packages/ruby-head-wasm-wasi/Gemfile index 5e83ff0b18..35a14732c1 100644 --- a/packages/npm-packages/ruby-head-wasm-wasi/Gemfile +++ b/packages/npm-packages/ruby-head-wasm-wasi/Gemfile @@ -2,11 +2,11 @@ source "https://rubygems.org" -# We build ./vendor/cache/js-local.gem just before evaluating this Gemfile +# We build ./vendor/cache/js-{version}.gem just before evaluating this Gemfile # so that Bundler builds extensions even from the local gem. (gem extensions # from "path:" gems are not built by Bundler.) # Thus even we specify version of "js" gem here, it should always installed -# from the ./vendor/cache/js-local.gem, not from rubygems.org. To achieve this, +# from the ./vendor/cache/js-{version}.gem, not from rubygems.org. To achieve this, # we always use non-exist version during development. require_relative "../../gems/js/lib/js/version.rb" gem "js", JS::VERSION diff --git a/rakelib/packaging.rake b/rakelib/packaging.rake index fa15f6f10f..85aebd0e93 100644 --- a/rakelib/packaging.rake +++ b/rakelib/packaging.rake @@ -49,7 +49,7 @@ def vendor_gem_cache(pkg) mkdir_p vendor_cache_dir require_relative "../packages/gems/js/lib/js/version" sh "gem", "-C", "packages/gems/js", "build", "-o", - File.join(vendor_cache_dir, "js-local.gem") + File.join(vendor_cache_dir, "js-#{JS::VERSION}.gem") end namespace :npm do From 4d8d2a802ae4bd5f8f8b556ac5834bb13a6944e6 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Wed, 21 Aug 2024 04:51:46 +0000 Subject: [PATCH 12/21] Use the head version of Ruby as buildruby --- .github/workflows/build.yml | 8 +++---- bin/setup | 22 ------------------- .../ruby-3.2-wasm-wasi/.gitignore | 2 ++ .../ruby-3.2-wasm-wasi/Gemfile.lock | 2 +- .../ruby-3.3-wasm-wasi/.gitignore | 2 ++ .../ruby-3.3-wasm-wasi/Gemfile.lock | 2 +- .../ruby-head-wasm-wasi/Gemfile.lock | 2 +- 7 files changed, 11 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 12668e4641..b9cc39090a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,7 +31,7 @@ jobs: fetch-depth: 0 - uses: ruby/setup-ruby@v1 with: - ruby-version: "3.3" + ruby-version: "head" bundler-cache: true - run: ./bin/setup - run: bundle exec rake check:type @@ -77,7 +77,7 @@ jobs: - uses: actions/checkout@v4 - uses: ruby/setup-ruby@v1 with: - ruby-version: "3.3" + ruby-version: "head" bundler-cache: true - run: ./bin/setup - run: rake ci:pin_build_manifest @@ -150,7 +150,7 @@ jobs: - uses: ruby/setup-ruby@v1 if: ${{ matrix.entry.test != '' }} with: - ruby-version: "3.3" + ruby-version: "head" bundler-cache: false - name: rake ${{ matrix.entry.test }} run: | @@ -176,7 +176,7 @@ jobs: registry-url: https://registry.npmjs.org/ - uses: ruby/setup-ruby@v1 with: - ruby-version: "3.3" + ruby-version: "head" bundler-cache: true - run: ./bin/setup - run: echo "PREREL_NAME=${{ inputs.prerel_name }}" >> $GITHUB_ENV diff --git a/bin/setup b/bin/setup index 8fdd6f73ab..cf84638259 100755 --- a/bin/setup +++ b/bin/setup @@ -6,28 +6,6 @@ set -vx root="$(cd "$(dirname "$0")/.." && pwd)" env BUNDLE_GEMFILE="$root/Gemfile" bundle install -for gemfile in $root/packages/npm-packages/*/Gemfile; do - # Skip ruby-head-wasm-wasi's Gemfile because it does not need to be installed here - if [[ "$gemfile" == *"/ruby-head-wasm-wasi/Gemfile" ]]; then - continue - fi - - # FIXME: This is a workaround for the following issue: - # 1. `bundle install` does not support auto-self-upgrade for pre-release bundler versions suffixed with ".dev" - # 2. ruby-head-wasm-wasi's component build depends on 2.6.0.dev, which added --target-rbconfig support - # 3. If the "bundle" command here is earlier than 2.6.0.dev, "bundle install" does *not* self-upgrade to - # the specified pre-release version, and it overwrites the Gemfile.lock with the earlier command version. - # 4. Overwritten Gemfile.lock with the earlier bundler version causes auto-self-downgrade when running - # "bundle install --target-rbconfig" inside "rbwasm build" command, then it fails because the earlier - # bundler version does not support --target-rbconfig. - # - # Thus, we temporarily discard the Gemfile.lock changes here to prevent the auto-self-downgrade issue. - # This workaround should be removed once we drop static Ruby builds from ruby-head-wasm-wasi or - # a stable bundler version is released with --target-rbconfig support. - cp "$gemfile.lock" "$gemfile.lock.orig" - env BUNDLE_GEMFILE="$gemfile" bundle install - mv "$gemfile.lock.orig" "$gemfile.lock" -done # Build vendored jco if Rust toolchain is available and submodule is checked out if command -v rustc && [ -f vendor/jco/package.json ]; then diff --git a/packages/npm-packages/ruby-3.2-wasm-wasi/.gitignore b/packages/npm-packages/ruby-3.2-wasm-wasi/.gitignore index fad2dcd07e..cbb80a5cd4 100644 --- a/packages/npm-packages/ruby-3.2-wasm-wasi/.gitignore +++ b/packages/npm-packages/ruby-3.2-wasm-wasi/.gitignore @@ -1,2 +1,4 @@ *.tgz +/tmp /bundle +/vendor diff --git a/packages/npm-packages/ruby-3.2-wasm-wasi/Gemfile.lock b/packages/npm-packages/ruby-3.2-wasm-wasi/Gemfile.lock index 5addb36ca0..16c73025e2 100644 --- a/packages/npm-packages/ruby-3.2-wasm-wasi/Gemfile.lock +++ b/packages/npm-packages/ruby-3.2-wasm-wasi/Gemfile.lock @@ -26,4 +26,4 @@ DEPENDENCIES test-unit BUNDLED WITH - 2.5.9 + 2.6.0.dev diff --git a/packages/npm-packages/ruby-3.3-wasm-wasi/.gitignore b/packages/npm-packages/ruby-3.3-wasm-wasi/.gitignore index fad2dcd07e..cbb80a5cd4 100644 --- a/packages/npm-packages/ruby-3.3-wasm-wasi/.gitignore +++ b/packages/npm-packages/ruby-3.3-wasm-wasi/.gitignore @@ -1,2 +1,4 @@ *.tgz +/tmp /bundle +/vendor diff --git a/packages/npm-packages/ruby-3.3-wasm-wasi/Gemfile.lock b/packages/npm-packages/ruby-3.3-wasm-wasi/Gemfile.lock index 5addb36ca0..16c73025e2 100644 --- a/packages/npm-packages/ruby-3.3-wasm-wasi/Gemfile.lock +++ b/packages/npm-packages/ruby-3.3-wasm-wasi/Gemfile.lock @@ -26,4 +26,4 @@ DEPENDENCIES test-unit BUNDLED WITH - 2.5.9 + 2.6.0.dev diff --git a/packages/npm-packages/ruby-head-wasm-wasi/Gemfile.lock b/packages/npm-packages/ruby-head-wasm-wasi/Gemfile.lock index 97198a4c02..f5bf0d9958 100644 --- a/packages/npm-packages/ruby-head-wasm-wasi/Gemfile.lock +++ b/packages/npm-packages/ruby-head-wasm-wasi/Gemfile.lock @@ -13,7 +13,7 @@ GEM PLATFORMS ruby - wasm32-wasi + x86_64-linux DEPENDENCIES js (= 2.6.2.dev) From 9e1ebb3f16f23e270e7e70dfe60d904ac94f8adb Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Wed, 4 Sep 2024 08:51:17 +0000 Subject: [PATCH 13/21] Stop executing rbwasm under bundle exec within this repository beause it requires gems listed in the Gemfile to be installed --- lib/ruby_wasm/packager/core.rb | 1 + rakelib/packaging.rake | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/ruby_wasm/packager/core.rb b/lib/ruby_wasm/packager/core.rb index 1b7a08a265..a1a8a77289 100644 --- a/lib/ruby_wasm/packager/core.rb +++ b/lib/ruby_wasm/packager/core.rb @@ -1,4 +1,5 @@ require "forwardable" +require "pathname" class RubyWasm::Packager::Core def initialize(packager) diff --git a/rakelib/packaging.rake b/rakelib/packaging.rake index 85aebd0e93..7a8125ff8c 100644 --- a/rakelib/packaging.rake +++ b/rakelib/packaging.rake @@ -11,8 +11,6 @@ def npm_pkg_build_command(pkg) # Skip if the package does not require building ruby return nil unless pkg[:ruby_version] && pkg[:target] [ - "bundle", - "exec", exe_rbwasm, "build", "--ruby-version", From dd88a9c21e58d08911e3e49884b23616d13f7151 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Wed, 4 Sep 2024 08:56:14 +0000 Subject: [PATCH 14/21] Write the matrix to a file instead of stdout --- .github/workflows/build.yml | 4 ++-- rakelib/ci.rake | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b9cc39090a..bc1bb38dd2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -90,8 +90,8 @@ jobs: - name: Set matrix id: set-matrix run: | - rake ci:rake_task_matrix > matrix.json - echo "entries=$(cat matrix.json)" >> $GITHUB_OUTPUT + rake ci:rake_task_matrix + echo "entries=$(cat ci_matrix.json)" >> $GITHUB_OUTPUT rake-tasks: strategy: diff --git a/rakelib/ci.rake b/rakelib/ci.rake index 727c1b2e55..1b90b3a549 100644 --- a/rakelib/ci.rake +++ b/rakelib/ci.rake @@ -95,7 +95,8 @@ end namespace :ci do task :rake_task_matrix do - print JSON.generate(rake_task_matrix.flat_map { |_, entries| entries }) + content = JSON.generate(rake_task_matrix.flat_map { |_, entries| entries }) + File.write("ci_matrix.json", content) end task :pin_build_manifest do From d3f126c1d396ecc6ed5bd34bfb9a3518935d7e40 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Wed, 4 Sep 2024 10:26:34 +0000 Subject: [PATCH 15/21] Install the latest Ruby in builder image --- builders/wasm32-unknown-wasip1/Dockerfile | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/builders/wasm32-unknown-wasip1/Dockerfile b/builders/wasm32-unknown-wasip1/Dockerfile index f954469717..33050078ef 100644 --- a/builders/wasm32-unknown-wasip1/Dockerfile +++ b/builders/wasm32-unknown-wasip1/Dockerfile @@ -34,6 +34,17 @@ RUN set -eux pipefail; \ sh -s -- -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION; \ chmod -R a+w $RUSTUP_HOME $CARGO_HOME +# Install the latest Ruby to use the latest Bundler for cross-building C extension gems. +ADD --keep-git-dir=true https://github.com/ruby/ruby.git /buildruby +RUN set -eux; \ + cd /buildruby; \ + ./autogen.sh; \ + ./configure --prefix=/usr/local --disable-install-doc; \ + make -j$(nproc); \ + make install; \ + cd /; \ + rm -rf /buildruby + ENV BUNDLE_PATH=/usr/local/gems RUN set -eux; \ mkdir -p $BUNDLE_PATH; \ From 1576fd08672d7bc1d7a151c47a14cc8167b5e4f9 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Wed, 4 Sep 2024 10:54:17 +0000 Subject: [PATCH 16/21] Use compiled API as fallback for rb-sys --- ext/ruby_wasm/extconf.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ext/ruby_wasm/extconf.rb b/ext/ruby_wasm/extconf.rb index 48b4f12841..7d03b19a8d 100644 --- a/ext/ruby_wasm/extconf.rb +++ b/ext/ruby_wasm/extconf.rb @@ -3,4 +3,7 @@ require "mkmf" require "rb_sys/mkmf" -create_rust_makefile("ruby_wasm/ruby_wasm") +create_rust_makefile("ruby_wasm/ruby_wasm") do |r| + # We require head Ruby, so we need to fallback to compiled API + r.use_stable_api_compiled_fallback = true +end From b428456bcb34a07d0185196c034afd5df84ffc25 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Wed, 4 Sep 2024 11:15:36 +0000 Subject: [PATCH 17/21] Invalidate existing cache --- lib/ruby_wasm/cli.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/ruby_wasm/cli.rb b/lib/ruby_wasm/cli.rb index 7df2ae58fe..22e58a5c31 100644 --- a/lib/ruby_wasm/cli.rb +++ b/lib/ruby_wasm/cli.rb @@ -335,6 +335,9 @@ def do_print_ruby_cache_key(packager) ruby_core_build = packager.ruby_core_build require "digest" digest = Digest::SHA256.new + # The build system key is used to invalidate the cache when the build system is updated. + build_system_key = 1 + digest.update(build_system_key.to_s) ruby_core_build.cache_key(digest) hexdigest = digest.hexdigest require "json" From 0508bfbff484ac7de48d12f4192119bf9c4b1aa6 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Wed, 4 Sep 2024 12:27:49 +0000 Subject: [PATCH 18/21] Run rbwasm under bundle exec for non-CM builds to integrate with Bundler in-process --- rakelib/packaging.rake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rakelib/packaging.rake b/rakelib/packaging.rake index 7a8125ff8c..edbb150d96 100644 --- a/rakelib/packaging.rake +++ b/rakelib/packaging.rake @@ -80,13 +80,16 @@ namespace :npm do mkdir_p dist_dir if pkg[:target].start_with?("wasm32-unknown-wasi") Dir.chdir(cwd || base_dir) do + sh "bundle", "install" sh env, + "bundle", "exec", *build_command, "--no-stdlib", "-o", File.join(dist_dir, "ruby.wasm") sh env, + "bundle", "exec", *build_command, "-o", File.join(dist_dir, "ruby.debug+stdlib.wasm") From 9d479ac14964e29b2aa23415e905b5b2313dddc9 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Wed, 4 Sep 2024 13:19:24 +0000 Subject: [PATCH 19/21] Install Ruby to /opt/ruby instead of /usr/local Seems like extension build process is confused by the presence of /usr/local/include/ruby/config.h --- builders/wasm32-unknown-wasip1/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/builders/wasm32-unknown-wasip1/Dockerfile b/builders/wasm32-unknown-wasip1/Dockerfile index 33050078ef..ccd32fabed 100644 --- a/builders/wasm32-unknown-wasip1/Dockerfile +++ b/builders/wasm32-unknown-wasip1/Dockerfile @@ -39,11 +39,13 @@ ADD --keep-git-dir=true https://github.com/ruby/ruby.git /buildruby RUN set -eux; \ cd /buildruby; \ ./autogen.sh; \ - ./configure --prefix=/usr/local --disable-install-doc; \ + mkdir -p /opt/ruby; \ + ./configure --prefix=/opt/ruby --disable-install-doc; \ make -j$(nproc); \ make install; \ cd /; \ rm -rf /buildruby +ENV PATH=/opt/ruby/bin:$PATH ENV BUNDLE_PATH=/usr/local/gems RUN set -eux; \ From 06176dc0336c6a6468c06481744c3015b2099f7a Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Wed, 4 Sep 2024 14:58:55 +0000 Subject: [PATCH 20/21] Use the latest version of webrick for URI change in Ruby 3.4 --- Gemfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 9280078ddf..d8814b7149 100644 --- a/Gemfile +++ b/Gemfile @@ -11,7 +11,8 @@ group :development do end group :check do - gem "webrick" + # Use the latest version of webrick for URI change in Ruby 3.4 + gem "webrick", github: "ruby/webrick", ref: "0c600e169bd4ae267cb5eeb6197277c848323bbe" gem "syntax_tree", "~> 3.5" gem "steep" end From 4ddccfc1f70496149c83c5106e625fd00fb769e5 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Wed, 4 Sep 2024 15:31:09 +0000 Subject: [PATCH 21/21] Require "/bundle/bundler/setup.rb" on Component Model build --- packages/npm-packages/ruby-wasm-wasi/src/vm.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/npm-packages/ruby-wasm-wasi/src/vm.ts b/packages/npm-packages/ruby-wasm-wasi/src/vm.ts index 5eecf6b7a9..c7f723fadd 100644 --- a/packages/npm-packages/ruby-wasm-wasi/src/vm.ts +++ b/packages/npm-packages/ruby-wasm-wasi/src/vm.ts @@ -126,7 +126,15 @@ export class RubyVM { this.guest.rubySysinit(c_args); this.guest.rubyOptions(c_args); try { - this.eval(`require "/bundle/setup"`); + this.eval(` + # Require Bundler standalone setup + if File.exist?("/bundle/bundler/setup.rb") + require "/bundle/bundler/setup.rb" + elsif File.exist?("/bundle/setup.rb") + # For non-CM builds, which doesn't use Bundler's standalone mode + require "/bundle/setup.rb" + end + `); } catch (e) { console.warn("Failed to load /bundle/setup", e); }