ruby · kateinoigakukun · Sep 4, 2024 · Jul 3, 2024 · Jul 4, 2024 · Jul 17, 2024
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -31,7 +31,7 @@ jobs:
          fetch-depth: 0
      - uses: ruby/setup-ruby@v1
        with:
-          ruby-version: "3.3"
+          ruby-version: "head"
          bundler-cache: true
      - run: ./bin/setup
      - run: bundle exec rake check:type
@@ -77,7 +77,7 @@ jobs:
      - uses: actions/checkout@v4
      - uses: ruby/setup-ruby@v1
        with:
-          ruby-version: "3.3"
+          ruby-version: "head"
          bundler-cache: true
      - run: ./bin/setup
      - run: rake ci:pin_build_manifest
@@ -90,8 +90,8 @@ jobs:
      - name: Set matrix
        id: set-matrix
        run: |
-          rake ci:rake_task_matrix > matrix.json
-          echo "entries=$(cat matrix.json)" >> $GITHUB_OUTPUT
+          rake ci:rake_task_matrix
+          echo "entries=$(cat ci_matrix.json)" >> $GITHUB_OUTPUT

  rake-tasks:
    strategy:
@@ -142,21 +142,21 @@ jobs:
        if: ${{ inputs.prerel_name != '' && matrix.entry.prerelease != '' }}
      - name: rake ${{ matrix.entry.task }}
        run: ./build-exec rake --verbose ${{ matrix.entry.task }}
+      - uses: actions/upload-artifact@v4
+        if: ${{ matrix.entry.artifact }}
+        with:
+          name: ${{ matrix.entry.artifact_name }}
+          path: ${{ matrix.entry.artifact }}
      - uses: ruby/setup-ruby@v1
        if: ${{ matrix.entry.test != '' }}
        with:
-          ruby-version: "3.3"
+          ruby-version: "head"
          bundler-cache: false
      - name: rake ${{ matrix.entry.test }}
        run: |
          bundle install --with=check --without=development
          rake ${{ matrix.entry.test }}
        if: ${{ matrix.entry.test != '' }}
-      - uses: actions/upload-artifact@v4
-        if: ${{ matrix.entry.artifact }}
-        with:
-          name: ${{ matrix.entry.artifact_name }}
-          path: ${{ matrix.entry.artifact }}

  release-artifacts:
    needs: [rake-tasks]
@@ -176,7 +176,7 @@ jobs:
          registry-url: https://registry.npmjs.org/
      - uses: ruby/setup-ruby@v1
        with:
-          ruby-version: "3.3"
+          ruby-version: "head"
          bundler-cache: true
      - run: ./bin/setup
      - run: echo "PREREL_NAME=${{ inputs.prerel_name }}" >> $GITHUB_ENV

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -117,6 +117,7 @@ $ gh run download <run-id>
 $ for pkg in cross-gem/pkg/ruby_wasm-*; do gem push $pkg; done
 $ gem build && gem push ruby_wasm-*.gem && rm ruby_wasm-*.gem
 $ (cd packages/gems/js/ && gem build && gem push js-*.gem && rm js-*.gem)
+$ rake bump_dev_version
 ```

 ## Release Channels

diff --git a/Gemfile b/Gemfile
@@ -11,7 +11,8 @@ group :development do
 end

 group :check do
-  gem "webrick"
+  # Use the latest version of webrick for URI change in Ruby 3.4
+  gem "webrick", github: "ruby/webrick", ref: "0c600e169bd4ae267cb5eeb6197277c848323bbe"
  gem "syntax_tree", "~> 3.5"
  gem "steep"
 end
diff --git a/Rakefile b/Rakefile
@@ -28,20 +28,20 @@ NPM_PACKAGES = [
  {
    name: "ruby-head-wasm-wasi",
    ruby_version: "head",
-    gemfile: "packages/npm-packages/ruby-wasm-wasi/Gemfile",
+    gemfile: "packages/npm-packages/ruby-head-wasm-wasi/Gemfile",
    target: "wasm32-unknown-wasip1",
    enable_component_model: true,
  },
  {
    name: "ruby-3.3-wasm-wasi",
    ruby_version: "3.3",
-    gemfile: "packages/npm-packages/ruby-wasm-wasi/Gemfile",
+    gemfile: "packages/npm-packages/ruby-3.3-wasm-wasi/Gemfile",
    target: "wasm32-unknown-wasip1"
  },
  {
    name: "ruby-3.2-wasm-wasi",
    ruby_version: "3.2",
-    gemfile: "packages/npm-packages/ruby-wasm-wasi/Gemfile",
+    gemfile: "packages/npm-packages/ruby-3.2-wasm-wasi/Gemfile",
    target: "wasm32-unknown-wasip1"
  },
  { name: "ruby-wasm-wasi", target: "wasm32-unknown-wasip1" }

diff --git a/bin/setup b/bin/setup
@@ -6,7 +6,6 @@ set -vx
 root="$(cd "$(dirname "$0")/.." && pwd)"

 env BUNDLE_GEMFILE="$root/Gemfile" bundle install
-env BUNDLE_GEMFILE="$root/packages/npm-packages/ruby-wasm-wasi/Gemfile" bundle install

 # Build vendored jco if Rust toolchain is available and submodule is checked out
 if command -v rustc && [ -f vendor/jco/package.json ]; then

diff --git a/builders/wasm32-unknown-wasip1/Dockerfile b/builders/wasm32-unknown-wasip1/Dockerfile
@@ -34,6 +34,19 @@ RUN set -eux pipefail; \
    sh -s -- -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION; \
  chmod -R a+w $RUSTUP_HOME $CARGO_HOME

+# Install the latest Ruby to use the latest Bundler for cross-building C extension gems.
+ADD --keep-git-dir=true https://github.com/ruby/ruby.git /buildruby
+RUN set -eux; \
+  cd /buildruby; \
+  ./autogen.sh; \
+  mkdir -p /opt/ruby; \
+  ./configure --prefix=/opt/ruby --disable-install-doc; \
+  make -j$(nproc); \
+  make install; \
+  cd /; \
+  rm -rf /buildruby
+ENV PATH=/opt/ruby/bin:$PATH
+
 ENV BUNDLE_PATH=/usr/local/gems
 RUN set -eux; \
  mkdir -p $BUNDLE_PATH; \

diff --git a/ext/ruby_wasm/extconf.rb b/ext/ruby_wasm/extconf.rb
@@ -3,4 +3,7 @@
 require "mkmf"
 require "rb_sys/mkmf"

-create_rust_makefile("ruby_wasm/ruby_wasm")
+create_rust_makefile("ruby_wasm/ruby_wasm") do |r|
+  # We require head Ruby, so we need to fallback to compiled API
+  r.use_stable_api_compiled_fallback = true
+end
diff --git a/ext/ruby_wasm/src/lib.rs b/ext/ruby_wasm/src/lib.rs
@@ -252,6 +252,9 @@ impl WasiVirt {
            // Disable sockets for now since `sockets/ip-name-lookup` is not
            // supported by @bytecodealliance/preview2-shim yet
            virt.sockets(false);
+            // Disable http for now since `http` is not supported by
+            // wasmtime yet
+            virt.http(false);
            Ok(())
        })
    }

diff --git a/lib/ruby_wasm/build/product/crossruby.rb b/lib/ruby_wasm/build/product/crossruby.rb
@@ -71,14 +71,14 @@ def do_extconf(executor, crossruby)
        return
      end
      objdir = product_build_dir crossruby
-      rbconfig_rb = Dir.glob(File.join(crossruby.dest_dir, "usr/local/lib/ruby/*/wasm32-wasi/rbconfig.rb")).first
+      rbconfig_rb = crossruby.rbconfig_rb
      raise "rbconfig.rb not found" unless rbconfig_rb
      extconf_args = [
        "-C", objdir,
        "#{@srcdir}/extconf.rb",
        "--target-rbconfig=#{rbconfig_rb}",
      ]
-      extconf_args << "--enable-component-model" if @features.support_component_model?
+      extconf_args << "--disable-component-model" unless @features.support_component_model?
      executor.system crossruby.baseruby_path, *extconf_args
    end

@@ -111,7 +111,7 @@ def do_legacy_extconf(executor, crossruby)
        "-I#{crossruby.build_dir}",
        "--",
      ]
-      extconf_args << "--enable-component-model" if @features.support_component_model?
+      extconf_args << "--disable-component-model" unless @features.support_component_model?
      # Clear RUBYOPT to avoid loading unrelated bundle setup
      executor.system crossruby.baseruby_path,
                      *extconf_args,
@@ -301,6 +301,10 @@ def extinit_c_erb
      File.expand_path("../crossruby/extinit.c.erb", __FILE__)
    end

+    def rbconfig_rb
+      Dir.glob(File.join(dest_dir, "usr/local/lib/ruby/*/wasm32-wasi/rbconfig.rb")).first
+    end
+
    def baseruby_path
      File.join(@baseruby.install_dir, "bin/ruby")
    end

diff --git a/lib/ruby_wasm/cli.rb b/lib/ruby_wasm/cli.rb
@@ -310,7 +310,10 @@ def self.bundled_patches_path

    def derive_packager(options)
      __skip__ = definition = nil
-      __skip__ = if defined?(Bundler) && !options[:disable_gems]
+      features = RubyWasm::FeatureSet.derive_from_env
+      # The head ruby & dynamic linking uses "bundle" command to build gems instead of in-process integration.
+      use_in_process_gem_building = !(options[:ruby_version] == "head" && features.support_dynamic_linking?)
+      __skip__ = if defined?(Bundler) && !options[:disable_gems] && use_in_process_gem_building
        begin
          # Silence Bundler UI if --print-ruby-cache-key is specified not to bother the JSON output.
          level = options[:print_ruby_cache_key] ? :silent : Bundler.ui.level
@@ -321,17 +324,20 @@ def derive_packager(options)
          Bundler.ui.level = old_level
        end
      end
-      RubyWasm.logger.info "Using Gemfile: #{definition.gemfiles}" if definition
+      RubyWasm.logger.info "Using Gemfile: #{definition.gemfiles.map(&:to_s).join(", ")}" if definition
      RubyWasm::Packager.new(
        root, build_config(options), definition,
-        features: RubyWasm::FeatureSet.derive_from_env
+        features: features,
      )
    end

    def do_print_ruby_cache_key(packager)
      ruby_core_build = packager.ruby_core_build
      require "digest"
      digest = Digest::SHA256.new
+      # The build system key is used to invalidate the cache when the build system is updated.
+      build_system_key = 1
+      digest.update(build_system_key.to_s)
      ruby_core_build.cache_key(digest)
      hexdigest = digest.hexdigest
      require "json"

diff --git a/lib/ruby_wasm/packager.rb b/lib/ruby_wasm/packager.rb
@@ -34,7 +34,7 @@ def package(executor, dest_dir, options)

    ruby_core.build_gem_exts(executor, fs.bundle_dir)

-    fs.package_gems
+    fs.package_gems unless features.support_component_model?
    fs.remove_non_runtime_files(executor)
    if options[:stdlib]
      options[:without_stdlib_components].each do |component|

diff --git a/lib/ruby_wasm/packager/core.rb b/lib/ruby_wasm/packager/core.rb
@@ -1,4 +1,5 @@
 require "forwardable"
+require "pathname"

 class RubyWasm::Packager::Core
  def initialize(packager)
@@ -19,7 +20,6 @@ def build(executor, options)
  def build_strategy
    @build_strategy ||=
      begin
-        has_exts = @packager.specs.any? { |spec| spec.extensions.any? }
        if @packager.features.support_dynamic_linking?
          DynamicLinking.new(@packager)
        else
@@ -59,14 +59,6 @@ def specs_with_extensions
      end
    end

-    def wasi_exec_model
-      # TODO: Detect WASI exec-model from binary exports (_start or _initialize)
-      use_js_gem = @packager.specs.any? do |spec|
-        spec.name == "js"
-      end
-      use_js_gem ? "reactor" : "command"
-    end
-
    def with_unbundled_env(&block)
      __skip__ = if defined?(Bundler)
        Bundler.with_unbundled_env(&block)
@@ -138,12 +130,16 @@ def _link_gem_exts(executor, build, ruby_root, gem_home, module_bytes)
        wasi_sdk_path = toolchain.wasi_sdk_path
        libraries << File.join(wasi_sdk_path, "share/wasi-sysroot/lib/wasm32-wasi", lib)
      end
-      wasi_adapter = RubyWasm::Packager::ComponentAdapter.wasi_snapshot_preview1(wasi_exec_model)
-      adapters = [wasi_adapter]
      dl_openable_libs = []
      dl_openable_libs << [File.dirname(ruby_root), Dir.glob(File.join(ruby_root, "lib", "ruby", "**", "*.so"))]
      dl_openable_libs << [gem_home, Dir.glob(File.join(gem_home, "**", "*.so"))]

+      has_js_so = dl_openable_libs.any? do |root, libs|
+        libs.any? { |lib| lib.end_with?("/js.so") }
+      end
+      wasi_adapter = RubyWasm::Packager::ComponentAdapter.wasi_snapshot_preview1(has_js_so ? "reactor" : "command")
+      adapters = [wasi_adapter]
+
      linker = RubyWasmExt::ComponentLink.new
      linker.use_built_in_libdl(true)
      linker.stub_missing_functions(false)
@@ -187,31 +183,43 @@ def _build_gem_exts(executor, build, gem_home)
        baseruby.build(executor)
      end

-      exts = specs_with_extensions.flat_map do |spec, exts|
-        exts.map do |ext|
-          ext_feature = File.dirname(ext) # e.g. "ext/cgi/escape"
-          ext_srcdir = File.join(spec.full_gem_path, ext_feature)
-          ext_relative_path = File.join(spec.full_name, ext_feature)
-          prod = RubyWasm::CrossRubyExtProduct.new(
-            ext_srcdir,
-            build.toolchain,
-            features: @packager.features,
-            ext_relative_path: ext_relative_path
-          )
-          [prod, spec]
-        end
-      end
+      crossruby = build.crossruby
+      rbconfig_rb = crossruby.rbconfig_rb

-      exts.each do |prod, spec|
-        libdir = File.join(gem_home, "gems", spec.full_name, spec.raw_require_paths.first)
-        extra_mkargs = [
-          "sitearchdir=#{libdir}",
-          "sitelibdir=#{libdir}",
-        ]
-        executor.begin_section prod.class, prod.name, "Building"
-        prod.build(executor, build.crossruby, extra_mkargs)
-        executor.end_section prod.class, prod.name
-      end
+      options = @packager.full_build_options
+      target_triplet = options[:target]
+
+      local_path = File.join("bundle", target_triplet)
+      env = {
+        "BUNDLE_APP_CONFIG" => File.join(".bundle", target_triplet),
+        "BUNDLE_PATH" => local_path,
+        "BUNDLE_WITHOUT" => "build",
+        # Do not auto-switch bundler version by Gemfile.lock
+        "BUNDLE_VERSION" => "system",
+        # FIXME: BUNDLE_PATH is set as a installation destination here, but
+        # it is also used as a source of gems to be loaded by RubyGems itself.
+        # RubyGems loads "psych" gem and if Gemfile includes "psych" gem,
+        # RubyGems tries to load "psych" gem from BUNDLE_PATH at the second
+        # time of "bundle install" command. But the extension of "psych" gem
+        # under BUNDLE_PATH is built for Wasm target, not for host platform,
+        # so it fails to load the extension.
+        #
+        # Thus we preload psych from the default LOAD_PATH here to avoid
+        # loading Wasm version of psych.so via `Kernel#require` patched by
+        # RubyGems.
+        "RUBYOPT" => "-rpsych",
+      }
+
+      args = [
+        File.join(baseruby.install_dir, "bin", "bundle"),
+        "install",
+        "--standalone",
+        "--target-rbconfig",
+        rbconfig_rb,
+      ]
+
+      executor.system(*args, env: env)
+      executor.cp_r(local_path, gem_home)
    end

    def cache_key(digest)
@@ -337,6 +345,14 @@ def build_gem_exts(executor, gem_home)
      # No-op because we already built extensions as part of the Ruby build
    end

+    def wasi_exec_model
+      # TODO: Detect WASI exec-model from binary exports (_start or _initialize)
+      use_js_gem = @packager.specs.any? do |spec|
+        spec.name == "js"
+      end
+      use_js_gem ? "reactor" : "command"
+    end
+
    def link_gem_exts(executor, ruby_root, gem_home, module_bytes)
      return module_bytes unless @packager.features.support_component_model?


diff --git a/lib/ruby_wasm/packager/file_system.rb b/lib/ruby_wasm/packager/file_system.rb
@@ -44,7 +44,6 @@ def remove_stdlib_component(executor, component)
    when "enc"
      # Remove all encodings except for encdb.so and transdb.so
      enc_dir = File.join(@ruby_root, "lib", "ruby", ruby_version, "wasm32-wasi", "enc")
-      puts File.join(enc_dir, "**/*.so")
      Dir.glob(File.join(enc_dir, "**/*.so")).each do |entry|
        next if entry.end_with?("encdb.so", "transdb.so")
        RubyWasm.logger.debug "Removing stdlib encoding: #{entry}"

diff --git a/lib/ruby_wasm/version.rb b/lib/ruby_wasm/version.rb
@@ -1,3 +1,3 @@
 module RubyWasm
-  VERSION = "2.6.2"
+  VERSION = "2.6.2.dev"
 end
diff --git a/packages/gems/js/ext/js/extconf.rb b/packages/gems/js/ext/js/extconf.rb
@@ -11,7 +11,7 @@

 $objs = %w[js-core.o witapi-core.o]

-use_component_model = enable_config("component-model", false)
+use_component_model = enable_config("component-model", true)
 $stderr.print "Building with component model: "
 $stderr.puts use_component_model ? "\e[1;32myes\e[0m" : "\e[1;31mno\e[0m"
 if use_component_model

diff --git a/packages/gems/js/lib/js/version.rb b/packages/gems/js/lib/js/version.rb
@@ -1,3 +1,3 @@
 module JS
-  VERSION = "2.6.2"
+  VERSION = "2.6.2.dev"
 end
diff --git a/packages/npm-packages/ruby-3.2-wasm-wasi/.gitignore b/packages/npm-packages/ruby-3.2-wasm-wasi/.gitignore
@@ -1 +1,4 @@
 *.tgz
+/tmp
+/bundle
+/vendor
diff --git a/packages/npm-packages/ruby-wasm-wasi/Gemfile → ...s/npm-packages/ruby-3.2-wasm-wasi/Gemfile b/packages/npm-packages/ruby-wasm-wasi/Gemfile → ...s/npm-packages/ruby-3.2-wasm-wasi/Gemfile
-Original file line number
+Diff line change
@@ -1 +1,4 @@
     *.tgz
+    /tmp
+    /bundle
+    /vendor