Initial aarch64/KVM support

.github/workflows/ValidatePullRequest.yml

@@ -60,11 +60,13 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
+        arch: [X64, arm64]
         config: [debug, release]
     uses: ./.github/workflows/dep_build_guests.yml
     secrets: inherit
     with:
       docs_only: ${{ needs.docs-pr.outputs.docs-only }}
+      arch: ${{ matrix.arch }}
       config: ${{ matrix.config }}
 
   # Code checks (fmt, clippy, MSRV) - runs in parallel with build-guests
@@ -92,14 +94,27 @@ jobs:
       fail-fast: true
       matrix:
         hypervisor: ['hyperv-ws2025', mshv3, kvm]
-        cpu: [amd, intel]
+        cpu: [amd, intel, apple]
+        arch: [X64, arm64]
         config: [debug, release]
+        exclude:
+        - cpu: apple
+          hypervisor: hyperv-ws2025
+        - cpu: apple
+          hypervisor: mshv3
+        - cpu: amd
+          arch: arm64
+        - cpu: intel
+          arch: arm64
+        - cpu: apple
+          arch: X64
     uses: ./.github/workflows/dep_build_test.yml
     secrets: inherit
-    with: 
+    with:
       docs_only: ${{ needs.docs-pr.outputs.docs-only }}
       hypervisor: ${{ matrix.hypervisor }}
       cpu: ${{ matrix.cpu }}
+      arch: ${{ matrix.arch }}
       config: ${{ matrix.config }}
 
   # Run examples - needs guest artifacts, runs in parallel with build-test
@@ -115,14 +130,27 @@ jobs:
       fail-fast: true
       matrix:
         hypervisor: ['hyperv-ws2025', mshv3, kvm]
-        cpu: [amd, intel]
+        cpu: [amd, intel, apple]
+        arch: [X64, arm64]
         config: [debug, release]
+        exclude:
+        - cpu: apple
+          hypervisor: hyperv-ws2025
+        - cpu: apple
+          hypervisor: mshv3
+        - cpu: amd
+          arch: arm64
+        - cpu: intel
+          arch: arm64
+        - cpu: apple
+          arch: X64
     uses: ./.github/workflows/dep_run_examples.yml
     secrets: inherit
     with: 
       docs_only: ${{ needs.docs-pr.outputs.docs-only }}
       hypervisor: ${{ matrix.hypervisor }}
       cpu: ${{ matrix.cpu }}
+      arch: ${{ matrix.arch }}
       config: ${{ matrix.config }}
 
   fuzzing:

.github/workflows/dep_build_guests.yml

@@ -10,6 +10,10 @@ on:
         required: false
         type: string
         default: "false"
+      arch:
+        description: CPU architecture for the build (passed from caller matrix)
+        required: true
+        type: string
       config:
         description: Build configuration (debug or release)
         required: true
@@ -29,7 +33,17 @@ jobs:
   build-guests:
     if: ${{ inputs.docs_only == 'false' }}
     timeout-minutes: 15
-    runs-on: [self-hosted, Linux, X64, "1ES.Pool=hld-kvm-amd", "JobId=build-guests-${{ inputs.config }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}"]
+    runs-on: ${{ fromJson(
+      format('["self-hosted", "Linux", "{0}", "{1}" {2}]',
+        inputs.arch,
+        inputs.arch == 'X64' && '1ES.Pool=hld-kvm-amd' || 'ubuntu-24.04',
+        inputs.arch == 'X64' && format(', "JobId=build-guests-{0}-{1}-{2}-{3}"',
+                                  inputs.config,
+                                  github.run_id,
+                                  github.run_number,
+                                  github.run_attempt)
+                             || ''
+      )) }}
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 
@@ -52,12 +66,12 @@ jobs:
         with:
           path: |
             src/tests/rust_guests/target/sysroot
-          key: sysroot-linux-${{ inputs.config }}-${{ hashFiles('rust-toolchain.toml') }}
+          key: sysroot-linux-${{ inputs.arch }}-${{ inputs.config }}-${{ hashFiles('rust-toolchain.toml') }}
 
       - name: Rust cache
         uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4  # v2.9.1
         with:
-          shared-key: "guests-${{ inputs.config }}"
+          shared-key: "guests-${{ inputs.arch }}-${{ inputs.config }}"
           cache-on-failure: "true"
           # Only save on main as caches are not shared across branches.
           # https://docs.github.com/en/actions/reference/workflows-and-actions/dependency-caching#restrictions-for-accessing-a-cache
@@ -82,7 +96,7 @@ jobs:
       - name: Upload Rust guests
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
         with:
-          name: rust-guests-${{ inputs.config }}
+          name: rust-guests-${{ inputs.arch }}-${{ inputs.config }}
           path: |
             src/tests/rust_guests/bin/${{ inputs.config }}/
           retention-days: 1
@@ -91,13 +105,13 @@ jobs:
       - name: Upload C guests
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
         with:
-          name: c-guests-${{ inputs.config }}
+          name: c-guests-${{ inputs.arch }}-${{ inputs.config }}
           path: src/tests/c_guests/bin/${{ inputs.config }}/
           retention-days: 1
           if-no-files-found: error
 
       - name: Upload interface.wasm
-        if: inputs.config == 'debug'
+        if: inputs.config == 'debug' && inputs.arch == 'X64'
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
         with:
           name: interface-wasm
@@ -106,7 +120,7 @@ jobs:
           if-no-files-found: error
 
       - name: Upload twoworlds.wasm
-        if: inputs.config == 'debug'
+        if: inputs.config == 'debug' && inputs.arch == 'X64'
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
         with:
           name: twoworlds-wasm

.github/workflows/dep_build_test.yml

@@ -19,6 +19,10 @@ on:
         required: true
         type: string
       cpu:
+        description: CPU vendor for the build (passed from caller matrix)
+        required: true
+        type: string
+      arch:
         description: CPU architecture for the build (passed from caller matrix)
         required: true
         type: string
@@ -39,14 +43,11 @@ jobs:
     if: ${{ inputs.docs_only == 'false' }}
     timeout-minutes: 45
     runs-on: ${{ fromJson(
-        format('["self-hosted", "{0}", "X64", "1ES.Pool=hld-{1}-{2}", "JobId=build-and-test-{3}-{4}-{5}-{6}"]', 
-          inputs.hypervisor == 'hyperv-ws2025' && 'Windows' || 'Linux', 
-          inputs.hypervisor == 'hyperv-ws2025' && 'win2025' || inputs.hypervisor == 'mshv3' && 'azlinux3-mshv' || inputs.hypervisor, 
-          inputs.cpu,
-          inputs.config,
-          github.run_id,
-          github.run_number,
-          github.run_attempt)) }} 
+        format('["self-hosted", "{0}", "{1}", {2} {3}]',
+          inputs.hypervisor == 'hyperv-ws2025' && 'Windows' || 'Linux',
+          inputs.arch,
+          inputs.arch == 'X64' && format('1ES.pool=hld-{0}-{1}', inputs.hypervisor == 'hyperv-ws2025' && 'win2025' || inputs.hypervisor == 'mshv3' && 'azlinux3-mshv' || inputs.hypervisor, inputs.cpu) || inputs.hypervisor == 'kvm' && '"kvm", "ubuntu-24.04"' || '',
+          inputs.arch == 'X64' && format(', "JobId=build-and-test-{0}-{1}-{2}-{3}"', inputs.config, github.run_id, github.run_number, github.run_attempt) || 'self-hosted')) }}
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 
@@ -64,7 +65,7 @@ jobs:
       - name: Rust cache
         uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4  # v2.9.1
         with:
-          shared-key: "${{ runner.os }}-${{ inputs.hypervisor }}-${{ inputs.config }}"
+          shared-key: "${{ inputs.arch }}-${{ runner.os }}-${{ inputs.hypervisor }}-${{ inputs.config }}"
           cache-on-failure: "true"
           # Only save on main as caches are not shared across branches.
           # https://docs.github.com/en/actions/reference/workflows-and-actions/dependency-caching#restrictions-for-accessing-a-cache
@@ -73,13 +74,13 @@ jobs:
       - name: Download Rust guests
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
         with:
-          name: rust-guests-${{ inputs.config }}
+          name: rust-guests-${{ inputs.arch }}-${{ inputs.config }}
           path: src/tests/rust_guests/bin/${{ inputs.config }}/
 
       - name: Download C guests
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
         with:
-          name: c-guests-${{ inputs.config }}
+          name: c-guests-${{ inputs.arch }}-${{ inputs.config }}
           path: src/tests/c_guests/bin/${{ inputs.config }}/
 
       - name: Download interface.wasm

.github/workflows/dep_run_examples.yml

@@ -19,6 +19,10 @@ on:
         required: true
         type: string
       cpu:
+        description: CPU vendor for the build (passed from caller matrix)
+        required: true
+        type: string
+      arch:
         description: CPU architecture for the build (passed from caller matrix)
         required: true
         type: string
@@ -39,14 +43,11 @@ jobs:
     if: ${{ inputs.docs_only == 'false' }}
     timeout-minutes: 15
     runs-on: ${{ fromJson(
-        format('["self-hosted", "{0}", "X64", "1ES.Pool=hld-{1}-{2}", "JobId=run-examples-{3}-{4}-{5}-{6}"]', 
-          inputs.hypervisor == 'hyperv-ws2025' && 'Windows' || 'Linux', 
-          inputs.hypervisor == 'hyperv-ws2025' && 'win2025' || inputs.hypervisor == 'mshv3' && 'azlinux3-mshv' || inputs.hypervisor, 
-          inputs.cpu,
-          inputs.config,
-          github.run_id,
-          github.run_number,
-          github.run_attempt)) }} 
+        format('["self-hosted", "{0}", "{1}", {2} {3}]',
+          inputs.hypervisor == 'hyperv-ws2025' && 'Windows' || 'Linux',
+          inputs.arch,
+          inputs.arch == 'X64' && format('1ES.pool=hld-{0}-{1}', inputs.hypervisor == 'hyperv-ws2025' && 'win2025' || inputs.hypervisor == 'mshv3' && 'azlinux3-mshv' || inputs.hypervisor, inputs.cpu) || inputs.hypervisor == 'kvm' && '"kvm", "ubuntu-24.04"' || '',
+          inputs.arch == 'X64' && format(', "JobId=run-examples-{0}-{1}-{2}-{3}"', inputs.config, github.run_id, github.run_number, github.run_attempt) || '')) }}
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 
@@ -64,19 +65,19 @@ jobs:
       - name: Rust cache
         uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4  # v2.9.1
         with:
-          shared-key: "${{ runner.os }}-${{ inputs.hypervisor }}-${{ inputs.config }}"
+          shared-key: "${{ inputs.arch }}-${{ runner.os }}-${{ inputs.hypervisor }}-${{ inputs.config }}"
           save-if: "false"
 
       - name: Download Rust guests
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
         with:
-          name: rust-guests-${{ inputs.config }}
+          name: rust-guests-${{ inputs.arch }}-${{ inputs.config }}
           path: src/tests/rust_guests/bin/${{ inputs.config }}/
 
       - name: Download C guests
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
         with:
-          name: c-guests-${{ inputs.config }}
+          name: c-guests-${{ inputs.arch }}-${{ inputs.config }}
           path: src/tests/c_guests/bin/${{ inputs.config }}/
 
       - name: Run Rust examples - windows

Justfile

@@ -24,10 +24,12 @@ export CROSS_CONTAINER_GID := if path_exists("/dev/kvm") == "true" { kvm-gid } e
 root := justfile_directory()
 
 default-target := "debug"
+hyperlight-target-arch := env("HYPERLIGHT_TARGET", arch())
+hyperlight-target := if hyperlight-target-arch == "x86_64" { "x86_64-hyperlight-none" } else if hyperlight-target-arch == "aarch64" { "aarch64-hyperlight-none" } else { error("Unsupported architecture: " + arch()) }
 # All three guest crates share one workspace under src/tests/rust_guests,
 # so they share one target dir and hyperlight-libc / hyperlight-guest-bin
 # get compiled once per profile instead of once per crate.
-rust_guests_target := "src/tests/rust_guests/target/x86_64-hyperlight-none"
+rust_guests_target := "src/tests/rust_guests/target/" + hyperlight-target
 simpleguest_source := rust_guests_target
 dummyguest_source := rust_guests_target
 witguest_source := rust_guests_target
@@ -314,7 +316,7 @@ fmt-apply: (ensure-nightly-fmt)
     cargo +{{nightly-toolchain}} fmt --manifest-path src/hyperlight_guest_capi/Cargo.toml
 
 clippy target=default-target: (witguest-wit)
-    {{ cargo-cmd }} clippy --all-targets --all-features --profile={{ if target == "debug" { "dev" } else { target } }}  {{ target-triple-flag }} -- -D warnings
+    {{ cargo-cmd }} clippy --all-targets {{ if hyperlight-target-arch == "x86_64" { "--all-features" } else { "" } }} --profile={{ if target == "debug" { "dev" } else { target } }}  {{ target-triple-flag }} -- -D warnings
 
 # for use on a linux host-machine when cross-compiling to windows. Uses the windows-gnu which should be sufficient for most purposes
 clippyw target=default-target: (witguest-wit)
@@ -339,7 +341,7 @@ clippy-exhaustive target=default-target: (witguest-wit)
     ./hack/clippy-package-features.sh hyperlight-testing {{ target }} {{ target-triple }}
     ./hack/clippy-package-features.sh hyperlight-component-macro  {{ target }} {{ target-triple }}
     ./hack/clippy-package-features.sh hyperlight-component-util {{ target }} {{ target-triple }}
-    ./hack/clippy-package-features.sh hyperlight-guest-tracing {{ target }}
+    {{ if hyperlight-target-arch == "x86_64" { "./hack/clippy-package-features.sh hyperlight-guest-tracing " + target } else { "" } }}
     just clippy-guests {{ target }}
 
 # Test a specific package with all feature combinations

c.just

@@ -1,8 +1,9 @@
 mkdir := if os() == "windows" { "mkdir -f -p" } else { "mkdir -p"} 
 
 # Elf options
+hyperlight-target-c := if hyperlight-target-arch == "x86_64" { "x86_64-unknown-linux-none" } else if hyperlight-target-arch == "aarch64" { "aarch64-unknown-linux-none" } else { error("Unsupported architecture: " + hyperlight-target-arch) }
 # We don't support stack protectors at the moment, but Arch Linux clang auto-enables them for -linux platforms, so explicitly disable them.
-c-compile-options-elf := '-nostdlibinc -H --target=x86_64-unknown-linux-none -fno-stack-protector -fstack-clash-protection -mstack-probe-size=4096 -fPIC'
+c-compile-options-elf := "-nostdlibinc -H --target=" + hyperlight-target-c + " -fno-stack-protector -fstack-clash-protection -mstack-probe-size=4096 -fPIC"
 c-include-flags-elf := "-I " + root / "src/hyperlight_guest_capi/include/"  + " -I " + root / "src/hyperlight_libc/third_party/picolibc/libc/include/" + " -I " + root / "src/hyperlight_libc/third_party/picolibc/libc/stdio/" + " -I " + root / "src/hyperlight_libc/include/"
 c-linker-options-elf := '--entry "entrypoint" --nostdlib -pie --no-dynamic-linker'
 c-flags-debug-elf := '-O0'
@@ -19,7 +20,7 @@ compile-c-guest target=default-target:
 
 link-c-guest target=default-target:
     # elf
-    cd src/tests/c_guests/c_simpleguest && ld.lld -o out/{{target}}/simpleguest {{c-linker-options-elf}} out/{{target}}/main.o -l hyperlight_guest_capi -L "{{justfile_directory()}}/target/x86_64-hyperlight-none/{{target}}"
+    cd src/tests/c_guests/c_simpleguest && ld.lld -o out/{{target}}/simpleguest {{c-linker-options-elf}} out/{{target}}/main.o -l hyperlight_guest_capi -L "{{justfile_directory()}}/target/{{hyperlight-target}}/{{target}}"
 
 move-c-guests target=default-target:
     # elf

docs/paging-development-notes.md

@@ -172,3 +172,9 @@ below the exception stack within the scratch region.
 
 Hyperlight unconditionally uses 48-bit virtual addresses (4-level
 paging) and enables PAE.  The guest is always entered in long mode.
+
+## aarch64
+
+Hyperlight unconditionally uses 48-bit virtual addresses. Hyperlight
+presently only uses addresses in the lower (ttbr0) half of the address range.
+

flake.nix

@@ -51,7 +51,7 @@
               toolchainVersionAttrs = args;
             };
           })) // {
-            targetPlatforms = [ "x86_64-linux" ];
+            targetPlatforms = [ "aarch64-linux" "x86_64-linux" ];
             badTargetPlatforms = [ ];
           };
           overrideRustPkg = pkg: self.lib.makeOverridable (origArgs:
@@ -68,21 +68,21 @@
               args = [ "-c" "declare > $out" ];
             });
         in {
-          shells.default = gcrootForShell devShells.x86_64-linux.default;
+          shells.x86_64-linux.default = gcrootForShell devShells.x86_64-linux.default;
+          shells.aarch64-linux.default = gcrootForShell devShells.aarch64-linux.default;
         };
-      devShells.x86_64-linux.default =
-        let pkgs = import nixpkgs {
-              system = "x86_64-linux";
-              overlays = [ (import (nixpkgs-mozilla + "/rust-overlay.nix")) overlays.fix-rust ];
-            };
-        in with pkgs; let
+      devShells = nixpkgs.lib.genAttrs nixpkgs.lib.systems.flakeExposed (system: {
+        default = let pkgs = import nixpkgs {
+          inherit system;
+          overlays = [ (import (nixpkgs-mozilla + "/rust-overlay.nix")) overlays.fix-rust ];
+        }; in with pkgs; let
           customisedRustChannelOf = args:
             lib.flip builtins.mapAttrs (rustChannelOf args) (_: pkg: pkg.override {
               targets = [
                 "x86_64-unknown-linux-gnu"
                 "x86_64-pc-windows-msvc" "x86_64-unknown-none"
                 "wasm32-wasip1" "wasm32-wasip2" "wasm32-unknown-unknown"
-                "i686-unknown-linux-gnu"
+                "i686-unknown-linux-gnu" "aarch64-unknown-none"
               ];
               extensions = [ "rust-src" ] ++ (if args.channel == "nightly" then [ "miri-preview" ] else []);
             });
@@ -225,10 +225,11 @@
             src = fetchFromGitHub {
               owner = "hyperlight-dev";
               repo = "cargo-hyperlight";
-              tag = "v${version}";
-              hash = "sha256-xq4/c69N0wG/I8WOYVloo0J0JqoSIKiWWtECdSKrsxo=";
+              rev = "28ac7b57e8e7b83f80bd601f1fab334aa3ae6d4a";
+              hash = "sha256-a/mvPEDJycrCbmd826SmFdasE8BFtMkCsefCNR5JnkM=";
             };
-            cargoHash = "sha256-muiMVrK1TydQiMitihfo7xYidqUIIQ+Hw3BIeo5rLFw=";
+            cargoHash = "sha256-wLapaao8qcB/toltV/xjQ7SXXcfh2J19nw6jWljmb2s=";
+            doCheck = false;
           };
         in (buildRustPackageClang (mkDerivationAttrs: {
           pname = "hyperlight";
@@ -278,5 +279,6 @@
         })).overrideAttrs(oA: {
           hardeningDisable = [ "all" ];
         });
+      });
     };
 }